From bcb28f2e9498ad2803ceee1b6f77730556449e11 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Fri, 28 Sep 2018 13:56:10 -0700 Subject: [PATCH] [Storage] Bucket Lock (#3727) * [Storage] Replay GCS Bucket Lock (#3645) * Add support for defaultEventBasedHold * Add support for Blob eventBasedHold * Add support for blob temporary holds * Remove Beta launch annotations for CMEK * Add support for lockRetentionPolicy and RetentionPolicy * Update FakeStorageRPC * codacy-bot review fix (final only) * Update getRetentionPolicyIsLocked() -> retentionPolicyIsLocked() and misc. * Address comments * Address comments * Address comments. * Small nits * Fix additional comments * [Storage] Bucket lock samples and bug fixes. (#3709) * Fix issue in tests, write bucket lock samples, and resolve issue in client library * Update documentation around Boolean returns to clarify different return caes. * Fix broken unit test * Address feedback * Fix typo in comments --- .../contrib/nio/testing/FakeStorageRpc.java | 6 + .../java/com/google/cloud/storage/Blob.java | 19 +- .../com/google/cloud/storage/BlobInfo.java | 328 ++++++++------ .../java/com/google/cloud/storage/Bucket.java | 48 +- .../com/google/cloud/storage/BucketInfo.java | 414 ++++++++++++------ .../com/google/cloud/storage/Storage.java | 33 +- .../com/google/cloud/storage/StorageImpl.java | 17 + .../cloud/storage/spi/v1/HttpStorageRpc.java | 17 + .../storage/spi/v1/HttpStorageRpcSpans.java | 2 + .../cloud/storage/spi/v1/StorageRpc.java | 9 +- .../storage/testing/RemoteStorageHelper.java | 7 + .../google/cloud/storage/BlobInfoTest.java | 15 + .../com/google/cloud/storage/BlobTest.java | 16 + .../google/cloud/storage/BucketInfoTest.java | 16 + .../com/google/cloud/storage/BucketTest.java | 33 ++ .../google/cloud/storage/StorageImplTest.java | 25 +- .../cloud/storage/it/ITStorageTest.java | 152 ++++++- .../storage/snippets/StorageSnippets.java | 277 ++++++++++++ .../storage/snippets/ITStorageSnippets.java | 106 ++++- 19 files changed, 1255 insertions(+), 285 deletions(-) diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing/FakeStorageRpc.java b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing/FakeStorageRpc.java index f8c4125b8557..d27664683db8 100644 --- a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing/FakeStorageRpc.java +++ b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing/FakeStorageRpc.java @@ -65,6 +65,7 @@ *
  • createBatch *
  • checksums, etags *
  • IAM operations
    • + *
  • BucketLock operations
    • * * */ @@ -520,6 +521,11 @@ public Notification createNotification(String bucket, Notification notification) throw new UnsupportedOperationException(); } + @Override + public Bucket lockRetentionPolicy(Bucket bucket, Map options) { + throw new UnsupportedOperationException(); + } + @Override public ServiceAccount getServiceAccount(String projectId) { return null; diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java index a5d14126e0f6..b29ce5455040 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java @@ -402,13 +402,30 @@ Builder setCustomerEncryption(CustomerEncryption customerEncryption) { return this; } - @GcpLaunchStage.Beta @Override Builder setKmsKeyName(String kmsKeyName) { infoBuilder.setKmsKeyName(kmsKeyName); return this; } + @Override + public Builder setEventBasedHold(Boolean eventBasedHold) { + infoBuilder.setEventBasedHold(eventBasedHold); + return this; + } + + @Override + public Builder setTemporaryHold(Boolean temporaryHold) { + infoBuilder.setTemporaryHold(temporaryHold); + return this; + } + + @Override + Builder setRetentionExpirationTime(Long retentionExpirationTime) { + infoBuilder.setRetentionExpirationTime(retentionExpirationTime); + return this; + } + @Override public Blob build() { return new Blob(storage, infoBuilder); diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java index 386daa63390a..d356a8d813ff 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java @@ -47,7 +47,7 @@ * Google Storage object metadata. * * @see Concepts and - * Terminology + * Terminology */ public class BlobInfo implements Serializable { @@ -85,10 +85,11 @@ public StorageObject apply(BlobInfo blobInfo) { private final boolean isDirectory; private final CustomerEncryption customerEncryption; private final String kmsKeyName; + private final Boolean eventBasedHold; + private final Boolean temporaryHold; + private final Long retentionExpirationTime; - /** - * This class is meant for internal use only. Users are discouraged from using this class. - */ + /** This class is meant for internal use only. Users are discouraged from using this class. */ public static final class ImmutableEmptyMap extends AbstractMap { @Override @@ -113,16 +114,12 @@ public static class CustomerEncryption implements Serializable { this.keySha256 = keySha256; } - /** - * Returns the algorithm used to encrypt the blob. - */ + /** Returns the algorithm used to encrypt the blob. */ public String getEncryptionAlgorithm() { return encryptionAlgorithm; } - /** - * Returns the SHA256 hash of the encryption key. - */ + /** Returns the SHA256 hash of the encryption key. */ public String getKeySha256() { return keySha256; } @@ -144,8 +141,8 @@ public final int hashCode() { public final boolean equals(Object obj) { return obj == this || obj != null - && obj.getClass().equals(CustomerEncryption.class) - && Objects.equals(toPb(), ((CustomerEncryption) obj).toPb()); + && obj.getClass().equals(CustomerEncryption.class) + && Objects.equals(toPb(), ((CustomerEncryption) obj).toPb()); } StorageObject.CustomerEncryption toPb() { @@ -155,19 +152,15 @@ StorageObject.CustomerEncryption toPb() { } static CustomerEncryption fromPb(StorageObject.CustomerEncryption customerEncryptionPb) { - return new CustomerEncryption(customerEncryptionPb.getEncryptionAlgorithm(), - customerEncryptionPb.getKeySha256()); + return new CustomerEncryption( + customerEncryptionPb.getEncryptionAlgorithm(), customerEncryptionPb.getKeySha256()); } } - /** - * Builder for {@code BlobInfo}. - */ + /** Builder for {@code BlobInfo}. */ public abstract static class Builder { - /** - * Sets the blob identity. - */ + /** Sets the blob identity. */ public abstract Builder setBlobId(BlobId blobId); abstract Builder setGeneratedId(String generatedId); @@ -213,7 +206,7 @@ public abstract static class Builder { * Sets the blob's access control configuration. * * @see + * href="https://cloud.google.com/storage/docs/access-control#About-Access-Control-Lists"> * About Access Control Lists */ public abstract Builder setAcl(List acl); @@ -229,31 +222,27 @@ public abstract static class Builder { /** * Sets the MD5 hash of blob's data. MD5 value must be encoded in base64. * - * @see - * Hashes and ETags: Best Practices + * @see Hashes and ETags: + * Best Practices */ public abstract Builder setMd5(String md5); /** - * Sets the CRC32C checksum of blob's data as described in - * RFC 4960, Appendix B; encoded in + * Sets the CRC32C checksum of blob's data as described in RFC 4960, Appendix B; encoded in * base64 in big-endian order. * - * @see - * Hashes and ETags: Best Practices + * @see Hashes and ETags: + * Best Practices */ public abstract Builder setCrc32c(String crc32c); abstract Builder setMediaLink(String mediaLink); - /** - * Sets the blob's storage class. - */ + /** Sets the blob's storage class. */ public abstract Builder setStorageClass(StorageClass storageClass); - /** - * Sets the blob's user provided metadata. - */ + /** Sets the blob's user provided metadata. */ public abstract Builder setMetadata(Map metadata); abstract Builder setMetageneration(Long metageneration); @@ -268,16 +257,17 @@ public abstract static class Builder { abstract Builder setCustomerEncryption(CustomerEncryption customerEncryption); - /** - * - * Sets the blob's kmsKeyName. - */ - @GcpLaunchStage.Beta abstract Builder setKmsKeyName(String kmsKeyName); - /** - * Creates a {@code BlobInfo} object. - */ + /** Sets the blob's event-based hold. */ + public abstract Builder setEventBasedHold(Boolean eventBasedHold); + + /** Sets the blob's temporary hold. */ + public abstract Builder setTemporaryHold(Boolean temporaryHold); + + abstract Builder setRetentionExpirationTime(Long retentionExpirationTime); + + /** Creates a {@code BlobInfo} object. */ public abstract BlobInfo build(); } @@ -308,6 +298,9 @@ static final class BuilderImpl extends Builder { private CustomerEncryption customerEncryption; private StorageClass storageClass; private String kmsKeyName; + private Boolean eventBasedHold; + private Boolean temporaryHold; + private Long retentionExpirationTime; BuilderImpl(BlobId blobId) { this.blobId = blobId; @@ -339,6 +332,9 @@ static final class BuilderImpl extends Builder { isDirectory = blobInfo.isDirectory; storageClass = blobInfo.storageClass; kmsKeyName = blobInfo.kmsKeyName; + eventBasedHold = blobInfo.eventBasedHold; + temporaryHold = blobInfo.temporaryHold; + retentionExpirationTime = blobInfo.retentionExpirationTime; } @Override @@ -439,8 +435,10 @@ Builder setMediaLink(String mediaLink) { @Override public Builder setMetadata(Map metadata) { - this.metadata = metadata != null - ? new HashMap<>(metadata) : Data.>nullOf(ImmutableEmptyMap.class); + this.metadata = + metadata != null + ? new HashMap<>(metadata) + : Data.>nullOf(ImmutableEmptyMap.class); return this; } @@ -486,13 +484,30 @@ Builder setCustomerEncryption(CustomerEncryption customerEncryption) { return this; } - @GcpLaunchStage.Beta @Override Builder setKmsKeyName(String kmsKeyName) { this.kmsKeyName = kmsKeyName; return this; } + @Override + public Builder setEventBasedHold(Boolean eventBasedHold) { + this.eventBasedHold = eventBasedHold; + return this; + } + + @Override + public Builder setTemporaryHold(Boolean temporaryHold) { + this.temporaryHold = temporaryHold; + return this; + } + + @Override + Builder setRetentionExpirationTime(Long retentionExpirationTime) { + this.retentionExpirationTime = retentionExpirationTime; + return this; + } + @Override public BlobInfo build() { checkNotNull(blobId); @@ -526,32 +541,27 @@ public BlobInfo build() { isDirectory = firstNonNull(builder.isDirectory, Boolean.FALSE); storageClass = builder.storageClass; kmsKeyName = builder.kmsKeyName; + eventBasedHold = builder.eventBasedHold; + temporaryHold = builder.temporaryHold; + retentionExpirationTime = builder.retentionExpirationTime; } - /** - * Returns the blob's identity. - */ + /** Returns the blob's identity. */ public BlobId getBlobId() { return blobId; } - /** - * Returns the name of the containing bucket. - */ + /** Returns the name of the containing bucket. */ public String getBucket() { return getBlobId().getBucket(); } - /** - * Returns the service-generated for the blob. - */ + /** Returns the service-generated for the blob. */ public String getGeneratedId() { return generatedId; } - /** - * Returns the blob's name. - */ + /** Returns the blob's name. */ public String getName() { return getBlobId().getName(); } @@ -575,9 +585,7 @@ public List getAcl() { return acl; } - /** - * Returns the blob's owner. This will always be the uploader of the blob. - */ + /** Returns the blob's owner. This will always be the uploader of the blob. */ public Acl.Entity getOwner() { return owner; } @@ -628,9 +636,9 @@ public String getContentLanguage() { } /** - * Returns the number of components that make up this blob. Components are accumulated through - * the {@link Storage#compose(Storage.ComposeRequest)} operation and are limited to a count of - * 1024, counting 1 for each non-composite component blob and componentCount for each composite + * Returns the number of components that make up this blob. Components are accumulated through the + * {@link Storage#compose(Storage.ComposeRequest)} operation and are limited to a count of 1024, + * counting 1 for each non-composite component blob and componentCount for each composite * component blob. This value is set only for composite blobs. * * @see Component Count @@ -649,9 +657,7 @@ public String getEtag() { return etag; } - /** - * Returns the URI of this blob as a string. - */ + /** Returns the URI of this blob as a string. */ public String getSelfLink() { return selfLink; } @@ -659,83 +665,71 @@ public String getSelfLink() { /** * Returns the MD5 hash of blob's data encoded in base64. * - * @see - * Hashes and ETags: Best Practices + * @see Hashes and ETags: + * Best Practices */ public String getMd5() { return Data.isNull(md5) ? null : md5; } /** - * Returns the CRC32C checksum of blob's data as described in - * RFC 4960, Appendix B; encoded in + * Returns the CRC32C checksum of blob's data as described in RFC 4960, Appendix B; encoded in * base64 in big-endian order. * - * @see - * Hashes and ETags: Best Practices + * @see Hashes and ETags: + * Best Practices */ public String getCrc32c() { return Data.isNull(crc32c) ? null : crc32c; } - /** - * Returns the blob's media download link. - */ + /** Returns the blob's media download link. */ public String getMediaLink() { return mediaLink; } - /** - * Returns blob's user provided metadata. - */ + /** Returns blob's user provided metadata. */ public Map getMetadata() { return metadata == null || Data.isNull(metadata) ? null : Collections.unmodifiableMap(metadata); } - /** - * Returns blob's data generation. Used for blob versioning. - */ + /** Returns blob's data generation. Used for blob versioning. */ public Long getGeneration() { return getBlobId().getGeneration(); } /** - * Returns blob's metageneration. Used for preconditions and for detecting changes in metadata. - * A metageneration number is only meaningful in the context of a particular generation of a + * Returns blob's metageneration. Used for preconditions and for detecting changes in metadata. A + * metageneration number is only meaningful in the context of a particular generation of a * particular blob. */ public Long getMetageneration() { return metageneration; } - /** - * Returns the deletion time of the blob. - */ + /** Returns the deletion time of the blob. */ public Long getDeleteTime() { return deleteTime; } - /** - * Returns the last modification time of the blob's metadata. - */ + /** Returns the last modification time of the blob's metadata. */ public Long getUpdateTime() { return updateTime; } - /** - * Returns the creation time of the blob. - */ + /** Returns the creation time of the blob. */ public Long getCreateTime() { return createTime; } /** * Returns {@code true} if the current blob represents a directory. This can only happen if the - * blob is returned by {@link Storage#list(String, Storage.BlobListOption...)} when the - * {@link Storage.BlobListOption#currentDirectory()} option is used. When this is the case only - * {@link #getBlobId()} and {@link #getSize()} are set for the current blob: - * {@link BlobId#getName()} ends with the '/' character, {@link BlobId#getGeneration()} returns - * {@code null} and {@link #getSize()} is {@code 0}. + * blob is returned by {@link Storage#list(String, Storage.BlobListOption...)} when the {@link + * Storage.BlobListOption#currentDirectory()} option is used. When this is the case only {@link + * #getBlobId()} and {@link #getSize()} are set for the current blob: {@link BlobId#getName()} + * ends with the '/' character, {@link BlobId#getGeneration()} returns {@code null} and {@link + * #getSize()} is {@code 0}. */ public boolean isDirectory() { return isDirectory; @@ -749,24 +743,77 @@ public CustomerEncryption getCustomerEncryption() { return customerEncryption; } - /** - * Returns the storage class of the blob. - */ + /** Returns the storage class of the blob. */ public StorageClass getStorageClass() { return storageClass; } - /** - * Returns the Cloud KMS key used to encrypt the blob, if any. - */ - @GcpLaunchStage.Beta + /** Returns the Cloud KMS key used to encrypt the blob, if any. */ public String getKmsKeyName() { return kmsKeyName; } /** - * Returns a builder for the current blob. + * Returns a {@code Boolean} with either {@code true}, {@code null} and in certain cases {@code + * false}. + * + *

    Case 1: {@code true} the field {@link + * com.google.cloud.storage.Storage.BlobField#EVENT_BASED_HOLD} is selected in a {@link + * Storage#get(BlobId, Storage.BlobGetOption...)} and event-based hold for the blob is enabled. + * + *

    Case 2.1: {@code null} the field {@link + * com.google.cloud.storage.Storage.BlobField#EVENT_BASED_HOLD} is selected in a {@link + * Storage#get(BlobId, Storage.BlobGetOption...)}, but event-based hold for the blob is not + * enabled. This case can be considered implicitly {@code false}. + * + *

    Case 2.2: {@code null} the field {@link + * com.google.cloud.storage.Storage.BlobField#EVENT_BASED_HOLD} is not selected in a {@link + * Storage#get(BlobId, Storage.BlobGetOption...)}, and the state for this field is unknown. + * + *

    Case 3: {@code false} event-based hold is explicitly set to false using in a {@link + * Builder#setEventBasedHold(Boolean)} client side for a follow-up request e.g. {@link + * Storage#update(BlobInfo, Storage.BlobTargetOption...)} in which case the value of event-based + * hold will remain {@code false} for the given instance. + */ + public Boolean getEventBasedHold() { + return Data.isNull(eventBasedHold) ? null : eventBasedHold; + } + + /** + * Returns a {@code Boolean} with either {@code true}, {@code null} and in certain cases {@code + * false}. + * + *

    Case 1: {@code true} the field {@link + * com.google.cloud.storage.Storage.BlobField#TEMPORARY_HOLD} is selected in a {@link + * Storage#get(BlobId, Storage.BlobGetOption...)} and temporary hold for the blob is enabled. + * + *

    Case 2.1: {@code null} the field {@link + * com.google.cloud.storage.Storage.BlobField#TEMPORARY_HOLD} is selected in a {@link + * Storage#get(BlobId, Storage.BlobGetOption...)}, but temporary hold for the blob is not enabled. + * This case can be considered implicitly {@code false}. + * + *

    Case 2.2: {@code null} the field {@link + * com.google.cloud.storage.Storage.BlobField#TEMPORARY_HOLD} is not selected in a {@link + * Storage#get(BlobId, Storage.BlobGetOption...)}, and the state for this field is unknown. + * + *

    Case 3: {@code false} event-based hold is explicitly set to false using in a {@link + * Builder#setEventBasedHold(Boolean)} client side for a follow-up request e.g. {@link + * Storage#update(BlobInfo, Storage.BlobTargetOption...)} in which case the value of temporary + * hold will remain {@code false} for the given instance. + */ + public Boolean getTemporaryHold() { + return Data.isNull(temporaryHold) ? null : temporaryHold; + } + + /** + * Returns the retention expiration time of the blob as {@code Long}, if a retention period is + * defined. If retention period is not defined this value returns {@code null} */ + public Long getRetentionExpirationTime() { + return Data.isNull(retentionExpirationTime) ? null : retentionExpirationTime; + } + + /** Returns a builder for the current blob. */ public Builder toBuilder() { return new BuilderImpl(this); } @@ -792,19 +839,22 @@ public int hashCode() { public boolean equals(Object obj) { return obj == this || obj != null - && obj.getClass().equals(BlobInfo.class) - && Objects.equals(toPb(), ((BlobInfo) obj).toPb()); + && obj.getClass().equals(BlobInfo.class) + && Objects.equals(toPb(), ((BlobInfo) obj).toPb()); } StorageObject toPb() { StorageObject storageObject = blobId.toPb(); if (acl != null) { - storageObject.setAcl(Lists.transform(acl, new Function() { - @Override - public ObjectAccessControl apply(Acl acl) { - return acl.toObjectPb(); - } - })); + storageObject.setAcl( + Lists.transform( + acl, + new Function() { + @Override + public ObjectAccessControl apply(Acl acl) { + return acl.toObjectPb(); + } + })); } if (deleteTime != null) { storageObject.setTimeDeleted(new DateTime(deleteTime)); @@ -829,15 +879,20 @@ public ObjectAccessControl apply(Acl acl) { if (metadata != null && !Data.isNull(metadata)) { pbMetadata = Maps.newHashMapWithExpectedSize(metadata.size()); for (Map.Entry entry : metadata.entrySet()) { - pbMetadata.put(entry.getKey(), - firstNonNull(entry.getValue(), Data.nullOf(String.class))); + pbMetadata.put( + entry.getKey(), firstNonNull(entry.getValue(), Data.nullOf(String.class))); } } if (customerEncryption != null) { storageObject.setCustomerEncryption(customerEncryption.toPb()); } + if (retentionExpirationTime != null) { + storageObject.setRetentionExpirationTime(new DateTime(retentionExpirationTime)); + } storageObject.setKmsKeyName(kmsKeyName); + storageObject.setEventBasedHold(eventBasedHold); + storageObject.setTemporaryHold(temporaryHold); storageObject.setMetadata(pbMetadata); storageObject.setCacheControl(cacheControl); storageObject.setContentEncoding(contentEncoding); @@ -855,37 +910,27 @@ public ObjectAccessControl apply(Acl acl) { return storageObject; } - /** - * Returns a {@code BlobInfo} builder where blob identity is set using the provided values. - */ + /** Returns a {@code BlobInfo} builder where blob identity is set using the provided values. */ public static Builder newBuilder(BucketInfo bucketInfo, String name) { return newBuilder(bucketInfo.getName(), name); } - /** - * Returns a {@code BlobInfo} builder where blob identity is set using the provided values. - */ + /** Returns a {@code BlobInfo} builder where blob identity is set using the provided values. */ public static Builder newBuilder(String bucket, String name) { return newBuilder(BlobId.of(bucket, name)); } - /** - * Returns a {@code BlobInfo} builder where blob identity is set using the provided values. - */ + /** Returns a {@code BlobInfo} builder where blob identity is set using the provided values. */ public static Builder newBuilder(BucketInfo bucketInfo, String name, Long generation) { return newBuilder(bucketInfo.getName(), name, generation); } - /** - * Returns a {@code BlobInfo} builder where blob identity is set using the provided values. - */ + /** Returns a {@code BlobInfo} builder where blob identity is set using the provided values. */ public static Builder newBuilder(String bucket, String name, Long generation) { return newBuilder(BlobId.of(bucket, name, generation)); } - /** - * Returns a {@code BlobInfo} builder where blob identity is set using the provided value. - */ + /** Returns a {@code BlobInfo} builder where blob identity is set using the provided value. */ public static Builder newBuilder(BlobId blobId) { return new BuilderImpl(blobId); } @@ -950,13 +995,15 @@ static BlobInfo fromPb(StorageObject storageObject) { builder.setOwner(Acl.Entity.fromPb(storageObject.getOwner().getEntity())); } if (storageObject.getAcl() != null) { - builder.setAcl(Lists.transform(storageObject.getAcl(), - new Function() { - @Override - public Acl apply(ObjectAccessControl objectAccessControl) { - return Acl.fromPb(objectAccessControl); - } - })); + builder.setAcl( + Lists.transform( + storageObject.getAcl(), + new Function() { + @Override + public Acl apply(ObjectAccessControl objectAccessControl) { + return Acl.fromPb(objectAccessControl); + } + })); } if (storageObject.containsKey("isDirectory")) { builder.setIsDirectory(Boolean.TRUE); @@ -971,6 +1018,15 @@ public Acl apply(ObjectAccessControl objectAccessControl) { if (storageObject.getKmsKeyName() != null) { builder.setKmsKeyName(storageObject.getKmsKeyName()); } + if (storageObject.getEventBasedHold() != null) { + builder.setEventBasedHold(storageObject.getEventBasedHold()); + } + if (storageObject.getTemporaryHold() != null) { + builder.setTemporaryHold(storageObject.getTemporaryHold()); + } + if (storageObject.getRetentionExpirationTime() != null) { + builder.setRetentionExpirationTime(storageObject.getRetentionExpirationTime().getValue()); + } return builder.build(); } } diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java index 179e8a5cbfef..c06ed6f6958a 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java @@ -628,13 +628,36 @@ public Builder setLabels(Map labels) { return this; } - @GcpLaunchStage.Beta @Override public Builder setDefaultKmsKeyName(String defaultKmsKeyName) { infoBuilder.setDefaultKmsKeyName(defaultKmsKeyName); return this; } + @Override + public Builder setDefaultEventBasedHold(Boolean defaultEventBasedHold) { + infoBuilder.setDefaultEventBasedHold(defaultEventBasedHold); + return this; + } + + @Override + Builder setRetentionEffectiveTime(Long retentionEffectiveTime) { + infoBuilder.setRetentionEffectiveTime(retentionEffectiveTime); + return this; + } + + @Override + Builder setRetentionPolicyIsLocked(Boolean retentionIsLocked) { + infoBuilder.setRetentionPolicyIsLocked(retentionIsLocked); + return this; + } + + @Override + public Builder setRetentionPeriod(Long retentionPeriod) { + infoBuilder.setRetentionPeriod(retentionPeriod); + return this; + } + @Override public Bucket build() { return new Bucket(storage, infoBuilder); @@ -1111,6 +1134,29 @@ public List listDefaultAcls() { return storage.listDefaultAcls(getName()); } + /** + * Locks bucket retention policy. Requires a local metageneration value in the request. Review example below. + * + *

    Accepts an optional userProject {@link BucketTargetOption} option which defines the project id + * to assign operational costs. + * + *

    Warning: Once a retention policy is locked, it can't be unlocked, removed, or shortened. + * + *

    Example of locking a retention policy on a bucket, only if its local metageneration value matches the bucket's + * service metageneration otherwise a {@link StorageException} is thrown. + * 

     {@code
+   * String bucketName = "my_unique_bucket";
+   * Bucket bucket = storage.get(bucketName, BucketGetOption.fields(BucketField.METAGENERATION));
+   * storage.lockRetentionPolicy(bucket, BucketTargetOption.metagenerationMatch());
+   * }
    + * + * @return a {@code Bucket} object of the locked bucket + * @throws StorageException upon failure + */ + public Bucket lockRetentionPolicy(BucketTargetOption... options) { + return storage.lockRetentionPolicy(this, options); + } + /** * Returns the bucket's {@code Storage} object used to issue requests. */ diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java index 6e0b3e2c38e9..0c417a8aee0d 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java @@ -50,7 +50,7 @@ * Google Storage bucket metadata; * * @see Concepts and - * Terminology + * Terminology */ public class BucketInfo implements Serializable { @@ -88,6 +88,10 @@ public com.google.api.services.storage.model.Bucket apply(BucketInfo bucketInfo) private final StorageClass storageClass; private final Map labels; private final String defaultKmsKeyName; + private final Boolean defaultEventBasedHold; + private final Long retentionEffectiveTime; + private final Boolean retentionPolicyIsLocked; + private final Long retentionPeriod; /** * Base class for bucket's delete rules. Allows to configure automatic deletion of blobs and blobs @@ -102,7 +106,11 @@ public abstract static class DeleteRule implements Serializable { private final Type type; public enum Type { - AGE, CREATE_BEFORE, NUM_NEWER_VERSIONS, IS_LIVE, UNKNOWN + AGE, + CREATE_BEFORE, + NUM_NEWER_VERSIONS, + IS_LIVE, + UNKNOWN } DeleteRule(Type type) { @@ -179,8 +187,8 @@ public static class AgeDeleteRule extends DeleteRule { * Creates an {@code AgeDeleteRule} object. * * @param daysToLive blobs' Time To Live expressed in days. The time when the age condition is - * considered to be satisfied is computed by adding {@code daysToLive} days to the - * midnight following blob's creation time in UTC. + * considered to be satisfied is computed by adding {@code daysToLive} days to the midnight + * following blob's creation time in UTC. */ public AgeDeleteRule(int daysToLive) { super(Type.AGE); @@ -218,8 +226,7 @@ private void writeObject(ObjectOutputStream out) throws IOException { out.writeUTF(rule.toString()); } - private void readObject(ObjectInputStream in) throws IOException, - ClassNotFoundException { + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { in.defaultReadObject(); rule = new JacksonFactory().fromString(in.readUTF(), Rule.class); } @@ -306,8 +313,8 @@ public static class IsLiveDeleteRule extends DeleteRule { /** * Creates an {@code IsLiveDeleteRule} object. * - * @param isLive if set to {@code true} live blobs meet the delete condition. If set to - * {@code false} delete condition is met by archived blobs. + * @param isLive if set to {@code true} live blobs meet the delete condition. If set to {@code + * false} delete condition is met by archived blobs. */ public IsLiveDeleteRule(boolean isLive) { super(Type.IS_LIVE); @@ -324,16 +331,11 @@ void populateCondition(Rule.Condition condition) { } } - /** - * Builder for {@code BucketInfo}. - */ + /** Builder for {@code BucketInfo}. */ public abstract static class Builder { - Builder() { - } + Builder() {} - /** - * Sets the bucket's name. - */ + /** Sets the bucket's name. */ public abstract Builder setName(String name); abstract Builder setGeneratedId(String generatedId); @@ -343,10 +345,8 @@ public abstract static class Builder { abstract Builder setSelfLink(String selfLink); /** - * Sets whether a user accessing the bucket or an object it contains should assume the transit costs - * related to the access. - * - * GcpLaunchStage.Alpha + * Sets whether a user accessing the bucket or an object it contains should assume the transit + * costs related to the access. */ public abstract Builder setRequesterPays(Boolean requesterPays); @@ -362,9 +362,7 @@ public abstract static class Builder { */ public abstract Builder setIndexPage(String indexPage); - /** - * Sets the custom object to return when a requested resource is not found. - */ + /** Sets the custom object to return when a requested resource is not found. */ public abstract Builder setNotFoundPage(String notFoundPage); /** @@ -376,15 +374,15 @@ public abstract static class Builder { /** * Sets the bucket's storage class. This defines how blobs in the bucket are stored and - * determines the SLA and the cost of storage. A list of supported values is available - * here. + * determines the SLA and the cost of storage. A list of supported values is available here. */ public abstract Builder setStorageClass(StorageClass storageClass); /** * Sets the bucket's location. Data for blobs in the bucket resides in physical storage within - * this region. A list of supported values is available - * here. + * this region. A list of supported values is available here. */ public abstract Builder setLocation(String location); @@ -397,8 +395,8 @@ public abstract static class Builder { /** * Sets the bucket's Cross-Origin Resource Sharing (CORS) configuration. * - * @see - * Cross-Origin Resource Sharing (CORS) + * @see Cross-Origin Resource + * Sharing (CORS) */ public abstract Builder setCors(Iterable cors); @@ -406,7 +404,7 @@ public abstract static class Builder { * Sets the bucket's access control configuration. * * @see + * href="https://cloud.google.com/storage/docs/access-control#About-Access-Control-Lists"> * About Access Control Lists */ public abstract Builder setAcl(Iterable acl); @@ -416,25 +414,31 @@ public abstract static class Builder { * configuration is specified. * * @see + * href="https://cloud.google.com/storage/docs/access-control#About-Access-Control-Lists"> * About Access Control Lists */ public abstract Builder setDefaultAcl(Iterable acl); - /** - * Sets the label of this bucket. - */ + /** Sets the label of this bucket. */ public abstract Builder setLabels(Map labels); - /** - * Sets the default Cloud KMS key name for this bucket. - */ - @GcpLaunchStage.Beta + /** Sets the default Cloud KMS key name for this bucket. */ public abstract Builder setDefaultKmsKeyName(String defaultKmsKeyName); + /** Sets the default event-based hold for this bucket. */ + public abstract Builder setDefaultEventBasedHold(Boolean defaultEventBasedHold); + + abstract Builder setRetentionEffectiveTime(Long retentionEffectiveTime); + + abstract Builder setRetentionPolicyIsLocked(Boolean retentionPolicyIsLocked); + /** - * Creates a {@code BucketInfo} object. + * If policy is not locked this value can be cleared, increased, and decreased. If policy is + * locked the retention period can only be increased. */ + public abstract Builder setRetentionPeriod(Long retentionPeriod); + + /** Creates a {@code BucketInfo} object. */ public abstract BucketInfo build(); } @@ -459,6 +463,10 @@ static final class BuilderImpl extends Builder { private List defaultAcl; private Map labels; private String defaultKmsKeyName; + private Boolean defaultEventBasedHold; + private Long retentionEffectiveTime; + private Boolean retentionPolicyIsLocked; + private Long retentionPeriod; BuilderImpl(String name) { this.name = name; @@ -484,6 +492,10 @@ static final class BuilderImpl extends Builder { labels = bucketInfo.labels; requesterPays = bucketInfo.requesterPays; defaultKmsKeyName = bucketInfo.defaultKmsKeyName; + defaultEventBasedHold = bucketInfo.defaultEventBasedHold; + retentionEffectiveTime = bucketInfo.retentionEffectiveTime; + retentionPolicyIsLocked = bucketInfo.retentionPolicyIsLocked; + retentionPeriod = bucketInfo.retentionPeriod; } @Override @@ -516,7 +528,6 @@ public Builder setVersioningEnabled(Boolean enable) { return this; } - /** GcpLaunchStage.Alpha */ @Override public Builder setRequesterPays(Boolean enable) { this.requesterPays = firstNonNull(enable, Data.nullOf(Boolean.class)); @@ -595,11 +606,37 @@ public Builder setLabels(Map labels) { return this; } - @GcpLaunchStage.Beta @Override public Builder setDefaultKmsKeyName(String defaultKmsKeyName) { - this.defaultKmsKeyName = defaultKmsKeyName != null - ? defaultKmsKeyName : Data.nullOf(String.class); + this.defaultKmsKeyName = + defaultKmsKeyName != null ? defaultKmsKeyName : Data.nullOf(String.class); + return this; + } + + @Override + public Builder setDefaultEventBasedHold(Boolean defaultEventBasedHold) { + this.defaultEventBasedHold = + firstNonNull(defaultEventBasedHold, Data.nullOf(Boolean.class)); + return this; + } + + @Override + Builder setRetentionEffectiveTime(Long retentionEffectiveTime) { + this.retentionEffectiveTime = + firstNonNull(retentionEffectiveTime, Data.nullOf(Long.class)); + return this; + } + + @Override + Builder setRetentionPolicyIsLocked(Boolean retentionPolicyIsLocked) { + this.retentionPolicyIsLocked = + firstNonNull(retentionPolicyIsLocked, Data.nullOf(Boolean.class)); + return this; + } + + @Override + public Builder setRetentionPeriod(Long retentionPeriod) { + this.retentionPeriod = firstNonNull(retentionPeriod, Data.nullOf(Long.class)); return this; } @@ -630,49 +667,72 @@ public BucketInfo build() { labels = builder.labels; requesterPays = builder.requesterPays; defaultKmsKeyName = builder.defaultKmsKeyName; + defaultEventBasedHold = builder.defaultEventBasedHold; + retentionEffectiveTime = builder.retentionEffectiveTime; + retentionPolicyIsLocked = builder.retentionPolicyIsLocked; + retentionPeriod = builder.retentionPeriod; } - /** - * Returns the service-generated id for the bucket. - */ + /** Returns the service-generated id for the bucket. */ public String getGeneratedId() { return generatedId; } - /** - * Returns the bucket's name. - */ + /** Returns the bucket's name. */ public String getName() { return name; } - /** - * Returns the bucket's owner. This is always the project team's owner group. - */ + /** Returns the bucket's owner. This is always the project team's owner group. */ public Entity getOwner() { return owner; } - /** - * Returns the URI of this bucket as a string. - */ + /** Returns the URI of this bucket as a string. */ public String getSelfLink() { return selfLink; } /** - * Returns {@code true} if versioning is fully enabled for this bucket, {@code false} otherwise. + * Returns a {@code Boolean} with either {@code true}, {@code null} and in certain cases {@code + * false}. + * + *

    Case 1: {@code true} the field {@link + * com.google.cloud.storage.Storage.BucketField#VERSIONING} is selected in a {@link + * Storage#get(String, Storage.BucketGetOption...)} and versions for the bucket is enabled. + * + *

    Case 2.1: {@code null} the field {@link + * com.google.cloud.storage.Storage.BucketField#VERSIONING} is selected in a {@link + * Storage#get(String, Storage.BucketGetOption...)}, but versions for the bucket is not enabled. + * This case can be considered implicitly {@code false}. + * + *

    Case 2.2: {@code null} the field {@link + * com.google.cloud.storage.Storage.BucketField#VERSIONING} is not selected in a {@link + * Storage#get(String, Storage.BucketGetOption...)}, and the state for this field is unknown. + * + *

    Case 3: {@code false} versions is explicitly set to false client side for a follow-up + * request for example {@link Storage#update(BucketInfo, Storage.BucketTargetOption...)} in which + * case the value of versions will remain {@code false} for for the given instance. */ public Boolean versioningEnabled() { return Data.isNull(versioningEnabled) ? null : versioningEnabled; } - /** - * Returns {@code true} if a user accessing the bucket or an object it contains should assume the transit costs - * related to the access, {@code false} otherwise. + * Returns a {@code Boolean} with either {@code true}, {@code false}, and in a specific case + * {@code null}. + * + *

    Case 1: {@code true} the field {@link com.google.cloud.storage.Storage.BucketField#BILLING} + * is selected in a {@link Storage#get(String, Storage.BucketGetOption...)} and requester pays for + * the bucket is enabled. + * + *

    Case 2: {@code false} the field {@link com.google.cloud.storage.Storage.BucketField#BILLING} + * in a {@link Storage#get(String, Storage.BucketGetOption...)} is selected and requester pays for + * the bucket is disable. * - * GcpLaunchStage.Alpha + *

    Case 3: {@code null} the field {@link com.google.cloud.storage.Storage.BucketField#BILLING} + * in a {@link Storage#get(String, Storage.BucketGetOption...)} is not selected, the value is + * unknown. */ public Boolean requesterPays() { return Data.isNull(requesterPays) ? null : requesterPays; @@ -686,9 +746,7 @@ public String getIndexPage() { return indexPage; } - /** - * Returns the custom object to return when a requested resource is not found. - */ + /** Returns the custom object to return when a requested resource is not found. */ public String getNotFoundPage() { return notFoundPage; } @@ -711,16 +769,12 @@ public String getEtag() { return etag; } - /** - * Returns the time at which the bucket was created. - */ + /** Returns the time at which the bucket was created. */ public Long getCreateTime() { return createTime; } - /** - * Returns the metadata generation of this bucket. - */ + /** Returns the metadata generation of this bucket. */ public Long getMetageneration() { return metageneration; } @@ -748,8 +802,8 @@ public StorageClass getStorageClass() { /** * Returns the bucket's Cross-Origin Resource Sharing (CORS) configuration. * - * @see - * Cross-Origin Resource Sharing (CORS) + * @see Cross-Origin Resource Sharing + * (CORS) */ public List getCors() { return cors; @@ -775,24 +829,78 @@ public List getDefaultAcl() { return defaultAcl; } - /** - * Returns the labels for this bucket. - */ + /** Returns the labels for this bucket. */ public Map getLabels() { return labels; } - /** - * Returns the default Cloud KMS key to be applied to newly inserted objects in this bucket. - */ - @GcpLaunchStage.Beta + /** Returns the default Cloud KMS key to be applied to newly inserted objects in this bucket. */ public String getDefaultKmsKeyName() { return defaultKmsKeyName; } /** - * Returns a builder for the current bucket. + * Returns a {@code Boolean} with either {@code true}, {@code null} and in certain cases {@code + * false}. + * + *

    Case 1: {@code true} the field {@link + * com.google.cloud.storage.Storage.BucketField#DEFAULT_EVENT_BASED_HOLD} is selected in a {@link + * Storage#get(String, Storage.BucketGetOption...)} and default event-based hold for the bucket is + * enabled. + * + *

    Case 2.1: {@code null} the field {@link + * com.google.cloud.storage.Storage.BucketField#DEFAULT_EVENT_BASED_HOLD} is selected in a {@link + * Storage#get(String, Storage.BucketGetOption...)}, but default event-based hold for the bucket + * is not enabled. This case can be considered implicitly {@code false}. + * + *

    Case 2.2: {@code null} the field {@link + * com.google.cloud.storage.Storage.BucketField#DEFAULT_EVENT_BASED_HOLD} is not selected in a + * {@link Storage#get(String, Storage.BucketGetOption...)}, and the state for this field is + * unknown. + * + *

    Case 3: {@code false} default event-based hold is explicitly set to false using in a {@link + * Builder#setDefaultEventBasedHold(Boolean)} client side for a follow-up request e.g. {@link + * Storage#update(BucketInfo, Storage.BucketTargetOption...)} in which case the value of default + * event-based hold will remain {@code false} for the given instance. */ + public Boolean getDefaultEventBasedHold() { + return Data.isNull(defaultEventBasedHold) ? null : defaultEventBasedHold; + } + + /** + * Returns the retention effective time a policy took effect if a retention policy is defined as a + * {@code Long}. + */ + public Long getRetentionEffectiveTime() { + return retentionEffectiveTime; + } + + /** + * Returns a {@code Boolean} with either {@code true} or {@code null}. + * + *

    Case 1: {@code true} the field {@link + * com.google.cloud.storage.Storage.BucketField#RETENTION_POLICY} is selected in a {@link + * Storage#get(String, Storage.BucketGetOption...)} and retention policy for the bucket is locked. + * + *

    Case 2.1: {@code null} the field {@link + * com.google.cloud.storage.Storage.BucketField#RETENTION_POLICY} is selected in a {@link + * Storage#get(String, Storage.BucketGetOption...)}, but retention policy for the bucket is not + * locked. This case can be considered implicitly {@code false}. + * + *

    Case 2.2: {@code null} the field {@link + * com.google.cloud.storage.Storage.BucketField#RETENTION_POLICY} is not selected in a {@link + * Storage#get(String, Storage.BucketGetOption...)}, and the state for this field is unknown. + */ + public Boolean retentionPolicyIsLocked() { + return Data.isNull(retentionPolicyIsLocked) ? null : retentionPolicyIsLocked; + } + + /** Returns the retention policy retention period. */ + public Long getRetentionPeriod() { + return retentionPeriod; + } + + /** Returns a builder for the current bucket. */ public Builder toBuilder() { return new BuilderImpl(this); } @@ -806,15 +914,13 @@ public int hashCode() { public boolean equals(Object obj) { return obj == this || obj != null - && obj.getClass().equals(BucketInfo.class) - && Objects.equals(toPb(), ((BucketInfo) obj).toPb()); + && obj.getClass().equals(BucketInfo.class) + && Objects.equals(toPb(), ((BucketInfo) obj).toPb()); } @Override public String toString() { - return MoreObjects.toStringHelper(this) - .add("name", name) - .toString(); + return MoreObjects.toStringHelper(this).add("name", name).toString(); } com.google.api.services.storage.model.Bucket toPb() { @@ -839,20 +945,26 @@ com.google.api.services.storage.model.Bucket toPb() { bucketPb.setCors(transform(cors, Cors.TO_PB_FUNCTION)); } if (acl != null) { - bucketPb.setAcl(transform(acl, new Function() { - @Override - public BucketAccessControl apply(Acl acl) { - return acl.toBucketPb(); - } - })); + bucketPb.setAcl( + transform( + acl, + new Function() { + @Override + public BucketAccessControl apply(Acl acl) { + return acl.toBucketPb(); + } + })); } if (defaultAcl != null) { - bucketPb.setDefaultObjectAcl(transform(defaultAcl, new Function() { - @Override - public ObjectAccessControl apply(Acl acl) { - return acl.toObjectPb(); - } - })); + bucketPb.setDefaultObjectAcl( + transform( + defaultAcl, + new Function() { + @Override + public ObjectAccessControl apply(Acl acl) { + return acl.toObjectPb(); + } + })); } if (owner != null) { bucketPb.setOwner(new Owner().setEntity(owner.toPb())); @@ -874,12 +986,15 @@ public ObjectAccessControl apply(Acl acl) { } if (deleteRules != null) { Lifecycle lifecycle = new Lifecycle(); - lifecycle.setRule(transform(deleteRules, new Function() { - @Override - public Rule apply(DeleteRule deleteRule) { - return deleteRule.toPb(); - } - })); + lifecycle.setRule( + transform( + deleteRules, + new Function() { + @Override + public Rule apply(DeleteRule deleteRule) { + return deleteRule.toPb(); + } + })); bucketPb.setLifecycle(lifecycle); } if (labels != null) { @@ -888,19 +1003,36 @@ public Rule apply(DeleteRule deleteRule) { if (defaultKmsKeyName != null) { bucketPb.setEncryption(new Encryption().setDefaultKmsKeyName(defaultKmsKeyName)); } + if (defaultEventBasedHold != null) { + bucketPb.setDefaultEventBasedHold(defaultEventBasedHold); + } + if (retentionPeriod != null) { + if (Data.isNull(retentionPeriod)) { + bucketPb.setRetentionPolicy( + Data.nullOf(Bucket.RetentionPolicy.class)); + } else { + Bucket.RetentionPolicy retentionPolicy = new Bucket.RetentionPolicy(); + retentionPolicy.setRetentionPeriod(retentionPeriod); + if (retentionEffectiveTime != null) { + retentionPolicy.setEffectiveTime(new DateTime(retentionEffectiveTime)); + } + if (retentionPolicyIsLocked != null) { + retentionPolicy.setIsLocked(retentionPolicyIsLocked); + } + bucketPb.setRetentionPolicy(retentionPolicy); + + } + } + return bucketPb; } - /** - * Creates a {@code BucketInfo} object for the provided bucket name. - */ + /** Creates a {@code BucketInfo} object for the provided bucket name. */ public static BucketInfo of(String name) { return newBuilder(name).build(); } - /** - * Returns a {@code BucketInfo} builder where the bucket's name is set to the provided name. - */ + /** Returns a {@code BucketInfo} builder where the bucket's name is set to the provided name. */ public static Builder newBuilder(String name) { return new BuilderImpl(name); } @@ -932,21 +1064,26 @@ static BucketInfo fromPb(com.google.api.services.storage.model.Bucket bucketPb) builder.setCors(transform(bucketPb.getCors(), Cors.FROM_PB_FUNCTION)); } if (bucketPb.getAcl() != null) { - builder.setAcl(transform(bucketPb.getAcl(), new Function() { - @Override - public Acl apply(BucketAccessControl bucketAccessControl) { - return Acl.fromPb(bucketAccessControl); - } - })); + builder.setAcl( + transform( + bucketPb.getAcl(), + new Function() { + @Override + public Acl apply(BucketAccessControl bucketAccessControl) { + return Acl.fromPb(bucketAccessControl); + } + })); } if (bucketPb.getDefaultObjectAcl() != null) { - builder.setDefaultAcl(transform(bucketPb.getDefaultObjectAcl(), - new Function() { - @Override - public Acl apply(ObjectAccessControl objectAccessControl) { - return Acl.fromPb(objectAccessControl); - } - })); + builder.setDefaultAcl( + transform( + bucketPb.getDefaultObjectAcl(), + new Function() { + @Override + public Acl apply(ObjectAccessControl objectAccessControl) { + return Acl.fromPb(objectAccessControl); + } + })); } if (bucketPb.getOwner() != null) { builder.setOwner(Entity.fromPb(bucketPb.getOwner().getEntity())); @@ -960,13 +1097,15 @@ public Acl apply(ObjectAccessControl objectAccessControl) { builder.setNotFoundPage(website.getNotFoundPage()); } if (bucketPb.getLifecycle() != null && bucketPb.getLifecycle().getRule() != null) { - builder.setDeleteRules(transform(bucketPb.getLifecycle().getRule(), - new Function() { - @Override - public DeleteRule apply(Rule rule) { - return DeleteRule.fromPb(rule); - } - })); + builder.setDeleteRules( + transform( + bucketPb.getLifecycle().getRule(), + new Function() { + @Override + public DeleteRule apply(Rule rule) { + return DeleteRule.fromPb(rule); + } + })); } if (bucketPb.getLabels() != null) { builder.setLabels(bucketPb.getLabels()); @@ -976,9 +1115,26 @@ public DeleteRule apply(Rule rule) { builder.setRequesterPays(billing.getRequesterPays()); } Encryption encryption = bucketPb.getEncryption(); - if (encryption != null && encryption.getDefaultKmsKeyName() != null && !encryption.getDefaultKmsKeyName().isEmpty()) { + if (encryption != null + && encryption.getDefaultKmsKeyName() != null + && !encryption.getDefaultKmsKeyName().isEmpty()) { builder.setDefaultKmsKeyName(encryption.getDefaultKmsKeyName()); } + if (bucketPb.getDefaultEventBasedHold() != null) { + builder.setDefaultEventBasedHold(bucketPb.getDefaultEventBasedHold()); + } + Bucket.RetentionPolicy retentionPolicy = bucketPb.getRetentionPolicy(); + if (retentionPolicy != null) { + if (retentionPolicy.getEffectiveTime() != null) { + builder.setRetentionEffectiveTime(retentionPolicy.getEffectiveTime().getValue()); + } + if (retentionPolicy.getIsLocked() != null) { + builder.setRetentionPolicyIsLocked(retentionPolicy.getIsLocked()); + } + if (retentionPolicy.getRetentionPeriod() != null) { + builder.setRetentionPeriod(retentionPolicy.getRetentionPeriod()); + } + } return builder.build(); } } diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 78f9390a67e4..c00ef60e94aa 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -93,9 +93,10 @@ enum BucketField implements FieldSelector { CORS("cors"), STORAGE_CLASS("storageClass"), ETAG("etag"), - @GcpLaunchStage.Beta ENCRYPTION("encryption"), - BILLING("billing"); + BILLING("billing"), + DEFAULT_EVENT_BASED_HOLD("defaultEventBasedHold"), + RETENTION_POLICY("retentionPolicy"); static final List REQUIRED_FIELDS = ImmutableList.of(NAME); @@ -136,8 +137,11 @@ enum BlobField implements FieldSelector { SIZE("size"), STORAGE_CLASS("storageClass"), TIME_DELETED("timeDeleted"), - @GcpLaunchStage.Beta + TIME_CREATED("timeCreated"), KMS_KEY_NAME("kmsKeyName"), + EVENT_BASED_HOLD("eventBasedHold"), + TEMPORARY_HOLD("temporaryHold"), + RETENTION_EXPIRATION_TIME("retentionExpirationTime"), UPDATED("updated"); static final List REQUIRED_FIELDS = ImmutableList.of(BUCKET, NAME); @@ -388,7 +392,6 @@ public static BlobTargetOption encryptionKey(String key) { /** * Returns an option to set a customer-managed key for server-side encryption of the blob. */ - @GcpLaunchStage.Beta public static BlobTargetOption kmsKeyName(String kmsKeyName) { return new BlobTargetOption(StorageRpc.Option.KMS_KEY_NAME, kmsKeyName); } @@ -550,7 +553,6 @@ public static BlobWriteOption encryptionKey(String key) { * * @param kmsKeyName the KMS key resource id */ - @GcpLaunchStage.Beta public static BlobWriteOption kmsKeyName(String kmsKeyName) { return new BlobWriteOption(Option.KMS_KEY_NAME, kmsKeyName); } @@ -1537,6 +1539,27 @@ public static Builder newBuilder() { */ Bucket get(String bucket, BucketGetOption... options); + /** + * Locks bucket retention policy. Requires a local metageneration value in the request. Review example below. + * + *

    Accepts an optional userProject {@link BucketTargetOption} option which defines the project id + * to assign operational costs. + * + *

    Warning: Once a retention policy is locked, it can't be unlocked, removed, or shortened. + * + *

    Example of locking a retention policy on a bucket, only if its local metageneration value matches the bucket's + * service metageneration otherwise a {@link StorageException} is thrown. + * 

     {@code
+   * String bucketName = "my_unique_bucket";
+   * Bucket bucket = storage.get(bucketName, BucketGetOption.fields(BucketField.METAGENERATION));
+   * storage.lockRetentionPolicy(bucket, BucketTargetOption.metagenerationMatch());
+   * }
    + * + * @return a {@code Bucket} object of the locked bucket + * @throws StorageException upon failure + */ + Bucket lockRetentionPolicy(BucketInfo bucket, BucketTargetOption... options); + /** * Returns the requested blob or {@code null} if not found. * diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index e49445d13646..787388006571 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -983,6 +983,23 @@ public Boolean apply(String permission) { } } + @Override + public Bucket lockRetentionPolicy(BucketInfo bucketInfo, BucketTargetOption... options) { + final com.google.api.services.storage.model.Bucket bucketPb = bucketInfo.toPb(); + final Map optionsMap = optionMap(bucketInfo, options); + try { + return Bucket.fromPb(this, runWithRetries( + new Callable() { + @Override + public com.google.api.services.storage.model.Bucket call() { + return storageRpc.lockRetentionPolicy(bucketPb, optionsMap); + } + }, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock())); + } catch (RetryHelperException e) { + throw StorageException.translateAndThrow(e); + } + } + @Override public ServiceAccount getServiceAccount(final String projectId) { try { diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index c6b2e668742d..19c9324518d7 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -1208,6 +1208,23 @@ public Notification createNotification(String bucket, Notification notification) } } + @Override + public Bucket lockRetentionPolicy(Bucket bucket, Map options) { + Span span = startSpan(HttpStorageRpcSpans.SPAN_LOCK_RETENTION_POLICY); + Scope scope = tracer.withSpan(span); + try { + return storage.buckets().lockRetentionPolicy(bucket.getName(), Option.IF_METAGENERATION_MATCH.getLong(options)) + .setUserProject(Option.USER_PROJECT.getString(options)).execute(); + } catch (IOException ex) { + span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); + throw translate(ex); + } finally { + scope.close(); + span.end(); + } + + } + @Override public ServiceAccount getServiceAccount(String projectId) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_GET_SERVICE_ACCOUNT); diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpcSpans.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpcSpans.java index ceb8a3c83e53..41c877f8eefc 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpcSpans.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpcSpans.java @@ -90,6 +90,7 @@ class HttpStorageRpcSpans { "listNotifications(String)"); static final String SPAN_NAME_CREATE_NOTIFICATION = getTraceSpanName( "createNotification(String,Notification)"); + static final String SPAN_LOCK_RETENTION_POLICY = getTraceSpanName("lockRetentionPolicy(String,Long)"); static final String SPAN_NAME_GET_SERVICE_ACCOUNT = getTraceSpanName( "getServiceAccount(String)"); static final String SPAN_NAME_BATCH_SUBMIT = getTraceSpanName( @@ -136,6 +137,7 @@ class HttpStorageRpcSpans { SPAN_NAME_LIST_NOTIFICATIONS, SPAN_NAME_CREATE_NOTIFICATION, SPAN_NAME_GET_SERVICE_ACCOUNT, + SPAN_LOCK_RETENTION_POLICY, SPAN_NAME_BATCH_SUBMIT); static String getTraceSpanName(String methodDescriptor) { diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java index 780d8b864cfd..2a62a66a9967 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java @@ -57,7 +57,6 @@ enum Option { FIELDS("fields"), CUSTOMER_SUPPLIED_KEY("customerSuppliedKey"), USER_PROJECT("userProject"), - @GcpLaunchStage.Beta KMS_KEY_NAME("kmsKeyName"); private final String value; @@ -460,6 +459,14 @@ void write(String uploadId, byte[] toWrite, int toWriteOffset, long destOffset, */ Notification createNotification(String bucket, Notification notification); + /** + * Lock retention policy for the provided bucket. + * + * @return a {@code Bucket} object of the locked bucket + * @throws StorageException upon failure + */ + Bucket lockRetentionPolicy(Bucket bucket, Map options); + /** * Returns the service account associated with the given project. * diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/RemoteStorageHelper.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/RemoteStorageHelper.java index f637f5cee4cf..666b9ed6449b 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/RemoteStorageHelper.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/RemoteStorageHelper.java @@ -20,6 +20,7 @@ import com.google.api.gax.retrying.RetrySettings; import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.http.HttpTransportOptions; +import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Bucket; @@ -80,6 +81,12 @@ public void run() { for (Bucket bucket : buckets.iterateAll()) { if (bucket.getCreateTime() < olderThan) { try { + for (Blob blob : bucket.list(BlobListOption.fields(Storage.BlobField.EVENT_BASED_HOLD, + Storage.BlobField.TEMPORARY_HOLD)).iterateAll()) { + if(blob.getEventBasedHold() == true || blob.getTemporaryHold() == true) { + storage.update(blob.toBuilder().setTemporaryHold(false).setEventBasedHold(false).build()); + } + } forceDelete(storage, bucket.getName()); } catch (Exception e) { // Ignore the exception, maybe the bucket is being deleted by someone else. diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java index 460bdb1ea811..88914f34a4c8 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobInfoTest.java @@ -68,6 +68,9 @@ public class BlobInfoTest { new CustomerEncryption(ENCRYPTION_ALGORITHM, KEY_SHA256); private static final String KMS_KEY_NAME = "projects/p/locations/kr-loc/keyRings/kr/cryptoKeys/key"; private static final StorageClass STORAGE_CLASS = StorageClass.COLDLINE; + private static final Boolean EVENT_BASED_HOLD = true; + private static final Boolean TEMPORARY_HOLD = true; + private static final Long RETENTION_EXPIRATION_TIME = 10L; private static final BlobInfo BLOB_INFO = BlobInfo.newBuilder("b", "n", GENERATION) .setAcl(ACL) @@ -93,6 +96,9 @@ public class BlobInfoTest { .setCreateTime(CREATE_TIME) .setStorageClass(STORAGE_CLASS) .setKmsKeyName(KMS_KEY_NAME) + .setEventBasedHold(EVENT_BASED_HOLD) + .setTemporaryHold(TEMPORARY_HOLD) + .setRetentionExpirationTime(RETENTION_EXPIRATION_TIME) .build(); private static final BlobInfo DIRECTORY_INFO = BlobInfo.newBuilder("b", "n/") .setSize(0L) @@ -156,6 +162,9 @@ public void testBuilder() { assertEquals(CREATE_TIME, BLOB_INFO.getCreateTime()); assertEquals(STORAGE_CLASS, BLOB_INFO.getStorageClass()); assertEquals(KMS_KEY_NAME, BLOB_INFO.getKmsKeyName()); + assertEquals(EVENT_BASED_HOLD, BLOB_INFO.getEventBasedHold()); + assertEquals(TEMPORARY_HOLD, BLOB_INFO.getTemporaryHold()); + assertEquals(RETENTION_EXPIRATION_TIME, BLOB_INFO.getRetentionExpirationTime()); assertFalse(BLOB_INFO.isDirectory()); assertEquals("b", DIRECTORY_INFO.getBucket()); assertEquals("n/", DIRECTORY_INFO.getName()); @@ -212,6 +221,9 @@ private void compareBlobs(BlobInfo expected, BlobInfo value) { assertEquals(expected.getUpdateTime(), value.getUpdateTime()); assertEquals(expected.getStorageClass(), value.getStorageClass()); assertEquals(expected.getKmsKeyName(), value.getKmsKeyName()); + assertEquals(expected.getEventBasedHold(), value.getEventBasedHold()); + assertEquals(expected.getTemporaryHold(), value.getTemporaryHold()); + assertEquals(expected.getRetentionExpirationTime(), value.getRetentionExpirationTime()); } private void compareCustomerEncryptions(CustomerEncryption expected, CustomerEncryption value) { @@ -260,6 +272,9 @@ public void testToPbAndFromPb() { assertNull(blobInfo.getUpdateTime()); assertNull(blobInfo.getStorageClass()); assertNull(blobInfo.getKmsKeyName()); + assertNull(blobInfo.getEventBasedHold()); + assertNull(blobInfo.getTemporaryHold()); + assertNull(blobInfo.getRetentionExpirationTime()); assertTrue(blobInfo.isDirectory()); } diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java index f1b52a526b71..8db0793409a6 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java @@ -92,6 +92,9 @@ public class BlobTest { private static final BlobInfo.CustomerEncryption CUSTOMER_ENCRYPTION = new BlobInfo.CustomerEncryption(ENCRYPTION_ALGORITHM, KEY_SHA256); private static final String KMS_KEY_NAME = "projects/p/locations/kr-loc/keyRings/kr/cryptoKeys/key"; + private static final Boolean EVENT_BASED_HOLD = true; + private static final Boolean TEMPORARY_HOLD = true; + private static final Long RETENTION_EXPIRATION_TIME = 10L; private static final BlobInfo FULL_BLOB_INFO = BlobInfo.newBuilder("b", "n", GENERATION) .setAcl(ACLS) .setComponentCount(COMPONENT_COUNT) @@ -115,6 +118,9 @@ public class BlobTest { .setCreateTime(CREATE_TIME) .setCustomerEncryption(CUSTOMER_ENCRYPTION) .setKmsKeyName(KMS_KEY_NAME) + .setEventBasedHold(EVENT_BASED_HOLD) + .setTemporaryHold(TEMPORARY_HOLD) + .setRetentionExpirationTime(RETENTION_EXPIRATION_TIME) .build(); private static final BlobInfo BLOB_INFO = BlobInfo.newBuilder("b", "n") .setMetageneration(42L) @@ -472,6 +478,9 @@ public void testBuilder() { .setCreateTime(CREATE_TIME) .setCustomerEncryption(CUSTOMER_ENCRYPTION) .setKmsKeyName(KMS_KEY_NAME) + .setEventBasedHold(EVENT_BASED_HOLD) + .setTemporaryHold(TEMPORARY_HOLD) + .setRetentionExpirationTime(RETENTION_EXPIRATION_TIME) .setDeleteTime(DELETE_TIME) .setEtag(ETAG) .setGeneratedId(GENERATED_ID) @@ -496,6 +505,10 @@ public void testBuilder() { assertEquals(CRC32, blob.getCrc32c()); assertEquals(CREATE_TIME, blob.getCreateTime()); assertEquals(CUSTOMER_ENCRYPTION, blob.getCustomerEncryption()); + assertEquals(KMS_KEY_NAME, blob.getKmsKeyName()); + assertEquals(EVENT_BASED_HOLD, blob.getEventBasedHold()); + assertEquals(TEMPORARY_HOLD, blob.getTemporaryHold()); + assertEquals(RETENTION_EXPIRATION_TIME, blob.getRetentionExpirationTime()); assertEquals(DELETE_TIME, blob.getDeleteTime()); assertEquals(ETAG, blob.getEtag()); assertEquals(GENERATED_ID, blob.getGeneratedId()); @@ -527,6 +540,9 @@ public void testBuilder() { assertNull(blob.getCreateTime()); assertNull(blob.getCustomerEncryption()); assertNull(blob.getKmsKeyName()); + assertNull(blob.getEventBasedHold()); + assertNull(blob.getTemporaryHold()); + assertNull(blob.getRetentionExpirationTime()); assertNull(blob.getDeleteTime()); assertNull(blob.getEtag()); assertNull(blob.getGeneratedId()); diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java index d31dd0246e0b..f371af93bc0a 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java @@ -64,6 +64,10 @@ public class BucketInfoTest { private static final Boolean VERSIONING_ENABLED = true; private static final Map BUCKET_LABELS = ImmutableMap.of("label1", "value1"); private static final Boolean REQUESTER_PAYS = true; + private static final Boolean DEFAULT_EVENT_BASED_HOLD = true; + private static final Long RETENTION_EFFECTIVE_TIME = 10L; + private static final Long RETENTION_PERIOD = 10L; + private static final Boolean RETENTION_POLICY_IS_LOCKED = false; private static final BucketInfo BUCKET_INFO = BucketInfo.newBuilder("b") .setAcl(ACL) .setEtag(ETAG) @@ -83,6 +87,10 @@ public class BucketInfoTest { .setLabels(BUCKET_LABELS) .setRequesterPays(REQUESTER_PAYS) .setDefaultKmsKeyName(DEFAULT_KMS_KEY_NAME) + .setDefaultEventBasedHold(DEFAULT_EVENT_BASED_HOLD) + .setRetentionEffectiveTime(RETENTION_EFFECTIVE_TIME) + .setRetentionPeriod(RETENTION_PERIOD) + .setRetentionPolicyIsLocked(RETENTION_POLICY_IS_LOCKED) .build(); @Test @@ -128,6 +136,10 @@ public void testBuilder() { assertEquals(VERSIONING_ENABLED, BUCKET_INFO.versioningEnabled()); assertEquals(BUCKET_LABELS, BUCKET_INFO.getLabels()); assertEquals(REQUESTER_PAYS, BUCKET_INFO.requesterPays()); + assertEquals(DEFAULT_EVENT_BASED_HOLD, BUCKET_INFO.getDefaultEventBasedHold()); + assertEquals(RETENTION_EFFECTIVE_TIME, BUCKET_INFO.getRetentionEffectiveTime()); + assertEquals(RETENTION_PERIOD, BUCKET_INFO.getRetentionPeriod()); + assertEquals(RETENTION_POLICY_IS_LOCKED, BUCKET_INFO.retentionPolicyIsLocked()); } @Test @@ -158,6 +170,10 @@ private void compareBuckets(BucketInfo expected, BucketInfo value) { assertEquals(expected.versioningEnabled(), value.versioningEnabled()); assertEquals(expected.getLabels(), value.getLabels()); assertEquals(expected.requesterPays(), value.requesterPays()); + assertEquals(expected.getDefaultEventBasedHold(), value.getDefaultEventBasedHold()); + assertEquals(expected.getRetentionEffectiveTime(), value.getRetentionEffectiveTime()); + assertEquals(expected.getRetentionPeriod(), value.getRetentionPeriod()); + assertEquals(expected.retentionPolicyIsLocked(), value.retentionPolicyIsLocked()); } @Test diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java index 707a9f3c7565..9f0324b205b3 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java @@ -82,6 +82,10 @@ public class BucketTest { private static final Map BUCKET_LABELS = ImmutableMap.of("label1", "value1"); private static final Boolean REQUESTER_PAYS = true; private static final String USER_PROJECT = "test-project"; + private static final Boolean DEFAULT_EVENT_BASED_HOLD = true; + private static final Long RETENTION_EFFECTIVE_TIME = 10L; + private static final Long RETENTION_PERIOD = 10L; + private static final Boolean RETENTION_POLICY_IS_LOCKED = false; private static final BucketInfo FULL_BUCKET_INFO = BucketInfo.newBuilder("b") .setAcl(ACLS) .setEtag(ETAG) @@ -101,6 +105,10 @@ public class BucketTest { .setLabels(BUCKET_LABELS) .setRequesterPays(REQUESTER_PAYS) .setDefaultKmsKeyName(DEFAULT_KMS_KEY_NAME) + .setDefaultEventBasedHold(DEFAULT_EVENT_BASED_HOLD) + .setRetentionEffectiveTime(RETENTION_EFFECTIVE_TIME) + .setRetentionPeriod(RETENTION_PERIOD) + .setRetentionPolicyIsLocked(RETENTION_POLICY_IS_LOCKED) .build(); private static final BucketInfo BUCKET_INFO = BucketInfo.newBuilder("b").setMetageneration(42L).build(); @@ -659,6 +667,23 @@ public void testListDefaultAcls() throws Exception { assertEquals(ACLS, bucket.listDefaultAcls()); } + @Test + public void testLockRetention() throws Exception { + initializeExpectedBucket(5); + Bucket expectedRetentionLockedBucket = expectedBucket.toBuilder().setRetentionPeriod(RETENTION_PERIOD) + .setRetentionPolicyIsLocked(true).build(); + expect(storage.getOptions()).andReturn(mockOptions).times(2); + expect(storage.lockRetentionPolicy(expectedRetentionLockedBucket, Storage.BucketTargetOption.metagenerationMatch(), + Storage.BucketTargetOption.userProject(USER_PROJECT))).andReturn(expectedRetentionLockedBucket); + replay(storage); + initializeBucket(); + Bucket lockedRetentionPolicyBucket = new Bucket(storage, new BucketInfo.BuilderImpl(expectedRetentionLockedBucket)); + Bucket actualRetentionLockedBucket = lockedRetentionPolicyBucket + .lockRetentionPolicy(Storage.BucketTargetOption.metagenerationMatch(), + Storage.BucketTargetOption.userProject(USER_PROJECT)); + assertEquals(expectedRetentionLockedBucket, actualRetentionLockedBucket); + } + @Test public void testToBuilder() { expect(storage.getOptions()).andReturn(mockOptions).times(4); @@ -694,6 +719,10 @@ public void testBuilder() { .setLabels(BUCKET_LABELS) .setRequesterPays(REQUESTER_PAYS) .setDefaultKmsKeyName(DEFAULT_KMS_KEY_NAME) + .setDefaultEventBasedHold(DEFAULT_EVENT_BASED_HOLD) + .setRetentionEffectiveTime(RETENTION_EFFECTIVE_TIME) + .setRetentionPeriod(RETENTION_PERIOD) + .setRetentionPolicyIsLocked(RETENTION_POLICY_IS_LOCKED) .build(); assertEquals("b", bucket.getName()); assertEquals(ACLS, bucket.getAcl()); @@ -714,6 +743,10 @@ public void testBuilder() { assertEquals(BUCKET_LABELS, bucket.getLabels()); assertEquals(REQUESTER_PAYS, bucket.requesterPays()); assertEquals(DEFAULT_KMS_KEY_NAME, bucket.getDefaultKmsKeyName()); + assertEquals(DEFAULT_EVENT_BASED_HOLD, bucket.getDefaultEventBasedHold()); + assertEquals(RETENTION_EFFECTIVE_TIME, bucket.getRetentionEffectiveTime()); + assertEquals(RETENTION_PERIOD, bucket.getRetentionPeriod()); + assertEquals(RETENTION_POLICY_IS_LOCKED, bucket.retentionPolicyIsLocked()); assertEquals(storage.getOptions(), bucket.getStorage().getOptions()); } } diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index 9fea3887c464..574c02ea001f 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -92,6 +92,7 @@ public class StorageImplTest { private static final String BUCKET_NAME1 = "b1"; private static final String BUCKET_NAME2 = "b2"; + private static final String BUCKET_NAME3 = "b3"; private static final String BLOB_NAME1 = "n1"; private static final String BLOB_NAME2 = "n2"; private static final String BLOB_NAME3 = "n3"; @@ -104,11 +105,15 @@ public class StorageImplTest { new SecretKeySpec(BaseEncoding.base64().decode(BASE64_KEY), "AES256"); private static final String KMS_KEY_NAME = "projects/gcloud-devel/locations/us/keyRings/gcs_kms_key_ring_us/cryptoKeys/key"; + private static final Long RETENTION_PERIOD = 10L; + private static final String USER_PROJECT = "test-project"; // BucketInfo objects private static final BucketInfo BUCKET_INFO1 = BucketInfo.newBuilder(BUCKET_NAME1).setMetageneration(42L).build(); private static final BucketInfo BUCKET_INFO2 = BucketInfo.newBuilder(BUCKET_NAME2).build(); + private static final BucketInfo BUCKET_INFO3 = BucketInfo.newBuilder(BUCKET_NAME3) + .setRetentionPeriod(RETENTION_PERIOD).setRetentionPolicyIsLocked(true).setMetageneration(42L).build(); // BlobInfo objects private static final BlobInfo BLOB_INFO1 = @@ -128,10 +133,16 @@ public class StorageImplTest { Storage.BucketTargetOption.metagenerationMatch(); private static final Storage.BucketTargetOption BUCKET_TARGET_PREDEFINED_ACL = Storage.BucketTargetOption.predefinedAcl(Storage.PredefinedAcl.PRIVATE); + private static final Storage.BucketTargetOption BUCKET_TARGET_USER_PROJECT = + Storage.BucketTargetOption.userProject(USER_PROJECT); private static final Map BUCKET_TARGET_OPTIONS = ImmutableMap.of( StorageRpc.Option.IF_METAGENERATION_MATCH, BUCKET_INFO1.getMetageneration(), StorageRpc.Option.PREDEFINED_ACL, BUCKET_TARGET_PREDEFINED_ACL.getValue()); + private static final Map BUCKET_TARGET_OPTIONS_LOCK_RETENTION_POLICY = + ImmutableMap.of( + StorageRpc.Option.IF_METAGENERATION_MATCH, BUCKET_INFO3.getMetageneration(), + StorageRpc.Option.USER_PROJECT, USER_PROJECT); // Blob target options (create, update, compose) private static final BlobTargetOption BLOB_TARGET_GENERATION = BlobTargetOption.generationMatch(); @@ -320,7 +331,7 @@ public long millisTime() { private Storage storage; private Blob expectedBlob1, expectedBlob2, expectedBlob3; - private Bucket expectedBucket1, expectedBucket2; + private Bucket expectedBucket1, expectedBucket2, expectedBucket3; @Rule public ExpectedException thrown = ExpectedException.none(); @@ -367,6 +378,7 @@ private void initializeServiceDependentObjects() { expectedBlob3 = new Blob(storage, new BlobInfo.BuilderImpl(BLOB_INFO3)); expectedBucket1 = new Bucket(storage, new BucketInfo.BuilderImpl(BUCKET_INFO1)); expectedBucket2 = new Bucket(storage, new BucketInfo.BuilderImpl(BUCKET_INFO2)); + expectedBucket3 = new Bucket(storage, new BucketInfo.BuilderImpl(BUCKET_INFO3)); } @Test @@ -2325,6 +2337,17 @@ public void testTestIamPermissionsNonNull() { assertEquals(expectedPermissions, storage.testIamPermissions(BUCKET_NAME1, checkedPermissions)); } + @Test + public void testLockRetentionPolicy() { + EasyMock.expect(storageRpcMock + .lockRetentionPolicy(BUCKET_INFO3.toPb(), BUCKET_TARGET_OPTIONS_LOCK_RETENTION_POLICY)) + .andReturn(BUCKET_INFO3.toPb()); + EasyMock.replay(storageRpcMock); + initializeService(); + Bucket bucket = storage.lockRetentionPolicy(BUCKET_INFO3, BUCKET_TARGET_METAGENERATION, BUCKET_TARGET_USER_PROJECT); + assertEquals(expectedBucket3, bucket); + } + @Test public void testGetServiceAccount() { EasyMock.expect(storageRpcMock.getServiceAccount("projectId")) diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index a1330e946738..5b407b30bf32 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -132,6 +132,8 @@ public class ITStorageTest { private static final byte[] COMPRESSED_CONTENT = BaseEncoding.base64() .decode("H4sIAAAAAAAAAPNIzcnJV3DPz0/PSVVwzskvTVEILskvSkxPVQQA/LySchsAAAA="); private static final Map BUCKET_LABELS = ImmutableMap.of("label1", "value1"); + private static final Long RETENTION_PERIOD = 5L; + private static final Long RETENTION_PERIOD_IN_MILLISECONDS = RETENTION_PERIOD * 1000; private static final String SERVICE_ACCOUNT_EMAIL_SUFFIX = "@gs-project-accounts.iam.gserviceaccount.com"; private static final String KMS_KEY_RING_NAME = "gcs_test_kms_key_ring"; private static final String KMS_KEY_RING_LOCATION = "us"; @@ -2038,7 +2040,155 @@ public void testListBucketDefaultKmsKeyName() throws ExecutionException, Interru } @Test - public void testGetServiceAccount() throws InterruptedException { + public void testRetentionPolicyNoLock() throws ExecutionException, InterruptedException { + String bucketName = RemoteStorageHelper.generateBucketName(); + Bucket remoteBucket = storage.create(BucketInfo.newBuilder(bucketName) + .setRetentionPeriod(RETENTION_PERIOD).build()); + try { + assertEquals(RETENTION_PERIOD, remoteBucket.getRetentionPeriod()); + assertNotNull(remoteBucket.getRetentionEffectiveTime()); + assertNull(remoteBucket.retentionPolicyIsLocked()); + remoteBucket = storage.get(bucketName, Storage.BucketGetOption.fields(BucketField.RETENTION_POLICY)); + assertEquals(RETENTION_PERIOD, remoteBucket.getRetentionPeriod()); + assertNotNull(remoteBucket.getRetentionEffectiveTime()); + assertNull(remoteBucket.retentionPolicyIsLocked()); + String blobName = "test-create-with-retention-policy-hold"; + BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, blobName).build(); + Blob remoteBlob = storage.create(blobInfo); + assertNotNull(remoteBlob.getRetentionExpirationTime()); + remoteBucket = remoteBucket.toBuilder().setRetentionPeriod(null).build().update(); + assertNull(remoteBucket.getRetentionPeriod()); + remoteBucket = remoteBucket.toBuilder().setRetentionPeriod(null).build().update(); + assertNull(remoteBucket.getRetentionPeriod()); + } finally { + RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); + } + } + + @Test + public void testRetentionPolicyLock() throws ExecutionException, InterruptedException { + retentionPolicyLockRequesterPays(true); + retentionPolicyLockRequesterPays(false); + } + + private void retentionPolicyLockRequesterPays(boolean requesterPays) throws ExecutionException, InterruptedException { + String projectId = remoteStorageHelper.getOptions().getProjectId(); + String bucketName = RemoteStorageHelper.generateBucketName(); + BucketInfo bucketInfo; + if (requesterPays) { + bucketInfo = BucketInfo.newBuilder(bucketName).setRetentionPeriod(RETENTION_PERIOD) + .setRequesterPays(true).build(); + } else { + bucketInfo = BucketInfo.newBuilder(bucketName).setRetentionPeriod(RETENTION_PERIOD).build(); + } + Bucket remoteBucket = storage.create(bucketInfo); + try { + assertNull(remoteBucket.retentionPolicyIsLocked()); + assertNotNull(remoteBucket.getRetentionEffectiveTime()); + assertNotNull(remoteBucket.getMetageneration()); + if (requesterPays) { + remoteBucket = storage.lockRetentionPolicy(remoteBucket, Storage.BucketTargetOption.metagenerationMatch(), + Storage.BucketTargetOption.userProject(projectId)); + } else { + remoteBucket = storage.lockRetentionPolicy(remoteBucket, Storage.BucketTargetOption.metagenerationMatch()); + } + assertTrue(remoteBucket.retentionPolicyIsLocked()); + assertNotNull(remoteBucket.getRetentionEffectiveTime()); + } finally { + RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); + } + } + + @Test + public void testAttemptObjectDeleteWithRetentionPolicy() throws ExecutionException, InterruptedException { + String bucketName = RemoteStorageHelper.generateBucketName(); + Bucket remoteBucket = storage.create(BucketInfo.newBuilder(bucketName) + .setRetentionPeriod(RETENTION_PERIOD).build()); + assertEquals(RETENTION_PERIOD, remoteBucket.getRetentionPeriod()); + String blobName = "test-create-with-retention-policy"; + BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, blobName).build(); + Blob remoteBlob = storage.create(blobInfo); + assertNotNull(remoteBlob.getRetentionExpirationTime()); + try { + remoteBlob.delete(); + fail("Expected failure on delete from retentionPolicy"); + } catch (StorageException ex) { + // expected + } finally { + Thread.sleep(RETENTION_PERIOD_IN_MILLISECONDS); + RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); + } + } + + @Test + public void testEnableDisableBucketDefaultEventBasedHold() throws ExecutionException, InterruptedException { + String bucketName = RemoteStorageHelper.generateBucketName(); + Bucket remoteBucket = storage.create(BucketInfo.newBuilder(bucketName).setDefaultEventBasedHold(true).build()); + try { + assertTrue(remoteBucket.getDefaultEventBasedHold()); + remoteBucket = storage.get(bucketName, Storage.BucketGetOption.fields(BucketField.DEFAULT_EVENT_BASED_HOLD)); + assertTrue(remoteBucket.getDefaultEventBasedHold()); + String blobName = "test-create-with-event-based-hold"; + BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, blobName).build(); + Blob remoteBlob = storage.create(blobInfo); + assertTrue(remoteBlob.getEventBasedHold()); + remoteBlob = storage.get(blobInfo.getBlobId(), Storage.BlobGetOption.fields(BlobField.EVENT_BASED_HOLD)); + assertTrue(remoteBlob.getEventBasedHold()); + remoteBlob = remoteBlob.toBuilder().setEventBasedHold(false).build().update(); + assertFalse(remoteBlob.getEventBasedHold()); + remoteBucket = remoteBucket.toBuilder().setDefaultEventBasedHold(false).build().update(); + assertFalse(remoteBucket.getDefaultEventBasedHold()); + } finally { + RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); + } + } + + @Test + public void testEnableDisableTemporaryHold() { + String blobName = "test-create-with-temporary-hold"; + BlobInfo blobInfo = BlobInfo.newBuilder(BUCKET, blobName).setTemporaryHold(true).build(); + Blob remoteBlob = storage.create(blobInfo); + assertTrue(remoteBlob.getTemporaryHold()); + remoteBlob = storage.get(remoteBlob.getBlobId(), Storage.BlobGetOption.fields(BlobField.TEMPORARY_HOLD)); + assertTrue(remoteBlob.getTemporaryHold()); + remoteBlob = remoteBlob.toBuilder().setTemporaryHold(false).build().update(); + assertFalse(remoteBlob.getTemporaryHold()); + } + + @Test + public void testAttemptObjectDeleteWithEventBasedHold() { + String blobName = "test-create-with-event-based-hold"; + BlobInfo blobInfo = BlobInfo.newBuilder(BUCKET, blobName).setEventBasedHold(true).build(); + Blob remoteBlob = storage.create(blobInfo); + assertTrue(remoteBlob.getEventBasedHold()); + try { + remoteBlob.delete(); + fail("Expected failure on delete from eventBasedHold"); + } catch (StorageException ex) { + // expected + } finally { + remoteBlob.toBuilder().setEventBasedHold(false).build().update(); + } + } + + @Test + public void testAttemptDeletionObjectTemporaryHold() { + String blobName = "test-create-with-temporary-hold"; + BlobInfo blobInfo = BlobInfo.newBuilder(BUCKET, blobName).setTemporaryHold(true).build(); + Blob remoteBlob = storage.create(blobInfo); + assertTrue(remoteBlob.getTemporaryHold()); + try { + remoteBlob.delete(); + fail("Expected failure on delete from temporaryHold"); + } catch (StorageException ex) { + // expected + } finally { + remoteBlob.toBuilder().setEventBasedHold(false).build().update(); + } + } + + @Test + public void testGetServiceAccount() { String projectId = remoteStorageHelper.getOptions().getProjectId(); ServiceAccount serviceAccount = storage.getServiceAccount(projectId); assertNotNull(serviceAccount); diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/StorageSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/StorageSnippets.java index 9d943c87331e..7d72f9df4af3 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/StorageSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/StorageSnippets.java @@ -57,6 +57,7 @@ import com.google.cloud.storage.StorageClass; import com.google.cloud.storage.StorageException; import com.google.cloud.storage.StorageOptions; +import java.util.Date; import java.io.ByteArrayInputStream; import java.io.FileInputStream; @@ -1113,4 +1114,280 @@ public Bucket setDefaultKmsKey(String bucketName, String kmsKeyName) throws Stor // [END storage_set_bucket_default_kms_key] return bucket; } + + /** Example of displaying Blob metadata */ + public void getBlobMetadata(String bucketName, String blobName) throws StorageException { + // [START storage_get_metadata] + // Instantiate a Google Cloud Storage client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name of a bucket, e.g. "my-bucket" + // String bucketName = "my-bucket"; + + // The name of a blob, e.g. "my-blob" + // String blobName = "my-blob"; + + // Select all fields + // Fields can be selected individually e.g. Storage.BlobField.CACHE_CONTROL + Blob blob = storage.get(bucketName, blobName, BlobGetOption.fields(Storage.BlobField.values())); + + // Print blob metadata + System.out.println("Bucket: " + blob.getBucket()); + System.out.println("CacheControl: " + blob.getCacheControl()); + System.out.println("ComponentCount: " + blob.getComponentCount()); + System.out.println("ContentDisposition: " + blob.getContentDisposition()); + System.out.println("ContentEncoding: " + blob.getContentEncoding()); + System.out.println("ContentLanguage: " + blob.getContentLanguage()); + System.out.println("ContentType: " + blob.getContentType()); + System.out.println("Crc32c: " + blob.getCrc32c()); + System.out.println("ETag: " + blob.getEtag()); + System.out.println("Generation: " + blob.getGeneration()); + System.out.println("Id: " + blob.getBlobId()); + System.out.println("KmsKeyName: " + blob.getKmsKeyName()); + System.out.println("Md5Hash: " + blob.getMd5()); + System.out.println("MediaLink: " + blob.getMediaLink()); + System.out.println("Metageneration: " + blob.getMetageneration()); + System.out.println("Name: " + blob.getName()); + System.out.println("Size: " + blob.getSize()); + System.out.println("StorageClass: " + blob.getStorageClass()); + System.out.println("TimeCreated: " + new Date(blob.getCreateTime())); + System.out.println("Last Metadata Update: " + new Date(blob.getUpdateTime())); + Boolean temporaryHoldIsEnabled = (blob.getTemporaryHold() != null && blob.getTemporaryHold()); + System.out.println("temporaryHold: " + (temporaryHoldIsEnabled ? "enabled" : "disabled")); + Boolean eventBasedHoldIsEnabled = (blob.getEventBasedHold() != null && blob.getEventBasedHold()); + System.out.println("eventBasedHold: " + (eventBasedHoldIsEnabled ? "enabled" : "disabled")); + if (blob.getRetentionExpirationTime() != null) { + System.out.println("retentionExpirationTime: " + new Date(blob.getRetentionExpirationTime())); + } + if (blob.getMetadata() != null) { + System.out.println("\n\n\nUser metadata:"); + for (Map.Entry userMetadata : blob.getMetadata().entrySet()) { + System.out.println(userMetadata.getKey() + "=" + userMetadata.getValue()); + } + } + // [END storage_get_metadata] + } + + /** Example of setting a retention policy on a bucket */ + public Bucket setRetentionPolicy(String bucketName, Long retentionPeriod) + throws StorageException { + // [START storage_set_retention_policy] + // Instantiate a Google Cloud Storage client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name of a bucket, e.g. "my-bucket" + // String bucketName = "my-bucket"; + + // The retention period for objects in bucket + // Long retentionPeriod = 3600L; // 1 hour in seconds + + Bucket bucketWithRetentionPolicy = + storage.update( + BucketInfo.newBuilder(bucketName).setRetentionPeriod(retentionPeriod).build()); + + System.out.println( + "Retention period for " + bucketName + " is now " + bucketWithRetentionPolicy.getRetentionPeriod()); + // [END storage_set_retention_policy] + return bucketWithRetentionPolicy; + } + + /** Example of removing a retention policy on a bucket */ + public Bucket removeRetentionPolicy(String bucketName) throws StorageException, IllegalArgumentException { + // [START storage_remove_retention_policy] + // Instantiate a Google Cloud Storage client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name of a bucket, e.g. "my-bucket" + // String bucketName = "my-bucket"; + + Bucket bucket = storage.get(bucketName, BucketGetOption.fields(BucketField.RETENTION_POLICY)); + if (bucket.retentionPolicyIsLocked() != null && bucket.retentionPolicyIsLocked()) { + throw new IllegalArgumentException("Unable to remove retention period as retention policy is locked."); + } + + Bucket bucketWithoutRetentionPolicy = bucket.toBuilder().setRetentionPeriod(null).build().update(); + + System.out.println("Retention period for " + bucketName + " has been removed"); + // [END storage_remove_retention_policy] + return bucketWithoutRetentionPolicy; + } + + /** Example of how to get a bucket's retention policy */ + public Bucket getRetentionPolicy(String bucketName) throws StorageException { + // [START storage_get_retention_policy] + // Instantiate a Google Cloud Storage client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name of a bucket, e.g. "my-bucket" + // String bucketName = "my-bucket"; + + Bucket bucket = storage.get(bucketName, BucketGetOption.fields(BucketField.RETENTION_POLICY)); + + System.out.println("Retention Policy for " + bucketName); + System.out.println("Retention Period: " + bucket.getRetentionPeriod()); + if (bucket.retentionPolicyIsLocked() != null && bucket.retentionPolicyIsLocked()) { + System.out.println("Retention Policy is locked"); + } + if (bucket.getRetentionEffectiveTime() != null) { + System.out.println("Effective Time: " + new Date(bucket.getRetentionEffectiveTime())); + } + // [END storage_get_retention_policy] + return bucket; + } + + /** Example of how to lock a bucket retention policy */ + public Bucket lockRetentionPolicy(String bucketName) throws StorageException { + // [START storage_lock_retention_policy] + // Instantiate a Google Cloud Storage client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name of a bucket, e.g. "my-bucket" + // String bucketName = "my-bucket"; + + Bucket bucket = + storage.get(bucketName, Storage.BucketGetOption.fields(BucketField.METAGENERATION)); + Bucket lockedBucket = + bucket.lockRetentionPolicy(Storage.BucketTargetOption.metagenerationMatch()); + + System.out.println("Retention period for " + bucketName + " is now locked"); + System.out.println( + "Retention policy effective as of " + new Date(lockedBucket.getRetentionEffectiveTime())); + // [END storage_lock_retention_policy] + return lockedBucket; + } + + /** Example of how to enable default event-based hold for a bucket */ + public Bucket enableDefaultEventBasedHold(String bucketName) throws StorageException { + // [START storage_enable_default_event_based_hold] + // Instantiate a Google Cloud Storage client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name of a bucket, e.g. "my-bucket" + // String bucketName = "my-bucket"; + + Bucket bucket = + storage.update(BucketInfo.newBuilder(bucketName).setDefaultEventBasedHold(true).build()); + + System.out.println("Default event-based hold was enabled for " + bucketName); + // [END storage_enable_default_event_based_hold] + return bucket; + } + + /** Example of how to disable default event-based hold for a bucket */ + public Bucket disableDefaultEventBasedHold(String bucketName) throws StorageException { + // [START storage_disable_default_event_based_hold] + // Instantiate a Google Cloud Storage client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name of a bucket, e.g. "my-bucket" + // String bucketName = "my-bucket"; + + Bucket bucket = + storage.update(BucketInfo.newBuilder(bucketName).setDefaultEventBasedHold(false).build()); + + System.out.println("Default event-based hold was disabled for " + bucketName); + // [END storage_disable_default_event_based_hold] + return bucket; + } + + /** Example of how to get default event-based hold for a bucket */ + public Bucket getDefaultEventBasedHold(String bucketName) throws StorageException { + // [START storage_get_default_event_based_hold] + // Instantiate a Google Cloud Storage client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name of a bucket, e.g. "my-bucket" + // String bucketName = "my-bucket"; + + Bucket bucket = + storage.get(bucketName, BucketGetOption.fields(BucketField.DEFAULT_EVENT_BASED_HOLD)); + + if (bucket.getDefaultEventBasedHold() != null && bucket.getDefaultEventBasedHold()) { + System.out.println("Default event-based hold is enabled for " + bucketName); + } else { + System.out.println("Default event-based hold is not enabled for " + bucketName); + } + // [END storage_get_default_event_based_hold] + return bucket; + } + + /** Example of how to set event-based hold for a blob */ + public Blob setEventBasedHold(String bucketName, String blobName) throws StorageException { + // [START storage_set_event_based_hold] + // Instantiate a Google Cloud Storage client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name of a bucket, e.g. "my-bucket" + // String bucketName = "my-bucket"; + + // The name of a blob, e.g. "my-blob" + // String blobName = "my-blob"; + + BlobId blobId = BlobId.of(bucketName, blobName); + Blob blob = storage.update(BlobInfo.newBuilder(blobId).setEventBasedHold(true).build()); + + System.out.println("Event-based hold was set for " + blobName); + // [END storage_set_event_based_hold] + return blob; + } + + /** Example of how to release event-based hold for a blob */ + public Blob releaseEventBasedHold(String bucketName, String blobName) throws StorageException { + // [START storage_release_event_based_hold] + // Instantiate a Google Cloud Storage client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name of a bucket, e.g. "my-bucket" + // String bucketName = "my-bucket"; + + // The name of a blob, e.g. "my-blob" + // String blobName = "my-blob"; + + BlobId blobId = BlobId.of(bucketName, blobName); + Blob blob = storage.update(BlobInfo.newBuilder(blobId).setEventBasedHold(false).build()); + + System.out.println("Event-based hold was released for " + blobName); + // [END storage_release_event_based_hold] + return blob; + } + + /** Example of how to set a temporary hold for a blob */ + public Blob setTemporaryHold(String bucketName, String blobName) throws StorageException { + // [START storage_set_temporary_hold] + // Instantiate a Google Cloud Storage client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name of a bucket, e.g. "my-bucket" + // String bucketName = "my-bucket"; + + // The name of a blob, e.g. "my-blob" + // String blobName = "my-blob"; + + BlobId blobId = BlobId.of(bucketName, blobName); + Blob blob = storage.update(BlobInfo.newBuilder(blobId).setTemporaryHold(true).build()); + + System.out.println("Temporary hold was set for " + blobName); + // [END storage_set_temporary_hold] + return blob; + } + + /** Example of how to release a temporary hold for a blob */ + public Blob releaseTemporaryHold(String bucketName, String blobName) throws StorageException { + // [START storage_release_temporary_hold] + // Instantiate a Google Cloud Storage client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name of a bucket, e.g. "my-bucket" + // String bucketName = "my-bucket"; + + // The name of a blob, e.g. "my-blob" + // String blobName = "my-blob"; + + BlobId blobId = BlobId.of(bucketName, blobName); + Blob blob = storage.update(BlobInfo.newBuilder(blobId).setTemporaryHold(false).build()); + + System.out.println("Temporary hold was released for " + blobName); + // [END storage_release_temporary_hold] + return blob; + } } diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/storage/snippets/ITStorageSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/storage/snippets/ITStorageSnippets.java index fffd0c3eb5b5..edfbdad30514 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/storage/snippets/ITStorageSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/storage/snippets/ITStorageSnippets.java @@ -38,6 +38,7 @@ import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageException; import com.google.cloud.storage.testing.RemoteStorageHelper; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterators; import com.google.common.collect.Sets; @@ -48,13 +49,15 @@ import org.junit.rules.ExpectedException; import org.junit.rules.Timeout; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.PrintStream; import java.net.URL; import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.ArrayList; +import java.util.Date; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -74,7 +77,7 @@ public class ITStorageSnippets { private static final String KMS_KEY_NAME = "projects/gcloud-devel/locations/us/" + "keyRings/gcs_kms_key_ring_us/cryptoKeys/key"; - + private static final Long RETENTION_PERIOD = 5L; // 5 seconds private static Storage storage; private static StorageSnippets storageSnippets; private static List bucketsToCleanUp; @@ -87,20 +90,22 @@ public class ITStorageSnippets { public static void beforeClass() { RemoteStorageHelper helper = RemoteStorageHelper.create(); storage = helper.getOptions().getService(); - bucketsToCleanUp = new ArrayList(); storageSnippets = new StorageSnippets(storage); storageSnippets.createBucket(BUCKET); - bucketsToCleanUp.add(BUCKET); } @AfterClass public static void afterClass() throws ExecutionException, InterruptedException { if (storage != null) { - for (String bucket : bucketsToCleanUp) { - boolean wasDeleted = RemoteStorageHelper.forceDelete(storage, bucket, 5, TimeUnit.SECONDS); - if (!wasDeleted && log.isLoggable(Level.WARNING)) { - log.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", bucket); - } + // In beforeClass, we make buckets auto-delete blobs older than a day old. + // Here, delete all buckets older than 2 days. They should already be empty and easy. + long cleanTime = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(2); + long cleanTimeout = System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(1); + RemoteStorageHelper.cleanBuckets(storage, cleanTime, cleanTimeout); + + boolean wasDeleted = RemoteStorageHelper.forceDelete(storage, BUCKET, 1, TimeUnit.MINUTES); + if (!wasDeleted && log.isLoggable(Level.WARNING)) { + log.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", BUCKET); } } } @@ -109,7 +114,6 @@ public static void afterClass() throws ExecutionException, InterruptedException public void testCreateBucketWithStorageClassAndLocation() throws ExecutionException, InterruptedException { String tempBucket = RemoteStorageHelper.generateBucketName(); - bucketsToCleanUp.add(tempBucket); Bucket bucket = storageSnippets.createBucketWithStorageClassAndLocation(tempBucket); @@ -435,6 +439,45 @@ public void testBlobDownload() throws Exception { assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); } + @Test + public void testGetBlobMetadata() { + String blobName = "test-create-empty-blob"; + BlobId blobId = BlobId.of(BUCKET, blobName); + BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setMetadata(ImmutableMap.of("k", "v")).build(); + Blob remoteBlob = storage.create(blobInfo, BLOB_BYTE_CONTENT); + assertNotNull(remoteBlob); + final ByteArrayOutputStream snippetOutputCapture = new ByteArrayOutputStream(); + System.setOut(new PrintStream(snippetOutputCapture)); + storageSnippets.getBlobMetadata(BUCKET, blobName); + String snippetOutput = snippetOutputCapture.toString(); + System.setOut(System.out); + assertTrue(snippetOutput.contains("Bucket: " + remoteBlob.getBucket())); + assertTrue(snippetOutput.contains("Bucket: " + remoteBlob.getBucket())); + assertTrue(snippetOutput.contains("CacheControl: " + remoteBlob.getCacheControl())); + assertTrue(snippetOutput.contains("ComponentCount: " + remoteBlob.getComponentCount())); + assertTrue(snippetOutput.contains("ContentDisposition: " + remoteBlob.getContentDisposition())); + assertTrue(snippetOutput.contains("ContentEncoding: " + remoteBlob.getContentEncoding())); + assertTrue(snippetOutput.contains("ContentLanguage: " + remoteBlob.getContentLanguage())); + assertTrue(snippetOutput.contains("ContentType: " + remoteBlob.getContentType())); + assertTrue(snippetOutput.contains("Crc32c: " + remoteBlob.getCrc32c())); + assertTrue(snippetOutput.contains("ETag: " + remoteBlob.getEtag())); + assertTrue(snippetOutput.contains("Generation: " + remoteBlob.getGeneration())); + assertTrue(snippetOutput.contains("Id: " + remoteBlob.getBlobId())); + assertTrue(snippetOutput.contains("KmsKeyName: " + remoteBlob.getKmsKeyName())); + assertTrue(snippetOutput.contains("Md5Hash: " + remoteBlob.getMd5())); + assertTrue(snippetOutput.contains("MediaLink: " + remoteBlob.getMediaLink())); + assertTrue(snippetOutput.contains("Metageneration: " + remoteBlob.getMetageneration())); + assertTrue(snippetOutput.contains("Name: " + remoteBlob.getName())); + assertTrue(snippetOutput.contains("Size: " + remoteBlob.getSize())); + assertTrue(snippetOutput.contains("StorageClass: " + remoteBlob.getStorageClass())); + assertTrue(snippetOutput.contains("TimeCreated: " + new Date(remoteBlob.getCreateTime()))); + assertTrue(snippetOutput.contains("Last Metadata Update: " + new Date(remoteBlob.getUpdateTime()))); + assertTrue(snippetOutput.contains("temporaryHold: disabled")); + assertTrue(snippetOutput.contains("eventBasedHold: disabled")); + assertTrue(snippetOutput.contains("User metadata:")); + assertTrue(snippetOutput.contains("k=v")); + } + @Test public void testRequesterPays() throws Exception { Bucket bucket = storageSnippets.enableRequesterPays(BUCKET); @@ -461,4 +504,47 @@ public void testDefaultKMSKey() { // Remove default key storageSnippets.setDefaultKmsKey(BUCKET, null); } + + @Test + public void testBucketRetention() { + Bucket bucket = storageSnippets.setRetentionPolicy(BUCKET, RETENTION_PERIOD); + assertEquals(bucket.getRetentionPeriod(), RETENTION_PERIOD); + assertNotNull(bucket.getRetentionEffectiveTime()); + bucket = storageSnippets.getRetentionPolicy(BUCKET); + assertEquals(bucket.getRetentionPeriod(), RETENTION_PERIOD); + assertNotNull(bucket.getRetentionEffectiveTime()); + assertNull(bucket.retentionPolicyIsLocked()); + bucket = storageSnippets.enableDefaultEventBasedHold(BUCKET); + assertTrue(bucket.getDefaultEventBasedHold()); + bucket = storageSnippets.getDefaultEventBasedHold(BUCKET); + assertTrue(bucket.getDefaultEventBasedHold()); + String blobName = "test-create-empty-blob-retention-policy"; + Blob remoteBlob = bucket.create(blobName, BLOB_BYTE_CONTENT); + assertTrue(remoteBlob.getEventBasedHold()); + remoteBlob = storageSnippets.setEventBasedHold(BUCKET, blobName); + assertTrue(remoteBlob.getEventBasedHold()); + remoteBlob = storageSnippets.releaseEventBasedHold(BUCKET, blobName); + assertFalse(remoteBlob.getEventBasedHold()); + assertNotNull(remoteBlob.getRetentionExpirationTime()); + bucket = storageSnippets.removeRetentionPolicy(BUCKET); + assertNull(bucket.getRetentionPeriod()); + assertNull(bucket.getRetentionEffectiveTime()); + bucket = storageSnippets.disableDefaultEventBasedHold(BUCKET); + assertFalse(bucket.getDefaultEventBasedHold()); + remoteBlob = storageSnippets.setTemporaryHold(BUCKET, blobName); + assertTrue(remoteBlob.getTemporaryHold()); + remoteBlob = storageSnippets.releaseTemporaryHold(BUCKET, blobName); + assertFalse(remoteBlob.getTemporaryHold()); + } + + @Test + public void testLockRetentionPolicy() { + String tempBucket = RemoteStorageHelper.generateBucketName(); + Bucket bucket = storageSnippets.createBucket(tempBucket); + assertNotNull(bucket); + bucket = storageSnippets.setRetentionPolicy(tempBucket, RETENTION_PERIOD); + assertEquals(bucket.getRetentionPeriod(), RETENTION_PERIOD); + bucket = storageSnippets.lockRetentionPolicy(tempBucket); + assertTrue(bucket.retentionPolicyIsLocked()); + } }