Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add object retention feature #2277

Merged
merged 16 commits into from
Nov 30, 2023
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,20 +50,20 @@ If you are using Maven without the BOM, add this to your dependencies:
If you are using Gradle 5.x or later, add this to your dependencies:

```Groovy
implementation platform('com.google.cloud:libraries-bom:26.24.0')
implementation platform('com.google.cloud:libraries-bom:26.25.0')

implementation 'com.google.cloud:google-cloud-storage'
```
If you are using Gradle without BOM, add this to your dependencies:

```Groovy
implementation 'com.google.cloud:google-cloud-storage:2.28.0'
implementation 'com.google.cloud:google-cloud-storage:2.29.0'
```

If you are using SBT, add this to your dependencies:

```Scala
libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "2.28.0"
libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "2.29.0"
```
<!-- {x-version-update-end} -->

Expand Down Expand Up @@ -428,7 +428,7 @@ Java is a registered trademark of Oracle and/or its affiliates.
[kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-storage/java11.html
[stability-image]: https://img.shields.io/badge/stability-stable-green
[maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-storage.svg
[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-storage/2.28.0
[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-storage/2.29.0
[authentication]: https://github.com/googleapis/google-cloud-java#authentication
[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes
[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
import com.google.cloud.storage.Acl.Role;
import com.google.cloud.storage.Acl.User;
import com.google.cloud.storage.BlobInfo.CustomerEncryption;
import com.google.cloud.storage.BlobInfo.Retention;
import com.google.cloud.storage.BucketInfo.Autoclass;
import com.google.cloud.storage.BucketInfo.CustomPlacementConfig;
import com.google.cloud.storage.BucketInfo.IamConfiguration;
Expand All @@ -63,6 +64,7 @@
import com.google.cloud.storage.BucketInfo.LifecycleRule.LifecycleCondition;
import com.google.cloud.storage.BucketInfo.LifecycleRule.SetStorageClassLifecycleAction;
import com.google.cloud.storage.BucketInfo.Logging;
import com.google.cloud.storage.BucketInfo.ObjectRetention;
import com.google.cloud.storage.BucketInfo.PublicAccessPrevention;
import com.google.cloud.storage.Conversions.Codec;
import com.google.cloud.storage.Cors.Origin;
Expand Down Expand Up @@ -114,6 +116,9 @@ final class ApiaryConversions {
Codec.of(this::iamConfigEncode, this::iamConfigDecode);
private final Codec<Autoclass, Bucket.Autoclass> autoclassCodec =
Codec.of(this::autoclassEncode, this::autoclassDecode);

private final Codec<ObjectRetention, Bucket.ObjectRetention> objectRetentionCodec =
Codec.of(this::objectRetentionEncode, this::objectRetentionDecode);
private final Codec<LifecycleRule, Rule> lifecycleRuleCodec =
Codec.of(this::lifecycleRuleEncode, this::lifecycleRuleDecode);
private final Codec<LifecycleCondition, Condition> lifecycleConditionCodec =
Expand All @@ -124,6 +129,9 @@ final class ApiaryConversions {
private final Codec<CustomerEncryption, StorageObject.CustomerEncryption>
customerEncryptionCodec =
Codec.of(this::customerEncryptionEncode, this::customerEncryptionDecode);

private final Codec<Retention, StorageObject.Retention> retentionCodec =
Codec.of(this::retentionEncode, this::retentionDecode);
private final Codec<BlobId, StorageObject> blobIdCodec =
Codec.of(this::blobIdEncode, this::blobIdDecode);
private final Codec<BlobInfo, StorageObject> blobInfoCodec =
Expand Down Expand Up @@ -238,6 +246,11 @@ private StorageObject blobInfoEncode(BlobInfo from) {
from.getRetentionExpirationTimeOffsetDateTime(),
dateTimeCodec::encode,
to::setRetentionExpirationTime);
if (from.getRetention() == null) {
to.setRetention(Data.nullOf(StorageObject.Retention.class));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What was the conclusion on detecting if a user set retention to null vs. it not being modified?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It doesn't work, due to the modified fields not being passed along to the final patch call. I talked to ben and we agreed to just do it this way, there's precedent for it in the logging encoding

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for clarifying, is this value mutable once set?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Only certain cases work (i.e. going from Unlocked to Locked, reducing the retainUntilTime on an unlocked policy, and setting to null), and you have to pass in overrideUnlockedRetention for any of those cases to work, but yes

} else {
to.setRetention(this.retentionEncode(from.getRetention()));
}
to.setKmsKeyName(from.getKmsKeyName());
to.setEventBasedHold(from.getEventBasedHold());
to.setTemporaryHold(from.getTemporaryHold());
Expand Down Expand Up @@ -306,6 +319,7 @@ private BlobInfo blobInfoDecode(StorageObject from) {
from.getRetentionExpirationTime(),
dateTimeCodec::decode,
to::setRetentionExpirationTimeOffsetDateTime);
ifNonNull(from.getRetention(), this::retentionDecode, to::setRetention);
return to.build();
}

Expand All @@ -331,6 +345,20 @@ private CustomerEncryption customerEncryptionDecode(StorageObject.CustomerEncryp
return new CustomerEncryption(from.getEncryptionAlgorithm(), from.getKeySha256());
}

private StorageObject.Retention retentionEncode(Retention from) {
StorageObject.Retention to = new StorageObject.Retention();
ifNonNull(from.getMode(), Retention.Mode::toString, to::setMode);
ifNonNull(from.getRetainUntilTime(), dateTimeCodec::encode, to::setRetainUntilTime);
return to;
}

private Retention retentionDecode(StorageObject.Retention from) {
Retention.Builder to = Retention.newBuilder();
ifNonNull(from.getMode(), Retention.Mode::valueOf, to::setMode);
ifNonNull(from.getRetainUntilTime(), dateTimeCodec::decode, to::setRetainUntilTime);
return to.build();
}

private Bucket bucketInfoEncode(BucketInfo from) {
Bucket to = new Bucket();
ifNonNull(from.getProject(), projectNameCodec::encode, p -> to.set(PROJECT_ID_FIELD_NAME, p));
Expand Down Expand Up @@ -400,6 +428,7 @@ private Bucket bucketInfoEncode(BucketInfo from) {
from.getCustomPlacementConfig(),
this::customPlacementConfigEncode,
to::setCustomPlacementConfig);
ifNonNull(from.getObjectRetention(), this::objectRetentionEncode, to::setObjectRetention);
return to;
}

Expand Down Expand Up @@ -450,7 +479,7 @@ private BucketInfo bucketInfoDecode(com.google.api.services.storage.model.Bucket
from.getCustomPlacementConfig(),
this::customPlacementConfigDecode,
to::setCustomPlacementConfig);

ifNonNull(from.getObjectRetention(), this::objectRetentionDecode, to::setObjectRetention);
return to.build();
}

Expand Down Expand Up @@ -494,6 +523,18 @@ private Autoclass autoclassDecode(Bucket.Autoclass from) {
return to.build();
}

private Bucket.ObjectRetention objectRetentionEncode(ObjectRetention from) {
Bucket.ObjectRetention to = new Bucket.ObjectRetention();
ifNonNull(from.getMode(), ObjectRetention.Mode::toString, to::setMode);
return to;
}

private ObjectRetention objectRetentionDecode(Bucket.ObjectRetention from) {
ObjectRetention.Builder to = ObjectRetention.newBuilder();
ifNonNull(from.getMode(), ObjectRetention.Mode::valueOf, to::setMode);
return to.build();
}

private UniformBucketLevelAccess ublaEncode(IamConfiguration from) {
UniformBucketLevelAccess to = new UniformBucketLevelAccess();
to.setEnabled(from.isUniformBucketLevelAccessEnabled());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -526,6 +526,12 @@ Builder setRetentionExpirationTimeOffsetDateTime(OffsetDateTime retentionExpirat
return this;
}

@Override
public Builder setRetention(Retention retention) {
infoBuilder.setRetention(retention);
return this;
}

@Override
public Blob build() {
return new Blob(storage, infoBuilder);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,10 @@
import static com.google.common.base.Preconditions.checkNotNull;

import com.google.api.client.util.Data;
import com.google.api.core.ApiFunction;
import com.google.api.core.BetaApi;
import com.google.cloud.StringEnumType;
import com.google.cloud.StringEnumValue;
import com.google.cloud.storage.Storage.BlobField;
import com.google.cloud.storage.TransportCompatibility.Transport;
import com.google.cloud.storage.UnifiedOpts.NamedField;
Expand Down Expand Up @@ -104,6 +107,7 @@ public class BlobInfo implements Serializable {
private final Boolean eventBasedHold;
private final Boolean temporaryHold;
private final OffsetDateTime retentionExpirationTime;
private final Retention retention;
private final transient ImmutableSet<NamedField> modifiedFields;

/** This class is meant for internal use only. Users are discouraged from using this class. */
Expand Down Expand Up @@ -168,6 +172,116 @@ public final boolean equals(Object o) {
}
}

/**
* Defines a blob's Retention policy. Can only be used on objects in a retention-enabled bucket.
*/
public static class Retention implements Serializable {
BenWhitehead marked this conversation as resolved.
Show resolved Hide resolved

public static final class Mode extends StringEnumValue {
BenWhitehead marked this conversation as resolved.
Show resolved Hide resolved
private static final long serialVersionUID = 1973143582659557184L;

private Mode(String constant) {
super(constant);
}

private static final ApiFunction<String, Mode> CONSTRUCTOR = Mode::new;

private static final StringEnumType<Mode> type =
new StringEnumType<>(Mode.class, CONSTRUCTOR);

public static final Mode UNLOCKED = type.createAndRegister("Unlocked");

public static final Mode LOCKED = type.createAndRegister("Locked");

public static Mode valueOfStrict(String constant) {
return type.valueOfStrict(constant);
}

public static Mode valueOf(String constant) {
return type.valueOf(constant);
}

public static Mode[] values() {
return type.values();
}
}

private static final long serialVersionUID = 5046718464542688444L;

private Mode mode;

private OffsetDateTime retainUntilTime;

/** Returns the retention policy's Mode. Can be Locked or Unlocked. */
public Mode getMode() {
return mode;
}

/** Returns what time this object will be retained until, if the mode is Locked. */
public OffsetDateTime getRetainUntilTime() {
return retainUntilTime;
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof Retention)) {
return false;
}
Retention that = (Retention) o;
return Objects.equals(mode, that.mode)
&& Objects.equals(retainUntilTime, that.retainUntilTime);
}

@Override
public int hashCode() {
return Objects.hash(mode, retainUntilTime);
}

@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("mode", mode)
.add("retainUntilTime", retainUntilTime)
.toString();
}

public static Builder newBuilder() {
return new Builder();
}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can't remember, do we usually do toBuilder() for the nested model classes or only the top level ones?


private Retention() {}

public Retention(Builder builder) {
this.mode = builder.mode;
this.retainUntilTime = builder.retainUntilTime;
}

public static final class Builder {

private Mode mode;
private OffsetDateTime retainUntilTime;

/** Sets the retention policy's Mode. Can be Locked or Unlocked. */
public Builder setMode(Mode mode) {
this.mode = mode;
return this;
}

/** Sets what time this object will be retained until, if the mode is Locked. */
public Builder setRetainUntilTime(OffsetDateTime retainUntilTime) {
this.retainUntilTime = retainUntilTime;
return this;
}

public Retention build() {
return new Retention(this);
}
}
}

/** Builder for {@code BlobInfo}. */
public abstract static class Builder {

Expand Down Expand Up @@ -408,6 +522,8 @@ Builder setRetentionExpirationTimeOffsetDateTime(OffsetDateTime retentionExpirat
return setRetentionExpirationTime(millisOffsetDateTimeCodec.decode(retentionExpirationTime));
}

public abstract Builder setRetention(Retention retention);

/** Creates a {@code BlobInfo} object. */
public abstract BlobInfo build();

Expand Down Expand Up @@ -506,6 +622,7 @@ static final class BuilderImpl extends Builder {
private Boolean eventBasedHold;
private Boolean temporaryHold;
private OffsetDateTime retentionExpirationTime;
private Retention retention;
private final ImmutableSet.Builder<NamedField> modifiedFields = ImmutableSet.builder();

BuilderImpl(BlobId blobId) {
Expand Down Expand Up @@ -543,6 +660,7 @@ static final class BuilderImpl extends Builder {
eventBasedHold = blobInfo.eventBasedHold;
temporaryHold = blobInfo.temporaryHold;
retentionExpirationTime = blobInfo.retentionExpirationTime;
this.retention = blobInfo.retention;
}

@Override
Expand Down Expand Up @@ -916,6 +1034,15 @@ Builder setRetentionExpirationTimeOffsetDateTime(OffsetDateTime retentionExpirat
return this;
}

@Override
public Builder setRetention(Retention retention) {
if (!Objects.equals(this.retention, retention)) {
modifiedFields.add(BlobField.RETENTION);
}
this.retention = retention;
return this;
}

@Override
public BlobInfo build() {
checkNotNull(blobId);
Expand Down Expand Up @@ -1139,6 +1266,7 @@ Builder clearRetentionExpirationTime() {
eventBasedHold = builder.eventBasedHold;
temporaryHold = builder.temporaryHold;
retentionExpirationTime = builder.retentionExpirationTime;
retention = builder.retention;
modifiedFields = builder.modifiedFields.build();
}

Expand Down Expand Up @@ -1532,6 +1660,11 @@ public OffsetDateTime getRetentionExpirationTimeOffsetDateTime() {
return retentionExpirationTime;
}

/** Returns the object's Retention policy. */
public Retention getRetention() {
return retention;
}

/** Returns a builder for the current blob. */
public Builder toBuilder() {
return new BuilderImpl(this);
Expand Down Expand Up @@ -1581,6 +1714,7 @@ public int hashCode() {
kmsKeyName,
eventBasedHold,
temporaryHold,
retention,
retentionExpirationTime);
}

Expand Down Expand Up @@ -1622,7 +1756,8 @@ public boolean equals(Object o) {
&& Objects.equals(kmsKeyName, blobInfo.kmsKeyName)
&& Objects.equals(eventBasedHold, blobInfo.eventBasedHold)
&& Objects.equals(temporaryHold, blobInfo.temporaryHold)
&& Objects.equals(retentionExpirationTime, blobInfo.retentionExpirationTime);
&& Objects.equals(retentionExpirationTime, blobInfo.retentionExpirationTime)
&& Objects.equals(retention, blobInfo.retention);
}

ImmutableSet<NamedField> getModifiedFields() {
Expand Down
Loading