Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(storage): support soft delete timestamps #13728

Merged
merged 2 commits into from
Mar 5, 2024
Merged

feat(storage): support soft delete timestamps #13728

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
20e71ac
feat(storage): support soft delete timestamps
coryan Mar 4, 2024
ab544ef
Address review comments
coryan Mar 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion google/cloud/storage/internal/grpc/object_metadata_parser.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,14 @@ storage::ObjectMetadata FromProto(google::storage::v2::Object object,
metadata.set_custom_time(
google::cloud::internal::ToChronoTimePoint(object.custom_time()));
}

if (object.has_soft_delete_time()) {
metadata.set_soft_delete_time(
google::cloud::internal::ToChronoTimePoint(object.soft_delete_time()));
}
if (object.has_hard_delete_time()) {
metadata.set_hard_delete_time(
google::cloud::internal::ToChronoTimePoint(object.hard_delete_time()));
}
return metadata;
}

Expand Down
12 changes: 11 additions & 1 deletion google/cloud/storage/internal/grpc/object_metadata_parser_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,14 @@ TEST(GrpcClientFromProto, ObjectSimple) {
key_sha256_bytes: "01234567"
}
etag: "test-etag"
soft_delete_time {
seconds: 1709555696
nanos: 987654321
}
hard_delete_time {
seconds: 1710160496
nanos: 987654321
}
)""",
&input));

Expand Down Expand Up @@ -149,7 +157,9 @@ TEST(GrpcClientFromProto, ObjectSimple) {
"encryptionAlgorithm": "test-encryption-algorithm",
"keySha256": "MDEyMzQ1Njc="
},
"etag": "test-etag"
"etag": "test-etag",
"softDeleteTime": "2024-03-04T12:34:56.987654321Z",
"hardDeleteTime": "2024-03-11T12:34:56.987654321Z"
})""");
ASSERT_STATUS_OK(expected);

Expand Down
16 changes: 16 additions & 0 deletions google/cloud/storage/internal/object_metadata_parser.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,20 @@ Status ParseUpdated(ObjectMetadata& meta, nlohmann::json const& json) {
return Status{};
}

Status ParseSoftDeleteTime(ObjectMetadata& meta, nlohmann::json const& json) {
auto v = ParseTimestampField(json, "softDeleteTime");
if (!v) return std::move(v).status();
meta.set_soft_delete_time(*std::move(v));
return Status{};
}

Status ParseHardDeleteTime(ObjectMetadata& meta, nlohmann::json const& json) {
auto v = ParseTimestampField(json, "hardDeleteTime");
if (!v) return std::move(v).status();
meta.set_hard_delete_time(*std::move(v));
return Status{};
}

} // namespace

StatusOr<ObjectMetadata> ObjectMetadataParser::FromJson(
Expand Down Expand Up @@ -260,6 +274,8 @@ StatusOr<ObjectMetadata> ObjectMetadataParser::FromJson(
ParseTimeDeleted,
ParseTimeStorageClassUpdated,
ParseUpdated,
ParseSoftDeleteTime,
ParseHardDeleteTime,
};
ObjectMetadata meta;
for (auto const& p : parsers) {
Expand Down
13 changes: 11 additions & 2 deletions google/cloud/storage/object_metadata.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,11 @@ bool operator==(ObjectMetadata const& lhs, ObjectMetadata const& rhs) {
&& lhs.time_created_ == rhs.time_created_ //
&& lhs.time_deleted_ == rhs.time_deleted_ //
&& (lhs.time_storage_class_updated_ ==
rhs.time_storage_class_updated_) //
&& lhs.updated_ == rhs.updated_;
rhs.time_storage_class_updated_) //
&& lhs.updated_ == rhs.updated_ //
&& lhs.soft_delete_time_ == rhs.soft_delete_time_ //
&& lhs.hard_delete_time_ == rhs.hard_delete_time_ //
;
}

std::ostream& operator<<(std::ostream& os, ObjectMetadata const& rhs) {
Expand Down Expand Up @@ -118,6 +121,12 @@ std::ostream& operator<<(std::ostream& os, ObjectMetadata const& rhs) {
if (rhs.has_custom_time()) {
os << ", custom_time=" << FormatRfc3339(rhs.custom_time());
}
if (rhs.has_soft_delete_time()) {
os << ", soft_delete_time=" << FormatRfc3339(rhs.soft_delete_time());
}
if (rhs.has_hard_delete_time()) {
os << ", hard_delete_time=" << FormatRfc3339(rhs.hard_delete_time());
}
return os << "}";
}

Expand Down
56 changes: 56 additions & 0 deletions google/cloud/storage/object_metadata.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,60 @@ class ObjectMetadata {
return *this;
}

/// Returns true if the object has a soft delete timestamp.
bool has_soft_delete_time() const { return soft_delete_time_.has_value(); }
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not have one method that returns the optional<>?

Is it consistency with has_owner(), for example

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All other timestamps (and all other scalars really) return a default value when not set. If we had a time machine I would probably go back and do something different, but I am not sure it is worth changing the approach on only new fields.


/**
* This is the time when the object became soft-deleted.
*
* Soft-deleted objects are only accessible if a `soft_delete_policy` is
* enabled in their bucket. Also see `hard_delete_time()`.
*/
std::chrono::system_clock::time_point soft_delete_time() const {
return soft_delete_time_.value_or(std::chrono::system_clock::time_point{});
}

/// @note This is only intended for mocking.
ObjectMetadata& set_soft_delete_time(
std::chrono::system_clock::time_point v) {
soft_delete_time_ = v;
return *this;
}

/// @note This is only intended for mocking.
ObjectMetadata& reset_soft_delete_time() {
soft_delete_time_.reset();
return *this;
}

/// Returns true if the object has a hard delete timestamp.
bool has_hard_delete_time() const { return hard_delete_time_.has_value(); }

/**
* The time when the object will be permanently deleted.
*
* Soft-deleted objects are permanently deleted after some time, based on the
* `soft_delete_policy` in their bucket. This is only set on soft-deleted
* objects, and indicates the earliest time at which the object will be
* permanently deleted.
*/
std::chrono::system_clock::time_point hard_delete_time() const {
return hard_delete_time_.value_or(std::chrono::system_clock::time_point{});
}

/// @note This is only intended for mocking.
ObjectMetadata& set_hard_delete_time(
std::chrono::system_clock::time_point v) {
hard_delete_time_ = v;
return *this;
}

/// @note This is only intended for mocking.
ObjectMetadata& reset_hard_delete_time() {
hard_delete_time_.reset();
return *this;
}

friend bool operator==(ObjectMetadata const& lhs, ObjectMetadata const& rhs);
friend bool operator!=(ObjectMetadata const& lhs, ObjectMetadata const& rhs) {
return !(lhs == rhs);
Expand Down Expand Up @@ -545,6 +599,8 @@ class ObjectMetadata {
std::chrono::system_clock::time_point time_deleted_;
std::chrono::system_clock::time_point time_storage_class_updated_;
std::chrono::system_clock::time_point updated_;
absl::optional<std::chrono::system_clock::time_point> soft_delete_time_;
absl::optional<std::chrono::system_clock::time_point> hard_delete_time_;
};

std::ostream& operator<<(std::ostream& os, ObjectMetadata const& rhs);
Expand Down
59 changes: 58 additions & 1 deletion google/cloud/storage/object_metadata_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,9 @@ ObjectMetadata CreateObjectMetadataForTest() {
"timeDeleted": "2018-05-19T19:32:24Z",
"timeStorageClassUpdated": "2018-05-19T19:31:34Z",
"updated": "2018-05-19T19:31:24Z",
"customTime": "2020-08-10T12:34:56Z"
"customTime": "2020-08-10T12:34:56Z",
"softDeleteTime": "2024-03-04T12:34:56.789Z",
"hardDeleteTime": "2024-03-11T12:34:56.789Z"
})""";
return internal::ObjectMetadataParser::FromString(text).value();
}
Expand Down Expand Up @@ -184,6 +186,14 @@ TEST(ObjectMetadataTest, Parse) {
EXPECT_EQ(magic_timestamp + 10, duration_cast<std::chrono::seconds>(
actual.updated().time_since_epoch())
.count());
// Use `date -u +%s --date=2024-03-04T12:34:56Z` to get the magic number:
EXPECT_EQ(actual.soft_delete_time(),
std::chrono::system_clock::from_time_t(1709555696L) +
std::chrono::milliseconds(789));
// Use `date -u +%s --date=2024-03-11T12:34:56Z` to get the magic number:
EXPECT_EQ(actual.hard_delete_time(),
std::chrono::system_clock::from_time_t(1710160496L) +
std::chrono::milliseconds(789));
}

/// @test Verify that the IOStream operator works as expected.
Expand All @@ -203,6 +213,9 @@ TEST(ObjectMetadataTest, IOStream) {
EXPECT_THAT(actual, HasSubstr("size=102400"));
EXPECT_THAT(actual, HasSubstr("temporary_hold=true"));
EXPECT_THAT(actual, HasSubstr("custom_time=2020-08-10T12:34:56Z"));

EXPECT_THAT(actual, HasSubstr("soft_delete_time=2024-03-04T12:34:56.789Z"));
EXPECT_THAT(actual, HasSubstr("hard_delete_time=2024-03-11T12:34:56.789Z"));
}

/// @test Verify that ObjectMetadataJsonForCompose works as expected.
Expand Down Expand Up @@ -541,6 +554,50 @@ TEST(ObjectMetadataTest, InsertMetadata) {
EXPECT_NE(expected, copy);
}

/// @test Verify we can change the softDeleteTime field.
TEST(ObjectMetadataTest, SetSoftDeleteTime) {
auto const expected = CreateObjectMetadataForTest();
auto copy = expected;
auto tp =
google::cloud::internal::ParseRfc3339("2020-08-11T09:00:00Z").value();
copy.set_soft_delete_time(tp);
EXPECT_TRUE(expected.has_soft_delete_time());
EXPECT_TRUE(copy.has_soft_delete_time());
EXPECT_EQ(tp, copy.soft_delete_time());
EXPECT_NE(expected, copy);
}

/// @test Verify we can reset the softDeleteTime field.
TEST(ObjectMetadataTest, ResetSoftDeleteTime) {
auto const expected = CreateObjectMetadataForTest();
auto copy = expected;
copy.reset_soft_delete_time();
EXPECT_FALSE(copy.has_soft_delete_time());
EXPECT_NE(expected, copy);
}

/// @test Verify we can change the hardDeleteTime field.
TEST(ObjectMetadataTest, SetHardDeleteTime) {
auto const expected = CreateObjectMetadataForTest();
auto copy = expected;
auto tp =
google::cloud::internal::ParseRfc3339("2020-08-11T09:00:00Z").value();
copy.set_hard_delete_time(tp);
EXPECT_TRUE(expected.has_hard_delete_time());
EXPECT_TRUE(copy.has_hard_delete_time());
EXPECT_EQ(tp, copy.hard_delete_time());
EXPECT_NE(expected, copy);
}

/// @test Verify we can reset the softDeleteTime field.
TEST(ObjectMetadataTest, ResetHardDeleteTime) {
auto const expected = CreateObjectMetadataForTest();
auto copy = expected;
copy.reset_hard_delete_time();
EXPECT_FALSE(copy.has_hard_delete_time());
EXPECT_NE(expected, copy);
}

TEST(ObjectMetadataPatchBuilder, SetAcl) {
ObjectMetadataPatchBuilder builder;
builder.SetAcl({internal::ObjectAccessControlParser::FromString(
Expand Down