Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(GCS+gRPC): synthetic object metadata links #7563

Merged
merged 1 commit into from
Nov 5, 2021
Merged

feat(GCS+gRPC): synthetic object metadata links #7563

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 24 additions & 5 deletions google/cloud/storage/internal/grpc_client.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ StatusOr<ResumableUploadResponse> GrpcClient::QueryResumableUpload(
response.last_committed_byte = 0;
}
if (status->has_resource()) {
response.payload = FromProto(status->resource());
response.payload = FromProto(status->resource(), options());
response.upload_state = ResumableUploadResponse::kDone;
}
return response;
Expand Down Expand Up @@ -314,7 +314,9 @@ StatusOr<ObjectMetadata> GrpcClient::InsertObjectMedia(

auto response = stream->Close();
if (!response) return std::move(response).status();
if (response->has_resource()) return FromProto(response->resource());
if (response->has_resource()) {
return FromProto(response->resource(), options());
}
return ObjectMetadata{};
}

Expand Down Expand Up @@ -714,7 +716,8 @@ StatusOr<google::storage::v2::Object::CustomerEncryption> GrpcClient::ToProto(
return result;
}

ObjectMetadata GrpcClient::FromProto(google::storage::v2::Object object) {
ObjectMetadata GrpcClient::FromProto(google::storage::v2::Object object,
Options const& options) {
auto bucket_id = [](google::storage::v2::Object const& object) {
auto const& bucket_name = object.bucket();
auto const pos = bucket_name.find_last_of('/');
Expand All @@ -729,6 +732,22 @@ ObjectMetadata GrpcClient::FromProto(google::storage::v2::Object object) {
metadata.generation_ = object.generation();
metadata.id_ = metadata.bucket() + "/" + metadata.name() + "/" +
std::to_string(metadata.generation());
auto const metadata_endpoint = [&options]() -> std::string {
if (options.get<RestEndpointOption>() != "https://storage.googleapis.com") {
return options.get<RestEndpointOption>();
}
return "https://www.googleapis.com";
}();
auto const path = [&options]() -> std::string {
if (!options.has<TargetApiVersionOption>()) return "/storage/v1";
return "/storage/" + options.get<TargetApiVersionOption>();
}();
auto const rel_path = "/b/" + metadata.bucket() + "/o/" + metadata.name();
metadata.self_link_ = metadata_endpoint + path + rel_path;
metadata.media_link_ =
options.get<RestEndpointOption>() + "/download" + path + rel_path +
"?generation=" + std::to_string(metadata.generation()) + "&alt=media";

metadata.metageneration_ = object.metageneration();
if (object.has_owner()) {
metadata.owner_ = FromProto(*object.mutable_owner());
Expand Down Expand Up @@ -935,7 +954,7 @@ StatusOr<google::storage::v2::WriteObjectRequest> GrpcClient::ToProto(
}

ResumableUploadResponse GrpcClient::FromProto(
google::storage::v2::WriteObjectResponse const& p) {
google::storage::v2::WriteObjectResponse const& p, Options const& options) {
ResumableUploadResponse response;
response.upload_state = ResumableUploadResponse::kInProgress;
if (p.has_persisted_size() && p.persisted_size() > 0) {
Expand All @@ -946,7 +965,7 @@ ResumableUploadResponse GrpcClient::FromProto(
response.last_committed_byte = 0;
}
if (p.has_resource()) {
response.payload = FromProto(p.resource());
response.payload = FromProto(p.resource(), options);
response.upload_state = ResumableUploadResponse::kDone;
}
return response;
Expand Down
8 changes: 6 additions & 2 deletions google/cloud/storage/internal/grpc_client.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ class GrpcClient : public RawClient,
std::string const& upload_url);
//@}

Options const& options() const { return options_; }

ClientOptions const& client_options() const override;

StatusOr<ListBucketsResponse> ListBuckets(
Expand Down Expand Up @@ -184,7 +186,8 @@ class GrpcClient : public RawClient,
static CustomerEncryption FromProto(
google::storage::v2::Object::CustomerEncryption rhs);

static ObjectMetadata FromProto(google::storage::v2::Object object);
static ObjectMetadata FromProto(google::storage::v2::Object object,
Options const& options);

static google::storage::v2::ObjectAccessControl ToProto(
ObjectAccessControl const& acl);
Expand All @@ -202,7 +205,8 @@ class GrpcClient : public RawClient,
static StatusOr<google::storage::v2::WriteObjectRequest> ToProto(
InsertObjectMediaRequest const& request);
static ResumableUploadResponse FromProto(
google::storage::v2::WriteObjectResponse const&);
google::storage::v2::WriteObjectResponse const& p,
Options const& options);
static StatusOr<google::storage::v2::StartResumableWriteRequest> ToProto(
ResumableUploadRequest const& request);
static google::storage::v2::QueryWriteStatusRequest ToProto(
Expand Down
6 changes: 5 additions & 1 deletion google/cloud/storage/internal/grpc_client_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,8 @@ TEST(GrpcClientFromProto, ObjectSimple) {
"kind": "storage#object",
"bucket": "test-bucket",
"generation": 2345,
"selfLink": "https://www.googleapis.com/storage/v1/b/test-bucket/o/test-object-name",
"mediaLink": "https://storage.googleapis.com/download/storage/v1/b/test-bucket/o/test-object-name?generation=2345&alt=media",
"owner": {
"entity": "test-entity",
"entityId": "test-entity-id"
Expand All @@ -144,7 +146,9 @@ TEST(GrpcClientFromProto, ObjectSimple) {
})""");
EXPECT_STATUS_OK(expected);

auto actual = GrpcClient::FromProto(input);
auto actual = GrpcClient::FromProto(
input,
Options{}.set<RestEndpointOption>("https://storage.googleapis.com"));
EXPECT_EQ(actual, *expected);
}

Expand Down
3 changes: 2 additions & 1 deletion google/cloud/storage/internal/grpc_resumable_upload_session.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,8 @@ StatusOr<ResumableUploadResponse> GrpcResumableUploadSession::UploadGeneric(
return last_response_;
}
done_ = final_chunk;
last_response_ = GrpcClient::FromProto(*std::move(result));
last_response_ =
GrpcClient::FromProto(*std::move(result), client_->options());
return last_response_;
};

Expand Down
3 changes: 2 additions & 1 deletion google/cloud/storage/object_metadata.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,8 @@ std::ostream& operator<<(std::ostream& os, ObjectMetadata const& rhs) {
<< rhs.event_based_hold() << ", generation=" << rhs.generation()
<< ", id=" << rhs.id() << ", kind=" << rhs.kind()
<< ", kms_key_name=" << rhs.kms_key_name()
<< ", md5_hash=" << rhs.md5_hash() << ", media_link=" << rhs.media_link();
<< ", md5_hash=" << rhs.md5_hash() << ", media_link=" << rhs.media_link()
<< ", ";
if (!rhs.metadata_.empty()) {
os << "metadata."
<< absl::StrJoin(rhs.metadata_, ", metadata.", absl::PairFormatter("="));
Expand Down
4 changes: 2 additions & 2 deletions google/cloud/storage/tests/object_basic_crud_integration_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,12 +92,12 @@ TEST_F(ObjectBasicCRUDIntegrationTest, BasicCRUD) {
EXPECT_EQ(get_meta->kind(), insert_meta->kind());
EXPECT_EQ(get_meta->kms_key_name(), insert_meta->kms_key_name());
EXPECT_EQ(get_meta->md5_hash(), insert_meta->md5_hash());
// EXPECT_EQ(get_meta->media_link(), insert_meta->media_link());
EXPECT_EQ(get_meta->media_link(), insert_meta->media_link());
EXPECT_EQ(get_meta->metageneration(), insert_meta->metageneration());
// EXPECT_EQ(get_meta->owner(), insert_meta->owner());
EXPECT_EQ(get_meta->retention_expiration_time(),
insert_meta->retention_expiration_time());
// EXPECT_EQ(get_meta->self_link(), insert_meta->self_link());
EXPECT_EQ(get_meta->self_link(), insert_meta->self_link());
EXPECT_EQ(get_meta->size(), insert_meta->size());
EXPECT_EQ(get_meta->storage_class(), insert_meta->storage_class());
EXPECT_EQ(get_meta->temporary_hold(), insert_meta->temporary_hold());
Expand Down