From 00f6d2da4179eb3f4f55a1fe1da04047697c5999 Mon Sep 17 00:00:00 2001 From: Rahul Kesharwani <42969463+rahulKQL@users.noreply.github.com> Date: Thu, 14 May 2020 19:05:48 +0530 Subject: [PATCH] feat: adding utility to transform protobuf into model object (#299) * feat: adding utility to transform protobuf into model object With this commit, User shall be able to transform protobuf object to bigtable client's specific model objects. * chore: addressed feedback comments - updated the JavaDoc - marked the utility as `@BetaApi` * chore: addressed feedback comments - Rephrased Javadoc to include more explanation - change `Mutation#fromProto` visibility to package only * chore: removed VisibleForTesting annotation As Mutation#fromProto is being used by RowMutation#fromProto, So removed `@VisibleForTesting` annotation * chore: extended unit tests to verify project and instance override --- .../bigtable/data/v2/models/BulkMutation.java | 21 ++++++++++++++++ .../v2/models/ConditionalRowMutation.java | 17 +++++++++++++ .../bigtable/data/v2/models/Mutation.java | 16 ++++++++++++ .../data/v2/models/ReadModifyWriteRow.java | 17 +++++++++++++ .../bigtable/data/v2/models/RowMutation.java | 20 +++++++++++++++ .../data/v2/models/BulkMutationTest.java | 25 +++++++++++++++++++ .../v2/models/ConditionalRowMutationTest.java | 25 +++++++++++++++++++ .../bigtable/data/v2/models/MutationTest.java | 16 ++++++++++++ .../v2/models/ReadModifyWriteRowTest.java | 24 ++++++++++++++++++ .../data/v2/models/RowMutationTest.java | 24 ++++++++++++++++++ 10 files changed, 205 insertions(+) diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/BulkMutation.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/BulkMutation.java index fb45e8e458..95d10872ec 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/BulkMutation.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/BulkMutation.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigtable.data.v2.models; +import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.bigtable.v2.MutateRowsRequest; import com.google.cloud.bigtable.data.v2.internal.NameUtil; @@ -114,6 +115,26 @@ public MutateRowsRequest toProto(RequestContext requestContext) { .build(); } + /** + * Wraps the protobuf {@link MutateRowsRequest}. + * + *

This is meant for advanced usage only. Please ensure that the MutateRowsRequest does not use + * server side timestamps. The BigtableDataClient assumes that mutation present in BulkMutation + * are idempotent and is configured to enable retries by default. If serverside timestamps are + * enabled then that can lead to duplicate mutations. + * + *

WARNING: when applied, the resulting mutation object will ignore the project id and instance + * id in the table_name and instead apply the configuration in the client. + */ + @BetaApi + public static BulkMutation fromProto(@Nonnull MutateRowsRequest request) { + BulkMutation bulkMutation = + BulkMutation.create(NameUtil.extractTableIdFromTableName(request.getTableName())); + bulkMutation.builder = request.toBuilder(); + + return bulkMutation; + } + /** Creates a copy of {@link BulkMutation}. */ @Override public BulkMutation clone() { diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutation.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutation.java index 15cc2be1f4..ac4c548942 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutation.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutation.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigtable.data.v2.models; +import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.bigtable.v2.CheckAndMutateRowRequest; import com.google.cloud.bigtable.data.v2.internal.NameUtil; @@ -136,4 +137,20 @@ public CheckAndMutateRowRequest toProto(RequestContext requestContext) { .setAppProfileId(requestContext.getAppProfileId()) .build(); } + + /** + * Wraps the protobuf {@link CheckAndMutateRowRequest}. + * + *

WARNING: Please note that the table_name will be overwritten by the configuration in the + * BigtableDataClient. + */ + @BetaApi + public static ConditionalRowMutation fromProto(@Nonnull CheckAndMutateRowRequest request) { + String tableId = NameUtil.extractTableIdFromTableName(request.getTableName()); + ConditionalRowMutation rowMutation = + ConditionalRowMutation.create(tableId, request.getRowKey()); + rowMutation.builder = request.toBuilder(); + + return rowMutation; + } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java index 2a6b655ee4..326d78cfe0 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java @@ -87,6 +87,22 @@ public static Mutation fromProtoUnsafe(List pro return mutation; } + /** + * Constructs a row mutation from an existing protobuf object. + * + *

Callers must ensure that the protobuf argument is not using serverside timestamps. The + * client assumes that all mutations are idempotent and will retry in case of transient errors. + * This can lead to row duplication. + * + *

When applied, the resulting Mutation object will ignore the project id and instance id in + * the table_name and instead apply the configuration in the client + */ + static Mutation fromProto(List protos) { + Mutation mutation = new Mutation(false); + mutation.mutations.addAll(protos); + return mutation; + } + private Mutation(boolean allowServersideTimestamp) { this.allowServersideTimestamp = allowServersideTimestamp; } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRow.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRow.java index 26253b9cbc..5fa483d1bd 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRow.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRow.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigtable.data.v2.models; +import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.bigtable.v2.ReadModifyWriteRowRequest; import com.google.bigtable.v2.ReadModifyWriteRule; @@ -137,4 +138,20 @@ public ReadModifyWriteRowRequest toProto(RequestContext requestContext) { .setAppProfileId(requestContext.getAppProfileId()) .build(); } + + /** + * Wraps the protobuf {@link ReadModifyWriteRowRequest}. + * + *

WARNING: Please note that the table_name will be overwritten by the configuration in the + * BigtableDataClient. + */ + @BetaApi + public static ReadModifyWriteRow fromProto(@Nonnull ReadModifyWriteRowRequest request) { + String tableId = NameUtil.extractTableIdFromTableName(request.getTableName()); + + ReadModifyWriteRow row = ReadModifyWriteRow.create(tableId, request.getRowKey()); + row.builder = request.toBuilder(); + + return row; + } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutation.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutation.java index 37b658e8c9..6c5456aac3 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutation.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutation.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigtable.data.v2.models; +import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.bigtable.v2.MutateRowRequest; import com.google.bigtable.v2.MutateRowsRequest; @@ -214,4 +215,23 @@ public MutateRowsRequest toBulkProto(RequestContext requestContext) { Entry.newBuilder().setRowKey(key).addAllMutations(mutation.getMutations()).build()) .build(); } + + /** + * Wraps the protobuf {@link MutateRowRequest}. + * + *

This is meant for advanced usage only. Please ensure that the MutateRowRequest does not use + * server side timestamps. The BigtableDataClient assumes that RowMutations are idempotent and is + * configured to enable retries by default. If serverside timestamps are enabled, this can lead to + * duplicate mutations. + * + *

WARNING: when applied, the resulting mutation object will ignore the project id and instance + * id in the table_name and instead apply the configuration in the client. + */ + @BetaApi + public static RowMutation fromProto(@Nonnull MutateRowRequest request) { + String tableId = NameUtil.extractTableIdFromTableName(request.getTableName()); + + return RowMutation.create( + tableId, request.getRowKey(), Mutation.fromProto(request.getMutationsList())); + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/BulkMutationTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/BulkMutationTest.java index 8b07249735..dc06a58f5c 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/BulkMutationTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/BulkMutationTest.java @@ -147,4 +147,29 @@ public void addRowMutationEntry() { bulkMutation.add(entry); assertThat(bulkMutation.toProto(REQUEST_CONTEXT).getEntriesList()).contains(entry.toProto()); } + + @Test + public void fromProtoTest() { + BulkMutation expected = + BulkMutation.create(TABLE_ID) + .add( + "key", + Mutation.create().setCell("fake-family", "fake-qualifier", 10_000L, "fake-value")); + + MutateRowsRequest protoRequest = expected.toProto(REQUEST_CONTEXT); + BulkMutation actualBulkMutation = BulkMutation.fromProto(protoRequest); + + assertThat(actualBulkMutation.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest); + + String projectId = "fresh-project"; + String instanceId = "fresh-instance"; + String appProfile = "fresh-app-profile"; + MutateRowsRequest overriddenRequest = + actualBulkMutation.toProto(RequestContext.create(projectId, instanceId, appProfile)); + + assertThat(overriddenRequest).isNotEqualTo(protoRequest); + assertThat(overriddenRequest.getTableName()) + .matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID)); + assertThat(overriddenRequest.getAppProfileId()).matches(appProfile); + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutationTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutationTest.java index 044e54f4ef..8a626bb846 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutationTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutationTest.java @@ -162,4 +162,29 @@ public void serializationTest() throws IOException, ClassNotFoundException { ConditionalRowMutation actual = (ConditionalRowMutation) ois.readObject(); assertThat(actual.toProto(REQUEST_CONTEXT)).isEqualTo(expected.toProto(REQUEST_CONTEXT)); } + + @Test + public void fromProtoTest() { + ConditionalRowMutation mutation = + ConditionalRowMutation.create(TABLE_ID, TEST_KEY) + .condition(Filters.FILTERS.key().regex("test")) + .then(Mutation.create().setCell("family1", "qualifier1", 10_000L, "value")) + .otherwise(Mutation.create().deleteFamily("family")); + + CheckAndMutateRowRequest protoRequest = mutation.toProto(REQUEST_CONTEXT); + ConditionalRowMutation actualRequest = ConditionalRowMutation.fromProto(protoRequest); + + assertThat(actualRequest.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest); + + String projectId = "fresh-project"; + String instanceId = "fresh-instance"; + String appProfile = "fresh-app-profile"; + CheckAndMutateRowRequest overriddenRequest = + actualRequest.toProto(RequestContext.create(projectId, instanceId, appProfile)); + + assertThat(overriddenRequest).isNotEqualTo(protoRequest); + assertThat(overriddenRequest.getTableName()) + .matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID)); + assertThat(overriddenRequest.getAppProfileId()).matches(appProfile); + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/MutationTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/MutationTest.java index 2583d51b34..450925ab0a 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/MutationTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/MutationTest.java @@ -254,4 +254,20 @@ public void testWithLongValue() { .setValue(ByteString.copyFrom(Longs.toByteArray(20_000L))) .build()); } + + @Test + public void fromProtoTest() { + mutation + .setCell( + "fake-family", + ByteString.copyFromUtf8("fake-qualifier"), + 1_000, + ByteString.copyFromUtf8("fake-value")) + .deleteCells("fake-family", ByteString.copyFromUtf8("fake-qualifier")) + .deleteFamily("fake-family2"); + + List protoMutation = mutation.getMutations(); + + assertThat(Mutation.fromProto(protoMutation).getMutations()).isEqualTo(protoMutation); + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRowTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRowTest.java index a318a61f0e..b9e6f9e7ba 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRowTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRowTest.java @@ -113,4 +113,28 @@ public void serializationTest() throws IOException, ClassNotFoundException { ReadModifyWriteRow actual = (ReadModifyWriteRow) ois.readObject(); assertThat(actual.toProto(REQUEST_CONTEXT)).isEqualTo(expected.toProto(REQUEST_CONTEXT)); } + + @Test + public void fromProtoTest() { + ReadModifyWriteRow expected = + ReadModifyWriteRow.create(TABLE_ID, "row-key") + .increment("fake-family", ByteString.copyFromUtf8("fake-qualifier"), 1) + .append("fake-family", "fake-qualifier", "fake-value"); + + ReadModifyWriteRowRequest protoRequest = expected.toProto(REQUEST_CONTEXT); + ReadModifyWriteRow actualRequest = ReadModifyWriteRow.fromProto(protoRequest); + + assertThat(actualRequest.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest); + + String projectId = "fresh-project"; + String instanceId = "fresh-instance"; + String appProfile = "fresh-app-profile"; + ReadModifyWriteRowRequest overriddenRequest = + actualRequest.toProto(RequestContext.create(projectId, instanceId, appProfile)); + + assertThat(overriddenRequest).isNotEqualTo(protoRequest); + assertThat(overriddenRequest.getTableName()) + .matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID)); + assertThat(overriddenRequest.getAppProfileId()).matches(appProfile); + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/RowMutationTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/RowMutationTest.java index 553d7ce9d5..b401ad5ef3 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/RowMutationTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/RowMutationTest.java @@ -139,4 +139,28 @@ public void testWithLongValue() { .setValue(ByteString.copyFrom(Longs.toByteArray(100_000L))) .build()); } + + @Test + public void fromProtoTest() { + RowMutation rowMutation = + RowMutation.create("fake-table", "fake-key") + .setCell("fake-family", "fake-qualifier-1", "fake-value") + .setCell("fake-family", "fake-qualifier-2", 30_000L, "fake-value-2"); + + MutateRowRequest protoRequest = rowMutation.toProto(REQUEST_CONTEXT); + RowMutation actualRequest = RowMutation.fromProto(protoRequest); + + assertThat(actualRequest.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest); + + String projectId = "fresh-project"; + String instanceId = "fresh-instance"; + String appProfile = "fresh-app-profile"; + MutateRowRequest overriddenRequest = + actualRequest.toProto(RequestContext.create(projectId, instanceId, appProfile)); + + assertThat(overriddenRequest).isNotEqualTo(protoRequest); + assertThat(overriddenRequest.getTableName()) + .matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID)); + assertThat(overriddenRequest.getAppProfileId()).matches(appProfile); + } }