From 00f6d2da4179eb3f4f55a1fe1da04047697c5999 Mon Sep 17 00:00:00 2001 From: Rahul Kesharwani <42969463+rahulKQL@users.noreply.github.com> Date: Thu, 14 May 2020 19:05:48 +0530 Subject: [PATCH] feat: adding utility to transform protobuf into model object (#299) * feat: adding utility to transform protobuf into model object With this commit, User shall be able to transform protobuf object to bigtable client's specific model objects. * chore: addressed feedback comments - updated the JavaDoc - marked the utility as `@BetaApi` * chore: addressed feedback comments - Rephrased Javadoc to include more explanation - change `Mutation#fromProto` visibility to package only * chore: removed VisibleForTesting annotation As Mutation#fromProto is being used by RowMutation#fromProto, So removed `@VisibleForTesting` annotation * chore: extended unit tests to verify project and instance override --- .../bigtable/data/v2/models/BulkMutation.java | 21 ++++++++++++++++ .../v2/models/ConditionalRowMutation.java | 17 +++++++++++++ .../bigtable/data/v2/models/Mutation.java | 16 ++++++++++++ .../data/v2/models/ReadModifyWriteRow.java | 17 +++++++++++++ .../bigtable/data/v2/models/RowMutation.java | 20 +++++++++++++++ .../data/v2/models/BulkMutationTest.java | 25 +++++++++++++++++++ .../v2/models/ConditionalRowMutationTest.java | 25 +++++++++++++++++++ .../bigtable/data/v2/models/MutationTest.java | 16 ++++++++++++ .../v2/models/ReadModifyWriteRowTest.java | 24 ++++++++++++++++++ .../data/v2/models/RowMutationTest.java | 24 ++++++++++++++++++ 10 files changed, 205 insertions(+) diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/BulkMutation.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/BulkMutation.java index fb45e8e458..95d10872ec 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/BulkMutation.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/BulkMutation.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigtable.data.v2.models; +import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.bigtable.v2.MutateRowsRequest; import com.google.cloud.bigtable.data.v2.internal.NameUtil; @@ -114,6 +115,26 @@ public MutateRowsRequest toProto(RequestContext requestContext) { .build(); } + /** + * Wraps the protobuf {@link MutateRowsRequest}. + * + *
This is meant for advanced usage only. Please ensure that the MutateRowsRequest does not use + * server side timestamps. The BigtableDataClient assumes that mutation present in BulkMutation + * are idempotent and is configured to enable retries by default. If serverside timestamps are + * enabled then that can lead to duplicate mutations. + * + *
WARNING: when applied, the resulting mutation object will ignore the project id and instance + * id in the table_name and instead apply the configuration in the client. + */ + @BetaApi + public static BulkMutation fromProto(@Nonnull MutateRowsRequest request) { + BulkMutation bulkMutation = + BulkMutation.create(NameUtil.extractTableIdFromTableName(request.getTableName())); + bulkMutation.builder = request.toBuilder(); + + return bulkMutation; + } + /** Creates a copy of {@link BulkMutation}. */ @Override public BulkMutation clone() { diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutation.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutation.java index 15cc2be1f4..ac4c548942 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutation.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutation.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigtable.data.v2.models; +import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.bigtable.v2.CheckAndMutateRowRequest; import com.google.cloud.bigtable.data.v2.internal.NameUtil; @@ -136,4 +137,20 @@ public CheckAndMutateRowRequest toProto(RequestContext requestContext) { .setAppProfileId(requestContext.getAppProfileId()) .build(); } + + /** + * Wraps the protobuf {@link CheckAndMutateRowRequest}. + * + *
WARNING: Please note that the table_name will be overwritten by the configuration in the
+ * BigtableDataClient.
+ */
+ @BetaApi
+ public static ConditionalRowMutation fromProto(@Nonnull CheckAndMutateRowRequest request) {
+ String tableId = NameUtil.extractTableIdFromTableName(request.getTableName());
+ ConditionalRowMutation rowMutation =
+ ConditionalRowMutation.create(tableId, request.getRowKey());
+ rowMutation.builder = request.toBuilder();
+
+ return rowMutation;
+ }
}
diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java
index 2a6b655ee4..326d78cfe0 100644
--- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java
+++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java
@@ -87,6 +87,22 @@ public static Mutation fromProtoUnsafe(List Callers must ensure that the protobuf argument is not using serverside timestamps. The
+ * client assumes that all mutations are idempotent and will retry in case of transient errors.
+ * This can lead to row duplication.
+ *
+ * When applied, the resulting Mutation object will ignore the project id and instance id in
+ * the table_name and instead apply the configuration in the client
+ */
+ static Mutation fromProto(List WARNING: Please note that the table_name will be overwritten by the configuration in the
+ * BigtableDataClient.
+ */
+ @BetaApi
+ public static ReadModifyWriteRow fromProto(@Nonnull ReadModifyWriteRowRequest request) {
+ String tableId = NameUtil.extractTableIdFromTableName(request.getTableName());
+
+ ReadModifyWriteRow row = ReadModifyWriteRow.create(tableId, request.getRowKey());
+ row.builder = request.toBuilder();
+
+ return row;
+ }
}
diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutation.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutation.java
index 37b658e8c9..6c5456aac3 100644
--- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutation.java
+++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutation.java
@@ -15,6 +15,7 @@
*/
package com.google.cloud.bigtable.data.v2.models;
+import com.google.api.core.BetaApi;
import com.google.api.core.InternalApi;
import com.google.bigtable.v2.MutateRowRequest;
import com.google.bigtable.v2.MutateRowsRequest;
@@ -214,4 +215,23 @@ public MutateRowsRequest toBulkProto(RequestContext requestContext) {
Entry.newBuilder().setRowKey(key).addAllMutations(mutation.getMutations()).build())
.build();
}
+
+ /**
+ * Wraps the protobuf {@link MutateRowRequest}.
+ *
+ * This is meant for advanced usage only. Please ensure that the MutateRowRequest does not use
+ * server side timestamps. The BigtableDataClient assumes that RowMutations are idempotent and is
+ * configured to enable retries by default. If serverside timestamps are enabled, this can lead to
+ * duplicate mutations.
+ *
+ * WARNING: when applied, the resulting mutation object will ignore the project id and instance
+ * id in the table_name and instead apply the configuration in the client.
+ */
+ @BetaApi
+ public static RowMutation fromProto(@Nonnull MutateRowRequest request) {
+ String tableId = NameUtil.extractTableIdFromTableName(request.getTableName());
+
+ return RowMutation.create(
+ tableId, request.getRowKey(), Mutation.fromProto(request.getMutationsList()));
+ }
}
diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/BulkMutationTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/BulkMutationTest.java
index 8b07249735..dc06a58f5c 100644
--- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/BulkMutationTest.java
+++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/BulkMutationTest.java
@@ -147,4 +147,29 @@ public void addRowMutationEntry() {
bulkMutation.add(entry);
assertThat(bulkMutation.toProto(REQUEST_CONTEXT).getEntriesList()).contains(entry.toProto());
}
+
+ @Test
+ public void fromProtoTest() {
+ BulkMutation expected =
+ BulkMutation.create(TABLE_ID)
+ .add(
+ "key",
+ Mutation.create().setCell("fake-family", "fake-qualifier", 10_000L, "fake-value"));
+
+ MutateRowsRequest protoRequest = expected.toProto(REQUEST_CONTEXT);
+ BulkMutation actualBulkMutation = BulkMutation.fromProto(protoRequest);
+
+ assertThat(actualBulkMutation.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest);
+
+ String projectId = "fresh-project";
+ String instanceId = "fresh-instance";
+ String appProfile = "fresh-app-profile";
+ MutateRowsRequest overriddenRequest =
+ actualBulkMutation.toProto(RequestContext.create(projectId, instanceId, appProfile));
+
+ assertThat(overriddenRequest).isNotEqualTo(protoRequest);
+ assertThat(overriddenRequest.getTableName())
+ .matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID));
+ assertThat(overriddenRequest.getAppProfileId()).matches(appProfile);
+ }
}
diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutationTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutationTest.java
index 044e54f4ef..8a626bb846 100644
--- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutationTest.java
+++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutationTest.java
@@ -162,4 +162,29 @@ public void serializationTest() throws IOException, ClassNotFoundException {
ConditionalRowMutation actual = (ConditionalRowMutation) ois.readObject();
assertThat(actual.toProto(REQUEST_CONTEXT)).isEqualTo(expected.toProto(REQUEST_CONTEXT));
}
+
+ @Test
+ public void fromProtoTest() {
+ ConditionalRowMutation mutation =
+ ConditionalRowMutation.create(TABLE_ID, TEST_KEY)
+ .condition(Filters.FILTERS.key().regex("test"))
+ .then(Mutation.create().setCell("family1", "qualifier1", 10_000L, "value"))
+ .otherwise(Mutation.create().deleteFamily("family"));
+
+ CheckAndMutateRowRequest protoRequest = mutation.toProto(REQUEST_CONTEXT);
+ ConditionalRowMutation actualRequest = ConditionalRowMutation.fromProto(protoRequest);
+
+ assertThat(actualRequest.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest);
+
+ String projectId = "fresh-project";
+ String instanceId = "fresh-instance";
+ String appProfile = "fresh-app-profile";
+ CheckAndMutateRowRequest overriddenRequest =
+ actualRequest.toProto(RequestContext.create(projectId, instanceId, appProfile));
+
+ assertThat(overriddenRequest).isNotEqualTo(protoRequest);
+ assertThat(overriddenRequest.getTableName())
+ .matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID));
+ assertThat(overriddenRequest.getAppProfileId()).matches(appProfile);
+ }
}
diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/MutationTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/MutationTest.java
index 2583d51b34..450925ab0a 100644
--- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/MutationTest.java
+++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/MutationTest.java
@@ -254,4 +254,20 @@ public void testWithLongValue() {
.setValue(ByteString.copyFrom(Longs.toByteArray(20_000L)))
.build());
}
+
+ @Test
+ public void fromProtoTest() {
+ mutation
+ .setCell(
+ "fake-family",
+ ByteString.copyFromUtf8("fake-qualifier"),
+ 1_000,
+ ByteString.copyFromUtf8("fake-value"))
+ .deleteCells("fake-family", ByteString.copyFromUtf8("fake-qualifier"))
+ .deleteFamily("fake-family2");
+
+ List