From 27716d9afdf7a3872e9733cd032b1fe1093a5175 Mon Sep 17 00:00:00 2001 From: yirutang Date: Fri, 4 Sep 2020 16:21:09 -0700 Subject: [PATCH 1/2] fix: update schema compat check with backend type changes --- .../storage/v1alpha2/SchemaCompatibility.java | 24 +++++++++---------- .../v1alpha2/SchemaCompatibilityTest.java | 7 +++--- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/SchemaCompatibility.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/SchemaCompatibility.java index bbf3bfc62c..f15d6c3261 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/SchemaCompatibility.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/SchemaCompatibility.java @@ -156,8 +156,8 @@ private static boolean isCompatibleWithBQDate(Descriptors.FieldDescriptor.Type f } private static boolean isCompatibleWithBQDatetime(Descriptors.FieldDescriptor.Type field) { - if (field == Descriptors.FieldDescriptor.Type.INT64 - || field == Descriptors.FieldDescriptor.Type.SFIXED64) { + if (field == Descriptors.FieldDescriptor.Type.STRING + || field == Descriptors.FieldDescriptor.Type.INT64) { return true; } return false; @@ -174,7 +174,7 @@ private static boolean isCompatibleWithBQFloat(Descriptors.FieldDescriptor.Type } private static boolean isCompatibleWithBQGeography(Descriptors.FieldDescriptor.Type field) { - if (field == Descriptors.FieldDescriptor.Type.BYTES) { + if (field == Descriptors.FieldDescriptor.Type.STRING) { return true; } return false; @@ -201,15 +201,10 @@ private static boolean isCompatibleWithBQNumeric(Descriptors.FieldDescriptor.Typ || field == Descriptors.FieldDescriptor.Type.FIXED32 || field == Descriptors.FieldDescriptor.Type.FIXED64 || field == Descriptors.FieldDescriptor.Type.SFIXED32 - || field == Descriptors.FieldDescriptor.Type.SFIXED64) { - return true; - } - - if (field == Descriptors.FieldDescriptor.Type.BYTES) { - return true; - } - - if (field == Descriptors.FieldDescriptor.Type.FLOAT + || field == Descriptors.FieldDescriptor.Type.SFIXED64 + || field == Descriptors.FieldDescriptor.Type.STRING + || field == Descriptors.FieldDescriptor.Type.BYTES + || field == Descriptors.FieldDescriptor.Type.FLOAT || field == Descriptors.FieldDescriptor.Type.DOUBLE) { return true; } @@ -235,7 +230,7 @@ private static boolean isCompatibleWithBQString(Descriptors.FieldDescriptor.Type private static boolean isCompatibleWithBQTime(Descriptors.FieldDescriptor.Type field) { if (field == Descriptors.FieldDescriptor.Type.INT64 - || field == Descriptors.FieldDescriptor.Type.SFIXED64) { + || field == Descriptors.FieldDescriptor.Type.STRING) { return true; } @@ -348,6 +343,9 @@ private void protoFieldTypeIsCompatibleWithBQFieldType( case "NUMERIC": match = isCompatibleWithBQNumeric(protoType); break; + case "BIGNUMERIC": + match = isCompatibleWithBQNumeric(protoType); + break; case "RECORD": if (allMessageTypes.size() > NestingLimit) { throw new IllegalArgumentException( diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/SchemaCompatibilityTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/SchemaCompatibilityTest.java index 801bf7f078..7ec70e51db 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/SchemaCompatibilityTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/SchemaCompatibilityTest.java @@ -587,7 +587,7 @@ public void testBQDatetime() { .build())); SchemaCompatibility compact = SchemaCompatibility.getInstance(mockBigquery); HashSet compatible = - new HashSet<>(Arrays.asList(Int64Type.getDescriptor(), SFixed64Type.getDescriptor())); + new HashSet<>(Arrays.asList(Int64Type.getDescriptor(), StringType.getDescriptor())); for (Descriptors.Descriptor descriptor : type_descriptors) { if (compatible.contains(descriptor)) { @@ -649,7 +649,7 @@ public void testBQGeography() { .build())); SchemaCompatibility compact = SchemaCompatibility.getInstance(mockBigquery); HashSet compatible = - new HashSet<>(Arrays.asList(BytesType.getDescriptor())); + new HashSet<>(Arrays.asList(StringType.getDescriptor())); for (Descriptors.Descriptor descriptor : type_descriptors) { if (compatible.contains(descriptor)) { @@ -730,6 +730,7 @@ public void testBQNumeric() { SFixed32Type.getDescriptor(), SFixed64Type.getDescriptor(), BytesType.getDescriptor(), + StringType.getDescriptor(), FloatType.getDescriptor(), DoubleType.getDescriptor())); @@ -879,7 +880,7 @@ public void testBQTime() { .build())); SchemaCompatibility compact = SchemaCompatibility.getInstance(mockBigquery); HashSet compatible = - new HashSet<>(Arrays.asList(Int64Type.getDescriptor(), SFixed64Type.getDescriptor())); + new HashSet<>(Arrays.asList(Int64Type.getDescriptor(), StringType.getDescriptor())); for (Descriptors.Descriptor descriptor : type_descriptors) { if (compatible.contains(descriptor)) { From d3a289860c1c039faa3bdbb0c394142b294f7163 Mon Sep 17 00:00:00 2001 From: yirutang Date: Tue, 8 Sep 2020 12:21:55 -0700 Subject: [PATCH 2/2] remove BIGNUMERIC, the client library is not populated with the type --- .../cloud/bigquery/storage/v1alpha2/SchemaCompatibility.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/SchemaCompatibility.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/SchemaCompatibility.java index f15d6c3261..6943993b05 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/SchemaCompatibility.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/SchemaCompatibility.java @@ -343,9 +343,6 @@ private void protoFieldTypeIsCompatibleWithBQFieldType( case "NUMERIC": match = isCompatibleWithBQNumeric(protoType); break; - case "BIGNUMERIC": - match = isCompatibleWithBQNumeric(protoType); - break; case "RECORD": if (allMessageTypes.size() > NestingLimit) { throw new IllegalArgumentException(