From e80b512ad299a40c722f33a3d34eca30217a209f Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Fri, 9 Sep 2022 13:52:53 -0400 Subject: [PATCH 01/15] feat: add reference file schema option for federated formats --- .../bigquery/ExternalTableDefinition.java | 38 +++++++++ .../cloud/bigquery/LoadJobConfiguration.java | 40 ++++++++- .../cloud/bigquery/it/ITBigQueryTest.java | 81 ++++++++++++++++++- 3 files changed, 157 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java index 6ca64a8d7..2da009652 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java @@ -157,6 +157,14 @@ public Builder setHivePartitioningOptions(HivePartitioningOptions hivePartitioni return setHivePartitioningOptionsInner(hivePartitioningOptions); }; + /** + * When creating an external table, the user can provide a reference file + * with the table schema. This is enabled for the following formats: AVRO, + * PARQUET, ORC. + * @param referenceFileSchemaUri or {@code null} for none + */ + public abstract Builder setReferenceFileSchemaUri(String referenceFileSchemaUri); + abstract Builder setHivePartitioningOptionsInner( HivePartitioningOptions hivePartitioningOptions); @@ -250,6 +258,9 @@ public F getFormatOptions() { @Nullable public abstract Boolean getAutodetect(); + @Nullable + public abstract String getReferenceFileSchemaUri(); + /** * [Experimental] Returns the HivePartitioningOptions when the data layout follows Hive * partitioning convention @@ -317,6 +328,16 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC if (getAutodetect() != null) { externalConfigurationPb.setAutodetect(getAutodetect()); } + if(FormatOptions.AVRO.equals(getFormatOptions().getType()) || + FormatOptions.PARQUET.equals(getFormatOptions().getType()) || + FormatOptions.ORC.equals(getFormatOptions().getType())) { + + if (getReferenceFileSchemaUri() != null) { + externalConfigurationPb.setReferenceFileSchemaUri(getReferenceFileSchemaUri()); + + } + } + if (getHivePartitioningOptions() != null) { externalConfigurationPb.setHivePartitioningOptions(getHivePartitioningOptions().toPb()); } @@ -486,6 +507,14 @@ static ExternalTableDefinition fromPb(Table tablePb) { builder.setHivePartitioningOptions( HivePartitioningOptions.fromPb(externalDataConfiguration.getHivePartitioningOptions())); } + if(FormatOptions.AVRO.equals(externalDataConfiguration.getSourceFormat()) || + FormatOptions.PARQUET.equals(externalDataConfiguration.getSourceFormat()) || + FormatOptions.ORC.equals(externalDataConfiguration.getSourceFormat())) { + if (externalDataConfiguration.getReferenceFileSchemaUri() != null){ + builder.setReferenceFileSchemaUri(externalDataConfiguration.getReferenceFileSchemaUri()); + } + } + } return builder.build(); } @@ -538,10 +567,19 @@ static ExternalTableDefinition fromExternalDataConfiguration( if (externalDataConfiguration.getAutodetect() != null) { builder.setAutodetect(externalDataConfiguration.getAutodetect()); } + if(FormatOptions.AVRO.equals(externalDataConfiguration.getSourceFormat()) || + FormatOptions.PARQUET.equals(externalDataConfiguration.getSourceFormat()) || + FormatOptions.ORC.equals(externalDataConfiguration.getSourceFormat())) { + if (externalDataConfiguration.getReferenceFileSchemaUri() != null) { + builder.setReferenceFileSchemaUri(externalDataConfiguration.getReferenceFileSchemaUri()); + } + } + if (externalDataConfiguration.getHivePartitioningOptions() != null) { builder.setHivePartitioningOptions( HivePartitioningOptions.fromPb(externalDataConfiguration.getHivePartitioningOptions())); } + return builder.build(); } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java index 2f625acf5..f16ff15a9 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java @@ -56,6 +56,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load private final Long jobTimeoutMs; private final RangePartitioning rangePartitioning; private final HivePartitioningOptions hivePartitioningOptions; + private final String referenceFileSchemaUri; public static final class Builder extends JobConfiguration.Builder implements LoadConfiguration.Builder { @@ -81,6 +82,7 @@ public static final class Builder extends JobConfiguration.Builder SOURCE_URIS = ImmutableList.of( + "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/a-twitter.avro", + "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/b-twitter.avro", + "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/c-twitter.avro"); + + // Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema + String referenceFileSchema = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/a-twitter.avro"; + + + LoadJobConfiguration loadJobConfiguration = + LoadJobConfiguration.newBuilder(tableId, SOURCE_URIS, FormatOptions.avro()) + .setReferenceFileSchemaUri(referenceFileSchema).build(); + + Job job = bigquery.create(JobInfo.of(loadJobConfiguration)); + // Blocks until this load table job completes its execution, either failing or succeeding. + job = job.waitFor(); + if (job.isDone()) { + System.out.println("Avro data from GCS successfully loaded in a table"); + } else { + System.out.println( + "BigQuery job was unable to load into the table due to an error:" + + job.getStatus().getError()); + } + } catch (BigQueryException | InterruptedException e) { + System.out.println("Column not added during load append \n" + e.toString()); + } + + } + + @Test + public void testReferenceFileSchemaUriForParquet() { + try { + String destinationTableName = "test_reference_file_schema_parquet"; + TableId tableId = TableId.of(DATASET, destinationTableName); + + // By default, the table should have c-twitter schema because it is lexicographically last. + // a-twitter schema (username, tweet, timestamp, likes) + // b-twitter schema (username, tweet, timestamp) + // c-twitter schema (username, tweet) + List SOURCE_URIS = ImmutableList.of( + "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/a-twitter.parquet", + "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/b-twitter.parquet", + "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/c-twitter.parquet"); + + // Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema + String referenceFileSchema = "gs://avro-test-bucket/a-twitter.parquet"; + + + LoadJobConfiguration loadJobConfiguration = + LoadJobConfiguration.newBuilder(tableId, SOURCE_URIS, FormatOptions.parquet()) + .setReferenceFileSchemaUri(referenceFileSchema).build(); + + Job job = bigquery.create(JobInfo.of(loadJobConfiguration)); + // Blocks until this load table job completes its execution, either failing or succeeding. + job = job.waitFor(); + if (job.isDone()) { + System.out.println("Parquet data from GCS successfully loaded in a table"); + } else { + System.out.println( + "BigQuery job was unable to load into the table due to an error:" + + job.getStatus().getError()); + } + } catch (BigQueryException | InterruptedException e) { + System.out.println("Column not added during load append \n" + e.toString()); + } + + } + + } From 940d263ec8f739da0e3e41c8456ceb7bffbb8cfa Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 9 Sep 2022 17:55:38 +0000 Subject: [PATCH 02/15] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../bigquery/ExternalTableDefinition.java | 30 ++++++------ .../cloud/bigquery/LoadJobConfiguration.java | 22 ++++----- .../cloud/bigquery/it/ITBigQueryTest.java | 48 ++++++++++++------- 3 files changed, 55 insertions(+), 45 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java index 2da009652..e031348e3 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java @@ -158,10 +158,10 @@ public Builder setHivePartitioningOptions(HivePartitioningOptions hivePartitioni }; /** - * When creating an external table, the user can provide a reference file - * with the table schema. This is enabled for the following formats: AVRO, - * PARQUET, ORC. - * @param referenceFileSchemaUri or {@code null} for none + * When creating an external table, the user can provide a reference file with the table schema. + * This is enabled for the following formats: AVRO, PARQUET, ORC. + * + * @param referenceFileSchemaUri or {@code null} for none */ public abstract Builder setReferenceFileSchemaUri(String referenceFileSchemaUri); @@ -328,13 +328,12 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC if (getAutodetect() != null) { externalConfigurationPb.setAutodetect(getAutodetect()); } - if(FormatOptions.AVRO.equals(getFormatOptions().getType()) || - FormatOptions.PARQUET.equals(getFormatOptions().getType()) || - FormatOptions.ORC.equals(getFormatOptions().getType())) { + if (FormatOptions.AVRO.equals(getFormatOptions().getType()) + || FormatOptions.PARQUET.equals(getFormatOptions().getType()) + || FormatOptions.ORC.equals(getFormatOptions().getType())) { if (getReferenceFileSchemaUri() != null) { externalConfigurationPb.setReferenceFileSchemaUri(getReferenceFileSchemaUri()); - } } @@ -507,14 +506,13 @@ static ExternalTableDefinition fromPb(Table tablePb) { builder.setHivePartitioningOptions( HivePartitioningOptions.fromPb(externalDataConfiguration.getHivePartitioningOptions())); } - if(FormatOptions.AVRO.equals(externalDataConfiguration.getSourceFormat()) || - FormatOptions.PARQUET.equals(externalDataConfiguration.getSourceFormat()) || - FormatOptions.ORC.equals(externalDataConfiguration.getSourceFormat())) { - if (externalDataConfiguration.getReferenceFileSchemaUri() != null){ + if (FormatOptions.AVRO.equals(externalDataConfiguration.getSourceFormat()) + || FormatOptions.PARQUET.equals(externalDataConfiguration.getSourceFormat()) + || FormatOptions.ORC.equals(externalDataConfiguration.getSourceFormat())) { + if (externalDataConfiguration.getReferenceFileSchemaUri() != null) { builder.setReferenceFileSchemaUri(externalDataConfiguration.getReferenceFileSchemaUri()); } } - } return builder.build(); } @@ -567,9 +565,9 @@ static ExternalTableDefinition fromExternalDataConfiguration( if (externalDataConfiguration.getAutodetect() != null) { builder.setAutodetect(externalDataConfiguration.getAutodetect()); } - if(FormatOptions.AVRO.equals(externalDataConfiguration.getSourceFormat()) || - FormatOptions.PARQUET.equals(externalDataConfiguration.getSourceFormat()) || - FormatOptions.ORC.equals(externalDataConfiguration.getSourceFormat())) { + if (FormatOptions.AVRO.equals(externalDataConfiguration.getSourceFormat()) + || FormatOptions.PARQUET.equals(externalDataConfiguration.getSourceFormat()) + || FormatOptions.ORC.equals(externalDataConfiguration.getSourceFormat())) { if (externalDataConfiguration.getReferenceFileSchemaUri() != null) { builder.setReferenceFileSchemaUri(externalDataConfiguration.getReferenceFileSchemaUri()); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java index f16ff15a9..af73ceacc 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java @@ -202,16 +202,14 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur this.hivePartitioningOptions = HivePartitioningOptions.fromPb(loadConfigurationPb.getHivePartitioningOptions()); } - if(FormatOptions.AVRO.equals(loadConfigurationPb.getSourceFormat()) || - FormatOptions.PARQUET.equals(loadConfigurationPb.getSourceFormat()) || - FormatOptions.ORC.equals(loadConfigurationPb.getSourceFormat())) { + if (FormatOptions.AVRO.equals(loadConfigurationPb.getSourceFormat()) + || FormatOptions.PARQUET.equals(loadConfigurationPb.getSourceFormat()) + || FormatOptions.ORC.equals(loadConfigurationPb.getSourceFormat())) { if (loadConfigurationPb.getReferenceFileSchemaUri() != null) { this.referenceFileSchemaUri = loadConfigurationPb.getReferenceFileSchemaUri(); - } } - } @Override @@ -365,10 +363,10 @@ public Builder setHivePartitioningOptions(HivePartitioningOptions hivePartitioni } /** - * When creating an external table, the user can provide a reference file - * with the table schema. This is enabled for the following formats: AVRO, - * PARQUET, ORC. - * @param referenceFileSchemaUri or {@code null} for none + * When creating an external table, the user can provide a reference file with the table schema. + * This is enabled for the following formats: AVRO, PARQUET, ORC. + * + * @param referenceFileSchemaUri or {@code null} for none */ public Builder setReferenceFileSchemaUri(String referenceFileSchemaUri) { this.referenceFileSchemaUri = referenceFileSchemaUri; @@ -658,9 +656,9 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { if (hivePartitioningOptions != null) { loadConfigurationPb.setHivePartitioningOptions(hivePartitioningOptions.toPb()); } - if(FormatOptions.AVRO.equals(loadConfigurationPb.getSourceFormat()) || - FormatOptions.PARQUET.equals(loadConfigurationPb.getSourceFormat()) || - FormatOptions.ORC.equals(loadConfigurationPb.getSourceFormat())) { + if (FormatOptions.AVRO.equals(loadConfigurationPb.getSourceFormat()) + || FormatOptions.PARQUET.equals(loadConfigurationPb.getSourceFormat()) + || FormatOptions.ORC.equals(loadConfigurationPb.getSourceFormat())) { if (referenceFileSchemaUri != null) { loadConfigurationPb.setReferenceFileSchemaUri(referenceFileSchemaUri); } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 1c5a5bd7e..784142f78 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -4597,18 +4597,28 @@ public void testReferenceFileSchemaUriForAvro() { // a-twitter schema (username, tweet, timestamp, likes) // b-twitter schema (username, tweet, timestamp) // c-twitter schema (username, tweet) - List SOURCE_URIS = ImmutableList.of( - "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/a-twitter.avro", - "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/b-twitter.avro", - "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/c-twitter.avro"); + List SOURCE_URIS = + ImmutableList.of( + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/a-twitter.avro", + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/b-twitter.avro", + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/c-twitter.avro"); // Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema - String referenceFileSchema = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/a-twitter.avro"; - + String referenceFileSchema = + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/a-twitter.avro"; LoadJobConfiguration loadJobConfiguration = LoadJobConfiguration.newBuilder(tableId, SOURCE_URIS, FormatOptions.avro()) - .setReferenceFileSchemaUri(referenceFileSchema).build(); + .setReferenceFileSchemaUri(referenceFileSchema) + .build(); Job job = bigquery.create(JobInfo.of(loadJobConfiguration)); // Blocks until this load table job completes its execution, either failing or succeeding. @@ -4623,7 +4633,6 @@ public void testReferenceFileSchemaUriForAvro() { } catch (BigQueryException | InterruptedException e) { System.out.println("Column not added during load append \n" + e.toString()); } - } @Test @@ -4636,18 +4645,25 @@ public void testReferenceFileSchemaUriForParquet() { // a-twitter schema (username, tweet, timestamp, likes) // b-twitter schema (username, tweet, timestamp) // c-twitter schema (username, tweet) - List SOURCE_URIS = ImmutableList.of( - "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/a-twitter.parquet", - "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/b-twitter.parquet", - "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/c-twitter.parquet"); + List SOURCE_URIS = + ImmutableList.of( + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/a-twitter.parquet", + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/b-twitter.parquet", + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/c-twitter.parquet"); // Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema String referenceFileSchema = "gs://avro-test-bucket/a-twitter.parquet"; - LoadJobConfiguration loadJobConfiguration = LoadJobConfiguration.newBuilder(tableId, SOURCE_URIS, FormatOptions.parquet()) - .setReferenceFileSchemaUri(referenceFileSchema).build(); + .setReferenceFileSchemaUri(referenceFileSchema) + .build(); Job job = bigquery.create(JobInfo.of(loadJobConfiguration)); // Blocks until this load table job completes its execution, either failing or succeeding. @@ -4662,7 +4678,5 @@ public void testReferenceFileSchemaUriForParquet() { } catch (BigQueryException | InterruptedException e) { System.out.println("Column not added during load append \n" + e.toString()); } - - } - } +} From 44c26464b56b8ab69fe77539e0aaba321c85a929 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 9 Sep 2022 17:56:55 +0000 Subject: [PATCH 03/15] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../bigquery/ExternalTableDefinition.java | 30 ++++++------ .../cloud/bigquery/LoadJobConfiguration.java | 22 ++++----- .../cloud/bigquery/it/ITBigQueryTest.java | 48 ++++++++++++------- 3 files changed, 55 insertions(+), 45 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java index 2da009652..e031348e3 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java @@ -158,10 +158,10 @@ public Builder setHivePartitioningOptions(HivePartitioningOptions hivePartitioni }; /** - * When creating an external table, the user can provide a reference file - * with the table schema. This is enabled for the following formats: AVRO, - * PARQUET, ORC. - * @param referenceFileSchemaUri or {@code null} for none + * When creating an external table, the user can provide a reference file with the table schema. + * This is enabled for the following formats: AVRO, PARQUET, ORC. + * + * @param referenceFileSchemaUri or {@code null} for none */ public abstract Builder setReferenceFileSchemaUri(String referenceFileSchemaUri); @@ -328,13 +328,12 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC if (getAutodetect() != null) { externalConfigurationPb.setAutodetect(getAutodetect()); } - if(FormatOptions.AVRO.equals(getFormatOptions().getType()) || - FormatOptions.PARQUET.equals(getFormatOptions().getType()) || - FormatOptions.ORC.equals(getFormatOptions().getType())) { + if (FormatOptions.AVRO.equals(getFormatOptions().getType()) + || FormatOptions.PARQUET.equals(getFormatOptions().getType()) + || FormatOptions.ORC.equals(getFormatOptions().getType())) { if (getReferenceFileSchemaUri() != null) { externalConfigurationPb.setReferenceFileSchemaUri(getReferenceFileSchemaUri()); - } } @@ -507,14 +506,13 @@ static ExternalTableDefinition fromPb(Table tablePb) { builder.setHivePartitioningOptions( HivePartitioningOptions.fromPb(externalDataConfiguration.getHivePartitioningOptions())); } - if(FormatOptions.AVRO.equals(externalDataConfiguration.getSourceFormat()) || - FormatOptions.PARQUET.equals(externalDataConfiguration.getSourceFormat()) || - FormatOptions.ORC.equals(externalDataConfiguration.getSourceFormat())) { - if (externalDataConfiguration.getReferenceFileSchemaUri() != null){ + if (FormatOptions.AVRO.equals(externalDataConfiguration.getSourceFormat()) + || FormatOptions.PARQUET.equals(externalDataConfiguration.getSourceFormat()) + || FormatOptions.ORC.equals(externalDataConfiguration.getSourceFormat())) { + if (externalDataConfiguration.getReferenceFileSchemaUri() != null) { builder.setReferenceFileSchemaUri(externalDataConfiguration.getReferenceFileSchemaUri()); } } - } return builder.build(); } @@ -567,9 +565,9 @@ static ExternalTableDefinition fromExternalDataConfiguration( if (externalDataConfiguration.getAutodetect() != null) { builder.setAutodetect(externalDataConfiguration.getAutodetect()); } - if(FormatOptions.AVRO.equals(externalDataConfiguration.getSourceFormat()) || - FormatOptions.PARQUET.equals(externalDataConfiguration.getSourceFormat()) || - FormatOptions.ORC.equals(externalDataConfiguration.getSourceFormat())) { + if (FormatOptions.AVRO.equals(externalDataConfiguration.getSourceFormat()) + || FormatOptions.PARQUET.equals(externalDataConfiguration.getSourceFormat()) + || FormatOptions.ORC.equals(externalDataConfiguration.getSourceFormat())) { if (externalDataConfiguration.getReferenceFileSchemaUri() != null) { builder.setReferenceFileSchemaUri(externalDataConfiguration.getReferenceFileSchemaUri()); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java index f16ff15a9..af73ceacc 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java @@ -202,16 +202,14 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur this.hivePartitioningOptions = HivePartitioningOptions.fromPb(loadConfigurationPb.getHivePartitioningOptions()); } - if(FormatOptions.AVRO.equals(loadConfigurationPb.getSourceFormat()) || - FormatOptions.PARQUET.equals(loadConfigurationPb.getSourceFormat()) || - FormatOptions.ORC.equals(loadConfigurationPb.getSourceFormat())) { + if (FormatOptions.AVRO.equals(loadConfigurationPb.getSourceFormat()) + || FormatOptions.PARQUET.equals(loadConfigurationPb.getSourceFormat()) + || FormatOptions.ORC.equals(loadConfigurationPb.getSourceFormat())) { if (loadConfigurationPb.getReferenceFileSchemaUri() != null) { this.referenceFileSchemaUri = loadConfigurationPb.getReferenceFileSchemaUri(); - } } - } @Override @@ -365,10 +363,10 @@ public Builder setHivePartitioningOptions(HivePartitioningOptions hivePartitioni } /** - * When creating an external table, the user can provide a reference file - * with the table schema. This is enabled for the following formats: AVRO, - * PARQUET, ORC. - * @param referenceFileSchemaUri or {@code null} for none + * When creating an external table, the user can provide a reference file with the table schema. + * This is enabled for the following formats: AVRO, PARQUET, ORC. + * + * @param referenceFileSchemaUri or {@code null} for none */ public Builder setReferenceFileSchemaUri(String referenceFileSchemaUri) { this.referenceFileSchemaUri = referenceFileSchemaUri; @@ -658,9 +656,9 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { if (hivePartitioningOptions != null) { loadConfigurationPb.setHivePartitioningOptions(hivePartitioningOptions.toPb()); } - if(FormatOptions.AVRO.equals(loadConfigurationPb.getSourceFormat()) || - FormatOptions.PARQUET.equals(loadConfigurationPb.getSourceFormat()) || - FormatOptions.ORC.equals(loadConfigurationPb.getSourceFormat())) { + if (FormatOptions.AVRO.equals(loadConfigurationPb.getSourceFormat()) + || FormatOptions.PARQUET.equals(loadConfigurationPb.getSourceFormat()) + || FormatOptions.ORC.equals(loadConfigurationPb.getSourceFormat())) { if (referenceFileSchemaUri != null) { loadConfigurationPb.setReferenceFileSchemaUri(referenceFileSchemaUri); } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 1c5a5bd7e..784142f78 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -4597,18 +4597,28 @@ public void testReferenceFileSchemaUriForAvro() { // a-twitter schema (username, tweet, timestamp, likes) // b-twitter schema (username, tweet, timestamp) // c-twitter schema (username, tweet) - List SOURCE_URIS = ImmutableList.of( - "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/a-twitter.avro", - "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/b-twitter.avro", - "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/c-twitter.avro"); + List SOURCE_URIS = + ImmutableList.of( + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/a-twitter.avro", + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/b-twitter.avro", + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/c-twitter.avro"); // Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema - String referenceFileSchema = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/a-twitter.avro"; - + String referenceFileSchema = + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/a-twitter.avro"; LoadJobConfiguration loadJobConfiguration = LoadJobConfiguration.newBuilder(tableId, SOURCE_URIS, FormatOptions.avro()) - .setReferenceFileSchemaUri(referenceFileSchema).build(); + .setReferenceFileSchemaUri(referenceFileSchema) + .build(); Job job = bigquery.create(JobInfo.of(loadJobConfiguration)); // Blocks until this load table job completes its execution, either failing or succeeding. @@ -4623,7 +4633,6 @@ public void testReferenceFileSchemaUriForAvro() { } catch (BigQueryException | InterruptedException e) { System.out.println("Column not added during load append \n" + e.toString()); } - } @Test @@ -4636,18 +4645,25 @@ public void testReferenceFileSchemaUriForParquet() { // a-twitter schema (username, tweet, timestamp, likes) // b-twitter schema (username, tweet, timestamp) // c-twitter schema (username, tweet) - List SOURCE_URIS = ImmutableList.of( - "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/a-twitter.parquet", - "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/b-twitter.parquet", - "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/c-twitter.parquet"); + List SOURCE_URIS = + ImmutableList.of( + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/a-twitter.parquet", + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/b-twitter.parquet", + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/c-twitter.parquet"); // Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema String referenceFileSchema = "gs://avro-test-bucket/a-twitter.parquet"; - LoadJobConfiguration loadJobConfiguration = LoadJobConfiguration.newBuilder(tableId, SOURCE_URIS, FormatOptions.parquet()) - .setReferenceFileSchemaUri(referenceFileSchema).build(); + .setReferenceFileSchemaUri(referenceFileSchema) + .build(); Job job = bigquery.create(JobInfo.of(loadJobConfiguration)); // Blocks until this load table job completes its execution, either failing or succeeding. @@ -4662,7 +4678,5 @@ public void testReferenceFileSchemaUriForParquet() { } catch (BigQueryException | InterruptedException e) { System.out.println("Column not added during load append \n" + e.toString()); } - - } - } +} From 29c0ec0d0bbdd26c163872ac5f46c920e542d211 Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Fri, 9 Sep 2022 14:03:33 -0400 Subject: [PATCH 04/15] chore: fix clirr check --- google-cloud-bigquery/clirr-ignored-differences.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/google-cloud-bigquery/clirr-ignored-differences.xml b/google-cloud-bigquery/clirr-ignored-differences.xml index 2ad26f946..7a882777a 100644 --- a/google-cloud-bigquery/clirr-ignored-differences.xml +++ b/google-cloud-bigquery/clirr-ignored-differences.xml @@ -14,4 +14,9 @@ com.google.api.services.bigquery.model.GetQueryResultsResponse getQueryResultsWithRowLimit(java.lang.String, java.lang.String, java.lang.String, java.lang.Integer) getQueryResultsWithRowLimit is just used by ConnectionImpl at the moment so it should be fine to update the signature instead of writing an overloaded method + + 7013 + com/google/cloud/bigquery/ExternalTableDefinition* + *ReferenceFileSchemaUri(*) + \ No newline at end of file From e0926cf0f543dd9cbc548843abc814afb5cf245b Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Fri, 9 Sep 2022 15:25:52 -0400 Subject: [PATCH 05/15] chore: add assertion to tests --- .../cloud/bigquery/it/ITBigQueryTest.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 784142f78..be6b58673 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -65,6 +65,7 @@ import com.google.cloud.bigquery.ExternalTableDefinition; import com.google.cloud.bigquery.ExtractJobConfiguration; import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.Field.Mode; import com.google.cloud.bigquery.FieldList; import com.google.cloud.bigquery.FieldValue; import com.google.cloud.bigquery.FieldValue.Attribute; @@ -4592,6 +4593,11 @@ public void testReferenceFileSchemaUriForAvro() { try { String destinationTableName = "test_reference_file_schema_avro"; TableId tableId = TableId.of(DATASET, destinationTableName); + Schema expectedSchema = Schema.of( + Field.newBuilder("username", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("timestamp", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build()); // By default, the table should have c-twitter schema because it is lexicographically last. // a-twitter schema (username, tweet, timestamp, likes) @@ -4630,6 +4636,11 @@ public void testReferenceFileSchemaUriForAvro() { "BigQuery job was unable to load into the table due to an error:" + job.getStatus().getError()); } + + LoadJobConfiguration actualLoadJobConfiguration = job.getConfiguration(); + Table generatedTable = bigquery.getTable(actualLoadJobConfiguration.getDestinationTable()); + + assertEquals(expectedSchema, generatedTable.getDefinition().getSchema()); } catch (BigQueryException | InterruptedException e) { System.out.println("Column not added during load append \n" + e.toString()); } @@ -4640,6 +4651,11 @@ public void testReferenceFileSchemaUriForParquet() { try { String destinationTableName = "test_reference_file_schema_parquet"; TableId tableId = TableId.of(DATASET, destinationTableName); + Schema expectedSchema = Schema.of( + Field.newBuilder("username", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("timestamp", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build()); // By default, the table should have c-twitter schema because it is lexicographically last. // a-twitter schema (username, tweet, timestamp, likes) @@ -4658,7 +4674,9 @@ public void testReferenceFileSchemaUriForParquet() { + "/bigquery/federated-formats-reference-file-schema/c-twitter.parquet"); // Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema - String referenceFileSchema = "gs://avro-test-bucket/a-twitter.parquet"; + String referenceFileSchema = "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/a-twitter.parquet"; LoadJobConfiguration loadJobConfiguration = LoadJobConfiguration.newBuilder(tableId, SOURCE_URIS, FormatOptions.parquet()) @@ -4675,6 +4693,10 @@ public void testReferenceFileSchemaUriForParquet() { "BigQuery job was unable to load into the table due to an error:" + job.getStatus().getError()); } + LoadJobConfiguration actualLoadJobConfiguration = job.getConfiguration(); + Table generatedTable = bigquery.getTable(actualLoadJobConfiguration.getDestinationTable()); + + assertEquals(expectedSchema, generatedTable.getDefinition().getSchema()); } catch (BigQueryException | InterruptedException e) { System.out.println("Column not added during load append \n" + e.toString()); } From 15e005df9b45a33341168cd42f6b41a312863067 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 9 Sep 2022 19:28:32 +0000 Subject: [PATCH 06/15] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../cloud/bigquery/it/ITBigQueryTest.java | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index be6b58673..61dbf9981 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -4593,11 +4593,16 @@ public void testReferenceFileSchemaUriForAvro() { try { String destinationTableName = "test_reference_file_schema_avro"; TableId tableId = TableId.of(DATASET, destinationTableName); - Schema expectedSchema = Schema.of( - Field.newBuilder("username", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("timestamp", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build()); + Schema expectedSchema = + Schema.of( + Field.newBuilder("username", StandardSQLTypeName.STRING) + .setMode(Mode.NULLABLE) + .build(), + Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("timestamp", StandardSQLTypeName.STRING) + .setMode(Mode.NULLABLE) + .build(), + Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build()); // By default, the table should have c-twitter schema because it is lexicographically last. // a-twitter schema (username, tweet, timestamp, likes) @@ -4651,11 +4656,16 @@ public void testReferenceFileSchemaUriForParquet() { try { String destinationTableName = "test_reference_file_schema_parquet"; TableId tableId = TableId.of(DATASET, destinationTableName); - Schema expectedSchema = Schema.of( - Field.newBuilder("username", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("timestamp", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build()); + Schema expectedSchema = + Schema.of( + Field.newBuilder("username", StandardSQLTypeName.STRING) + .setMode(Mode.NULLABLE) + .build(), + Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("timestamp", StandardSQLTypeName.STRING) + .setMode(Mode.NULLABLE) + .build(), + Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build()); // By default, the table should have c-twitter schema because it is lexicographically last. // a-twitter schema (username, tweet, timestamp, likes) @@ -4674,9 +4684,10 @@ public void testReferenceFileSchemaUriForParquet() { + "/bigquery/federated-formats-reference-file-schema/c-twitter.parquet"); // Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema - String referenceFileSchema = "gs://" - + CLOUD_SAMPLES_DATA - + "/bigquery/federated-formats-reference-file-schema/a-twitter.parquet"; + String referenceFileSchema = + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/a-twitter.parquet"; LoadJobConfiguration loadJobConfiguration = LoadJobConfiguration.newBuilder(tableId, SOURCE_URIS, FormatOptions.parquet()) From 0c6bb2c2208d61aa39ea24036ea6903b4f86dd36 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 9 Sep 2022 19:30:31 +0000 Subject: [PATCH 07/15] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../cloud/bigquery/it/ITBigQueryTest.java | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index be6b58673..61dbf9981 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -4593,11 +4593,16 @@ public void testReferenceFileSchemaUriForAvro() { try { String destinationTableName = "test_reference_file_schema_avro"; TableId tableId = TableId.of(DATASET, destinationTableName); - Schema expectedSchema = Schema.of( - Field.newBuilder("username", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("timestamp", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build()); + Schema expectedSchema = + Schema.of( + Field.newBuilder("username", StandardSQLTypeName.STRING) + .setMode(Mode.NULLABLE) + .build(), + Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("timestamp", StandardSQLTypeName.STRING) + .setMode(Mode.NULLABLE) + .build(), + Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build()); // By default, the table should have c-twitter schema because it is lexicographically last. // a-twitter schema (username, tweet, timestamp, likes) @@ -4651,11 +4656,16 @@ public void testReferenceFileSchemaUriForParquet() { try { String destinationTableName = "test_reference_file_schema_parquet"; TableId tableId = TableId.of(DATASET, destinationTableName); - Schema expectedSchema = Schema.of( - Field.newBuilder("username", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("timestamp", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build()); + Schema expectedSchema = + Schema.of( + Field.newBuilder("username", StandardSQLTypeName.STRING) + .setMode(Mode.NULLABLE) + .build(), + Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("timestamp", StandardSQLTypeName.STRING) + .setMode(Mode.NULLABLE) + .build(), + Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build()); // By default, the table should have c-twitter schema because it is lexicographically last. // a-twitter schema (username, tweet, timestamp, likes) @@ -4674,9 +4684,10 @@ public void testReferenceFileSchemaUriForParquet() { + "/bigquery/federated-formats-reference-file-schema/c-twitter.parquet"); // Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema - String referenceFileSchema = "gs://" - + CLOUD_SAMPLES_DATA - + "/bigquery/federated-formats-reference-file-schema/a-twitter.parquet"; + String referenceFileSchema = + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/a-twitter.parquet"; LoadJobConfiguration loadJobConfiguration = LoadJobConfiguration.newBuilder(tableId, SOURCE_URIS, FormatOptions.parquet()) From fe4416dc1393dfa5cdac625c6f94bab09817847a Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Fri, 9 Sep 2022 16:53:55 -0400 Subject: [PATCH 08/15] chore: add create external table tests --- .../cloud/bigquery/it/ITBigQueryTest.java | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index be6b58673..a4722a923 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -4701,4 +4701,66 @@ public void testReferenceFileSchemaUriForParquet() { System.out.println("Column not added during load append \n" + e.toString()); } } + + @Test + public void testCreateExternalTableWithReferenceFileSchemaAvro(){ + String destinationTableName = "test_create_external_table_reference_file_schema_avro"; + TableId tableId = TableId.of(DATASET, destinationTableName); + Schema expectedSchema = Schema.of( + Field.newBuilder("username", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("timestamp", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build()); + String CLOUD_SAMPLES_DATA = "cloud-samples-data"; + + // By default, the table should have c-twitter schema because it is lexicographically last. + // a-twitter schema (username, tweet, timestamp, likes) + // b-twitter schema (username, tweet, timestamp) + // c-twitter schema (username, tweet) + String SOURCE_URI = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/*.avro"; + + // Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema + String referenceFileSchema = + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/a-twitter.avro"; + + ExternalTableDefinition externalTableDefinition = + ExternalTableDefinition.newBuilder(SOURCE_URI, FormatOptions.avro()).setReferenceFileSchemaUri(referenceFileSchema).build(); + TableInfo tableInfo = TableInfo.of(tableId, externalTableDefinition); + Table createdTable = bigquery.create(tableInfo); + Table generatedTable = bigquery.getTable(createdTable.getTableId()); + assertEquals(expectedSchema, generatedTable.getDefinition().getSchema()); + } + + @Test + public void testCreateExternalTableWithReferenceFileSchemaParquet(){ + String destinationTableName = "test_create_external_table_reference_file_schema_parquet"; + TableId tableId = TableId.of(DATASET, destinationTableName); + Schema expectedSchema = Schema.of( + Field.newBuilder("username", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("timestamp", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build()); + String CLOUD_SAMPLES_DATA = "cloud-samples-data"; + + // By default, the table should have c-twitter schema because it is lexicographically last. + // a-twitter schema (username, tweet, timestamp, likes) + // b-twitter schema (username, tweet, timestamp) + // c-twitter schema (username, tweet) + String SOURCE_URI = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/*.parquet"; + + // Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema + String referenceFileSchema = + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/a-twitter.parquet"; + + ExternalTableDefinition externalTableDefinition = + ExternalTableDefinition.newBuilder(SOURCE_URI, FormatOptions.parquet()).setReferenceFileSchemaUri(referenceFileSchema).build(); + TableInfo tableInfo = TableInfo.of(tableId, externalTableDefinition); + Table createdTable = bigquery.create(tableInfo); + Table generatedTable = bigquery.getTable(createdTable.getTableId()); + assertEquals(expectedSchema, generatedTable.getDefinition().getSchema()); + } } From e7da327c3fdb12f1ff2ea074f012e80205b9855a Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 9 Sep 2022 20:58:16 +0000 Subject: [PATCH 09/15] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../cloud/bigquery/it/ITBigQueryTest.java | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 033a1e467..4becd070a 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -4714,21 +4714,25 @@ public void testReferenceFileSchemaUriForParquet() { } @Test - public void testCreateExternalTableWithReferenceFileSchemaAvro(){ + public void testCreateExternalTableWithReferenceFileSchemaAvro() { String destinationTableName = "test_create_external_table_reference_file_schema_avro"; TableId tableId = TableId.of(DATASET, destinationTableName); - Schema expectedSchema = Schema.of( - Field.newBuilder("username", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("timestamp", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build()); + Schema expectedSchema = + Schema.of( + Field.newBuilder("username", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("timestamp", StandardSQLTypeName.STRING) + .setMode(Mode.NULLABLE) + .build(), + Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build()); String CLOUD_SAMPLES_DATA = "cloud-samples-data"; // By default, the table should have c-twitter schema because it is lexicographically last. // a-twitter schema (username, tweet, timestamp, likes) // b-twitter schema (username, tweet, timestamp) // c-twitter schema (username, tweet) - String SOURCE_URI = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/*.avro"; + String SOURCE_URI = + "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/*.avro"; // Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema String referenceFileSchema = @@ -4737,7 +4741,9 @@ public void testCreateExternalTableWithReferenceFileSchemaAvro(){ + "/bigquery/federated-formats-reference-file-schema/a-twitter.avro"; ExternalTableDefinition externalTableDefinition = - ExternalTableDefinition.newBuilder(SOURCE_URI, FormatOptions.avro()).setReferenceFileSchemaUri(referenceFileSchema).build(); + ExternalTableDefinition.newBuilder(SOURCE_URI, FormatOptions.avro()) + .setReferenceFileSchemaUri(referenceFileSchema) + .build(); TableInfo tableInfo = TableInfo.of(tableId, externalTableDefinition); Table createdTable = bigquery.create(tableInfo); Table generatedTable = bigquery.getTable(createdTable.getTableId()); @@ -4745,21 +4751,27 @@ public void testCreateExternalTableWithReferenceFileSchemaAvro(){ } @Test - public void testCreateExternalTableWithReferenceFileSchemaParquet(){ + public void testCreateExternalTableWithReferenceFileSchemaParquet() { String destinationTableName = "test_create_external_table_reference_file_schema_parquet"; TableId tableId = TableId.of(DATASET, destinationTableName); - Schema expectedSchema = Schema.of( - Field.newBuilder("username", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("timestamp", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build()); + Schema expectedSchema = + Schema.of( + Field.newBuilder("username", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(), + Field.newBuilder("timestamp", StandardSQLTypeName.STRING) + .setMode(Mode.NULLABLE) + .build(), + Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build()); String CLOUD_SAMPLES_DATA = "cloud-samples-data"; // By default, the table should have c-twitter schema because it is lexicographically last. // a-twitter schema (username, tweet, timestamp, likes) // b-twitter schema (username, tweet, timestamp) // c-twitter schema (username, tweet) - String SOURCE_URI = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/*.parquet"; + String SOURCE_URI = + "gs://" + + CLOUD_SAMPLES_DATA + + "/bigquery/federated-formats-reference-file-schema/*.parquet"; // Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema String referenceFileSchema = @@ -4768,7 +4780,9 @@ public void testCreateExternalTableWithReferenceFileSchemaParquet(){ + "/bigquery/federated-formats-reference-file-schema/a-twitter.parquet"; ExternalTableDefinition externalTableDefinition = - ExternalTableDefinition.newBuilder(SOURCE_URI, FormatOptions.parquet()).setReferenceFileSchemaUri(referenceFileSchema).build(); + ExternalTableDefinition.newBuilder(SOURCE_URI, FormatOptions.parquet()) + .setReferenceFileSchemaUri(referenceFileSchema) + .build(); TableInfo tableInfo = TableInfo.of(tableId, externalTableDefinition); Table createdTable = bigquery.create(tableInfo); Table generatedTable = bigquery.getTable(createdTable.getTableId()); From c8b1180b67bfc62509892cf5c973ba19ffb3fbdc Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Mon, 12 Sep 2022 12:10:32 -0400 Subject: [PATCH 10/15] chore: delete table for external table after testing --- .../test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 4becd070a..58d95ecfe 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -4787,5 +4787,7 @@ public void testCreateExternalTableWithReferenceFileSchemaParquet() { Table createdTable = bigquery.create(tableInfo); Table generatedTable = bigquery.getTable(createdTable.getTableId()); assertEquals(expectedSchema, generatedTable.getDefinition().getSchema()); + boolean success = bigquery.delete(tableId); + assertEquals(true, success); } } From 3f352137e1933251f10e49e7fc1c43c209a130f7 Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Mon, 12 Sep 2022 12:11:14 -0400 Subject: [PATCH 11/15] comment --- .../test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 58d95ecfe..e5ceaf663 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -4787,6 +4787,7 @@ public void testCreateExternalTableWithReferenceFileSchemaParquet() { Table createdTable = bigquery.create(tableInfo); Table generatedTable = bigquery.getTable(createdTable.getTableId()); assertEquals(expectedSchema, generatedTable.getDefinition().getSchema()); + // clean up after test to avoid conflict with other tests boolean success = bigquery.delete(tableId); assertEquals(true, success); } From 3f815daeadafc19a19449eb0dbd11fa0666f67d9 Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Mon, 12 Sep 2022 12:39:19 -0400 Subject: [PATCH 12/15] cleanup --- .../com/google/cloud/bigquery/it/ITBigQueryTest.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index e5ceaf663..eab91fbb3 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -4646,6 +4646,9 @@ public void testReferenceFileSchemaUriForAvro() { Table generatedTable = bigquery.getTable(actualLoadJobConfiguration.getDestinationTable()); assertEquals(expectedSchema, generatedTable.getDefinition().getSchema()); + // clean up after test to avoid conflict with other tests + boolean success = bigquery.delete(tableId); + assertEquals(true, success); } catch (BigQueryException | InterruptedException e) { System.out.println("Column not added during load append \n" + e.toString()); } @@ -4708,6 +4711,9 @@ public void testReferenceFileSchemaUriForParquet() { Table generatedTable = bigquery.getTable(actualLoadJobConfiguration.getDestinationTable()); assertEquals(expectedSchema, generatedTable.getDefinition().getSchema()); + // clean up after test to avoid conflict with other tests + boolean success = bigquery.delete(tableId); + assertEquals(true, success); } catch (BigQueryException | InterruptedException e) { System.out.println("Column not added during load append \n" + e.toString()); } @@ -4748,6 +4754,9 @@ public void testCreateExternalTableWithReferenceFileSchemaAvro() { Table createdTable = bigquery.create(tableInfo); Table generatedTable = bigquery.getTable(createdTable.getTableId()); assertEquals(expectedSchema, generatedTable.getDefinition().getSchema()); + // clean up after test to avoid conflict with other tests + boolean success = bigquery.delete(tableId); + assertEquals(true, success); } @Test From c4ffd072d4c279e07cd012f38bb238614db14e6b Mon Sep 17 00:00:00 2001 From: Neenu1995 Date: Mon, 12 Sep 2022 14:23:09 -0400 Subject: [PATCH 13/15] chore: remove enforced login from library code --- .../bigquery/ExternalTableDefinition.java | 18 ++---------------- .../cloud/bigquery/LoadJobConfiguration.java | 9 --------- .../cloud/bigquery/it/ITBigQueryTest.java | 16 ++-------------- 3 files changed, 4 insertions(+), 39 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java index e031348e3..e8dc34d7f 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java @@ -328,14 +328,9 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC if (getAutodetect() != null) { externalConfigurationPb.setAutodetect(getAutodetect()); } - if (FormatOptions.AVRO.equals(getFormatOptions().getType()) - || FormatOptions.PARQUET.equals(getFormatOptions().getType()) - || FormatOptions.ORC.equals(getFormatOptions().getType())) { - if (getReferenceFileSchemaUri() != null) { externalConfigurationPb.setReferenceFileSchemaUri(getReferenceFileSchemaUri()); } - } if (getHivePartitioningOptions() != null) { externalConfigurationPb.setHivePartitioningOptions(getHivePartitioningOptions().toPb()); @@ -506,13 +501,9 @@ static ExternalTableDefinition fromPb(Table tablePb) { builder.setHivePartitioningOptions( HivePartitioningOptions.fromPb(externalDataConfiguration.getHivePartitioningOptions())); } - if (FormatOptions.AVRO.equals(externalDataConfiguration.getSourceFormat()) - || FormatOptions.PARQUET.equals(externalDataConfiguration.getSourceFormat()) - || FormatOptions.ORC.equals(externalDataConfiguration.getSourceFormat())) { if (externalDataConfiguration.getReferenceFileSchemaUri() != null) { builder.setReferenceFileSchemaUri(externalDataConfiguration.getReferenceFileSchemaUri()); } - } } return builder.build(); } @@ -565,14 +556,9 @@ static ExternalTableDefinition fromExternalDataConfiguration( if (externalDataConfiguration.getAutodetect() != null) { builder.setAutodetect(externalDataConfiguration.getAutodetect()); } - if (FormatOptions.AVRO.equals(externalDataConfiguration.getSourceFormat()) - || FormatOptions.PARQUET.equals(externalDataConfiguration.getSourceFormat()) - || FormatOptions.ORC.equals(externalDataConfiguration.getSourceFormat())) { - if (externalDataConfiguration.getReferenceFileSchemaUri() != null) { - builder.setReferenceFileSchemaUri(externalDataConfiguration.getReferenceFileSchemaUri()); - } + if (externalDataConfiguration.getReferenceFileSchemaUri() != null) { + builder.setReferenceFileSchemaUri(externalDataConfiguration.getReferenceFileSchemaUri()); } - if (externalDataConfiguration.getHivePartitioningOptions() != null) { builder.setHivePartitioningOptions( HivePartitioningOptions.fromPb(externalDataConfiguration.getHivePartitioningOptions())); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java index af73ceacc..dfc399a75 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java @@ -202,13 +202,8 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur this.hivePartitioningOptions = HivePartitioningOptions.fromPb(loadConfigurationPb.getHivePartitioningOptions()); } - if (FormatOptions.AVRO.equals(loadConfigurationPb.getSourceFormat()) - || FormatOptions.PARQUET.equals(loadConfigurationPb.getSourceFormat()) - || FormatOptions.ORC.equals(loadConfigurationPb.getSourceFormat())) { - if (loadConfigurationPb.getReferenceFileSchemaUri() != null) { this.referenceFileSchemaUri = loadConfigurationPb.getReferenceFileSchemaUri(); - } } } @@ -656,12 +651,8 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { if (hivePartitioningOptions != null) { loadConfigurationPb.setHivePartitioningOptions(hivePartitioningOptions.toPb()); } - if (FormatOptions.AVRO.equals(loadConfigurationPb.getSourceFormat()) - || FormatOptions.PARQUET.equals(loadConfigurationPb.getSourceFormat()) - || FormatOptions.ORC.equals(loadConfigurationPb.getSourceFormat())) { if (referenceFileSchemaUri != null) { loadConfigurationPb.setReferenceFileSchemaUri(referenceFileSchemaUri); - } } jobConfiguration.setLoad(loadConfigurationPb); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index eab91fbb3..9a4b18a48 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -4634,13 +4634,7 @@ public void testReferenceFileSchemaUriForAvro() { Job job = bigquery.create(JobInfo.of(loadJobConfiguration)); // Blocks until this load table job completes its execution, either failing or succeeding. job = job.waitFor(); - if (job.isDone()) { - System.out.println("Avro data from GCS successfully loaded in a table"); - } else { - System.out.println( - "BigQuery job was unable to load into the table due to an error:" - + job.getStatus().getError()); - } + assertEquals(true, job.isDone()); LoadJobConfiguration actualLoadJobConfiguration = job.getConfiguration(); Table generatedTable = bigquery.getTable(actualLoadJobConfiguration.getDestinationTable()); @@ -4700,13 +4694,7 @@ public void testReferenceFileSchemaUriForParquet() { Job job = bigquery.create(JobInfo.of(loadJobConfiguration)); // Blocks until this load table job completes its execution, either failing or succeeding. job = job.waitFor(); - if (job.isDone()) { - System.out.println("Parquet data from GCS successfully loaded in a table"); - } else { - System.out.println( - "BigQuery job was unable to load into the table due to an error:" - + job.getStatus().getError()); - } + assertEquals(true, job.isDone()); LoadJobConfiguration actualLoadJobConfiguration = job.getConfiguration(); Table generatedTable = bigquery.getTable(actualLoadJobConfiguration.getDestinationTable()); From bcac00b96b5cd8ffd607ecbc3d7529c1f4647311 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 12 Sep 2022 18:26:30 +0000 Subject: [PATCH 14/15] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../cloud/bigquery/ExternalTableDefinition.java | 12 ++++++------ .../google/cloud/bigquery/LoadJobConfiguration.java | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java index e8dc34d7f..cb327a3a9 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java @@ -328,9 +328,9 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC if (getAutodetect() != null) { externalConfigurationPb.setAutodetect(getAutodetect()); } - if (getReferenceFileSchemaUri() != null) { - externalConfigurationPb.setReferenceFileSchemaUri(getReferenceFileSchemaUri()); - } + if (getReferenceFileSchemaUri() != null) { + externalConfigurationPb.setReferenceFileSchemaUri(getReferenceFileSchemaUri()); + } if (getHivePartitioningOptions() != null) { externalConfigurationPb.setHivePartitioningOptions(getHivePartitioningOptions().toPb()); @@ -501,9 +501,9 @@ static ExternalTableDefinition fromPb(Table tablePb) { builder.setHivePartitioningOptions( HivePartitioningOptions.fromPb(externalDataConfiguration.getHivePartitioningOptions())); } - if (externalDataConfiguration.getReferenceFileSchemaUri() != null) { - builder.setReferenceFileSchemaUri(externalDataConfiguration.getReferenceFileSchemaUri()); - } + if (externalDataConfiguration.getReferenceFileSchemaUri() != null) { + builder.setReferenceFileSchemaUri(externalDataConfiguration.getReferenceFileSchemaUri()); + } } return builder.build(); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java index dfc399a75..ee142be01 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java @@ -202,8 +202,8 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur this.hivePartitioningOptions = HivePartitioningOptions.fromPb(loadConfigurationPb.getHivePartitioningOptions()); } - if (loadConfigurationPb.getReferenceFileSchemaUri() != null) { - this.referenceFileSchemaUri = loadConfigurationPb.getReferenceFileSchemaUri(); + if (loadConfigurationPb.getReferenceFileSchemaUri() != null) { + this.referenceFileSchemaUri = loadConfigurationPb.getReferenceFileSchemaUri(); } } @@ -651,8 +651,8 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { if (hivePartitioningOptions != null) { loadConfigurationPb.setHivePartitioningOptions(hivePartitioningOptions.toPb()); } - if (referenceFileSchemaUri != null) { - loadConfigurationPb.setReferenceFileSchemaUri(referenceFileSchemaUri); + if (referenceFileSchemaUri != null) { + loadConfigurationPb.setReferenceFileSchemaUri(referenceFileSchemaUri); } jobConfiguration.setLoad(loadConfigurationPb); From 5d55222a75c0205ceb7e057a27ee898809c17276 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 12 Sep 2022 18:26:58 +0000 Subject: [PATCH 15/15] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../cloud/bigquery/ExternalTableDefinition.java | 12 ++++++------ .../google/cloud/bigquery/LoadJobConfiguration.java | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java index e8dc34d7f..cb327a3a9 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java @@ -328,9 +328,9 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC if (getAutodetect() != null) { externalConfigurationPb.setAutodetect(getAutodetect()); } - if (getReferenceFileSchemaUri() != null) { - externalConfigurationPb.setReferenceFileSchemaUri(getReferenceFileSchemaUri()); - } + if (getReferenceFileSchemaUri() != null) { + externalConfigurationPb.setReferenceFileSchemaUri(getReferenceFileSchemaUri()); + } if (getHivePartitioningOptions() != null) { externalConfigurationPb.setHivePartitioningOptions(getHivePartitioningOptions().toPb()); @@ -501,9 +501,9 @@ static ExternalTableDefinition fromPb(Table tablePb) { builder.setHivePartitioningOptions( HivePartitioningOptions.fromPb(externalDataConfiguration.getHivePartitioningOptions())); } - if (externalDataConfiguration.getReferenceFileSchemaUri() != null) { - builder.setReferenceFileSchemaUri(externalDataConfiguration.getReferenceFileSchemaUri()); - } + if (externalDataConfiguration.getReferenceFileSchemaUri() != null) { + builder.setReferenceFileSchemaUri(externalDataConfiguration.getReferenceFileSchemaUri()); + } } return builder.build(); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java index dfc399a75..ee142be01 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java @@ -202,8 +202,8 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur this.hivePartitioningOptions = HivePartitioningOptions.fromPb(loadConfigurationPb.getHivePartitioningOptions()); } - if (loadConfigurationPb.getReferenceFileSchemaUri() != null) { - this.referenceFileSchemaUri = loadConfigurationPb.getReferenceFileSchemaUri(); + if (loadConfigurationPb.getReferenceFileSchemaUri() != null) { + this.referenceFileSchemaUri = loadConfigurationPb.getReferenceFileSchemaUri(); } } @@ -651,8 +651,8 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { if (hivePartitioningOptions != null) { loadConfigurationPb.setHivePartitioningOptions(hivePartitioningOptions.toPb()); } - if (referenceFileSchemaUri != null) { - loadConfigurationPb.setReferenceFileSchemaUri(referenceFileSchemaUri); + if (referenceFileSchemaUri != null) { + loadConfigurationPb.setReferenceFileSchemaUri(referenceFileSchemaUri); } jobConfiguration.setLoad(loadConfigurationPb);