From 0f8e11e2432f1fc7c02be637e1cb0206edbf468f Mon Sep 17 00:00:00 2001 From: VitaliiMaltsev <39538064+VitaliiMaltsev@users.noreply.github.com> Date: Fri, 15 Apr 2022 19:17:46 +0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20BigQuery=20Destination=20failed?= =?UTF-8?q?=20conditional=20binding=20(#12068)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * BigQuery Destination check conditional binding * fixed checkstyle * BigQuery Destination fixed bug with contitional binding * update CHANGELOG * update CHANGELOG * add explaining comment * bump version * auto-bump connector version Co-authored-by: Octavia Squidington III --- .../main/resources/seed/destination_definitions.yaml | 2 +- .../src/main/resources/seed/destination_specs.yaml | 2 +- .../connectors/destination-bigquery/Dockerfile | 2 +- .../destination/bigquery/BigQueryDestination.java | 12 +++++++----- docs/integrations/destinations/bigquery.md | 1 + 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index a24409273104..6a522e9fbf79 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -24,7 +24,7 @@ - name: BigQuery destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 dockerRepository: airbyte/destination-bigquery - dockerImageTag: 1.1.0 + dockerImageTag: 1.1.1 documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery icon: bigquery.svg resourceRequirements: diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 2ecc807d2053..b22d58065ae5 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -285,7 +285,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-bigquery:1.1.0" +- dockerImage: "airbyte/destination-bigquery:1.1.1" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery" connectionSpecification: diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index 2eff9daaae37..ceab2eaaa910 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.1.0 +LABEL io.airbyte.version=1.1.1 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index f44fe4ea4bec..978f5abff05f 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -32,6 +32,7 @@ import io.airbyte.integrations.destination.bigquery.uploader.BigQueryUploaderFactory; import io.airbyte.integrations.destination.bigquery.uploader.UploaderType; import io.airbyte.integrations.destination.bigquery.uploader.config.UploaderConfig; +import io.airbyte.integrations.destination.gcs.GcsDestination; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.integrations.destination.gcs.GcsNameTransformer; import io.airbyte.integrations.destination.gcs.GcsStorageOperations; @@ -135,12 +136,13 @@ public AirbyteConnectionStatus checkStorageIamPermissions(final JsonNode config) .toList(); if (!missingPermissions.isEmpty()) { - LOGGER.error("Please make sure you account has all of these permissions:{}", REQUIRED_PERMISSIONS); + LOGGER.warn("Please make sure you account has all of these permissions:{}", REQUIRED_PERMISSIONS); + // if user or service account has a conditional binding for processing handling in the GCS bucket, + // testIamPermissions will not work properly, so we use the standard check method of GCS destination + final GcsDestination gcsDestination = new GcsDestination(); + final JsonNode gcsJsonNodeConfig = BigQueryUtils.getGcsJsonNodeConfig(config); + return gcsDestination.check(gcsJsonNodeConfig); - return new AirbyteConnectionStatus() - .withStatus(AirbyteConnectionStatus.Status.FAILED) - .withMessage("Could not connect to the Gcs bucket with the provided configuration. " - + "Missing permissions: " + missingPermissions); } return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index b65a11d50ad4..6916d66d93b0 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -207,6 +207,7 @@ This uploads data directly from your source to BigQuery. While this is faster to | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | +| 1.1.1 | 2022-04-15 | [12068](https://github.com/airbytehq/airbyte/pull/12068) | Fixed bug with GCS bucket conditional binding | | 1.1.0 | 2022-04-06 | [11776](https://github.com/airbytehq/airbyte/pull/11776) | Use serialized buffering strategy to reduce memory consumption. | | 1.0.2 | 2022-03-30 | [11620](https://github.com/airbytehq/airbyte/pull/11620) | Updated spec | | 1.0.1 | 2022-03-24 | [11350](https://github.com/airbytehq/airbyte/pull/11350) | Improve check performance |