From 16d99b9b0cbbe3b0d4012cdbae3517929d9ea1f7 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 27 Jun 2022 12:02:45 -0600 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Destination=20Redshift:=20use=20?= =?UTF-8?q?s3=20bucket=20path=20for=20s3=20staging=20operations=20(#13916)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- .../connectors/destination-redshift/Dockerfile | 2 +- .../destination/redshift/RedshiftStagingS3Destination.java | 2 +- .../redshift/operations/RedshiftS3StagingSqlOperations.java | 5 ++++- docs/integrations/destinations/redshift.md | 1 + 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index bcb24bf5476c..f7fa338ac00b 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -225,7 +225,7 @@ - name: Redshift destinationDefinitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc dockerRepository: airbyte/destination-redshift - dockerImageTag: 0.3.44 + dockerImageTag: 0.3.45 documentationUrl: https://docs.airbyte.io/integrations/destinations/redshift icon: redshift.svg resourceRequirements: diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 69cb5050f1a2..424bf5674254 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -3617,7 +3617,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-redshift:0.3.44" +- dockerImage: "airbyte/destination-redshift:0.3.45" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/redshift" connectionSpecification: diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index 77b4c81f967a..07b6136dd82f 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.44 +LABEL io.airbyte.version=0.3.45 LABEL io.airbyte.name=airbyte/destination-redshift diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java index 47d2520bb8f1..5dd6d2313adb 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java @@ -67,7 +67,7 @@ public AirbyteConnectionStatus check(final JsonNode config) { .withMessage( "You cannot use ephemeral keys and disable purging your staging data. This would produce S3 objects that you cannot decrypt."); } - S3Destination.attemptS3WriteAndDelete(new S3StorageOperations(new RedshiftSQLNameTransformer(), s3Config.getS3Client(), s3Config), s3Config, ""); + S3Destination.attemptS3WriteAndDelete(new S3StorageOperations(new RedshiftSQLNameTransformer(), s3Config.getS3Client(), s3Config), s3Config, s3Config.getBucketPath()); final NamingConventionTransformer nameTransformer = getNamingResolver(); final RedshiftS3StagingSqlOperations redshiftS3StagingSqlOperations = diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java index 494ee50ff56a..020347014537 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java @@ -63,7 +63,10 @@ public String getStageName(String namespace, String streamName) { @Override public String getStagingPath(UUID connectionId, String namespace, String streamName, DateTime writeDatetime) { - return nameTransformer.applyDefaultCase(String.format("%s/%s_%02d_%02d_%02d_%s/", + final String bucketPath = s3Config.getBucketPath(); + final String prefix = bucketPath.isEmpty() ? "" : bucketPath + (bucketPath.endsWith("/") ? "" : "/"); + return nameTransformer.applyDefaultCase(String.format("%s%s/%s_%02d_%02d_%02d_%s/", + prefix, getStageName(namespace, streamName), writeDatetime.year().get(), writeDatetime.monthOfYear().get(), diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index 0518c683edf3..dabc92c7ff68 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -138,6 +138,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.45 | 2022-06-25 | [\#13916](https://github.com/airbytehq/airbyte/pull/13916) | Use the configured bucket path for S3 staging operations. | | 0.3.44 | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | | 0.3.43 | 2022-06-24 | [\#13690](https://github.com/airbytehq/airbyte/pull/13690) | Improved discovery for NOT SUPER column | | 0.3.42 | 2022-06-21 | [\#14013](https://github.com/airbytehq/airbyte/pull/14013) | Add an option to use encryption with staging in Redshift Destination |