From d14b4423ea9552bb674bc7be8e982564434182b0 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sun, 25 Sep 2022 21:45:00 -0700 Subject: [PATCH] Publish R2 destination (#17122) * Format code * Update doc * Add r2 to build doc * Update readme * Add r2 to destination definitions * Move test resource to base-java-s3 * Remove redundant dependency * Add missing dependency * Format json * Remove s3-destination-base-integration-test setting.gradle root * Add missing lzo dependency * Fix import in destination jdbc --- .../main/resources/icons/cloudflare-r2.svg | 4 + .../seed/destination_definitions.yaml | 7 + .../resources/seed/destination_specs.yaml | 276 ++++++++++++++++++ .../bases/base-java-s3/build.gradle | 2 + .../destination/s3/S3DestinationConfig.java | 6 +- .../s3/S3DestinationConfigTest.java | 1 + .../json_field_name_updater/test_case.json | 0 .../json_conversion_test_cases.json | 120 ++------ .../type_conversion_test_cases.json | 0 .../build.gradle | 1 - ...3AvroParquetDestinationAcceptanceTest.java | 1 - airbyte-integrations/builds.md | 1 + .../destination/gcs/GcsDestination.java | 3 +- .../jdbc/copy/s3/S3StreamCopierTest.java | 1 + .../connectors/destination-r2/README.md | 2 + .../destination/r2/R2Destination.java | 2 +- .../r2/R2AvroDestinationAcceptanceTest.java | 5 + .../r2/R2CsvDestinationAcceptanceTest.java | 5 + .../R2CsvGzipDestinationAcceptanceTest.java | 5 + .../r2/R2JsonlDestinationAcceptanceTest.java | 5 + .../R2JsonlGzipDestinationAcceptanceTest.java | 5 + .../R2ParquetDestinationAcceptanceTest.java | 5 + .../destination/s3/S3Destination.java | 2 +- docs/integrations/README.md | 1 + docs/integrations/destinations/r2.md | 7 +- settings.gradle | 3 - 26 files changed, 359 insertions(+), 111 deletions(-) create mode 100644 airbyte-config/init/src/main/resources/icons/cloudflare-r2.svg rename airbyte-integrations/{connectors/destination-s3 => bases/base-java-s3}/src/test/resources/parquet/json_field_name_updater/test_case.json (100%) rename airbyte-integrations/{connectors/destination-s3 => bases/base-java-s3}/src/test/resources/parquet/json_schema_converter/json_conversion_test_cases.json (95%) rename airbyte-integrations/{connectors/destination-s3 => bases/base-java-s3}/src/test/resources/parquet/json_schema_converter/type_conversion_test_cases.json (100%) diff --git a/airbyte-config/init/src/main/resources/icons/cloudflare-r2.svg b/airbyte-config/init/src/main/resources/icons/cloudflare-r2.svg new file mode 100644 index 000000000000..9a99e21ef39c --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/cloudflare-r2.svg @@ -0,0 +1,4 @@ + + diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 4ddbd1763acd..d93d5c8d9876 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -70,6 +70,13 @@ dockerImageTag: 0.1.12 documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse releaseStage: alpha +- name: Cloudflare R2 + destinationDefinitionId: 0fb07be9-7c3b-4336-850d-5efc006152ee + dockerRepository: airbyte/destination-r2 + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/destinations/r2 + icon: cloudflare-r2.svg + releaseStage: alpha - name: Databricks Lakehouse destinationDefinitionId: 072d5540-f236-4294-ba7c-ade8fd918496 dockerRepository: airbyte/destination-databricks diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index f321e4b92a86..4600dc34265c 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -973,6 +973,282 @@ - "overwrite" - "append" - "append_dedup" +- dockerImage: "airbyte/destination-r2:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/destinations/r2" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "R2 Destination Spec" + type: "object" + required: + - "account_id" + - "access_key_id" + - "secret_access_key" + - "s3_bucket_name" + - "s3_bucket_path" + - "format" + properties: + account_id: + type: "string" + description: "Cloudflare account ID" + title: "Cloudflare account ID" + examples: + - "12345678aa1a1a11111aaa1234567abc" + order: 0 + access_key_id: + type: "string" + description: "The access key ID to access the R2 bucket. Airbyte requires\ + \ Read and Write permissions to the given bucket. Read more here." + title: "R2 Key ID *" + airbyte_secret: true + examples: + - "A012345678910EXAMPLE" + order: 1 + secret_access_key: + type: "string" + description: "The corresponding secret to the access key ID. Read more here" + title: "R2 Access Key *" + airbyte_secret: true + examples: + - "a012345678910ABCDEFGHAbCdEfGhEXAMPLEKEY" + order: 2 + s3_bucket_name: + title: "R2 Bucket Name" + type: "string" + description: "The name of the R2 bucket. Read more here." + examples: + - "r2_sync" + order: 3 + s3_bucket_path: + title: "R2 Bucket Path" + description: "Directory under the R2 bucket where data will be written." + type: "string" + examples: + - "data_sync/test" + order: 4 + format: + title: "Output Format *" + type: "object" + description: "Format of the data output. See here for more details" + oneOf: + - title: "Avro: Apache Avro" + required: + - "format_type" + - "compression_codec" + properties: + format_type: + title: "Format Type *" + type: "string" + enum: + - "Avro" + default: "Avro" + order: 0 + compression_codec: + title: "Compression Codec *" + description: "The compression algorithm used to compress data. Default\ + \ to no compression." + type: "object" + oneOf: + - title: "No Compression" + required: + - "codec" + properties: + codec: + type: "string" + enum: + - "no compression" + default: "no compression" + - title: "Deflate" + required: + - "codec" + - "compression_level" + properties: + codec: + type: "string" + enum: + - "Deflate" + default: "Deflate" + compression_level: + title: "Deflate Level" + description: "0: no compression & fastest, 9: best compression\ + \ & slowest." + type: "integer" + default: 0 + minimum: 0 + maximum: 9 + - title: "bzip2" + required: + - "codec" + properties: + codec: + type: "string" + enum: + - "bzip2" + default: "bzip2" + - title: "xz" + required: + - "codec" + - "compression_level" + properties: + codec: + type: "string" + enum: + - "xz" + default: "xz" + compression_level: + title: "Compression Level" + description: "See here for details." + type: "integer" + default: 6 + minimum: 0 + maximum: 9 + - title: "zstandard" + required: + - "codec" + - "compression_level" + properties: + codec: + type: "string" + enum: + - "zstandard" + default: "zstandard" + compression_level: + title: "Compression Level" + description: "Negative levels are 'fast' modes akin to lz4 or\ + \ snappy, levels above 9 are generally for archival purposes,\ + \ and levels above 18 use a lot of memory." + type: "integer" + default: 3 + minimum: -5 + maximum: 22 + include_checksum: + title: "Include Checksum" + description: "If true, include a checksum with each data block." + type: "boolean" + default: false + - title: "snappy" + required: + - "codec" + properties: + codec: + type: "string" + enum: + - "snappy" + default: "snappy" + order: 1 + - title: "CSV: Comma-Separated Values" + required: + - "format_type" + - "flattening" + properties: + format_type: + title: "Format Type *" + type: "string" + enum: + - "CSV" + default: "CSV" + flattening: + type: "string" + title: "Normalization (Flattening)" + description: "Whether the input json data should be normalized (flattened)\ + \ in the output CSV. Please refer to docs for details." + default: "No flattening" + enum: + - "No flattening" + - "Root level flattening" + compression: + title: "Compression" + type: "object" + description: "Whether the output files should be compressed. If compression\ + \ is selected, the output filename will have an extra extension\ + \ (GZIP: \".csv.gz\")." + oneOf: + - title: "No Compression" + requires: + - "compression_type" + properties: + compression_type: + type: "string" + enum: + - "No Compression" + default: "No Compression" + - title: "GZIP" + requires: + - "compression_type" + properties: + compression_type: + type: "string" + enum: + - "GZIP" + default: "GZIP" + - title: "JSON Lines: Newline-delimited JSON" + required: + - "format_type" + properties: + format_type: + title: "Format Type *" + type: "string" + enum: + - "JSONL" + default: "JSONL" + compression: + title: "Compression" + type: "object" + description: "Whether the output files should be compressed. If compression\ + \ is selected, the output filename will have an extra extension\ + \ (GZIP: \".jsonl.gz\")." + oneOf: + - title: "No Compression" + requires: "compression_type" + properties: + compression_type: + type: "string" + enum: + - "No Compression" + default: "No Compression" + - title: "GZIP" + requires: "compression_type" + properties: + compression_type: + type: "string" + enum: + - "GZIP" + default: "GZIP" + order: 5 + s3_path_format: + title: "R2 Path Format (Optional)" + description: "Format string on how data will be organized inside the R2\ + \ bucket directory. Read more here" + type: "string" + examples: + - "${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_" + order: 6 + file_name_pattern: + type: "string" + description: "The pattern allows you to set the file-name format for the\ + \ R2 staging file(s)" + title: "R2 Filename pattern (Optional)" + examples: + - "{date}" + - "{date:yyyy_MM}" + - "{timestamp}" + - "{part_number}" + - "{sync_id}" + order: 7 + supportsIncremental: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: + - "overwrite" + - "append" - dockerImage: "airbyte/destination-databricks:0.2.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/databricks" diff --git a/airbyte-integrations/bases/base-java-s3/build.gradle b/airbyte-integrations/bases/base-java-s3/build.gradle index f5296ef7462c..eac8f75291d4 100644 --- a/airbyte-integrations/bases/base-java-s3/build.gradle +++ b/airbyte-integrations/bases/base-java-s3/build.gradle @@ -13,6 +13,7 @@ dependencies { implementation ('org.apache.parquet:parquet-avro:1.12.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} implementation ('com.github.airbytehq:json-avro-converter:1.0.1') { exclude group: 'ch.qos.logback', module: 'logback-classic'} + implementation group: 'com.hadoop.gplcompression', name: 'hadoop-lzo', version: '0.4.20' // parquet implementation ('org.apache.hadoop:hadoop-common:3.3.3') { @@ -31,6 +32,7 @@ dependencies { testImplementation 'org.apache.commons:commons-lang3:3.11' testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' + testImplementation "org.mockito:mockito-inline:4.1.0" testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1' diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java index f75ae7bd38c2..bd959f491494 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java @@ -137,9 +137,9 @@ public static S3DestinationConfig getS3DestinationConfig(@Nonnull final JsonNode builder = builder.withEndpoint(endpoint); } builder = builder.withCheckIntegrity(false) - // https://developers.cloudflare.com/r2/platform/s3-compatibility/api/#implemented-object-level-operations - // 3 or less - .withUploadThreadsCount(S3StorageOperations.R2_UPLOAD_THREADS); + // https://developers.cloudflare.com/r2/platform/s3-compatibility/api/#implemented-object-level-operations + // 3 or less + .withUploadThreadsCount(S3StorageOperations.R2_UPLOAD_THREADS); } default -> { if (config.has(S_3_ENDPOINT)) { diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3DestinationConfigTest.java b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3DestinationConfigTest.java index ca4429a6c660..c804cebd0196 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3DestinationConfigTest.java +++ b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3DestinationConfigTest.java @@ -99,4 +99,5 @@ public void testGetS3DestinationConfigCF_R2Provider() { assertThat(awsCredentials.getAWSSecretKey()).isEqualTo("paste-secret-access-key-here"); assertThat(result.isCheckIntegrity()).isEqualTo(false); } + } diff --git a/airbyte-integrations/connectors/destination-s3/src/test/resources/parquet/json_field_name_updater/test_case.json b/airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_field_name_updater/test_case.json similarity index 100% rename from airbyte-integrations/connectors/destination-s3/src/test/resources/parquet/json_field_name_updater/test_case.json rename to airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_field_name_updater/test_case.json diff --git a/airbyte-integrations/connectors/destination-s3/src/test/resources/parquet/json_schema_converter/json_conversion_test_cases.json b/airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_schema_converter/json_conversion_test_cases.json similarity index 95% rename from airbyte-integrations/connectors/destination-s3/src/test/resources/parquet/json_schema_converter/json_conversion_test_cases.json rename to airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_schema_converter/json_conversion_test_cases.json index 5077309f5fe4..8f5928534904 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test/resources/parquet/json_schema_converter/json_conversion_test_cases.json +++ b/airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_schema_converter/json_conversion_test_cases.json @@ -1655,44 +1655,26 @@ "namespace": "namespace23", "appendAirbyteFields": true, "jsonSchema": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "same_record_name_field": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "sub_field_1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "any_of_field": { "anyOf": [ { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "same_record_name_field": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "sub_field_2": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } @@ -1742,10 +1724,7 @@ "fields": [ { "name": "sub_field_1", - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "default": null }, { @@ -1784,10 +1763,7 @@ "fields": [ { "name": "sub_field_2", - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "default": null }, { @@ -1851,44 +1827,26 @@ "namespace": "namespace24", "appendAirbyteFields": true, "jsonSchema": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "same_record_name_field": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "sub_field_1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "any_of_field": { "allOf": [ { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "same_record_name_field": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "sub_field_2": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } @@ -1938,10 +1896,7 @@ "fields": [ { "name": "sub_field_1", - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "default": null }, { @@ -1980,10 +1935,7 @@ "fields": [ { "name": "sub_field_2", - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "default": null }, { @@ -2047,44 +1999,26 @@ "namespace": "namespace25", "appendAirbyteFields": true, "jsonSchema": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "same_record_name_field": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "sub_field_1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "any_of_field": { "anyOf": [ { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "same_record_name_field": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "sub_field_2": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } @@ -2134,10 +2068,7 @@ "fields": [ { "name": "sub_field_1", - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "default": null }, { @@ -2176,10 +2107,7 @@ "fields": [ { "name": "sub_field_2", - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "default": null }, { diff --git a/airbyte-integrations/connectors/destination-s3/src/test/resources/parquet/json_schema_converter/type_conversion_test_cases.json b/airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_schema_converter/type_conversion_test_cases.json similarity index 100% rename from airbyte-integrations/connectors/destination-s3/src/test/resources/parquet/json_schema_converter/type_conversion_test_cases.json rename to airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_schema_converter/type_conversion_test_cases.json diff --git a/airbyte-integrations/bases/s3-destination-base-integration-test/build.gradle b/airbyte-integrations/bases/s3-destination-base-integration-test/build.gradle index 976ba25bcbfc..c39130f24999 100644 --- a/airbyte-integrations/bases/s3-destination-base-integration-test/build.gradle +++ b/airbyte-integrations/bases/s3-destination-base-integration-test/build.gradle @@ -7,7 +7,6 @@ dependencies { implementation project(':airbyte-protocol:protocol-models') implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:bases:base-java-s3') - implementation project(':airbyte-integrations:bases:base-java-s3') implementation(enforcedPlatform('org.junit:junit-bom:5.8.2')) implementation 'org.junit.jupiter:junit-jupiter-api' diff --git a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.java b/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.java index 1be301c90a80..96dd7b96db97 100644 --- a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.java @@ -29,7 +29,6 @@ import org.apache.avro.Schema.Field; import org.apache.avro.Schema.Type; import org.apache.avro.generic.GenericData.Record; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 038d99ed7869..405d1a325b32 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -150,6 +150,7 @@ | MySQL | [![destination-mysql](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-mysql%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-mysql) | | Postgres | [![destination-postgres](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-postgres%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-postgres) | | Pulsar | [![destination-pulsar](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-pulsar%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-pulsar) | +| R2 | (Not Setup) | | Redshift | [![destination-redshift](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-redshift%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-redshift) | | Rockset | [![destination-rockset](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-rockset%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-rockset) | | S3 | [![destination-s3](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-s3%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-s3) | diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java index 73b007b6da7d..5a7624501d8b 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java @@ -64,7 +64,8 @@ public AirbyteConnectionStatus check(final JsonNode config) { .withStatus(Status.FAILED) .withMessage(message); } catch (final Exception e) { - LOGGER.error("Exception attempting to access the AWS bucket: {}. Please make sure you account has all of these roles: {}", e.getMessage(), EXPECTED_ROLES); + LOGGER.error("Exception attempting to access the AWS bucket: {}. Please make sure you account has all of these roles: {}", e.getMessage(), + EXPECTED_ROLES); AirbyteTraceMessageUtility.emitConfigErrorTrace(e, e.getMessage()); return new AirbyteConnectionStatus() .withStatus(AirbyteConnectionStatus.Status.FAILED) diff --git a/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java index e80cf7323e5b..a0f487d59018 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java @@ -17,6 +17,7 @@ import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.destination.jdbc.SqlOperations; +import io.airbyte.integrations.destination.s3.S3DestinationConfig; import io.airbyte.integrations.destination.s3.csv.CsvSheetGenerator; import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig; import io.airbyte.integrations.destination.s3.csv.S3CsvWriter; diff --git a/airbyte-integrations/connectors/destination-r2/README.md b/airbyte-integrations/connectors/destination-r2/README.md index ac6a940fba9d..3e560f7ec44b 100644 --- a/airbyte-integrations/connectors/destination-r2/README.md +++ b/airbyte-integrations/connectors/destination-r2/README.md @@ -3,6 +3,8 @@ This is the repository for the R2 destination connector in Java. For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.io/integrations/destinations/r2). +**Currently, no integration test has been set up for this connector, which requires either a local R2 container, or a remote R2 account.** + ## Local development #### Building via Gradle diff --git a/airbyte-integrations/connectors/destination-r2/src/main/java/io/airbyte/integrations/destination/r2/R2Destination.java b/airbyte-integrations/connectors/destination-r2/src/main/java/io/airbyte/integrations/destination/r2/R2Destination.java index a38bbb83351a..dc72c1bec10c 100644 --- a/airbyte-integrations/connectors/destination-r2/src/main/java/io/airbyte/integrations/destination/r2/R2Destination.java +++ b/airbyte-integrations/connectors/destination-r2/src/main/java/io/airbyte/integrations/destination/r2/R2Destination.java @@ -16,9 +16,9 @@ public static void main(String[] args) throws Exception { new IntegrationRunner(new R2Destination()).run(args); } - @Override public StorageProvider storageProvider() { return StorageProvider.CF_R2; } + } diff --git a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2AvroDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2AvroDestinationAcceptanceTest.java index 687b1acecaed..7ae744f1db6e 100644 --- a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2AvroDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2AvroDestinationAcceptanceTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.r2; import io.airbyte.integrations.destination.s3.S3BaseAvroDestinationAcceptanceTest; @@ -14,4 +18,5 @@ protected String getImageName() { public StorageProvider storageProvider() { return StorageProvider.CF_R2; } + } diff --git a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2CsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2CsvDestinationAcceptanceTest.java index 696e83b34878..55f2a04e7e1a 100644 --- a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2CsvDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2CsvDestinationAcceptanceTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.r2; import io.airbyte.integrations.destination.s3.S3BaseCsvDestinationAcceptanceTest; @@ -14,4 +18,5 @@ protected String getImageName() { public StorageProvider storageProvider() { return StorageProvider.CF_R2; } + } diff --git a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2CsvGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2CsvGzipDestinationAcceptanceTest.java index 1521fb266b1b..75d8367ac6a3 100644 --- a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2CsvGzipDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2CsvGzipDestinationAcceptanceTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.r2; import io.airbyte.integrations.destination.s3.S3BaseCsvGzipDestinationAcceptanceTest; @@ -14,4 +18,5 @@ protected String getImageName() { public StorageProvider storageProvider() { return StorageProvider.CF_R2; } + } diff --git a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2JsonlDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2JsonlDestinationAcceptanceTest.java index 33cbf3c028bf..d08eb59199e2 100644 --- a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2JsonlDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2JsonlDestinationAcceptanceTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.r2; import io.airbyte.integrations.destination.s3.S3BaseJsonlDestinationAcceptanceTest; @@ -14,4 +18,5 @@ protected String getImageName() { public StorageProvider storageProvider() { return StorageProvider.CF_R2; } + } diff --git a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2JsonlGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2JsonlGzipDestinationAcceptanceTest.java index 997a3e65d457..752fc38172f7 100644 --- a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2JsonlGzipDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2JsonlGzipDestinationAcceptanceTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.r2; import io.airbyte.integrations.destination.s3.S3BaseJsonlGzipDestinationAcceptanceTest; @@ -14,4 +18,5 @@ protected String getImageName() { public StorageProvider storageProvider() { return StorageProvider.CF_R2; } + } diff --git a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2ParquetDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2ParquetDestinationAcceptanceTest.java index c3296b2de2db..8cf667afa519 100644 --- a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2ParquetDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2ParquetDestinationAcceptanceTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.r2; import io.airbyte.integrations.destination.s3.S3BaseParquetDestinationAcceptanceTest; @@ -19,4 +23,5 @@ protected String getImageName() { public StorageProvider storageProvider() { return StorageProvider.CF_R2; } + } diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java index 393afae67b48..935ded778b2b 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java @@ -8,7 +8,6 @@ public class S3Destination extends BaseS3Destination { - public S3Destination() {} public S3Destination(final S3DestinationConfigFactory s3DestinationConfigFactory) { @@ -23,4 +22,5 @@ public static void main(String[] args) throws Exception { public StorageProvider storageProvider() { return StorageProvider.AWS_S3; } + } diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 2a4ea131ef5e..2400c0c005cb 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -211,6 +211,7 @@ For more information about the grading system, see [Product Release Stages](http | [Oracle](destinations/oracle.md) | Alpha | Yes | | [Postgres](destinations/postgres.md) | Alpha | Yes | | [Pulsar](destinations/pulsar.md) | Alpha | No | +| [R2](destinations/r2.md) | Alpha | No | | [RabbitMQ](destinations/rabbitmq.md) | Alpha | No | | [Redis](destinations/redis.md) | Alpha | No | | [Redshift](destinations/redshift.md) | Beta | Yes | diff --git a/docs/integrations/destinations/r2.md b/docs/integrations/destinations/r2.md index 893d5b0f26c6..37f25fea20bc 100644 --- a/docs/integrations/destinations/r2.md +++ b/docs/integrations/destinations/r2.md @@ -279,7 +279,6 @@ Under the hood, an Airbyte data stream in JSON schema is first converted to an A ## CHANGELOG -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.1.0 | 2022-08-12 | [\#15296](https://github.com/airbytehq/airbyte/pull/15296) | Initial release. | - +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:-----------------------------------------------------------|:--------| +| 0.1.0 | 2022-09-25 | [\#15296](https://github.com/airbytehq/airbyte/pull/15296) | Initial release. | diff --git a/settings.gradle b/settings.gradle index 9f387dd740ac..09a5cbf56b82 100644 --- a/settings.gradle +++ b/settings.gradle @@ -149,6 +149,3 @@ if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD" } } } -include 'airbyte-integrations:bases:s3-destination-base-integration-test' -findProject(':airbyte-integrations:bases:s3-destination-base-integration-test')?.name = 's3-destination-base-integration-test' -