diff --git a/airbyte-integrations/connectors/destination-s3/build.gradle b/airbyte-integrations/connectors/destination-s3/build.gradle index 49da4da3cdf0..b9b660b7a76e 100644 --- a/airbyte-integrations/connectors/destination-s3/build.gradle +++ b/airbyte-integrations/connectors/destination-s3/build.gradle @@ -24,7 +24,7 @@ dependencies { implementation group: 'org.apache.hadoop', name: 'hadoop-common', version: '3.3.3' implementation group: 'org.apache.hadoop', name: 'hadoop-aws', version: '3.3.3' implementation group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-core', version: '3.3.3' - implementation group: 'org.apache.parquet', name: 'parquet-avro', version: '1.12.0' + implementation group: 'org.apache.parquet', name: 'parquet-avro', version: '1.12.3' implementation group: 'com.github.airbytehq', name: 'json-avro-converter', version: '1.0.1' testImplementation 'org.apache.commons:commons-lang3:3.11' diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java index f025a0a0a8b9..b94c007f816a 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java @@ -39,7 +39,8 @@ public class ParquetSerializedBufferTest { "field1", 10000, "column2", "string value", "another field", true, - "nested_column", Map.of("array_column", List.of(1, 2, 3)))); + "nested_column", Map.of("array_column", List.of(1, 2, 3)), + "datetime_with_timezone", "2022-05-12T15:35:44.192950Z")); private static final String STREAM = "stream1"; private static final AirbyteStreamNameNamespacePair streamPair = new AirbyteStreamNameNamespacePair(STREAM, null); private static final AirbyteRecordMessage message = new AirbyteRecordMessage() @@ -50,7 +51,8 @@ public class ParquetSerializedBufferTest { Field.of("field1", JsonSchemaType.NUMBER), Field.of("column2", JsonSchemaType.STRING), Field.of("another field", JsonSchemaType.BOOLEAN), - Field.of("nested_column", JsonSchemaType.OBJECT)); + Field.of("nested_column", JsonSchemaType.OBJECT), + Field.of("datetime_with_timezone", JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE)); private static final ConfiguredAirbyteCatalog catalog = CatalogHelpers.createConfiguredAirbyteCatalog(STREAM, null, FIELDS); @Test @@ -60,7 +62,7 @@ public void testUncompressedParquetWriter() throws Exception { "format_type", "parquet"), "s3_bucket_name", "test", "s3_bucket_region", "us-east-2"))); - runTest(195L, 205L, config, getExpectedString()); + runTest(195L, 215L, config, getExpectedString()); } @Test @@ -72,7 +74,7 @@ public void testCompressedParquetWriter() throws Exception { "s3_bucket_name", "test", "s3_bucket_region", "us-east-2"))); // TODO: Compressed parquet is the same size as uncompressed?? - runTest(195L, 205L, config, getExpectedString()); + runTest(195L, 215L, config, getExpectedString()); } private static String getExpectedString() { @@ -80,6 +82,7 @@ private static String getExpectedString() { + "\"field1\": 10000.0, \"another_field\": true, " + "\"nested_column\": {\"_airbyte_additional_properties\": {\"array_column\": \"[1,2,3]\"}}, " + "\"column2\": \"string value\", " + + "\"datetime_with_timezone\": 1652369744192000, " + "\"_airbyte_additional_properties\": null}"; }