From 25d955c83f6ca81076cabfcfab8d26a17d3995e0 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Fri, 3 Jun 2022 17:44:55 -0300 Subject: [PATCH 01/10] add flag to apply flatten to arrays --- .../java/io/airbyte/commons/json/Jsons.java | 16 ++++++- .../io/airbyte/commons/json/JsonsTest.java | 48 ++++++++++++++++--- .../operations/RedshiftSqlOperations.java | 2 +- .../persistence/job/tracker/JobTracker.java | 2 +- 4 files changed, 57 insertions(+), 11 deletions(-) diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java index a92e2c49985c..58d7b3788a6e 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @@ -230,16 +230,28 @@ public static int getIntOrZero(final JsonNode json, final List keys) { /** * Flattens an ObjectNode, or dumps it into a {null: value} map if it's not an object. + * this is used for JobTracker where is arrays aren't flatten and only converted to strings + * and Redshift SUPER type now reuse the function and need to arrays be flatten in the + * correct format */ @SuppressWarnings("PMD.ForLoopCanBeForeach") - public static Map flatten(final JsonNode node) { + public static Map flatten(final JsonNode node, final Boolean applyFlattenToArray) { if (node.isObject()) { final Map output = new HashMap<>(); for (final Iterator> it = node.fields(); it.hasNext();) { final Entry entry = it.next(); final String field = entry.getKey(); final JsonNode value = entry.getValue(); - mergeMaps(output, field, flatten(value)); + mergeMaps(output, field, flatten(value, false)); + } + return output; + } else if (node.isArray() && applyFlattenToArray) { + final Map output = new HashMap<>(); + final int arrayLen = node.size(); + for (int i = 0; i < arrayLen; i++) { + final String field = String.format("[%d]", i); + final JsonNode value = node.get(i); + mergeMaps(output, field, flatten(value, true)); } return output; } else { diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java index d1a844c02a5e..00a7b93f3602 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java @@ -21,8 +21,11 @@ import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; class JsonsTest { @@ -60,7 +63,8 @@ void testSerializeJsonNode() { Jsons.serialize(Jsons.jsonNode(ImmutableMap.of( TEST, ABC, TEST2, DEF)))); - // issue: 5878 add test for binary node serialization, binary data are serialized into base64 + // issue: 5878 add test for binary node serialization, binary data are + // serialized into base64 assertEquals( "{\"test\":\"dGVzdA==\"}", Jsons.serialize(Jsons.jsonNode(ImmutableMap.of( @@ -83,7 +87,8 @@ void testDeserializeToJsonNode() { assertEquals( "[{\"str\":\"abc\"},{\"str\":\"abc\"}]", Jsons.deserialize("[{\"str\":\"abc\"},{\"str\":\"abc\"}]").toString()); - // issue: 5878 add test for binary node deserialization, for now should be base64 string + // issue: 5878 add test for binary node deserialization, for now should be + // base64 string assertEquals( "{\"test\":\"dGVzdA==\"}", Jsons.deserialize("{\"test\":\"dGVzdA==\"}").toString()); @@ -157,7 +162,8 @@ void testToObject() { assertEquals( Lists.newArrayList(expected), - Jsons.object(Jsons.jsonNode(Lists.newArrayList(expected)), new TypeReference>() {})); + Jsons.object(Jsons.jsonNode(Lists.newArrayList(expected)), new TypeReference>() { + })); assertEquals( new ToClass(), @@ -173,7 +179,8 @@ void testTryToObject() { assertEquals( Optional.of(expected), - Jsons.tryObject(Jsons.deserialize(SERIALIZED_JSON), new TypeReference() {})); + Jsons.tryObject(Jsons.deserialize(SERIALIZED_JSON), new TypeReference() { + })); final ToClass emptyExpected = new ToClass(); assertEquals( @@ -182,7 +189,8 @@ void testTryToObject() { assertEquals( Optional.of(emptyExpected), - Jsons.tryObject(Jsons.deserialize("{\"str1\":\"abc\"}"), new TypeReference() {})); + Jsons.tryObject(Jsons.deserialize("{\"str1\":\"abc\"}"), new TypeReference() { + })); } @@ -230,7 +238,8 @@ void testToPrettyString() { @Test void testGetOptional() { - final JsonNode json = Jsons.deserialize("{ \"abc\": { \"def\": \"ghi\" }, \"jkl\": {}, \"mno\": \"pqr\", \"stu\": null }"); + final JsonNode json = Jsons + .deserialize("{ \"abc\": { \"def\": \"ghi\" }, \"jkl\": {}, \"mno\": \"pqr\", \"stu\": null }"); assertEquals(Optional.of(Jsons.jsonNode("ghi")), Jsons.getOptional(json, "abc", "def")); assertEquals(Optional.of(Jsons.emptyObject()), Jsons.getOptional(json, "jkl")); @@ -260,6 +269,30 @@ void testGetEstimatedByteSize() { assertEquals(Jsons.toBytes(json).length, Jsons.getEstimatedByteSize(json)); } + @Test + void testFlatten__noArrays() { + final JsonNode json = Jsons.deserialize("{ \"abc\": { \"def\": \"ghi\" }, \"jkl\": true, \"pqr\": 1 }"); + Map expected = Stream.of(new Object[][] { + { "abc.def", "ghi" }, + { "jkl", true }, + { "pqr", 1 }, + }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); + assertEquals(Jsons.flatten(json, false), expected); + } + + @Test + void testFlatten__withArrays() { + final JsonNode json = Jsons + .deserialize("{ \"abc\": [{ \"def\": \"ghi\" }, { \"fed\": \"ihg\" }], \"jkl\": true, \"pqr\": 1 }"); + Map expected = Stream.of(new Object[][] { + { "abc.[0].def", "ghi" }, + { "abc.[1].fed", "ihg" }, + { "jkl", true }, + { "pqr", 1 }, + }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); + assertEquals(Jsons.flatten(json, true), expected); + } + private static class ToClass { @JsonProperty("str") @@ -271,7 +304,8 @@ private static class ToClass { @JsonProperty("numLong") long numLong; - public ToClass() {} + public ToClass() { + } public ToClass(final String str, final Integer num, final long numLong) { this.str = str; diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java index 423cf9834520..5b5ae21b11f0 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java @@ -112,7 +112,7 @@ public boolean isValidData(final JsonNode data) { // check VARCHAR limits for VARCHAR fields within the SUPER object, if overall object is valid if (isValid) { - final Map dataMap = Jsons.flatten(data); + final Map dataMap = Jsons.flatten(data, true); for (final Object value : dataMap.values()) { if (value instanceof String stringValue) { final int stringDataSize = stringValue.getBytes(StandardCharsets.UTF_8).length; diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/tracker/JobTracker.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/tracker/JobTracker.java index 86c0b154891c..d851b7ac302e 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/tracker/JobTracker.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/tracker/JobTracker.java @@ -246,7 +246,7 @@ private static Map configToMetadata(final JsonNode config, final // * Otherwise, do some basic conversions to value-ish data. // It would be a weird thing to declare const: null, but in that case we don't want to report null // anyway, so explicitly use hasNonNull. - return Jsons.flatten(config); + return Jsons.flatten(config, false); } else if (schema.has("oneOf")) { // If this schema is a oneOf, then find the first sub-schema which the config matches // and use that sub-schema to convert the config to a map From 56d79f4de64afe38115fa668247e3a1664cd385c Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Fri, 3 Jun 2022 18:26:43 -0300 Subject: [PATCH 02/10] run format change docstirng --- .../src/main/java/io/airbyte/commons/json/Jsons.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java index 58d7b3788a6e..5f4aa241a891 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @@ -229,10 +229,10 @@ public static int getIntOrZero(final JsonNode json, final List keys) { } /** - * Flattens an ObjectNode, or dumps it into a {null: value} map if it's not an object. - * this is used for JobTracker where is arrays aren't flatten and only converted to strings - * and Redshift SUPER type now reuse the function and need to arrays be flatten in the - * correct format + * Flattens an ObjectNode, or dumps it into a {null: value} map if it's not an object. When + * applyFlattenToArray is true, each element in the array will be one entry in the returned map. + * This behavior is used in the Redshift SUPER type. When it is false, the whole array will be one + * entry. This is used in the JobTracker. */ @SuppressWarnings("PMD.ForLoopCanBeForeach") public static Map flatten(final JsonNode node, final Boolean applyFlattenToArray) { From a2838a282f6966de87183c5fd1a49ec2450684a6 Mon Sep 17 00:00:00 2001 From: Adam Bloom Date: Tue, 27 Sep 2022 12:14:52 -0600 Subject: [PATCH 03/10] fix recursion --- .../src/main/java/io/airbyte/commons/json/Jsons.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java index 5f4aa241a891..1f04ed9bd5c3 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @@ -242,7 +242,7 @@ public static Map flatten(final JsonNode node, final Boolean app final Entry entry = it.next(); final String field = entry.getKey(); final JsonNode value = entry.getValue(); - mergeMaps(output, field, flatten(value, false)); + mergeMaps(output, field, flatten(value, applyFlattenToArray)); } return output; } else if (node.isArray() && applyFlattenToArray) { @@ -251,7 +251,7 @@ public static Map flatten(final JsonNode node, final Boolean app for (int i = 0; i < arrayLen; i++) { final String field = String.format("[%d]", i); final JsonNode value = node.get(i); - mergeMaps(output, field, flatten(value, true)); + mergeMaps(output, field, flatten(value, applyFlattenToArray)); } return output; } else { From cf38f735581f0ad7dd33d3afd882e5eb1b72a957 Mon Sep 17 00:00:00 2001 From: Adam Bloom Date: Tue, 27 Sep 2022 12:25:53 -0600 Subject: [PATCH 04/10] add additional unit test cases for array flattening --- .../io/airbyte/commons/json/JsonsTest.java | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java index 00a7b93f3602..b7550f91e9aa 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java @@ -277,11 +277,23 @@ void testFlatten__noArrays() { { "jkl", true }, { "pqr", 1 }, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); - assertEquals(Jsons.flatten(json, false), expected); + assertEquals(expected, Jsons.flatten(json, false)); } @Test - void testFlatten__withArrays() { + void testFlatten__withArraysNoApplyFlatten() { + final JsonNode json = Jsons + .deserialize("{ \"abc\": [{ \"def\": \"ghi\" }, { \"fed\": \"ihg\" }], \"jkl\": true, \"pqr\": 1 }"); + Map expected = Stream.of(new Object[][] { + { "abc", "[{\"def\":\"ghi\"},{\"fed\":\"ihg\"}]" }, + { "jkl", true }, + { "pqr", 1 }, + }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); + assertEquals(expected, Jsons.flatten(json, false)); + } + + @Test + void testFlatten__withArraysApplyFlatten() { final JsonNode json = Jsons .deserialize("{ \"abc\": [{ \"def\": \"ghi\" }, { \"fed\": \"ihg\" }], \"jkl\": true, \"pqr\": 1 }"); Map expected = Stream.of(new Object[][] { @@ -290,7 +302,21 @@ void testFlatten__withArrays() { { "jkl", true }, { "pqr", 1 }, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); - assertEquals(Jsons.flatten(json, true), expected); + assertEquals(expected, Jsons.flatten(json, true)); + } + + @Test + void testFlatten__withArraysApplyFlattenNested() { + final JsonNode json = Jsons + .deserialize( + "{ \"abc\": [{ \"def\": {\"ghi\": [\"xyz\"] }}, { \"fed\": \"ihg\" }], \"jkl\": true, \"pqr\": 1 }"); + Map expected = Stream.of(new Object[][] { + { "abc.[0].def.ghi.[0]", "xyz" }, + { "abc.[1].fed", "ihg" }, + { "jkl", true }, + { "pqr", 1 }, + }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); + assertEquals(expected, Jsons.flatten(json, true)); } private static class ToClass { From 52d05c8a4723d87e4dd1b836ef0abd87efd3a156 Mon Sep 17 00:00:00 2001 From: Adam Bloom Date: Tue, 27 Sep 2022 12:31:42 -0600 Subject: [PATCH 05/10] add backward compatibility function --- .../src/main/java/io/airbyte/commons/json/Jsons.java | 9 +++++++++ .../test/java/io/airbyte/commons/json/JsonsTest.java | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java index 1f04ed9bd5c3..77ac363d90b0 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @@ -274,6 +274,15 @@ public static Map flatten(final JsonNode node, final Boolean app } } + /** + * Flattens an ObjectNode, or dumps it into a {null: value} map if it's not an object. + * New usage of this function is best to explicitly declare the intended array mode. + * This version is provided for backward compatibility. + */ + public static Map flatten(final JsonNode node) { + return flatten(node, false); + } + /** * Prepend all keys in subMap with prefix, then merge that map into originalMap. *

diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java index b7550f91e9aa..e8628d9f62d4 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java @@ -291,6 +291,18 @@ void testFlatten__withArraysNoApplyFlatten() { }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); assertEquals(expected, Jsons.flatten(json, false)); } + + @Test + void testFlatten__checkBackwardCompatiblity() { + final JsonNode json = Jsons + .deserialize("{ \"abc\": [{ \"def\": \"ghi\" }, { \"fed\": \"ihg\" }], \"jkl\": true, \"pqr\": 1 }"); + Map expected = Stream.of(new Object[][] { + { "abc", "[{\"def\":\"ghi\"},{\"fed\":\"ihg\"}]" }, + { "jkl", true }, + { "pqr", 1 }, + }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); + assertEquals(expected, Jsons.flatten(json)); + } @Test void testFlatten__withArraysApplyFlatten() { From e81a5f1d0df2480b2e761077c83003e247f482be Mon Sep 17 00:00:00 2001 From: Adam Bloom Date: Tue, 27 Sep 2022 12:39:44 -0600 Subject: [PATCH 06/10] bump dest-redshift version and add changelog --- airbyte-integrations/connectors/destination-redshift/Dockerfile | 2 +- docs/integrations/destinations/redshift.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index adbab7cd3ef6..cc49956af5b6 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.50 +LABEL io.airbyte.version=0.3.51 LABEL io.airbyte.name=airbyte/destination-redshift diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index c7de6d268eb4..def68a1967b0 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -141,6 +141,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.51 | 2022-09-27 | [\#17273](https://github.com/airbytehq/airbyte/pull/17273) | Fixed handling of arrays in SUPER maximum size check | | 0.3.50 | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage | | 0.3.49 | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) | | 0.3.48 | 2022-09-01 | | Added JDBC URL params | From 682c55e6c277d471bb53a7998fd1fa6f294899ae Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Fri, 14 Oct 2022 14:47:07 -0300 Subject: [PATCH 07/10] format --- .../java/io/airbyte/commons/json/Jsons.java | 6 +-- .../io/airbyte/commons/json/JsonsTest.java | 48 +++++++++---------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java index 77ac363d90b0..9fcd2ac61cb7 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @@ -275,9 +275,9 @@ public static Map flatten(final JsonNode node, final Boolean app } /** - * Flattens an ObjectNode, or dumps it into a {null: value} map if it's not an object. - * New usage of this function is best to explicitly declare the intended array mode. - * This version is provided for backward compatibility. + * Flattens an ObjectNode, or dumps it into a {null: value} map if it's not an object. New usage of + * this function is best to explicitly declare the intended array mode. This version is provided for + * backward compatibility. */ public static Map flatten(final JsonNode node) { return flatten(node, false); diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java index e8628d9f62d4..cd67fd0190e2 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java @@ -162,8 +162,7 @@ void testToObject() { assertEquals( Lists.newArrayList(expected), - Jsons.object(Jsons.jsonNode(Lists.newArrayList(expected)), new TypeReference>() { - })); + Jsons.object(Jsons.jsonNode(Lists.newArrayList(expected)), new TypeReference>() {})); assertEquals( new ToClass(), @@ -179,8 +178,7 @@ void testTryToObject() { assertEquals( Optional.of(expected), - Jsons.tryObject(Jsons.deserialize(SERIALIZED_JSON), new TypeReference() { - })); + Jsons.tryObject(Jsons.deserialize(SERIALIZED_JSON), new TypeReference() {})); final ToClass emptyExpected = new ToClass(); assertEquals( @@ -189,8 +187,7 @@ void testTryToObject() { assertEquals( Optional.of(emptyExpected), - Jsons.tryObject(Jsons.deserialize("{\"str1\":\"abc\"}"), new TypeReference() { - })); + Jsons.tryObject(Jsons.deserialize("{\"str1\":\"abc\"}"), new TypeReference() {})); } @@ -273,9 +270,9 @@ void testGetEstimatedByteSize() { void testFlatten__noArrays() { final JsonNode json = Jsons.deserialize("{ \"abc\": { \"def\": \"ghi\" }, \"jkl\": true, \"pqr\": 1 }"); Map expected = Stream.of(new Object[][] { - { "abc.def", "ghi" }, - { "jkl", true }, - { "pqr", 1 }, + {"abc.def", "ghi"}, + {"jkl", true}, + {"pqr", 1}, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); assertEquals(expected, Jsons.flatten(json, false)); } @@ -285,9 +282,9 @@ void testFlatten__withArraysNoApplyFlatten() { final JsonNode json = Jsons .deserialize("{ \"abc\": [{ \"def\": \"ghi\" }, { \"fed\": \"ihg\" }], \"jkl\": true, \"pqr\": 1 }"); Map expected = Stream.of(new Object[][] { - { "abc", "[{\"def\":\"ghi\"},{\"fed\":\"ihg\"}]" }, - { "jkl", true }, - { "pqr", 1 }, + {"abc", "[{\"def\":\"ghi\"},{\"fed\":\"ihg\"}]"}, + {"jkl", true}, + {"pqr", 1}, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); assertEquals(expected, Jsons.flatten(json, false)); } @@ -297,22 +294,22 @@ void testFlatten__checkBackwardCompatiblity() { final JsonNode json = Jsons .deserialize("{ \"abc\": [{ \"def\": \"ghi\" }, { \"fed\": \"ihg\" }], \"jkl\": true, \"pqr\": 1 }"); Map expected = Stream.of(new Object[][] { - { "abc", "[{\"def\":\"ghi\"},{\"fed\":\"ihg\"}]" }, - { "jkl", true }, - { "pqr", 1 }, + {"abc", "[{\"def\":\"ghi\"},{\"fed\":\"ihg\"}]"}, + {"jkl", true}, + {"pqr", 1}, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); assertEquals(expected, Jsons.flatten(json)); } - + @Test void testFlatten__withArraysApplyFlatten() { final JsonNode json = Jsons .deserialize("{ \"abc\": [{ \"def\": \"ghi\" }, { \"fed\": \"ihg\" }], \"jkl\": true, \"pqr\": 1 }"); Map expected = Stream.of(new Object[][] { - { "abc.[0].def", "ghi" }, - { "abc.[1].fed", "ihg" }, - { "jkl", true }, - { "pqr", 1 }, + {"abc.[0].def", "ghi"}, + {"abc.[1].fed", "ihg"}, + {"jkl", true}, + {"pqr", 1}, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); assertEquals(expected, Jsons.flatten(json, true)); } @@ -323,10 +320,10 @@ void testFlatten__withArraysApplyFlattenNested() { .deserialize( "{ \"abc\": [{ \"def\": {\"ghi\": [\"xyz\"] }}, { \"fed\": \"ihg\" }], \"jkl\": true, \"pqr\": 1 }"); Map expected = Stream.of(new Object[][] { - { "abc.[0].def.ghi.[0]", "xyz" }, - { "abc.[1].fed", "ihg" }, - { "jkl", true }, - { "pqr", 1 }, + {"abc.[0].def.ghi.[0]", "xyz"}, + {"abc.[1].fed", "ihg"}, + {"jkl", true}, + {"pqr", 1}, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); assertEquals(expected, Jsons.flatten(json, true)); } @@ -342,8 +339,7 @@ private static class ToClass { @JsonProperty("numLong") long numLong; - public ToClass() { - } + public ToClass() {} public ToClass(final String str, final Integer num, final long numLong) { this.str = str; From ddfe66b7a66f2983c634a65531c14ab2da75118c Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 8 Dec 2022 17:31:22 -0300 Subject: [PATCH 08/10] bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- airbyte-integrations/connectors/destination-redshift/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index e081e623f076..9f770397dbcf 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -271,7 +271,7 @@ - name: Redshift destinationDefinitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc dockerRepository: airbyte/destination-redshift - dockerImageTag: 0.3.51 + dockerImageTag: 0.3.52 documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift icon: redshift.svg normalizationRepository: airbyte/normalization-redshift diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index cc49956af5b6..24a17530da92 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.51 +LABEL io.airbyte.version=0.3.52 LABEL io.airbyte.name=airbyte/destination-redshift From a23d9bc42b1db1e2807066e59736c5187a80b8f0 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 8 Dec 2022 17:32:45 -0300 Subject: [PATCH 09/10] fix pmd test --- .../io/airbyte/commons/json/JsonsTest.java | 49 ++++++++++--------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java index cd67fd0190e2..3d1d50e6f21d 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java @@ -34,6 +34,11 @@ class JsonsTest { private static final String SERIALIZED_JSON2 = "{\"str\":\"abc\"}"; private static final String ABC = "abc"; private static final String DEF = "def"; + private static final String GHI = "ghi"; + private static final String JKL = "jkl"; + private static final String MNO = "mno"; + private static final String PQR = "pqr"; + private static final String STU = "stu"; private static final String TEST = "test"; private static final String TEST2 = "test2"; private static final String XYZ = "xyz"; @@ -238,24 +243,24 @@ void testGetOptional() { final JsonNode json = Jsons .deserialize("{ \"abc\": { \"def\": \"ghi\" }, \"jkl\": {}, \"mno\": \"pqr\", \"stu\": null }"); - assertEquals(Optional.of(Jsons.jsonNode("ghi")), Jsons.getOptional(json, "abc", "def")); - assertEquals(Optional.of(Jsons.emptyObject()), Jsons.getOptional(json, "jkl")); - assertEquals(Optional.of(Jsons.jsonNode("pqr")), Jsons.getOptional(json, "mno")); - assertEquals(Optional.of(Jsons.jsonNode(null)), Jsons.getOptional(json, "stu")); + assertEquals(Optional.of(Jsons.jsonNode(GHI)), Jsons.getOptional(json, ABC, DEF)); + assertEquals(Optional.of(Jsons.emptyObject()), Jsons.getOptional(json, JKL)); + assertEquals(Optional.of(Jsons.jsonNode(PQR)), Jsons.getOptional(json, MNO)); + assertEquals(Optional.of(Jsons.jsonNode(null)), Jsons.getOptional(json, STU)); assertEquals(Optional.empty(), Jsons.getOptional(json, XYZ)); assertEquals(Optional.empty(), Jsons.getOptional(json, ABC, XYZ)); assertEquals(Optional.empty(), Jsons.getOptional(json, ABC, DEF, XYZ)); - assertEquals(Optional.empty(), Jsons.getOptional(json, ABC, "jkl", XYZ)); - assertEquals(Optional.empty(), Jsons.getOptional(json, "stu", XYZ)); + assertEquals(Optional.empty(), Jsons.getOptional(json, ABC, JKL, XYZ)); + assertEquals(Optional.empty(), Jsons.getOptional(json, STU, XYZ)); } @Test void testGetStringOrNull() { final JsonNode json = Jsons.deserialize("{ \"abc\": { \"def\": \"ghi\" }, \"jkl\": \"mno\", \"pqr\": 1 }"); - assertEquals("ghi", Jsons.getStringOrNull(json, ABC, DEF)); - assertEquals("mno", Jsons.getStringOrNull(json, "jkl")); - assertEquals("1", Jsons.getStringOrNull(json, "pqr")); + assertEquals(GHI, Jsons.getStringOrNull(json, ABC, DEF)); + assertEquals(MNO, Jsons.getStringOrNull(json, JKL)); + assertEquals("1", Jsons.getStringOrNull(json, PQR)); assertNull(Jsons.getStringOrNull(json, ABC, DEF, XYZ)); assertNull(Jsons.getStringOrNull(json, XYZ)); } @@ -270,9 +275,9 @@ void testGetEstimatedByteSize() { void testFlatten__noArrays() { final JsonNode json = Jsons.deserialize("{ \"abc\": { \"def\": \"ghi\" }, \"jkl\": true, \"pqr\": 1 }"); Map expected = Stream.of(new Object[][] { - {"abc.def", "ghi"}, - {"jkl", true}, - {"pqr", 1}, + {"abc.def", GHI}, + {JKL, true}, + {PQR, 1}, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); assertEquals(expected, Jsons.flatten(json, false)); } @@ -282,9 +287,9 @@ void testFlatten__withArraysNoApplyFlatten() { final JsonNode json = Jsons .deserialize("{ \"abc\": [{ \"def\": \"ghi\" }, { \"fed\": \"ihg\" }], \"jkl\": true, \"pqr\": 1 }"); Map expected = Stream.of(new Object[][] { - {"abc", "[{\"def\":\"ghi\"},{\"fed\":\"ihg\"}]"}, - {"jkl", true}, - {"pqr", 1}, + {ABC, "[{\"def\":\"ghi\"},{\"fed\":\"ihg\"}]"}, + {JKL, true}, + {PQR, 1}, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); assertEquals(expected, Jsons.flatten(json, false)); } @@ -294,9 +299,9 @@ void testFlatten__checkBackwardCompatiblity() { final JsonNode json = Jsons .deserialize("{ \"abc\": [{ \"def\": \"ghi\" }, { \"fed\": \"ihg\" }], \"jkl\": true, \"pqr\": 1 }"); Map expected = Stream.of(new Object[][] { - {"abc", "[{\"def\":\"ghi\"},{\"fed\":\"ihg\"}]"}, - {"jkl", true}, - {"pqr", 1}, + {ABC, "[{\"def\":\"ghi\"},{\"fed\":\"ihg\"}]"}, + {JKL, true}, + {PQR, 1}, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); assertEquals(expected, Jsons.flatten(json)); } @@ -308,8 +313,8 @@ void testFlatten__withArraysApplyFlatten() { Map expected = Stream.of(new Object[][] { {"abc.[0].def", "ghi"}, {"abc.[1].fed", "ihg"}, - {"jkl", true}, - {"pqr", 1}, + {JKL, true}, + {PQR, 1}, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); assertEquals(expected, Jsons.flatten(json, true)); } @@ -322,8 +327,8 @@ void testFlatten__withArraysApplyFlattenNested() { Map expected = Stream.of(new Object[][] { {"abc.[0].def.ghi.[0]", "xyz"}, {"abc.[1].fed", "ihg"}, - {"jkl", true}, - {"pqr", 1}, + {JKL, true}, + {PQR, 1}, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); assertEquals(expected, Jsons.flatten(json, true)); } From 0e55dd516f7fb7657ab23150d527eefd92fd358f Mon Sep 17 00:00:00 2001 From: Adam Bloom Date: Thu, 29 Dec 2022 07:38:39 -0700 Subject: [PATCH 10/10] use original method signature --- .../java/io/airbyte/persistence/job/tracker/JobTracker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/tracker/JobTracker.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/tracker/JobTracker.java index d851b7ac302e..86c0b154891c 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/tracker/JobTracker.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/tracker/JobTracker.java @@ -246,7 +246,7 @@ private static Map configToMetadata(final JsonNode config, final // * Otherwise, do some basic conversions to value-ish data. // It would be a weird thing to declare const: null, but in that case we don't want to report null // anyway, so explicitly use hasNonNull. - return Jsons.flatten(config, false); + return Jsons.flatten(config); } else if (schema.has("oneOf")) { // If this schema is a oneOf, then find the first sub-schema which the config matches // and use that sub-schema to convert the config to a map