diff --git a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java index fe787abfd617..ae7f5e91493c 100644 --- a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java +++ b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java @@ -10,7 +10,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import io.airbyte.commons.json.JsonSchemas; -import io.airbyte.commons.json.JsonSchemas.FieldNameOrList; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; import io.airbyte.commons.util.MoreLists; @@ -33,6 +32,8 @@ */ public class CatalogHelpers { + private static final String ITEMS_KEY = "items"; + public static AirbyteCatalog createAirbyteCatalog(final String streamName, final Field... fields) { return new AirbyteCatalog().withStreams(Lists.newArrayList(createAirbyteStream(streamName, fields))); } @@ -222,7 +223,7 @@ protected static List, JsonNode>> getFullyQualifiedFieldNamesW final Set> fieldNamesThatAreOneOfs = new HashSet<>(); return JsonSchemas.traverseJsonSchemaWithCollector(jsonSchema, (node, basicPath) -> { - final List fieldName = basicPath.stream().filter(fieldOrList -> !fieldOrList.isList()).map(FieldNameOrList::getFieldName).toList(); + final List fieldName = basicPath.stream().map(fieldOrList -> fieldOrList.isList() ? ITEMS_KEY : fieldOrList.getFieldName()).toList(); return Pair.of(fieldName, node); }) .stream() diff --git a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java index ae3cc50d8738..a77de87a0552 100644 --- a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java +++ b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java @@ -82,7 +82,7 @@ void testGetFieldNames() throws IOException { final JsonNode node = Jsons.deserialize(MoreResources.readResource("valid_schema.json")); final Set actualFieldNames = CatalogHelpers.getAllFieldNames(node); final List expectedFieldNames = - List.of("CAD", "DKK", "HKD", "HUF", "ISK", "PHP", "date", "nestedkey", "somekey", "something", "something2", "文"); + List.of("CAD", "DKK", "HKD", "HUF", "ISK", "PHP", "date", "nestedkey", "somekey", "something", "something2", "文", "someArray", "items", "oldName"); // sort so that the diff is easier to read. assertEquals(expectedFieldNames.stream().sorted().toList(), actualFieldNames.stream().sorted().toList()); @@ -109,7 +109,17 @@ void testGetCatalogDiff() throws IOException { FieldTransform.createRemoveFieldTransform(List.of("HKD"), schema1.get("properties").get("HKD")), FieldTransform.createUpdateFieldTransform(List.of("CAD"), new UpdateFieldSchemaTransform( schema1.get("properties").get("CAD"), - schema2.get("properties").get("CAD"))))))) + schema2.get("properties").get("CAD"))), + FieldTransform.createUpdateFieldTransform(List.of("someArray"), new UpdateFieldSchemaTransform( + schema1.get("properties").get("someArray"), + schema2.get("properties").get("someArray"))), + FieldTransform.createUpdateFieldTransform(List.of("someArray", "items"), new UpdateFieldSchemaTransform( + schema1.get("properties").get("someArray").get("items"), + schema2.get("properties").get("someArray").get("items"))), + FieldTransform.createRemoveFieldTransform(List.of("someArray", "items", "oldName"), + schema1.get("properties").get("someArray").get("items").get("properties").get("oldName")), + FieldTransform.createAddFieldTransform(List.of("someArray", "items", "newName"), + schema2.get("properties").get("someArray").get("items").get("properties").get("newName")))))) .sorted(STREAM_TRANSFORM_COMPARATOR) .toList(); assertEquals(expectedDiff, actualDiff.stream().sorted(STREAM_TRANSFORM_COMPARATOR).toList()); diff --git a/airbyte-protocol/protocol-models/src/test/resources/valid_schema.json b/airbyte-protocol/protocol-models/src/test/resources/valid_schema.json index a5b7b656f3e2..951ba79fa3ca 100644 --- a/airbyte-protocol/protocol-models/src/test/resources/valid_schema.json +++ b/airbyte-protocol/protocol-models/src/test/resources/valid_schema.json @@ -44,6 +44,27 @@ } } ] + }, + "someArray": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "object", + "null" + ], + "properties": { + "oldName": { + "type": [ + "string", + "null" + ], + "maxLength": 100 + } + } + } } } } diff --git a/airbyte-protocol/protocol-models/src/test/resources/valid_schema2.json b/airbyte-protocol/protocol-models/src/test/resources/valid_schema2.json index f84e8458be7c..7cabd6f71d50 100644 --- a/airbyte-protocol/protocol-models/src/test/resources/valid_schema2.json +++ b/airbyte-protocol/protocol-models/src/test/resources/valid_schema2.json @@ -24,6 +24,27 @@ "patternProperties": { ".+": {} } + }, + "someArray": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "object", + "null" + ], + "properties": { + "newName": { + "type": [ + "string", + "null" + ], + "maxLength": 100 + } + } + } } } }