[#14909] Fix for duplicate key isue when diffing catalogs with arrays (…

…#14922)
airbytehq · Jul 22, 2022 · e838cd6 · e838cd6
1 parent c3b3be1
commit e838cd6
Show file tree

Hide file tree

Showing 4 changed files with 57 additions and 4 deletions.
diff --git a/...yte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java b/...yte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java
@@ -10,7 +10,6 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import io.airbyte.commons.json.JsonSchemas;
-import io.airbyte.commons.json.JsonSchemas.FieldNameOrList;
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.commons.util.MoreIterators;
 import io.airbyte.commons.util.MoreLists;
@@ -33,6 +32,8 @@
  */
 public class CatalogHelpers {
 
+  private static final String ITEMS_KEY = "items";
+
   public static AirbyteCatalog createAirbyteCatalog(final String streamName, final Field... fields) {
     return new AirbyteCatalog().withStreams(Lists.newArrayList(createAirbyteStream(streamName, fields)));
   }
@@ -222,7 +223,7 @@ protected static List<Pair<List<String>, JsonNode>> getFullyQualifiedFieldNamesW
     final Set<List<String>> fieldNamesThatAreOneOfs = new HashSet<>();
 
     return JsonSchemas.traverseJsonSchemaWithCollector(jsonSchema, (node, basicPath) -> {
-      final List<String> fieldName = basicPath.stream().filter(fieldOrList -> !fieldOrList.isList()).map(FieldNameOrList::getFieldName).toList();
+      final List<String> fieldName = basicPath.stream().map(fieldOrList -> fieldOrList.isList() ? ITEMS_KEY : fieldOrList.getFieldName()).toList();
       return Pair.of(fieldName, node);
     })
         .stream()

diff --git a/...protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/...protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java
@@ -82,7 +82,7 @@ void testGetFieldNames() throws IOException {
     final JsonNode node = Jsons.deserialize(MoreResources.readResource("valid_schema.json"));
     final Set<String> actualFieldNames = CatalogHelpers.getAllFieldNames(node);
     final List<String> expectedFieldNames =
-        List.of("CAD", "DKK", "HKD", "HUF", "ISK", "PHP", "date", "nestedkey", "somekey", "something", "something2", "文");
+        List.of("CAD", "DKK", "HKD", "HUF", "ISK", "PHP", "date", "nestedkey", "somekey", "something", "something2", "文", "someArray", "items", "oldName");
 
     // sort so that the diff is easier to read.
     assertEquals(expectedFieldNames.stream().sorted().toList(), actualFieldNames.stream().sorted().toList());
@@ -109,7 +109,17 @@ void testGetCatalogDiff() throws IOException {
             FieldTransform.createRemoveFieldTransform(List.of("HKD"), schema1.get("properties").get("HKD")),
             FieldTransform.createUpdateFieldTransform(List.of("CAD"), new UpdateFieldSchemaTransform(
                 schema1.get("properties").get("CAD"),
-                schema2.get("properties").get("CAD")))))))
+                schema2.get("properties").get("CAD"))),
+            FieldTransform.createUpdateFieldTransform(List.of("someArray"), new UpdateFieldSchemaTransform(
+                schema1.get("properties").get("someArray"),
+                schema2.get("properties").get("someArray"))),
+            FieldTransform.createUpdateFieldTransform(List.of("someArray", "items"), new UpdateFieldSchemaTransform(
+                schema1.get("properties").get("someArray").get("items"),
+                schema2.get("properties").get("someArray").get("items"))),
+            FieldTransform.createRemoveFieldTransform(List.of("someArray", "items", "oldName"),
+                schema1.get("properties").get("someArray").get("items").get("properties").get("oldName")),
+            FieldTransform.createAddFieldTransform(List.of("someArray", "items", "newName"),
+                schema2.get("properties").get("someArray").get("items").get("properties").get("newName"))))))
         .sorted(STREAM_TRANSFORM_COMPARATOR)
         .toList();
     assertEquals(expectedDiff, actualDiff.stream().sorted(STREAM_TRANSFORM_COMPARATOR).toList());

diff --git a/airbyte-protocol/protocol-models/src/test/resources/valid_schema.json b/airbyte-protocol/protocol-models/src/test/resources/valid_schema.json
@@ -44,6 +44,27 @@
           }
         }
       ]
+    },
+    "someArray": {
+      "type": [
+        "array",
+        "null"
+      ],
+      "items": {
+        "type": [
+          "object",
+          "null"
+        ],
+        "properties": {
+          "oldName": {
+            "type": [
+              "string",
+              "null"
+            ],
+            "maxLength": 100
+          }
+        }
+      }
     }
   }
 }
diff --git a/airbyte-protocol/protocol-models/src/test/resources/valid_schema2.json b/airbyte-protocol/protocol-models/src/test/resources/valid_schema2.json
@@ -24,6 +24,27 @@
       "patternProperties": {
         ".+": {}
       }
+    },
+    "someArray": {
+      "type": [
+        "array",
+        "null"
+      ],
+      "items": {
+        "type": [
+          "object",
+          "null"
+        ],
+        "properties": {
+          "newName": {
+            "type": [
+              "string",
+              "null"
+            ],
+            "maxLength": 100
+          }
+        }
+      }
     }
   }
 }