Merge branch 'master' into grubberr/10384-source-github

airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataTypeTestArgumentProvider.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.integrations.standardtest.destination;
+
+import java.util.stream.Stream;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.ArgumentsProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DataTypeTestArgumentProvider implements ArgumentsProvider {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(DataTypeTestArgumentProvider.class);
+
+  public static final CatalogMessageTestConfigWithCompatibility BASIC_TEST =
+      new CatalogMessageTestConfigWithCompatibility("data_type_basic_test_catalog.json", "data_type_basic_test_messages.txt",
+          new TestCompatibility(true, false, false));
+  public static final CatalogMessageTestConfigWithCompatibility ARRAY_TEST =
+      new CatalogMessageTestConfigWithCompatibility("data_type_array_test_catalog.json", "data_type_array_test_messages.txt",
+          new TestCompatibility(true, true, false));
+  public static final CatalogMessageTestConfigWithCompatibility OBJECT_TEST =
+      new CatalogMessageTestConfigWithCompatibility("data_type_object_test_catalog.json", "data_type_object_test_messages.txt",
+          new TestCompatibility(true, false, true));
+  public static final CatalogMessageTestConfigWithCompatibility OBJECT_WITH_ARRAY_TEST =
+      new CatalogMessageTestConfigWithCompatibility("data_type_array_object_test_catalog.json", "data_type_array_object_test_messages.txt",
+          new TestCompatibility(true, true, true));
+
+  @Override
+  public Stream<? extends Arguments> provideArguments(ExtensionContext context) throws Exception {
+    return Stream.of(
+        getArguments(BASIC_TEST),
+        getArguments(ARRAY_TEST),
+        getArguments(OBJECT_TEST),
+        getArguments(OBJECT_WITH_ARRAY_TEST));
+  }
+
+  private Arguments getArguments(CatalogMessageTestConfigWithCompatibility testConfig) {
+    return Arguments.of(testConfig.messageFile, testConfig.catalogFile, testConfig.testCompatibility);
+  }
+
+  public record TestCompatibility(boolean requireBasicCompatibility,
+                                  boolean requireArrayCompatibility,
+                                  boolean requireObjectCompatibility) {
+
+    public boolean isTestCompatible(boolean supportBasicDataTypeTest, boolean supportArrayDataTypeTest, boolean supportObjectDataTypeTest) {
+      LOGGER.info("---- Data type test compatibility ----");
+      LOGGER.info("| Data type test | Require | Support |");
+      LOGGER.info("| Basic test     | {}   | {}   |", (requireBasicCompatibility ? "true " : "false"),
+          (supportBasicDataTypeTest ? "true " : "false"));
+      LOGGER.info("| Array test     | {}   | {}   |", (requireArrayCompatibility ? "true " : "false"),
+          (supportArrayDataTypeTest ? "true " : "false"));
+      LOGGER.info("| Object test    | {}   | {}   |", (requireObjectCompatibility ? "true " : "false"),
+          (supportObjectDataTypeTest ? "true " : "false"));
+      LOGGER.info("--------------------------------------");
+
+      if (requireBasicCompatibility && !supportBasicDataTypeTest) {
+        LOGGER.warn("The destination doesn't support required Basic data type test. The test is skipped!");
+        return false;
+      }
+      if (requireArrayCompatibility && !supportArrayDataTypeTest) {
+        LOGGER.warn("The destination doesn't support required Array data type test. The test is skipped!");
+        return false;
+      }
+      if (requireObjectCompatibility && !supportObjectDataTypeTest) {
+        LOGGER.warn("The destination doesn't support required Object data type test. The test is skipped!");
+        return false;
+      }
+
+      return true;
+    }
+
+  }
+
+  public static class CatalogMessageTestConfigWithCompatibility extends DataArgumentsProvider.CatalogMessageTestConfigPair {
+
+    final TestCompatibility testCompatibility;
+
+    public CatalogMessageTestConfigWithCompatibility(String catalogFile, String messageFile, TestCompatibility testCompatibility) {
+      super(catalogFile, messageFile);
+      this.testCompatibility = testCompatibility;
+    }
+
+  }
+
+}

airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java

@@ -53,6 +53,7 @@
 import io.airbyte.workers.protocols.airbyte.AirbyteDestination;
 import io.airbyte.workers.protocols.airbyte.DefaultAirbyteDestination;
 import io.airbyte.workers.test_helpers.EntrypointEnvChecker;
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Instant;
@@ -1301,4 +1302,71 @@ public void testStressPerformance() throws Exception {
           + "\n"
           + "Praesent finibus scelerisque elit, accumsan condimentum risus mattis vitae. Donec tristique hendrerit facilisis. Curabitur metus purus, venenatis non elementum id, finibus eu augue. Quisque posuere rhoncus ligula, et vehicula erat pulvinar at. Pellentesque vel quam vel lectus tincidunt congue quis id sapien. Ut efficitur mauris vitae pretium iaculis. Aliquam consectetur iaculis nisi vitae laoreet. Integer vel odio quis diam mattis tempor eget nec est. Donec iaculis facilisis neque, at dictum magna vestibulum ut. Sed malesuada non nunc ac consequat. Maecenas tempus lectus a nisl congue, ac venenatis diam viverra. Nam ac justo id nulla iaculis lobortis in eu ligula. Vivamus et ligula id sapien efficitur aliquet. Curabitur est justo, tempus vitae mollis quis, tincidunt vitae felis. Vestibulum molestie laoreet justo, nec mollis purus vulputate at.";
 
+  protected boolean supportBasicDataTypeTest() {
+    return false;
+  }
+
+  protected boolean supportArrayDataTypeTest() {
+    return false;
+  }
+
+  protected boolean supportObjectDataTypeTest() {
+    return false;
+  }
+
+  private boolean checkTestCompatibility(final DataTypeTestArgumentProvider.TestCompatibility testCompatibility) {
+    return testCompatibility.isTestCompatible(supportBasicDataTypeTest(), supportArrayDataTypeTest(), supportObjectDataTypeTest());
+  }
+
+  @ParameterizedTest
+  @ArgumentsSource(DataTypeTestArgumentProvider.class)
+  public void testDataTypeTestWithNormalization(final String messagesFilename,
+                                                final String catalogFilename,
+                                                final DataTypeTestArgumentProvider.TestCompatibility testCompatibility)
+      throws Exception {
+    if (!checkTestCompatibility(testCompatibility))
+      return;
+
+    final AirbyteCatalog catalog = readCatalogFromFile(catalogFilename);
+    final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog);
+    final List<AirbyteMessage> messages = readMessagesFromFile(messagesFilename);
+
+    if (supportsNormalization()) {
+      LOGGER.info("Normalization is supported! Run test with normalization.");
+      runAndCheckWithNormalization(messages, configuredCatalog, catalog);
+    } else {
+      LOGGER.info("Normalization is not supported! Run test without normalization.");
+      runAndCheckWithoutNormalization(messages, configuredCatalog, catalog);
+    }
+  }
+
+  private AirbyteCatalog readCatalogFromFile(String catalogFilename) throws IOException {
+    return Jsons.deserialize(MoreResources.readResource(catalogFilename), AirbyteCatalog.class);
+  }
+
+  private List<AirbyteMessage> readMessagesFromFile(String messagesFilename) throws IOException {
+    return MoreResources.readResource(messagesFilename).lines()
+        .map(record -> Jsons.deserialize(record, AirbyteMessage.class)).collect(Collectors.toList());
+  }
+
+  private void runAndCheckWithNormalization(final List<AirbyteMessage> messages,
+                                            final ConfiguredAirbyteCatalog configuredCatalog,
+                                            final AirbyteCatalog catalog)
+      throws Exception {
+    final JsonNode config = getConfig();
+    runSyncAndVerifyStateOutput(config, messages, configuredCatalog, true);
+
+    final List<AirbyteRecordMessage> actualMessages = retrieveNormalizedRecords(catalog, getDefaultSchema(config));
+    assertSameMessages(messages, actualMessages, true);
+  }
+
+  private void runAndCheckWithoutNormalization(final List<AirbyteMessage> messages,
+                                               final ConfiguredAirbyteCatalog configuredCatalog,
+                                               final AirbyteCatalog catalog)
+      throws Exception {
+    final JsonNode config = getConfig();
+    runSyncAndVerifyStateOutput(config, messages, configuredCatalog, false);
+    retrieveRawRecordsAndAssertSameMessages(catalog, messages, getDefaultSchema(config));
+  }
+
 }

airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_catalog.json

@@ -0,0 +1,46 @@
+{
+  "streams": [
+    {
+      "name": "object_array_test_1",
+      "json_schema": {
+        "type": ["object"],
+        "properties": {
+          "property_string": {
+            "type": ["string"]
+          },
+          "property_array": {
+            "type": ["array"],
+            "items": {
+              "type": ["object"],
+              "properties": {
+                "property_string": { "type": "string" },
+                "property_date": { "type": "string", "format": "date" },
+                "property_timestamp_with_timezone": {
+                  "type": "string",
+                  "format": "date-time",
+                  "airbyte_type": "timestamp_with_timezone"
+                },
+                "property_timestamp_without_timezone": {
+                  "type": "string",
+                  "format": "date-time",
+                  "airbyte_type": "timestamp_without_timezone"
+                },
+                "property_number": { "type": "number" },
+                "property_big_number": {
+                  "type": "string",
+                  "airbyte_type": "big_number"
+                },
+                "property_integer": { "type": "integer" },
+                "property_big_integer": {
+                  "type": "string",
+                  "airbyte_type": "big_integer"
+                },
+                "property_boolean": { "type": "boolean" }
+              }
+            }
+          }
+        }
+      }
+    }
+  ]
+}

airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_messages.txt

@@ -0,0 +1,2 @@
+{"type": "RECORD", "record": {"stream": "object_array_test_1", "emitted_at": 1602637589100, "data": { "property_string" : "qqq", "property_array" : [ { "property_string": "foo bar", "property_date": "2021-01-23", "property_timestamp_with_timezone": "2022-11-22T01:23:45+00:00", "property_timestamp_without_timezone": "2022-11-22T01:23:45", "property_number": 56.78, "property_big_number": "100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.1234", "property_integer": 42, "property_big_integer": "123141241234124123141241234124123141241234124123141241234124123141241234124", "property_boolean": true } ] }}}
+{"type": "STATE", "state": { "data": {"start_date": "2022-02-14"}}}

airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_catalog.json

@@ -0,0 +1,72 @@
+{
+  "streams": [
+    {
+      "name": "array_test_1",
+      "json_schema": {
+        "properties": {
+          "string_array": {
+            "type": ["array"],
+            "items": {
+              "type": ["string"]
+            }
+          },
+          "array_date": {
+            "type": ["array"],
+            "items": {
+              "type": "string",
+              "format": "date"
+            }
+          },
+          "array_timestamp_with_timezone": {
+            "type": ["array"],
+            "items": {
+              "type": "string",
+              "format": "date-time",
+              "airbyte_type": "timestamp_with_timezone"
+            }
+          },
+          "array_timestamp_without_timezone": {
+            "type": ["array"],
+            "items": {
+              "type": "string",
+              "format": "date-time",
+              "airbyte_type": "timestamp_without_timezone"
+            }
+          },
+          "array_number": {
+            "type": ["array"],
+            "items": {
+              "type": "number"
+            }
+          },
+          "array_big_number": {
+            "type": ["array"],
+            "items": {
+              "type": "string",
+              "airbyte_type": "big_number"
+            }
+          },
+          "array_integer": {
+            "type": ["array"],
+            "items": {
+              "type": "integer"
+            }
+          },
+          "array_big_integer": {
+            "type": ["array"],
+            "items": {
+              "type": "string",
+              "airbyte_type": "big_integer"
+            }
+          },
+          "array_boolean": {
+            "type": ["array"],
+            "items": {
+              "type": "boolean"
+            }
+          }
+        }
+      }
+    }
+  ]
+}

airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_messages.txt

@@ -0,0 +1,2 @@
+{"type": "RECORD", "record": {"stream": "array_test_1", "emitted_at": 1602637589100, "data": { "string_array" : ["foo bar", "some random special characters: ࠈൡሗ"], "array_date" : ["2021-01-23", "1504-02-29"], "array_timestamp_with_timezone" : ["2022-11-22T01:23:45+05:00", "9999-12-21T01:23:45-05:00"], "array_timestamp_without_timezone" : ["2022-11-22T01:23:45", "1504-02-29T01:23:45"], "array_number" : [56.78, 0, -12345.678], "array_big_number" : ["-12345.678", "100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.1234"], "array_integer" : [42, 0, 12345], "array_big_integer" : ["0", "123141241234124123141241234124123141241234124123141241234124123141241234124"], "array_boolean" : [true, false] }}}
+{"type": "STATE", "state": { "data": {"start_date": "2022-02-14"}}}

airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_catalog.json

@@ -0,0 +1,101 @@
+{
+  "streams": [
+    {
+      "name": "string_test_1",
+      "json_schema": {
+        "properties": {
+          "data": {
+            "type": "string"
+          }
+        }
+      }
+    },
+    {
+      "name": "date_test_1",
+      "json_schema": {
+        "properties": {
+          "data": {
+            "type": "string",
+            "format": "date"
+          }
+        }
+      }
+    },
+    {
+      "name": "datetime_test_1",
+      "json_schema": {
+        "properties": {
+          "data": {
+            "type": "string",
+            "format": "date-time",
+            "airbyte_type": "timestamp_with_timezone"
+          }
+        }
+      }
+    },
+    {
+      "name": "datetime_test_2",
+      "json_schema": {
+        "properties": {
+          "data": {
+            "type": "string",
+            "format": "date-time",
+            "airbyte_type": "timestamp_without_timezone"
+          }
+        }
+      }
+    },
+    {
+      "name": "number_test_1",
+      "json_schema": {
+        "properties": {
+          "data": {
+            "type": "number"
+          }
+        }
+      }
+    },
+    {
+      "name": "bignumber_test_1",
+      "json_schema": {
+        "properties": {
+          "data": {
+            "type": "string",
+            "airbyte_type": "big_number"
+          }
+        }
+      }
+    },
+    {
+      "name": "integer_test_1",
+      "json_schema": {
+        "properties": {
+          "data": {
+            "type": "integer"
+          }
+        }
+      }
+    },
+    {
+      "name": "big_integer_test_1",
+      "json_schema": {
+        "properties": {
+          "data": {
+            "type": "string",
+            "airbyte_type": "big_integer"
+          }
+        }
+      }
+    },
+    {
+      "name": "boolean_test_1",
+      "json_schema": {
+        "properties": {
+          "data": {
+            "type": "boolean"
+          }
+        }
+      }
+    }
+  ]
+}