Local_json, Maridb, Meilisearch, Mongodb, Mqtt destinations : Enable…

… DAT tests (#12073) * add Boolean, Number, DateTimeWithTZ compare methods * Improve value comparison * Compare inherit objects element by element * Move comparison methods to a new class not to overload the test class. Basic - old implementation which provides backward compatibility of this PR. Advanced - comparator required for destinations with enabled DAT tests. * format * Move common method to ComparatorUtils. + Review update * format * review * review * mark resolveIdentifier method as deprecated * remove size objects validation. We iterate expected elements and compare it with actualObject elements. Expected might have empty elements and they are equal missing element in the actual Object. Also, actualObject might contain additional elements which is not an exception. * enable DAT tests for local_json, maridb, meilisearch, mongodb, mqtt
airbytehq · Apr 19, 2022 · c8c4cf5 · c8c4cf5
1 parent d358229
commit c8c4cf5
Show file tree

Hide file tree

Showing 6 changed files with 134 additions and 10 deletions.
diff --git a/...va/io/airbyte/integrations/destination/local_json/LocalJsonDestinationAcceptanceTest.java b/...va/io/airbyte/integrations/destination/local_json/LocalJsonDestinationAcceptanceTest.java
@@ -12,6 +12,9 @@
 import io.airbyte.integrations.base.JavaBaseConstants;
 import io.airbyte.integrations.destination.StandardNameTransformer;
 import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
+import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator;
+import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
+
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
@@ -49,6 +52,26 @@ protected JsonNode getFailCheckConfig() {
   @Override
   public void testCheckConnectionInvalidCredentials() {}
 
+  @Override
+  protected TestDataComparator getTestDataComparator() {
+    return new AdvancedTestDataComparator();
+  }
+
+  @Override
+  protected boolean supportBasicDataTypeTest() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportArrayDataTypeTest() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportObjectDataTypeTest() {
+    return true;
+  }
+
   @Override
   protected List<JsonNode> retrieveRecords(final TestDestinationEnv testEnv,
                                            final String streamName,

diff --git a/...va/io/airbyte/integrations/destination/mariadb_columnstore/MariaDbTestDataComparator.java b/...va/io/airbyte/integrations/destination/mariadb_columnstore/MariaDbTestDataComparator.java
@@ -0,0 +1,22 @@
+package io.airbyte.integrations.destination.mariadb_columnstore;
+
+import io.airbyte.integrations.destination.ExtendedNameTransformer;
+import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class MariaDbTestDataComparator extends AdvancedTestDataComparator {
+
+    private final ExtendedNameTransformer namingResolver = new MariadbColumnstoreNameTransformer();
+
+    @Override
+    protected List<String> resolveIdentifier(final String identifier) {
+        final List<String> result = new ArrayList<>();
+        final String resolved = namingResolver.getIdentifier(identifier);
+        result.add(identifier);
+        result.add(resolved);
+
+        return result;
+    }
+}
diff --git a/...grations/destination/mariadb_columnstore/MariadbColumnstoreDestinationAcceptanceTest.java b/...grations/destination/mariadb_columnstore/MariadbColumnstoreDestinationAcceptanceTest.java
@@ -17,6 +17,9 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
+
+import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator;
+import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testcontainers.containers.MariaDBContainer;
@@ -68,6 +71,26 @@ protected String getDefaultSchema(final JsonNode config) {
     return config.get("database").asText();
   }
 
+  @Override
+  protected TestDataComparator getTestDataComparator() {
+    return new MariaDbTestDataComparator();
+  }
+
+  @Override
+  protected boolean supportBasicDataTypeTest() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportArrayDataTypeTest() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportObjectDataTypeTest() {
+    return true;
+  }
+
   @Override
   protected List<JsonNode> retrieveRecords(TestDestinationEnv testEnv,
                                            String streamName,
@@ -98,16 +121,6 @@ private static JdbcDatabase getDatabase(final JsonNode config) {
         MariadbColumnstoreDestination.DRIVER_CLASS);
   }
 
-  @Override
-  protected List<String> resolveIdentifier(final String identifier) {
-    final List<String> result = new ArrayList<>();
-    final String resolved = namingResolver.getIdentifier(identifier);
-    result.add(identifier);
-    result.add(resolved);
-
-    return result;
-  }
-
   @Override
   protected void setup(TestDestinationEnv testEnv) throws Exception {
     DockerImageName mcsImage = DockerImageName.parse("fengdi/columnstore:1.5.2").asCompatibleSubstituteFor("mariadb");

diff --git a/...io/airbyte/integrations/destination/meilisearch/MeiliSearchDestinationAcceptanceTest.java b/...io/airbyte/integrations/destination/meilisearch/MeiliSearchDestinationAcceptanceTest.java
@@ -20,6 +20,9 @@
 import java.util.Comparator;
 import java.util.List;
 import java.util.stream.Collectors;
+
+import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator;
+import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.utility.DockerImageName;
 
@@ -73,6 +76,26 @@ protected JsonNode getFailCheckConfig() {
     return invalidConfig;
   }
 
+  @Override
+  protected TestDataComparator getTestDataComparator() {
+    return new AdvancedTestDataComparator();
+  }
+
+  @Override
+  protected boolean supportBasicDataTypeTest() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportArrayDataTypeTest() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportObjectDataTypeTest() {
+    return true;
+  }
+
   @Override
   protected List<JsonNode> retrieveRecords(final TestDestinationEnv env,
                                            final String streamName,

diff --git a/...on/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java b/...on/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java
@@ -63,6 +63,26 @@ protected JsonNode getFailCheckConfig() {
         .build());
   }
 
+  @Override
+  protected TestDataComparator getTestDataComparator() {
+    return new AdvancedTestDataComparator();
+  }
+
+  @Override
+  protected boolean supportBasicDataTypeTest() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportArrayDataTypeTest() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportObjectDataTypeTest() {
+    return true;
+  }
+
   @Override
   protected List<JsonNode> retrieveRecords(final TestDestinationEnv testEnv,
                                            final String streamName,

diff --git a/...egration/java/io/airbyte/integrations/destination/mqtt/MqttDestinationAcceptanceTest.java b/...egration/java/io/airbyte/integrations/destination/mqtt/MqttDestinationAcceptanceTest.java
@@ -23,6 +23,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+
+import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator;
+import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
 import org.eclipse.paho.client.mqttv3.MqttClient;
 import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
 import org.eclipse.paho.client.mqttv3.MqttException;
@@ -77,6 +80,26 @@ protected JsonNode getFailCheckConfig() {
         .build());
   }
 
+  @Override
+  protected TestDataComparator getTestDataComparator() {
+    return new AdvancedTestDataComparator();
+  }
+
+  @Override
+  protected boolean supportBasicDataTypeTest() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportArrayDataTypeTest() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportObjectDataTypeTest() {
+    return true;
+  }
+
   @Override
   protected boolean implementsNamespaces() {
     return true;