S3 destination: added data types tests (#12098)

* add Boolean, Number, DateTimeWithTZ compare methods * Improve value comparison * Compare inherit objects element by element * Move comparison methods to a new class not to overload the test class. Basic - old implementation which provides backward compatibility of this PR. Advanced - comparator required for destinations with enabled DAT tests. * format * Move common method to ComparatorUtils. + Review update * format * review * review * mark resolveIdentifier method as deprecated * remove size objects validation. We iterate expected elements and compare it with actualObject elements. Expected might have empty elements and they are equal missing element in the actual Object. Also, actualObject might contain additional elements which is not an exception. * S3 destination: enable DAT tests for s3 Co-authored-by: andrii.leonets <aleonets@gmail.com>
airbytehq · Apr 19, 2022 · 78c81af · 78c81af
1 parent 1fe7222
commit 78c81af
Show file tree

Hide file tree

Showing 5 changed files with 80 additions and 0 deletions.
diff --git a/...egration/java/io/airbyte/integrations/destination/s3/S3AvroDestinationAcceptanceTest.java b/...egration/java/io/airbyte/integrations/destination/s3/S3AvroDestinationAcceptanceTest.java
@@ -12,6 +12,7 @@
 import io.airbyte.integrations.destination.s3.avro.AvroConstants;
 import io.airbyte.integrations.destination.s3.avro.JsonFieldNameUpdater;
 import io.airbyte.integrations.destination.s3.util.AvroRecordHelper;
+import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -67,4 +68,9 @@ protected List<JsonNode> retrieveRecords(final TestDestinationEnv testEnv,
     return jsonRecords;
   }
 
+  @Override
+  protected TestDataComparator getTestDataComparator() {
+    return new S3AvroParquetTestDataComparator();
+  }
+
 }
diff --git a/...egration/java/io/airbyte/integrations/destination/s3/S3AvroParquetTestDataComparator.java b/...egration/java/io/airbyte/integrations/destination/s3/S3AvroParquetTestDataComparator.java
@@ -0,0 +1,36 @@
+package io.airbyte.integrations.destination.s3;
+
+import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+
+public class S3AvroParquetTestDataComparator extends AdvancedTestDataComparator {
+
+  @Override
+  protected boolean compareDateValues(String airbyteMessageValue, String destinationValue) {
+    var destinationDate = LocalDate.ofEpochDay(Long.parseLong(destinationValue));
+    var expectedDate = LocalDate.parse(airbyteMessageValue, DateTimeFormatter.ofPattern(AIRBYTE_DATE_FORMAT));
+    return expectedDate.equals(destinationDate);
+  }
+
+  private Instant getInstantFromEpoch(String epochValue) {
+    return Instant.ofEpochMilli(Long.parseLong(epochValue.replaceAll("000$", "")));
+  }
+
+  @Override
+  protected ZonedDateTime parseDestinationDateWithTz(String destinationValue) {
+    return ZonedDateTime.ofInstant(getInstantFromEpoch(destinationValue), ZoneOffset.UTC);
+  }
+
+  @Override
+  protected boolean compareDateTimeValues(String airbyteMessageValue, String destinationValue) {
+    var format = DateTimeFormatter.ofPattern(AIRBYTE_DATETIME_FORMAT);
+    LocalDateTime dateTime = LocalDateTime.ofInstant(getInstantFromEpoch(destinationValue), ZoneOffset.UTC);
+    return super.compareDateTimeValues(airbyteMessageValue, format.format(dateTime));
+  }
+
+}
diff --git a/...tegration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java b/...tegration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java
@@ -76,12 +76,22 @@ private static JsonNode getJsonNode(final Map<String, String> input, final Map<S
         case "boolean" -> json.put(key, Boolean.valueOf(value));
         case "integer" -> json.put(key, Integer.valueOf(value));
         case "number" -> json.put(key, Double.valueOf(value));
+        case "" -> addNoTypeValue(json, key, value);
         default -> json.put(key, value);
       }
     }
     return json;
   }
 
+  private static void addNoTypeValue(ObjectNode json, String key, String value) {
+    if (value != null && (value.matches("^\\[.*\\]$")) || value.matches("^\\{.*\\}$")) {
+      var newNode = Jsons.deserialize(value);
+      json.set(key, newNode);
+    } else {
+      json.put(key, value);
+    }
+  }
+
   @Override
   protected List<JsonNode> retrieveRecords(final TestDestinationEnv testEnv,
                                            final String streamName,

diff --git a/...-integration/java/io/airbyte/integrations/destination/s3/S3DestinationAcceptanceTest.java b/...-integration/java/io/airbyte/integrations/destination/s3/S3DestinationAcceptanceTest.java
@@ -18,6 +18,8 @@
 import io.airbyte.integrations.destination.NamingConventionTransformer;
 import io.airbyte.integrations.destination.s3.util.S3NameTransformer;
 import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
+import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator;
+import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
 import java.nio.file.Path;
 import java.util.Comparator;
 import java.util.LinkedList;
@@ -165,4 +167,24 @@ protected void tearDown(final TestDestinationEnv testEnv) {
     }
   }
 
+  @Override
+  protected TestDataComparator getTestDataComparator() {
+    return new AdvancedTestDataComparator();
+  }
+
+  @Override
+  protected boolean supportBasicDataTypeTest() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportArrayDataTypeTest() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportObjectDataTypeTest() {
+    return true;
+  }
+
 }
diff --git a/...ation/java/io/airbyte/integrations/destination/s3/S3ParquetDestinationAcceptanceTest.java b/...ation/java/io/airbyte/integrations/destination/s3/S3ParquetDestinationAcceptanceTest.java
@@ -13,6 +13,7 @@
 import io.airbyte.integrations.destination.s3.avro.JsonFieldNameUpdater;
 import io.airbyte.integrations.destination.s3.parquet.S3ParquetWriter;
 import io.airbyte.integrations.destination.s3.util.AvroRecordHelper;
+import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -71,4 +72,9 @@ protected List<JsonNode> retrieveRecords(final TestDestinationEnv testEnv,
     return jsonRecords;
   }
 
+  @Override
+  protected TestDataComparator getTestDataComparator() {
+    return new S3AvroParquetTestDataComparator();
+  }
+
 }