-
Notifications
You must be signed in to change notification settings - Fork 4.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
S3 destination: added data types tests (#12098)
* add Boolean, Number, DateTimeWithTZ compare methods * Improve value comparison * Compare inherit objects element by element * Move comparison methods to a new class not to overload the test class. Basic - old implementation which provides backward compatibility of this PR. Advanced - comparator required for destinations with enabled DAT tests. * format * Move common method to ComparatorUtils. + Review update * format * review * review * mark resolveIdentifier method as deprecated * remove size objects validation. We iterate expected elements and compare it with actualObject elements. Expected might have empty elements and they are equal missing element in the actual Object. Also, actualObject might contain additional elements which is not an exception. * S3 destination: enable DAT tests for s3 Co-authored-by: andrii.leonets <aleonets@gmail.com>
- Loading branch information
1 parent
1fe7222
commit 78c81af
Showing
5 changed files
with
80 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
36 changes: 36 additions & 0 deletions
36
...egration/java/io/airbyte/integrations/destination/s3/S3AvroParquetTestDataComparator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
package io.airbyte.integrations.destination.s3; | ||
|
||
import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; | ||
import java.time.Instant; | ||
import java.time.LocalDate; | ||
import java.time.LocalDateTime; | ||
import java.time.ZoneOffset; | ||
import java.time.ZonedDateTime; | ||
import java.time.format.DateTimeFormatter; | ||
|
||
public class S3AvroParquetTestDataComparator extends AdvancedTestDataComparator { | ||
|
||
@Override | ||
protected boolean compareDateValues(String airbyteMessageValue, String destinationValue) { | ||
var destinationDate = LocalDate.ofEpochDay(Long.parseLong(destinationValue)); | ||
var expectedDate = LocalDate.parse(airbyteMessageValue, DateTimeFormatter.ofPattern(AIRBYTE_DATE_FORMAT)); | ||
return expectedDate.equals(destinationDate); | ||
} | ||
|
||
private Instant getInstantFromEpoch(String epochValue) { | ||
return Instant.ofEpochMilli(Long.parseLong(epochValue.replaceAll("000$", ""))); | ||
} | ||
|
||
@Override | ||
protected ZonedDateTime parseDestinationDateWithTz(String destinationValue) { | ||
return ZonedDateTime.ofInstant(getInstantFromEpoch(destinationValue), ZoneOffset.UTC); | ||
} | ||
|
||
@Override | ||
protected boolean compareDateTimeValues(String airbyteMessageValue, String destinationValue) { | ||
var format = DateTimeFormatter.ofPattern(AIRBYTE_DATETIME_FORMAT); | ||
LocalDateTime dateTime = LocalDateTime.ofInstant(getInstantFromEpoch(destinationValue), ZoneOffset.UTC); | ||
return super.compareDateTimeValues(airbyteMessageValue, format.format(dateTime)); | ||
} | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters