Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Local_json, Maridb, Meilisearch, Mongodb, Mqtt destinations : Enable DAT tests #12073

Merged
merged 21 commits into from
Apr 19, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
44a6d81
add Boolean, Number, DateTimeWithTZ compare methods
DoNotPanicUA Mar 11, 2022
656148a
Merge remote-tracking branch 'origin/master' into aleonets/add-dat-te…
DoNotPanicUA Mar 22, 2022
46c3a1b
Improve value comparison
DoNotPanicUA Mar 30, 2022
2a4b384
Merge remote-tracking branch 'origin/master' into aleonets/add-dat-te…
DoNotPanicUA Mar 30, 2022
cd53764
Compare inherit objects element by element
DoNotPanicUA Apr 6, 2022
4cd50de
Merge remote-tracking branch 'origin/aleonets/add-dat-tests-ph1' into…
DoNotPanicUA Apr 6, 2022
3c911c9
Move comparison methods to a new class not to overload the test class.
DoNotPanicUA Apr 7, 2022
f28b9af
format
DoNotPanicUA Apr 8, 2022
af1cff8
Merge remote-tracking branch 'origin/master' into aleonets/add-dat-te…
DoNotPanicUA Apr 10, 2022
06764b2
Merge remote-tracking branch 'origin/master' into aleonets/add-dat-te…
DoNotPanicUA Apr 11, 2022
4ddee6c
Move common method to ComparatorUtils. + Review update
DoNotPanicUA Apr 12, 2022
127b4ba
format
DoNotPanicUA Apr 12, 2022
ed890d3
review
DoNotPanicUA Apr 13, 2022
4007837
Merge remote-tracking branch 'origin/master' into aleonets/add-dat-te…
DoNotPanicUA Apr 13, 2022
f1e860b
review
DoNotPanicUA Apr 13, 2022
5b59eb1
mark resolveIdentifier method as deprecated
DoNotPanicUA Apr 13, 2022
c2106a4
remove size objects validation. We iterate expected elements and comp…
DoNotPanicUA Apr 13, 2022
28625d6
Merge remote-tracking branch 'origin/master' into aleonets/add-dat-te…
DoNotPanicUA Apr 14, 2022
bf9b84c
Merge remote-tracking branch 'origin/master' into aleonets/add-dat-te…
DoNotPanicUA Apr 15, 2022
cb2bac6
enable DAT tests for local_json, maridb, meilisearch, mongodb, mqtt
DoNotPanicUA Apr 15, 2022
b5c6b10
Merge remote-tracking branch 'origin/master' into aleonets/dat-LM
DoNotPanicUA Apr 19, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
import io.airbyte.integrations.base.JavaBaseConstants;
import io.airbyte.integrations.destination.StandardNameTransformer;
import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator;
import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
Expand Down Expand Up @@ -49,6 +52,26 @@ protected JsonNode getFailCheckConfig() {
@Override
public void testCheckConnectionInvalidCredentials() {}

@Override
protected TestDataComparator getTestDataComparator() {
return new AdvancedTestDataComparator();
}

@Override
protected boolean supportBasicDataTypeTest() {
return true;
}

@Override
protected boolean supportArrayDataTypeTest() {
return true;
}

@Override
protected boolean supportObjectDataTypeTest() {
return true;
}

@Override
protected List<JsonNode> retrieveRecords(final TestDestinationEnv testEnv,
final String streamName,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package io.airbyte.integrations.destination.mariadb_columnstore;

import io.airbyte.integrations.destination.ExtendedNameTransformer;
import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator;

import java.util.ArrayList;
import java.util.List;

public class MariaDbTestDataComparator extends AdvancedTestDataComparator {

private final ExtendedNameTransformer namingResolver = new MariadbColumnstoreNameTransformer();

@Override
protected List<String> resolveIdentifier(final String identifier) {
final List<String> result = new ArrayList<>();
final String resolved = namingResolver.getIdentifier(identifier);
result.add(identifier);
result.add(resolved);

return result;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;

import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator;
import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.MariaDBContainer;
Expand Down Expand Up @@ -68,6 +71,26 @@ protected String getDefaultSchema(final JsonNode config) {
return config.get("database").asText();
}

@Override
protected TestDataComparator getTestDataComparator() {
return new MariaDbTestDataComparator();
}

@Override
protected boolean supportBasicDataTypeTest() {
return true;
}

@Override
protected boolean supportArrayDataTypeTest() {
return true;
}

@Override
protected boolean supportObjectDataTypeTest() {
return true;
}

@Override
protected List<JsonNode> retrieveRecords(TestDestinationEnv testEnv,
String streamName,
Expand Down Expand Up @@ -98,16 +121,6 @@ private static JdbcDatabase getDatabase(final JsonNode config) {
MariadbColumnstoreDestination.DRIVER_CLASS);
}

@Override
protected List<String> resolveIdentifier(final String identifier) {
final List<String> result = new ArrayList<>();
final String resolved = namingResolver.getIdentifier(identifier);
result.add(identifier);
result.add(resolved);

return result;
}

@Override
protected void setup(TestDestinationEnv testEnv) throws Exception {
DockerImageName mcsImage = DockerImageName.parse("fengdi/columnstore:1.5.2").asCompatibleSubstituteFor("mariadb");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;

import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator;
import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.utility.DockerImageName;

Expand Down Expand Up @@ -73,6 +76,26 @@ protected JsonNode getFailCheckConfig() {
return invalidConfig;
}

@Override
protected TestDataComparator getTestDataComparator() {
return new AdvancedTestDataComparator();
}

@Override
protected boolean supportBasicDataTypeTest() {
return true;
}

@Override
protected boolean supportArrayDataTypeTest() {
return true;
}

@Override
protected boolean supportObjectDataTypeTest() {
return true;
}

@Override
protected List<JsonNode> retrieveRecords(final TestDestinationEnv env,
final String streamName,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@
import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
import java.util.ArrayList;
import java.util.List;

import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator;
import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
import org.bson.Document;
import org.testcontainers.containers.MongoDBContainer;

Expand Down Expand Up @@ -60,6 +63,26 @@ protected JsonNode getFailCheckConfig() {
.build());
}

@Override
protected TestDataComparator getTestDataComparator() {
return new AdvancedTestDataComparator();
}

@Override
protected boolean supportBasicDataTypeTest() {
return true;
}

@Override
protected boolean supportArrayDataTypeTest() {
return true;
}

@Override
protected boolean supportObjectDataTypeTest() {
return true;
}

@Override
protected List<JsonNode> retrieveRecords(final TestDestinationEnv testEnv,
final String streamName,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
import java.util.List;
import java.util.Map;
import java.util.UUID;

import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator;
import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
import org.eclipse.paho.client.mqttv3.MqttClient;
import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
import org.eclipse.paho.client.mqttv3.MqttException;
Expand Down Expand Up @@ -77,6 +80,26 @@ protected JsonNode getFailCheckConfig() {
.build());
}

@Override
protected TestDataComparator getTestDataComparator() {
return new AdvancedTestDataComparator();
}

@Override
protected boolean supportBasicDataTypeTest() {
return true;
}

@Override
protected boolean supportArrayDataTypeTest() {
return true;
}

@Override
protected boolean supportObjectDataTypeTest() {
return true;
}

@Override
protected boolean implementsNamespaces() {
return true;
Expand Down