From 92631e1aac0b6443d7ae64bacccd90ea3309bfa1 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Mon, 19 Jul 2021 17:04:12 +0300 Subject: [PATCH 1/3] [4583] Fixed MySQL destination of fails is source has a column with json data --- .../destination/mysql/MySQLSqlOperations.java | 2 +- .../mysql/MySQLDestinationAcceptanceTest.java | 51 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java index 820922273ad8..0803fa5ab990 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java +++ b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java @@ -80,7 +80,7 @@ private void loadDataIntoTable(JdbcDatabase database, String absoluteFile = "'" + tmpFile.getAbsolutePath() + "'"; String query = String.format( - "LOAD DATA LOCAL INFILE %s INTO TABLE %s.%s FIELDS TERMINATED BY ',' ENCLOSED BY '\"' LINES TERMINATED BY '\\r\\n'", + "LOAD DATA LOCAL INFILE %s INTO TABLE %s.%s FIELDS TERMINATED BY ',' ENCLOSED BY '\"' ESCAPED BY '\\\"' LINES TERMINATED BY '\\r\\n'", absoluteFile, schemaName, tmpTableName); try (Statement stmt = connection.createStatement()) { diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java index 94f4157118a9..09fa8f94d026 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java @@ -28,12 +28,21 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Databases; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteMessage.Type; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.CatalogHelpers; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import java.sql.SQLException; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -204,6 +213,48 @@ public void testCustomDbtTransformations() throws Exception { // super.testCustomDbtTransformations(); } + @Test + public void testJsonSync() throws Exception { + final String catalogAsText = "{\n" + + " \"streams\": [\n" + + " {\n" + + " \"name\": \"exchange_rate\",\n" + + " \"json_schema\": {\n" + + " \"properties\": {\n" + + " \"id\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"data\": {\n" + + " \"type\": \"string\"\n" + + " }" + + " }\n" + + " }\n" + + " }\n" + + " ]\n" + + "}\n"; + + final AirbyteCatalog catalog = Jsons.deserialize(catalogAsText, AirbyteCatalog.class); + final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog); + final List messages = Lists.newArrayList( + new AirbyteMessage() + .withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withStream(catalog.getStreams().get(0).getName()) + .withEmittedAt(Instant.now().toEpochMilli()) + .withData(Jsons.jsonNode(ImmutableMap.builder() + .put("id", 1) + .put("data", "{\"name\":\"Conferência Faturamento - Custo - Taxas - Margem - Resumo ano inicial até -2\",\"description\":null}") + .build()))), + new AirbyteMessage() + .withType(Type.STATE) + .withState(new AirbyteStateMessage().withData(Jsons.jsonNode(ImmutableMap.of("checkpoint", 2))))); + + final JsonNode config = getConfig(); + final String defaultSchema = getDefaultSchema(config); + runSyncAndVerifyStateOutput(config, messages, configuredCatalog, false); + retrieveRawRecordsAndAssertSameMessages(catalog, messages, defaultSchema); + } + @Override @Test public void testLineBreakCharacters() { From 88f12679829ebda19262f899ea08d33a7eeaaab5 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Mon, 19 Jul 2021 17:39:20 +0300 Subject: [PATCH 2/3] [4583] Fixed codestyle --- .../integrations/destination/mysql/MySQLSqlOperations.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java index 0803fa5ab990..aee61eda1c4c 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java +++ b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java @@ -80,7 +80,7 @@ private void loadDataIntoTable(JdbcDatabase database, String absoluteFile = "'" + tmpFile.getAbsolutePath() + "'"; String query = String.format( - "LOAD DATA LOCAL INFILE %s INTO TABLE %s.%s FIELDS TERMINATED BY ',' ENCLOSED BY '\"' ESCAPED BY '\\\"' LINES TERMINATED BY '\\r\\n'", + "LOAD DATA LOCAL INFILE %s INTO TABLE %s.%s FIELDS TERMINATED BY ',' ENCLOSED BY '\"' ESCAPED BY '\\\"' LINES TERMINATED BY '\\r\\n'", absoluteFile, schemaName, tmpTableName); try (Statement stmt = connection.createStatement()) { From 1cd0c8661fa642af31effc0d05d283d3b123e443 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Mon, 19 Jul 2021 18:02:04 +0300 Subject: [PATCH 3/3] [4583] Bumped version --- .../ca81ee7c-3163-4246-af40-094cc31e5e42.json | 2 +- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- airbyte-integrations/connectors/destination-mysql/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/ca81ee7c-3163-4246-af40-094cc31e5e42.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/ca81ee7c-3163-4246-af40-094cc31e5e42.json index f47db2ef0a50..1d6d3ad835ee 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/ca81ee7c-3163-4246-af40-094cc31e5e42.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/ca81ee7c-3163-4246-af40-094cc31e5e42.json @@ -2,6 +2,6 @@ "destinationDefinitionId": "ca81ee7c-3163-4246-af40-094cc31e5e42", "name": "MySQL", "dockerRepository": "airbyte/destination-mysql", - "dockerImageTag": "0.1.7", + "dockerImageTag": "0.1.8", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mysql" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 272bbc4334f4..3b50ac0037ee 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -58,7 +58,7 @@ - destinationDefinitionId: ca81ee7c-3163-4246-af40-094cc31e5e42 name: MySQL dockerRepository: airbyte/destination-mysql - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 documentationUrl: https://docs.airbyte.io/integrations/destinations/mysql - destinationDefinitionId: d4353156-9217-4cad-8dd7-c108fd4f74cf name: MS SQL Server diff --git a/airbyte-integrations/connectors/destination-mysql/Dockerfile b/airbyte-integrations/connectors/destination-mysql/Dockerfile index bf0cbe5fdd9a..5bc950ab07e3 100644 --- a/airbyte-integrations/connectors/destination-mysql/Dockerfile +++ b/airbyte-integrations/connectors/destination-mysql/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/destination-mysql