diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java index 77ac363d90b0..9fcd2ac61cb7 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @@ -275,9 +275,9 @@ public static Map flatten(final JsonNode node, final Boolean app } /** - * Flattens an ObjectNode, or dumps it into a {null: value} map if it's not an object. - * New usage of this function is best to explicitly declare the intended array mode. - * This version is provided for backward compatibility. + * Flattens an ObjectNode, or dumps it into a {null: value} map if it's not an object. New usage of + * this function is best to explicitly declare the intended array mode. This version is provided for + * backward compatibility. */ public static Map flatten(final JsonNode node) { return flatten(node, false); diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java index e8628d9f62d4..cd67fd0190e2 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java @@ -162,8 +162,7 @@ void testToObject() { assertEquals( Lists.newArrayList(expected), - Jsons.object(Jsons.jsonNode(Lists.newArrayList(expected)), new TypeReference>() { - })); + Jsons.object(Jsons.jsonNode(Lists.newArrayList(expected)), new TypeReference>() {})); assertEquals( new ToClass(), @@ -179,8 +178,7 @@ void testTryToObject() { assertEquals( Optional.of(expected), - Jsons.tryObject(Jsons.deserialize(SERIALIZED_JSON), new TypeReference() { - })); + Jsons.tryObject(Jsons.deserialize(SERIALIZED_JSON), new TypeReference() {})); final ToClass emptyExpected = new ToClass(); assertEquals( @@ -189,8 +187,7 @@ void testTryToObject() { assertEquals( Optional.of(emptyExpected), - Jsons.tryObject(Jsons.deserialize("{\"str1\":\"abc\"}"), new TypeReference() { - })); + Jsons.tryObject(Jsons.deserialize("{\"str1\":\"abc\"}"), new TypeReference() {})); } @@ -273,9 +270,9 @@ void testGetEstimatedByteSize() { void testFlatten__noArrays() { final JsonNode json = Jsons.deserialize("{ \"abc\": { \"def\": \"ghi\" }, \"jkl\": true, \"pqr\": 1 }"); Map expected = Stream.of(new Object[][] { - { "abc.def", "ghi" }, - { "jkl", true }, - { "pqr", 1 }, + {"abc.def", "ghi"}, + {"jkl", true}, + {"pqr", 1}, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); assertEquals(expected, Jsons.flatten(json, false)); } @@ -285,9 +282,9 @@ void testFlatten__withArraysNoApplyFlatten() { final JsonNode json = Jsons .deserialize("{ \"abc\": [{ \"def\": \"ghi\" }, { \"fed\": \"ihg\" }], \"jkl\": true, \"pqr\": 1 }"); Map expected = Stream.of(new Object[][] { - { "abc", "[{\"def\":\"ghi\"},{\"fed\":\"ihg\"}]" }, - { "jkl", true }, - { "pqr", 1 }, + {"abc", "[{\"def\":\"ghi\"},{\"fed\":\"ihg\"}]"}, + {"jkl", true}, + {"pqr", 1}, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); assertEquals(expected, Jsons.flatten(json, false)); } @@ -297,22 +294,22 @@ void testFlatten__checkBackwardCompatiblity() { final JsonNode json = Jsons .deserialize("{ \"abc\": [{ \"def\": \"ghi\" }, { \"fed\": \"ihg\" }], \"jkl\": true, \"pqr\": 1 }"); Map expected = Stream.of(new Object[][] { - { "abc", "[{\"def\":\"ghi\"},{\"fed\":\"ihg\"}]" }, - { "jkl", true }, - { "pqr", 1 }, + {"abc", "[{\"def\":\"ghi\"},{\"fed\":\"ihg\"}]"}, + {"jkl", true}, + {"pqr", 1}, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); assertEquals(expected, Jsons.flatten(json)); } - + @Test void testFlatten__withArraysApplyFlatten() { final JsonNode json = Jsons .deserialize("{ \"abc\": [{ \"def\": \"ghi\" }, { \"fed\": \"ihg\" }], \"jkl\": true, \"pqr\": 1 }"); Map expected = Stream.of(new Object[][] { - { "abc.[0].def", "ghi" }, - { "abc.[1].fed", "ihg" }, - { "jkl", true }, - { "pqr", 1 }, + {"abc.[0].def", "ghi"}, + {"abc.[1].fed", "ihg"}, + {"jkl", true}, + {"pqr", 1}, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); assertEquals(expected, Jsons.flatten(json, true)); } @@ -323,10 +320,10 @@ void testFlatten__withArraysApplyFlattenNested() { .deserialize( "{ \"abc\": [{ \"def\": {\"ghi\": [\"xyz\"] }}, { \"fed\": \"ihg\" }], \"jkl\": true, \"pqr\": 1 }"); Map expected = Stream.of(new Object[][] { - { "abc.[0].def.ghi.[0]", "xyz" }, - { "abc.[1].fed", "ihg" }, - { "jkl", true }, - { "pqr", 1 }, + {"abc.[0].def.ghi.[0]", "xyz"}, + {"abc.[1].fed", "ihg"}, + {"jkl", true}, + {"pqr", 1}, }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); assertEquals(expected, Jsons.flatten(json, true)); } @@ -342,8 +339,7 @@ private static class ToClass { @JsonProperty("numLong") long numLong; - public ToClass() { - } + public ToClass() {} public ToClass(final String str, final Integer num, final long numLong) { this.str = str; diff --git a/airbyte-integrations/connectors/destination-bitdotio/src/main/java/io/airbyte/integrations/destination/postgres/BitDotIoDestination.java b/airbyte-integrations/connectors/destination-bitdotio/src/main/java/io/airbyte/integrations/destination/postgres/BitDotIoDestination.java index 092171ad6ecb..db60379700bb 100644 --- a/airbyte-integrations/connectors/destination-bitdotio/src/main/java/io/airbyte/integrations/destination/postgres/BitDotIoDestination.java +++ b/airbyte-integrations/connectors/destination-bitdotio/src/main/java/io/airbyte/integrations/destination/postgres/BitDotIoDestination.java @@ -1,27 +1,28 @@ -/* * Copyright (c) 2022 Airbyte, Inc., all rights reserved. */ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.postgres; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode ; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.json.Jsons; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination; -import io.airbyte.integrations.destination.postgres.PostgresDestination; import io.airbyte.protocol.models.ConnectorSpecification; import java.net.URI; import java.net.URISyntaxException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class BitDotIoDestination extends SpecModifyingDestination implements Destination { private static final Logger LOGGER = LoggerFactory.getLogger(BitDotIoDestination.class); - + public BitDotIoDestination() { super(new PostgresDestination()); } @@ -32,7 +33,7 @@ public ConnectorSpecification modifySpec(final ConnectorSpecification originalSp String json = "[ \"database\", \"username\", \"password\"] "; ObjectMapper objectMapper = new ObjectMapper(); - + JsonNode jsonNode; try { jsonNode = objectMapper.readTree(json); diff --git a/airbyte-integrations/connectors/destination-bitdotio/src/test-integration/java/io/airbyte/integrations/destination/postgres/BitDotIoDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bitdotio/src/test-integration/java/io/airbyte/integrations/destination/postgres/BitDotIoDestinationAcceptanceTest.java index d0b46ece172a..93343b24dd9f 100644 --- a/airbyte-integrations/connectors/destination-bitdotio/src/test-integration/java/io/airbyte/integrations/destination/postgres/BitDotIoDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-bitdotio/src/test-integration/java/io/airbyte/integrations/destination/postgres/BitDotIoDestinationAcceptanceTest.java @@ -14,54 +14,57 @@ import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.file.Files; +import java.nio.file.Path; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.stream.Collectors; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; - import org.jooq.DSLContext; import org.jooq.SQLDialect; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class BitDotIoDestinationAcceptanceTest extends DestinationAcceptanceTest { + public class BitDotIoConfig { - private String username = ""; - private String database = ""; - private String password = ""; - public String getUsername() { - return username; - } - public String getDatabase() { - return database; - } - public String getPassword() { - return password; - } - public BitDotIoConfig(String username, String database, String password) - { + + private String username = ""; + private String database = ""; + private String password = ""; + + public String getUsername() { + return username; + } + + public String getDatabase() { + return database; + } + + public String getPassword() { + return password; + } + + public BitDotIoConfig(String username, String database, String password) { this.username = username; this.database = database; this.password = password; } public String getJdbcUrl() { - String jdbcUrl = ""; - try { - jdbcUrl = "jdbc:postgresql://db.bit.io:5432" + "/" + URLEncoder.encode(database, "UTF-8") + "?sslmode=require"; - } catch (UnsupportedEncodingException e) { - // Should never happen - e.printStackTrace(); - } - return jdbcUrl; + String jdbcUrl = ""; + try { + jdbcUrl = "jdbc:postgresql://db.bit.io:5432" + "/" + URLEncoder.encode(database, "UTF-8") + "?sslmode=require"; + } catch (UnsupportedEncodingException e) { + // Should never happen + e.printStackTrace(); + } + return jdbcUrl; } - } + } private static final Logger LOGGER = LoggerFactory.getLogger(BitDotIoDestinationAcceptanceTest.class); private BitDotIoConfig cfg; @@ -88,8 +91,8 @@ protected JsonNode getConfig() { .put(JdbcUtils.USERNAME_KEY, cfg.getUsername()) .put(JdbcUtils.PASSWORD_KEY, cfg.getPassword()) .put(JdbcUtils.DATABASE_KEY, cfg.getDatabase()) - .put(JdbcUtils.SSL_KEY, true ) - .put("sslmode", "require" ) + .put(JdbcUtils.SSL_KEY, true) + .put("sslmode", "require") .put("ssl_mode", ImmutableMap.builder().put("mode", "require").build()) .build()); } @@ -104,7 +107,7 @@ protected JsonNode getFailCheckConfig() { .put(JdbcUtils.PASSWORD_KEY, "wrong password") .put(JdbcUtils.DATABASE_KEY, cfg.getDatabase()) .put(JdbcUtils.SSL_KEY, true) - .put("sslmode", "require" ) + .put("sslmode", "require") .put("ssl_mode", ImmutableMap.builder().put("mode", "require").build()) .build()); } @@ -173,6 +176,7 @@ private List retrieveRecordsFromTable(final String tableName, final St .collect(Collectors.toList())); } } + @Override protected void setup(final TestDestinationEnv testEnv) throws Exception { if (!Files.exists(CREDENTIALS_PATH)) { @@ -187,8 +191,9 @@ protected void setup(final TestDestinationEnv testEnv) throws Exception { final String database = credentialsJson.get(CONFIG_BITIO_DATABASE).asText(); final String password = credentialsJson.get(CONFIG_BITIO_CONNECT_PASSWORD).asText(); - this.cfg = new BitDotIoConfig(username, database, password) ; + this.cfg = new BitDotIoConfig(username, database, password); } + @Override protected void tearDown(final TestDestinationEnv testEnv) { try (final DSLContext dslContext = DSLContextFactory.create( @@ -200,29 +205,32 @@ protected void tearDown(final TestDestinationEnv testEnv) { Database db = new Database(dslContext); List tables = db.query( - ctx -> ctx - .fetch( "SELECT table_name FROM information_schema.tables WHERE table_type='BASE TABLE' AND table_schema not IN ('pg_catalog', 'information_schema');") - .stream() - .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat())) - .map(Jsons::deserialize) - .collect(Collectors.toList())); - for (JsonNode node : tables) { - db.query(ctx -> ctx.fetch(String.format("DROP TABLE IF EXISTS %s CASCADE", node.get("table_name")))); - } + ctx -> ctx + .fetch( + "SELECT table_name FROM information_schema.tables WHERE table_type='BASE TABLE' AND table_schema not IN ('pg_catalog', 'information_schema');") + .stream() + .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat())) + .map(Jsons::deserialize) + .collect(Collectors.toList())); + for (JsonNode node : tables) { + db.query(ctx -> ctx.fetch(String.format("DROP TABLE IF EXISTS %s CASCADE", node.get("table_name")))); + } List schemas = db.query( - ctx -> ctx - .fetch( "SELECT DISTINCT table_schema FROM information_schema.tables WHERE table_type='BASE TABLE' AND table_schema not IN ('public', 'pg_catalog', 'information_schema');") - .stream() - .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat())) - .map(Jsons::deserialize) - .collect(Collectors.toList())); - for (JsonNode node : schemas) { - db.query(ctx -> ctx.fetch(String.format("DROP SCHEMA IF EXISTS %s CASCADE", node.get("table_schema")))); - } + ctx -> ctx + .fetch( + "SELECT DISTINCT table_schema FROM information_schema.tables WHERE table_type='BASE TABLE' AND table_schema not IN ('public', 'pg_catalog', 'information_schema');") + .stream() + .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat())) + .map(Jsons::deserialize) + .collect(Collectors.toList())); + for (JsonNode node : schemas) { + db.query(ctx -> ctx.fetch(String.format("DROP SCHEMA IF EXISTS %s CASCADE", node.get("table_schema")))); + } } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } LOGGER.info("Finished acceptance test for bit.io"); } -} \ No newline at end of file + +} diff --git a/airbyte-integrations/connectors/destination-bitdotio/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-bitdotio/src/test/resources/expected_spec.json index ca0f9bf4b315..fc4d0d7ce417 100644 --- a/airbyte-integrations/connectors/destination-bitdotio/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/destination-bitdotio/src/test/resources/expected_spec.json @@ -8,11 +8,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Postgres Destination Spec", "type": "object", - "required": [ - "database", - "username", - "password" - ], + "required": ["database", "username", "password"], "additionalProperties": true, "properties": { "database": { @@ -25,9 +21,7 @@ "title": "Default Schema", "description": "The default schema tables are written to if the source does not specify a namespace. The usual value for this field is \"public\".", "type": "string", - "examples": [ - "public" - ], + "examples": ["public"], "default": "public", "order": 3 }, diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java index 3c694621e23a..6cd6b925e982 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java +++ b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java @@ -387,7 +387,8 @@ protected String getWrappedColumnNames(final JdbcDatabase database, final Connection connection, final List columnNames, final String schemaName, - final String tableName) throws SQLException { + final String tableName) + throws SQLException { return sourceOperations.enquoteIdentifierList(connection, columnNames); } diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json index c362f65b2358..09560a7a843c 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json @@ -1,80 +1,80 @@ { "customers": { - "updated_at": "2022-06-22T03:50:13-07:00" + "updated_at": "2022-06-22T03:50:13-07:00" }, "orders": { - "updated_at": "2022-10-10T06:21:53-07:00" + "updated_at": "2022-10-10T06:21:53-07:00" }, "draft_orders": { - "updated_at": "2022-10-08T05:07:29-07:00" + "updated_at": "2022-10-08T05:07:29-07:00" }, "products": { - "updated_at": "2022-10-10T06:21:56-07:00" + "updated_at": "2022-10-10T06:21:56-07:00" }, "abandoned_checkouts": {}, "metafields": { - "updated_at": "2022-05-30T23:42:02-07:00" + "updated_at": "2022-05-30T23:42:02-07:00" }, "collects": { - "id": 29427031703740 + "id": 29427031703740 }, "custom_collections": { - "updated_at": "2022-10-08T04:44:51-07:00" + "updated_at": "2022-10-08T04:44:51-07:00" }, "order_refunds": { - "created_at": "2022-10-10T06:21:53-07:00", - "orders": { - "updated_at": "2022-10-10T06:21:53-07:00" - } + "created_at": "2022-10-10T06:21:53-07:00", + "orders": { + "updated_at": "2022-10-10T06:21:53-07:00" + } }, "order_risks": { - "id": 6446736474301, - "orders": { - "updated_at": "2022-03-07T02:09:04-08:00" - } + "id": 6446736474301, + "orders": { + "updated_at": "2022-03-07T02:09:04-08:00" + } }, "transactions": { - "created_at": "2022-10-10T06:21:52-07:00", - "orders": { - "updated_at": "2022-10-10T06:21:53-07:00" - } + "created_at": "2022-10-10T06:21:52-07:00", + "orders": { + "updated_at": "2022-10-10T06:21:53-07:00" + } }, "tender_transactions": { - "processed_at": "2022-10-10T06:21:52-07:00" + "processed_at": "2022-10-10T06:21:52-07:00" }, "pages": { - "updated_at": "2022-10-08T08:07:00-07:00" + "updated_at": "2022-10-08T08:07:00-07:00" }, "price_rules": { - "updated_at": "2021-09-10T06:48:10-07:00" + "updated_at": "2021-09-10T06:48:10-07:00" }, "discount_codes": { - "price_rules": { - "updated_at": "2021-09-10T06:48:10-07:00" - }, + "price_rules": { "updated_at": "2021-09-10T06:48:10-07:00" + }, + "updated_at": "2021-09-10T06:48:10-07:00" }, "inventory_items": { - "products": { - "updated_at": "2022-03-17T03:10:35-07:00" - }, - "updated_at": "2022-03-06T14:12:20-08:00" + "products": { + "updated_at": "2022-03-17T03:10:35-07:00" + }, + "updated_at": "2022-03-06T14:12:20-08:00" }, "inventory_levels": { - "locations": {}, - "updated_at": "2022-10-10T06:21:56-07:00" + "locations": {}, + "updated_at": "2022-10-10T06:21:56-07:00" }, "fulfillment_orders": { - "id": 5567724486845, - "orders": { - "updated_at": "2022-10-10T06:05:29-07:00" - } + "id": 5567724486845, + "orders": { + "updated_at": "2022-10-10T06:05:29-07:00" + } }, "fulfillments": { - "updated_at": "2022-06-22T03:50:14-07:00", - "orders": { - "updated_at": "2022-10-10T06:05:29-07:00" - } + "updated_at": "2022-06-22T03:50:14-07:00", + "orders": { + "updated_at": "2022-10-10T06:05:29-07:00" + } }, "balance_transactions": {} -} \ No newline at end of file +}