diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 91ccc25b58f5..423b9b411ea6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -870,7 +870,7 @@ - name: MySQL sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad dockerRepository: airbyte/source-mysql - dockerImageTag: 1.0.11 + dockerImageTag: 1.0.12 documentationUrl: https://docs.airbyte.com/integrations/sources/mysql icon: mysql.svg sourceType: database @@ -1070,7 +1070,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.22 + dockerImageTag: 1.0.23 documentationUrl: https://docs.airbyte.com/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 26c4c525c4f6..763213e73282 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -8167,7 +8167,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mysql:1.0.11" +- dockerImage: "airbyte/source-mysql:1.0.12" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/mysql" connectionSpecification: @@ -10152,7 +10152,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.22" +- dockerImage: "airbyte/source-postgres:1.0.23" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index e359a2331889..5529b35668c4 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -47,6 +47,7 @@ public class JdbcUtils { // NOTE: this is the plural version of SCHEMA_KEY public static final String SCHEMAS_KEY = "schemas"; public static final String SSL_KEY = "ssl"; + public static final List SSL_MODE_DISABLE = List.of("disable", "disabled"); public static final String SSL_MODE_KEY = "ssl_mode"; public static final String TLS_KEY = "tls"; public static final String USERNAME_KEY = "username"; @@ -111,10 +112,17 @@ public static Map parseJdbcParameters(final String jdbcPropertie * (e.g. non-zero integers, string true, etc) * * @param config A configuration used to check Jdbc connection - * @return true: if ssl has not been set or it has been set with true, false: in all other cases + * @return true: if ssl has not been set and ssl mode not equals disabled or it has been set with + * true, false: in all other cases */ public static boolean useSsl(final JsonNode config) { - return !config.has(SSL_KEY) || config.get(SSL_KEY).asBoolean(); + if (!config.has(SSL_KEY)) { + if (config.has(SSL_MODE_KEY) && config.get(SSL_MODE_KEY).has(MODE_KEY)) { + return !SSL_MODE_DISABLE.contains(config.get(SSL_MODE_KEY).get(MODE_KEY).asText()); + } else + return true; + } else + return config.get(SSL_KEY).asBoolean(); } } diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java index d6a4b139a2bf..92435db61ce4 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java @@ -210,6 +210,42 @@ void testUssSslWithSslSetAndValueIntegerTrue() { assertTrue(sslSet); } + @Test + void testUseSslWithEmptySslKeyAndSslModeVerifyFull() { + final JsonNode config = Jsons.jsonNode(ImmutableMap.builder() + .put("host", PSQL_DB.getHost()) + .put("port", PSQL_DB.getFirstMappedPort()) + .put("database", dbName) + .put("username", PSQL_DB.getUsername()) + .put("password", PSQL_DB.getPassword()) + .put("ssl_mode", ImmutableMap.builder() + .put("mode", "verify-full") + .put("ca_certificate", "test_ca_cert") + .put("client_certificate", "test_client_cert") + .put("client_key", "test_client_key") + .put("client_key_password", "test_pass") + .build()) + .build()); + final boolean sslSet = JdbcUtils.useSsl(config); + assertTrue(sslSet); + } + + @Test + void testUseSslWithEmptySslKeyAndSslModeDisable() { + final JsonNode config = Jsons.jsonNode(ImmutableMap.builder() + .put("host", PSQL_DB.getHost()) + .put("port", PSQL_DB.getFirstMappedPort()) + .put("database", dbName) + .put("username", PSQL_DB.getUsername()) + .put("password", PSQL_DB.getPassword()) + .put("ssl_mode", ImmutableMap.builder() + .put("mode", "disable") + .build()) + .build()); + final boolean sslSet = JdbcUtils.useSsl(config); + assertFalse(sslSet); + } + private static void createTableWithAllTypes(final Connection connection) throws SQLException { // jdbctype not included because they are not directly supported in postgres: TINYINT, LONGVARCHAR, // VARBINAR, LONGVARBINARY diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index 86a16ad9c7f9..fc05749d57fa 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -16,6 +16,6 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.11 +LABEL io.airbyte.version=1.0.12 LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java index eb58bd9c10c1..c5516cb56e15 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java @@ -22,10 +22,13 @@ import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.Source; +import io.airbyte.integrations.base.ssh.SshBastionContainer; import io.airbyte.integrations.base.ssh.SshHelpers; +import io.airbyte.integrations.base.ssh.SshTunnel; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.integrations.source.mysql.MySqlSource; import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; @@ -39,10 +42,8 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; + import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; @@ -51,12 +52,15 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.Network; class MySqlStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { protected static final String TEST_USER = "test"; protected static final String TEST_PASSWORD = "test"; protected static MySQLContainer container; + private static final SshBastionContainer bastion = new SshBastionContainer(); + private static final Network network = Network.newNetwork(); protected Database database; protected DSLContext dslContext; @@ -328,6 +332,33 @@ void testStrictSSLUnsecuredWithTunnel() throws Exception { assertTrue(actual.getMessage().contains("Could not connect with provided SSH configuration.")); } + + @Test + void testCheckWithSSlModeDisabled() throws Exception { + try (final MySQLContainer db = new MySQLContainer<>("mysql:8.0").withNetwork(network)) { + bastion.initAndStartBastion(network); + db.start(); + final JsonNode configWithSSLModeDisabled = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, Objects.requireNonNull(db.getContainerInfo() + .getNetworkSettings() + .getNetworks() + .entrySet().stream() + .findFirst() + .get().getValue().getIpAddress())) + .put(JdbcUtils.PORT_KEY, db.getExposedPorts().get(0)) + .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) + .put(JdbcUtils.SCHEMAS_KEY, List.of("public")) + .put(JdbcUtils.USERNAME_KEY, db.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) + .put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, "disable"))); + + final AirbyteConnectionStatus actual = source.check(configWithSSLModeDisabled); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, actual.getStatus()); + } finally { + bastion.stopAndClose(); + } + } + @Override protected boolean supportsPerStream() { return true; diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile index b8186ad82952..3889a9a1e60a 100644 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql/Dockerfile @@ -16,6 +16,6 @@ ENV APPLICATION source-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.11 +LABEL io.airbyte.version=1.0.12 LABEL io.airbyte.name=airbyte/source-mysql diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 8c43729edb76..fff90a2cc674 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.22 +LABEL io.airbyte.version=1.0.23 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 77a80d05f0b8..fc8ad82a1e2a 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.22 +LABEL io.airbyte.version=1.0.23 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncryptTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncryptTest.java new file mode 100644 index 000000000000..b31de10fcc58 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncryptTest.java @@ -0,0 +1,77 @@ +package io.airbyte.integrations.source.postgres; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.base.ssh.SshBastionContainer; +import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.protocol.models.AirbyteConnectionStatus; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.PostgreSQLContainer; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class PostgresSourceStrictEncryptTest { + private static final SshBastionContainer bastion = new SshBastionContainer(); + private static final Network network = Network.newNetwork(); + + @Test + void testCheckWithSSlModeDisable() throws Exception { + + try (PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(network)) { + bastion.initAndStartBastion(network); + db.start(); + + // stop to enforce ssl for ssl_mode disable + final ImmutableMap.Builder builderWithSSLModeDisable = getDatabaseConfigBuilderWithSSLMode(db, "disable"); + final JsonNode configWithSSLModeDisable = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, builderWithSSLModeDisable); + final AirbyteConnectionStatus connectionStatusForDisabledMode = new PostgresSourceStrictEncrypt().check(configWithSSLModeDisable); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatusForDisabledMode.getStatus()); + + } finally { + bastion.stopAndClose(); + } + } + + @Test + void testCheckWithSSlModePrefer() throws Exception { + + try (PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(network)) { + bastion.initAndStartBastion(network); + db.start(); + //continue to enforce ssl because ssl mode is prefer + final ImmutableMap.Builder builderWithSSLModePrefer = getDatabaseConfigBuilderWithSSLMode(db, "prefer"); + final JsonNode configWithSSLModePrefer = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, builderWithSSLModePrefer); + final AirbyteConnectionStatus connectionStatusForPreferredMode = new PostgresSourceStrictEncrypt().check(configWithSSLModePrefer); + assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatusForPreferredMode.getStatus()); + assertEquals("State code: 08004; Message: The server does not support SSL.", connectionStatusForPreferredMode.getMessage()); + + } finally { + bastion.stopAndClose(); + } + } + + private ImmutableMap.Builder getDatabaseConfigBuilderWithSSLMode(PostgreSQLContainer db, String sslMode) { + return ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, Objects.requireNonNull(db.getContainerInfo() + .getNetworkSettings() + .getNetworks() + .entrySet().stream() + .findFirst() + .get().getValue().getIpAddress())) + .put(JdbcUtils.PORT_KEY, db.getExposedPorts().get(0)) + .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) + .put(JdbcUtils.SCHEMAS_KEY, List.of("public")) + .put(JdbcUtils.USERNAME_KEY, db.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) + .put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, sslMode)); + } + + + +} diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index b2a78ecf2869..bc03813c2407 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -252,6 +252,7 @@ WHERE actor_definition_id ='435bb9a5-7887-4809-aa58-28c27df0d7ad' AND (configura ## Changelog | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.12 | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025) | Stop enforce SSL if ssl mode is disabled | | 1.0.11 | 2022-11-03 | [18851](https://github.com/airbytehq/airbyte/pull/18851) | Fix bug with unencrypted CDC connections | | 1.0.10 | 2022-11-02 | [18619](https://github.com/airbytehq/airbyte/pull/18619) | Fix bug with handling Tinyint(1) Unsigned values as boolean | | 1.0.9 | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538) | Encode database name | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 4d73c604a67e..7844f663ea82 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -400,6 +400,7 @@ The root causes is that the WALs needed for the incremental sync has been remove | Version | Date | Pull Request | Subject | |:--------|:-----------|:-------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.23 | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025) | Stop enforce SSL if ssl mode is disabled | | 1.0.22 | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538) | Encode database name | | 1.0.21 | 2022-10-25 | [18256](https://github.com/airbytehq/airbyte/pull/18256) | Disable allow and prefer ssl modes in CDC mode | | 1.0.20 | 2022-10-25 | [18383](https://github.com/airbytehq/airbyte/pull/18383) | Better SSH error handling + messages |