Skip to content

Commit

Permalink
Postgres/MySQL Source Strict Encrypt: stop enforce SSL if ssl mode di…
Browse files Browse the repository at this point in the history
…sabled (#19025)

* Postgres/MySQL Source Strict Encrypt: stop enforce SSL if ssl mode disabled

* fixed checkstyle

* updated changelog

* add tests

* replaced MySQL test to mysql-strict-encrypt module

* fixed Connection Refused for mysql test

* replaced Postgres Source strict-encrypt tests into new class

* bump version

* auto-bump connector version

* auto-bump connector version

Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
  • Loading branch information
VitaliiMaltsev and octavia-squidington-iii authored Nov 9, 2022
1 parent f54f4af commit c72b75e
Show file tree
Hide file tree
Showing 12 changed files with 168 additions and 14 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -870,7 +870,7 @@
- name: MySQL
sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad
dockerRepository: airbyte/source-mysql
dockerImageTag: 1.0.11
dockerImageTag: 1.0.12
documentationUrl: https://docs.airbyte.com/integrations/sources/mysql
icon: mysql.svg
sourceType: database
Expand Down Expand Up @@ -1070,7 +1070,7 @@
- name: Postgres
sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
dockerRepository: airbyte/source-postgres
dockerImageTag: 1.0.22
dockerImageTag: 1.0.23
documentationUrl: https://docs.airbyte.com/integrations/sources/postgres
icon: postgresql.svg
sourceType: database
Expand Down
4 changes: 2 additions & 2 deletions airbyte-config/init/src/main/resources/seed/source_specs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8167,7 +8167,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-mysql:1.0.11"
- dockerImage: "airbyte/source-mysql:1.0.12"
spec:
documentationUrl: "https://docs.airbyte.com/integrations/sources/mysql"
connectionSpecification:
Expand Down Expand Up @@ -10152,7 +10152,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-postgres:1.0.22"
- dockerImage: "airbyte/source-postgres:1.0.23"
spec:
documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
connectionSpecification:
Expand Down
12 changes: 10 additions & 2 deletions airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ public class JdbcUtils {
// NOTE: this is the plural version of SCHEMA_KEY
public static final String SCHEMAS_KEY = "schemas";
public static final String SSL_KEY = "ssl";
public static final List<String> SSL_MODE_DISABLE = List.of("disable", "disabled");
public static final String SSL_MODE_KEY = "ssl_mode";
public static final String TLS_KEY = "tls";
public static final String USERNAME_KEY = "username";
Expand Down Expand Up @@ -111,10 +112,17 @@ public static Map<String, String> parseJdbcParameters(final String jdbcPropertie
* (e.g. non-zero integers, string true, etc)
*
* @param config A configuration used to check Jdbc connection
* @return true: if ssl has not been set or it has been set with true, false: in all other cases
* @return true: if ssl has not been set and ssl mode not equals disabled or it has been set with
* true, false: in all other cases
*/
public static boolean useSsl(final JsonNode config) {
return !config.has(SSL_KEY) || config.get(SSL_KEY).asBoolean();
if (!config.has(SSL_KEY)) {
if (config.has(SSL_MODE_KEY) && config.get(SSL_MODE_KEY).has(MODE_KEY)) {
return !SSL_MODE_DISABLE.contains(config.get(SSL_MODE_KEY).get(MODE_KEY).asText());
} else
return true;
} else
return config.get(SSL_KEY).asBoolean();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,42 @@ void testUssSslWithSslSetAndValueIntegerTrue() {
assertTrue(sslSet);
}

@Test
void testUseSslWithEmptySslKeyAndSslModeVerifyFull() {
final JsonNode config = Jsons.jsonNode(ImmutableMap.builder()
.put("host", PSQL_DB.getHost())
.put("port", PSQL_DB.getFirstMappedPort())
.put("database", dbName)
.put("username", PSQL_DB.getUsername())
.put("password", PSQL_DB.getPassword())
.put("ssl_mode", ImmutableMap.builder()
.put("mode", "verify-full")
.put("ca_certificate", "test_ca_cert")
.put("client_certificate", "test_client_cert")
.put("client_key", "test_client_key")
.put("client_key_password", "test_pass")
.build())
.build());
final boolean sslSet = JdbcUtils.useSsl(config);
assertTrue(sslSet);
}

@Test
void testUseSslWithEmptySslKeyAndSslModeDisable() {
final JsonNode config = Jsons.jsonNode(ImmutableMap.builder()
.put("host", PSQL_DB.getHost())
.put("port", PSQL_DB.getFirstMappedPort())
.put("database", dbName)
.put("username", PSQL_DB.getUsername())
.put("password", PSQL_DB.getPassword())
.put("ssl_mode", ImmutableMap.builder()
.put("mode", "disable")
.build())
.build());
final boolean sslSet = JdbcUtils.useSsl(config);
assertFalse(sslSet);
}

private static void createTableWithAllTypes(final Connection connection) throws SQLException {
// jdbctype not included because they are not directly supported in postgres: TINYINT, LONGVARCHAR,
// VARBINAR, LONGVARBINARY
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,6 @@ ENV APPLICATION source-mysql-strict-encrypt

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=1.0.11
LABEL io.airbyte.version=1.0.12

LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,13 @@
import io.airbyte.db.factory.DatabaseDriver;
import io.airbyte.db.jdbc.JdbcUtils;
import io.airbyte.integrations.base.Source;
import io.airbyte.integrations.base.ssh.SshBastionContainer;
import io.airbyte.integrations.base.ssh.SshHelpers;
import io.airbyte.integrations.base.ssh.SshTunnel;
import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest;
import io.airbyte.integrations.source.mysql.MySqlSource;
import io.airbyte.integrations.source.relationaldb.models.DbStreamState;
import io.airbyte.integrations.util.HostPortResolver;
import io.airbyte.protocol.models.AirbyteCatalog;
import io.airbyte.protocol.models.AirbyteConnectionStatus;
import io.airbyte.protocol.models.AirbyteConnectionStatus.Status;
Expand All @@ -39,10 +42,8 @@
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.*;

import org.jooq.DSLContext;
import org.jooq.SQLDialect;
import org.junit.jupiter.api.AfterAll;
Expand All @@ -51,12 +52,15 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.MySQLContainer;
import org.testcontainers.containers.Network;

class MySqlStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest {

protected static final String TEST_USER = "test";
protected static final String TEST_PASSWORD = "test";
protected static MySQLContainer<?> container;
private static final SshBastionContainer bastion = new SshBastionContainer();
private static final Network network = Network.newNetwork();

protected Database database;
protected DSLContext dslContext;
Expand Down Expand Up @@ -328,6 +332,33 @@ void testStrictSSLUnsecuredWithTunnel() throws Exception {
assertTrue(actual.getMessage().contains("Could not connect with provided SSH configuration."));
}


@Test
void testCheckWithSSlModeDisabled() throws Exception {
try (final MySQLContainer<?> db = new MySQLContainer<>("mysql:8.0").withNetwork(network)) {
bastion.initAndStartBastion(network);
db.start();
final JsonNode configWithSSLModeDisabled = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, ImmutableMap.builder()
.put(JdbcUtils.HOST_KEY, Objects.requireNonNull(db.getContainerInfo()
.getNetworkSettings()
.getNetworks()
.entrySet().stream()
.findFirst()
.get().getValue().getIpAddress()))
.put(JdbcUtils.PORT_KEY, db.getExposedPorts().get(0))
.put(JdbcUtils.DATABASE_KEY, db.getDatabaseName())
.put(JdbcUtils.SCHEMAS_KEY, List.of("public"))
.put(JdbcUtils.USERNAME_KEY, db.getUsername())
.put(JdbcUtils.PASSWORD_KEY, db.getPassword())
.put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, "disable")));

final AirbyteConnectionStatus actual = source.check(configWithSSLModeDisabled);
assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, actual.getStatus());
} finally {
bastion.stopAndClose();
}
}

@Override
protected boolean supportsPerStream() {
return true;
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-mysql/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,6 @@ ENV APPLICATION source-mysql

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=1.0.11
LABEL io.airbyte.version=1.0.12

LABEL io.airbyte.name=airbyte/source-mysql
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=1.0.22
LABEL io.airbyte.version=1.0.23
LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-postgres/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=1.0.22
LABEL io.airbyte.version=1.0.23
LABEL io.airbyte.name=airbyte/source-postgres
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package io.airbyte.integrations.source.postgres;

import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.ImmutableMap;
import io.airbyte.db.jdbc.JdbcUtils;
import io.airbyte.integrations.base.ssh.SshBastionContainer;
import io.airbyte.integrations.base.ssh.SshTunnel;
import io.airbyte.protocol.models.AirbyteConnectionStatus;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.Network;
import org.testcontainers.containers.PostgreSQLContainer;

import java.util.List;
import java.util.Map;
import java.util.Objects;

import static org.junit.jupiter.api.Assertions.assertEquals;

public class PostgresSourceStrictEncryptTest {
private static final SshBastionContainer bastion = new SshBastionContainer();
private static final Network network = Network.newNetwork();

@Test
void testCheckWithSSlModeDisable() throws Exception {

try (PostgreSQLContainer<?> db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(network)) {
bastion.initAndStartBastion(network);
db.start();

// stop to enforce ssl for ssl_mode disable
final ImmutableMap.Builder<Object, Object> builderWithSSLModeDisable = getDatabaseConfigBuilderWithSSLMode(db, "disable");
final JsonNode configWithSSLModeDisable = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, builderWithSSLModeDisable);
final AirbyteConnectionStatus connectionStatusForDisabledMode = new PostgresSourceStrictEncrypt().check(configWithSSLModeDisable);
assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatusForDisabledMode.getStatus());

} finally {
bastion.stopAndClose();
}
}

@Test
void testCheckWithSSlModePrefer() throws Exception {

try (PostgreSQLContainer<?> db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(network)) {
bastion.initAndStartBastion(network);
db.start();
//continue to enforce ssl because ssl mode is prefer
final ImmutableMap.Builder<Object, Object> builderWithSSLModePrefer = getDatabaseConfigBuilderWithSSLMode(db, "prefer");
final JsonNode configWithSSLModePrefer = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, builderWithSSLModePrefer);
final AirbyteConnectionStatus connectionStatusForPreferredMode = new PostgresSourceStrictEncrypt().check(configWithSSLModePrefer);
assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatusForPreferredMode.getStatus());
assertEquals("State code: 08004; Message: The server does not support SSL.", connectionStatusForPreferredMode.getMessage());

} finally {
bastion.stopAndClose();
}
}

private ImmutableMap.Builder<Object, Object> getDatabaseConfigBuilderWithSSLMode(PostgreSQLContainer<?> db, String sslMode) {
return ImmutableMap.builder()
.put(JdbcUtils.HOST_KEY, Objects.requireNonNull(db.getContainerInfo()
.getNetworkSettings()
.getNetworks()
.entrySet().stream()
.findFirst()
.get().getValue().getIpAddress()))
.put(JdbcUtils.PORT_KEY, db.getExposedPorts().get(0))
.put(JdbcUtils.DATABASE_KEY, db.getDatabaseName())
.put(JdbcUtils.SCHEMAS_KEY, List.of("public"))
.put(JdbcUtils.USERNAME_KEY, db.getUsername())
.put(JdbcUtils.PASSWORD_KEY, db.getPassword())
.put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, sslMode));
}



}
1 change: 1 addition & 0 deletions docs/integrations/sources/mysql.md
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,7 @@ WHERE actor_definition_id ='435bb9a5-7887-4809-aa58-28c27df0d7ad' AND (configura
## Changelog
| Version | Date | Pull Request | Subject |
|:--------|:-----------|:-----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------|
| 1.0.12 | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025) | Stop enforce SSL if ssl mode is disabled |
| 1.0.11 | 2022-11-03 | [18851](https://github.com/airbytehq/airbyte/pull/18851) | Fix bug with unencrypted CDC connections |
| 1.0.10 | 2022-11-02 | [18619](https://github.com/airbytehq/airbyte/pull/18619) | Fix bug with handling Tinyint(1) Unsigned values as boolean |
| 1.0.9 | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538) | Encode database name |
Expand Down
1 change: 1 addition & 0 deletions docs/integrations/sources/postgres.md
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,7 @@ The root causes is that the WALs needed for the incremental sync has been remove

| Version | Date | Pull Request | Subject |
|:--------|:-----------|:-------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| 1.0.23 | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025) | Stop enforce SSL if ssl mode is disabled |
| 1.0.22 | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538) | Encode database name |
| 1.0.21 | 2022-10-25 | [18256](https://github.com/airbytehq/airbyte/pull/18256) | Disable allow and prefer ssl modes in CDC mode |
| 1.0.20 | 2022-10-25 | [18383](https://github.com/airbytehq/airbyte/pull/18383) | Better SSH error handling + messages |
Expand Down

0 comments on commit c72b75e

Please sign in to comment.