From bd5be9bf217b91d28cbaab9d003c8d6464c30d57 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 10 May 2023 03:47:45 +0200 Subject: [PATCH 01/15] connectors-ci: fix postgres integration testing --- .../integrations/util/HostPortResolver.java | 20 ++++++++++++++----- ...resSourceSSLCertificateAcceptanceTest.java | 12 +++++------ .../CdcPostgresSourceAcceptanceTest.java | 2 ++ .../sources/PostgresSourceAcceptanceTest.java | 2 ++ 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java index 89eaa857a916..a01cd1f796eb 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java @@ -7,19 +7,29 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.Objects; +import java.util.Optional; import org.testcontainers.containers.GenericContainer; public class HostPortResolver { public static String resolveHost(GenericContainer container) { - return System.getProperty("os.name").toLowerCase().startsWith("mac") - ? getIpAddress(container) - : container.getHost(); + return getIpAddress(container); + } + + public static String resolveHost(GenericContainer container, boolean fromHostMachine) { + return fromHostMachine ? container.getHost() : getIpAddress(container); } public static int resolvePort(GenericContainer container) { - return System.getProperty("os.name").toLowerCase().startsWith("mac") ? (Integer) container.getExposedPorts().get(0) - : container.getFirstMappedPort(); + return (Integer) container.getExposedPorts().stream().findFirst().get(); + } + + public static int resolvePort(GenericContainer container, boolean fromHostMachine) { + if (fromHostMachine) { + return container.getFirstMappedPort(); + } else { + return (Integer) container.getExposedPorts().stream().findFirst().get(); + } } public static String resolveIpAddress(GenericContainer container) { diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceSSLCertificateAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceSSLCertificateAcceptanceTest.java index 80bccbd0f08f..a557d381e596 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceSSLCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceSSLCertificateAcceptanceTest.java @@ -18,6 +18,7 @@ import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -62,11 +63,10 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() .put("method", "Standard") .build()); - final var containerOuterAddress = SshHelpers.getOuterContainerAddress(container); - final var containerInnerAddress = SshHelpers.getInnerContainerAddress(container); + config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", containerInnerAddress.left) - .put("port", containerInnerAddress.right) + .put("host", HostPortResolver.resolveHost(container)) + .put("port", HostPortResolver.resolvePort(container)) .put("database", container.getDatabaseName()) .put("schemas", Jsons.jsonNode(List.of("public"))) .put("username", "postgres") @@ -81,8 +81,8 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc config.get("password").asText(), DatabaseDriver.POSTGRESQL.getDriverClassName(), String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - containerOuterAddress.left, - containerOuterAddress.right, + container.getHost(), + container.getFirstMappedPort(), config.get("database").asText()), SQLDialect.POSTGRES)) { final Database database = new Database(dslContext); diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java index 6bb5c4a1c69d..6afe43213e12 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java @@ -66,6 +66,8 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc .put("publication", PUBLICATION) .put("initial_waiting_seconds", INITIAL_WAITING_SECONDS) .build()); + + config = Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java index 160686d16f09..b8e57c5e04a5 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java @@ -67,6 +67,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc container = new PostgreSQLContainer<>("postgres:13-alpine"); container.start(); + final String username = container.getUsername(); final String password = container.getPassword(); final List schemas = List.of("public"); @@ -98,6 +99,7 @@ private JsonNode getConfig(final String username, final String password, final L final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() .put("method", "Standard") .build()); + return Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) From bae1c1400f1a5eb938090da71b3e5dd9b98617cf Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 10 May 2023 04:13:48 +0200 Subject: [PATCH 02/15] fix mysql --- .../integrations/source/mysql/CdcMysqlSourceTest.java | 5 +++-- .../source/mysql/MySqlJdbcSourceAcceptanceTest.java | 9 +++++---- .../integrations/source/mysql/MySqlSourceTests.java | 5 +++-- .../source/mysql/MySqlSslJdbcSourceAcceptanceTest.java | 5 +++-- .../integrations/source/mysql/MySqlStressTest.java | 9 +++++---- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java index 96e1c7937231..76bd9c4c89d0 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java @@ -33,6 +33,7 @@ import io.airbyte.integrations.debezium.CdcSourceTest; import io.airbyte.integrations.debezium.CdcTargetPosition; import io.airbyte.integrations.debezium.internals.mysql.MySqlCdcTargetPosition; +import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -95,8 +96,8 @@ private void init() { .build()); config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", container.getHost()) - .put("port", container.getFirstMappedPort()) + .put("host", HostPortResolver.resolveHost(container)) + .put("port", HostPortResolver.resolvePort(container)) .put("database", DB_NAME) .put("username", container.getUsername()) .put("password", container.getPassword()) diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java index 8d61becf068c..1a60710feda6 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java @@ -22,6 +22,7 @@ import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -83,8 +84,8 @@ static void init() throws Exception { public void setup() throws Exception { environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) .put(JdbcUtils.DATABASE_KEY, Strings.addRandomSuffix("db", "_", 10)) .put(JdbcUtils.USERNAME_KEY, TEST_USER) .put(JdbcUtils.PASSWORD_KEY, TEST_PASSWORD.call()) @@ -95,8 +96,8 @@ public void setup() throws Exception { config.get(JdbcUtils.PASSWORD_KEY).asText(), DatabaseDriver.MYSQL.getDriverClassName(), String.format("jdbc:mysql://%s:%s", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asText()), + container.getHost(), + container.getFirstMappedPort()), SQLDialect.MYSQL); database = new Database(dslContext); diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java index 1880b371469f..9f6a8757a7d4 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java @@ -19,6 +19,7 @@ import io.airbyte.commons.util.MoreIterators; import io.airbyte.db.Database; import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; @@ -79,8 +80,8 @@ public void testSettingTimezones() throws Exception { private static JsonNode getConfig(final MySQLContainer dbContainer, final String dbName, final String jdbcParams) { return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, dbContainer.getHost()) - .put(JdbcUtils.PORT_KEY, dbContainer.getFirstMappedPort()) + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(dbContainer)) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(dbContainer)) .put(JdbcUtils.DATABASE_KEY, dbName) .put(JdbcUtils.USERNAME_KEY, TEST_USER) .put(JdbcUtils.PASSWORD_KEY, TEST_PASSWORD) diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java index cb6b1ead8267..71bf99f0185c 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java @@ -13,6 +13,7 @@ import io.airbyte.db.factory.DSLContextFactory; import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.util.HostPortResolver; import org.jooq.SQLDialect; import org.junit.jupiter.api.BeforeEach; @@ -34,8 +35,8 @@ public void setup() throws Exception { config.get(JdbcUtils.PASSWORD_KEY).asText(), DatabaseDriver.MYSQL.getDriverClassName(), String.format("jdbc:mysql://%s:%s?%s", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asText(), + container.getHost(), + container.getFirstMappedPort(), String.join("&", SSL_PARAMETERS)), SQLDialect.MYSQL); database = new Database(dslContext); diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java index 5820451ed254..ee91231aa249 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java @@ -15,6 +15,7 @@ import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcStressTest; +import io.airbyte.integrations.util.HostPortResolver; import java.sql.Connection; import java.sql.DriverManager; import java.util.Optional; @@ -54,8 +55,8 @@ static void init() throws Exception { @BeforeEach public void setup() throws Exception { config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) .put(JdbcUtils.DATABASE_KEY, Strings.addRandomSuffix("db", "_", 10)) .put(JdbcUtils.USERNAME_KEY, TEST_USER) .put(JdbcUtils.PASSWORD_KEY, TEST_PASSWORD.call()) @@ -66,8 +67,8 @@ public void setup() throws Exception { config.get(JdbcUtils.PASSWORD_KEY).asText(), DatabaseDriver.MYSQL.getDriverClassName(), String.format("jdbc:mysql://%s:%s", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asText()), + container.getHost(), + container.getFirstMappedPort()), SQLDialect.MYSQL); database = new Database(dslContext); From bbab004f744ca7b4c92822c5e791ac32958538e4 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 10 May 2023 04:15:41 +0200 Subject: [PATCH 03/15] fix conflicts --- .../ci_connector_ops/pipelines/tests/java_connectors.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py index 6499dec0baaa..8942c4bbac40 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py @@ -72,9 +72,8 @@ async def run_all_tests(context: ConnectorContext) -> List[StepResult]: """ context.secrets_dir = await secrets.get_connector_secret_dir(context) - step_results = await run_steps([BuildConnectorImage(context, LOCAL_BUILD_PLATFORM), UnitTests(context)]) - if any([result.status is StepStatus.FAILURE for result in step_results]): - return step_results + step_results = await run_steps([BuildConnectorImage(context, LOCAL_BUILD_PLATFORM)]) + if context.connector.supports_normalization: normalization_image = f"{context.connector.normalization_repository}:dev" context.logger.info(f"This connector supports normalization: will build {normalization_image}.") From 6199f353440581f06a70b816316cea0e52434ca2 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 10 May 2023 04:16:25 +0200 Subject: [PATCH 04/15] revert java connectors pipeline changes --- .../ci_connector_ops/pipelines/tests/java_connectors.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py index 8942c4bbac40..6499dec0baaa 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py @@ -72,8 +72,9 @@ async def run_all_tests(context: ConnectorContext) -> List[StepResult]: """ context.secrets_dir = await secrets.get_connector_secret_dir(context) - step_results = await run_steps([BuildConnectorImage(context, LOCAL_BUILD_PLATFORM)]) - + step_results = await run_steps([BuildConnectorImage(context, LOCAL_BUILD_PLATFORM), UnitTests(context)]) + if any([result.status is StepStatus.FAILURE for result in step_results]): + return step_results if context.connector.supports_normalization: normalization_image = f"{context.connector.normalization_repository}:dev" context.logger.info(f"This connector supports normalization: will build {normalization_image}.") From 36087e7882ba33330039b022128b21fbb8417c4f Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 10 May 2023 04:31:04 +0200 Subject: [PATCH 05/15] Revert "fix mysql" This reverts commit bae1c1400f1a5eb938090da71b3e5dd9b98617cf. --- .../integrations/source/mysql/CdcMysqlSourceTest.java | 5 ++--- .../source/mysql/MySqlJdbcSourceAcceptanceTest.java | 9 ++++----- .../integrations/source/mysql/MySqlSourceTests.java | 5 ++--- .../source/mysql/MySqlSslJdbcSourceAcceptanceTest.java | 5 ++--- .../integrations/source/mysql/MySqlStressTest.java | 9 ++++----- 5 files changed, 14 insertions(+), 19 deletions(-) diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java index 6f9bd600cb6d..9f9f00c53004 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java @@ -32,7 +32,6 @@ import io.airbyte.integrations.base.Source; import io.airbyte.integrations.debezium.CdcSourceTest; import io.airbyte.integrations.debezium.internals.mysql.MySqlCdcTargetPosition; -import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -95,8 +94,8 @@ private void init() { .build()); config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", HostPortResolver.resolveHost(container)) - .put("port", HostPortResolver.resolvePort(container)) + .put("host", container.getHost()) + .put("port", container.getFirstMappedPort()) .put("database", DB_NAME) .put("username", container.getUsername()) .put("password", container.getPassword()) diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java index 1a60710feda6..8d61becf068c 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java @@ -22,7 +22,6 @@ import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.integrations.source.relationaldb.models.DbStreamState; -import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -84,8 +83,8 @@ static void init() throws Exception { public void setup() throws Exception { environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) + .put(JdbcUtils.HOST_KEY, container.getHost()) + .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) .put(JdbcUtils.DATABASE_KEY, Strings.addRandomSuffix("db", "_", 10)) .put(JdbcUtils.USERNAME_KEY, TEST_USER) .put(JdbcUtils.PASSWORD_KEY, TEST_PASSWORD.call()) @@ -96,8 +95,8 @@ public void setup() throws Exception { config.get(JdbcUtils.PASSWORD_KEY).asText(), DatabaseDriver.MYSQL.getDriverClassName(), String.format("jdbc:mysql://%s:%s", - container.getHost(), - container.getFirstMappedPort()), + config.get(JdbcUtils.HOST_KEY).asText(), + config.get(JdbcUtils.PORT_KEY).asText()), SQLDialect.MYSQL); database = new Database(dslContext); diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java index 708ce6364639..6c1ce06b76db 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java @@ -18,7 +18,6 @@ import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; @@ -77,8 +76,8 @@ public void testSettingTimezones() throws Exception { private static JsonNode getConfig(final MySQLContainer dbContainer, final String dbName, final String jdbcParams) { return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(dbContainer)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(dbContainer)) + .put(JdbcUtils.HOST_KEY, dbContainer.getHost()) + .put(JdbcUtils.PORT_KEY, dbContainer.getFirstMappedPort()) .put(JdbcUtils.DATABASE_KEY, dbName) .put(JdbcUtils.USERNAME_KEY, TEST_USER) .put(JdbcUtils.PASSWORD_KEY, TEST_PASSWORD) diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java index 71bf99f0185c..cb6b1ead8267 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java @@ -13,7 +13,6 @@ import io.airbyte.db.factory.DSLContextFactory; import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.util.HostPortResolver; import org.jooq.SQLDialect; import org.junit.jupiter.api.BeforeEach; @@ -35,8 +34,8 @@ public void setup() throws Exception { config.get(JdbcUtils.PASSWORD_KEY).asText(), DatabaseDriver.MYSQL.getDriverClassName(), String.format("jdbc:mysql://%s:%s?%s", - container.getHost(), - container.getFirstMappedPort(), + config.get(JdbcUtils.HOST_KEY).asText(), + config.get(JdbcUtils.PORT_KEY).asText(), String.join("&", SSL_PARAMETERS)), SQLDialect.MYSQL); database = new Database(dslContext); diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java index ee91231aa249..5820451ed254 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java @@ -15,7 +15,6 @@ import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcStressTest; -import io.airbyte.integrations.util.HostPortResolver; import java.sql.Connection; import java.sql.DriverManager; import java.util.Optional; @@ -55,8 +54,8 @@ static void init() throws Exception { @BeforeEach public void setup() throws Exception { config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) + .put(JdbcUtils.HOST_KEY, container.getHost()) + .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) .put(JdbcUtils.DATABASE_KEY, Strings.addRandomSuffix("db", "_", 10)) .put(JdbcUtils.USERNAME_KEY, TEST_USER) .put(JdbcUtils.PASSWORD_KEY, TEST_PASSWORD.call()) @@ -67,8 +66,8 @@ public void setup() throws Exception { config.get(JdbcUtils.PASSWORD_KEY).asText(), DatabaseDriver.MYSQL.getDriverClassName(), String.format("jdbc:mysql://%s:%s", - container.getHost(), - container.getFirstMappedPort()), + config.get(JdbcUtils.HOST_KEY).asText(), + config.get(JdbcUtils.PORT_KEY).asText()), SQLDialect.MYSQL); database = new Database(dslContext); From 56e6eeea367818db5d72052287a508c7575c3947 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 10 May 2023 04:39:45 +0200 Subject: [PATCH 06/15] use retry plugin to handle flaky tests --- .../connectors/source-postgres/build.gradle | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index ebfac92a1fb5..2d56ee138fc3 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -4,6 +4,7 @@ plugins { id 'airbyte-integration-test-java' id 'airbyte-performance-test-java' id 'airbyte-connector-acceptance-test' + id 'org.gradle.test-retry' version '1.5.2' } application { @@ -38,3 +39,11 @@ dependencies { integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) performanceTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } + +test { + retry { + maxRetries = 3 + maxFailures = 20 + failOnPassedAfterRetry = true + } +} From 25d345604024229a38213303d4f67321dee41873 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 10 May 2023 03:03:43 +0000 Subject: [PATCH 07/15] Automated Change --- .../java/io/airbyte/integrations/util/HostPortResolver.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java index a01cd1f796eb..e7cb24cc7b4d 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java @@ -7,7 +7,6 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.Objects; -import java.util.Optional; import org.testcontainers.containers.GenericContainer; public class HostPortResolver { @@ -21,7 +20,7 @@ public static String resolveHost(GenericContainer container, boolean fromHostMac } public static int resolvePort(GenericContainer container) { - return (Integer) container.getExposedPorts().stream().findFirst().get(); + return (Integer) container.getExposedPorts().stream().findFirst().get(); } public static int resolvePort(GenericContainer container, boolean fromHostMachine) { From ebc4665526853c56be156b6024fe0289a40e029a Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 10 May 2023 05:29:57 +0200 Subject: [PATCH 08/15] use S3 cache --- .github/actions/run-dagger-pipeline/action.yml | 2 ++ .../workflows/connector_integration_test_single_dagger.yml | 4 ++++ .github/workflows/connector_nightly_builds_dagger.yml | 2 ++ airbyte-integrations/connectors/source-postgres/build.gradle | 4 +--- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml index d1d7aaae9b7f..fb553e050c24 100644 --- a/.github/actions/run-dagger-pipeline/action.yml +++ b/.github/actions/run-dagger-pipeline/action.yml @@ -52,3 +52,5 @@ runs: CI_GIT_REVISION: ${{ github.event.pull_request.head.sha }} CI_CONTEXT: "${{ inputs.context }}" CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} + S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/connector_integration_test_single_dagger.yml b/.github/workflows/connector_integration_test_single_dagger.yml index 4de2192913cb..b48fa940908a 100644 --- a/.github/workflows/connector_integration_test_single_dagger.yml +++ b/.github/workflows/connector_integration_test_single_dagger.yml @@ -64,6 +64,8 @@ jobs: CI_GIT_REVISION: ${{ github.sha }} CI_CONTEXT: "manual" CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} + S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - name: Run airbyte-ci connectors test [PULL REQUESTS] if: github.event_name == 'pull_request' run: | @@ -88,3 +90,5 @@ jobs: CI_GIT_REVISION: ${{ github.event.pull_request.head.sha }} CI_CONTEXT: "pull_request" CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} + S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/connector_nightly_builds_dagger.yml b/.github/workflows/connector_nightly_builds_dagger.yml index 1c84d0cd06f3..cdb8c45e99e6 100644 --- a/.github/workflows/connector_nightly_builds_dagger.yml +++ b/.github/workflows/connector_nightly_builds_dagger.yml @@ -68,3 +68,5 @@ jobs: CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }} CI_CONTEXT: "nightly_builds" CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} + S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index 2d56ee138fc3..ec20ebde8155 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -43,7 +43,5 @@ dependencies { test { retry { maxRetries = 3 - maxFailures = 20 - failOnPassedAfterRetry = true } -} +} \ No newline at end of file From b3e5e1036002ed98afae7b4e05271921e513463a Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 10 May 2023 05:32:43 +0200 Subject: [PATCH 09/15] use S3 cache --- .../ci_connector_ops/pipelines/actions/environments.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py index 4920da5c54a2..682ff7c1aa40 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py @@ -414,6 +414,10 @@ def with_gradle( .with_mounted_directory("/airbyte", context.get_repo_dir(".", include=include)) .with_mounted_cache("/airbyte/.gradle", airbyte_gradle_cache, sharing=CacheSharingMode.LOCKED) .with_workdir("/airbyte") + .with_secret_variable( + "S3_BUILD_CACHE_ACCESS_KEY_ID", context.dagger_client.host().env_variable("S3_BUILD_CACHE_ACCESS_KEY_ID").secret() + ) + .with_secret_variable("S3_BUILD_CACHE_SECRET_KEY", context.dagger_client.host().env_variable("S3_BUILD_CACHE_SECRET_KEY").secret()) ) if bind_to_docker_host: return with_bound_docker_host(context, openjdk_with_docker, shared_tmp_volume, docker_service_name=docker_service_name) From 93597e709fde6e8fe162d34c97726ee81f4b0237 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 10 May 2023 05:56:00 +0200 Subject: [PATCH 10/15] clean HostPortResolver --- .../integrations/util/HostPortResolver.java | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java index e7cb24cc7b4d..52a85eeda700 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java @@ -15,22 +15,10 @@ public static String resolveHost(GenericContainer container) { return getIpAddress(container); } - public static String resolveHost(GenericContainer container, boolean fromHostMachine) { - return fromHostMachine ? container.getHost() : getIpAddress(container); - } - public static int resolvePort(GenericContainer container) { return (Integer) container.getExposedPorts().stream().findFirst().get(); } - - public static int resolvePort(GenericContainer container, boolean fromHostMachine) { - if (fromHostMachine) { - return container.getFirstMappedPort(); - } else { - return (Integer) container.getExposedPorts().stream().findFirst().get(); - } - } - + public static String resolveIpAddress(GenericContainer container) { return getIpAddress(container); } From 1a1721c80f26355f6bf9272278d13eeffb48cf66 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 10 May 2023 04:00:18 +0000 Subject: [PATCH 11/15] Automated Commit - Formatting Changes --- .../java/io/airbyte/integrations/util/HostPortResolver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java index 52a85eeda700..f0adef160f77 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java @@ -18,7 +18,7 @@ public static String resolveHost(GenericContainer container) { public static int resolvePort(GenericContainer container) { return (Integer) container.getExposedPorts().stream().findFirst().get(); } - + public static String resolveIpAddress(GenericContainer container) { return getIpAddress(container); } From 761796f6d7e74d0587bb454998c66cef9f1190d9 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 10 May 2023 13:37:50 +0200 Subject: [PATCH 12/15] increase INITIAL_WAITING_SECONDS on CDC tests to reduce flakyness --- .../sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java | 2 +- .../sources/CdcPostgresSourceAcceptanceTest.java | 2 +- .../sources/CdcWalLogsPostgresSourceDatatypeTest.java | 2 +- .../integrations/source/postgres/CdcPostgresSourceTest.java | 2 +- .../source/postgres/PostgresCdcGetPublicizedTablesTest.java | 2 +- .../airbyte/integrations/source/postgres/PostgresSpecTest.java | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java index c7c3e62184d8..0c5bd6e5f3e3 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java @@ -23,7 +23,7 @@ public class CdcInitialSnapshotPostgresSourceDatatypeTest extends AbstractPostgr private static final String SCHEMA_NAME = "test"; private static final String SLOT_NAME_BASE = "debezium_slot"; private static final String PUBLICATION = "publication"; - private static final int INITIAL_WAITING_SECONDS = 5; + private static final int INITIAL_WAITING_SECONDS = 30; @Override protected Database setupDatabase() throws Exception { diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java index f092c56f8a5e..6b18911e752a 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java @@ -45,7 +45,7 @@ public class CdcPostgresSourceAcceptanceTest extends AbstractPostgresSourceAccep private static final String STREAM_NAME = "id_and_name"; private static final String STREAM_NAME2 = "starships"; protected static final String PUBLICATION = "publication"; - protected static final int INITIAL_WAITING_SECONDS = 5; + protected static final int INITIAL_WAITING_SECONDS = 30; protected PostgreSQLContainer container; protected JsonNode config; diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java index ef8230a7505c..b591065733b8 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java @@ -30,7 +30,7 @@ public class CdcWalLogsPostgresSourceDatatypeTest extends AbstractPostgresSource private static final String SCHEMA_NAME = "test"; private static final String SLOT_NAME_BASE = "debezium_slot"; private static final String PUBLICATION = "publication"; - private static final int INITIAL_WAITING_SECONDS = 15; + private static final int INITIAL_WAITING_SECONDS = 30; private JsonNode stateAfterFirstSync; @Override diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java index c8999b2082fe..0bf12801b9e4 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java @@ -75,7 +75,7 @@ public class CdcPostgresSourceTest extends CdcSourceTest { protected static final String SLOT_NAME_BASE = "debezium_slot"; protected static final String PUBLICATION = "publication"; - protected static final int INITIAL_WAITING_SECONDS = 5; + protected static final int INITIAL_WAITING_SECONDS = 30; private PostgreSQLContainer container; protected String dbName; diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCdcGetPublicizedTablesTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCdcGetPublicizedTablesTest.java index fd1bb6473d70..4240c96f53ab 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCdcGetPublicizedTablesTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCdcGetPublicizedTablesTest.java @@ -41,7 +41,7 @@ class PostgresCdcGetPublicizedTablesTest { private static final String SCHEMA_NAME = "public"; private static final String PUBLICATION = "publication_test_12"; private static final String REPLICATION_SLOT = "replication_slot_test_12"; - protected static final int INITIAL_WAITING_SECONDS = 5; + protected static final int INITIAL_WAITING_SECONDS = 30; private static PostgreSQLContainer container; private JsonNode config; diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSpecTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSpecTest.java index 750b320f0d81..0124f1e62d34 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSpecTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSpecTest.java @@ -140,7 +140,7 @@ void testLsnCommitBehaviourProperty(final String commitBehaviour) { .put("replication_slot", "replication_slot") .put("publication", "PUBLICATION") .put("plugin", "pgoutput") - .put("initial_waiting_seconds", 5) + .put("initial_waiting_seconds", 30) .put("lsn_commit_behaviour", commitBehaviour) .build()); From 279b461fe8a491823be2780b6442127c4830e3a4 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 10 May 2023 13:45:50 +0200 Subject: [PATCH 13/15] remove secrets from run-dagger-pipeline/action.yml --- .github/actions/run-dagger-pipeline/action.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml index fb553e050c24..d1d7aaae9b7f 100644 --- a/.github/actions/run-dagger-pipeline/action.yml +++ b/.github/actions/run-dagger-pipeline/action.yml @@ -52,5 +52,3 @@ runs: CI_GIT_REVISION: ${{ github.event.pull_request.head.sha }} CI_CONTEXT: "${{ inputs.context }}" CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} - S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} From c03db6bea5bccb4e88ab6f6725d3505062175362 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 10 May 2023 14:16:29 +0200 Subject: [PATCH 14/15] set CI env var on gradle container --- .../pipelines/actions/environments.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py index 682ff7c1aa40..ec3daf531684 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py @@ -408,17 +408,24 @@ def with_gradle( .with_env_variable("VERSION", "23.0.1") .with_exec(["sh", "-c", "curl -fsSL https://get.docker.com | sh"]) .with_exec(["mkdir", "/root/.gradle"]) - .with_mounted_cache("/root/.gradle", root_gradle_cache, sharing=CacheSharingMode.LOCKED) - .with_exec(["rm", "-f", "/root/.gradle/caches/journal-1/journal-1.lock"]) .with_exec(["mkdir", "/airbyte"]) .with_mounted_directory("/airbyte", context.get_repo_dir(".", include=include)) .with_mounted_cache("/airbyte/.gradle", airbyte_gradle_cache, sharing=CacheSharingMode.LOCKED) .with_workdir("/airbyte") - .with_secret_variable( - "S3_BUILD_CACHE_ACCESS_KEY_ID", context.dagger_client.host().env_variable("S3_BUILD_CACHE_ACCESS_KEY_ID").secret() - ) - .with_secret_variable("S3_BUILD_CACHE_SECRET_KEY", context.dagger_client.host().env_variable("S3_BUILD_CACHE_SECRET_KEY").secret()) ) + if context.is_ci: + openjdk_with_docker = ( + openjdk_with_docker.with_env_variable("CI", "true") + .with_secret_variable( + "S3_BUILD_CACHE_ACCESS_KEY_ID", context.dagger_client.host().env_variable("S3_BUILD_CACHE_ACCESS_KEY_ID").secret() + ) + .with_secret_variable( + "S3_BUILD_CACHE_SECRET_KEY", context.dagger_client.host().env_variable("S3_BUILD_CACHE_SECRET_KEY").secret() + ) + ) + else: + openjdk_with_docker = openjdk_with_docker.with_mounted_cache("/root/.gradle", root_gradle_cache, sharing=CacheSharingMode.LOCKED) + if bind_to_docker_host: return with_bound_docker_host(context, openjdk_with_docker, shared_tmp_volume, docker_service_name=docker_service_name) else: From 275e4776a0a937b7094109c502d84f5621815f55 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 10 May 2023 14:50:38 +0200 Subject: [PATCH 15/15] remove retry from build.gradle --- .../connectors/source-postgres/build.gradle | 7 ------- 1 file changed, 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index ec20ebde8155..ebfac92a1fb5 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -4,7 +4,6 @@ plugins { id 'airbyte-integration-test-java' id 'airbyte-performance-test-java' id 'airbyte-connector-acceptance-test' - id 'org.gradle.test-retry' version '1.5.2' } application { @@ -39,9 +38,3 @@ dependencies { integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) performanceTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } - -test { - retry { - maxRetries = 3 - } -} \ No newline at end of file