-
Notifications
You must be signed in to change notification settings - Fork 4.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
connectors-ci: fix postgres integration testing #25942
Changes from all commits
bd5be9b
8eedd50
bae1c14
70a16b0
bbab004
6199f35
36087e7
56e6eee
25d3456
ebc4665
bda7e31
b3e5e10
93597e7
1a1721c
761796f
a8a788c
279b461
c03db6b
275e477
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,6 +17,7 @@ | |
import io.airbyte.db.factory.DatabaseDriver; | ||
import io.airbyte.integrations.base.ssh.SshHelpers; | ||
import io.airbyte.integrations.standardtest.source.TestDestinationEnv; | ||
import io.airbyte.integrations.util.HostPortResolver; | ||
import io.airbyte.protocol.models.Field; | ||
import io.airbyte.protocol.models.JsonSchemaType; | ||
import io.airbyte.protocol.models.v0.CatalogHelpers; | ||
|
@@ -60,11 +61,10 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc | |
final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() | ||
.put("method", "Standard") | ||
.build()); | ||
final var containerOuterAddress = SshHelpers.getOuterContainerAddress(container); | ||
final var containerInnerAddress = SshHelpers.getInnerContainerAddress(container); | ||
|
||
config = Jsons.jsonNode(ImmutableMap.builder() | ||
.put("host", containerInnerAddress.left) | ||
.put("port", containerInnerAddress.right) | ||
.put("host", HostPortResolver.resolveHost(container)) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is this going to return the inner address and port of a container? This was the reason tests were originally failing when run on Mac.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. thanks |
||
.put("port", HostPortResolver.resolvePort(container)) | ||
.put("database", container.getDatabaseName()) | ||
.put("schemas", Jsons.jsonNode(List.of("public"))) | ||
.put("username", "postgres") | ||
|
@@ -79,8 +79,8 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc | |
config.get("password").asText(), | ||
DatabaseDriver.POSTGRESQL.getDriverClassName(), | ||
String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), | ||
containerOuterAddress.left, | ||
containerOuterAddress.right, | ||
container.getHost(), | ||
container.getFirstMappedPort(), | ||
config.get("database").asText()), | ||
SQLDialect.POSTGRES)) { | ||
final Database database = new Database(dslContext); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -23,7 +23,7 @@ public class CdcInitialSnapshotPostgresSourceDatatypeTest extends AbstractPostgr | |
private static final String SCHEMA_NAME = "test"; | ||
private static final String SLOT_NAME_BASE = "debezium_slot"; | ||
private static final String PUBLICATION = "publication"; | ||
private static final int INITIAL_WAITING_SECONDS = 5; | ||
private static final int INITIAL_WAITING_SECONDS = 30; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is the difference on dagger expected? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I faced flaky results on my local machine too and increasing the waiting time mitigate these problems on both local and dagger. I found a PR from Sergio on another connector that did it for the same reasons. |
||
|
||
@Override | ||
protected Database setupDatabase() throws Exception { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -408,13 +408,24 @@ def with_gradle( | |
.with_env_variable("VERSION", "23.0.1") | ||
.with_exec(["sh", "-c", "curl -fsSL https://get.docker.com | sh"]) | ||
.with_exec(["mkdir", "/root/.gradle"]) | ||
.with_mounted_cache("/root/.gradle", root_gradle_cache, sharing=CacheSharingMode.LOCKED) | ||
.with_exec(["rm", "-f", "/root/.gradle/caches/journal-1/journal-1.lock"]) | ||
.with_exec(["mkdir", "/airbyte"]) | ||
.with_mounted_directory("/airbyte", context.get_repo_dir(".", include=include)) | ||
.with_mounted_cache("/airbyte/.gradle", airbyte_gradle_cache, sharing=CacheSharingMode.LOCKED) | ||
.with_workdir("/airbyte") | ||
) | ||
if context.is_ci: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @alafanechere Can we leave a comment around this that explains
|
||
openjdk_with_docker = ( | ||
openjdk_with_docker.with_env_variable("CI", "true") | ||
.with_secret_variable( | ||
"S3_BUILD_CACHE_ACCESS_KEY_ID", context.dagger_client.host().env_variable("S3_BUILD_CACHE_ACCESS_KEY_ID").secret() | ||
) | ||
.with_secret_variable( | ||
"S3_BUILD_CACHE_SECRET_KEY", context.dagger_client.host().env_variable("S3_BUILD_CACHE_SECRET_KEY").secret() | ||
) | ||
) | ||
else: | ||
openjdk_with_docker = openjdk_with_docker.with_mounted_cache("/root/.gradle", root_gradle_cache, sharing=CacheSharingMode.LOCKED) | ||
|
||
if bind_to_docker_host: | ||
return with_bound_docker_host(context, openjdk_with_docker, shared_tmp_volume, docker_service_name=docker_service_name) | ||
else: | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is going to break the clickhouse integration tests on Mac I think.
The original change was #14701 back in July last year
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
source-clickhouse that is