Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ensure database initialization in test container #17697

Merged
merged 13 commits into from
Oct 13, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
import org.jooq.DSLContext;
import org.jooq.SQLDialect;
import org.testcontainers.containers.PostgreSQLContainer;
import org.testcontainers.ext.ScriptUtils;
import org.testcontainers.jdbc.JdbcDatabaseDelegate;

/**
* Helper class for migration development. See README for details.
Expand Down Expand Up @@ -47,6 +49,9 @@ private static PostgreSQLContainer<?> createContainer() {
.withUsername("docker")
.withPassword("docker");
container.start();

initializeDatabase(container);

return container;
}

Expand Down Expand Up @@ -106,6 +111,12 @@ private void dumpSchema() {
}
}

private static void initializeDatabase(final PostgreSQLContainer container) {
final var containerDelegate = new JdbcDatabaseDelegate(container, "");
ScriptUtils.runInitScript(containerDelegate, "configs_database/schema.sql");
ScriptUtils.runInitScript(containerDelegate, "jobs_database/schema.sql");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jdpgrailsdev, I think this is incorrect. Each migration dev center should run specific its own init script, i.e. the config migration dev center should only run the configs initial script.

Currently when I run schema dump for one database, it will include the schemas for both configs and jobs database.

The schema dump is still correct in production because it is updated by ConfigsDatabaseMigratorTest and JobsDatabaseMigratorTest, which only does not have this behavior.

}

public static void main(final String[] args) {
final MigrationDevCenter devCenter;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,25 @@ create table "public"."airbyte_configs_migrations"(
constraint "airbyte_configs_migrations_pk"
primary key ("installed_rank")
);
create table "public"."airbyte_metadata"(
"key" varchar(255) not null,
"value" varchar(255) null,
constraint "airbyte_metadata_pkey"
primary key ("key")
);
create table "public"."attempts"(
"id" int8 generated by default as identity not null,
"job_id" int8 null,
"attempt_number" int4 null,
"log_path" varchar(255) null,
"output" jsonb null,
"status" any null,
"created_at" timestamptz(35) null,
"updated_at" timestamptz(35) null,
"ended_at" timestamptz(35) null,
constraint "attempts_pkey"
primary key ("id")
);
create table "public"."connection"(
"id" uuid not null,
"namespace_definition" namespace_definition_type not null,
Expand Down Expand Up @@ -124,6 +143,18 @@ create table "public"."connection_operation"(
"operation_id"
)
);
create table "public"."jobs"(
"id" int8 generated by default as identity not null,
"config_type" any null,
"scope" varchar(255) null,
"config" jsonb null,
"status" any null,
"started_at" timestamptz(35) null,
"created_at" timestamptz(35) null,
"updated_at" timestamptz(35) null,
constraint "jobs_pkey"
primary key ("id")
);
create table "public"."operation"(
"id" uuid not null,
"workspace_id" uuid not null,
Expand Down Expand Up @@ -273,6 +304,12 @@ create index "actor_oauth_parameter_workspace_definition_idx" on "public"."actor
);
create unique index "airbyte_configs_migrations_pk" on "public"."airbyte_configs_migrations"("installed_rank" asc);
create index "airbyte_configs_migrations_s_idx" on "public"."airbyte_configs_migrations"("success" asc);
create unique index "airbyte_metadata_pkey" on "public"."airbyte_metadata"("key" asc);
create unique index "attempts_pkey" on "public"."attempts"("id" asc);
create unique index "job_attempt_idx" on "public"."attempts"(
"job_id" asc,
"attempt_number" asc
);
create index "connection_destination_id_idx" on "public"."connection"("destination_id" asc);
create unique index "connection_pkey" on "public"."connection"("id" asc);
create index "connection_source_id_idx" on "public"."connection"("source_id" asc);
Expand All @@ -282,6 +319,7 @@ create unique index "connection_operation_pkey" on "public"."connection_operatio
"connection_id" asc,
"operation_id" asc
);
create unique index "jobs_pkey" on "public"."jobs"("id" asc);
create unique index "operation_pkey" on "public"."operation"("id" asc);
create unique index "state__connection_id__stream_name__namespace__uq" on "public"."state"(
"connection_id" asc,
Expand Down
16 changes: 16 additions & 0 deletions airbyte-db/db-lib/src/main/resources/jobs_database/schema_dump.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,16 @@
// It is also not used by any piece of code to generate anything.
// It doesn't contain the enums created in the database and the default values might also be buggy.

create table "public"."airbyte_configs"(
"id" int8 generated by default as identity not null,
"config_id" varchar(36) not null,
"config_type" varchar(60) not null,
"config_blob" jsonb not null,
"created_at" timestamptz(35) not null default null,
"updated_at" timestamptz(35) not null default null,
constraint "airbyte_configs_pkey"
primary key ("id")
);
create table "public"."airbyte_jobs_migrations"(
"installed_rank" int4 not null,
"version" varchar(50) null,
Expand Down Expand Up @@ -86,6 +96,12 @@ alter table "public"."sync_stats"
add constraint "sync_stats_attempt_id_fkey"
foreign key ("attempt_id")
references "public"."attempts" ("id");
create index "airbyte_configs_id_idx" on "public"."airbyte_configs"("config_id" asc);
create unique index "airbyte_configs_pkey" on "public"."airbyte_configs"("id" asc);
create unique index "airbyte_configs_type_id_idx" on "public"."airbyte_configs"(
"config_type" asc,
"config_id" asc
);
create unique index "airbyte_jobs_migrations_pk" on "public"."airbyte_jobs_migrations"("installed_rank" asc);
create index "airbyte_jobs_migrations_s_idx" on "public"."airbyte_jobs_migrations"("success" asc);
create unique index "airbyte_metadata_pkey" on "public"."airbyte_metadata"("key" asc);
Expand Down