diff --git a/api/src/main/java/marquez/MarquezApp.java b/api/src/main/java/marquez/MarquezApp.java index 37a7b2c16e..15406d95c2 100644 --- a/api/src/main/java/marquez/MarquezApp.java +++ b/api/src/main/java/marquez/MarquezApp.java @@ -28,7 +28,7 @@ import marquez.api.filter.JobRedirectFilter; import marquez.api.filter.exclusions.Exclusions; import marquez.api.filter.exclusions.ExclusionsConfig; -import marquez.cli.DbMigrationCommand; +import marquez.cli.DbMigrateCommand; import marquez.cli.DbRetentionCommand; import marquez.cli.MetadataCommand; import marquez.cli.SeedCommand; @@ -90,6 +90,7 @@ public void initialize(@NonNull Bootstrap bootstrap) { new EnvironmentVariableSubstitutor(ERROR_ON_UNDEFINED))); // Add CLI commands + bootstrap.addCommand(new DbMigrateCommand()); bootstrap.addCommand(new DbRetentionCommand()); bootstrap.addCommand(new MetadataCommand()); bootstrap.addCommand(new SeedCommand()); @@ -202,12 +203,6 @@ public void registerResources( } } - @Override - protected void addDefaultCommands(Bootstrap bootstrap) { - bootstrap.addCommand(new DbMigrationCommand<>(this)); - super.addDefaultCommands(bootstrap); - } - private void registerServlets(@NonNull Environment env) { log.debug("Registering servlets..."); diff --git a/api/src/main/java/marquez/cli/DbMigrateCommand.java b/api/src/main/java/marquez/cli/DbMigrateCommand.java new file mode 100644 index 0000000000..2b295ff51a --- /dev/null +++ b/api/src/main/java/marquez/cli/DbMigrateCommand.java @@ -0,0 +1,42 @@ +/* + * Copyright 2018-2022 contributors to the Marquez project + * SPDX-License-Identifier: Apache-2.0 + */ + +package marquez.cli; + +import io.dropwizard.cli.ConfiguredCommand; +import io.dropwizard.db.DataSourceFactory; +import io.dropwizard.db.ManagedDataSource; +import io.dropwizard.setup.Bootstrap; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import marquez.MarquezConfig; +import marquez.db.DbMigration; +import net.sourceforge.argparse4j.inf.Namespace; + +/** + * A command to manually run database migrations. This command to be used to run migrations + * decoupled from application deployment. + */ +@Slf4j +public class DbMigrateCommand extends ConfiguredCommand { + + public DbMigrateCommand() { + super("db-migrate", "A command to manually run database migrations."); + } + + @Override + protected void run( + @NonNull Bootstrap bootstrap, + @NonNull Namespace namespace, + @NonNull MarquezConfig configuration) + throws Exception { + + final DataSourceFactory sourceFactory = configuration.getDataSourceFactory(); + final ManagedDataSource source = + sourceFactory.build(bootstrap.getMetricRegistry(), "MarquezApp-source"); + + DbMigration.migrateDbOrError(configuration.getFlywayFactory(), source, true); + } +} diff --git a/api/src/main/java/marquez/cli/DbMigrationCommand.java b/api/src/main/java/marquez/cli/DbMigrationCommand.java deleted file mode 100644 index 34953fddcd..0000000000 --- a/api/src/main/java/marquez/cli/DbMigrationCommand.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2018-2022 contributors to the Marquez project - * SPDX-License-Identifier: Apache-2.0 - */ - -package marquez.cli; - -import io.dropwizard.Application; -import io.dropwizard.cli.EnvironmentCommand; -import io.dropwizard.db.DataSourceFactory; -import io.dropwizard.db.ManagedDataSource; -import io.dropwizard.jdbi3.JdbiFactory; -import io.dropwizard.setup.Environment; -import javax.sql.DataSource; -import lombok.extern.slf4j.Slf4j; -import marquez.db.migrations.V57_1__BackfillFacets; -import net.sourceforge.argparse4j.inf.Namespace; -import net.sourceforge.argparse4j.inf.Subparser; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.jackson2.Jackson2Plugin; -import org.jdbi.v3.postgres.PostgresPlugin; -import org.jdbi.v3.sqlobject.SqlObjectPlugin; - -/** - * A command to manually run database migrations when needed. This migration requires a heavy DB - * operation which can be done asynchronously (with limited API downtime) due to separate migration - * command. - */ -@Slf4j -public class DbMigrationCommand extends EnvironmentCommand { - - private static final String DB_MIGRATE = "db-migrate"; - private static final String MIGRATION_V57_DESCRIPTION = - """ - A command to manually run V57 database migration. - Please refer to https://github.com/MarquezProject/marquez/blob/main/api/src/main/resources/marquez/db/migration/V57__readme.md for more details. - """; - - private static final String COMMAND_DESCRIPTION = - """ - A command to manually run database migrations. - Extra parameters are required to specify the migration to run. - """; - - /** - * Creates a new environment command. - * - * @param application the application providing this command - */ - public DbMigrationCommand(Application application) { - super(application, DB_MIGRATE, COMMAND_DESCRIPTION); - } - - @Override - public void configure(Subparser subparser) { - subparser - .addArgument("--chunkSize") - .dest("chunkSize") - .type(Integer.class) - .required(false) - .setDefault(V57_1__BackfillFacets.DEFAULT_CHUNK_SIZE) - .help("amount of lineage_events rows processed in a single SQL query and transaction."); - - subparser - .addArgument("--version") - .dest("version") - .type(String.class) - .required(true) - .help("migration version to apply like 'v57'"); - - addFileArgument(subparser); - } - - @Override - protected void run( - Environment environment, Namespace namespace, marquez.MarquezConfig configuration) - throws Exception { - - final DataSourceFactory sourceFactory = configuration.getDataSourceFactory(); - final DataSource source = sourceFactory.build(environment.metrics(), "MarquezApp-source"); - final JdbiFactory factory = new JdbiFactory(); - - Jdbi jdbi = - factory - .build( - environment, - configuration.getDataSourceFactory(), - (ManagedDataSource) source, - "postgresql-command") - .installPlugin(new SqlObjectPlugin()) - .installPlugin(new PostgresPlugin()) - .installPlugin(new Jackson2Plugin()); - - MarquezMigrations.valueOf(namespace.getString("version")).run(jdbi, namespace); - } - - enum MarquezMigrations { - v57 { - public void run(Jdbi jdbi, Namespace namespace) throws Exception { - log.info("Running V57_1__BackfillFacets migration"); - V57_1__BackfillFacets migration = new V57_1__BackfillFacets(); - migration.setManual(true); - migration.setJdbi(jdbi); - migration.setChunkSize(namespace.getInt("chunkSize")); - migration.migrate(null); - } - }; - - public void run(Jdbi jdbi, Namespace namespace) throws Exception { - throw new UnsupportedOperationException(); - } - } -} diff --git a/api/src/main/java/marquez/db/DbMigration.java b/api/src/main/java/marquez/db/DbMigration.java index c46ea11bdb..a1c139d8b1 100644 --- a/api/src/main/java/marquez/db/DbMigration.java +++ b/api/src/main/java/marquez/db/DbMigration.java @@ -25,7 +25,7 @@ public static void migrateDbOrError(@NonNull final DataSource source) { public static void migrateDbOrError( @NonNull final FlywayFactory flywayFactory, @NonNull final DataSource source, - final boolean migrateDbOnStartup) { + final boolean migrateNow) { final Flyway flyway = flywayFactory.build(source); // Only attempt a database migration if there are pending changes to be applied, // or on the initialization of a new database. Otherwise, error on pending changes @@ -33,7 +33,7 @@ public static void migrateDbOrError( if (!hasPendingDbMigrations(flyway)) { log.info("No pending migrations found, skipping..."); return; - } else if (!migrateDbOnStartup && hasDbMigrationsApplied(flyway)) { + } else if (!migrateNow && hasDbMigrationsApplied(flyway)) { errorOnPendingDbMigrations(flyway); } // Attempt to perform a database migration. An exception is thrown on failed migration attempts