diff --git a/README.md b/README.md index aa95624..f18a594 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Installation Add the following dependency to your project's build.gradle file: ```kts -implementation('app.meetacy.wdater:wdater:$wdaterVersion') +implementation("app.meetacy.wdater:wdater:$wdaterVersion") ``` Replace `$wdaterVersion` with the latest version from release. @@ -81,13 +81,7 @@ Usage ```kotlin object UsersTable : Table() { - val USER_ID = long("USER_ID").autoIncrement() - val ACCESS_HASH = varchar("ACCESS_HASH", length = HASH_LENGTH) - val NICKNAME = varchar("NICKNAME", length = NICKNAME_MAX_LIMIT) val USERNAME = varchar("USERNAME", length = USERNAME_MAX_LIMIT).nullable() - val EMAIL = varchar("EMAIL", length = EMAIL_MAX_LIMIT).nullable() - val EMAIL_VERIFIED = bool("EMAIL_VERIFIED").default(false) - val AVATAR_ID = long("AVATAR_ID").nullable() } ``` @@ -101,6 +95,24 @@ Usage You can also pass a list of migrations to update multiple steps at once. +Auto Migrations +--------------- + +To use automatic migrations, you need to include this dependency: + +```kts +implementation("app.meetacy.wdater:auto-migrations:$wdaterVersion") +``` + +Then you can use this factory function to create a migration: + +```kotlin +fun AutoMigration( + vararg tables: Table, + fromVersion: Int, + toVersion: Int = fromVersion + 1 +): AutoMigration +``` Future Plans for Wdater ----------------------- diff --git a/auto-migrations/build.gradle.kts b/auto-migrations/build.gradle.kts new file mode 100644 index 0000000..2620b57 --- /dev/null +++ b/auto-migrations/build.gradle.kts @@ -0,0 +1,10 @@ +plugins { + id("library-convention") +} + +version = libs.versions.wdater.get() + +dependencies { + implementation(libs.exposedCore) + api(projects.wdater) +} diff --git a/auto-migrations/src/main/kotlin/app/meetacy/database/updater/auto/AutoMigration.kt b/auto-migrations/src/main/kotlin/app/meetacy/database/updater/auto/AutoMigration.kt new file mode 100644 index 0000000..168958e --- /dev/null +++ b/auto-migrations/src/main/kotlin/app/meetacy/database/updater/auto/AutoMigration.kt @@ -0,0 +1,169 @@ +package app.meetacy.database.updater.auto + +import app.meetacy.database.updater.Migration +import app.meetacy.database.updater.MigrationContext +import app.meetacy.database.updater.log.Logger +import app.meetacy.database.updater.log.SQL +import org.jetbrains.exposed.sql.* +import org.jetbrains.exposed.sql.vendors.ColumnMetadata + +public fun AutoMigration( + vararg tables: Table, + fromVersion: Int, + toVersion: Int = fromVersion + 1 +): AutoMigration = AutoMigration(tables.toList(), fromVersion, toVersion) + +/** + * Auto migrations do not support renaming/deleting + */ +public class AutoMigration( + private val tables: List, + override val fromVersion: Int, + override val toVersion: Int = fromVersion + 1 +) : Migration { + + override suspend fun MigrationContext.migrate() { + logger.log("Trying to auto migrate from current state") + migrateTables() + migrateColumns() + } + + private fun MigrationContext.migrateTables() { + logger.log("Migrating tables...") + val logger = logger["tables-migration"] + + val rawNames = transaction.db.dialect.allTablesNames() + logger.log("Tables from server: $rawNames") + val schema = transaction.connection.schema + logger.log("Default schema: $schema") + + val tablesOnServer = rawNames.map { name -> name.removePrefix("$schema.") } + + createMissingTables(logger, tablesOnServer) + } + + private fun MigrationContext.createMissingTables( + baseLogger: Logger, + tablesOnServer: List + ) { + baseLogger.log("Creating missing tables...") + val logger = baseLogger["create"] + + val newTables = tables.filter { table -> + tablesOnServer.none { name -> normalizedTableName(table) == name } + } + + if (newTables.isEmpty()) { + logger.log("No new tables were found") + return + } + + logger.log("New tables were found: $newTables, creating them...") + + val create = SchemaUtils.createStatements(*newTables.toTypedArray()) + execInBatch(logger, create) + + logger.log("New tables were created") + } + + private fun MigrationContext.migrateColumns() { + logger.log("Migrating columns...") + val logger = logger["columns-migration"] + val columnsOnServer = transaction.db.dialect.tableColumns(*tables.toTypedArray()) + createMissingColumns(logger, columnsOnServer) + val updatedColumnOnServer = transaction.db.dialect.tableColumns(*tables.toTypedArray()) + modifyAllColumns(logger, updatedColumnOnServer) + logger.log("Migration completed!") + } + + private fun MigrationContext.createMissingColumns( + baseLogger: Logger, + databaseColumns: Map> + ) { + baseLogger.log("Creating missing columns...") + val parentLogger = baseLogger["create"] + + tables.forEach { table -> + parentLogger.log("Working on: ${normalizedTableName(table)}") + + val logger = parentLogger[normalizedTableName(table)] + val columnsOnServer = databaseColumns.getValue(table) + + val newColumns = table.columns.filter { column -> + columnsOnServer.none { metadata -> normalizedColumnName(column) == metadata.name } + } + + if (newColumns.isEmpty()) { + logger.log("No new columns were found") + return@forEach + } + + logger.log("New columns were found: ${newColumns.map { normalizedColumnName(it) } }, creating them...") + + val create = newColumns.flatMap { column -> column.createStatement() } + execInBatch(logger, create) + + logger.log("New columns were created") + } + } + + private fun MigrationContext.modifyAllColumns( + baseLogger: Logger, + databaseColumns: Map> + ) { + baseLogger.log("Modifying all columns...") + val parentLogger = baseLogger["modify-all"] + + tables.forEach { table -> + parentLogger.log("Working on ${normalizedTableName(table)}") + val logger = parentLogger[normalizedTableName(table)] + + val columnsOnServer = databaseColumns.getValue(table) + + val modify = table.columns.map { column -> + column to columnsOnServer.first { it.name == normalizedColumnName(column) } + }.flatMap { (column, columnOnServer) -> + // exposed doesn't support migration of auto inc + if (column.columnType.isAutoInc) return@flatMap emptyList() + + column.modifyStatements( + ColumnDiff( + nullability = column.columnType.nullable != columnOnServer.nullable, + autoInc = false, + defaults = true, + caseSensitiveName = true + ) + ) + } + execInBatch(logger, modify) + logger.log("Columns were modified") + } + } + + private fun MigrationContext.execInBatch( + baseLogger: Logger, + statements: List + ) { + statements.forEach(baseLogger.SQL::log) + transaction.execInBatch(statements) + } + + private fun MigrationContext.normalizedTableName(table: Table): String { + return table.nameInDatabaseCase().removePrefix("${transaction.connection.schema}.") + } + + private fun MigrationContext.normalizedColumnName(column: Column<*>): String { + return column.nameInDatabaseCase() + } + + override val displayName: String = "AutoMigration" + + private fun MigrationContext.cannotMigrate(): Nothing { + transaction.rollback() + throw CannotAutoMigrateException() + } + + public class CannotAutoMigrateException : RuntimeException( + /* message = */"Ambiguity occurred while migrating, cannot make auto migration for current state" + ) +} diff --git a/auto-migrations/src/main/kotlin/app/meetacy/database/updater/auto/NoOpColumnType.kt b/auto-migrations/src/main/kotlin/app/meetacy/database/updater/auto/NoOpColumnType.kt new file mode 100644 index 0000000..b82f56a --- /dev/null +++ b/auto-migrations/src/main/kotlin/app/meetacy/database/updater/auto/NoOpColumnType.kt @@ -0,0 +1,7 @@ +package app.meetacy.database.updater.auto + +import org.jetbrains.exposed.sql.ColumnType + +internal object NoOpColumnType : ColumnType() { + override fun sqlType(): String = error("No operation") +} diff --git a/auto-migrations/src/main/kotlin/app/meetacy/database/updater/auto/Table.kt b/auto-migrations/src/main/kotlin/app/meetacy/database/updater/auto/Table.kt new file mode 100644 index 0000000..53d86f3 --- /dev/null +++ b/auto-migrations/src/main/kotlin/app/meetacy/database/updater/auto/Table.kt @@ -0,0 +1,13 @@ +package app.meetacy.database.updater.auto + +import org.jetbrains.exposed.sql.Table + +internal fun Table.dropColumnStatement(columnName: String): List { + val column = Table(tableName).registerColumn(columnName, NoOpColumnType) + return column.dropStatement() +} + +internal fun Table.createColumnStatement(columnName: String): List { + val column = Table(tableName).registerColumn(columnName, NoOpColumnType) + return column.createStatement() +} diff --git a/auto-migrations/src/main/kotlin/app/meetacy/database/updater/auto/TableName.kt b/auto-migrations/src/main/kotlin/app/meetacy/database/updater/auto/TableName.kt new file mode 100644 index 0000000..5d18460 --- /dev/null +++ b/auto-migrations/src/main/kotlin/app/meetacy/database/updater/auto/TableName.kt @@ -0,0 +1,11 @@ +package app.meetacy.database.updater.auto + +internal data class TableName( + val schema: Schema, + val name: String +) { + sealed interface Schema { + object Default : Schema + data class Custom(val string: String) : Schema + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index df56e08..15547b6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,7 +2,7 @@ kotlin = "1.8.10" exposed = "0.41.1" -wdater = "0.0.1" +wdater = "0.0.2" [libraries] diff --git a/settings.gradle.kts b/settings.gradle.kts index 0cc1820..920734f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -17,3 +17,5 @@ dependencyResolutionManagement { } includeBuild("build-logic") + +include("auto-migrations") diff --git a/src/main/kotlin/app/meetacy/database/updater/Migration.kt b/src/main/kotlin/app/meetacy/database/updater/Migration.kt index 87e71e0..676e1ff 100644 --- a/src/main/kotlin/app/meetacy/database/updater/Migration.kt +++ b/src/main/kotlin/app/meetacy/database/updater/Migration.kt @@ -15,6 +15,11 @@ public interface Migration { */ public val toVersion: Int get() = fromVersion + 1 + /** + * The string used as name when printing this migration + */ + public val displayName: String get() = "Migration" + /** * Function that defines the migration logic */ diff --git a/src/main/kotlin/app/meetacy/database/updater/MigrationContext.kt b/src/main/kotlin/app/meetacy/database/updater/MigrationContext.kt index 225a5c0..b5d15eb 100644 --- a/src/main/kotlin/app/meetacy/database/updater/MigrationContext.kt +++ b/src/main/kotlin/app/meetacy/database/updater/MigrationContext.kt @@ -1,5 +1,6 @@ package app.meetacy.database.updater +import app.meetacy.database.updater.log.Logger import org.jetbrains.exposed.sql.Column import org.jetbrains.exposed.sql.Table.Dual.default import org.jetbrains.exposed.sql.Transaction @@ -10,7 +11,10 @@ import org.jetbrains.exposed.sql.Transaction * * @param transaction The database transaction in which the migration is executed. */ -public class MigrationContext(public val transaction: Transaction) { +public class MigrationContext( + public val transaction: Transaction, + public val logger: Logger +) { /** * Creates the column in the associated table. diff --git a/src/main/kotlin/app/meetacy/database/updater/Wdater.kt b/src/main/kotlin/app/meetacy/database/updater/Wdater.kt index c86b5a1..f25f6c9 100644 --- a/src/main/kotlin/app/meetacy/database/updater/Wdater.kt +++ b/src/main/kotlin/app/meetacy/database/updater/Wdater.kt @@ -1,6 +1,8 @@ package app.meetacy.database.updater import kotlinx.coroutines.Dispatchers +import org.jetbrains.exposed.sql.name +import org.jetbrains.exposed.sql.transactions.TransactionManager import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction /** @@ -22,6 +24,7 @@ public fun Wdater(block: WdaterConfig.Builder.() -> Unit): Wdater { public class Wdater(private val config: WdaterConfig = WdaterConfig()) { private val db = config.database private val storage = config.storage + private val logger = config.logger["wdater"] /** * Update the database with the specified migrations. @@ -38,25 +41,54 @@ public class Wdater(private val config: WdaterConfig = WdaterConfig()) { * @param migrations The list of [Migration] instances representing the migrations to be executed. */ public suspend fun update(migrations: List) { + logDependencies(migrations) + logger.log("Process of database migration started") val fromVersion = storage.getSchemaVersion() - // Skip migrations if there is no version if (fromVersion == null) { + logger.log("No schema version was found, initializing database...") initializeDatabase(migrations) } else { + logger.log("Detected schema version is: $fromVersion, running migrations...") val migratedVersion = migrate(fromVersion, migrations) + logger.log("Saving the final version of schema using storage") storage.setSchemaVersion(migratedVersion) } + logger.log("Completed!") } private suspend fun initializeDatabase(migrations: List) { - val maxVersion = migrations.maxOfOrNull { it.toVersion } ?: config.defaultSchemaVersion + val logger = logger["db-init"] + + val maxMigrationsVersion = migrations.maxOfOrNull { it.toVersion } + + if (maxMigrationsVersion == null) { + logger.log("There is no migrations, so I will use config.defaultSchemaVersion (${config.defaultSchemaVersion}) as the latest schema version") + } else { + logger.log("The latest schema version was found in migrations: $maxMigrationsVersion") + } + + val maxVersion = maxMigrationsVersion ?: config.defaultSchemaVersion + + logger.log("Saving $maxVersion schema version using storage") storage.setSchemaVersion(maxVersion) + + if (config.initializer is DatabaseInitializer.Empty) { + logger.log("No initializer provided") + } else { + logger.log("Running custom initializer...") + } + newSuspendedTransaction(Dispatchers.IO, db) { with(config.initializer) { - MigrationContext(transaction = this@newSuspendedTransaction).initialize() + MigrationContext( + transaction = this@newSuspendedTransaction, + logger = logger + ).initialize() } } + + logger.log("Completed!") } /** @@ -70,20 +102,63 @@ public class Wdater(private val config: WdaterConfig = WdaterConfig()) { fromVersion: Int, migrations: List ): Int { + val logger = logger["migrating"] + // If we did not find any migration to perform, assume we should stop there val migration = migrations.find { migration -> migration.fromVersion == fromVersion - } ?: return fromVersion + } + + if (migration == null) { + logger.log("No migrations were found, therefore the ending point of migration is $fromVersion") + return fromVersion + } + + val migrationLogger = logger[stringifyMigration(migration)] + migrationLogger.log("Migration found, running it...") newSuspendedTransaction(Dispatchers.IO, db) { with(migration) { - MigrationContext(transaction = this@newSuspendedTransaction).migrate() + MigrationContext( + transaction = this@newSuspendedTransaction, + logger = logger[stringifyMigration(migration)] + ).migrate() } } + migrationLogger.log("Migration completed, current schema version is ${migration.toVersion}") + logger.log("Searching for the next migration...") + return migrate(migration.toVersion, migrations) } + private fun logDependencies(migrations: List) { + val logger = logger["dependencies"] + + if (db == null) { + logger.log("* Database: ${TransactionManager.defaultDatabase?.name} (from global object)") + } else { + logger.log("* Database: ${db.name} (passed explicitly)") + } + + if (storage is WdaterTable) { + logger.log("* Migrations Storage: Default implementation is used for writing, table '${storage.tableName}'") + } else { + logger.log("* Migrations Storage: Custom implementation") + } + + val initializerEmpty = config.initializer is DatabaseInitializer.Empty + logger.log("* Initializer: ${if (initializerEmpty) "unspecified" else "specified"}") + + logger.log("* Migrations: ${stringifyMigrations(migrations)}") + } + + private fun stringifyMigrations(migrations: List): String = + migrations.map(::stringifyMigration).toString() + + private fun stringifyMigration(migration: Migration): String = + "${migration.displayName}{${migration.fromVersion} > ${migration.toVersion}}" + /** * Configure the [Wdater] instance using the provided configuration block. * diff --git a/src/main/kotlin/app/meetacy/database/updater/WdaterConfig.kt b/src/main/kotlin/app/meetacy/database/updater/WdaterConfig.kt index f530129..ef0b9cf 100644 --- a/src/main/kotlin/app/meetacy/database/updater/WdaterConfig.kt +++ b/src/main/kotlin/app/meetacy/database/updater/WdaterConfig.kt @@ -1,5 +1,6 @@ package app.meetacy.database.updater +import app.meetacy.database.updater.log.Logger import org.jetbrains.exposed.sql.Database /** @@ -9,17 +10,25 @@ import org.jetbrains.exposed.sql.Database * @property storage The storage implementation for managing migrations. * @property defaultSchemaVersion The default schema version. * @property initializer The initializer for performing database initialization. + * @property logger The logger implementation to log process of migration. */ public class WdaterConfig( public val database: Database? = null, public val storage: WdaterStorage = WdaterTable(database = database), public val defaultSchemaVersion: Int = 0, - public val initializer: DatabaseInitializer = DatabaseInitializer.Empty + public val initializer: DatabaseInitializer = DatabaseInitializer.Empty, + public val logger: Logger = Logger.Simple() ) { /** * Creates a new [Builder] instance to configure the [WdaterConfig]. */ - public fun builder(): Builder = Builder(database, storage, defaultSchemaVersion) + public fun builder(): Builder = Builder( + database = database, + storage = storage, + defaultSchemaVersion = defaultSchemaVersion, + initializer = initializer, + logger = logger + ) /** * Builder class for configuring the [WdaterConfig]. @@ -28,7 +37,8 @@ public class WdaterConfig( public var database: Database? = null, public var storage: WdaterStorage = WdaterTable(database = database), public var defaultSchemaVersion: Int = 0, - public var initializer: DatabaseInitializer = DatabaseInitializer.Empty + public var initializer: DatabaseInitializer = DatabaseInitializer.Empty, + public var logger: Logger = Logger.Simple() ) { /** * Creates the table storage configuration for managing migrations. @@ -66,7 +76,8 @@ public class WdaterConfig( database = database, storage = storage, defaultSchemaVersion = defaultSchemaVersion, - initializer = initializer + initializer = initializer, + logger = logger ) } } diff --git a/src/main/kotlin/app/meetacy/database/updater/log/Logger.kt b/src/main/kotlin/app/meetacy/database/updater/log/Logger.kt new file mode 100644 index 0000000..88638cd --- /dev/null +++ b/src/main/kotlin/app/meetacy/database/updater/log/Logger.kt @@ -0,0 +1,57 @@ +package app.meetacy.database.updater.log + +import java.text.SimpleDateFormat +import java.util.* + +public interface Logger { + public fun log(message: String = "") + + public operator fun get(tag: String): Logger + + public class Simple( + private val includeTimestamp: Boolean = true, + private val tag: String? = null, + private val delimiter: String = " > " + ) : Logger { + override fun log(message: String) { + simpleLog(message, includeTimestamp, tag) + } + + override fun get(tag: String): Logger { + this.tag ?: return Simple(includeTimestamp, tag, delimiter) + return Simple(includeTimestamp, tag = "${this.tag}$delimiter$tag", delimiter) + } + } + + public object None : Logger { + override fun get(tag: String): None = this + + override fun log(message: String) {} + } +} + +private val format = SimpleDateFormat("dd-MM-yyyy/hh:mm:ss") + +private fun simpleLog( + message: String, + includeTimestamp: Boolean, + tag: String? +) { + val prettyDate = format.format(Date()) + + val log = buildString { + if (includeTimestamp) { + append(prettyDate) + } + if (tag != null) { + if (includeTimestamp) append(" ") + append("[$tag]") + } + if (includeTimestamp || tag != null) { + append(": ") + } + append(message) + } + + println(log) +} diff --git a/src/main/kotlin/app/meetacy/database/updater/log/SQL.kt b/src/main/kotlin/app/meetacy/database/updater/log/SQL.kt new file mode 100644 index 0000000..b4b2b4b --- /dev/null +++ b/src/main/kotlin/app/meetacy/database/updater/log/SQL.kt @@ -0,0 +1,3 @@ +package app.meetacy.database.updater.log + +public val Logger.SQL: Logger get() = this["SQL"]