From 7f962b3b21b00c2b4510734a95dfb53a61d766f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Jan=20Czocha=C5=84ski?= Date: Wed, 15 Mar 2023 14:55:16 +0100 Subject: [PATCH] Load migrations from *.sql files in a directory --- .github/workflows/ci.yml | 15 +++ examples/from-directory/Cargo.toml | 22 +++++ .../migrations/01-friend_car-up.sql | 13 +++ .../migrations/02-add_birthday_column-up.sql | 2 + .../migrations/03-add_animal_table-down.sql | 1 + .../migrations/03-add_animal_table-up.sql | 1 + examples/from-directory/src/main.rs | 62 ++++++++++++ rusqlite_migration/Cargo.toml | 4 + rusqlite_migration/src/asynch.rs | 22 +++++ rusqlite_migration/src/errors.rs | 3 + rusqlite_migration/src/lib.rs | 34 +++++++ rusqlite_migration/src/loader.rs | 95 +++++++++++++++++++ rusqlite_migration_tests/Cargo.toml | 3 +- .../tests/from_dir_test.rs | 34 +++++++ 14 files changed, 310 insertions(+), 1 deletion(-) create mode 100644 examples/from-directory/Cargo.toml create mode 100644 examples/from-directory/migrations/01-friend_car-up.sql create mode 100644 examples/from-directory/migrations/02-add_birthday_column-up.sql create mode 100644 examples/from-directory/migrations/03-add_animal_table-down.sql create mode 100644 examples/from-directory/migrations/03-add_animal_table-up.sql create mode 100644 examples/from-directory/src/main.rs create mode 100644 rusqlite_migration/src/loader.rs create mode 100644 rusqlite_migration_tests/tests/from_dir_test.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 552bfd2..436acb7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,21 @@ jobs: command: test args: --features async-tokio-rusqlite + test_from_directory: + name: Test feature from-directory + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - uses: actions-rs/cargo@v1 + with: + command: test + args: --features from-directory + test_examples: name: Test examples runs-on: ubuntu-latest diff --git a/examples/from-directory/Cargo.toml b/examples/from-directory/Cargo.toml new file mode 100644 index 0000000..ba76dce --- /dev/null +++ b/examples/from-directory/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "example-from-directory" +version = "0.1.0" +edition = "2018" +publish = false + +[dependencies] +rusqlite_migration = { path = "../../rusqlite_migration", features = [ + "from-directory", +] } +log = "0.4" +simple-logging = "2.0.2" +env_logger = "0.10" +anyhow = "1" +lazy_static = "1.4.0" +mktemp = "0.5" +include_dir = "0.7.3" + +[dependencies.rusqlite] +version = ">=0.23.0" +default-features = false +features = [] diff --git a/examples/from-directory/migrations/01-friend_car-up.sql b/examples/from-directory/migrations/01-friend_car-up.sql new file mode 100644 index 0000000..19f4a1e --- /dev/null +++ b/examples/from-directory/migrations/01-friend_car-up.sql @@ -0,0 +1,13 @@ +CREATE TABLE friend( + friend_id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + email TEXT UNIQUE, + phone TEXT UNIQUE, + picture BLOB +); + +CREATE TABLE car( + registration_plate TEXT PRIMARY KEY, + cost REAL NOT NULL, + bought_on TEXT NOT NULL +); diff --git a/examples/from-directory/migrations/02-add_birthday_column-up.sql b/examples/from-directory/migrations/02-add_birthday_column-up.sql new file mode 100644 index 0000000..858d223 --- /dev/null +++ b/examples/from-directory/migrations/02-add_birthday_column-up.sql @@ -0,0 +1,2 @@ +ALTER TABLE friend ADD COLUMN birthday TEXT; +ALTER TABLE friend ADD COLUMN comment TEXT; \ No newline at end of file diff --git a/examples/from-directory/migrations/03-add_animal_table-down.sql b/examples/from-directory/migrations/03-add_animal_table-down.sql new file mode 100644 index 0000000..66912bf --- /dev/null +++ b/examples/from-directory/migrations/03-add_animal_table-down.sql @@ -0,0 +1 @@ +DROP TABLE animal; \ No newline at end of file diff --git a/examples/from-directory/migrations/03-add_animal_table-up.sql b/examples/from-directory/migrations/03-add_animal_table-up.sql new file mode 100644 index 0000000..084a56b --- /dev/null +++ b/examples/from-directory/migrations/03-add_animal_table-up.sql @@ -0,0 +1 @@ +CREATE TABLE animal(name TEXT); \ No newline at end of file diff --git a/examples/from-directory/src/main.rs b/examples/from-directory/src/main.rs new file mode 100644 index 0000000..ff01e30 --- /dev/null +++ b/examples/from-directory/src/main.rs @@ -0,0 +1,62 @@ +use anyhow::Result; +use include_dir::{include_dir, Dir}; +use lazy_static::lazy_static; +use rusqlite::{params, Connection}; +use rusqlite_migration::Migrations; + +// Test that migrations are working +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn migrations_test() { + assert!(MIGRATIONS.validate().is_ok()); + } +} + +static MIGRATIONS_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/from-directory/migrations"); + +// Define migrations. These are applied atomically. +lazy_static! { + static ref MIGRATIONS: Migrations<'static> = + Migrations::from_directory(&MIGRATIONS_DIR).unwrap(); +} + +pub fn init_db() -> Result { + let mut conn = Connection::open("./my_db.db3")?; + + // Update the database schema, atomically + MIGRATIONS.to_latest(&mut conn)?; + + Ok(conn) +} + +pub fn main() { + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("trace")).init(); + + let mut conn = init_db().unwrap(); + + // Apply some PRAGMA. These are often better applied outside of migrations, as some needs to be + // executed for each connection (like `foreign_keys`) or to be executed outside transactions + // (`journal_mode` is a noop in a transaction). + conn.pragma_update(None, "journal_mode", "WAL").unwrap(); + conn.pragma_update(None, "foreign_keys", "ON").unwrap(); + + // Use the db 🥳 + conn.execute( + "INSERT INTO friend (name, birthday) VALUES (?1, ?2)", + params!["John", "1970-01-01"], + ) + .unwrap(); + + conn.execute("INSERT INTO animal (name) VALUES (?1)", params!["dog"]) + .unwrap(); + + // If we want to revert the last migration + MIGRATIONS.to_version(&mut conn, 2).unwrap(); + + // The table was removed + conn.execute("INSERT INTO animal (name) VALUES (?1)", params!["cat"]) + .unwrap_err(); +} diff --git a/rusqlite_migration/Cargo.toml b/rusqlite_migration/Cargo.toml index 783539b..07b4942 100644 --- a/rusqlite_migration/Cargo.toml +++ b/rusqlite_migration/Cargo.toml @@ -17,7 +17,11 @@ default = [] ### Enable support for async migrations with the use of `tokio-rusqlite` async-tokio-rusqlite = ["dep:tokio-rusqlite", "dep:tokio"] +### Enable loading migrations from *.sql files in a given directory +from-directory = ["dep:include_dir"] + [dependencies] +include_dir = { version = "0.7.3", features = ["glob"], optional = true } tokio = { version = "1.25", features = ["macros"], optional = true } tokio-rusqlite = { version = "0.3.0", optional = true } log = "0.4" diff --git a/rusqlite_migration/src/asynch.rs b/rusqlite_migration/src/asynch.rs index 8325f63..158a582 100644 --- a/rusqlite_migration/src/asynch.rs +++ b/rusqlite_migration/src/asynch.rs @@ -3,6 +3,9 @@ use tokio_rusqlite::Connection as AsyncConnection; use crate::errors::Result; use crate::{Migrations, SchemaVersion, M}; +#[cfg(feature = "from-directory")] +use include_dir::Dir; + /// Adapter to make `Migrations` available in an async context. #[derive(Debug, PartialEq, Eq, Clone)] pub struct AsyncMigrations { @@ -29,6 +32,25 @@ impl AsyncMigrations { } } + /// Proxy implementation of the same method in the [Migrations](crate::Migrations::from_directory) struct. + /// + /// # Example + /// + /// ``` + /// use rusqlite_migration::AsyncMigrations; + /// use include_dir::{Dir, include_dir}; + /// + /// static MIGRATION_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/../examples/from-directory/migrations"); + /// let migrations = AsyncMigrations::from_directory(&MIGRATION_DIR).unwrap(); + /// ``` + #[allow(clippy::missing_errors_doc)] + #[cfg(feature = "from-directory")] + pub fn from_directory(dir: &'static Dir<'static>) -> Result { + Ok(Self { + migrations: Migrations::from_directory(dir)?, + }) + } + /// Asynchronous version of the same method in the [Migrations](crate::Migrations::current_version) struct. /// /// # Example diff --git a/rusqlite_migration/src/errors.rs b/rusqlite_migration/src/errors.rs index 7f1672b..dd2208d 100644 --- a/rusqlite_migration/src/errors.rs +++ b/rusqlite_migration/src/errors.rs @@ -27,6 +27,8 @@ pub enum Error { ForeignKeyCheck(ForeignKeyCheckError), /// Error returned by the migration hook Hook(String), + /// Error returned when loading migrations from directory + FileLoad(String), } impl Error { @@ -55,6 +57,7 @@ impl std::error::Error for Error { Error::MigrationDefinition(e) => Some(e), Error::ForeignKeyCheck(e) => Some(e), Error::Hook(_) => None, + Error::FileLoad(_) => None, } } } diff --git a/rusqlite_migration/src/lib.rs b/rusqlite_migration/src/lib.rs index c4cf460..c4d6afb 100644 --- a/rusqlite_migration/src/lib.rs +++ b/rusqlite_migration/src/lib.rs @@ -97,6 +97,14 @@ use log::{debug, info, trace, warn}; use rusqlite::NO_PARAMS; use rusqlite::{Connection, OptionalExtension, Transaction}; +#[cfg(feature = "from-directory")] +use include_dir::Dir; + +#[cfg(feature = "from-directory")] +mod loader; +#[cfg(feature = "from-directory")] +use loader::from_dir; + #[cfg(feature = "async-tokio-rusqlite")] mod asynch; mod errors; @@ -380,6 +388,32 @@ impl<'m> Migrations<'m> { Self { ms } } + /// Creates a set of migrations from a given directory without scanning subdirectories. + /// The migrations are loaded and stored in the binary. + /// + /// The migration directory pointed to by `include_dir!()` must contain + /// files with an `sql` extension with file names in accordance with the given template: + /// `{usize id indicating the order}-{convinient migration name}-{up or down}.sql` + /// + /// # Example + /// + /// ``` + /// use rusqlite_migration::Migrations; + /// use include_dir::{Dir, include_dir}; + /// + /// static MIGRATION_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/../examples/from-directory/migrations"); + /// let migrations = Migrations::from_directory(&MIGRATION_DIR).unwrap(); + /// ``` + /// + /// Errors: + /// + /// Returns [`Error::FileLoad`] in case the file names are incorrect, + /// contain incorrect data or cannot be loaded. + #[cfg(feature = "from-directory")] + pub fn from_directory(dir: &'static Dir<'static>) -> Result { + Ok(Self { ms: from_dir(dir)? }) + } + /// Performs allocations transparently. pub fn new_iter>>(ms: I) -> Self { use std::iter::FromIterator; diff --git a/rusqlite_migration/src/loader.rs b/rusqlite_migration/src/loader.rs new file mode 100644 index 0000000..30b2cf1 --- /dev/null +++ b/rusqlite_migration/src/loader.rs @@ -0,0 +1,95 @@ +use std::collections::BTreeMap; + +use crate::{Error, Result, M}; +use include_dir::{Dir, DirEntry}; +use log::trace; + +#[derive(Debug, Clone)] +struct MigrationFile { + up: Option<&'static str>, + down: Option<&'static str>, +} + +pub fn from_dir(dir: &'static Dir<'static>) -> Result>> { + let mut migrations: BTreeMap = BTreeMap::new(); + + let entries = dir + .find("*.sql") + .map_err(|_| Error::FileLoad("Incorrect glob pattern".to_string()))? + .flat_map(|entry| match entry { + DirEntry::File(f) => Some(f), + DirEntry::Dir(_) => None, + }); + + for entry in entries { + trace!("Loading migration from {:?}", entry.path().file_name()); + + let file_name = + entry + .path() + .file_name() + .and_then(|f| f.to_str()) + .ok_or(Error::FileLoad(format!( + "Could not extract file name from {:?}", + entry.path() + )))?; + + let content = entry.contents_utf8().ok_or(Error::FileLoad(format!( + "Could not load contents from {file_name}" + )))?; + + let id = file_name + .split_once('-') + .ok_or(Error::FileLoad(format!( + "Could not extract migration id from file name {file_name}" + )))? + .0 + .parse::() + .map_err(|e| { + Error::FileLoad(format!( + "Could not parse migration id from file name {file_name} as usize: {e}" + )) + })?; + + if file_name.ends_with("-up.sql") { + migrations + .entry(id) + .and_modify(|e| e.up = Some(content)) + .or_insert(MigrationFile { + up: Some(content), + down: None, + }); + } else if file_name.ends_with("-down.sql") { + migrations + .entry(id) + .and_modify(|e| e.down = Some(content)) + .or_insert(MigrationFile { + up: None, + down: Some(content), + }); + } else { + return Err(Error::FileLoad(format!( + "Incorrect file name: {file_name}, should end with -up.sql or -down.sql" + ))); + } + } + + if let Some(migration) = migrations.values().find(|m| m.up.is_none()) { + return Err(Error::FileLoad(format!( + "Missing upward migration file for migration {migration:?}" + ))); + } + + if migrations.values().any(|v| v.up.is_none()) {} + + Ok(migrations + .values() + .flat_map(|v| { + if let Some(up) = v.up { + return Some(M::up(up).down(v.down.unwrap_or_default())); + } else { + None + } + }) + .collect()) +} diff --git a/rusqlite_migration_tests/Cargo.toml b/rusqlite_migration_tests/Cargo.toml index dbecc7f..a8a9f91 100644 --- a/rusqlite_migration_tests/Cargo.toml +++ b/rusqlite_migration_tests/Cargo.toml @@ -20,7 +20,7 @@ log = "0.4" [dependencies.rusqlite_migration] path = "../rusqlite_migration" -features = ["async-tokio-rusqlite"] +features = ["async-tokio-rusqlite", "from-directory"] [dependencies.rusqlite] version = ">=0.23.0" @@ -34,6 +34,7 @@ env_logger = "0.10" anyhow = "1" lazy_static = "1.4.0" mktemp = "0.5" +include_dir = "0.7.3" [[test]] name = "integration_tests" diff --git a/rusqlite_migration_tests/tests/from_dir_test.rs b/rusqlite_migration_tests/tests/from_dir_test.rs new file mode 100644 index 0000000..4eb4d66 --- /dev/null +++ b/rusqlite_migration_tests/tests/from_dir_test.rs @@ -0,0 +1,34 @@ +use std::num::NonZeroUsize; + +use include_dir::{include_dir, Dir}; + +use rusqlite::{params, Connection}; +use rusqlite_migration::{Migrations, SchemaVersion}; + +static MIGRATIONS_DIR: Dir = + include_dir!("$CARGO_MANIFEST_DIR/../examples/from-directory/migrations"); + +#[test] +fn main_test() { + simple_logging::log_to_stderr(log::LevelFilter::Trace); + + let mut conn = Connection::open_in_memory().unwrap(); + // Define migrations + + { + let migrations = Migrations::from_directory(&MIGRATIONS_DIR).unwrap(); + + migrations.to_latest(&mut conn).unwrap(); + + assert_eq!( + Ok(SchemaVersion::Inside(NonZeroUsize::new(3).unwrap())), + migrations.current_version(&conn) + ); + + conn.execute( + "INSERT INTO friend (name, birthday) VALUES (?1, ?2)", + params!["John", "1970-01-01"], + ) + .unwrap(); + } +}