Load migrations from *.sql files in a directory

.github/workflows/ci.yml

@@ -65,6 +65,21 @@ jobs:
           command: test
           args: --features async-tokio-rusqlite
 
+  test_from_directory:
+    name: Test feature from-directory
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --features from-directory
+
   test_examples:
     name: Test examples
     runs-on: ubuntu-latest

examples/from-directory/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "example-from-directory"
+version = "0.1.0"
+edition = "2018"
+publish = false
+
+[dependencies]
+rusqlite_migration = { path = "../../rusqlite_migration", features = [
+    "from-directory",
+] }
+log = "0.4"
+simple-logging = "2.0.2"
+env_logger = "0.10"
+anyhow = "1"
+lazy_static = "1.4.0"
+mktemp = "0.5"
+include_dir = "0.7.3"
+
+[dependencies.rusqlite]
+version = ">=0.23.0"
+default-features = false
+features = []

examples/from-directory/migrations/01-friend_car-up.sql

@@ -0,0 +1,13 @@
+CREATE TABLE friend(
+    friend_id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    email TEXT UNIQUE,
+    phone TEXT UNIQUE,
+    picture BLOB
+);
+
+CREATE TABLE car(
+    registration_plate TEXT PRIMARY KEY,
+    cost REAL NOT NULL,
+    bought_on TEXT NOT NULL
+);

examples/from-directory/migrations/02-add_birthday_column-up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE friend ADD COLUMN birthday TEXT;
+ALTER TABLE friend ADD COLUMN comment TEXT;

examples/from-directory/migrations/03-add_animal_table-down.sql

@@ -0,0 +1 @@
+DROP TABLE animal;

examples/from-directory/migrations/03-add_animal_table-up.sql

@@ -0,0 +1 @@
+CREATE TABLE animal(name TEXT);

examples/from-directory/src/main.rs

@@ -0,0 +1,62 @@
+use anyhow::Result;
+use include_dir::{include_dir, Dir};
+use lazy_static::lazy_static;
+use rusqlite::{params, Connection};
+use rusqlite_migration::Migrations;
+
+// Test that migrations are working
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn migrations_test() {
+        assert!(MIGRATIONS.validate().is_ok());
+    }
+}
+
+static MIGRATIONS_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/from-directory/migrations");
+
+// Define migrations. These are applied atomically.
+lazy_static! {
+    static ref MIGRATIONS: Migrations<'static> =
+        Migrations::from_directory(&MIGRATIONS_DIR).unwrap();
+}
+
+pub fn init_db() -> Result<Connection> {
+    let mut conn = Connection::open("./my_db.db3")?;
+
+    // Update the database schema, atomically
+    MIGRATIONS.to_latest(&mut conn)?;
+
+    Ok(conn)
+}
+
+pub fn main() {
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("trace")).init();
+
+    let mut conn = init_db().unwrap();
+
+    // Apply some PRAGMA. These are often better applied outside of migrations, as some needs to be
+    // executed for each connection (like `foreign_keys`) or to be executed outside transactions
+    // (`journal_mode` is a noop in a transaction).
+    conn.pragma_update(None, "journal_mode", "WAL").unwrap();
+    conn.pragma_update(None, "foreign_keys", "ON").unwrap();
+
+    // Use the db 🥳
+    conn.execute(
+        "INSERT INTO friend (name, birthday) VALUES (?1, ?2)",
+        params!["John", "1970-01-01"],
+    )
+    .unwrap();
+
+    conn.execute("INSERT INTO animal (name) VALUES (?1)", params!["dog"])
+        .unwrap();
+
+    // If we want to revert the last migration
+    MIGRATIONS.to_version(&mut conn, 2).unwrap();
+
+    // The table was removed
+    conn.execute("INSERT INTO animal (name) VALUES (?1)", params!["cat"])
+        .unwrap_err();
+}

rusqlite_migration/Cargo.toml

@@ -17,7 +17,11 @@ default = []
 ### Enable support for async migrations with the use of `tokio-rusqlite`
 async-tokio-rusqlite = ["dep:tokio-rusqlite", "dep:tokio"]
 
+### Enable loading migrations from *.sql files in a given directory
+from-directory = ["dep:include_dir"]
+
 [dependencies]
+include_dir = { version = "0.7.3", features = ["glob"], optional = true }
 tokio = { version = "1.25", features = ["macros"], optional = true }
 tokio-rusqlite = { version = "0.3.0", optional = true }
 log = "0.4"

rusqlite_migration/src/asynch.rs

@@ -3,6 +3,9 @@ use tokio_rusqlite::Connection as AsyncConnection;
 use crate::errors::Result;
 use crate::{Migrations, SchemaVersion, M};
 
+#[cfg(feature = "from-directory")]
+use include_dir::Dir;
+
 /// Adapter to make `Migrations` available in an async context.
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub struct AsyncMigrations {
@@ -29,6 +32,25 @@ impl AsyncMigrations {
         }
     }
 
+    /// Proxy implementation of the same method in the [Migrations](crate::Migrations::from_directory) struct.
+    ///
+    /// # Example
+    ///
+    /// ```
+    /// use rusqlite_migration::AsyncMigrations;
+    /// use include_dir::{Dir, include_dir};
+    ///
+    /// static MIGRATION_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/../examples/from-directory/migrations");
+    /// let migrations = AsyncMigrations::from_directory(&MIGRATION_DIR).unwrap();
+    /// ```
+    #[allow(clippy::missing_errors_doc)]
+    #[cfg(feature = "from-directory")]
+    pub fn from_directory(dir: &'static Dir<'static>) -> Result<Self> {
+        Ok(Self {
+            migrations: Migrations::from_directory(dir)?,
+        })
+    }
+
     /// Asynchronous version of the same method in the [Migrations](crate::Migrations::current_version) struct.
     ///
     /// # Example

rusqlite_migration/src/errors.rs

@@ -27,6 +27,8 @@ pub enum Error {
     ForeignKeyCheck(ForeignKeyCheckError),
     /// Error returned by the migration hook
     Hook(String),
+    /// Error returned when loading migrations from directory
+    FileLoad(String),
 }
 
 impl Error {
@@ -55,6 +57,7 @@ impl std::error::Error for Error {
             Error::MigrationDefinition(e) => Some(e),
             Error::ForeignKeyCheck(e) => Some(e),
             Error::Hook(_) => None,
+            Error::FileLoad(_) => None,
         }
     }
 }

rusqlite_migration/src/lib.rs

@@ -97,6 +97,14 @@ use log::{debug, info, trace, warn};
 use rusqlite::NO_PARAMS;
 use rusqlite::{Connection, OptionalExtension, Transaction};
 
+#[cfg(feature = "from-directory")]
+use include_dir::Dir;
+
+#[cfg(feature = "from-directory")]
+mod loader;
+#[cfg(feature = "from-directory")]
+use loader::from_dir;
+
 #[cfg(feature = "async-tokio-rusqlite")]
 mod asynch;
 mod errors;
@@ -380,6 +388,32 @@ impl<'m> Migrations<'m> {
         Self { ms }
     }
 
+    /// Creates a set of migrations from a given directory without scanning subdirectories.
+    /// The migrations are loaded and stored in the binary.
+    ///
+    /// The migration directory pointed to by `include_dir!()` must contain
+    /// files with an `sql` extension with file names in accordance with the given template:
+    /// `{usize id indicating the order}-{convinient migration name}-{up or down}.sql`
+    ///
+    /// # Example
+    ///
+    /// ```
+    /// use rusqlite_migration::Migrations;
+    /// use include_dir::{Dir, include_dir};
+    ///
+    /// static MIGRATION_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/../examples/from-directory/migrations");
+    /// let migrations = Migrations::from_directory(&MIGRATION_DIR).unwrap();
+    /// ```
+    ///
+    /// Errors:
+    ///
+    /// Returns [`Error::FileLoad`] in case the file names are incorrect,
+    /// contain incorrect data or cannot be loaded.
+    #[cfg(feature = "from-directory")]
+    pub fn from_directory(dir: &'static Dir<'static>) -> Result<Self> {
+        Ok(Self { ms: from_dir(dir)? })
+    }
+
     /// Performs allocations transparently.
     pub fn new_iter<I: IntoIterator<Item = M<'m>>>(ms: I) -> Self {
         use std::iter::FromIterator;

rusqlite_migration/src/loader.rs

@@ -0,0 +1,95 @@
+use std::collections::BTreeMap;
+
+use crate::{Error, Result, M};
+use include_dir::{Dir, DirEntry};
+use log::trace;
+
+#[derive(Debug, Clone)]
+struct MigrationFile {
+    up: Option<&'static str>,
+    down: Option<&'static str>,
+}
+
+pub fn from_dir(dir: &'static Dir<'static>) -> Result<Vec<M<'static>>> {
+    let mut migrations: BTreeMap<usize, MigrationFile> = BTreeMap::new();
+
+    let entries = dir
+        .find("*.sql")
+        .map_err(|_| Error::FileLoad("Incorrect glob pattern".to_string()))?
+        .flat_map(|entry| match entry {
+            DirEntry::File(f) => Some(f),
+            DirEntry::Dir(_) => None,
+        });
+
+    for entry in entries {
+        trace!("Loading migration from {:?}", entry.path().file_name());
+
+        let file_name =
+            entry
+                .path()
+                .file_name()
+                .and_then(|f| f.to_str())
+                .ok_or(Error::FileLoad(format!(
+                    "Could not extract file name from {:?}",
+                    entry.path()
+                )))?;
+
+        let content = entry.contents_utf8().ok_or(Error::FileLoad(format!(
+            "Could not load contents from {file_name}"
+        )))?;
+
+        let id = file_name
+            .split_once('-')
+            .ok_or(Error::FileLoad(format!(
+                "Could not extract migration id from file name {file_name}"
+            )))?
+            .0
+            .parse::<usize>()
+            .map_err(|e| {
+                Error::FileLoad(format!(
+                    "Could not parse migration id from file name {file_name} as usize: {e}"
+                ))
+            })?;
+
+        if file_name.ends_with("-up.sql") {
+            migrations
+                .entry(id)
+                .and_modify(|e| e.up = Some(content))
+                .or_insert(MigrationFile {
+                    up: Some(content),
+                    down: None,
+                });
+        } else if file_name.ends_with("-down.sql") {
+            migrations
+                .entry(id)
+                .and_modify(|e| e.down = Some(content))
+                .or_insert(MigrationFile {
+                    up: None,
+                    down: Some(content),
+                });
+        } else {
+            return Err(Error::FileLoad(format!(
+                "Incorrect file name: {file_name}, should end with -up.sql or -down.sql"
+            )));
+        }
+    }
+
+    if let Some(migration) = migrations.values().find(|m| m.up.is_none()) {
+        return Err(Error::FileLoad(format!(
+            "Missing upward migration file for migration {migration:?}"
+        )));
+    }
+
+    if migrations.values().any(|v| v.up.is_none()) {}
+
+    Ok(migrations
+        .values()
+        .flat_map(|v| {
+            if let Some(up) = v.up {
+                return Some(M::up(up).down(v.down.unwrap_or_default()));
+            } else {
+                None
+            }
+        })
+        .collect())
+}

rusqlite_migration_tests/Cargo.toml

@@ -20,7 +20,7 @@ log = "0.4"
 
 [dependencies.rusqlite_migration]
 path = "../rusqlite_migration"
-features = ["async-tokio-rusqlite"]
+features = ["async-tokio-rusqlite", "from-directory"]
 
 [dependencies.rusqlite]
 version = ">=0.23.0"
@@ -34,6 +34,7 @@ env_logger = "0.10"
 anyhow = "1"
 lazy_static = "1.4.0"
 mktemp = "0.5"
+include_dir = "0.7.3"
 
 [[test]]
 name = "integration_tests"

rusqlite_migration_tests/tests/from_dir_test.rs

@@ -0,0 +1,34 @@
+use std::num::NonZeroUsize;
+
+use include_dir::{include_dir, Dir};
+
+use rusqlite::{params, Connection};
+use rusqlite_migration::{Migrations, SchemaVersion};
+
+static MIGRATIONS_DIR: Dir =
+    include_dir!("$CARGO_MANIFEST_DIR/../examples/from-directory/migrations");
+
+#[test]
+fn main_test() {
+    simple_logging::log_to_stderr(log::LevelFilter::Trace);
+
+    let mut conn = Connection::open_in_memory().unwrap();
+    // Define migrations
+
+    {
+        let migrations = Migrations::from_directory(&MIGRATIONS_DIR).unwrap();
+
+        migrations.to_latest(&mut conn).unwrap();
+
+        assert_eq!(
+            Ok(SchemaVersion::Inside(NonZeroUsize::new(3).unwrap())),
+            migrations.current_version(&conn)
+        );
+
+        conn.execute(
+            "INSERT INTO friend (name, birthday) VALUES (?1, ?2)",
+            params!["John", "1970-01-01"],
+        )
+        .unwrap();
+    }
+}