diff --git a/packages/os/migrator.service b/packages/os/migrator.service index 1b911483305..1ecf01f6dd1 100644 --- a/packages/os/migrator.service +++ b/packages/os/migrator.service @@ -3,7 +3,7 @@ Description=Bottlerocket data store migrator [Service] Type=oneshot -ExecStart=/usr/bin/migrator --datastore-path /var/lib/bottlerocket/datastore/current --migration-directories /var/lib/bottlerocket-migrations --migrate-to-version-from-os-release +ExecStart=/usr/bin/migrator --datastore-path /var/lib/bottlerocket/datastore/current --migration-directory /var/lib/bottlerocket-migrations --root-path /usr/share/updog/root.json --metadata-directory /var/cache/bottlerocket-metadata --migrate-to-version-from-os-release RemainAfterExit=true StandardError=journal+console diff --git a/sources/Cargo.lock b/sources/Cargo.lock index a694671e66d..7666745fb10 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -1414,12 +1414,15 @@ dependencies = [ "cargo-readme 3.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "lz4 1.23.1 (registry+https://github.com/rust-lang/crates.io-index)", "nix 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "simplelog 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)", "snafu 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tough 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "update_metadata 0.1.0", ] @@ -2706,7 +2709,6 @@ dependencies = [ "lz4 1.23.1 (registry+https://github.com/rust-lang/crates.io-index)", "migrator 0.1.0", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.110 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/sources/api/migration/migrator/Cargo.toml b/sources/api/migration/migrator/Cargo.toml index 6c05ff12d2f..a8d3f06fcd3 100644 --- a/sources/api/migration/migrator/Cargo.toml +++ b/sources/api/migration/migrator/Cargo.toml @@ -18,10 +18,16 @@ semver = "0.9" simplelog = "0.7" snafu = "0.6" update_metadata = { path = "../../../updater/update_metadata" } +tempfile = "3.1.0" +tough = { version = "0.5.0" } +lz4 = "1.23.1" [build-dependencies] cargo-readme = "3.1" +[dev-dependencies] +tempfile = "3.1" + [[bin]] name = "migrator" path = "src/main.rs" diff --git a/sources/api/migration/migrator/src/args.rs b/sources/api/migration/migrator/src/args.rs index 63cf90b6b7c..416c86abbba 100644 --- a/sources/api/migration/migrator/src/args.rs +++ b/sources/api/migration/migrator/src/args.rs @@ -15,7 +15,10 @@ fn usage() -> ! { eprintln!( r"Usage: {} --datastore-path PATH - --migration-directories PATH[:PATH:PATH...] + --migration-directory PATH + --root-path PATH + --metadata-directory PATH + --working-directory PATH (--migrate-to-version x.y | --migrate-to-version-from-os-release) [ --no-color ] [ --log-level trace|debug|info|warn|error ]", @@ -34,8 +37,10 @@ fn usage_msg>(msg: S) -> ! { pub(crate) struct Args { pub(crate) datastore_path: PathBuf, pub(crate) log_level: LevelFilter, - pub(crate) migration_directories: Vec, + pub(crate) migration_directory: PathBuf, pub(crate) migrate_to_version: Version, + pub(crate) root_path: PathBuf, + pub(crate) metadata_directory: PathBuf, } impl Args { @@ -44,8 +49,10 @@ impl Args { // Required parameters. let mut datastore_path = None; let mut log_level = None; - let mut migration_directories = None; + let mut migration_directory = None; let mut migrate_to_version = None; + let mut root_path = None; + let mut metadata_path = None; let mut iter = args.skip(1); while let Some(arg) = iter.next() { @@ -84,16 +91,12 @@ impl Args { })); } - "--migration-directories" => { - let paths_str = iter.next().unwrap_or_else(|| { - usage_msg("Did not give argument to --migration-directories") + "--migration-directory" => { + let path_str = iter.next().unwrap_or_else(|| { + usage_msg("Did not give argument to --migration-directory") }); - trace!("Given --migration-directories: {}", paths_str); - let paths: Vec<_> = paths_str.split(':').map(PathBuf::from).collect(); - if paths.is_empty() { - usage_msg("Found no paths in argument to --migration-directories"); - } - migration_directories = Some(paths); + trace!("Given --migration-directory: {}", path_str); + migration_directory = Some(PathBuf::from(path_str)); } "--migrate-to-version" => { @@ -114,15 +117,36 @@ impl Args { migrate_to_version = Some(br.version_id) } - _ => usage(), + "--root-path" => { + let path_str = iter + .next() + .unwrap_or_else(|| usage_msg("Did not give argument to --root-path")); + trace!("Given --root-path: {}", path_str); + root_path = Some(PathBuf::from(path_str)); + } + + "--metadata-directory" => { + let path_str = iter.next().unwrap_or_else(|| { + usage_msg("Did not give argument to --metadata-directory") + }); + trace!("Given --metadata-directory: {}", path_str); + metadata_path = Some(PathBuf::from(path_str)); + } + _ => usage_msg(format!("Unable to parse input '{}'", arg)), } } Self { - datastore_path: datastore_path.unwrap_or_else(|| usage()), + datastore_path: datastore_path + .unwrap_or_else(|| usage_msg("--datastore_path is empty")), log_level: log_level.unwrap_or_else(|| LevelFilter::Info), - migration_directories: migration_directories.unwrap_or_else(|| usage()), - migrate_to_version: migrate_to_version.unwrap_or_else(|| usage()), + migration_directory: migration_directory + .unwrap_or_else(|| usage_msg("--migration_directory is empty")), + migrate_to_version: migrate_to_version + .unwrap_or_else(|| usage_msg("--migrate_to_version is empty")), + root_path: root_path.unwrap_or_else(|| usage_msg("--root_path is empty")), + metadata_directory: metadata_path + .unwrap_or_else(|| usage_msg("--metadata_directory is empty")), } } } diff --git a/sources/api/migration/migrator/src/error.rs b/sources/api/migration/migrator/src/error.rs index be150fad68f..ee1fc4a225a 100644 --- a/sources/api/migration/migrator/src/error.rs +++ b/sources/api/migration/migrator/src/error.rs @@ -1,7 +1,7 @@ //! This module owns the error type used by the migrator. use semver::Version; -use snafu::Snafu; +use snafu::{Backtrace, Snafu}; use std::io; use std::path::PathBuf; use std::process::{Command, Output}; @@ -13,6 +13,12 @@ pub(crate) enum Error { #[snafu(display("Internal error: {}", msg))] Internal { msg: String }, + #[snafu(display("Unable to create tempdir for migration binaries: '{}'", source))] + CreateRunDir { source: std::io::Error }, + + #[snafu(display("Unable to create tempdir for tough datastore: '{}'", source))] + CreateToughTempDir { source: std::io::Error }, + #[snafu(display("Data store path '{}' contains invalid UTF-8", path.display()))] DataStorePathNotUTF8 { path: PathBuf }, @@ -22,6 +28,21 @@ pub(crate) enum Error { #[snafu(display("Data store link '{}' points to /", path.display()))] DataStoreLinkToRoot { path: PathBuf }, + #[snafu(display("Unable to delete directory '{}': {}", path.display(), source))] + DeleteDirectory { + path: PathBuf, + source: std::io::Error, + }, + + #[snafu(display("Failed to convert '{}' to a URL", path.display()))] + DirectoryUrl { path: PathBuf, backtrace: Backtrace }, + + #[snafu(display("Error finding migration: {}", source))] + FindMigrations { + source: update_metadata::error::Error, + backtrace: Backtrace, + }, + #[snafu(display("Data store path '{}' contains invalid version: {}", path.display(), source))] InvalidDataStoreVersion { path: PathBuf, @@ -59,9 +80,58 @@ pub(crate) enum Error { #[snafu(display("Failed listing migration directory '{}': {}", dir.display(), source))] ListMigrations { dir: PathBuf, source: io::Error }, + #[snafu(display("Error loading manifest: {}", source))] + LoadManifest { + source: update_metadata::error::Error, + backtrace: Backtrace, + }, + + #[snafu(display("Error loading migration '{}': {}", migration, source))] + LoadMigration { + migration: String, + source: tough::error::Error, + backtrace: Backtrace, + }, + + #[snafu(display("Failed to decode LZ4-compressed migration {}: {}", migration, source))] + Lz4Decode { + migration: String, + source: std::io::Error, + backtrace: Backtrace, + }, + + #[snafu(display("Migration '{}' not found", migration))] + MigrationNotFound { + migration: String, + backtrace: Backtrace, + }, + + #[snafu(display("Error saving migration '{}': {}", path.display(), source))] + MigrationSave { + path: PathBuf, + source: std::io::Error, + backtrace: Backtrace, + }, + + #[snafu(display("Failed to open trusted root metadata file {}: {}", path.display(), source))] + OpenRoot { + path: PathBuf, + source: std::io::Error, + backtrace: Backtrace, + }, + + #[snafu(display("Unable to create URL from path '{}'", path.display()))] + PathUrl { path: PathBuf, backtrace: Backtrace }, + #[snafu(display("Failed reading migration directory entry: {}", source))] ReadMigrationEntry { source: io::Error }, + #[snafu(display("Failed to load TUF repo: {}", source))] + RepoLoad { + source: tough::error::Error, + backtrace: Backtrace, + }, + #[snafu(display("Failed reading metadata of '{}': {}", path.display(), source))] PathMetadata { path: PathBuf, source: io::Error }, @@ -70,9 +140,6 @@ pub(crate) enum Error { #[snafu(display("Migration path '{}' contains invalid UTF-8", path.display()))] MigrationNameNotUTF8 { path: PathBuf }, - - #[snafu(display("Logger setup error: {}", source))] - Logger { source: simplelog::TermLogError }, } /// Result alias containing our Error type. diff --git a/sources/api/migration/migrator/src/main.rs b/sources/api/migration/migrator/src/main.rs index aa7a64bc955..1a1c2fc395e 100644 --- a/sources/api/migration/migrator/src/main.rs +++ b/sources/api/migration/migrator/src/main.rs @@ -30,38 +30,39 @@ use simplelog::{Config as LogConfig, TermLogger, TerminalMode}; use snafu::{ensure, OptionExt, ResultExt}; use std::collections::HashSet; use std::env; -use std::fs::{self, Permissions}; +use std::fs::{self, File, OpenOptions, Permissions}; use std::os::unix::fs::{symlink, PermissionsExt}; use std::os::unix::io::AsRawFd; use std::path::{Path, PathBuf}; use std::process::{self, Command}; +use tempfile::TempDir; + +use update_metadata::{load_manifest, Direction, REPOSITORY_LIMITS}; -use update_metadata::MIGRATION_FILENAME_RE; mod args; -mod direction; mod error; use args::Args; -use direction::Direction; use error::Result; +use tough::ExpirationEnforcement; // Returning a Result from main makes it print a Debug representation of the error, but with Snafu // we have nice Display representations of the error, so we wrap "main" (run) and print any error. // https://github.com/shepmaster/snafu/issues/110 fn main() { - if let Err(e) = run() { + let args = Args::from_env(env::args()); + // TerminalMode::Mixed will send errors to stderr and anything less to stdout. + if let Err(e) = TermLogger::init(args.log_level, LogConfig::default(), TerminalMode::Mixed) { + eprintln!("{}", e); + process::exit(1); + } + if let Err(e) = run(&args) { eprintln!("{}", e); process::exit(1); } } -fn run() -> Result<()> { - let args = Args::from_env(env::args()); - - // TerminalMode::Mixed will send errors to stderr and anything less to stdout. - TermLogger::init(args.log_level, LogConfig::default(), TerminalMode::Mixed) - .context(error::Logger)?; - +fn run(args: &Args) -> Result<()> { // Get the directory we're working in. let datastore_dir = args .datastore_path @@ -71,7 +72,6 @@ fn run() -> Result<()> { })?; let current_version = get_current_version(&datastore_dir)?; - let direction = Direction::from_versions(¤t_version, &args.migrate_to_version) .unwrap_or_else(|| { info!( @@ -82,11 +82,33 @@ fn run() -> Result<()> { process::exit(0); }); - let migrations = find_migrations( - &args.migration_directories, - ¤t_version, - &args.migrate_to_version, - )?; + // Prepare to load the locally cached TUF repository to obtain the manifest. + let tough_datastore = TempDir::new().context(error::CreateToughTempDir)?; + let metadata_url = dir_url(&args.metadata_directory)?; + let migrations_url = dir_url(&args.migration_directory)?; + + // Failure to load the TUF repo at the expected location is a serious issue because updog should + // always create a TUF repo that contains at least the manifest, even if there are no migrations. + let repo = tough::Repository::load( + &tough::FilesystemTransport, + tough::Settings { + root: File::open(&args.root_path).context(error::OpenRoot { + path: args.root_path.clone(), + })?, + datastore: tough_datastore.path(), + metadata_base_url: metadata_url.as_str(), + targets_base_url: migrations_url.as_str(), + limits: REPOSITORY_LIMITS, + // if metadata has expired since the time that updog downloaded them, we do not want to + // fail the migration process, so we set expiration enforcement to unsafe. + expiration_enforcement: ExpirationEnforcement::Unsafe, + }, + ) + .context(error::RepoLoad)?; + let manifest = load_manifest(&repo).context(error::LoadManifest)?; + let migrations = + update_metadata::find_migrations(¤t_version, &args.migrate_to_version, &manifest) + .context(error::FindMigrations)?; if migrations.is_empty() { // Not all new OS versions need to change the data store format. If there's been no @@ -95,15 +117,19 @@ fn run() -> Result<()> { // have a chain of symlinks that could go past the maximum depth.) flip_to_new_version(&args.migrate_to_version, &args.datastore_path)?; } else { + // Prepare directory to save migrations to before running them. + // TODO - use pentacle instead of saving the migration binaries to disk before running them. + let rundir = TempDir::new().context(error::CreateRunDir)?; let copy_path = run_migrations( + &repo, direction, &migrations, &args.datastore_path, &args.migrate_to_version, + &rundir, )?; flip_to_new_version(&args.migrate_to_version, ©_path)?; } - Ok(()) } @@ -138,161 +164,6 @@ where Version::parse(version_str).context(error::InvalidDataStoreVersion { path: &patch }) } -/// Returns a list of all migrations found on disk. -/// -/// TODO: This does not yet handle migrations that have been replaced by newer versions - we only -/// look in one fixed location. We need to get the list of migrations from update metadata, and -/// only return those. That may also obviate the need for select_migrations. -fn find_migrations_on_disk

(dir: P) -> Result> -where - P: AsRef, -{ - let dir = dir.as_ref(); - let mut result = Vec::new(); - - trace!("Looking for potential migrations in {}", dir.display()); - let entries = fs::read_dir(dir).context(error::ListMigrations { dir })?; - for entry in entries { - let entry = entry.context(error::ReadMigrationEntry)?; - let path = entry.path(); - - // Just check that it's a file; other checks to determine whether we should actually run - // a file we find are done by select_migrations. - let file_type = entry - .file_type() - .context(error::PathMetadata { path: &path })?; - if !file_type.is_file() { - debug!( - "Skipping non-file in migration directory: {}", - path.display() - ); - continue; - } - - trace!("Found potential migration: {}", path.display()); - result.push(path); - } - - Ok(result) -} - -/// Returns the sublist of the given migrations that should be run, in the returned order, to move -/// from the 'from' version to the 'to' version. -fn select_migrations>( - from: &Version, - to: &Version, - paths: &[P], -) -> Result> { - // Intermediate result where we also store the version and name, needed for sorting - let mut sortable: Vec<(Version, String, PathBuf)> = Vec::new(); - - for path in paths { - let path = path.as_ref(); - - // We pull the applicable version and the migration name out of the filename. - let file_name = path - .file_name() - .context(error::Internal { - msg: "Found '/' as migration", - })? - .to_str() - .context(error::MigrationNameNotUTF8 { path: &path })?; - let captures = match MIGRATION_FILENAME_RE.captures(&file_name) { - Some(captures) => captures, - None => { - debug!( - "Skipping non-migration (bad name) in migration directory: {}", - path.display() - ); - continue; - } - }; - - let version_match = captures.name("version").context(error::Internal { - msg: "Migration name matched regex but we don't have a 'version' capture", - })?; - let version = Version::parse(version_match.as_str()) - .context(error::InvalidMigrationVersion { path: &path })?; - - let name_match = captures.name("name").context(error::Internal { - msg: "Migration name matched regex but we don't have a 'name' capture", - })?; - let name = name_match.as_str().to_string(); - - // We don't want to include migrations for the "from" version we're already on. - // Note on possible confusion: when going backward it's the higher version that knows - // how to undo its changes and take you to the lower version. For example, the v2 - // migration knows what changes it made to go from v1 to v2 and therefore how to go - // back from v2 to v1. See tests. - let applicable = if to > from && version > *from && version <= *to { - info!( - "Found applicable forward migration '{}': {} < ({}) <= {}", - file_name, from, version, to - ); - true - } else if to < from && version > *to && version <= *from { - info!( - "Found applicable backward migration '{}': {} >= ({}) > {}", - file_name, from, version, to - ); - true - } else { - debug!( - "Migration '{}' doesn't apply when going from {} to {}", - file_name, from, to - ); - false - }; - - if applicable { - sortable.push((version, name, path.to_path_buf())); - } - } - - // Sort the migrations using the metadata we stored -- version first, then name so that - // authors have some ordering control if necessary. - sortable.sort_unstable(); - - // For a Backward migration process, reverse the order. - if to < from { - sortable.reverse(); - } - - debug!( - "Sorted migrations: {:?}", - sortable - .iter() - // Want filename, which always applies for us, but fall back to name just in case - .map(|(_version, name, path)| path - .file_name() - .map(|osstr| osstr.to_string_lossy().into_owned()) - .unwrap_or_else(|| name.to_string())) - .collect::>() - ); - - // Get rid of the name; only needed it as a separate component for sorting - let result: Vec = sortable - .into_iter() - .map(|(_version, _name, path)| path) - .collect(); - - Ok(result) -} - -/// Given the versions we're migrating from and to, this will return an ordered list of paths to -/// migration binaries we should run to complete the migration on a data store. -// This separation allows for easy testing of select_migrations. -fn find_migrations

(paths: &[P], from: &Version, to: &Version) -> Result> -where - P: AsRef, -{ - let mut candidates = Vec::new(); - for path in paths { - candidates.extend(find_migrations_on_disk(path)?); - } - select_migrations(from, to, &candidates) -} - /// Generates a random ID, affectionately known as a 'rando', that can be used to avoid timing /// issues and identify unique migration attempts. fn rando() -> String { @@ -328,15 +199,18 @@ where /// /// The given data store is used as a starting point; each migration is given the output of the /// previous migration, and the final output becomes the new data store. -fn run_migrations( +fn run_migrations( + repository: &tough::Repository<'_, tough::FilesystemTransport>, direction: Direction, - migrations: &[P1], - source_datastore: P2, + migrations: &[S], + source_datastore: P1, new_version: &Version, + migrations_rundir: P2, ) -> Result where P1: AsRef, P2: AsRef, + S: AsRef, { // We start with the given source_datastore, updating this after each migration to point to the // output of the previous one. @@ -349,14 +223,34 @@ where let mut intermediate_datastores = HashSet::new(); for migration in migrations { + let migration = migration.as_ref(); + // get the migration from the repo + let lz4_bytes = repository + .read_target(migration) + .context(error::LoadMigration { migration })? + .context(error::MigrationNotFound { migration })?; + + // deflate with an lz4 decoder read + let mut reader = lz4::Decoder::new(lz4_bytes).context(error::Lz4Decode { migration })?; + + // TODO - remove this use of the filesystem when we add pentacle + let exec_path = migrations_rundir.as_ref().join(&migration); + { + let mut f = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&exec_path) + .context(error::MigrationSave { path: &exec_path })?; + let _ = std::io::copy(&mut reader, &mut f) + .context(error::MigrationSave { path: &exec_path })?; + } + // Ensure the migration is executable. - fs::set_permissions(migration.as_ref(), Permissions::from_mode(0o755)).context( - error::SetPermissions { - path: migration.as_ref(), - }, - )?; + fs::set_permissions(&exec_path, Permissions::from_mode(0o755)) + .context(error::SetPermissions { path: &exec_path })?; - let mut command = Command::new(migration.as_ref()); + let mut command = Command::new(&exec_path); // Point each migration in the right direction, and at the given data store. command.arg(direction.to_string()); @@ -397,7 +291,6 @@ where } ensure!(output.status.success(), error::MigrationFailure { output }); - source_datastore = &target_datastore; } @@ -407,7 +300,10 @@ where // Even if we fail to remove an intermediate data store, we've still migrated // successfully, and we don't want to fail the upgrade - just let someone know for // later cleanup. - trace!("Removing intermediate data store at {}", intermediate_datastore.display()); + trace!( + "Removing intermediate data store at {}", + intermediate_datastore.display() + ); if let Err(e) = fs::remove_dir_all(&intermediate_datastore) { error!( "Failed to remove intermediate data store at '{}': {}", @@ -580,60 +476,193 @@ where Ok(()) } +/// Converts a filepath into a URI formatted string +fn dir_url>(path: P) -> Result { + let path_str = path.as_ref().to_str().context(error::PathUrl { + path: path.as_ref(), + })?; + Ok(format!("file://{}", path_str)) +} + // =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= #[cfg(test)] mod test { use super::*; + use tempfile::TempDir; - #[test] - #[allow(unused_variables)] - fn select_migrations_works() { - // Migration paths for use in testing - let m00_1 = Path::new("migrate_v0.0.0_001"); - let m01_1 = Path::new("migrate_v0.0.1_001"); - let m01_2 = Path::new("migrate_v0.0.1_002"); - let m02_1 = Path::new("migrate_v0.0.2_001"); - let m03_1 = Path::new("migrate_v0.0.3_001"); - let m04_1 = Path::new("migrate_v0.0.4_001"); - let m04_2 = Path::new("migrate_v0.0.4_002"); - let all_migrations = vec![&m00_1, &m01_1, &m01_2, &m02_1, &m03_1, &m04_1, &m04_2]; - - // Versions for use in testing - let v00 = Version::new(0, 0, 0); - let v01 = Version::new(0, 0, 1); - let v02 = Version::new(0, 0, 2); - let v03 = Version::new(0, 0, 3); - let v04 = Version::new(0, 0, 4); - let v05 = Version::new(0, 0, 5); - - // Test going forward one minor version - assert_eq!( - select_migrations(&v01, &v02, &all_migrations).unwrap(), - vec![m02_1] - ); + pub fn test_data() -> PathBuf { + let mut p = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + p.pop(); + p.join("migrator").join("tests").join("data") + } - // Test going backward one minor version - assert_eq!( - select_migrations(&v02, &v01, &all_migrations).unwrap(), - vec![m02_1] - ); + struct MigrationTestInfo { + tmp: TempDir, + from_version: Version, + to_version: Version, + datastore: PathBuf, + } - // Test going forward a few minor versions - assert_eq!( - select_migrations(&v01, &v04, &all_migrations).unwrap(), - vec![m02_1, m03_1, m04_1, m04_2] - ); + impl MigrationTestInfo { + fn new(from_version: Version, to_version: Version) -> Self { + MigrationTestInfo { + tmp: TempDir::new().unwrap(), + from_version, + to_version, + datastore: PathBuf::default(), + } + } + } - // Test going backward a few minor versions - assert_eq!( - select_migrations(&v04, &v01, &all_migrations).unwrap(), - vec![m04_2, m04_1, m03_1, m02_1] - ); + /// Migrator relies on the datastore symlink structure to determine the 'from' version. + /// This function sets up the directory and symlinks to mock the datastore for migrator. + fn create_datastore_links(info: &mut MigrationTestInfo) { + info.datastore = info.tmp.path().join(format!( + "v{}.{}.{}_xyz", + info.from_version.major, info.from_version.minor, info.from_version.patch + )); + let datastore_version = info.tmp.path().join(format!( + "v{}.{}.{}", + info.from_version.major, info.from_version.minor, info.from_version.patch + )); + let datastore_minor = info.tmp.path().join(format!( + "v{}.{}", + info.from_version.major, info.from_version.minor + )); + let datastore_major = info + .tmp + .path() + .join(format!("v{}", info.from_version.major)); + let datastore_current = info.tmp.path().join("current"); + fs::create_dir_all(&info.datastore).unwrap(); + std::os::unix::fs::symlink(&info.datastore, &datastore_version).unwrap(); + std::os::unix::fs::symlink(&datastore_version, &datastore_minor).unwrap(); + std::os::unix::fs::symlink(&datastore_minor, &datastore_major).unwrap(); + std::os::unix::fs::symlink(&datastore_major, &datastore_current).unwrap(); + } + + fn root() -> PathBuf { + test_data() + .join("repository") + .join("metadata") + .join("1.root.json") + } - // Test no matching migrations - assert!(select_migrations(&v04, &v05, &all_migrations) - .unwrap() - .is_empty()); + fn tuf_metadata() -> PathBuf { + test_data().join("repository").join("metadata") + } + + fn tuf_targets() -> PathBuf { + test_data().join("repository").join("targets") + } + + /// Tests the migrator program end-to-end using the `run` function. + /// The test uses a locally stored tuf repo at `migrator/tests/data/repository`. + /// In the `manifest.json` we have specified the following migrations: + /// ``` + /// "(0.99.0, 0.99.1)": [ + /// "x-first-migration.lz4", + /// "a-second-migration.lz4" + /// ] + /// ``` + /// + /// The two 'migrations' are bash scripts with content like this: + /// + /// ``` + /// #!/bin/bash + /// set -eo pipefail + /// migration_name="x-first-migration" + /// datastore_parent_dir="$(dirname "${3}")" + /// outfile="${datastore_parent_dir}/result.txt" + /// echo "${migration_name}: writing a message to '${outfile}'" + /// echo "${migration_name}:" "${@}" >> "${outfile}" + /// ``` + /// + /// These 'migrations' use the --source-datastore path and take its parent. + /// Into this parent directory they write lines to a file named result.txt. + /// In the test we read the result.txt file to see that the migrations have been run in the + /// expected order. + /// + /// This test ensures that migrations run when migrating from an older to a newer version. + #[test] + fn migrate_forward() { + let from_version = Version::parse("0.99.0").unwrap(); + let to_version = Version::parse("0.99.1").unwrap(); + let mut info = MigrationTestInfo::new(from_version, to_version); + create_datastore_links(&mut info); + let args = Args { + datastore_path: info.datastore.clone(), + log_level: log::LevelFilter::Info, + migration_directory: tuf_targets(), + migrate_to_version: info.to_version.clone(), + root_path: root(), + metadata_directory: tuf_metadata(), + }; + run(&args).unwrap(); + // the migrations should write to a file named result.txt. + let output_file = info.tmp.path().join("result.txt"); + let contents = std::fs::read_to_string(&output_file).unwrap(); + let lines: Vec<&str> = contents.split('\n').collect(); + assert_eq!(lines.len(), 3); + let first_line = *lines.get(0).unwrap(); + if !first_line.contains("x-first-migration: --forward") { + panic!(format!( + "Expected the migration 'x-first-migration.sh' to run first and write \ + a message containing 'x-first-migration: --forward' to the output file. Instead found \ + '{}'", + first_line + )); + } + let second_line = *lines.get(1).unwrap(); + if !second_line.contains("a-second-migration: --forward") { + panic!(format!( + "Expected the migration 'a-second-migration.sh' to run second and write \ + a message containing 'a-second-migration: --forward' to the output file. Instead found \ + '{}'", + second_line + )); + } + } + + /// This test ensures that migrations run when migrating from a newer to an older version. + /// See `migrate_forward` for a description of how these tests work. + #[test] + fn migrate_backward() { + let from_version = Version::parse("0.99.1").unwrap(); + let to_version = Version::parse("0.99.0").unwrap(); + let mut info = MigrationTestInfo::new(from_version, to_version); + create_datastore_links(&mut info); + let args = Args { + datastore_path: info.datastore.clone(), + log_level: log::LevelFilter::Info, + migration_directory: tuf_targets(), + migrate_to_version: info.to_version.clone(), + root_path: root(), + metadata_directory: tuf_metadata(), + }; + run(&args).unwrap(); + let output_file = info.tmp.path().join("result.txt"); + let contents = std::fs::read_to_string(&output_file).unwrap(); + let lines: Vec<&str> = contents.split('\n').collect(); + assert_eq!(lines.len(), 3); + let first_line = *lines.get(0).unwrap(); + if !first_line.contains("a-second-migration: --backward") { + panic!(format!( + "Expected the migration 'a-second-migration.sh' to run first and write \ + a message containing 'a-second-migration: --backward' to the output file. Instead \ + found '{}'", + first_line + )); + } + let second_line = *lines.get(1).unwrap(); + if !second_line.contains("x-first-migration: --backward") { + panic!(format!( + "Expected the migration 'x-first-migration.sh' to run second and write \ + a message containing 'x-first-migration: --backward' to the output file. Instead \ + found '{}'", + second_line + )); + } } } diff --git a/sources/api/migration/migrator/tests/data/repository/metadata/1.root.json b/sources/api/migration/migrator/tests/data/repository/metadata/1.root.json new file mode 100644 index 00000000000..04bc68d6273 --- /dev/null +++ b/sources/api/migration/migrator/tests/data/repository/metadata/1.root.json @@ -0,0 +1,71 @@ +{ + "signed": { + "_type": "root", + "spec_version": "1.0.0", + "consistent_snapshot": true, + "roles": { + "root": { + "keyids": [ + "c8cd4d9a1436b604690001db35f42c4c07246439ad79649881268e323730eb7c" + ], + "threshold": 1 + }, + "snapshot": { + "keyids": [ + "c63047e43001b3ca29cd6c1ab76b72871fc2c14459cefca5360efa5c3243f276" + ], + "threshold": 1 + }, + "targets": { + "keyids": [ + "5444ff6d392fc17e2b5bea23d2694570f98db5e15591e80144cecf20198ba0a2" + ], + "threshold": 1 + }, + "timestamp": { + "keyids": [ + "29ac0c464b44a29482e1e3c21388844ba2e5f1eb4201e3f7abdd53ba3415032d" + ], + "threshold": 1 + } + }, + "version": 1, + "keys": { + "29ac0c464b44a29482e1e3c21388844ba2e5f1eb4201e3f7abdd53ba3415032d": { + "keytype": "rsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzidGkP8LZ2JiXmpSq9FF\nZ0K+LbrmYgKsuwUIs2g/hJ4M2e9Ie3PmyFTdinfXR7wdeQvfEwFqnfSsp/v0C4i9\nV5l3377sIpvf6LzIlaBQ8mnSX1+FYXkHwFHlRkJeATD6E8hghCWOVhDp2Re1dOK2\npmI8FiQ+KdHW0NIk+/N2wnC0Y3zfa0GRVLQjFkezyfm2slKwyj/zacBrE0TaL32J\naeZERkMi+pFFPCNrYjSZ1ZgQPnWWjauWYLEL/EDZ+tNGfVhgJ+ewAKMljEPJTWoV\nZIC1/go727ev5KuH1RQv8gJpbNoM6rYooSq41OoH6CEWZSF2YbbgWBld+iOSQqT6\nswIDAQAB\n-----END PUBLIC KEY-----" + }, + "scheme": "rsassa-pss-sha256" + }, + "5444ff6d392fc17e2b5bea23d2694570f98db5e15591e80144cecf20198ba0a2": { + "keytype": "rsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6OPWcABrlHyAsBYYAVyh\nYdKCsqINuyKDGb3QjODo020GtCb7II6v16k3FunwiZmYCjfMMgK/+P/hwtqul6zC\nPVbbgrP9s48xZ/FEB3ZdjC8DgW2hYz5ZzPoowWG1JD7FKYCVwf0YFsR7FUvMyPaQ\nkv+aCqKDOKiQ3cpzuqJ10XHrXnbvWjtielxEO/JS056/aC++FyTbqV54TrfGrhiV\nKyYPgsJPQfcLKnOZ6V1XSkPoG67VbNS+CZ76haq23IBYzhd5hPIt5Kp4bU489TuE\nxRxnxSJYfl8HKSrhNPfQCgKqqKZQdKD16w85fHpL2jTp1pJHXd5Oo5dRmN4JiQnW\nJQIDAQAB\n-----END PUBLIC KEY-----" + }, + "scheme": "rsassa-pss-sha256" + }, + "c63047e43001b3ca29cd6c1ab76b72871fc2c14459cefca5360efa5c3243f276": { + "keytype": "rsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoMDnLfKYaZLTXFT9DY2X\nQoZNSxwNTRcfajGdndAfYc0gpilXBwzywqybsWBLxRUgAODEpvjdzV9+KovvaKdO\nFZQ7vOcJrHST9eDuz4whb+bilzrx6bWDx9tMngCQTtSuLgTHm7yJH/MEPai72q8h\niTdTgU4rqPxoyNIMxLW17bXP561/3KqkggY0gHuiTQ/+EffSIgnj6oAXKV6zmMqE\nhC4Lmpt1M13kjC17oplM3v3V7VsPtkRGr9CVN7rZQxCnjIPQKzBdELZAaZUCfC/1\nHSL70gsQg0DL4TBOGda2Dh4skpt8EPQmURyRBkWHbrPSUB1WsS2OG1v7ZgdhrxjY\nawIDAQAB\n-----END PUBLIC KEY-----" + }, + "scheme": "rsassa-pss-sha256" + }, + "c8cd4d9a1436b604690001db35f42c4c07246439ad79649881268e323730eb7c": { + "keytype": "rsa", + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx0akwzfs+ytEv9dkJznl\ntPiRlUGXkOX/i4Z+IZ7fUf6TL9DKrO/b2VO3LhEJ9vrz0p3ZzRZjfo6THsCzbI8c\n/BIhldk7Z3W8u6nifjjWPDP+4bWYQmhv459CAQGBm5ZdaVLMmD99Wq7nXXt5V1T4\nHHOkYoF4GON08l2rKtXHZc3qKlMy6EAmP9t+EmFJC1CzKj0Q8H6KPIkXJGcoDwbu\nY0nd6kSJTOyACPtMkG1bhOi36kVme2tRr+WRQkveJKAv0f7iGw+XSFaqAfYK7PTi\nxT/VLXw0ad8y4xcHkRR/Hb1QcSDLaJr/KCi1y37AJo5M8rRux96JFbDpAh/jbxTd\nYQIDAQAB\n-----END PUBLIC KEY-----" + }, + "scheme": "rsassa-pss-sha256" + } + }, + "expires": "2020-05-17T00:00:00Z" + }, + "signatures": [ + { + "keyid": "c8cd4d9a1436b604690001db35f42c4c07246439ad79649881268e323730eb7c", + "sig": "123eb397a9ea8ea1381759f11933aabf101a1e358119fb1282b14f6582199df0a4078275726952dea559044ec6987a9000663913449ec38ce4084864753eecc82f9f2976b7230652c7ad36d436876c5e9a4bd1e3377bcd62f6c6d7534d36319fab69bac6b06a48ae804643ded7095620e6ac8e44be1f0a7319926a94cf75feb4a210b156253705ee74243924fc3c79215d947a6950e1e49e50ceb2960856ec78b7315e118d327db26676a6ab16e3db0ad6785271c7458bc0a4ed9521079fbead50999411f2ad0c6e8e64f3fa8ed5cc2593ac187486a4764acc1de25726b624f281e81475de66914aa6698184315980020f1d090c935a46adce6c647e065dfecb" + } + ] +} diff --git a/sources/api/migration/migrator/tests/data/repository/metadata/1589415932.snapshot.json b/sources/api/migration/migrator/tests/data/repository/metadata/1589415932.snapshot.json new file mode 100644 index 00000000000..e49728425d3 --- /dev/null +++ b/sources/api/migration/migrator/tests/data/repository/metadata/1589415932.snapshot.json @@ -0,0 +1,30 @@ +{ + "signed": { + "_type": "snapshot", + "spec_version": "1.0.0", + "version": 1589415932, + "expires": "2020-05-17T00:00:00Z", + "meta": { + "targets.json": { + "length": 1348, + "hashes": { + "sha256": "0a6d7bdea71298e9090da156d81cc71508394e40c31c5b85bff5dda74305bc3c" + }, + "version": 1589415932 + }, + "root.json": { + "length": 4128, + "hashes": { + "sha256": "2efb61bc599de91cdb8ca6a3b6666c911584b934bb79f7aa9fdd351c5a38e348" + }, + "version": 1 + } + } + }, + "signatures": [ + { + "keyid": "c63047e43001b3ca29cd6c1ab76b72871fc2c14459cefca5360efa5c3243f276", + "sig": "157525bdcc54551e66521fb6cc97ef185e1e3b9a768f095124cd70a137bcb08b7d25b69c4d2623a24f97d3437c3c74beecfddea6b9975785de16d4bb999ea37e07af973d59b79f916e47d10ed95ac7f7302902c16e2e84ca5cf91aca8edf86681d98611758c4acf0a2567fd89733865fb15060500273cdbd407ecca6c26ff564c8250b56c4425ead5d380626521e0ea6852b0bae045ce7cb8b4bb0d9b540d15d020bd1f0d1a5de54a0fd524ff4cbfffe630e73785a2e6b87cf5624d2c4b21a84e2e434286aaac6f98235aa1084c88f929ae61a49afed1bd941c141ae9f3bb63f0245fb1758a883f133d2c00ed27d784d7816b6bbdbd377545c1ba3e3fe68aa27" + } + ] +} diff --git a/sources/api/migration/migrator/tests/data/repository/metadata/1589415932.targets.json b/sources/api/migration/migrator/tests/data/repository/metadata/1589415932.targets.json new file mode 100644 index 00000000000..d83d8c05d0b --- /dev/null +++ b/sources/api/migration/migrator/tests/data/repository/metadata/1589415932.targets.json @@ -0,0 +1,34 @@ +{ + "signed": { + "_type": "targets", + "spec_version": "1.0.0", + "version": 1589415932, + "expires": "2020-05-17T00:00:00Z", + "targets": { + "x-first-migration.lz4": { + "length": 224, + "hashes": { + "sha256": "f0a66d3ffea99eebc24f1966d484b8616160e46f609eb7056d997b76b847bc12" + } + }, + "a-second-migration.lz4": { + "length": 225, + "hashes": { + "sha256": "b0815d371a7536c104f06fb836be42911856989fa5be4bf01541fdcf8ad51095" + } + }, + "manifest.json": { + "length": 136, + "hashes": { + "sha256": "9da13a22f501fe0b436871cf925ef8170ca5a14dda90013eac67a5df8b10bee4" + } + } + } + }, + "signatures": [ + { + "keyid": "5444ff6d392fc17e2b5bea23d2694570f98db5e15591e80144cecf20198ba0a2", + "sig": "2800a8880ff1faca821546dda95ec8d4ef43274a928fd2d4a5c6b1fabd24165c6d436e75af46e529542906b5072e84fe27bb707ec423a0618c86b258a0440d97fb4785dec3a94e6a1dc25c748ddbf4ee5b4d5d9b37d2029a3846b80dace9f20974dc22046a721e75765fe41ce2563ebaed517943e26e0c8f7a692db5456cd9dadd728a4bea63844ee29b4d86a7a015056af355933cfe5b26d7e6d3efe7f10ca9a7d5717a2dc39ea88324fa37b9e878bb233adf57ca5add9eac1338856b06ad587741edd33d95f6bfbe4b476515eb2db817a95daf2427765e739b80ead50cf6ac20a150924a3b2e34338935ae295059c3a891522e1a876c08da0ffe52937c71ee" + } + ] +} diff --git a/sources/api/migration/migrator/tests/data/repository/metadata/timestamp.json b/sources/api/migration/migrator/tests/data/repository/metadata/timestamp.json new file mode 100644 index 00000000000..941068203f8 --- /dev/null +++ b/sources/api/migration/migrator/tests/data/repository/metadata/timestamp.json @@ -0,0 +1,23 @@ +{ + "signed": { + "_type": "timestamp", + "spec_version": "1.0.0", + "version": 1589415932, + "expires": "2020-05-17T00:00:00Z", + "meta": { + "snapshot.json": { + "length": 1205, + "hashes": { + "sha256": "0d9ad8a4a567527900b0ba7b3c14821b17474543f2d47337a2eed08c15fa1a32" + }, + "version": 1589415932 + } + } + }, + "signatures": [ + { + "keyid": "29ac0c464b44a29482e1e3c21388844ba2e5f1eb4201e3f7abdd53ba3415032d", + "sig": "74a8a13696f15b7ced60f79b2c0e5fe2be5c6131842f3467281926e020e9011f92c4e6885c8c7662063a265881f7dfcfbc66b5eab6d97a15a9ad8a8a8c59153e60f44b47dfc776264e3918a5c793bcbd69cffa9b324e0720a3cf3659564c3d1616e7f2b09ec76cebbf3fba6a2e3adad818271151f70ee86b1555effc82392826978f3d0b5d136aa5ff9e2d3da12190a4d7be31d3cb5e7fe67b93ffc3e9e75b6b4187fcb4d19886819f7eb77132880daf9f0198dc2abf041f233913a00c2bc8ea5c07f7de0411f29e3e1bf684d51588e632e26b1651efd6eeaa2da8aa4ef2724744305b00cc3ab5b71848479cdccc3eddb0fb995c8af174cd8d8c6e1b7c8cabb7" + } + ] +} diff --git a/sources/api/migration/migrator/tests/data/repository/targets/9da13a22f501fe0b436871cf925ef8170ca5a14dda90013eac67a5df8b10bee4.manifest.json b/sources/api/migration/migrator/tests/data/repository/targets/9da13a22f501fe0b436871cf925ef8170ca5a14dda90013eac67a5df8b10bee4.manifest.json new file mode 100644 index 00000000000..bddfd4433e3 --- /dev/null +++ b/sources/api/migration/migrator/tests/data/repository/targets/9da13a22f501fe0b436871cf925ef8170ca5a14dda90013eac67a5df8b10bee4.manifest.json @@ -0,0 +1,9 @@ +{ + "updates": [], + "migrations": { + "(0.99.0, 0.99.1)": [ + "x-first-migration.lz4", + "a-second-migration.lz4" + ] + } +} \ No newline at end of file diff --git a/sources/api/migration/migrator/tests/data/repository/targets/a-second-migration.sh b/sources/api/migration/migrator/tests/data/repository/targets/a-second-migration.sh new file mode 100644 index 00000000000..15cb3126a23 --- /dev/null +++ b/sources/api/migration/migrator/tests/data/repository/targets/a-second-migration.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -eo pipefail +migration_name="a-second-migration" +datastore_parent_dir="$(dirname "${3}")" +outfile="${datastore_parent_dir}/result.txt" +echo "${migration_name}: writing a message to '${outfile}'" +echo "${migration_name}:" "${@}" >> "${outfile}" diff --git a/sources/api/migration/migrator/tests/data/repository/targets/b0815d371a7536c104f06fb836be42911856989fa5be4bf01541fdcf8ad51095.a-second-migration.lz4 b/sources/api/migration/migrator/tests/data/repository/targets/b0815d371a7536c104f06fb836be42911856989fa5be4bf01541fdcf8ad51095.a-second-migration.lz4 new file mode 100644 index 00000000000..f33329bbc70 Binary files /dev/null and b/sources/api/migration/migrator/tests/data/repository/targets/b0815d371a7536c104f06fb836be42911856989fa5be4bf01541fdcf8ad51095.a-second-migration.lz4 differ diff --git a/sources/api/migration/migrator/tests/data/repository/targets/f0a66d3ffea99eebc24f1966d484b8616160e46f609eb7056d997b76b847bc12.x-first-migration.lz4 b/sources/api/migration/migrator/tests/data/repository/targets/f0a66d3ffea99eebc24f1966d484b8616160e46f609eb7056d997b76b847bc12.x-first-migration.lz4 new file mode 100644 index 00000000000..08127efc719 Binary files /dev/null and b/sources/api/migration/migrator/tests/data/repository/targets/f0a66d3ffea99eebc24f1966d484b8616160e46f609eb7056d997b76b847bc12.x-first-migration.lz4 differ diff --git a/sources/api/migration/migrator/tests/data/repository/targets/x-first-migration.sh b/sources/api/migration/migrator/tests/data/repository/targets/x-first-migration.sh new file mode 100644 index 00000000000..6d178c697ed --- /dev/null +++ b/sources/api/migration/migrator/tests/data/repository/targets/x-first-migration.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -eo pipefail +migration_name="x-first-migration" +datastore_parent_dir="$(dirname "${3}")" +outfile="${datastore_parent_dir}/result.txt" +echo "${migration_name}: writing a message to '${outfile}'" +echo "${migration_name}:" "${@}" >> "${outfile}" diff --git a/sources/api/migration/migrator/src/direction.rs b/sources/updater/update_metadata/src/direction.rs similarity index 92% rename from sources/api/migration/migrator/src/direction.rs rename to sources/updater/update_metadata/src/direction.rs index ef250cc9679..e2d484274a1 100644 --- a/sources/api/migration/migrator/src/direction.rs +++ b/sources/updater/update_metadata/src/direction.rs @@ -2,13 +2,13 @@ //! is moving forward to a new version or rolling back to a previous version. use semver::Version; -use std::cmp::Ordering; +use std::cmp::{Ordering, Ord}; use std::fmt; /// Direction represents whether we're moving forward toward a newer version, or rolling back to /// an older version. #[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub(crate) enum Direction { +pub enum Direction { Forward, Backward, } @@ -25,7 +25,7 @@ impl fmt::Display for Direction { impl Direction { /// Determines the migration direction, given the outgoing ("from') and incoming ("to") /// versions. - pub(crate) fn from_versions(from: &Version, to: &Version) -> Option { + pub fn from_versions(from: &Version, to: &Version) -> Option { match from.cmp(&to) { Ordering::Less => Some(Direction::Forward), Ordering::Greater => Some(Direction::Backward), diff --git a/sources/updater/update_metadata/src/lib.rs b/sources/updater/update_metadata/src/lib.rs index c9c76ab6086..7a7229c8b38 100644 --- a/sources/updater/update_metadata/src/lib.rs +++ b/sources/updater/update_metadata/src/lib.rs @@ -1,43 +1,35 @@ #![warn(clippy::pedantic)] mod de; +mod direction; pub mod error; mod se; use crate::error::Result; use chrono::{DateTime, Duration, Utc}; -use lazy_static::lazy_static; +pub use direction::Direction; use parse_datetime::parse_offset; use rand::{thread_rng, Rng}; -use regex::Regex; use semver::Version; use serde::{Deserialize, Serialize}; use snafu::{ensure, OptionExt, ResultExt}; +use std::cmp::Ordering; use std::collections::BTreeMap; use std::fs; use std::fs::File; use std::ops::Bound::{Excluded, Included}; use std::path::Path; -use std::str::FromStr; -use tough; +use tough::{self, Limits}; pub const MAX_SEED: u32 = 2048; -lazy_static! { - /// Regular expression that will match migration file names and allow retrieving the - /// version and name components. - // Note: the version component is a simplified semver regex; we don't use any of the - // extensions, just a simple x.y.z, so this isn't as strict as it could be. - pub static ref MIGRATION_FILENAME_RE: Regex = - Regex::new(r"(?x)^ - migrate - _ - v? # optional 'v' prefix for humans - (?P[0-9]+\.[0-9]+\.[0-9]+[0-9a-zA-Z+-]*) - _ - (?P[a-zA-Z0-9-]+) - $").unwrap(); -} +/// These are the limits that Bottlerocket will use for the `tough` library. +pub const REPOSITORY_LIMITS: Limits = Limits { + max_root_size: 1024 * 1024, // 1 MiB + max_targets_size: 1024 * 1024 * 10, // 10 MiB + max_timestamp_size: 1024 * 1024, // 1 MiB + max_root_updates: 1024, +}; #[derive(Debug, PartialEq, Eq)] pub enum Wave { @@ -132,48 +124,6 @@ pub fn write_file(path: &Path, manifest: &Manifest) -> Result<()> { } impl Manifest { - pub fn add_migration( - &mut self, - append: bool, - from: Version, - to: Version, - migration_list: Vec, - ) -> Result<()> { - // Check each migration matches the filename conventions used by the migrator - for name in &migration_list { - let captures = MIGRATION_FILENAME_RE - .captures(&name) - .context(error::MigrationNaming)?; - - let version_match = captures - .name("version") - .context(error::BadRegexVersion { name })?; - let version = Version::from_str(version_match.as_str()) - .context(error::BadVersion { key: name })?; - ensure!( - version == to, - error::MigrationInvalidTarget { name, to, version } - ); - - let _ = captures - .name("name") - .context(error::BadRegexName { name })?; - } - - // If append is true, append the new migrations to the existing vec. - if append && self.migrations.contains_key(&(from.clone(), to.clone())) { - let migrations = self - .migrations - .get_mut(&(from.clone(), to.clone())) - .context(error::MigrationMutable { from, to })?; - migrations.extend_from_slice(&migration_list); - // Otherwise just overwrite the existing migrations - } else { - self.migrations.insert((from, to), migration_list); - } - Ok(()) - } - pub fn add_update( &mut self, image_version: Version, @@ -387,7 +337,32 @@ impl Update { } } -pub fn migration_targets(from: &Version, to: &Version, manifest: &Manifest) -> Result> { +pub fn find_migrations(from: &Version, to: &Version, manifest: &Manifest) -> Result> { + // early exit if there is no work to do. + if from == to { + return Ok(Vec::new()); + } + // express the versions in ascending order + let (lower, higher, is_reversed) = match from.cmp(to) { + Ordering::Less | Ordering::Equal => (from, to, false), + Ordering::Greater => (to, from, true), + }; + let mut migrations = find_migrations_forward(&lower, &higher, manifest)?; + // if the direction is backward, reverse the order of the migration list. + if is_reversed { + migrations = migrations.into_iter().rev().collect(); + } + Ok(migrations) +} + +/// Finds the migration from one version to another. The migration direction must be forward, that +/// is, `from` must be less than or equal to `to`. The caller may reverse the Vec returned by this +/// function to migrate backward. +fn find_migrations_forward( + from: &Version, + to: &Version, + manifest: &Manifest, +) -> Result> { let mut targets = Vec::new(); let mut version = from; while version != to { @@ -431,14 +406,14 @@ pub fn load_manifest(repository: &tough::Repository) -> } #[test] -fn test_migrations() { +fn test_migrations_forward() { // A manifest with four migration tuples starting at 1.0 and ending at 1.3. // There is a shortcut from 1.1 to 1.3, skipping 1.2 let path = "./tests/data/migrations.json"; let manifest: Manifest = serde_json::from_reader(File::open(path).unwrap()).unwrap(); let from = Version::parse("1.0.0").unwrap(); let to = Version::parse("1.5.0").unwrap(); - let targets = migration_targets(&from, &to, &manifest).unwrap(); + let targets = find_migrations(&from, &to, &manifest).unwrap(); assert!(targets.len() == 3); let mut i = targets.iter(); @@ -446,3 +421,19 @@ fn test_migrations() { assert!(i.next().unwrap() == "migration_1.1.0_b"); assert!(i.next().unwrap() == "migration_1.5.0_shortcut"); } + +#[test] +fn test_migrations_backward() { + // The same manifest as `test_migrations_forward` but this time we will migrate backward. + let path = "./tests/data/migrations.json"; + let manifest: Manifest = serde_json::from_reader(File::open(path).unwrap()).unwrap(); + let from = Version::parse("1.5.0").unwrap(); + let to = Version::parse("1.0.0").unwrap(); + let targets = find_migrations(&from, &to, &manifest).unwrap(); + + assert!(targets.len() == 3); + let mut i = targets.iter(); + assert!(i.next().unwrap() == "migration_1.5.0_shortcut"); + assert!(i.next().unwrap() == "migration_1.1.0_b"); + assert!(i.next().unwrap() == "migration_1.1.0_a"); +} diff --git a/sources/updater/updog/Cargo.toml b/sources/updater/updog/Cargo.toml index f861fa0d9f6..62c684b5d31 100644 --- a/sources/updater/updog/Cargo.toml +++ b/sources/updater/updog/Cargo.toml @@ -12,7 +12,6 @@ chrono = "0.4.9" log = "0.4" lz4 = "1.23.1" rand = "0.7.0" -regex = "1.1" reqwest = { version = "0.10.1", default-features = false, features = ["rustls-tls", "blocking"] } semver = "0.9.0" serde = { version = "1.0.100", features = ["derive"] } @@ -23,6 +22,7 @@ simplelog = "0.7" snafu = "0.6.0" toml = "0.5.1" tough = { version = "0.5.0", features = ["http"] } +tempfile = "3.1.0" update_metadata = { path = "../update_metadata" } structopt = "0.3" migrator = { path = "../../api/migration/migrator" } diff --git a/sources/updater/updog/src/error.rs b/sources/updater/updog/src/error.rs index 949806a9e9c..5c08f97b61d 100644 --- a/sources/updater/updog/src/error.rs +++ b/sources/updater/updog/src/error.rs @@ -31,8 +31,15 @@ pub(crate) enum Error { backtrace: Backtrace, }, - #[snafu(display("Failed to create metadata cache directory: {}", source))] + #[snafu(display("Failed to create metadata cache directory '{}': {}", path, source))] CreateMetadataCache { + path: &'static str, + source: std::io::Error, + backtrace: Backtrace, + }, + + #[snafu(display("Failed to create a tempdir for tough datastore: {}", source))] + CreateTempDir { source: std::io::Error, backtrace: Backtrace, }, @@ -235,6 +242,9 @@ pub(crate) enum Error { source: std::io::Error, backtrace: Backtrace, }, + + #[snafu(display("Failed to store manifest and migrations: {}", source))] + RepoCacheMigrations { source: tough::error::Error }, } impl std::convert::From for Error { diff --git a/sources/updater/updog/src/main.rs b/sources/updater/updog/src/main.rs index 21c87abe25f..102b943ebac 100644 --- a/sources/updater/updog/src/main.rs +++ b/sources/updater/updog/src/main.rs @@ -14,23 +14,28 @@ use signal_hook::{iterator::Signals, SIGTERM}; use signpost::State; use simplelog::{Config as LogConfig, LevelFilter, TermLogger, TerminalMode}; use snafu::{ensure, ErrorCompat, OptionExt, ResultExt}; -use std::fs::{self, File, OpenOptions, Permissions}; +use std::fs::{self, File, OpenOptions}; use std::io; -use std::os::unix::fs::PermissionsExt; use std::path::Path; use std::process; use std::str::FromStr; use std::thread; -use tough::{Limits, ExpirationEnforcement, Repository, Settings}; -use update_metadata::{load_manifest, migration_targets, Manifest, Update}; +use tempfile::TempDir; +use tough::{ExpirationEnforcement, Repository, Settings}; +use update_metadata::{find_migrations, load_manifest, Manifest, Update, REPOSITORY_LIMITS}; #[cfg(target_arch = "x86_64")] const TARGET_ARCH: &str = "x86_64"; #[cfg(target_arch = "aarch64")] const TARGET_ARCH: &str = "aarch64"; +/// The root.json file as required by TUF. const TRUSTED_ROOT_PATH: &str = "/usr/share/updog/root.json"; + +/// This is where we store the TUF targets used by migrator after reboot. const MIGRATION_PATH: &str = "/var/lib/bottlerocket-migrations"; + +/// This is where we store the TUF metadata used by migrator after reboot. const METADATA_PATH: &str = "/var/cache/bottlerocket-metadata"; #[derive(Debug, Deserialize, PartialEq)] @@ -104,23 +109,21 @@ fn load_config() -> Result { fn load_repository<'a>( transport: &'a HttpQueryTransport, config: &'a Config, + tough_datastore: &'a Path, ) -> Result> { - fs::create_dir_all(METADATA_PATH).context(error::CreateMetadataCache)?; + fs::create_dir_all(METADATA_PATH).context(error::CreateMetadataCache { + path: METADATA_PATH, + })?; Repository::load( transport, Settings { root: File::open(TRUSTED_ROOT_PATH).context(error::OpenRoot { path: TRUSTED_ROOT_PATH, })?, - datastore: Path::new(METADATA_PATH), + datastore: tough_datastore, metadata_base_url: &config.metadata_base_url, targets_base_url: &config.targets_base_url, - limits: Limits { - max_root_size: 1024 * 1024, // 1 MiB - max_targets_size: 1024 * 1024 * 10, // 10 MiB - max_timestamp_size: 1024 * 1024, // 1 MiB - max_root_updates: 1024, - }, + limits: REPOSITORY_LIMITS, expiration_enforcement: ExpirationEnforcement::Safe, }, ) @@ -210,20 +213,14 @@ fn retrieve_migrations( fs::create_dir(&dir).context(error::DirCreate { path: &dir })?; } - // download each migration, making sure they are executable and removing - // known extensions from our compression, e.g. .lz4 - let mut targets = migration_targets(start, target, &manifest)?; - targets.sort(); - for name in &targets { - let mut destination = dir.join(&name); - if destination.extension() == Some("lz4".as_ref()) { - destination.set_extension(""); - } - write_target_to_disk(repository, &name, &destination)?; - fs::set_permissions(&destination, Permissions::from_mode(0o755)) - .context(error::SetPermissions { path: destination })?; - } - + // find the list of migrations in the manifest based on our from and to versions. + let mut targets = find_migrations(start, target, &manifest)?; + // Even if there are no migrations, we need to make sure that we store the manifest so that + // migrator can independently and securely determine that there are no migrations. + targets.push("manifest.json".to_owned()); + repository + .cache(METADATA_PATH, MIGRATION_PATH, Some(&targets), true) + .context(error::RepoCacheMigrations)?; // Set a query parameter listing the required migrations transport .queries_get_mut() @@ -445,7 +442,8 @@ fn main_inner() -> Result<()> { let variant = arguments.variant.unwrap_or(current_release.variant_id); let transport = HttpQueryTransport::new(); set_common_query_params(&transport, ¤t_release.version_id, &config)?; - let repository = load_repository(&transport, &config)?; + let tough_datastore = TempDir::new().context(error::CreateTempDir)?; + let repository = load_repository(&transport, &config, tough_datastore.path())?; let manifest = load_manifest(&repository)?; match command {