From 4ac022b4781bf6aafbc3f1b7ff6ad3205fa4ff43 Mon Sep 17 00:00:00 2001 From: lklimek <842586+lklimek@users.noreply.github.com> Date: Thu, 28 Sep 2023 17:45:01 +0200 Subject: [PATCH] feat(drive): drive-abci verify grovedb CLI (#1427) --- Cargo.lock | 1 + docs/DRIVE-ABCI.md | 45 ++++++++ packages/rs-drive-abci/Cargo.toml | 3 + packages/rs-drive-abci/src/main.rs | 162 +++++++++++++++++++++++++++++ 4 files changed, 211 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index dbf5e00f1b..f4d89d8bae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1297,6 +1297,7 @@ dependencies = [ "prost", "rand", "regex", + "rocksdb", "rust_decimal", "rust_decimal_macros", "serde", diff --git a/docs/DRIVE-ABCI.md b/docs/DRIVE-ABCI.md index 2be4a34d24..82681b9d9e 100644 --- a/docs/DRIVE-ABCI.md +++ b/docs/DRIVE-ABCI.md @@ -76,3 +76,48 @@ export ABCI_LOG_EXAMPLE_MAX_FILES=10 This configuration specifies that logs for the "EXAMPLE" destination should be stored in the /var/log/example directory, with a verbosity level of 3. Colorful output should not be used, and the logs should be formatted in a human-readable and visually appealing manner. The maximum number of daily log files to store is set to 10. Ensure that you adjust the values according to your specific logging requirements. + +## Integrity checks + +## `drive-abci verify` + +The `drive-abci verify` command is used to verify the integrity of the database used by `drive-abci`. +This command will execute GroveDB hash integrity checks to ensure that the database is consistent +and free of corruption. + +### Usage + +To use the `drive-abci verify` command, simply run the following command: + +```bash +drive-abci verify +``` + +This will execute the GroveDB hash integrity checks and report any errors or inconsistencies found in the database. + +### Enforcing Integrity Checks + +You can also enforce GroveDB integrity checks during `drive-abci start` by creating a `.fsck` file in the database +directory (`DB_PATH`). This file should be created before starting `drive-abci`, and can be empty. + +When `drive-abci` starts up, it checks for the presence of the `.fsck` file in the database directory. +If the file is present, it executes the specified integrity checks. After the checks are completed, +the `.fsck` file is deleted from the database directory. + +### Example: Verifying consistency of `drive-abci` running in Docker + +To verify integrity of database when `drive-abci` runs in a Docker container, you can create a `.fsck` file in the +database directory and and restart the container. + +For example, for a drive-abci container `dashmate_ccc1e5c2_local_1-drive_abci-1`, you can execute the following commands: + +```bash +docker exec -ti dashmate_ccc1e5c2_local_1-drive_abci-1 touch db/.fsck +docker restart dashmate_ccc1e5c2_local_1-drive_abci-1 +``` + +You can check the result of verification in logs by running the following command: + +```bash +docker logs dashmate_ccc1e5c2_local_1-drive_abci-1 --tail 1000 2>&1 | grep 'grovedb verification' +``` diff --git a/packages/rs-drive-abci/Cargo.toml b/packages/rs-drive-abci/Cargo.toml index 5227d6ec04..e8346e1964 100644 --- a/packages/rs-drive-abci/Cargo.toml +++ b/packages/rs-drive-abci/Cargo.toml @@ -92,6 +92,9 @@ dpp = { path = "../rs-dpp", features = [ ] } drive = { path = "../rs-drive" } +# For tests of grovedb verify +rocksdb = { version = "0.21.0" } + [features] default = ["server", "mocks"] server = ["clap", "dotenvy"] diff --git a/packages/rs-drive-abci/src/main.rs b/packages/rs-drive-abci/src/main.rs index e2424ff8c2..9f8eb15c3d 100644 --- a/packages/rs-drive-abci/src/main.rs +++ b/packages/rs-drive-abci/src/main.rs @@ -9,6 +9,7 @@ use drive_abci::logging::{LogBuilder, LogConfig, Loggers}; use drive_abci::metrics::{Prometheus, DEFAULT_PROMETHEUS_PORT}; use drive_abci::rpc::core::DefaultCoreRPC; use itertools::Itertools; +use std::fs::remove_file; use std::path::PathBuf; use std::process::ExitCode; use tokio::runtime::Builder; @@ -35,6 +36,15 @@ enum Commands { /// Returns 0 on success. #[command()] Status, + + /// Verify integrity of database. + /// + /// This command will execute GroveDB hash integrity checks. + /// + /// You can also enforce grovedb integrity checks during `drive-abci start` + /// by creating `.fsck` file in database directory (`DB_PATH`). + #[command()] + Verify, } /// Server that accepts connections from Tenderdash, and @@ -76,6 +86,8 @@ impl Cli { fn run(self, config: PlatformConfig, cancel: CancellationToken) -> Result<(), String> { match self.command { Commands::Start => { + verify_grovedb(&config.db_path, false)?; + let core_rpc = DefaultCoreRPC::open( config.core.rpc.url().as_str(), config.core.rpc.username.clone(), @@ -95,6 +107,7 @@ impl Cli { } Commands::Config => dump_config(&config)?, Commands::Status => check_status(&config)?, + Commands::Verify => verify_grovedb(&config.db_path, true)?, }; Ok(()) @@ -233,6 +246,56 @@ fn check_status(config: &PlatformConfig) -> Result<(), String> { } } +/// Verify GroveDB integrity. +/// +/// This function will execute GroveDB integrity checks if one of the following conditions is met: +/// - `force` is `true` +/// - file `.fsck` in `config.db_path` exists +/// +/// After successful verification, .fsck file is removed. +fn verify_grovedb(db_path: &PathBuf, force: bool) -> Result<(), String> { + let fsck = PathBuf::from(db_path).join(".fsck"); + + if !force { + if !fsck.exists() { + return Ok(()); + } + tracing::info!( + "found {} file, starting grovedb verification", + fsck.display() + ); + } + + let grovedb = drive::grovedb::GroveDb::open(db_path).expect("open grovedb"); + let result = grovedb + .visualize_verify_grovedb() + .map_err(|e| e.to_string()); + + match result { + Ok(data) => { + for result in data { + tracing::warn!(?result, "grovedb verification") + } + tracing::info!("grovedb verification finished"); + + if fsck.exists() { + if let Err(e) = remove_file(&fsck) { + tracing::warn!( + error = ?e, + path =fsck.display().to_string(), + "grovedb verification: cannot remove .fsck file: please remove it manually to avoid running verification again", + ); + } + } + Ok(()) + } + Err(e) => { + tracing::error!("grovedb verification failed: {}", e); + Err(e) + } + } +} + fn load_config(path: &Option) -> PlatformConfig { if let Some(path) = path { if let Err(e) = dotenvy::from_path(path) { @@ -290,3 +353,102 @@ fn install_panic_hook(cancel: CancellationToken) { cancel.cancel(); })); } + +#[cfg(test)] +mod test { + use std::{ + fs, + path::{Path, PathBuf}, + }; + + use ::drive::{drive::Drive, fee_pools::epochs::paths::EpochProposers, query::Element}; + use dpp::block::epoch::Epoch; + use drive::fee_pools::epochs::epoch_key_constants; + + use platform_version::version::PlatformVersion; + use rocksdb::{IteratorMode, Options}; + + /// Setup drive database by creating initial state structure and inserting some data. + /// + /// Returns path to the database. + fn setup_db(tempdir: &Path) -> PathBuf { + let path = tempdir.join("db"); + fs::create_dir(&path).expect("create db dir"); + + let drive = Drive::open(&path, None).expect("open drive"); + + let platform_version = PlatformVersion::latest(); + drive + .create_initial_state_structure(None, platform_version) + .expect("should create root tree successfully"); + + let transaction = drive.grove.start_transaction(); + let epoch = Epoch::new(0).unwrap(); + + let i = 100; + + drive + .grove + .insert( + &epoch.get_path(), + epoch_key_constants::KEY_FEE_MULTIPLIER.as_slice(), + Element::Item((i as u128).to_be_bytes().to_vec(), None), + None, + Some(&transaction), + ) + .unwrap() + .expect("should insert data"); + + transaction.commit().unwrap(); + + path + } + + /// Open RocksDB and corrupt `n`-th item from `cf` column family. + fn corrupt_rocksdb_item(db_path: &PathBuf, cf: &str, n: usize) { + let mut db_opts = Options::default(); + + db_opts.create_missing_column_families(false); + db_opts.create_if_missing(false); + + let db = rocksdb::DB::open_cf(&db_opts, &db_path, vec!["roots", "meta", "aux"]).unwrap(); + + let cf_handle = db.cf_handle(cf).unwrap(); + let iter = db.iterator_cf(cf_handle, IteratorMode::Start); + + // let iter = db.iterator(IteratorMode::Start); + let mut i = 0; + for item in iter { + let (key, mut value) = item.unwrap(); + // println!("{} = {}", hex::encode(&key), hex::encode(value)); + tracing::trace!(cf, key=?hex::encode(&key), value=hex::encode(&value),"found item in rocksdb"); + + if i == n { + value[0] = !value[0]; + db.put_cf(cf_handle, &key, &value).unwrap(); + + tracing::debug!(cf, key=?hex::encode(&key), value=hex::encode(&value), "corrupt_rocksdb_item: corrupting item"); + return; + } + i += 1; + } + panic!( + "cannot corrupt db: cannot find {}-th item in rocksdb column family {}", + n, cf + ); + } + + #[test] + fn test_verify_grovedb_corrupt_0th_root() { + drive_abci::logging::init_for_tests(4); + let tempdir = tempfile::tempdir().unwrap(); + let db_path = setup_db(tempdir.path()); + + corrupt_rocksdb_item(&db_path, "roots", 0); + + let result = super::verify_grovedb(&db_path, true); + assert!(result.is_err()); + + println!("db path: {:?}", &db_path); + } +}