From aae606ffeca733a53ef922b393389bfbac1e3216 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 27 Sep 2023 10:52:30 +0200
Subject: [PATCH 1/4] feat(drive): drive-abci verify grovedb CLI

---
 Cargo.lock                         |  19 ++---
 packages/rs-drive-abci/Cargo.toml  |   3 +
 packages/rs-drive-abci/src/main.rs | 120 +++++++++++++++++++++++++++++
 packages/rs-drive/Cargo.toml       |  12 ++-
 4 files changed, 138 insertions(+), 16 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 6351136cee..50ba3c6149 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1299,6 +1299,7 @@ dependencies = [
  "prost",
  "rand",
  "regex",
+ "rocksdb",
  "rust_decimal",
  "rust_decimal_macros",
  "serde",
@@ -1754,8 +1755,7 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 [[package]]
 name = "grovedb"
 version = "1.0.0-rc.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2450b5791de217465b03891586fa847bea3b761362233504e201f72d19bb7449"
+source = "git+https://github.com/dashpay/grovedb?rev=75b4df1177a830a7a280d65a8c179c93c225fe73#75b4df1177a830a7a280d65a8c179c93c225fe73"
 dependencies = [
  "bincode 1.3.3",
  "grovedb-costs",
@@ -1777,8 +1777,7 @@ dependencies = [
 [[package]]
 name = "grovedb-costs"
 version = "1.0.0-rc.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f5d0318f90811a3001b9c37aeeb2bf5243a760ccb87b35b640578f27fd0b017b"
+source = "git+https://github.com/dashpay/grovedb?rev=75b4df1177a830a7a280d65a8c179c93c225fe73#75b4df1177a830a7a280d65a8c179c93c225fe73"
 dependencies = [
  "integer-encoding",
  "intmap",
@@ -1788,8 +1787,7 @@ dependencies = [
 [[package]]
 name = "grovedb-merk"
 version = "1.0.0-rc.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24a2c2282ad812281faff6ec45d2c9fa126727b3281be9951732d6173f32731f"
+source = "git+https://github.com/dashpay/grovedb?rev=75b4df1177a830a7a280d65a8c179c93c225fe73#75b4df1177a830a7a280d65a8c179c93c225fe73"
 dependencies = [
  "blake3",
  "byteorder",
@@ -1813,14 +1811,12 @@ dependencies = [
 [[package]]
 name = "grovedb-path"
 version = "1.0.0-rc.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b0ae650cd3e696e8016f3390c074719eba19a96bcb6b4e075567b5ca62678b"
+source = "git+https://github.com/dashpay/grovedb?rev=75b4df1177a830a7a280d65a8c179c93c225fe73#75b4df1177a830a7a280d65a8c179c93c225fe73"
 
 [[package]]
 name = "grovedb-storage"
 version = "1.0.0-rc.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7752c194cf97445cdc037cfea754fc363153bcc85fee0b02996d4e87c5d880aa"
+source = "git+https://github.com/dashpay/grovedb?rev=75b4df1177a830a7a280d65a8c179c93c225fe73#75b4df1177a830a7a280d65a8c179c93c225fe73"
 dependencies = [
  "blake3",
  "grovedb-costs",
@@ -1839,8 +1835,7 @@ dependencies = [
 [[package]]
 name = "grovedb-visualize"
 version = "1.0.0-rc.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b36deef35537c149ed523baee2a0ae9d0d09a8f3b025bca3d17fcbea573f9c08"
+source = "git+https://github.com/dashpay/grovedb?rev=75b4df1177a830a7a280d65a8c179c93c225fe73#75b4df1177a830a7a280d65a8c179c93c225fe73"
 dependencies = [
  "hex",
  "itertools",
diff --git a/packages/rs-drive-abci/Cargo.toml b/packages/rs-drive-abci/Cargo.toml
index e3074ca4c5..bf3720ef42 100644
--- a/packages/rs-drive-abci/Cargo.toml
+++ b/packages/rs-drive-abci/Cargo.toml
@@ -93,6 +93,9 @@ dpp = { path = "../rs-dpp", features = [
 ] }
 drive = { path = "../rs-drive" }
 
+# For tests of grovedb verify
+rocksdb = { version = "0.21.0" }
+
 [features]
 default = ["server", "mocks"]
 server = ["clap", "dotenvy"]
diff --git a/packages/rs-drive-abci/src/main.rs b/packages/rs-drive-abci/src/main.rs
index d753dcdbca..0d796746aa 100644
--- a/packages/rs-drive-abci/src/main.rs
+++ b/packages/rs-drive-abci/src/main.rs
@@ -9,6 +9,7 @@ use drive_abci::logging::{LogBuilder, LogConfig, Loggers};
 use drive_abci::metrics::{Prometheus, DEFAULT_PROMETHEUS_PORT};
 use drive_abci::rpc::core::DefaultCoreRPC;
 use itertools::Itertools;
+use std::fs::remove_file;
 use std::path::PathBuf;
 use std::process::ExitCode;
 use tokio::runtime::Builder;
@@ -35,6 +36,15 @@ enum Commands {
     /// Returns 0 on success.
     #[command()]
     Status,
+
+    /// Verify integrity of database.
+    ///
+    /// This command will execute GroveDB integrity checks.
+    ///
+    /// You can also enforce grovedb integrity checks during `drive-abci start`
+    /// by creating `.fsck` file in database directory (`DB_PATH`).
+    #[command()]
+    Verify,
 }
 
 /// Server that accepts connections from Tenderdash, and
@@ -76,6 +86,8 @@ impl Cli {
     fn run(self, config: PlatformConfig, cancel: CancellationToken) -> Result<(), String> {
         match self.command {
             Commands::Start => {
+                verify_grovedb(&config.db_path, false)?;
+
                 let core_rpc = DefaultCoreRPC::open(
                     config.core.rpc.url().as_str(),
                     config.core.rpc.username.clone(),
@@ -95,6 +107,7 @@ impl Cli {
             }
             Commands::Config => dump_config(&config)?,
             Commands::Status => check_status(&config)?,
+            Commands::Verify => verify_grovedb(&config.db_path, true)?,
         };
 
         Ok(())
@@ -228,6 +241,51 @@ fn check_status(config: &PlatformConfig) -> Result<(), String> {
     }
 }
 
+/// Verify GroveDB integrity.
+///
+/// This function will execute GroveDB integrity checks if one of the following conditions is met:
+/// - `force` is `true`
+/// - file `.fsck` in `config.db_path` exists
+///
+/// After successful verification, .fsck file is removed.
+fn verify_grovedb(db_path: &PathBuf, force: bool) -> Result<(), String> {
+    let fsck = PathBuf::from(db_path).join(".fsck");
+    if !force && !fsck.exists() {
+        tracing::info!("no {} file, grovedb verification skipped", fsck.display());
+
+        return Ok(());
+    }
+
+    let grovedb = drive::grovedb::GroveDb::open(db_path).expect("open grovedb");
+    let result = grovedb
+        .visualize_verify_grovedb()
+        .map_err(|e| e.to_string());
+
+    match result {
+        Ok(data) => {
+            for result in data {
+                tracing::warn!(?result, "grovedb verification")
+            }
+            tracing::info!("grovedb verification finished");
+
+            if fsck.exists() {
+                if let Err(e) = remove_file(&fsck) {
+                    tracing::warn!(
+                        error = ?e,
+                        path  =fsck.display().to_string(),
+                        "grovedb verification: cannot remove .fsck file: please remove it manually to avoid running verification again",
+                    );
+                }
+            }
+            Ok(())
+        }
+        Err(e) => {
+            tracing::error!("grovedb verification failed: {}", e);
+            Err(e)
+        }
+    }
+}
+
 fn load_config(path: &Option<PathBuf>) -> PlatformConfig {
     if let Some(path) = path {
         if let Err(e) = dotenvy::from_path(path) {
@@ -285,3 +343,65 @@ fn install_panic_hook(cancel: CancellationToken) {
         cancel.cancel();
     }));
 }
+
+#[cfg(test)]
+mod test {
+    use std::{fs, path::PathBuf, thread};
+
+    use ::drive::{
+        drive::Drive,
+        fee_pools::epochs::{epoch_key_constants, paths::EpochProposers},
+        query::Element,
+    };
+    use dpp::block::epoch::Epoch;
+
+    use platform_version::version::PlatformVersion;
+
+    fn setup_drive(path: &PathBuf) -> Drive {
+        let drive = Drive::open(path, None).expect("open drive");
+
+        let platform_version = PlatformVersion::latest();
+        drive
+            .create_initial_state_structure(None, platform_version)
+            .expect("should create root tree successfully");
+
+        let transaction = drive.grove.start_transaction();
+        let epoch = Epoch::new(0).unwrap();
+
+        drive
+            .grove
+            .insert(
+                &epoch.get_path(),
+                epoch_key_constants::KEY_FEE_MULTIPLIER.as_slice(),
+                Element::Item(123u128.to_be_bytes().to_vec(), None),
+                None,
+                Some(&transaction),
+            )
+            .unwrap()
+            .expect("should insert data");
+        transaction.commit().unwrap();
+
+        drive
+    }
+
+    #[test]
+    fn test_verify_grovedb() {
+        let tempdir = tempfile::tempdir().unwrap();
+
+        let mut db_path = tempdir.path().to_path_buf();
+        db_path.push("db");
+
+        fs::create_dir(&db_path).expect("create db dir");
+
+        let drive = setup_drive(&db_path);
+        // let grovedb = drive::grovedb::GroveDb::open(&db_path).expect("open grovedb");
+        drop(drive);
+
+        // TODO: Break data in grovedb here
+
+        super::verify_grovedb(&db_path, true).unwrap();
+
+        println!("db path: {:?}", &db_path);
+        thread::sleep(std::time::Duration::from_secs(30));
+    }
+}
diff --git a/packages/rs-drive/Cargo.toml b/packages/rs-drive/Cargo.toml
index d887d9a81d..cbb9bcfd00 100644
--- a/packages/rs-drive/Cargo.toml
+++ b/packages/rs-drive/Cargo.toml
@@ -43,10 +43,14 @@ rust_decimal = { version = "1.2.5", optional = true }
 rust_decimal_macros = { version = "1.25.0", optional = true }
 lazy_static = { version = "1.4.0", optional = true }
 mockall = { version = "0.11", optional = true }
-grovedb = { version = "1.0.0-rc.1", optional = true }
-grovedb-costs = { version = "1.0.0-rc.1", optional = true }
-grovedb-path = { version = "1.0.0-rc.1" }
-grovedb-storage = { version = "1.0.0-rc.1", optional = true }
+# grovedb = { version = "1.0.0-rc.1", optional = true }
+# grovedb-costs = { version = "1.0.0-rc.1", optional = true }
+# grovedb-path = { version = "1.0.0-rc.1" }
+# grovedb-storage = { version = "1.0.0-rc.1", optional = true }
+grovedb = { git = "https://github.com/dashpay/grovedb", rev = "75b4df1177a830a7a280d65a8c179c93c225fe73", optional = true }
+grovedb-costs = { git = "https://github.com/dashpay/grovedb", rev = "75b4df1177a830a7a280d65a8c179c93c225fe73", optional = true }
+grovedb-path = { git = "https://github.com/dashpay/grovedb", rev = "75b4df1177a830a7a280d65a8c179c93c225fe73" }
+grovedb-storage = { git = "https://github.com/dashpay/grovedb", rev = "75b4df1177a830a7a280d65a8c179c93c225fe73", optional = true }
 
 [dev-dependencies]
 criterion = "0.3.5"

From d3c621dccbf3a1f64e9772c7d665899fe4b85ad2 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 27 Sep 2023 15:56:54 +0200
Subject: [PATCH 2/4] test(drive): test verification func

---
 packages/rs-drive-abci/src/main.rs | 156 +++++++++++++++++++++++------
 1 file changed, 126 insertions(+), 30 deletions(-)

diff --git a/packages/rs-drive-abci/src/main.rs b/packages/rs-drive-abci/src/main.rs
index 0d796746aa..1aa80e3b6a 100644
--- a/packages/rs-drive-abci/src/main.rs
+++ b/packages/rs-drive-abci/src/main.rs
@@ -346,19 +346,25 @@ fn install_panic_hook(cancel: CancellationToken) {
 
 #[cfg(test)]
 mod test {
-    use std::{fs, path::PathBuf, thread};
-
-    use ::drive::{
-        drive::Drive,
-        fee_pools::epochs::{epoch_key_constants, paths::EpochProposers},
-        query::Element,
+    use std::{
+        fs,
+        path::{Path, PathBuf},
     };
+
+    use ::drive::{drive::Drive, fee_pools::epochs::paths::EpochProposers, query::Element};
     use dpp::block::epoch::Epoch;
 
     use platform_version::version::PlatformVersion;
+    use rocksdb::{IteratorMode, Options};
+
+    /// Setup drive database by creating initial state structure and inserting some data.
+    ///
+    /// Returns path to the database.
+    fn setup_db(tempdir: &Path) -> PathBuf {
+        let path = tempdir.join("db");
+        fs::create_dir(&path).expect("create db dir");
 
-    fn setup_drive(path: &PathBuf) -> Drive {
-        let drive = Drive::open(path, None).expect("open drive");
+        let drive = Drive::open(&path, None).expect("open drive");
 
         let platform_version = PlatformVersion::latest();
         drive
@@ -368,40 +374,130 @@ mod test {
         let transaction = drive.grove.start_transaction();
         let epoch = Epoch::new(0).unwrap();
 
-        drive
-            .grove
-            .insert(
-                &epoch.get_path(),
-                epoch_key_constants::KEY_FEE_MULTIPLIER.as_slice(),
-                Element::Item(123u128.to_be_bytes().to_vec(), None),
-                None,
-                Some(&transaction),
-            )
-            .unwrap()
-            .expect("should insert data");
+        for i in 0..10 {
+            drive
+                .grove
+                .insert(
+                    &epoch.get_path(),
+                    &(i as u128).to_be_bytes(), //   epoch_key_constants::KEY_FEE_MULTIPLIER.as_slice()
+                    Element::Item((i as u128).to_be_bytes().to_vec(), None),
+                    None,
+                    Some(&transaction),
+                )
+                .unwrap()
+                .expect("should insert data");
+        }
+
         transaction.commit().unwrap();
 
-        drive
+        path
+    }
+
+    /// Open RocksDB and delete `n`-th item from `cf` column family.
+    ///
+    /// If `cf` is an empty string, all column families are searched.
+    fn corrupt_rocksdb_item(db_path: &PathBuf, cf: &str, n: usize) {
+        // let cf = ColumnFamilyDescriptor::new("roots", Default::default());
+        let mut db_opts = Options::default();
+
+        db_opts.create_missing_column_families(false);
+        db_opts.create_if_missing(false);
+        let db = rocksdb::DB::open_cf(&db_opts, &db_path, vec!["roots", "meta", "aux"]).unwrap();
+
+        let iter = if !cf.is_empty() {
+            let cf_handle = db.cf_handle(cf).unwrap();
+            db.iterator_cf(cf_handle, IteratorMode::Start)
+        } else {
+            db.iterator(IteratorMode::Start)
+        };
+
+        // let iter = db.iterator(IteratorMode::Start);
+        let mut i = 0;
+        for item in iter {
+            let (key, _value) = item.unwrap();
+            // println!("{} = {}", hex::encode(&key), hex::encode(value));
+            tracing::trace!(cf, key=?hex::encode(&key), "found item in rocksdb");
+
+            if i == n {
+                db.delete(&key).unwrap();
+                tracing::debug!(cf, key=?hex::encode(&key), "corrupt_rocksdb_item: removed item from rocksdb");
+                return;
+            }
+            i += 1;
+        }
+        panic!(
+            "cannot corrupt db: cannot find {}-th item in rocksdb column family {}",
+            n, cf
+        );
     }
 
     #[test]
-    fn test_verify_grovedb() {
+    fn test_verify_grovedb_remove_any_10th_item() {
+        drive_abci::logging::init_for_tests(4);
         let tempdir = tempfile::tempdir().unwrap();
+        let db_path = setup_db(tempdir.path());
 
-        let mut db_path = tempdir.path().to_path_buf();
-        db_path.push("db");
+        corrupt_rocksdb_item(&db_path, "", 10);
 
-        fs::create_dir(&db_path).expect("create db dir");
+        let result = super::verify_grovedb(&db_path, true);
+        assert!(result.is_err());
+
+        println!("db path: {:?}", &db_path);
+    }
+
+    #[test]
+    fn test_verify_grovedb_remove_roots_0th_item() {
+        drive_abci::logging::init_for_tests(4);
+        let tempdir = tempfile::tempdir().unwrap();
+        let db_path = setup_db(tempdir.path());
 
-        let drive = setup_drive(&db_path);
-        // let grovedb = drive::grovedb::GroveDb::open(&db_path).expect("open grovedb");
-        drop(drive);
+        corrupt_rocksdb_item(&db_path, "roots", 0);
+
+        let result = super::verify_grovedb(&db_path, true);
+        assert!(result.is_err());
+
+        println!("db path: {:?}", &db_path);
+    }
+
+    #[test]
+    fn test_verify_grovedb_remove_roots_10th_item() {
+        drive_abci::logging::init_for_tests(4);
+        let tempdir = tempfile::tempdir().unwrap();
+        let db_path = setup_db(tempdir.path());
+
+        corrupt_rocksdb_item(&db_path, "roots", 10);
+
+        let result = super::verify_grovedb(&db_path, true);
+        assert!(result.is_err());
+
+        println!("db path: {:?}", &db_path);
+    }
+
+    #[test]
+    fn test_verify_grovedb_remove_meta_10th_item() {
+        drive_abci::logging::init_for_tests(4);
+        let tempdir = tempfile::tempdir().unwrap();
+        let db_path = setup_db(tempdir.path());
+
+        corrupt_rocksdb_item(&db_path, "meta", 10);
+
+        let result = super::verify_grovedb(&db_path, true);
+        assert!(result.is_err());
+
+        println!("db path: {:?}", &db_path);
+    }
+
+    #[test]
+    fn test_verify_grovedb_remove_aux_10th_item() {
+        drive_abci::logging::init_for_tests(4);
+        let tempdir = tempfile::tempdir().unwrap();
+        let db_path = setup_db(tempdir.path());
 
-        // TODO: Break data in grovedb here
+        corrupt_rocksdb_item(&db_path, "aux", 10);
 
-        super::verify_grovedb(&db_path, true).unwrap();
+        let result = super::verify_grovedb(&db_path, true);
+        assert!(result.is_err());
 
         println!("db path: {:?}", &db_path);
-        thread::sleep(std::time::Duration::from_secs(30));
     }
 }

From 790feca7b7a3defc70ede9716c1c4007fdf04d83 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Wed, 27 Sep 2023 16:43:05 +0200
Subject: [PATCH 3/4] test: verify grovedb

---
 packages/rs-drive-abci/src/main.rs | 78 ++++--------------------------
 1 file changed, 9 insertions(+), 69 deletions(-)

diff --git a/packages/rs-drive-abci/src/main.rs b/packages/rs-drive-abci/src/main.rs
index 1aa80e3b6a..1a5f8a3646 100644
--- a/packages/rs-drive-abci/src/main.rs
+++ b/packages/rs-drive-abci/src/main.rs
@@ -393,9 +393,7 @@ mod test {
         path
     }
 
-    /// Open RocksDB and delete `n`-th item from `cf` column family.
-    ///
-    /// If `cf` is an empty string, all column families are searched.
+    /// Open RocksDB and corrupt `n`-th item from `cf` column family.
     fn corrupt_rocksdb_item(db_path: &PathBuf, cf: &str, n: usize) {
         // let cf = ColumnFamilyDescriptor::new("roots", Default::default());
         let mut db_opts = Options::default();
@@ -404,23 +402,21 @@ mod test {
         db_opts.create_if_missing(false);
         let db = rocksdb::DB::open_cf(&db_opts, &db_path, vec!["roots", "meta", "aux"]).unwrap();
 
-        let iter = if !cf.is_empty() {
-            let cf_handle = db.cf_handle(cf).unwrap();
-            db.iterator_cf(cf_handle, IteratorMode::Start)
-        } else {
-            db.iterator(IteratorMode::Start)
-        };
+        let cf_handle = db.cf_handle(cf).unwrap();
+        let iter = db.iterator_cf(cf_handle, IteratorMode::Start);
 
         // let iter = db.iterator(IteratorMode::Start);
         let mut i = 0;
         for item in iter {
-            let (key, _value) = item.unwrap();
+            let (key, mut value) = item.unwrap();
             // println!("{} = {}", hex::encode(&key), hex::encode(value));
-            tracing::trace!(cf, key=?hex::encode(&key), "found item in rocksdb");
+            tracing::trace!(cf, key=?hex::encode(&key), value=hex::encode(&value),"found item in rocksdb");
 
             if i == n {
-                db.delete(&key).unwrap();
-                tracing::debug!(cf, key=?hex::encode(&key), "corrupt_rocksdb_item: removed item from rocksdb");
+                value[0] = !value[0];
+                db.put_cf(cf_handle, &key, &value).unwrap();
+
+                tracing::debug!(cf, key=?hex::encode(&key), value=hex::encode(&value), "corrupt_rocksdb_item: corrupting item");
                 return;
             }
             i += 1;
@@ -431,20 +427,6 @@ mod test {
         );
     }
 
-    #[test]
-    fn test_verify_grovedb_remove_any_10th_item() {
-        drive_abci::logging::init_for_tests(4);
-        let tempdir = tempfile::tempdir().unwrap();
-        let db_path = setup_db(tempdir.path());
-
-        corrupt_rocksdb_item(&db_path, "", 10);
-
-        let result = super::verify_grovedb(&db_path, true);
-        assert!(result.is_err());
-
-        println!("db path: {:?}", &db_path);
-    }
-
     #[test]
     fn test_verify_grovedb_remove_roots_0th_item() {
         drive_abci::logging::init_for_tests(4);
@@ -458,46 +440,4 @@ mod test {
 
         println!("db path: {:?}", &db_path);
     }
-
-    #[test]
-    fn test_verify_grovedb_remove_roots_10th_item() {
-        drive_abci::logging::init_for_tests(4);
-        let tempdir = tempfile::tempdir().unwrap();
-        let db_path = setup_db(tempdir.path());
-
-        corrupt_rocksdb_item(&db_path, "roots", 10);
-
-        let result = super::verify_grovedb(&db_path, true);
-        assert!(result.is_err());
-
-        println!("db path: {:?}", &db_path);
-    }
-
-    #[test]
-    fn test_verify_grovedb_remove_meta_10th_item() {
-        drive_abci::logging::init_for_tests(4);
-        let tempdir = tempfile::tempdir().unwrap();
-        let db_path = setup_db(tempdir.path());
-
-        corrupt_rocksdb_item(&db_path, "meta", 10);
-
-        let result = super::verify_grovedb(&db_path, true);
-        assert!(result.is_err());
-
-        println!("db path: {:?}", &db_path);
-    }
-
-    #[test]
-    fn test_verify_grovedb_remove_aux_10th_item() {
-        drive_abci::logging::init_for_tests(4);
-        let tempdir = tempfile::tempdir().unwrap();
-        let db_path = setup_db(tempdir.path());
-
-        corrupt_rocksdb_item(&db_path, "aux", 10);
-
-        let result = super::verify_grovedb(&db_path, true);
-        assert!(result.is_err());
-
-        println!("db path: {:?}", &db_path);
-    }
 }

From cb1aaa487e14fd8c525e10c8faa69de681f151d8 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Thu, 28 Sep 2023 11:08:17 +0200
Subject: [PATCH 4/4] doc: drive-abci verify

---
 docs/DRIVE-ABCI.md                 | 45 ++++++++++++++++++++++++++++++
 packages/rs-drive-abci/src/main.rs | 44 ++++++++++++++++-------------
 2 files changed, 70 insertions(+), 19 deletions(-)

diff --git a/docs/DRIVE-ABCI.md b/docs/DRIVE-ABCI.md
index 2be4a34d24..82681b9d9e 100644
--- a/docs/DRIVE-ABCI.md
+++ b/docs/DRIVE-ABCI.md
@@ -76,3 +76,48 @@ export ABCI_LOG_EXAMPLE_MAX_FILES=10
 This configuration specifies that logs for the "EXAMPLE" destination should be stored in the /var/log/example directory, with a verbosity level of 3. Colorful output should not be used, and the logs should be formatted in a human-readable and visually appealing manner. The maximum number of daily log files to store is set to 10.
 
 Ensure that you adjust the values according to your specific logging requirements.
+
+## Integrity checks
+
+## `drive-abci verify`
+
+The `drive-abci verify` command is used to verify the integrity of the database used by `drive-abci`.
+This command will execute GroveDB hash integrity checks to ensure that the database is consistent
+and free of corruption.
+
+### Usage
+
+To use the `drive-abci verify` command, simply run the following command:
+
+```bash
+drive-abci verify
+```
+
+This will execute the GroveDB hash integrity checks and report any errors or inconsistencies found in the database.
+
+### Enforcing Integrity Checks
+
+You can also enforce GroveDB integrity checks during `drive-abci start` by creating a `.fsck` file in the database
+directory (`DB_PATH`). This file should be created before starting `drive-abci`, and can be empty.
+
+When `drive-abci` starts up, it checks for the presence of the `.fsck` file in the database directory.
+If the file is present, it executes the specified integrity checks. After the checks are completed,
+the `.fsck` file is deleted from the database directory.
+
+### Example: Verifying consistency of  `drive-abci` running in Docker
+
+To verify integrity of database when `drive-abci` runs in a Docker container, you can create a `.fsck` file in the
+database directory and and restart the container.
+
+For example, for a drive-abci container `dashmate_ccc1e5c2_local_1-drive_abci-1`, you can execute the following commands:
+
+```bash
+docker exec -ti dashmate_ccc1e5c2_local_1-drive_abci-1 touch db/.fsck 
+docker restart dashmate_ccc1e5c2_local_1-drive_abci-1
+```
+
+You can check the result of verification in logs by running the following command:
+
+```bash
+docker logs dashmate_ccc1e5c2_local_1-drive_abci-1 --tail 1000 2>&1 | grep 'grovedb verification' 
+```
diff --git a/packages/rs-drive-abci/src/main.rs b/packages/rs-drive-abci/src/main.rs
index 5ebcc4f6ba..9f8eb15c3d 100644
--- a/packages/rs-drive-abci/src/main.rs
+++ b/packages/rs-drive-abci/src/main.rs
@@ -39,7 +39,7 @@ enum Commands {
 
     /// Verify integrity of database.
     ///
-    /// This command will execute GroveDB integrity checks.
+    /// This command will execute GroveDB hash integrity checks.
     ///
     /// You can also enforce grovedb integrity checks during `drive-abci start`
     /// by creating `.fsck` file in database directory (`DB_PATH`).
@@ -255,10 +255,15 @@ fn check_status(config: &PlatformConfig) -> Result<(), String> {
 /// After successful verification, .fsck file is removed.
 fn verify_grovedb(db_path: &PathBuf, force: bool) -> Result<(), String> {
     let fsck = PathBuf::from(db_path).join(".fsck");
-    if !force && !fsck.exists() {
-        tracing::info!("no {} file, grovedb verification skipped", fsck.display());
 
-        return Ok(());
+    if !force {
+        if !fsck.exists() {
+            return Ok(());
+        }
+        tracing::info!(
+            "found {} file, starting grovedb verification",
+            fsck.display()
+        );
     }
 
     let grovedb = drive::grovedb::GroveDb::open(db_path).expect("open grovedb");
@@ -358,6 +363,7 @@ mod test {
 
     use ::drive::{drive::Drive, fee_pools::epochs::paths::EpochProposers, query::Element};
     use dpp::block::epoch::Epoch;
+    use drive::fee_pools::epochs::epoch_key_constants;
 
     use platform_version::version::PlatformVersion;
     use rocksdb::{IteratorMode, Options};
@@ -379,19 +385,19 @@ mod test {
         let transaction = drive.grove.start_transaction();
         let epoch = Epoch::new(0).unwrap();
 
-        for i in 0..10 {
-            drive
-                .grove
-                .insert(
-                    &epoch.get_path(),
-                    &(i as u128).to_be_bytes(), //   epoch_key_constants::KEY_FEE_MULTIPLIER.as_slice()
-                    Element::Item((i as u128).to_be_bytes().to_vec(), None),
-                    None,
-                    Some(&transaction),
-                )
-                .unwrap()
-                .expect("should insert data");
-        }
+        let i = 100;
+
+        drive
+            .grove
+            .insert(
+                &epoch.get_path(),
+                epoch_key_constants::KEY_FEE_MULTIPLIER.as_slice(),
+                Element::Item((i as u128).to_be_bytes().to_vec(), None),
+                None,
+                Some(&transaction),
+            )
+            .unwrap()
+            .expect("should insert data");
 
         transaction.commit().unwrap();
 
@@ -400,11 +406,11 @@ mod test {
 
     /// Open RocksDB and corrupt `n`-th item from `cf` column family.
     fn corrupt_rocksdb_item(db_path: &PathBuf, cf: &str, n: usize) {
-        // let cf = ColumnFamilyDescriptor::new("roots", Default::default());
         let mut db_opts = Options::default();
 
         db_opts.create_missing_column_families(false);
         db_opts.create_if_missing(false);
+
         let db = rocksdb::DB::open_cf(&db_opts, &db_path, vec!["roots", "meta", "aux"]).unwrap();
 
         let cf_handle = db.cf_handle(cf).unwrap();
@@ -433,7 +439,7 @@ mod test {
     }
 
     #[test]
-    fn test_verify_grovedb_remove_roots_0th_item() {
+    fn test_verify_grovedb_corrupt_0th_root() {
         drive_abci::logging::init_for_tests(4);
         let tempdir = tempfile::tempdir().unwrap();
         let db_path = setup_db(tempdir.path());