feat: enable FlatState values inlining migration in the background (n…

…ear#9093)

Part of near#8243.

This PR enables the migration added in near#9037 to be executed in the background on the running node.
It supports graceful stop when the node is shut down. The implementation is heavily inspired by state sync background dumping to S3.

This PR also introduces a new column `DBCol::Misc`. For now it only stores the status of the migration, but it can hold any small pieces of data, similar to `DBCol::BlockMisc`.

`FlatStorageManager` is exposed as part of `RuntimeAdapter` in this PR. This is the first step in cleaning `RuntimeAdapter` from all other flat storage related methods, as the manager can be directly used instead.

Tested by manually running a node and checking metrics and log messages. After that flat storage was checked with `flat-storage verify` cmd.

chain/chain/src/types.rs

@@ -32,6 +32,7 @@ use near_primitives::version::{
     MIN_PROTOCOL_VERSION_NEP_92_FIX,
 };
 use near_primitives::views::{QueryRequest, QueryResponse};
+use near_store::flat::{FlatStorage, FlatStorageStatus};
 use near_store::{PartialStorage, ShardTries, Store, StoreUpdate, Trie, WrappedTrieChanges};
 
 pub use near_epoch_manager::EpochManagerAdapter;
@@ -306,6 +307,30 @@ pub trait RuntimeAdapter: Send + Sync {
 
     fn get_flat_storage_manager(&self) -> Option<FlatStorageManager>;
 
+    fn get_flat_storage_for_shard(&self, shard_uid: ShardUId) -> Option<FlatStorage>;
+
+    fn get_flat_storage_status(&self, shard_uid: ShardUId) -> FlatStorageStatus;
+
+    /// Creates flat storage state for given shard, assuming that all flat storage data
+    /// is already stored in DB.
+    /// TODO (#7327): consider returning flat storage creation errors here
+    fn create_flat_storage_for_shard(&self, shard_uid: ShardUId);
+
+    /// Removes flat storage state for shard, if it exists.
+    /// Used to clear old flat storage data from disk and memory before syncing to newer state.
+    fn remove_flat_storage_for_shard(
+        &self,
+        shard_uid: ShardUId,
+        epoch_id: &EpochId,
+    ) -> Result<(), Error>;
+
+    fn set_flat_storage_for_genesis(
+        &self,
+        genesis_block: &CryptoHash,
+        genesis_block_height: BlockHeight,
+        genesis_epoch_id: &EpochId,
+    ) -> Result<StoreUpdate, Error>;
+
     /// Validates a given signed transaction.
     /// If the state root is given, then the verification will use the account. Otherwise it will
     /// only validate the transaction math, limits and signatures.

core/store/src/flat/inlining_migration.rs

@@ -8,7 +8,6 @@ use crossbeam::channel;
 use itertools::Itertools;
 use near_primitives::hash::CryptoHash;
 use near_primitives::shard_layout::ShardUId;
-use near_primitives::state::FlatStateValue;
 use tracing::{debug, info};
 
 use crate::flat::store_helper::set_flat_state_values_inlining_migration_status;
@@ -22,7 +21,7 @@ use super::store_helper::{
     decode_flat_state_db_key, get_flat_state_values_inlining_migration_status,
 };
 use super::types::{FlatStateValuesInliningMigrationStatus, INLINE_DISK_VALUE_THRESHOLD};
-use super::FlatStorageManager;
+use super::{FlatStateValue, FlatStorageManager};
 
 pub struct FlatStateValuesInliningMigrationHandle {
     handle: JoinHandle<()>,
@@ -287,11 +286,10 @@ mod tests {
     use borsh::{BorshDeserialize, BorshSerialize};
     use near_primitives::hash::hash;
     use near_primitives::shard_layout::ShardLayout;
-    use near_primitives::state::FlatStateValue;
 
     use crate::flat::store_helper::encode_flat_state_db_key;
     use crate::flat::types::INLINE_DISK_VALUE_THRESHOLD;
-    use crate::flat::FlatStorageManager;
+    use crate::flat::{FlatStateValue, FlatStorageManager};
     use crate::{DBCol, NodeStorage, TrieCachingStorage};
 
     use super::inline_flat_state_values;