feat(trin-storage): add filter to store

src/bin/purge_invalid_history_content.rs

@@ -0,0 +1,103 @@
+use alloy_primitives::B256;
+use anyhow::Result;
+use clap::Parser;
+use discv5::enr::{CombinedKey, Enr};
+use tracing::info;
+
+use ethportal_api::{types::portal_wire::ProtocolId, HistoryContentKey};
+use portalnet::utils::db::{configure_node_data_dir, configure_trin_data_dir};
+use trin_storage::{
+    versioned::{create_store, ContentType, IdIndexedV1Store, IdIndexedV1StoreConfig},
+    PortalStorageConfigFactory,
+};
+use trin_utils::log::init_tracing_logger;
+
+/// iterates history store and removes any invalid network entries
+pub fn main() -> Result<()> {
+    init_tracing_logger();
+    let script_config = PurgeConfig::parse();
+
+    let trin_data_dir = configure_trin_data_dir(false /* ephemeral */)?;
+    let (node_data_dir, mut private_key) = configure_node_data_dir(
+        trin_data_dir,
+        script_config.private_key,
+        "mainnet".to_string(),
+    )?;
+    let enr_key = CombinedKey::secp256k1_from_bytes(private_key.as_mut_slice())
+        .expect("Failed to create ENR key");
+    let enr = Enr::empty(&enr_key).unwrap();
+    let node_id = enr.node_id();
+    info!("Purging data for NodeID: {node_id}");
+    info!("DB Path: {node_data_dir:?}");
+
+    let config = PortalStorageConfigFactory::new(
+        script_config.capacity as u64,
+        &["history".to_string()],
+        node_id,
+        node_data_dir,
+    )
+    .unwrap()
+    .create("history");
+    let config = IdIndexedV1StoreConfig::new(ContentType::History, ProtocolId::History, config);
+    let sql_connection_pool = config.sql_connection_pool.clone();
+    let store: IdIndexedV1Store<HistoryContentKey> =
+        create_store(ContentType::History, config, sql_connection_pool).unwrap();
+    let total_entry_count = store.usage_stats().entry_count;
+    info!("total entry count: {total_entry_count}");
+    let sql_connection_pool = store.config.sql_connection_pool.clone();
+    let lookup_result = sql_connection_pool
+        .get()
+        .unwrap()
+        .query_row(&lookup_query(), [], |row| row.get::<usize, u64>(0))
+        .expect("Failed to fetch history content");
+    info!("found {} epoch accumulators", lookup_result);
+    if script_config.evict {
+        let _: Vec<String> = sql_connection_pool
+            .get()
+            .unwrap()
+            .prepare(&delete_query())
+            .unwrap()
+            .query_map([], |row| row.get::<usize, String>(0))
+            .unwrap()
+            .map(|r| r.unwrap())
+            .collect();
+        let changes = sql_connection_pool.get().unwrap().changes();
+        info!("removed {} invalid history content values", changes,);
+    }
+    Ok(())
+}
+
+fn lookup_query() -> String {
+    r#"SELECT COUNT(*) as count FROM ii1_history WHERE hex(content_key) LIKE "03%""#.to_string()
+}
+
+fn delete_query() -> String {
+    r#"DELETE FROM ii1_history WHERE hex(content_key) LIKE '03%'"#.to_string()
+}
+
+// CLI Parameter Handling
+#[derive(Parser, Debug, PartialEq)]
+#[command(
+    name = "Trin DB Purge Invalid History Content",
+    about = "Remove invalid data from Trin History Store"
+)]
+pub struct PurgeConfig {
+    #[arg(
+        long,
+        help = "(unsafe) Hex private key to generate node id for database namespace (with 0x prefix)"
+    )]
+    pub private_key: Option<B256>,
+
+    #[arg(
+        long,
+        help = "Storage capacity. Must be larger than the current capacity otherwise it will prune data!"
+    )]
+    pub capacity: usize,
+
+    #[arg(
+        long,
+        help = "Actually evict the history data from db",
+        default_value = "false"
+    )]
+    pub evict: bool,
+}

trin-storage/src/versioned/id_indexed_v1/store.rs

@@ -41,7 +41,7 @@ pub struct PaginateResult<TContentKey> {
 #[derive(Debug)]
 pub struct IdIndexedV1Store<TContentKey: OverlayContentKey> {
     /// The configuration.
-    config: IdIndexedV1StoreConfig,
+    pub config: IdIndexedV1StoreConfig,
     /// The maximum distance between `NodeId` and content id that store should keep. Updated
     /// dynamically after pruning to the farthest distance still stored.
     radius: Distance,