From c5656daaac5992eaddba64cc2b43812612005f36 Mon Sep 17 00:00:00 2001 From: Igor Egorov Date: Fri, 31 Mar 2023 09:57:24 +0300 Subject: [PATCH 1/4] Fix db editor crash Signed-off-by: Igor Egorov --- core/utils/kagome_db_editor.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/utils/kagome_db_editor.cpp b/core/utils/kagome_db_editor.cpp index acac9548c8..33bccc9b83 100644 --- a/core/utils/kagome_db_editor.cpp +++ b/core/utils/kagome_db_editor.cpp @@ -402,16 +402,15 @@ int db_editor_main(int argc, const char **argv) { } } - auto db_cursor = buffer_storage->cursor(); - auto db_batch = buffer_storage->batch(); - auto res = check(db_cursor->seek(prefix)); + auto db_cursor = trie_buffer_storage->cursor(); + auto db_batch = trie_buffer_storage->batch(); + auto res = check(db_cursor->seekFirst()); int count = 0; { TicToc t2("Process DB.", log); - while (db_cursor->isValid() && db_cursor->key().has_value() - && boost::starts_with(db_cursor->key().value(), prefix)) { + while (db_cursor->isValid() && db_cursor->key().has_value()) { auto key = db_cursor->key().value(); - if (trie_tracker->tracked(key.view(prefix.size()))) { + if (trie_tracker->tracked(key)) { db_cursor->next().value(); continue; } From 852576d9c80540dbd81b77db161a713e7436033b Mon Sep 17 00:00:00 2001 From: Igor Egorov Date: Fri, 31 Mar 2023 11:05:42 +0300 Subject: [PATCH 2/4] Add a touch of aggressive pruning Signed-off-by: Igor Egorov --- core/storage/rocksdb/rocksdb.cpp | 17 +++++++++++++++++ core/storage/rocksdb/rocksdb.hpp | 7 +++++++ core/utils/kagome_db_editor.cpp | 1 + 3 files changed, 25 insertions(+) diff --git a/core/storage/rocksdb/rocksdb.cpp b/core/storage/rocksdb/rocksdb.cpp index 7cc619ce22..5e94042381 100644 --- a/core/storage/rocksdb/rocksdb.cpp +++ b/core/storage/rocksdb/rocksdb.cpp @@ -115,6 +115,23 @@ namespace kagome::storage { return space_ptr; } + void RocksDb::dropColumn(kagome::storage::Space space) { + auto space_name = spaceName(space); + auto column_it = + std::find_if(cf_handles_.begin(), + cf_handles_.end(), + [&space_name](const ColumnFamilyHandle &handle) { + return handle.name == space_name; + }); + if (cf_handles_.end() == column_it) { + throw DatabaseError::INVALID_ARGUMENT; + } + auto &column = *column_it; + db_->DropColumnFamily(column.handle); + db_->DestroyColumnFamilyHandle(column.handle); + db_->CreateColumnFamily({}, column.name, &column.handle); + } + RocksDbSpace::RocksDbSpace(std::weak_ptr storage, RocksDb::ColumnFamilyHandle column, log::Logger logger) diff --git a/core/storage/rocksdb/rocksdb.hpp b/core/storage/rocksdb/rocksdb.hpp index 4eb5a4bc76..a8a818d3c6 100644 --- a/core/storage/rocksdb/rocksdb.hpp +++ b/core/storage/rocksdb/rocksdb.hpp @@ -35,6 +35,13 @@ namespace kagome::storage { std::shared_ptr getSpace(Space space) override; + /** + * Implementation specific way to erase the whole space data. + * Not exposed at SpacedStorage level as only used in pruner. + * @param space - storage space identifier to clear + */ + void dropColumn(Space space); + friend class RocksDbSpace; friend class RocksDbBatch; diff --git a/core/utils/kagome_db_editor.cpp b/core/utils/kagome_db_editor.cpp index 33bccc9b83..7c3bc90467 100644 --- a/core/utils/kagome_db_editor.cpp +++ b/core/utils/kagome_db_editor.cpp @@ -250,6 +250,7 @@ int db_editor_main(int argc, const char **argv) { try { storage = storage::RocksDb::create(argv[DB_PATH], rocksdb::Options()).value(); + storage->dropColumn(storage::Space::kBlockBody); buffer_storage = storage->getSpace(storage::Space::kDefault); } catch (std::system_error &e) { log->error("{}", e.what()); From 11d7c4688134a7f86fbd8f1691e43f3be67f952b Mon Sep 17 00:00:00 2001 From: Igor Egorov Date: Fri, 31 Mar 2023 11:18:45 +0300 Subject: [PATCH 3/4] Fix help messages Signed-off-by: Igor Egorov --- core/application/impl/app_configuration_impl.cpp | 5 ++++- core/utils/kagome_db_editor.cpp | 11 ++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/core/application/impl/app_configuration_impl.cpp b/core/application/impl/app_configuration_impl.cpp index 01fe6ef24f..fa7a033935 100644 --- a/core/application/impl/app_configuration_impl.cpp +++ b/core/application/impl/app_configuration_impl.cpp @@ -785,6 +785,8 @@ namespace kagome::application { ("purge-wavm-cache", "purge WAVM runtime cache") ; + po::options_description db_editor_desc("kagome db-editor - to view help message for db editor"); + // clang-format on for (auto &[flag, name, dev] : devAccounts()) { @@ -804,7 +806,8 @@ namespace kagome::application { desc.add(blockhain_desc) .add(storage_desc) .add(network_desc) - .add(development_desc); + .add(development_desc) + .add(db_editor_desc); if (vm.count("help") > 0) { std::cout << desc << std::endl; diff --git a/core/utils/kagome_db_editor.cpp b/core/utils/kagome_db_editor.cpp index 7c3bc90467..2242f755c5 100644 --- a/core/utils/kagome_db_editor.cpp +++ b/core/utils/kagome_db_editor.cpp @@ -143,21 +143,14 @@ enum Command : uint8_t { COMPACT, DUMP }; void usage() { std::string help(R"( -Kagome DB Editor +Kagome DB Editor - a storage pruner. Allows to reduce occupied disk space. Usage: - kagome-db-editor + kagome db-editor full or relative path to kagome database. It is usually path polkadot/db inside base path set in kagome options. - root state hash in 0x prefixed hex format. [Optional] - - dump: dumps the state from the DB to file hex_full_state.yaml in - format ready for use in polkadot-test. - compact: compacts the kagome DB. Leaves only keys of the state passed - as an arguments. Removes all other keys. [Default] Example: - kagome-db-editor base-path/polkadot/db 0x1e22e dump kagome-db-editor base-path/polkadot/db )"); std::cout << help; From 52ed6b8b88163af39554f95b339b0f727cafd952 Mon Sep 17 00:00:00 2001 From: Igor Egorov Date: Fri, 31 Mar 2023 11:57:29 +0300 Subject: [PATCH 4/4] Add db operations error handling Signed-off-by: Igor Egorov --- core/storage/rocksdb/rocksdb.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/core/storage/rocksdb/rocksdb.cpp b/core/storage/rocksdb/rocksdb.cpp index 5e94042381..d7089537ef 100644 --- a/core/storage/rocksdb/rocksdb.cpp +++ b/core/storage/rocksdb/rocksdb.cpp @@ -127,9 +127,15 @@ namespace kagome::storage { throw DatabaseError::INVALID_ARGUMENT; } auto &column = *column_it; - db_->DropColumnFamily(column.handle); - db_->DestroyColumnFamilyHandle(column.handle); - db_->CreateColumnFamily({}, column.name, &column.handle); + auto e = [this](rocksdb::Status status) { + if (!status.ok()) { + logger_->error("DB operation failed: {}", status.ToString()); + throw status_as_error(status); + } + }; + e(db_->DropColumnFamily(column.handle)); + e(db_->DestroyColumnFamilyHandle(column.handle)); + e(db_->CreateColumnFamily({}, column.name, &column.handle)); } RocksDbSpace::RocksDbSpace(std::weak_ptr storage,