Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Kagome db-editor update #1552

Merged
merged 10 commits into from
Apr 10, 2023
5 changes: 4 additions & 1 deletion core/application/impl/app_configuration_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -785,6 +785,8 @@ namespace kagome::application {
("purge-wavm-cache", "purge WAVM runtime cache")
;

po::options_description db_editor_desc("kagome db-editor - to view help message for db editor");

// clang-format on

for (auto &[flag, name, dev] : devAccounts()) {
Expand All @@ -804,7 +806,8 @@ namespace kagome::application {
desc.add(blockhain_desc)
.add(storage_desc)
.add(network_desc)
.add(development_desc);
.add(development_desc)
.add(db_editor_desc);

if (vm.count("help") > 0) {
std::cout << desc << std::endl;
Expand Down
23 changes: 23 additions & 0 deletions core/storage/rocksdb/rocksdb.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,29 @@ namespace kagome::storage {
return space_ptr;
}

void RocksDb::dropColumn(kagome::storage::Space space) {
auto space_name = spaceName(space);
auto column_it =
std::find_if(cf_handles_.begin(),
cf_handles_.end(),
[&space_name](const ColumnFamilyHandle &handle) {
return handle.name == space_name;
});
if (cf_handles_.end() == column_it) {
throw DatabaseError::INVALID_ARGUMENT;
}
auto &column = *column_it;
auto e = [this](rocksdb::Status status) {
if (!status.ok()) {
logger_->error("DB operation failed: {}", status.ToString());
throw status_as_error(status);
}
};
e(db_->DropColumnFamily(column.handle));
e(db_->DestroyColumnFamilyHandle(column.handle));
e(db_->CreateColumnFamily({}, column.name, &column.handle));
}

RocksDbSpace::RocksDbSpace(std::weak_ptr<RocksDb> storage,
RocksDb::ColumnFamilyHandle column,
log::Logger logger)
Expand Down
7 changes: 7 additions & 0 deletions core/storage/rocksdb/rocksdb.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,13 @@ namespace kagome::storage {

std::shared_ptr<BufferStorage> getSpace(Space space) override;

/**
* Implementation specific way to erase the whole space data.
* Not exposed at SpacedStorage level as only used in pruner.
* @param space - storage space identifier to clear
*/
void dropColumn(Space space);

friend class RocksDbSpace;
friend class RocksDbBatch;

Expand Down
23 changes: 8 additions & 15 deletions core/utils/kagome_db_editor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -142,21 +142,14 @@ enum Command : uint8_t { COMPACT, DUMP };

void usage() {
std::string help(R"(
Kagome DB Editor
Kagome DB Editor - a storage pruner. Allows to reduce occupied disk space.
Usage:
kagome-db-editor <db-path> <root-state> <command>
kagome db-editor <db-path>

<db-path> full or relative path to kagome database. It is usually path
polkadot/db inside base path set in kagome options.
<root-state> root state hash in 0x prefixed hex format. [Optional]
<command>
dump: dumps the state from the DB to file hex_full_state.yaml in
format ready for use in polkadot-test.
compact: compacts the kagome DB. Leaves only keys of the state passed
as an arguments. Removes all other keys. [Default]

Example:
kagome-db-editor base-path/polkadot/db 0x1e22e dump
kagome-db-editor base-path/polkadot/db
)");
std::cout << help;
Expand Down Expand Up @@ -249,6 +242,7 @@ int db_editor_main(int argc, const char **argv) {
try {
storage =
storage::RocksDb::create(argv[DB_PATH], rocksdb::Options()).value();
storage->dropColumn(storage::Space::kBlockBody);
buffer_storage = storage->getSpace(storage::Space::kDefault);
} catch (std::system_error &e) {
log->error("{}", e.what());
Expand Down Expand Up @@ -397,16 +391,15 @@ int db_editor_main(int argc, const char **argv) {
}
}

auto db_cursor = buffer_storage->cursor();
auto db_batch = buffer_storage->batch();
auto res = check(db_cursor->seek(prefix));
auto db_cursor = trie_buffer_storage->cursor();
auto db_batch = trie_buffer_storage->batch();
auto res = check(db_cursor->seekFirst());
int count = 0;
{
TicToc t2("Process DB.", log);
while (db_cursor->isValid() && db_cursor->key().has_value()
&& boost::starts_with(db_cursor->key().value(), prefix)) {
while (db_cursor->isValid() && db_cursor->key().has_value()) {
auto key = db_cursor->key().value();
if (trie_tracker->tracked(key.view(prefix.size()))) {
if (trie_tracker->tracked(key)) {
db_cursor->next().value();
continue;
}
Expand Down