Refactor db-cli

CMakeLists.txt

@@ -48,4 +48,4 @@ endif()
 ################################################################################
 add_subdirectory(daemon)
 add_subdirectory(web)
-add_subdirectory(dbcli)
+add_subdirectory(db-cli)

cmake/db-cli-deps.cmake

@@ -0,0 +1,54 @@
+################################################################################
+# Dependencies for db-cli
+#
+# See also deps.cmake and bundled-deps.cmake
+################################################################################
+
+add_library(db-cli-deps INTERFACE)
+
+find_package(OpenSSL REQUIRED)
+target_link_libraries(db-cli-deps INTERFACE openssl::openssl)
+
+find_package(fmt REQUIRED)
+target_link_libraries(db-cli-deps INTERFACE fmt::fmt)
+
+find_package(CLI11 REQUIRED)
+target_link_libraries(db-cli-deps INTERFACE CLI11::CLI11)
+
+find_package(lmdb REQUIRED)
+target_link_libraries(db-cli-deps INTERFACE lmdb::lmdb)
+
+find_package(cereal REQUIRED)
+target_link_libraries(db-cli-deps INTERFACE cereal::cereal)
+
+find_package(libcoro REQUIRED)
+target_link_libraries(db-cli-deps INTERFACE libcoro::libcoro)
+
+find_package(Boost REQUIRED)
+target_link_libraries(db-cli-deps INTERFACE boost::boost)
+
+find_package(frozen REQUIRED)
+target_link_libraries(db-cli-deps INTERFACE frozen::frozen)
+
+find_package(LibArchive REQUIRED)
+target_link_libraries(db-cli-deps INTERFACE LibArchive::LibArchive)
+
+find_package(phmap REQUIRED)
+target_link_libraries(db-cli-deps INTERFACE phmap)
+
+
+################################################################################
+# Bundled Dependencies for db-cli
+#
+################################################################################
+
+include(FetchContent)
+
+FetchContent_Declare(
+    lmdbxx
+    GIT_REPOSITORY https://github.com/hoytech/lmdbxx
+    GIT_TAG        b12d1b12f4c9793aef8bed03d07ecbf789e810b4
+    GIT_PROGRESS TRUE
+)
+
+FetchContent_MakeAvailable(lmdbxx)

cmake/deps.cmake

@@ -10,7 +10,7 @@ find_package(OpenSSL REQUIRED)
 target_link_libraries(deps INTERFACE openssl::openssl)
 
 find_package(Boost REQUIRED)
-target_link_libraries(deps INTERFACE Boost::boost)
+target_link_libraries(deps INTERFACE boost::boost)
 
 find_package(date REQUIRED)
 target_link_libraries(deps INTERFACE date::date)

commitlint.config.ts

@@ -14,7 +14,7 @@ module.exports = {
       'always',
       [
         'daemon',
-        'dbcli',
+        'db-cli',
         'web',
       ]
     ],

conanfile.py

@@ -33,6 +33,7 @@ def requirements(self):
         self.requires("cereal/1.3.2")
         self.requires("libcoro/0.12.1")
         self.requires("scope-lite/0.2.0")
+        self.requires("cli11/2.4.2")
 
         if self.options.testing:
             print("Testing enabled")

daemon/utilities/MemoryLiterals.h

@@ -0,0 +1,57 @@
+/* === This file is part of bxt ===
+ *
+ *   SPDX-FileCopyrightText: 2024 Artem Grinev <agrinev@manjaro.org>
+ *   SPDX-License-Identifier: AGPL-3.0-or-later
+ *
+ */
+
+#pragma once
+
+#include <cstddef>
+
+namespace bxt::MemoryLiterals {
+
+constexpr std::size_t BinaryBase = 1024UL;
+constexpr std::size_t DecimalBase = 1000UL;
+
+constexpr std::size_t operator"" _KiB(unsigned long long size) {
+    return size * BinaryBase;
+}
+
+constexpr std::size_t operator"" _MiB(unsigned long long size) {
+    return size * BinaryBase * BinaryBase;
+}
+
+constexpr std::size_t operator"" _GiB(unsigned long long size) {
+    return size * BinaryBase * BinaryBase * BinaryBase;
+}
+
+constexpr std::size_t operator"" _TiB(unsigned long long size) {
+    return size * BinaryBase * BinaryBase * BinaryBase * BinaryBase;
+}
+
+constexpr std::size_t operator"" _PiB(unsigned long long size) {
+    return size * BinaryBase * BinaryBase * BinaryBase * BinaryBase * BinaryBase;
+}
+
+constexpr std::size_t operator"" _KB(unsigned long long size) {
+    return size * DecimalBase;
+}
+
+constexpr std::size_t operator"" _MB(unsigned long long size) {
+    return size * DecimalBase * DecimalBase;
+}
+
+constexpr std::size_t operator"" _GB(unsigned long long size) {
+    return size * DecimalBase * DecimalBase * DecimalBase;
+}
+
+constexpr std::size_t operator"" _TB(unsigned long long size) {
+    return size * DecimalBase * DecimalBase * DecimalBase * DecimalBase;
+}
+
+constexpr std::size_t operator"" _PB(unsigned long long size) {
+    return size * DecimalBase * DecimalBase * DecimalBase * DecimalBase * DecimalBase;
+}
+
+} // namespace bxt::MemoryLiterals

dbcli/CMakeLists.txt → db-cli/CMakeLists.txt

@@ -1,21 +1,23 @@
 ################################################################################
-# Set up dbcli utility
+# Set up db-cli utility
 ################################################################################
 
-add_executable(
-  dbcli
-  dbcli.cpp
+include("${CMAKE_SOURCE_DIR}/cmake/db-cli-deps.cmake")
+
+add_executable(db-cli
+  cli.cpp  
+  validation.h
   ../daemon/core/domain/enums/PoolLocation.cpp
   ../daemon/utilities/alpmdb/Desc.cpp
   ../daemon/utilities/alpmdb/PkgInfo.cpp
   ../daemon/utilities/alpmdb/DescFormatter.cpp
   ../daemon/utilities/libarchive/Reader.cpp
 )
 
-target_link_libraries(dbcli PRIVATE deps)
+target_link_libraries(db-cli PRIVATE db-cli-deps)
 
 target_include_directories(
-  dbcli
+  db-cli
   PRIVATE
     ${CURL_INCLUDE_DIR}
     ${lmdbxx_SOURCE_DIR}/include

db-cli/cli.cpp

@@ -0,0 +1,181 @@
+
+/* === This file is part of bxt ===
+ *
+ *   SPDX-FileCopyrightText: 2024 Artem Grinev <agrinev@manjaro.org>
+ *   SPDX-License-Identifier: AGPL-3.0-or-later
+ *
+ */
+// Local
+#include "validation.h"
+
+// bxt
+#include <persistence/box/record/PackageRecord.h>
+#include <utilities/lmdb/CerealSerializer.h>
+#include <utilities/MemoryLiterals.h>
+#include <utilities/to_string.h>
+
+// External
+#include <CLI/CLI.hpp>
+#include <coro/io_scheduler.hpp>
+#include <coro/sync_wait.hpp>
+#include <fmt/color.h>
+#include <fmt/core.h>
+#include <fmt/std.h>
+#include <lmdbxx/lmdb++.h>
+
+// STL
+#include <string>
+
+namespace bxt::cli {
+
+namespace {
+    using namespace bxt::MemoryLiterals;
+    constexpr size_t LmdbMaxDbs = 128;
+    constexpr size_t LmdbMapSize = 50_GiB;
+} // namespace
+
+using Serializer = bxt::Utilities::LMDB::CerealSerializer<bxt::Persistence::Box::PackageRecord>;
+namespace handlers {
+    int list(lmdb::txn& transaction, lmdb::dbi& db, std::string const& prefix) {
+        auto cursor = lmdb::cursor::open(transaction, db);
+        if (!prefix.empty()) {
+            std::string_view key;
+            if (!cursor.get(key, MDB_SET_RANGE) || !key.starts_with(prefix)) {
+                fmt::print(stderr, "No packages found with prefix {}\n", prefix);
+                return 1;
+            }
+            while (cursor.get(key, MDB_NEXT) && key.starts_with(prefix)) {
+                fmt::print("{}\n", key);
+            }
+        } else {
+            std::string_view key;
+            while (cursor.get(key, MDB_NEXT)) {
+                fmt::print("{}\n", key);
+            }
+        }
+        return 0;
+    }
+
+    int get(lmdb::txn& transaction, lmdb::dbi& db, std::string const& key) {
+        std::string_view data;
+        auto result = db.get(transaction, key, data);
+        if (!result) {
+            fmt::print(stderr, "Failed to retrieve value or value not found.\n");
+            return 1;
+        }
+
+        auto const package = Serializer::deserialize(std::string(data));
+        if (!package.has_value()) {
+            fmt::print(stderr, "Failed to deserialize package.\n");
+            return 1;
+        }
+
+        fmt::print("{}\nIs any arch: {}\nDescriptions:\n", package->id.to_string(),
+                   package->is_any_architecture);
+        for (auto const& [key, value] : package->descriptions) {
+            fmt::print("==={}===\nFilepath: {}\nSignature path: "
+                       "{}\nDescfile:\n\n{}\n",
+                       bxt::to_string(key), value.filepath.string(), value.signature_path->string(),
+                       value.descfile.desc);
+        }
+        return 0;
+    }
+
+    int delete_(lmdb::txn& transaction, lmdb::dbi& db, std::string const& key) {
+        auto result = db.del(transaction, key);
+        if (result) {
+            fmt::print("Value deleted successfully.\n");
+            return 0;
+        } else {
+            fmt::print(stderr, "Failed to delete value or value not found.\n");
+            return 1;
+        }
+    }
+
+    int validate(lmdb::txn& transaction, lmdb::dbi& db) {
+        Validator validator(transaction, db, false, false);
+        auto error_count = validator.validate_and_rebuild();
+        if (error_count == 0) {
+            fmt::print("No errors found.\n");
+        } else {
+            fmt::print("{} errors found.\n", error_count);
+        }
+        return error_count > 0 ? 1 : 0;
+    }
+
+    int rebuild(lmdb::txn& transaction, lmdb::dbi& db, bool rebuild_keys) {
+        Validator validator(transaction, db, true, rebuild_keys);
+
+        if (validator.validate_and_rebuild() == 0) {
+            fmt::print("Successfully rebuilt{} packages.\n",
+                       rebuild_keys ? " all package keys" : "");
+            transaction.commit();
+            return 0;
+        } else {
+            fmt::print(stderr, "Failed to rebuild packages.\n");
+            return 1;
+        }
+    }
+} // namespace handlers
+
+class DatabaseCli {
+public:
+    int run(int argc, char** argv) {
+        CLI::App app {"BXT Database CLI Tool"};
+        app.require_subcommand(1);
+
+        std::string prefix;
+        auto list = app.add_subcommand("list", "List packages");
+        list->add_option("prefix", prefix, "Package name prefix to filter by");
+
+        std::string get_key;
+        auto get = app.add_subcommand("get", "Get package details");
+        get->add_option("key", get_key, "Package key to retrieve")->required();
+
+        std::string del_key;
+        auto del = app.add_subcommand("del", "Delete package");
+        del->add_option("key", del_key, "Package key to delete")->required();
+
+        auto validate = app.add_subcommand("validate", "Validate database records");
+
+        bool rebuild_keys = false;
+        auto rebuild = app.add_subcommand("rebuild", "Rebuild database records");
+        rebuild->add_flag("--keys", rebuild_keys, "Rebuild package keys");
+
+        CLI11_PARSE(app, argc, argv);
+
+        auto lmdbenv = lmdb::env::create();
+        lmdbenv.set_mapsize(LmdbMapSize);
+        lmdbenv.set_max_dbs(LmdbMaxDbs);
+        std::error_code ec;
+
+        if (!std::filesystem::exists("./bxtd.lmdb", ec) || ec) {
+            fmt::print(stderr, "bxtd.lmdb not found\n");
+            return 1;
+        }
+
+        lmdbenv.open("./bxtd.lmdb");
+        auto transaction = lmdb::txn::begin(lmdbenv);
+        auto db = lmdb::dbi::open(transaction, "bxt::Box");
+
+        if (list->parsed()) {
+            return handlers::list(transaction, db, prefix);
+        } else if (get->parsed()) {
+            return handlers::get(transaction, db, get_key);
+        } else if (del->parsed()) {
+            return handlers::delete_(transaction, db, del_key);
+        } else if (validate->parsed()) {
+            return handlers::validate(transaction, db);
+        } else if (rebuild->parsed()) {
+            return handlers::rebuild(transaction, db, rebuild_keys);
+        }
+
+        return 0;
+    }
+};
+
+} // namespace bxt::cli
+
+int main(int argc, char** argv) {
+    return bxt::cli::DatabaseCli {}.run(argc, argv);
+}