From e4231dfe7dff13368c071c69cfbcc6d0f682bb1c Mon Sep 17 00:00:00 2001 From: Artem Grinev Date: Sat, 27 Apr 2024 15:43:19 +0000 Subject: [PATCH 01/10] feat: expose box path over https using caddy --- docker-compose.caddy.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.caddy.yml b/docker-compose.caddy.yml index 985a6e3a..178fe986 100644 --- a/docker-compose.caddy.yml +++ b/docker-compose.caddy.yml @@ -11,6 +11,7 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock - caddy_data:/data + - persistence_volume: /persistence restart: unless-stopped production: ports: !reset [] @@ -19,6 +20,9 @@ services: labels: caddy: "${CADDY_HOST}" caddy.reverse_proxy: "{{upstreams 8080}}" + caddy.1_handle_path: "/box/*" + caddy.1_handle_path.root: "* /persistence/box" + caddy.1_handle_path.file_server: "browse" depends_on: - caddy From 4c2e99ae8d99d2d1feb7e7ca2fcf1a8eefc75e90 Mon Sep 17 00:00:00 2001 From: Artem Grinev Date: Sat, 27 Apr 2024 15:50:59 +0000 Subject: [PATCH 02/10] fix(daemon): export symlinks to the box structure --- .../persistence/box/export/AlpmDBExporter.cpp | 49 +++++++++++++++++-- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/daemon/persistence/box/export/AlpmDBExporter.cpp b/daemon/persistence/box/export/AlpmDBExporter.cpp index c9c67088..a1814315 100644 --- a/daemon/persistence/box/export/AlpmDBExporter.cpp +++ b/daemon/persistence/box/export/AlpmDBExporter.cpp @@ -23,12 +23,36 @@ #include #include #include +#include #include #include #include namespace bxt::Persistence::Box { +void create_relative_symlink(const std::filesystem::path& target, + const std::filesystem::path& link) { + if (std::filesystem::is_symlink(link)) { return; } + std::error_code ec; + const auto relative_target = + std::filesystem::relative(target, link.parent_path(), ec); + + if (ec) { + logf("Failed to get relative symlink path. The error is \"{}\". Box " + "will be malformed, exiting.", + ec.message()); + exit(1); + } + + std::filesystem::create_symlink(relative_target, link, ec); + if (ec) { + logf("Failed to create symlink. The error is \"{}\". Box will be " + "malformed, exiting.", + ec.message()); + exit(1); + } +} + AlpmDBExporter::AlpmDBExporter( BoxOptions& box_options, PackageStoreBase& package_store, @@ -98,12 +122,29 @@ coro::task AlpmDBExporter::export_to_disk() { "Exporter: Description for \"{}/{}-{}\" is being added... ", std::string(section), name, *version); + const auto preferred_description = package.descriptions.at( + *Core::Domain::select_preferred_pool_location( + package.descriptions)); + Utilities::AlpmDb::DatabaseUtils::write_buffer_to_archive( *writer, fmt::format("{}-{}/desc", name, *version), - package.descriptions - .at(*Core::Domain::select_preferred_pool_location( - package.descriptions)) - .descfile.desc); + preferred_description.descfile.desc); + + const auto filepath_link = std::filesystem::absolute( + m_box_path / std::string(section) + / preferred_description.filepath.filename()); + + create_relative_symlink(preferred_description.filepath, + filepath_link); + + if (preferred_description.signature_path) { + const auto signature_link = std::filesystem::absolute( + m_box_path / std::string(section) + / preferred_description.signature_path->filename()); + + create_relative_symlink( + *preferred_description.signature_path, signature_link); + } logd("Exporter: Description for \"{}/{}-{}\" is added", std::string(section), name, *version); From 8cc2ba071eac5dacf1c578278ee7ab08a94172fb Mon Sep 17 00:00:00 2001 From: Artem Grinev Date: Sat, 27 Apr 2024 16:14:40 +0000 Subject: [PATCH 03/10] fix: handle missing preferred pool location --- daemon/persistence/box/export/AlpmDBExporter.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/daemon/persistence/box/export/AlpmDBExporter.cpp b/daemon/persistence/box/export/AlpmDBExporter.cpp index a1814315..1f1cac9e 100644 --- a/daemon/persistence/box/export/AlpmDBExporter.cpp +++ b/daemon/persistence/box/export/AlpmDBExporter.cpp @@ -104,10 +104,19 @@ coro::task AlpmDBExporter::export_to_disk() { const auto& [section, name] = *deserialized; + const auto preferred_location = + Core::Domain::select_preferred_pool_location( + package.descriptions); + + if (!preferred_location.has_value()) { + loge("Can't find preferred location for {}. " + "Skipping.", + name); + return Utilities::NavigationAction::Next; + } + const auto version = - package.descriptions - .at(*Core::Domain::select_preferred_pool_location( - package.descriptions)) + package.descriptions.at(*preferred_location) .descfile.get("VERSION"); if (!version.has_value()) { From 8bf720853211ef9830f3e30b4fba3763d0ea652b Mon Sep 17 00:00:00 2001 From: Artem Grinev Date: Mon, 29 Apr 2024 21:26:17 +0000 Subject: [PATCH 04/10] fix: fix volumes syntax error in compose file --- docker-compose.caddy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.caddy.yml b/docker-compose.caddy.yml index 178fe986..f4b14a7c 100644 --- a/docker-compose.caddy.yml +++ b/docker-compose.caddy.yml @@ -11,7 +11,7 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock - caddy_data:/data - - persistence_volume: /persistence + - persistence_volume:/persistence restart: unless-stopped production: ports: !reset [] From 8c7c567256ceaf6877f92d2d1de45a690ec16ffa Mon Sep 17 00:00:00 2001 From: Artem Grinev Date: Tue, 30 Apr 2024 06:37:49 +0000 Subject: [PATCH 05/10] refactor(daemon): separate exporter writer initialization --- .../persistence/box/export/AlpmDBExporter.cpp | 40 ++++++++++++++++--- .../persistence/box/export/AlpmDBExporter.h | 5 ++- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/daemon/persistence/box/export/AlpmDBExporter.cpp b/daemon/persistence/box/export/AlpmDBExporter.cpp index 1f1cac9e..e6046f04 100644 --- a/daemon/persistence/box/export/AlpmDBExporter.cpp +++ b/daemon/persistence/box/export/AlpmDBExporter.cpp @@ -82,14 +82,16 @@ coro::task AlpmDBExporter::export_to_disk() { for (const auto& section : m_dirty_sections) { logd("Exporter: \"{}\" export into the package manager format started", std::string(section)); - writers.emplace(section, Archive::Writer()); - archive_write_add_filter_zstd(writers.at(section)); - archive_write_set_format_pax_restricted(writers.at(section)); + auto writer = setup_writer(section); - writers.at(section).open_filename( - m_box_path / std::string(section) - / fmt::format("{}.db.tar.zst", section.repository)); + if (!writer.has_value()) { + logf("Exporter: Writer cannot be created, the error is \"{}\"", + writer.error().what()); + co_return; + } + + writers.emplace(section, std::move(*writer)); co_await m_package_store.accept( [this, writer = &writers.at(section)]( @@ -176,4 +178,30 @@ void AlpmDBExporter::add_dirty_sections( std::make_move_iterator(sections.end())); } +std::expected + AlpmDBExporter::setup_writer(const PackageSectionDTO& section) { + Archive::Writer writer; + + if (archive_write_add_filter_zstd(writer) < ARCHIVE_WARN) { + return bxt::make_error(std::move(writer)); + } + if (archive_write_set_format_pax_restricted(writer) < ARCHIVE_WARN) { + return bxt::make_error(std::move(writer)); + } + + const auto archive_path = + m_box_path / std::string(section) + / fmt::format("{}.db.tar.zst", section.repository); + + auto open_ok = writer.open_filename(archive_path); + if (!open_ok.has_value()) { return std::unexpected(open_ok.error()); } + + const auto archive_link = m_box_path / std::string(section) + / fmt::format("{}.db", section.repository); + + create_relative_symlink(archive_path, archive_link); + + return writer; +} + } // namespace bxt::Persistence::Box diff --git a/daemon/persistence/box/export/AlpmDBExporter.h b/daemon/persistence/box/export/AlpmDBExporter.h index 9903377e..22d6eca9 100644 --- a/daemon/persistence/box/export/AlpmDBExporter.h +++ b/daemon/persistence/box/export/AlpmDBExporter.h @@ -13,7 +13,8 @@ #include "persistence/box/BoxOptions.h" #include "persistence/box/export/ExporterBase.h" #include "persistence/box/store/PackageStoreBase.h" -#include "utilities/locked.h" +#include "utilities/libarchive/Error.h" +#include "utilities/libarchive/Writer.h" #include #include @@ -34,6 +35,8 @@ class AlpmDBExporter : public ExporterBase { std::set&& override) override; private: + std::expected + setup_writer(const PackageSectionDTO& section); std::filesystem::path m_box_path; std::set m_sections; PackageStoreBase& m_package_store; From eb883e382dac6bc2de0d684f312e8d9a84093a0b Mon Sep 17 00:00:00 2001 From: Artem Grinev Date: Sat, 4 May 2024 06:34:21 +0000 Subject: [PATCH 06/10] fix(daemon): improve AlpmDB exporter error handling --- .../persistence/box/export/AlpmDBExporter.cpp | 105 +++++++++++------- .../persistence/box/export/AlpmDBExporter.h | 3 +- daemon/utilities/alpmdb/Database.h | 28 ++++- daemon/utilities/errors/FsError.h | 15 +++ daemon/utilities/libarchive/Error.h | 9 +- daemon/utilities/libarchive/Writer.h | 4 +- 6 files changed, 112 insertions(+), 52 deletions(-) create mode 100644 daemon/utilities/errors/FsError.h diff --git a/daemon/persistence/box/export/AlpmDBExporter.cpp b/daemon/persistence/box/export/AlpmDBExporter.cpp index e6046f04..3bfd316d 100644 --- a/daemon/persistence/box/export/AlpmDBExporter.cpp +++ b/daemon/persistence/box/export/AlpmDBExporter.cpp @@ -30,27 +30,20 @@ namespace bxt::Persistence::Box { -void create_relative_symlink(const std::filesystem::path& target, - const std::filesystem::path& link) { - if (std::filesystem::is_symlink(link)) { return; } +std::expected + create_relative_symlink(const std::filesystem::path& target, + const std::filesystem::path& link) { + if (std::filesystem::is_symlink(link)) { return {}; } std::error_code ec; + const auto relative_target = std::filesystem::relative(target, link.parent_path(), ec); - - if (ec) { - logf("Failed to get relative symlink path. The error is \"{}\". Box " - "will be malformed, exiting.", - ec.message()); - exit(1); - } + if (ec) { return bxt::make_error(ec); } std::filesystem::create_symlink(relative_target, link, ec); - if (ec) { - logf("Failed to create symlink. The error is \"{}\". Box will be " - "malformed, exiting.", - ec.message()); - exit(1); - } + if (ec) { return bxt::make_error(ec); } + + return {}; } AlpmDBExporter::AlpmDBExporter( @@ -80,13 +73,14 @@ coro::task AlpmDBExporter::export_to_disk() { phmap::parallel_flat_hash_map writers; for (const auto& section : m_dirty_sections) { - logd("Exporter: \"{}\" export into the package manager format started", + logi("Exporter: \"{}\" export into the package manager format started", std::string(section)); auto writer = setup_writer(section); if (!writer.has_value()) { - logf("Exporter: Writer cannot be created, the error is \"{}\"", + logf("Exporter: Writer cannot be created, the error is \"{}\". " + "Stopping...", writer.error().what()); co_return; } @@ -101,7 +95,7 @@ coro::task AlpmDBExporter::export_to_disk() { if (!deserialized.has_value()) { loge("Exporter: Key \"{}\" is not valid. Skipping...", key); - return Utilities::NavigationAction::Next; + return Utilities::NavigationAction::Stop; } const auto& [section, name] = *deserialized; @@ -111,10 +105,10 @@ coro::task AlpmDBExporter::export_to_disk() { package.descriptions); if (!preferred_location.has_value()) { - loge("Can't find preferred location for {}. " - "Skipping.", + logf("Exporter: Can't find preferred location for \"{}\". " + "Stopping...", name); - return Utilities::NavigationAction::Next; + return Utilities::NavigationAction::Stop; } const auto version = @@ -122,14 +116,14 @@ coro::task AlpmDBExporter::export_to_disk() { .descfile.get("VERSION"); if (!version.has_value()) { - loge("Exporter: Version for \"{}\" is not valid. " - "Skipping...", + logf("Exporter: Version for \"{}\" is not valid. " + "Stopping...", key); - return Utilities::NavigationAction::Next; + return Utilities::NavigationAction::Stop; } - logd( + logi( "Exporter: Description for \"{}/{}-{}\" is being added... ", std::string(section), name, *version); @@ -137,34 +131,62 @@ coro::task AlpmDBExporter::export_to_disk() { *Core::Domain::select_preferred_pool_location( package.descriptions)); - Utilities::AlpmDb::DatabaseUtils::write_buffer_to_archive( - *writer, fmt::format("{}-{}/desc", name, *version), - preferred_description.descfile.desc); + if (auto write_ok = Utilities::AlpmDb::DatabaseUtils:: + write_buffer_to_archive( + *writer, fmt::format("{}-{}/desc", name, *version), + preferred_description.descfile.desc); + !write_ok) { + logf("Exporter: Can't write \"{}/{}-{}\" description to " + "the archive. The error is \"{}\"", + std::string(section), name, *version, + write_ok.error().what()); + + return Utilities::NavigationAction::Stop; + } const auto filepath_link = std::filesystem::absolute( m_box_path / std::string(section) / preferred_description.filepath.filename()); - create_relative_symlink(preferred_description.filepath, - filepath_link); + if (auto link_created_ok = create_relative_symlink( + preferred_description.filepath, filepath_link); + !link_created_ok) { + logf("Exporter: Can't link the package file for " + "\"{}/{}-{}\". The error is \"{}\". Stopping the " + "export...", + std::string(section), name, *version, + link_created_ok.error().what()); + + return Utilities::NavigationAction::Stop; + } if (preferred_description.signature_path) { const auto signature_link = std::filesystem::absolute( m_box_path / std::string(section) / preferred_description.signature_path->filename()); - create_relative_symlink( - *preferred_description.signature_path, signature_link); + if (auto link_created_ok = create_relative_symlink( + *preferred_description.signature_path, + signature_link); + !link_created_ok) { + logf("Exporter: Can't link the signature file for " + "\"{}/{}-{}\". The error is \"{}\". Stopping the " + "export...", + std::string(section), name, *version, + link_created_ok.error().what()); + + return Utilities::NavigationAction::Stop; + } } - logd("Exporter: Description for \"{}/{}-{}\" is added", + logi("Exporter: Description and link for \"{}/{}-{}\" is added", std::string(section), name, *version); return Utilities::NavigationAction::Next; }, std::string(section)); - logd("Exporter: \"{}\" export finished", std::string(section)); + logi("Exporter: \"{}\" export finished", std::string(section)); } m_dirty_sections.clear(); @@ -178,7 +200,7 @@ void AlpmDBExporter::add_dirty_sections( std::make_move_iterator(sections.end())); } -std::expected +std::expected AlpmDBExporter::setup_writer(const PackageSectionDTO& section) { Archive::Writer writer; @@ -193,13 +215,18 @@ std::expected m_box_path / std::string(section) / fmt::format("{}.db.tar.zst", section.repository); - auto open_ok = writer.open_filename(archive_path); - if (!open_ok.has_value()) { return std::unexpected(open_ok.error()); } + if (auto open_ok = writer.open_filename(archive_path); !open_ok) { + return std::unexpected(std::move(open_ok.error())); + } const auto archive_link = m_box_path / std::string(section) / fmt::format("{}.db", section.repository); - create_relative_symlink(archive_path, archive_link); + if (auto link_created_ok = + create_relative_symlink(archive_path, archive_link); + !link_created_ok) { + return std::unexpected(link_created_ok.error()); + } return writer; } diff --git a/daemon/persistence/box/export/AlpmDBExporter.h b/daemon/persistence/box/export/AlpmDBExporter.h index 22d6eca9..6b4c7bbd 100644 --- a/daemon/persistence/box/export/AlpmDBExporter.h +++ b/daemon/persistence/box/export/AlpmDBExporter.h @@ -13,6 +13,7 @@ #include "persistence/box/BoxOptions.h" #include "persistence/box/export/ExporterBase.h" #include "persistence/box/store/PackageStoreBase.h" +#include "utilities/Error.h" #include "utilities/libarchive/Error.h" #include "utilities/libarchive/Writer.h" @@ -35,7 +36,7 @@ class AlpmDBExporter : public ExporterBase { std::set&& override) override; private: - std::expected + std::expected setup_writer(const PackageSectionDTO& section); std::filesystem::path m_box_path; std::set m_sections; diff --git a/daemon/utilities/alpmdb/Database.h b/daemon/utilities/alpmdb/Database.h index 62b2f0d1..fa04bb24 100644 --- a/daemon/utilities/alpmdb/Database.h +++ b/daemon/utilities/alpmdb/Database.h @@ -47,9 +47,9 @@ coro::task> visitor); template -void write_buffer_to_archive(Archive::Writer& writer, - const std::string& name, - const TBuffer& buffer) { +Result write_buffer_to_archive(Archive::Writer& writer, + const std::string& name, + const TBuffer& buffer) { auto header = Archive::Header::default_file(); archive_entry_set_pathname(header, name.c_str()); @@ -57,10 +57,26 @@ void write_buffer_to_archive(Archive::Writer& writer, auto entry = writer.start_write(header); - if (!entry.has_value()) { return; } + if (!entry.has_value()) { + return bxt::make_error_with_source( + std::move(entry.error()), DatabaseError::ErrorType::IOError); + } - entry->write({buffer.begin(), buffer.end()}); - entry->finish(); + auto write_ok = entry->write({buffer.begin(), buffer.end()}); + + if (!write_ok.has_value()) { + return bxt::make_error_with_source( + std::move(write_ok.error()), DatabaseError::ErrorType::IOError); + } + + auto finish_ok = entry->finish(); + + if (!finish_ok.has_value()) { + return bxt::make_error_with_source( + std::move(finish_ok.error()), DatabaseError::ErrorType::IOError); + } + + return {}; } coro::task> diff --git a/daemon/utilities/errors/FsError.h b/daemon/utilities/errors/FsError.h new file mode 100644 index 00000000..26a4eb94 --- /dev/null +++ b/daemon/utilities/errors/FsError.h @@ -0,0 +1,15 @@ +/* === This file is part of bxt === + * + * SPDX-FileCopyrightText: 2024 Artem Grinev + * SPDX-License-Identifier: AGPL-3.0-or-later + * + */ +#pragma once +#include "utilities/Error.h" + +#include +namespace bxt { +struct FsError : public bxt::Error { + FsError(const std::error_code& ec) { message = ec.message(); } +}; +} // namespace bxt diff --git a/daemon/utilities/libarchive/Error.h b/daemon/utilities/libarchive/Error.h index f8406d90..e1c440f5 100644 --- a/daemon/utilities/libarchive/Error.h +++ b/daemon/utilities/libarchive/Error.h @@ -14,14 +14,17 @@ namespace Archive { -struct InvalidEntryError : public bxt::Error { +struct ArchiveError : public bxt::Error { + ArchiveError() { message = "Unknown Archive error"; } +}; + +struct InvalidEntryError : public ArchiveError { InvalidEntryError() { message = "The entry has no linked archive!"; } }; -struct LibArchiveError : public bxt::Error { +struct LibArchiveError : public ArchiveError { explicit LibArchiveError(archive* archive) { message = archive_error_string(archive); } }; - } // namespace Archive diff --git a/daemon/utilities/libarchive/Writer.h b/daemon/utilities/libarchive/Writer.h index b8e7bd52..a4468f71 100644 --- a/daemon/utilities/libarchive/Writer.h +++ b/daemon/utilities/libarchive/Writer.h @@ -31,9 +31,7 @@ class Writer { public: Entry() = default; - template - using Result = - std::expected>; + template using Result = std::expected; // Use the Result template for function return type Result write(const std::vector& data); From b9748ad4641fe236377b79bcefb6a448d4583ea4 Mon Sep 17 00:00:00 2001 From: Artem Grinev Date: Sat, 4 May 2024 14:44:33 +0000 Subject: [PATCH 07/10] refactor(daemon): modularize package export functions --- .../persistence/box/export/AlpmDBExporter.cpp | 236 ++++++++++-------- .../persistence/box/export/AlpmDBExporter.h | 11 +- 2 files changed, 147 insertions(+), 100 deletions(-) diff --git a/daemon/persistence/box/export/AlpmDBExporter.cpp b/daemon/persistence/box/export/AlpmDBExporter.cpp index 3bfd316d..2e987cc4 100644 --- a/daemon/persistence/box/export/AlpmDBExporter.cpp +++ b/daemon/persistence/box/export/AlpmDBExporter.cpp @@ -8,15 +8,15 @@ #include "core/application/dtos/PackageSectionDTO.h" #include "core/domain/enums/PoolLocation.h" -#include "persistence/box/record/PackageRecord.h" -#include "persistence/box/store/PackageStoreBase.h" +#include "utilities/Error.h" +#include "utilities/NavigationAction.h" #include "utilities/alpmdb/Database.h" -#include "utilities/libarchive/Writer.h" -#include "utilities/lmdb/Database.h" -#include "utilities/log/Logging.h" +#include "utilities/errors/FsError.h" +#include "utilities/libarchive/Error.h" #include #include +#include #include #include #include @@ -76,7 +76,9 @@ coro::task AlpmDBExporter::export_to_disk() { logi("Exporter: \"{}\" export into the package manager format started", std::string(section)); - auto writer = setup_writer(section); + /// TODO: We need to make this way more robust. At least a full backup + /// would work but maybe we can do something more smart... + auto writer = setup_alpmdb_writer(section); if (!writer.has_value()) { logf("Exporter: Writer cannot be created, the error is \"{}\". " @@ -89,99 +91,14 @@ coro::task AlpmDBExporter::export_to_disk() { co_await m_package_store.accept( [this, writer = &writers.at(section)]( - std::string_view key, - const PackageRecord& package) -> Utilities::NavigationAction { - const auto deserialized = PackageRecord::Id::from_string(key); - - if (!deserialized.has_value()) { - loge("Exporter: Key \"{}\" is not valid. Skipping...", key); - return Utilities::NavigationAction::Stop; - } - - const auto& [section, name] = *deserialized; - - const auto preferred_location = - Core::Domain::select_preferred_pool_location( - package.descriptions); - - if (!preferred_location.has_value()) { - logf("Exporter: Can't find preferred location for \"{}\". " - "Stopping...", - name); - return Utilities::NavigationAction::Stop; - } - - const auto version = - package.descriptions.at(*preferred_location) - .descfile.get("VERSION"); - - if (!version.has_value()) { - logf("Exporter: Version for \"{}\" is not valid. " - "Stopping...", - key); - - return Utilities::NavigationAction::Stop; - } - - logi( - "Exporter: Description for \"{}/{}-{}\" is being added... ", - std::string(section), name, *version); - - const auto preferred_description = package.descriptions.at( - *Core::Domain::select_preferred_pool_location( - package.descriptions)); - - if (auto write_ok = Utilities::AlpmDb::DatabaseUtils:: - write_buffer_to_archive( - *writer, fmt::format("{}-{}/desc", name, *version), - preferred_description.descfile.desc); - !write_ok) { - logf("Exporter: Can't write \"{}/{}-{}\" description to " - "the archive. The error is \"{}\"", - std::string(section), name, *version, - write_ok.error().what()); - - return Utilities::NavigationAction::Stop; - } - - const auto filepath_link = std::filesystem::absolute( - m_box_path / std::string(section) - / preferred_description.filepath.filename()); - - if (auto link_created_ok = create_relative_symlink( - preferred_description.filepath, filepath_link); - !link_created_ok) { - logf("Exporter: Can't link the package file for " - "\"{}/{}-{}\". The error is \"{}\". Stopping the " - "export...", - std::string(section), name, *version, - link_created_ok.error().what()); - + std::string_view key, const PackageRecord& package) { + if (auto export_ok = export_package(*writer, key, package); + !export_ok) { + logf(fmt::format("Exporter: {}. Stopping...", + export_ok.error())); return Utilities::NavigationAction::Stop; } - if (preferred_description.signature_path) { - const auto signature_link = std::filesystem::absolute( - m_box_path / std::string(section) - / preferred_description.signature_path->filename()); - - if (auto link_created_ok = create_relative_symlink( - *preferred_description.signature_path, - signature_link); - !link_created_ok) { - logf("Exporter: Can't link the signature file for " - "\"{}/{}-{}\". The error is \"{}\". Stopping the " - "export...", - std::string(section), name, *version, - link_created_ok.error().what()); - - return Utilities::NavigationAction::Stop; - } - } - - logi("Exporter: Description and link for \"{}/{}-{}\" is added", - std::string(section), name, *version); - return Utilities::NavigationAction::Next; }, std::string(section)); @@ -199,9 +116,9 @@ void AlpmDBExporter::add_dirty_sections( m_dirty_sections.insert(std::make_move_iterator(sections.begin()), std::make_move_iterator(sections.end())); } - +// Factory function for ALPM .db archive writer std::expected - AlpmDBExporter::setup_writer(const PackageSectionDTO& section) { + AlpmDBExporter::setup_alpmdb_writer(const PackageSectionDTO& section) { Archive::Writer writer; if (archive_write_add_filter_zstd(writer) < ARCHIVE_WARN) { @@ -231,4 +148,127 @@ std::expected return writer; } +struct PackageDetails { + PackageSectionDTO section; + std::string name; + std::string version; + PoolLocation preferred_location; +}; +// Validates and extracts essential package details from both key and record +std::expected + validate_package_key(std::string_view key, const PackageRecord& package) { + auto deserialized = PackageRecord::Id::from_string(key); + if (!deserialized.has_value()) { + return std::unexpected(fmt::format("Invalid package key: '{}'.", key)); + } + + const auto& [section, name] = *deserialized; + + auto location = select_preferred_pool_location(package.descriptions); + + if (!location.has_value()) { + return std::unexpected( + fmt::format("Can't select preferred location for '{}'", key)); + } + + std::optional version_string; + + auto description_it = package.descriptions.find(*location); + if (description_it == package.descriptions.end()) { + return std::unexpected(fmt::format( + "Package details not found for preferred location: '{}'.", key)); + } + if (!(version_string = description_it->second.descfile.get("VERSION")) + .has_value() + || version_string->empty()) { + return std::unexpected( + fmt::format("No valid version for package '{}'.", key)); + } + return PackageDetails {section, name, *version_string, *location}; +} + +// Writes the contents of package description file to section's ALPM Database +// archive +std::expected + write_package_description_to_alpmdb(Archive::Writer& alpmdb_writer, + const PackageDetails& details, + const PackageRecord& package) { + const auto& [section, name, version, preferred_location] = details; + auto desc = package.descriptions.at(preferred_location).descfile.desc; + auto desc_path = fmt::format("{}-{}/desc", name, version); + + if (auto write_ok = + Utilities::AlpmDb::DatabaseUtils::write_buffer_to_archive( + alpmdb_writer, desc_path, desc); + !write_ok) { + return std::unexpected( + fmt::format("Failed to write description for '{}/{}-{}'.", + std::string(section), name, version)); + } + return {}; +} + +// Symlinks (usually pool) package and optionally it's signature to the +// specified section +std::expected + symlink_package_files(const PackageDetails& details, + const PackageRecord& package, + const std::filesystem::path& box_path) { + auto [section, name, version, preferred_location] = details; + + auto description = package.descriptions.at(preferred_location); + + auto link_file_path = std::filesystem::absolute( + box_path / std::string(section) / description.filepath.filename()); + + if (auto link_ok = + create_relative_symlink(description.filepath, link_file_path); + !link_ok) { + return std::unexpected( + fmt::format("Failed to link package file for '{}/{}-{}'.", + std::string(section), name, version)); + } + + if (description.signature_path.has_value()) { + auto link_file_path = + std::filesystem::absolute(box_path / std::string(section) + / description.signature_path->filename()); + + if (auto link_ok = create_relative_symlink(*description.signature_path, + link_file_path); + !link_ok) { + return std::unexpected( + fmt::format("Failed to link signature file for '{}/{}-{}'.", + std::string(section), name, version)); + } + } + return {}; +} +// Exports package into the ALPM repository format by writing it's description +// into the .db file and symlinking package and signature files for specified +// section +std::expected + AlpmDBExporter::export_package(Archive::Writer& writer, + std::string_view key, + const PackageRecord& package) { + auto validated_details = validate_package_key(key, package); + if (!validated_details.has_value()) { + return std::unexpected(validated_details.error()); + } + + auto write_result = write_package_description_to_alpmdb( + writer, *validated_details, package); + if (!write_result.has_value()) { + return std::unexpected(write_result.error()); + } + + auto file_management_result = + symlink_package_files(*validated_details, package, m_box_path); + if (!file_management_result.has_value()) { + return std::unexpected(file_management_result.error()); + } + + return {}; +} + } // namespace bxt::Persistence::Box diff --git a/daemon/persistence/box/export/AlpmDBExporter.h b/daemon/persistence/box/export/AlpmDBExporter.h index 6b4c7bbd..86344a20 100644 --- a/daemon/persistence/box/export/AlpmDBExporter.h +++ b/daemon/persistence/box/export/AlpmDBExporter.h @@ -8,13 +8,14 @@ #pragma once #include "core/application/dtos/PackageSectionDTO.h" +#include "core/domain/entities/Section.h" #include "core/domain/repositories/RepositoryBase.h" #include "parallel_hashmap/phmap.h" #include "persistence/box/BoxOptions.h" #include "persistence/box/export/ExporterBase.h" #include "persistence/box/store/PackageStoreBase.h" #include "utilities/Error.h" -#include "utilities/libarchive/Error.h" +#include "utilities/errors/FsError.h" #include "utilities/libarchive/Writer.h" #include @@ -37,7 +38,13 @@ class AlpmDBExporter : public ExporterBase { private: std::expected - setup_writer(const PackageSectionDTO& section); + setup_alpmdb_writer(const PackageSectionDTO& section); + + std::expected + export_package(Archive::Writer& writer, + std::string_view key, + const PackageRecord& package); + std::filesystem::path m_box_path; std::set m_sections; PackageStoreBase& m_package_store; From eb1e9b169ed728d8d98eb4294814a3ab2d0b6b6e Mon Sep 17 00:00:00 2001 From: Artem Grinev Date: Sat, 4 May 2024 14:53:47 +0000 Subject: [PATCH 08/10] fix(daemon): improve symlink error handling if file exists --- daemon/persistence/box/export/AlpmDBExporter.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/daemon/persistence/box/export/AlpmDBExporter.cpp b/daemon/persistence/box/export/AlpmDBExporter.cpp index 2e987cc4..8eca64d0 100644 --- a/daemon/persistence/box/export/AlpmDBExporter.cpp +++ b/daemon/persistence/box/export/AlpmDBExporter.cpp @@ -24,18 +24,21 @@ #include #include #include -#include -#include -#include namespace bxt::Persistence::Box { - +// Creates the symlink relative to target std::expected create_relative_symlink(const std::filesystem::path& target, const std::filesystem::path& link) { - if (std::filesystem::is_symlink(link)) { return {}; } std::error_code ec; + if (std::filesystem::exists(link)) { + ec.assign(static_cast(std::errc::file_exists), + std::generic_category()); + return bxt::make_error(ec); + } + if (ec) { return bxt::make_error(ec); } + const auto relative_target = std::filesystem::relative(target, link.parent_path(), ec); if (ec) { return bxt::make_error(ec); } From c2b56f9223fd0091a9ad8d3c0a86c18373f5ec48 Mon Sep 17 00:00:00 2001 From: Artem Grinev Date: Sat, 4 May 2024 14:55:04 +0000 Subject: [PATCH 09/10] fix(daemon): cleanup section folder before export process --- .../persistence/box/export/AlpmDBExporter.cpp | 31 +++++++++++++++++++ .../persistence/box/export/AlpmDBExporter.h | 3 ++ 2 files changed, 34 insertions(+) diff --git a/daemon/persistence/box/export/AlpmDBExporter.cpp b/daemon/persistence/box/export/AlpmDBExporter.cpp index 8eca64d0..9a241fec 100644 --- a/daemon/persistence/box/export/AlpmDBExporter.cpp +++ b/daemon/persistence/box/export/AlpmDBExporter.cpp @@ -81,6 +81,8 @@ coro::task AlpmDBExporter::export_to_disk() { /// TODO: We need to make this way more robust. At least a full backup /// would work but maybe we can do something more smart... + if (!cleanup_section(section)) { co_return; } + auto writer = setup_alpmdb_writer(section); if (!writer.has_value()) { @@ -150,6 +152,35 @@ std::expected return writer; } +// Cleans up section before the export by removing all it's content +std::expected + AlpmDBExporter::cleanup_section(const PackageSectionDTO& section) { + std::error_code ec; + + std::filesystem::directory_iterator directory_iterator; + + constexpr auto handle_error = [](const auto& ec) { + logf("Exporter: Can't wipe the directory before the export, the " + "error is " + "\"{}\". Stopping...", + ec.message()); + return bxt::make_error(ec); + }; + + if (directory_iterator = std::filesystem::directory_iterator( + m_box_path / std::string(section), ec); + ec) { + return handle_error(ec); + } + + for (const auto& entry : directory_iterator) { + if (!std::filesystem::remove_all(entry.path(), ec)) { + return handle_error(ec); + } + } + + return {}; +} struct PackageDetails { PackageSectionDTO section; diff --git a/daemon/persistence/box/export/AlpmDBExporter.h b/daemon/persistence/box/export/AlpmDBExporter.h index 86344a20..1eb5297f 100644 --- a/daemon/persistence/box/export/AlpmDBExporter.h +++ b/daemon/persistence/box/export/AlpmDBExporter.h @@ -40,6 +40,9 @@ class AlpmDBExporter : public ExporterBase { std::expected setup_alpmdb_writer(const PackageSectionDTO& section); + std::expected + cleanup_section(const PackageSectionDTO& section); + std::expected export_package(Archive::Writer& writer, std::string_view key, From 0773cd1ac02c6da9fdf6fafbd3ebd1df054faff2 Mon Sep 17 00:00:00 2001 From: Artem Grinev Date: Wed, 8 May 2024 15:20:29 +0000 Subject: [PATCH 10/10] fix(daemon): remove redundant error code check --- daemon/persistence/box/export/AlpmDBExporter.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/daemon/persistence/box/export/AlpmDBExporter.cpp b/daemon/persistence/box/export/AlpmDBExporter.cpp index 9a241fec..de5e9b8d 100644 --- a/daemon/persistence/box/export/AlpmDBExporter.cpp +++ b/daemon/persistence/box/export/AlpmDBExporter.cpp @@ -37,7 +37,6 @@ std::expected std::generic_category()); return bxt::make_error(ec); } - if (ec) { return bxt::make_error(ec); } const auto relative_target = std::filesystem::relative(target, link.parent_path(), ec);