restructure into compress, extract, compare

components/core/CMakeLists.txt

@@ -548,9 +548,9 @@ set(SOURCE_FILES_unitTest
         tests/LogSuppressor.hpp
         tests/test-Array.cpp
         tests/test-BufferedFileReader.cpp
+        tests/test-clp_s-end_to_end.cpp
         tests/test-EncodedVariableInterpreter.cpp
         tests/test-encoding_methods.cpp
-        tests/test-end_to_end.cpp
         tests/test-ffi_IrUnitHandlerInterface.cpp
         tests/test-ffi_KeyValuePairLogEvent.cpp
         tests/test-ffi_SchemaTree.cpp

components/core/tests/test-clp_s-end_to_end.cpp

@@ -0,0 +1,159 @@
+#include <sys/wait.h>
+
+#include <cstdlib>
+#include <filesystem>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include <Catch2/single_include/catch2/catch.hpp>
+#include <fmt/format.h>
+
+#include "../src/clp_s/JsonConstructor.hpp"
+#include "../src/clp_s/JsonParser.hpp"
+
+constexpr std::string_view cTestEndToEndArchiveDirectory{"test-end-to-end-archive"};
+constexpr std::string_view cTestEndToEndOutputDirectory{"test-end-to-end-out"};
+constexpr std::string_view cTestEndToEndOutputSortedJson{"test-end-to-end_sorted.jsonl"};
+constexpr std::string_view cTestEndToEndInputFileDirectory{"test_log_files"};
+constexpr std::string_view cTestEndToEndInputFile{"test_no_floats_sorted.jsonl"};
+
+namespace {
+/**
+ * A class that deletes the directories and files created by test cases, both before and after each
+ * test case where the class is instantiated.
+ */
+class TestOutputCleaner {
+public:
+    TestOutputCleaner() { delete_files(); }
+
+    ~TestOutputCleaner() { delete_files(); }
+
+    // Delete copy & move constructors and assignment operators
+    TestOutputCleaner(TestOutputCleaner const&) = delete;
+    TestOutputCleaner(TestOutputCleaner&&) = delete;
+    auto operator=(TestOutputCleaner const&) -> TestOutputCleaner& = delete;
+    auto operator=(TestOutputCleaner&&) -> TestOutputCleaner& = delete;
+
+private:
+    static void delete_files() {
+        std::filesystem::remove_all(cTestEndToEndArchiveDirectory);
+        std::filesystem::remove_all(cTestEndToEndOutputDirectory);
+        std::filesystem::remove(cTestEndToEndOutputSortedJson);
+    }
+};
+
+auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path;
+auto get_test_input_local_path() -> std::string;
+void compress(bool structurize_arrays);
+auto extract() -> std::filesystem::path;
+void compare(std::filesystem::path const& extracted_json_path);
+
+auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path {
+    return std::filesystem::path{cTestEndToEndInputFileDirectory} / cTestEndToEndInputFile;
+}
+
+auto get_test_input_local_path() -> std::string {
+    std::filesystem::path const current_file_path{__FILE__};
+    auto const tests_dir{current_file_path.parent_path()};
+    return (tests_dir / get_test_input_path_relative_to_tests_dir()).string();
+}
+
+void compress(bool structurize_arrays) {
+    constexpr auto cDefaultTargetEncodedSize = 8ULL * 1024 * 1024 * 1024;  // 8 GiB
+    constexpr auto cDefaultMaxDocumentSize = 512ULL * 1024 * 1024;  // 512 MiB
+    constexpr auto cDefaultMinTableSize = 1ULL * 1024 * 1024;  // 1 MiB
+    constexpr auto cDefaultCompressionLevel = 3;
+    constexpr auto cDefaultPrintArchiveStats = false;
+
+    std::filesystem::create_directory(cTestEndToEndArchiveDirectory);
+    REQUIRE((std::filesystem::is_directory(cTestEndToEndArchiveDirectory)));
+
+    clp_s::JsonParserOption parser_option{};
+    parser_option.file_paths.push_back(get_test_input_local_path());
+    parser_option.archives_dir = cTestEndToEndArchiveDirectory;
+    parser_option.target_encoded_size = cDefaultTargetEncodedSize;
+    parser_option.max_document_size = cDefaultMaxDocumentSize;
+    parser_option.min_table_size = cDefaultMinTableSize;
+    parser_option.compression_level = cDefaultCompressionLevel;
+    parser_option.print_archive_stats = cDefaultPrintArchiveStats;
+    parser_option.structurize_arrays = structurize_arrays;
+
+    clp_s::JsonParser parser{parser_option};
+    REQUIRE(parser.parse());
+    parser.store();
+
+    REQUIRE((false == std::filesystem::is_empty(cTestEndToEndArchiveDirectory)));
+}
+
+auto extract() -> std::filesystem::path {
+    constexpr auto cDefaultOrdered = false;
+    constexpr auto cDefaultTargetOrderedChunkSize = 0;
+
+    std::filesystem::create_directory(cTestEndToEndOutputDirectory);
+    REQUIRE(std::filesystem::is_directory(cTestEndToEndOutputDirectory));
+
+    clp_s::JsonConstructorOption constructor_option{};
+    constructor_option.archives_dir = cTestEndToEndArchiveDirectory;
+    constructor_option.output_dir = cTestEndToEndOutputDirectory;
+    constructor_option.ordered = cDefaultOrdered;
+    constructor_option.target_ordered_chunk_size = cDefaultTargetOrderedChunkSize;
+    for (auto const& entry : std::filesystem::directory_iterator(constructor_option.archives_dir)) {
+        if (false == entry.is_directory()) {
+            // Skip non-directories
+            continue;
+        }
+
+        constructor_option.archive_id = entry.path().filename();
+        clp_s::JsonConstructor constructor{constructor_option};
+        constructor.store();
+    }
+    std::filesystem::path extracted_json_path{cTestEndToEndOutputDirectory};
+    extracted_json_path /= "original";
+    REQUIRE(std::filesystem::exists(extracted_json_path));
+
+    return extracted_json_path;
+}
+
+// Silence the checks below since our use of `std::system` is safe in the context of testing.
+// NOLINTBEGIN(cert-env33-c,concurrency-mt-unsafe)
+void compare(std::filesystem::path const& extracted_json_path) {
+    int result{std::system("command -v jq >/dev/null 2>&1")};
+    REQUIRE((0 == result));
+    std::string command = fmt::format(
+            "jq --sort-keys --compact-output '.' {} | sort > {}",
+            extracted_json_path.string(),
+            cTestEndToEndOutputSortedJson
+    );
+    result = std::system(command.c_str());
+    REQUIRE((0 == result));
+
+    REQUIRE((false == std::filesystem::is_empty(cTestEndToEndOutputSortedJson)));
+
+    result = std::system("command -v diff >/dev/null 2>&1");
+    REQUIRE((0 == result));
+    command = fmt::format(
+            "diff --unified {} {}  > /dev/null",
+            cTestEndToEndOutputSortedJson,
+            get_test_input_local_path()
+    );
+    result = std::system(command.c_str());
+    REQUIRE((true == WIFEXITED(result)));
+    REQUIRE((0 == WEXITSTATUS(result)));
+}
+
+// NOLINTEND(cert-env33-c,concurrency-mt-unsafe)
+}  // namespace
+
+// NOLINTNEXTLINE(readability-function-cognitive-complexity)
+TEST_CASE("clp-s-compress-extract-no-floats", "[clp-s][end-to-end]") {
+    auto structurize_arrays = GENERATE(true, false);
+
+    TestOutputCleaner const test_cleanup;
+
+    compress(structurize_arrays);
+
+    std::filesystem::path extracted_json_path = extract();
+
+    compare(extracted_json_path);
+}