test(clp-s): Add end-to-end test case for compression and extraction.

components/core/CMakeLists.txt

@@ -241,6 +241,42 @@ add_subdirectory(src/clp_s)
 add_subdirectory(src/reducer)
 
 set(SOURCE_FILES_clp_s_unitTest
+    src/clp_s/ArchiveReader.cpp
+    src/clp_s/ArchiveReader.hpp
+    src/clp_s/ArchiveWriter.cpp
+    src/clp_s/ArchiveWriter.hpp
+    src/clp_s/ColumnReader.cpp
+    src/clp_s/ColumnReader.hpp
+    src/clp_s/ColumnWriter.cpp
+    src/clp_s/ColumnWriter.hpp
+    src/clp_s/DictionaryEntry.cpp
+    src/clp_s/DictionaryEntry.hpp
+    src/clp_s/DictionaryWriter.cpp
+    src/clp_s/DictionaryWriter.hpp
+    src/clp_s/FileReader.cpp
+    src/clp_s/FileReader.hpp
+    src/clp_s/FileWriter.cpp
+    src/clp_s/FileWriter.hpp
+    src/clp_s/JsonConstructor.cpp
+    src/clp_s/JsonConstructor.hpp
+    src/clp_s/JsonFileIterator.cpp
+    src/clp_s/JsonFileIterator.hpp
+    src/clp_s/JsonParser.cpp
+    src/clp_s/JsonParser.hpp
+    src/clp_s/PackedStreamReader.cpp
+    src/clp_s/PackedStreamReader.hpp
+    src/clp_s/ReaderUtils.cpp
+    src/clp_s/ReaderUtils.hpp
+    src/clp_s/Schema.cpp
+    src/clp_s/Schema.hpp
+    src/clp_s/SchemaMap.cpp
+    src/clp_s/SchemaMap.hpp
+    src/clp_s/SchemaReader.cpp
+    src/clp_s/SchemaReader.hpp
+    src/clp_s/SchemaTree.cpp
+    src/clp_s/SchemaTree.hpp
+    src/clp_s/SchemaWriter.cpp
+    src/clp_s/SchemaWriter.hpp
     src/clp_s/search/AndExpr.cpp
     src/clp_s/search/AndExpr.hpp
     src/clp_s/search/BooleanLiteral.cpp
@@ -273,11 +309,24 @@ set(SOURCE_FILES_clp_s_unitTest
     src/clp_s/search/StringLiteral.hpp
     src/clp_s/search/Transformation.hpp
     src/clp_s/search/Value.hpp
-    src/clp_s/SchemaTree.hpp
+    src/clp_s/TimestampDictionaryReader.cpp
+    src/clp_s/TimestampDictionaryReader.hpp
+    src/clp_s/TimestampDictionaryWriter.cpp
+    src/clp_s/TimestampDictionaryWriter.hpp
+    src/clp_s/TimestampEntry.cpp
+    src/clp_s/TimestampEntry.hpp
     src/clp_s/TimestampPattern.cpp
     src/clp_s/TimestampPattern.hpp
     src/clp_s/Utils.cpp
     src/clp_s/Utils.hpp
+    src/clp_s/VariableDecoder.cpp
+    src/clp_s/VariableDecoder.hpp
+    src/clp_s/VariableEncoder.cpp
+    src/clp_s/VariableEncoder.hpp
+    src/clp_s/ZstdCompressor.cpp
+    src/clp_s/ZstdCompressor.hpp
+    src/clp_s/ZstdDecompressor.cpp
+    src/clp_s/ZstdDecompressor.hpp
 )
 
 set(SOURCE_FILES_unitTest
@@ -501,6 +550,7 @@ set(SOURCE_FILES_unitTest
         tests/test-BufferedFileReader.cpp
         tests/test-EncodedVariableInterpreter.cpp
         tests/test-encoding_methods.cpp
+        tests/test-end_to_end.cpp
         tests/test-ffi_IrUnitHandlerInterface.cpp
         tests/test-ffi_KeyValuePairLogEvent.cpp
         tests/test-ffi_SchemaTree.cpp
@@ -542,6 +592,8 @@ target_link_libraries(unitTest
         log_surgeon::log_surgeon
         LibArchive::LibArchive
         MariaDBClient::MariaDBClient
+        ${MONGOCXX_TARGET}
+        simdjson
         spdlog::spdlog
         OpenSSL::Crypto
         ${sqlite_LIBRARY_DEPENDENCIES}

components/core/tests/test-end_to_end.cpp

@@ -0,0 +1,117 @@
+#include <sys/wait.h>
+
+#include <cstdlib>
+#include <filesystem>
+#include <string>
+#include <vector>
+
+#include <Catch2/single_include/catch2/catch.hpp>
+
+#include "../src/clp_s/JsonConstructor.hpp"
+#include "../src/clp_s/JsonParser.hpp"
+
+auto const cDefaultTargetEncodedSize = 8ULL * 1024 * 1024 * 1024;  // 8 GB
+auto const cDefaultMaxDocumentSize = 512ULL * 1024 * 1024;  // 512 MB
+auto const cDefaultMinTableSize = 1ULL * 1024 * 1024;  // 1 MB
+auto const cDefaultCompressionLevel = 3;
+auto const cDefaultPrintArchiveStats = false;
+auto const cDefaultOrdered = false;
+auto const cDefaultTargetOrderedChunkSize = 0;
+
+std::string const cTestEndToEndArchiveDirectory{"test-end-to-end-archive"};
+std::string const cTestEndToEndOutputDirectory = "test-end-to-end-out";
+std::string const cTestEndToEndOutputSortedJson = "test-end-to-end_sorted.json";
+std::string const cTestEndToEndInputFileDirectory = "test_log_files";
+std::string const cTestEndToEndInputFile = "test_no_floats_sorted.json";
+
+namespace {
+
+class Cleanup {
+public:
+    Cleanup() {
+        std::filesystem::remove_all(cTestEndToEndArchiveDirectory);
+        std::filesystem::remove_all(cTestEndToEndOutputDirectory);
+        std::filesystem::remove(cTestEndToEndOutputSortedJson);
+    }
+
+    ~Cleanup() {
+        std::filesystem::remove_all(cTestEndToEndArchiveDirectory);
+        std::filesystem::remove_all(cTestEndToEndOutputDirectory);
+        std::filesystem::remove(cTestEndToEndOutputSortedJson);
+    }
+};
+
+auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path {
+    return std::filesystem::path{cTestEndToEndInputFileDirectory} / cTestEndToEndInputFile;
+}
+
+auto get_test_input_local_path() -> std::string {
+    std::filesystem::path const current_file_path{__FILE__};
+    auto const tests_dir{current_file_path.parent_path()};
+    return (tests_dir / get_test_input_path_relative_to_tests_dir()).string();
+}
+}  // namespace
+
+// NOLINTNEXTLINE(readability-function-cognitive-complexity)
+TEST_CASE("clp-s_compression_and_extraction_no_floats", "[clp-s][end-to-end]") {
+    auto structurize_arrays = GENERATE(true, false);
+
+    Cleanup const test_cleanup;
+
+    std::filesystem::create_directory(cTestEndToEndArchiveDirectory);
+    REQUIRE(std::filesystem::is_directory(cTestEndToEndArchiveDirectory));
+
+    clp_s::JsonParserOption parser_option{};
+    parser_option.file_paths.push_back(get_test_input_local_path());
+    parser_option.archives_dir = cTestEndToEndArchiveDirectory;
+    parser_option.target_encoded_size = cDefaultTargetEncodedSize;
+    parser_option.max_document_size = cDefaultMaxDocumentSize;
+    parser_option.min_table_size = cDefaultMinTableSize;
+    parser_option.compression_level = cDefaultCompressionLevel;
+    parser_option.print_archive_stats = cDefaultPrintArchiveStats;
+    parser_option.structurize_arrays = structurize_arrays;
+
+    clp_s::JsonParser parser(parser_option);
+    REQUIRE(parser.parse());
+    parser.store();
+
+    REQUIRE(false == std::filesystem::is_empty(cTestEndToEndArchiveDirectory));
+
+    std::filesystem::create_directory(cTestEndToEndOutputDirectory);
+    REQUIRE(std::filesystem::is_directory(cTestEndToEndOutputDirectory));
+
+    clp_s::JsonConstructorOption constructor_option{};
+    constructor_option.archives_dir = parser_option.archives_dir;
+    constructor_option.output_dir = cTestEndToEndOutputDirectory;
+    constructor_option.ordered = false;
+    constructor_option.target_ordered_chunk_size = 0;
+    for (auto const& entry : std::filesystem::directory_iterator(constructor_option.archives_dir)) {
+        if (false == entry.is_directory()) {
+            // Skip non-directories
+            continue;
+        }
+
+        constructor_option.archive_id = entry.path().filename();
+        clp_s::JsonConstructor constructor(constructor_option);
+        constructor.store();
+    }
+
+    std::string command = cTestEndToEndOutputDirectory + "/original";
+    REQUIRE(std::filesystem::exists(command.c_str()));
+
+    int result = std::system("command -v jq >/dev/null 2>&1");
+    REQUIRE(0 == result);
+    command = "jq -S -c '.' " + cTestEndToEndOutputDirectory + "/original | sort > "
+              + cTestEndToEndOutputSortedJson;
+    result = std::system(command.c_str());
+    REQUIRE(0 == result);
+
+    REQUIRE(false == std::filesystem::is_empty(cTestEndToEndOutputSortedJson));
+
+    result = std::system("command -v diff >/dev/null 2>&1");
+    REQUIRE(0 == result);
+    command = "diff -u " + cTestEndToEndOutputSortedJson + " " + get_test_input_local_path()
+              + " > /dev/null";
+    result = std::system(command.c_str());
+    REQUIRE(0 == WEXITSTATUS(result));
+}

components/core/tests/test_log_files/test_no_floats_sorted.json

@@ -0,0 +1,4 @@
+{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true}
+{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true}
+{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"non_empty_object2":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true}
+{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_supported":9223372036854776,"int64_min_supported":-9223372036854776,"int8_max":127,"int8_min":-128,"nonempty_array":[1,2,3,4,5],"null":null,"string":"short_string","true":true}

components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh

@@ -8,9 +8,11 @@ set -u
 
 dnf install -y \
     cmake \
+    diffutils \
     gcc-c++ \
     git \
     java-11-openjdk \
+    jq \
     libarchive-devel \
     libcurl-devel \
     libzstd-devel \

components/core/tools/scripts/lib_install/ubuntu-focal/install-prebuilt-packages.sh

@@ -12,11 +12,13 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
   checkinstall \
   cmake \
   curl \
+  diffutils \
   g++ \
   g++-10 \
   gcc \
   gcc-10 \
   git \
+  jq \
   libcurl4 \
   libcurl4-openssl-dev \
   libmariadb-dev \

components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh

@@ -13,7 +13,9 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
   cmake \
   curl \
   build-essential \
+  diffutils \
   git \
+  jq \
   libboost-filesystem-dev \
   libboost-iostreams-dev \
   libboost-program-options-dev \