From 03d65109b2ee7cf53e39c23f41a3e301d597f5cc Mon Sep 17 00:00:00 2001 From: Abigail Matthews Date: Fri, 29 Nov 2024 17:33:09 -0500 Subject: [PATCH] restructure into compress, extract, compare --- components/core/CMakeLists.txt | 2 +- .../core/tests/test-clp_s-end_to_end.cpp | 159 ++++++++++++++++++ components/core/tests/test-end_to_end.cpp | 127 -------------- ...orted.json => test_no_floats_sorted.jsonl} | 0 4 files changed, 160 insertions(+), 128 deletions(-) create mode 100644 components/core/tests/test-clp_s-end_to_end.cpp delete mode 100644 components/core/tests/test-end_to_end.cpp rename components/core/tests/test_log_files/{test_no_floats_sorted.json => test_no_floats_sorted.jsonl} (100%) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index d4f9ff918..1b4fdb1be 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -548,9 +548,9 @@ set(SOURCE_FILES_unitTest tests/LogSuppressor.hpp tests/test-Array.cpp tests/test-BufferedFileReader.cpp + tests/test-clp_s-end_to_end.cpp tests/test-EncodedVariableInterpreter.cpp tests/test-encoding_methods.cpp - tests/test-end_to_end.cpp tests/test-ffi_IrUnitHandlerInterface.cpp tests/test-ffi_KeyValuePairLogEvent.cpp tests/test-ffi_SchemaTree.cpp diff --git a/components/core/tests/test-clp_s-end_to_end.cpp b/components/core/tests/test-clp_s-end_to_end.cpp new file mode 100644 index 000000000..1009111b4 --- /dev/null +++ b/components/core/tests/test-clp_s-end_to_end.cpp @@ -0,0 +1,159 @@ +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include "../src/clp_s/JsonConstructor.hpp" +#include "../src/clp_s/JsonParser.hpp" + +constexpr std::string_view cTestEndToEndArchiveDirectory{"test-end-to-end-archive"}; +constexpr std::string_view cTestEndToEndOutputDirectory{"test-end-to-end-out"}; +constexpr std::string_view cTestEndToEndOutputSortedJson{"test-end-to-end_sorted.jsonl"}; +constexpr std::string_view cTestEndToEndInputFileDirectory{"test_log_files"}; +constexpr std::string_view cTestEndToEndInputFile{"test_no_floats_sorted.jsonl"}; + +namespace { +/** + * A class that deletes the directories and files created by test cases, both before and after each + * test case where the class is instantiated. + */ +class TestOutputCleaner { +public: + TestOutputCleaner() { delete_files(); } + + ~TestOutputCleaner() { delete_files(); } + + // Delete copy & move constructors and assignment operators + TestOutputCleaner(TestOutputCleaner const&) = delete; + TestOutputCleaner(TestOutputCleaner&&) = delete; + auto operator=(TestOutputCleaner const&) -> TestOutputCleaner& = delete; + auto operator=(TestOutputCleaner&&) -> TestOutputCleaner& = delete; + +private: + static void delete_files() { + std::filesystem::remove_all(cTestEndToEndArchiveDirectory); + std::filesystem::remove_all(cTestEndToEndOutputDirectory); + std::filesystem::remove(cTestEndToEndOutputSortedJson); + } +}; + +auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path; +auto get_test_input_local_path() -> std::string; +void compress(bool structurize_arrays); +auto extract() -> std::filesystem::path; +void compare(std::filesystem::path const& extracted_json_path); + +auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path { + return std::filesystem::path{cTestEndToEndInputFileDirectory} / cTestEndToEndInputFile; +} + +auto get_test_input_local_path() -> std::string { + std::filesystem::path const current_file_path{__FILE__}; + auto const tests_dir{current_file_path.parent_path()}; + return (tests_dir / get_test_input_path_relative_to_tests_dir()).string(); +} + +void compress(bool structurize_arrays) { + constexpr auto cDefaultTargetEncodedSize = 8ULL * 1024 * 1024 * 1024; // 8 GiB + constexpr auto cDefaultMaxDocumentSize = 512ULL * 1024 * 1024; // 512 MiB + constexpr auto cDefaultMinTableSize = 1ULL * 1024 * 1024; // 1 MiB + constexpr auto cDefaultCompressionLevel = 3; + constexpr auto cDefaultPrintArchiveStats = false; + + std::filesystem::create_directory(cTestEndToEndArchiveDirectory); + REQUIRE((std::filesystem::is_directory(cTestEndToEndArchiveDirectory))); + + clp_s::JsonParserOption parser_option{}; + parser_option.file_paths.push_back(get_test_input_local_path()); + parser_option.archives_dir = cTestEndToEndArchiveDirectory; + parser_option.target_encoded_size = cDefaultTargetEncodedSize; + parser_option.max_document_size = cDefaultMaxDocumentSize; + parser_option.min_table_size = cDefaultMinTableSize; + parser_option.compression_level = cDefaultCompressionLevel; + parser_option.print_archive_stats = cDefaultPrintArchiveStats; + parser_option.structurize_arrays = structurize_arrays; + + clp_s::JsonParser parser{parser_option}; + REQUIRE(parser.parse()); + parser.store(); + + REQUIRE((false == std::filesystem::is_empty(cTestEndToEndArchiveDirectory))); +} + +auto extract() -> std::filesystem::path { + constexpr auto cDefaultOrdered = false; + constexpr auto cDefaultTargetOrderedChunkSize = 0; + + std::filesystem::create_directory(cTestEndToEndOutputDirectory); + REQUIRE(std::filesystem::is_directory(cTestEndToEndOutputDirectory)); + + clp_s::JsonConstructorOption constructor_option{}; + constructor_option.archives_dir = cTestEndToEndArchiveDirectory; + constructor_option.output_dir = cTestEndToEndOutputDirectory; + constructor_option.ordered = cDefaultOrdered; + constructor_option.target_ordered_chunk_size = cDefaultTargetOrderedChunkSize; + for (auto const& entry : std::filesystem::directory_iterator(constructor_option.archives_dir)) { + if (false == entry.is_directory()) { + // Skip non-directories + continue; + } + + constructor_option.archive_id = entry.path().filename(); + clp_s::JsonConstructor constructor{constructor_option}; + constructor.store(); + } + std::filesystem::path extracted_json_path{cTestEndToEndOutputDirectory}; + extracted_json_path /= "original"; + REQUIRE(std::filesystem::exists(extracted_json_path)); + + return extracted_json_path; +} + +// Silence the checks below since our use of `std::system` is safe in the context of testing. +// NOLINTBEGIN(cert-env33-c,concurrency-mt-unsafe) +void compare(std::filesystem::path const& extracted_json_path) { + int result{std::system("command -v jq >/dev/null 2>&1")}; + REQUIRE((0 == result)); + std::string command = fmt::format( + "jq --sort-keys --compact-output '.' {} | sort > {}", + extracted_json_path.string(), + cTestEndToEndOutputSortedJson + ); + result = std::system(command.c_str()); + REQUIRE((0 == result)); + + REQUIRE((false == std::filesystem::is_empty(cTestEndToEndOutputSortedJson))); + + result = std::system("command -v diff >/dev/null 2>&1"); + REQUIRE((0 == result)); + command = fmt::format( + "diff --unified {} {} > /dev/null", + cTestEndToEndOutputSortedJson, + get_test_input_local_path() + ); + result = std::system(command.c_str()); + REQUIRE((true == WIFEXITED(result))); + REQUIRE((0 == WEXITSTATUS(result))); +} + +// NOLINTEND(cert-env33-c,concurrency-mt-unsafe) +} // namespace + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +TEST_CASE("clp-s-compress-extract-no-floats", "[clp-s][end-to-end]") { + auto structurize_arrays = GENERATE(true, false); + + TestOutputCleaner const test_cleanup; + + compress(structurize_arrays); + + std::filesystem::path extracted_json_path = extract(); + + compare(extracted_json_path); +} diff --git a/components/core/tests/test-end_to_end.cpp b/components/core/tests/test-end_to_end.cpp deleted file mode 100644 index 6b1648df3..000000000 --- a/components/core/tests/test-end_to_end.cpp +++ /dev/null @@ -1,127 +0,0 @@ -#include - -#include -#include -#include -#include -#include - -#include -#include - -#include "../src/clp_s/JsonConstructor.hpp" -#include "../src/clp_s/JsonParser.hpp" - -constexpr std::string_view cTestEndToEndArchiveDirectory{"test-end-to-end-archive"}; -constexpr std::string_view cTestEndToEndOutputDirectory{"test-end-to-end-out"}; -constexpr std::string_view cTestEndToEndOutputSortedJson{"test-end-to-end_sorted.json"}; -constexpr std::string_view cTestEndToEndInputFileDirectory{"test_log_files"}; -constexpr std::string_view cTestEndToEndInputFile{"test_no_floats_sorted.json"}; - -namespace { -/** - * Class with no data members who's contruction and destruction is soley - * for cleanup up files and directorys created by the test case. - */ -class Cleanup { -public: - Cleanup() { delete_files(); } - - ~Cleanup() { delete_files(); } - -private: - static void delete_files() { - std::filesystem::remove_all(cTestEndToEndArchiveDirectory); - std::filesystem::remove_all(cTestEndToEndOutputDirectory); - std::filesystem::remove(cTestEndToEndOutputSortedJson); - } -}; - -auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path { - return std::filesystem::path{cTestEndToEndInputFileDirectory} / cTestEndToEndInputFile; -} - -auto get_test_input_local_path() -> std::string { - std::filesystem::path const current_file_path{__FILE__}; - auto const tests_dir{current_file_path.parent_path()}; - return (tests_dir / get_test_input_path_relative_to_tests_dir()).string(); -} -} // namespace - -// NOLINTNEXTLINE(readability-function-cognitive-complexity) -TEST_CASE("clp-s_compression_and_extraction_no_floats", "[clp-s][end-to-end]") { - auto const default_target_encoded_size = 8ULL * 1024 * 1024 * 1024; // 8 GiB - auto const default_max_document_size = 512ULL * 1024 * 1024; // 512 MiB - auto const default_min_table_size = 1ULL * 1024 * 1024; // 1 MiB - auto const default_compression_level = 3; - auto const default_print_archive_stats = false; - auto const default_ordered = false; - auto const default_target_ordered_chunk_size = 0; - auto structurize_arrays = GENERATE(true, false); - - Cleanup const test_cleanup; - - std::filesystem::create_directory(cTestEndToEndArchiveDirectory); - REQUIRE(std::filesystem::is_directory(cTestEndToEndArchiveDirectory)); - - clp_s::JsonParserOption parser_option{}; - parser_option.file_paths.push_back(get_test_input_local_path()); - parser_option.archives_dir = cTestEndToEndArchiveDirectory; - parser_option.target_encoded_size = default_target_encoded_size; - parser_option.max_document_size = default_max_document_size; - parser_option.min_table_size = default_min_table_size; - parser_option.compression_level = default_compression_level; - parser_option.print_archive_stats = default_print_archive_stats; - parser_option.structurize_arrays = structurize_arrays; - - clp_s::JsonParser parser{parser_option}; - REQUIRE(parser.parse()); - parser.store(); - - REQUIRE(false == std::filesystem::is_empty(cTestEndToEndArchiveDirectory)); - - std::filesystem::create_directory(cTestEndToEndOutputDirectory); - REQUIRE(std::filesystem::is_directory(cTestEndToEndOutputDirectory)); - - clp_s::JsonConstructorOption constructor_option{}; - constructor_option.archives_dir = parser_option.archives_dir; - constructor_option.output_dir = cTestEndToEndOutputDirectory; - constructor_option.ordered = false; - constructor_option.target_ordered_chunk_size = 0; - for (auto const& entry : std::filesystem::directory_iterator(constructor_option.archives_dir)) { - if (false == entry.is_directory()) { - // Skip non-directories - continue; - } - - constructor_option.archive_id = entry.path().filename(); - clp_s::JsonConstructor constructor{constructor_option}; - constructor.store(); - } - - std::filesystem::path extracted_json_path{cTestEndToEndOutputDirectory}; - extracted_json_path /= "original"; - REQUIRE(std::filesystem::exists(extracted_json_path)); - - int result = std::system("command -v jq >/dev/null 2>&1"); - REQUIRE(0 == result); - std::string command = fmt::format( - "jq --sort-keys --compact-output '.' {}/original | sort > {}", - cTestEndToEndOutputDirectory, - cTestEndToEndOutputSortedJson - ); - result = std::system(command.c_str()); - REQUIRE(0 == result); - - REQUIRE(false == std::filesystem::is_empty(cTestEndToEndOutputSortedJson)); - - result = std::system("command -v diff >/dev/null 2>&1"); - REQUIRE(0 == result); - command = fmt::format( - "diff -u {} {} > /dev/null", - cTestEndToEndOutputSortedJson, - get_test_input_local_path() - ); - result = std::system(command.c_str()); - REQUIRE(0 == WEXITSTATUS(result)); -} diff --git a/components/core/tests/test_log_files/test_no_floats_sorted.json b/components/core/tests/test_log_files/test_no_floats_sorted.jsonl similarity index 100% rename from components/core/tests/test_log_files/test_no_floats_sorted.json rename to components/core/tests/test_log_files/test_no_floats_sorted.jsonl