forked from y-scope/clp
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
test(clp-s): Add end-to-end test case for compression and extraction. (…
- Loading branch information
1 parent
a1fd272
commit 95d4425
Showing
6 changed files
with
219 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,158 @@ | ||
| #include <sys/wait.h> | ||
|
|
||
| #include <cstdlib> | ||
| #include <filesystem> | ||
| #include <string> | ||
| #include <string_view> | ||
| #include <vector> | ||
|
|
||
| #include <Catch2/single_include/catch2/catch.hpp> | ||
| #include <fmt/format.h> | ||
|
|
||
| #include "../src/clp_s/JsonConstructor.hpp" | ||
| #include "../src/clp_s/JsonParser.hpp" | ||
|
|
||
| constexpr std::string_view cTestEndToEndArchiveDirectory{"test-end-to-end-archive"}; | ||
| constexpr std::string_view cTestEndToEndOutputDirectory{"test-end-to-end-out"}; | ||
| constexpr std::string_view cTestEndToEndOutputSortedJson{"test-end-to-end_sorted.jsonl"}; | ||
| constexpr std::string_view cTestEndToEndInputFileDirectory{"test_log_files"}; | ||
| constexpr std::string_view cTestEndToEndInputFile{"test_no_floats_sorted.jsonl"}; | ||
|
|
||
| namespace { | ||
| /** | ||
| * A class that deletes the directories and files created by test cases, both before and after each | ||
| * test case where the class is instantiated. | ||
| */ | ||
| class TestOutputCleaner { | ||
| public: | ||
| TestOutputCleaner() { delete_files(); } | ||
|
|
||
| ~TestOutputCleaner() { delete_files(); } | ||
|
|
||
| // Delete copy & move constructors and assignment operators | ||
| TestOutputCleaner(TestOutputCleaner const&) = delete; | ||
| TestOutputCleaner(TestOutputCleaner&&) = delete; | ||
| auto operator=(TestOutputCleaner const&) -> TestOutputCleaner& = delete; | ||
| auto operator=(TestOutputCleaner&&) -> TestOutputCleaner& = delete; | ||
|
|
||
| private: | ||
| static void delete_files() { | ||
| std::filesystem::remove_all(cTestEndToEndArchiveDirectory); | ||
| std::filesystem::remove_all(cTestEndToEndOutputDirectory); | ||
| std::filesystem::remove(cTestEndToEndOutputSortedJson); | ||
| } | ||
| }; | ||
|
|
||
| auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path; | ||
| auto get_test_input_local_path() -> std::string; | ||
| void compress(bool structurize_arrays); | ||
| auto extract() -> std::filesystem::path; | ||
| void compare(std::filesystem::path const& extracted_json_path); | ||
|
|
||
| auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path { | ||
| return std::filesystem::path{cTestEndToEndInputFileDirectory} / cTestEndToEndInputFile; | ||
| } | ||
|
|
||
| auto get_test_input_local_path() -> std::string { | ||
| std::filesystem::path const current_file_path{__FILE__}; | ||
| auto const tests_dir{current_file_path.parent_path()}; | ||
| return (tests_dir / get_test_input_path_relative_to_tests_dir()).string(); | ||
| } | ||
|
|
||
| void compress(bool structurize_arrays) { | ||
| constexpr auto cDefaultTargetEncodedSize = 8ULL * 1024 * 1024 * 1024; // 8 GiB | ||
| constexpr auto cDefaultMaxDocumentSize = 512ULL * 1024 * 1024; // 512 MiB | ||
| constexpr auto cDefaultMinTableSize = 1ULL * 1024 * 1024; // 1 MiB | ||
| constexpr auto cDefaultCompressionLevel = 3; | ||
| constexpr auto cDefaultPrintArchiveStats = false; | ||
|
|
||
| std::filesystem::create_directory(cTestEndToEndArchiveDirectory); | ||
| REQUIRE((std::filesystem::is_directory(cTestEndToEndArchiveDirectory))); | ||
|
|
||
| clp_s::JsonParserOption parser_option{}; | ||
| parser_option.file_paths.push_back(get_test_input_local_path()); | ||
| parser_option.archives_dir = cTestEndToEndArchiveDirectory; | ||
| parser_option.target_encoded_size = cDefaultTargetEncodedSize; | ||
| parser_option.max_document_size = cDefaultMaxDocumentSize; | ||
| parser_option.min_table_size = cDefaultMinTableSize; | ||
| parser_option.compression_level = cDefaultCompressionLevel; | ||
| parser_option.print_archive_stats = cDefaultPrintArchiveStats; | ||
| parser_option.structurize_arrays = structurize_arrays; | ||
|
|
||
| clp_s::JsonParser parser{parser_option}; | ||
| REQUIRE(parser.parse()); | ||
| parser.store(); | ||
|
|
||
| REQUIRE((false == std::filesystem::is_empty(cTestEndToEndArchiveDirectory))); | ||
| } | ||
|
|
||
| auto extract() -> std::filesystem::path { | ||
| constexpr auto cDefaultOrdered = false; | ||
| constexpr auto cDefaultTargetOrderedChunkSize = 0; | ||
|
|
||
| std::filesystem::create_directory(cTestEndToEndOutputDirectory); | ||
| REQUIRE(std::filesystem::is_directory(cTestEndToEndOutputDirectory)); | ||
|
|
||
| clp_s::JsonConstructorOption constructor_option{}; | ||
| constructor_option.archives_dir = cTestEndToEndArchiveDirectory; | ||
| constructor_option.output_dir = cTestEndToEndOutputDirectory; | ||
| constructor_option.ordered = cDefaultOrdered; | ||
| constructor_option.target_ordered_chunk_size = cDefaultTargetOrderedChunkSize; | ||
| for (auto const& entry : std::filesystem::directory_iterator(constructor_option.archives_dir)) { | ||
| if (false == entry.is_directory()) { | ||
| // Skip non-directories | ||
| continue; | ||
| } | ||
|
|
||
| constructor_option.archive_id = entry.path().filename(); | ||
| clp_s::JsonConstructor constructor{constructor_option}; | ||
| constructor.store(); | ||
| } | ||
| std::filesystem::path extracted_json_path{cTestEndToEndOutputDirectory}; | ||
| extracted_json_path /= "original"; | ||
| REQUIRE(std::filesystem::exists(extracted_json_path)); | ||
|
|
||
| return extracted_json_path; | ||
| } | ||
|
|
||
| // Silence the checks below since our use of `std::system` is safe in the context of testing. | ||
| // NOLINTBEGIN(cert-env33-c,concurrency-mt-unsafe) | ||
| void compare(std::filesystem::path const& extracted_json_path) { | ||
| int result{std::system("command -v jq >/dev/null 2>&1")}; | ||
| REQUIRE((0 == result)); | ||
| auto command = fmt::format( | ||
| "jq --sort-keys --compact-output '.' {} | sort > {}", | ||
| extracted_json_path.string(), | ||
| cTestEndToEndOutputSortedJson | ||
| ); | ||
| result = std::system(command.c_str()); | ||
| REQUIRE((0 == result)); | ||
|
|
||
| REQUIRE((false == std::filesystem::is_empty(cTestEndToEndOutputSortedJson))); | ||
|
|
||
| result = std::system("command -v diff >/dev/null 2>&1"); | ||
| REQUIRE((0 == result)); | ||
| command = fmt::format( | ||
| "diff --unified {} {} > /dev/null", | ||
| cTestEndToEndOutputSortedJson, | ||
| get_test_input_local_path() | ||
| ); | ||
| result = std::system(command.c_str()); | ||
| REQUIRE((true == WIFEXITED(result))); | ||
| REQUIRE((0 == WEXITSTATUS(result))); | ||
| } | ||
|
|
||
| // NOLINTEND(cert-env33-c,concurrency-mt-unsafe) | ||
| } // namespace | ||
|
|
||
| TEST_CASE("clp-s-compress-extract-no-floats", "[clp-s][end-to-end]") { | ||
| auto structurize_arrays = GENERATE(true, false); | ||
|
|
||
| TestOutputCleaner const test_cleanup; | ||
|
|
||
| compress(structurize_arrays); | ||
|
|
||
| auto extracted_json_path = extract(); | ||
|
|
||
| compare(extracted_json_path); | ||
| } |
4 changes: 4 additions & 0 deletions
4
components/core/tests/test_log_files/test_no_floats_sorted.jsonl
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| {"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_jq_losslessly_represents":9824299763229016,"int64_min_jq_losslessly_represents":-9007199254740992,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true} | ||
| {"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_jq_losslessly_represents":9824299763229016,"int64_min_jq_losslessly_represents":-9007199254740992,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_jq_losslessly_represents":9824299763229016,"int64_min_jq_losslessly_represents":-9007199254740992,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} | ||
| {"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_jq_losslessly_represents":9824299763229016,"int64_min_jq_losslessly_represents":-9007199254740992,"int8_max":127,"int8_min":-128,"nonempty_object":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_jq_losslessly_represents":9824299763229016,"int64_min_jq_losslessly_represents":-9007199254740992,"int8_max":127,"int8_min":-128,"non_empty_object2":{"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_array":[],"empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_jq_losslessly_represents":9824299763229016,"int64_min_jq_losslessly_represents":-9007199254740992,"int8_max":127,"int8_min":-128,"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true},"null":null,"string":"short_string","true":true} | ||
| {"clp_string":"uid=0, CPU usage:99.99%, \"user_name\"=YScope","empty_object":{},"false":false,"int16_max":32767,"int16_min":-32768,"int32_max":2147483647,"int32_min":-2147483648,"int64_max_jq_losslessly_represents":9824299763229016,"int64_min_jq_losslessly_represents":-9007199254740992,"int8_max":127,"int8_min":-128,"nonempty_array":[1,2,3,4,5],"null":null,"string":"short_string","true":true} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters