From ae00914a50c5a0d4721aaf5011c0d93488452522 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 17 Jul 2022 14:26:25 +0200 Subject: [PATCH 1/6] :children_crossing: add error message if test suite cannot be found Fixes #3584 --- README.md | 16 ++++++++++++++++ tests/src/make_test_data_available.hpp | 26 ++++++++++++++++++++++++++ tests/src/unit-binary_formats.cpp | 2 +- tests/src/unit-bjdata.cpp | 2 +- tests/src/unit-bson.cpp | 2 +- tests/src/unit-cbor.cpp | 2 +- tests/src/unit-inspection.cpp | 2 +- tests/src/unit-json_patch.cpp | 2 +- tests/src/unit-msgpack.cpp | 2 +- tests/src/unit-regression1.cpp | 2 +- tests/src/unit-testsuites.cpp | 3 ++- tests/src/unit-ubjson.cpp | 2 +- tests/src/unit-unicode1.cpp | 2 +- tests/src/unit-unicode2.cpp | 2 +- tests/src/unit-unicode3.cpp | 2 +- tests/src/unit-unicode4.cpp | 2 +- tests/src/unit-unicode5.cpp | 2 +- 17 files changed, 58 insertions(+), 15 deletions(-) create mode 100644 tests/src/make_test_data_available.hpp diff --git a/README.md b/README.md index 220b3a9f15..4a99a49320 100644 --- a/README.md +++ b/README.md @@ -1727,6 +1727,22 @@ $ ctest --output-on-failure Note that during the `ctest` stage, several JSON test files are downloaded from an [external repository](https://github.com/nlohmann/json_test_data). If policies forbid downloading artifacts during testing, you can download the files yourself and pass the directory with the test files via `-DJSON_TestDataDirectory=path` to CMake. Then, no Internet connectivity is required. See [issue #2189](https://github.com/nlohmann/json/issues/2189) for more information. +If the test suite is not found, several test suites will fail like this: + +``` +=============================================================================== +json/tests/src/make_test_data_available.hpp:21: +TEST CASE: check test suite is downloaded + +json/tests/src/make_test_data_available.hpp:23: FATAL ERROR: REQUIRE( utils::check_testsuite_downloaded() ) is NOT correct! + values: REQUIRE( false ) + logged: Test data not found in 'json/cmake-build-debug/json_test_data'. + Please execute target 'download_test_data' before running this test suite. + See for more information. + +=============================================================================== +``` + In case you have downloaded the library rather than checked out the code via Git, test `cmake_fetch_content_configure` will fail. Please execute `ctest -LE git_required` to skip these tests. See [issue #2189](https://github.com/nlohmann/json/issues/2189) for more information. Some tests change the installed files and hence make the whole process not reproducible. Please execute `ctest -LE not_reproducible` to skip these tests. See [issue #2324](https://github.com/nlohmann/json/issues/2324) for more information. diff --git a/tests/src/make_test_data_available.hpp b/tests/src/make_test_data_available.hpp new file mode 100644 index 0000000000..dc01fce5cd --- /dev/null +++ b/tests/src/make_test_data_available.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include // fopen, fclose, FILE +#include +#include + +namespace utils +{ + +inline bool check_testsuite_downloaded() +{ + std::FILE* file = std::fopen(TEST_DATA_DIRECTORY "/README.md", "r"); + if (!file) + { + return false; + } + std::fclose(file); + return true; +} + +TEST_CASE("check test suite is downloaded") +{ + REQUIRE_MESSAGE(utils::check_testsuite_downloaded(), "Test data not found in '" TEST_DATA_DIRECTORY "'. Please execute target 'download_test_data' before running this test suite. See for more information."); +} + +} // namespace utils diff --git a/tests/src/unit-binary_formats.cpp b/tests/src/unit-binary_formats.cpp index 05e889b0fc..f920c2adf3 100644 --- a/tests/src/unit-binary_formats.cpp +++ b/tests/src/unit-binary_formats.cpp @@ -33,7 +33,7 @@ SOFTWARE. using nlohmann::json; #include -#include +#include "make_test_data_available.hpp" TEST_CASE("Binary Formats" * doctest::skip()) { diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index 7442ea5885..ccf5ca590a 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -37,7 +37,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" #include "test_utils.hpp" namespace diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index 1920d31550..9f6b3f589b 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -34,7 +34,7 @@ using nlohmann::json; #include #include -#include +#include "make_test_data_available.hpp" #include "test_utils.hpp" TEST_CASE("BSON") diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 83c8ada54f..f233d493a0 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -37,7 +37,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" #include "test_utils.hpp" namespace diff --git a/tests/src/unit-inspection.cpp b/tests/src/unit-inspection.cpp index 49d2431d71..219d0f466d 100644 --- a/tests/src/unit-inspection.cpp +++ b/tests/src/unit-inspection.cpp @@ -34,7 +34,7 @@ using nlohmann::json; #include #include -#include +#include "make_test_data_available.hpp" TEST_CASE("object inspection") { diff --git a/tests/src/unit-json_patch.cpp b/tests/src/unit-json_patch.cpp index 4ece3a5059..cf411963d4 100644 --- a/tests/src/unit-json_patch.cpp +++ b/tests/src/unit-json_patch.cpp @@ -33,7 +33,7 @@ SOFTWARE. using nlohmann::json; #include -#include +#include "make_test_data_available.hpp" TEST_CASE("JSON patch") { diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 483f8ed462..8721998c6d 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -36,7 +36,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" #include "test_utils.hpp" namespace diff --git a/tests/src/unit-regression1.cpp b/tests/src/unit-regression1.cpp index 5a6e31e738..dc32b3b66c 100644 --- a/tests/src/unit-regression1.cpp +++ b/tests/src/unit-regression1.cpp @@ -40,7 +40,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" #ifdef JSON_HAS_CPP_17 #include diff --git a/tests/src/unit-testsuites.cpp b/tests/src/unit-testsuites.cpp index 7fbf831d94..cc163b9081 100644 --- a/tests/src/unit-testsuites.cpp +++ b/tests/src/unit-testsuites.cpp @@ -32,8 +32,9 @@ SOFTWARE. #include using nlohmann::json; +#include "make_test_data_available.hpp" #include -#include +#include "make_test_data_available.hpp" TEST_CASE("compliance tests from json.org") { diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index d60edf6655..c181912ee3 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -35,7 +35,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" #include "test_utils.hpp" namespace diff --git a/tests/src/unit-unicode1.cpp b/tests/src/unit-unicode1.cpp index a988ba8907..1fd4174664 100644 --- a/tests/src/unit-unicode1.cpp +++ b/tests/src/unit-unicode1.cpp @@ -37,7 +37,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" TEST_CASE("Unicode (1/5)" * doctest::skip()) { diff --git a/tests/src/unit-unicode2.cpp b/tests/src/unit-unicode2.cpp index 8438f176fe..ad994e99fd 100644 --- a/tests/src/unit-unicode2.cpp +++ b/tests/src/unit-unicode2.cpp @@ -39,7 +39,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" // this test suite uses static variables with non-trivial destructors DOCTEST_CLANG_SUPPRESS_WARNING_PUSH diff --git a/tests/src/unit-unicode3.cpp b/tests/src/unit-unicode3.cpp index 5bc1afe317..e5f8fec045 100644 --- a/tests/src/unit-unicode3.cpp +++ b/tests/src/unit-unicode3.cpp @@ -39,7 +39,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" // this test suite uses static variables with non-trivial destructors DOCTEST_CLANG_SUPPRESS_WARNING_PUSH diff --git a/tests/src/unit-unicode4.cpp b/tests/src/unit-unicode4.cpp index d51880dae4..38ff021239 100644 --- a/tests/src/unit-unicode4.cpp +++ b/tests/src/unit-unicode4.cpp @@ -39,7 +39,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" // this test suite uses static variables with non-trivial destructors DOCTEST_CLANG_SUPPRESS_WARNING_PUSH diff --git a/tests/src/unit-unicode5.cpp b/tests/src/unit-unicode5.cpp index b63ad16009..2175dafe7c 100644 --- a/tests/src/unit-unicode5.cpp +++ b/tests/src/unit-unicode5.cpp @@ -39,7 +39,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" // this test suite uses static variables with non-trivial destructors DOCTEST_CLANG_SUPPRESS_WARNING_PUSH From 9b6dbc5b9eb38dcdec027623a2653b9539594011 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 17 Jul 2022 14:26:25 +0200 Subject: [PATCH 2/6] :children_crossing: add error message if test suite cannot be found Fixes #3584 --- README.md | 16 ++++++++++++++++ tests/src/make_test_data_available.hpp | 26 ++++++++++++++++++++++++++ tests/src/unit-binary_formats.cpp | 2 +- tests/src/unit-bjdata.cpp | 2 +- tests/src/unit-bson.cpp | 2 +- tests/src/unit-cbor.cpp | 2 +- tests/src/unit-inspection.cpp | 2 +- tests/src/unit-json_patch.cpp | 2 +- tests/src/unit-msgpack.cpp | 2 +- tests/src/unit-regression1.cpp | 2 +- tests/src/unit-testsuites.cpp | 2 +- tests/src/unit-ubjson.cpp | 2 +- tests/src/unit-unicode1.cpp | 2 +- tests/src/unit-unicode2.cpp | 2 +- tests/src/unit-unicode3.cpp | 2 +- tests/src/unit-unicode4.cpp | 2 +- tests/src/unit-unicode5.cpp | 2 +- 17 files changed, 57 insertions(+), 15 deletions(-) create mode 100644 tests/src/make_test_data_available.hpp diff --git a/README.md b/README.md index 220b3a9f15..4a99a49320 100644 --- a/README.md +++ b/README.md @@ -1727,6 +1727,22 @@ $ ctest --output-on-failure Note that during the `ctest` stage, several JSON test files are downloaded from an [external repository](https://github.com/nlohmann/json_test_data). If policies forbid downloading artifacts during testing, you can download the files yourself and pass the directory with the test files via `-DJSON_TestDataDirectory=path` to CMake. Then, no Internet connectivity is required. See [issue #2189](https://github.com/nlohmann/json/issues/2189) for more information. +If the test suite is not found, several test suites will fail like this: + +``` +=============================================================================== +json/tests/src/make_test_data_available.hpp:21: +TEST CASE: check test suite is downloaded + +json/tests/src/make_test_data_available.hpp:23: FATAL ERROR: REQUIRE( utils::check_testsuite_downloaded() ) is NOT correct! + values: REQUIRE( false ) + logged: Test data not found in 'json/cmake-build-debug/json_test_data'. + Please execute target 'download_test_data' before running this test suite. + See for more information. + +=============================================================================== +``` + In case you have downloaded the library rather than checked out the code via Git, test `cmake_fetch_content_configure` will fail. Please execute `ctest -LE git_required` to skip these tests. See [issue #2189](https://github.com/nlohmann/json/issues/2189) for more information. Some tests change the installed files and hence make the whole process not reproducible. Please execute `ctest -LE not_reproducible` to skip these tests. See [issue #2324](https://github.com/nlohmann/json/issues/2324) for more information. diff --git a/tests/src/make_test_data_available.hpp b/tests/src/make_test_data_available.hpp new file mode 100644 index 0000000000..dc01fce5cd --- /dev/null +++ b/tests/src/make_test_data_available.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include // fopen, fclose, FILE +#include +#include + +namespace utils +{ + +inline bool check_testsuite_downloaded() +{ + std::FILE* file = std::fopen(TEST_DATA_DIRECTORY "/README.md", "r"); + if (!file) + { + return false; + } + std::fclose(file); + return true; +} + +TEST_CASE("check test suite is downloaded") +{ + REQUIRE_MESSAGE(utils::check_testsuite_downloaded(), "Test data not found in '" TEST_DATA_DIRECTORY "'. Please execute target 'download_test_data' before running this test suite. See for more information."); +} + +} // namespace utils diff --git a/tests/src/unit-binary_formats.cpp b/tests/src/unit-binary_formats.cpp index 05e889b0fc..f920c2adf3 100644 --- a/tests/src/unit-binary_formats.cpp +++ b/tests/src/unit-binary_formats.cpp @@ -33,7 +33,7 @@ SOFTWARE. using nlohmann::json; #include -#include +#include "make_test_data_available.hpp" TEST_CASE("Binary Formats" * doctest::skip()) { diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index 7442ea5885..ccf5ca590a 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -37,7 +37,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" #include "test_utils.hpp" namespace diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index 1920d31550..9f6b3f589b 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -34,7 +34,7 @@ using nlohmann::json; #include #include -#include +#include "make_test_data_available.hpp" #include "test_utils.hpp" TEST_CASE("BSON") diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 83c8ada54f..f233d493a0 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -37,7 +37,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" #include "test_utils.hpp" namespace diff --git a/tests/src/unit-inspection.cpp b/tests/src/unit-inspection.cpp index 49d2431d71..219d0f466d 100644 --- a/tests/src/unit-inspection.cpp +++ b/tests/src/unit-inspection.cpp @@ -34,7 +34,7 @@ using nlohmann::json; #include #include -#include +#include "make_test_data_available.hpp" TEST_CASE("object inspection") { diff --git a/tests/src/unit-json_patch.cpp b/tests/src/unit-json_patch.cpp index 4ece3a5059..cf411963d4 100644 --- a/tests/src/unit-json_patch.cpp +++ b/tests/src/unit-json_patch.cpp @@ -33,7 +33,7 @@ SOFTWARE. using nlohmann::json; #include -#include +#include "make_test_data_available.hpp" TEST_CASE("JSON patch") { diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 483f8ed462..8721998c6d 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -36,7 +36,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" #include "test_utils.hpp" namespace diff --git a/tests/src/unit-regression1.cpp b/tests/src/unit-regression1.cpp index 5a6e31e738..dc32b3b66c 100644 --- a/tests/src/unit-regression1.cpp +++ b/tests/src/unit-regression1.cpp @@ -40,7 +40,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" #ifdef JSON_HAS_CPP_17 #include diff --git a/tests/src/unit-testsuites.cpp b/tests/src/unit-testsuites.cpp index 7fbf831d94..d02f77029c 100644 --- a/tests/src/unit-testsuites.cpp +++ b/tests/src/unit-testsuites.cpp @@ -33,7 +33,7 @@ SOFTWARE. using nlohmann::json; #include -#include +#include "make_test_data_available.hpp" TEST_CASE("compliance tests from json.org") { diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index d60edf6655..c181912ee3 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -35,7 +35,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" #include "test_utils.hpp" namespace diff --git a/tests/src/unit-unicode1.cpp b/tests/src/unit-unicode1.cpp index a988ba8907..1fd4174664 100644 --- a/tests/src/unit-unicode1.cpp +++ b/tests/src/unit-unicode1.cpp @@ -37,7 +37,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" TEST_CASE("Unicode (1/5)" * doctest::skip()) { diff --git a/tests/src/unit-unicode2.cpp b/tests/src/unit-unicode2.cpp index 8438f176fe..ad994e99fd 100644 --- a/tests/src/unit-unicode2.cpp +++ b/tests/src/unit-unicode2.cpp @@ -39,7 +39,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" // this test suite uses static variables with non-trivial destructors DOCTEST_CLANG_SUPPRESS_WARNING_PUSH diff --git a/tests/src/unit-unicode3.cpp b/tests/src/unit-unicode3.cpp index 5bc1afe317..e5f8fec045 100644 --- a/tests/src/unit-unicode3.cpp +++ b/tests/src/unit-unicode3.cpp @@ -39,7 +39,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" // this test suite uses static variables with non-trivial destructors DOCTEST_CLANG_SUPPRESS_WARNING_PUSH diff --git a/tests/src/unit-unicode4.cpp b/tests/src/unit-unicode4.cpp index d51880dae4..38ff021239 100644 --- a/tests/src/unit-unicode4.cpp +++ b/tests/src/unit-unicode4.cpp @@ -39,7 +39,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" // this test suite uses static variables with non-trivial destructors DOCTEST_CLANG_SUPPRESS_WARNING_PUSH diff --git a/tests/src/unit-unicode5.cpp b/tests/src/unit-unicode5.cpp index b63ad16009..2175dafe7c 100644 --- a/tests/src/unit-unicode5.cpp +++ b/tests/src/unit-unicode5.cpp @@ -39,7 +39,7 @@ using nlohmann::json; #include #include #include -#include +#include "make_test_data_available.hpp" // this test suite uses static variables with non-trivial destructors DOCTEST_CLANG_SUPPRESS_WARNING_PUSH From bb50ac61bb44a9a80eec2ab6f239a84e01b8b9c2 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 17 Jul 2022 15:13:47 +0200 Subject: [PATCH 3/6] :children_crossing: add error message if test suite cannot be found Fixes #3584 --- tests/src/unit-testsuites.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/src/unit-testsuites.cpp b/tests/src/unit-testsuites.cpp index cc163b9081..d02f77029c 100644 --- a/tests/src/unit-testsuites.cpp +++ b/tests/src/unit-testsuites.cpp @@ -32,7 +32,6 @@ SOFTWARE. #include using nlohmann::json; -#include "make_test_data_available.hpp" #include #include "make_test_data_available.hpp" From 0841d58ab69f8974c1c9e1b2ed6e6621e419170b Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 18 Jul 2022 17:54:12 +0200 Subject: [PATCH 4/6] :rotating_light: fix warning --- tests/src/make_test_data_available.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/src/make_test_data_available.hpp b/tests/src/make_test_data_available.hpp index dc01fce5cd..0215256efb 100644 --- a/tests/src/make_test_data_available.hpp +++ b/tests/src/make_test_data_available.hpp @@ -1,6 +1,7 @@ #pragma once #include // fopen, fclose, FILE +#include // unique_ptr #include #include @@ -9,12 +10,11 @@ namespace utils inline bool check_testsuite_downloaded() { - std::FILE* file = std::fopen(TEST_DATA_DIRECTORY "/README.md", "r"); - if (!file) + std::unique_ptr file(std::fopen(TEST_DATA_DIRECTORY "/README.md", "r"), &std::fclose); + if (file) { return false; } - std::fclose(file); return true; } From 2f80a8ee34423ecbf34d64b0d37ed5f5c3829584 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 18 Jul 2022 22:50:34 +0200 Subject: [PATCH 5/6] :bug: fix logic --- tests/src/make_test_data_available.hpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/src/make_test_data_available.hpp b/tests/src/make_test_data_available.hpp index 0215256efb..fca1c8da8b 100644 --- a/tests/src/make_test_data_available.hpp +++ b/tests/src/make_test_data_available.hpp @@ -11,11 +11,7 @@ namespace utils inline bool check_testsuite_downloaded() { std::unique_ptr file(std::fopen(TEST_DATA_DIRECTORY "/README.md", "r"), &std::fclose); - if (file) - { - return false; - } - return true; + return file.operator bool(); } TEST_CASE("check test suite is downloaded") From 5c2521aa516c29e37fc2e49e6d419f2d88748ab5 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 19 Jul 2022 12:14:07 +0200 Subject: [PATCH 6/6] :ok_hand: apply review comment --- tests/src/make_test_data_available.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/make_test_data_available.hpp b/tests/src/make_test_data_available.hpp index fca1c8da8b..740c839278 100644 --- a/tests/src/make_test_data_available.hpp +++ b/tests/src/make_test_data_available.hpp @@ -11,7 +11,7 @@ namespace utils inline bool check_testsuite_downloaded() { std::unique_ptr file(std::fopen(TEST_DATA_DIRECTORY "/README.md", "r"), &std::fclose); - return file.operator bool(); + return file != nullptr; } TEST_CASE("check test suite is downloaded")