From e70020747d400c6443172af62c70ff3dd08d8698 Mon Sep 17 00:00:00 2001 From: Nebojsa Cvetkovic Date: Thu, 5 Dec 2024 00:47:54 +0000 Subject: [PATCH] BJData draft3 flag --- docs/mkdocs/docs/api/basic_json/to_bjdata.md | 12 +- .../docs/features/binary_formats/bjdata.md | 7 + .../nlohmann/detail/output/binary_writer.hpp | 16 +-- include/nlohmann/json.hpp | 13 +- single_include/nlohmann/json.hpp | 29 ++-- tests/src/unit-bjdata.cpp | 128 +++++++++++++----- 6 files changed, 145 insertions(+), 60 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json/to_bjdata.md b/docs/mkdocs/docs/api/basic_json/to_bjdata.md index 48598a5e61..a41a6b64ec 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bjdata.md +++ b/docs/mkdocs/docs/api/basic_json/to_bjdata.md @@ -4,13 +4,16 @@ // (1) static std::vector to_bjdata(const basic_json& j, const bool use_size = false, - const bool use_type = false); + const bool use_type = false, + const bool draft3_binary = false); // (2) static void to_bjdata(const basic_json& j, detail::output_adapter o, - const bool use_size = false, const bool use_type = false); + const bool use_size = false, const bool use_type = false, + const bool draft3_binary = false); static void to_bjdata(const basic_json& j, detail::output_adapter o, - const bool use_size = false, const bool use_type = false); + const bool use_size = false, const bool use_type = false, + const bool draft3_binary = false); ``` Serializes a given JSON value `j` to a byte vector using the BJData (Binary JData) serialization format. BJData aims to @@ -34,6 +37,9 @@ The exact mapping and its limitations is described on a [dedicated page](../../f `use_type` (in) : whether to add type annotations to container types (must be combined with `#!cpp use_size = true`); optional, + +`draft3_binary` (in) +: whether to use the draft 3 binary format (see [draft 3](../../features/binary_formats/bjdata.md#draft-3-binary-format)); optional, `#!cpp false` by default. ## Return value diff --git a/docs/mkdocs/docs/features/binary_formats/bjdata.md b/docs/mkdocs/docs/features/binary_formats/bjdata.md index c24290097f..0fd4962b78 100644 --- a/docs/mkdocs/docs/features/binary_formats/bjdata.md +++ b/docs/mkdocs/docs/features/binary_formats/bjdata.md @@ -139,6 +139,13 @@ The library uses the following mapping from JSON values types to BJData types ac in optimized arrays to designate binary data. This means that, unlike UBJSON, binary data can be both serialized and deserialized. + To preserve compatibility with BJData Draft 2, the Draft 3 optimized binary array must be explicitly enabled using + the `draft3_binary` parameter of [`to_bjdata`](../../api/basic_json/to_bjdata.md). + + In Draft2 mode (default), if the JSON data contains the binary type, the value stored is a list of integers, as + suggested by the BJData documentation. In particular, this means that the serialization and the deserialization of + JSON containing binary values into BJData and back will result in a different JSON object. + [BJDataBinArr]: https://github.com/NeuroJSON/bjdata/blob/master/Binary_JData_Specification.md#optimized-binary-array) ??? example diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index cf57585aab..e75283d122 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -738,7 +738,7 @@ class binary_writer */ void write_ubjson(const BasicJsonType& j, const bool use_count, const bool use_type, const bool add_prefix = true, - const bool use_bjdata = false) + const bool use_bjdata = false, const bool bjdata_draft3_binary = false) { switch (j.type()) { @@ -829,7 +829,7 @@ class binary_writer for (const auto& el : *j.m_data.m_value.array) { - write_ubjson(el, use_count, use_type, prefix_required, use_bjdata); + write_ubjson(el, use_count, use_type, prefix_required, use_bjdata, bjdata_draft3_binary); } if (!use_count) @@ -847,11 +847,11 @@ class binary_writer oa->write_character(to_char_type('[')); } - if (use_type && (use_bjdata || !j.m_data.m_value.binary->empty())) + if (use_type && ((use_bjdata && bjdata_draft3_binary) || !j.m_data.m_value.binary->empty())) { JSON_ASSERT(use_count); oa->write_character(to_char_type('$')); - oa->write_character(use_bjdata ? 'B' : 'U'); + oa->write_character(use_bjdata && bjdata_draft3_binary ? 'B' : 'U'); } if (use_count) @@ -887,7 +887,7 @@ class binary_writer { if (use_bjdata && j.m_data.m_value.object->size() == 3 && j.m_data.m_value.object->find("_ArrayType_") != j.m_data.m_value.object->end() && j.m_data.m_value.object->find("_ArraySize_") != j.m_data.m_value.object->end() && j.m_data.m_value.object->find("_ArrayData_") != j.m_data.m_value.object->end()) { - if (!write_bjdata_ndarray(*j.m_data.m_value.object, use_count, use_type)) // decode bjdata ndarray in the JData format (https://github.com/NeuroJSON/jdata) + if (!write_bjdata_ndarray(*j.m_data.m_value.object, use_count, use_type, bjdata_draft3_binary)) // decode bjdata ndarray in the JData format (https://github.com/NeuroJSON/jdata) { break; } @@ -931,7 +931,7 @@ class binary_writer oa->write_characters( reinterpret_cast(el.first.c_str()), el.first.size()); - write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata); + write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata, bjdata_draft3_binary); } if (!use_count) @@ -1615,7 +1615,7 @@ class binary_writer /*! @return false if the object is successfully converted to a bjdata ndarray, true if the type or size is invalid */ - bool write_bjdata_ndarray(const typename BasicJsonType::object_t& value, const bool use_count, const bool use_type) + bool write_bjdata_ndarray(const typename BasicJsonType::object_t& value, const bool use_count, const bool use_type, const bool bjdata_draft3_binary) { std::map bjdtype = {{"uint8", 'U'}, {"int8", 'i'}, {"uint16", 'u'}, {"int16", 'I'}, {"uint32", 'm'}, {"int32", 'l'}, {"uint64", 'M'}, {"int64", 'L'}, {"single", 'd'}, {"double", 'D'}, @@ -1649,7 +1649,7 @@ class binary_writer oa->write_character('#'); key = "_ArraySize_"; - write_ubjson(value.at(key), use_count, use_type, true, true); + write_ubjson(value.at(key), use_count, use_type, true, true, bjdata_draft3_binary); key = "_ArrayData_"; if (dtype == 'U' || dtype == 'C' || dtype == 'B') diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index e004e83082..8aaef8e279 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -4309,7 +4309,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/ static std::vector to_bjdata(const basic_json& j, const bool use_size = false, - const bool use_type = false) + const bool use_type = false, + const bool draft3_binary = false) { std::vector result; to_bjdata(j, result, use_size, use_type); @@ -4319,17 +4320,19 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief create a BJData serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/ static void to_bjdata(const basic_json& j, detail::output_adapter o, - const bool use_size = false, const bool use_type = false) + const bool use_size = false, const bool use_type = false, + const bool draft3_binary = false) { - binary_writer(o).write_ubjson(j, use_size, use_type, true, true); + binary_writer(o).write_ubjson(j, use_size, use_type, true, true, draft3_binary); } /// @brief create a BJData serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/ static void to_bjdata(const basic_json& j, detail::output_adapter o, - const bool use_size = false, const bool use_type = false) + const bool use_size = false, const bool use_type = false, + const bool draft3_binary = false) { - binary_writer(o).write_ubjson(j, use_size, use_type, true, true); + binary_writer(o).write_ubjson(j, use_size, use_type, true, true, draft3_binary); } /// @brief create a BSON serialization of a given JSON value diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 7eafd35663..3658963682 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -15992,7 +15992,7 @@ class binary_writer */ void write_ubjson(const BasicJsonType& j, const bool use_count, const bool use_type, const bool add_prefix = true, - const bool use_bjdata = false) + const bool use_bjdata = false, const bool bjdata_draft3_binary = false) { switch (j.type()) { @@ -16083,7 +16083,7 @@ class binary_writer for (const auto& el : *j.m_data.m_value.array) { - write_ubjson(el, use_count, use_type, prefix_required, use_bjdata); + write_ubjson(el, use_count, use_type, prefix_required, use_bjdata, bjdata_draft3_binary); } if (!use_count) @@ -16101,11 +16101,11 @@ class binary_writer oa->write_character(to_char_type('[')); } - if (use_type && (use_bjdata || !j.m_data.m_value.binary->empty())) + if (use_type && ((use_bjdata && bjdata_draft3_binary) || !j.m_data.m_value.binary->empty())) { JSON_ASSERT(use_count); oa->write_character(to_char_type('$')); - oa->write_character(use_bjdata ? 'B' : 'U'); + oa->write_character(use_bjdata && bjdata_draft3_binary ? 'B' : 'U'); } if (use_count) @@ -16141,7 +16141,7 @@ class binary_writer { if (use_bjdata && j.m_data.m_value.object->size() == 3 && j.m_data.m_value.object->find("_ArrayType_") != j.m_data.m_value.object->end() && j.m_data.m_value.object->find("_ArraySize_") != j.m_data.m_value.object->end() && j.m_data.m_value.object->find("_ArrayData_") != j.m_data.m_value.object->end()) { - if (!write_bjdata_ndarray(*j.m_data.m_value.object, use_count, use_type)) // decode bjdata ndarray in the JData format (https://github.com/NeuroJSON/jdata) + if (!write_bjdata_ndarray(*j.m_data.m_value.object, use_count, use_type, bjdata_draft3_binary)) // decode bjdata ndarray in the JData format (https://github.com/NeuroJSON/jdata) { break; } @@ -16185,7 +16185,7 @@ class binary_writer oa->write_characters( reinterpret_cast(el.first.c_str()), el.first.size()); - write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata); + write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata, bjdata_draft3_binary); } if (!use_count) @@ -16869,7 +16869,7 @@ class binary_writer /*! @return false if the object is successfully converted to a bjdata ndarray, true if the type or size is invalid */ - bool write_bjdata_ndarray(const typename BasicJsonType::object_t& value, const bool use_count, const bool use_type) + bool write_bjdata_ndarray(const typename BasicJsonType::object_t& value, const bool use_count, const bool use_type, const bool bjdata_draft3_binary) { std::map bjdtype = {{"uint8", 'U'}, {"int8", 'i'}, {"uint16", 'u'}, {"int16", 'I'}, {"uint32", 'm'}, {"int32", 'l'}, {"uint64", 'M'}, {"int64", 'L'}, {"single", 'd'}, {"double", 'D'}, @@ -16903,7 +16903,7 @@ class binary_writer oa->write_character('#'); key = "_ArraySize_"; - write_ubjson(value.at(key), use_count, use_type, true, true); + write_ubjson(value.at(key), use_count, use_type, true, true, bjdata_draft3_binary); key = "_ArrayData_"; if (dtype == 'U' || dtype == 'C' || dtype == 'B') @@ -23827,7 +23827,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/ static std::vector to_bjdata(const basic_json& j, const bool use_size = false, - const bool use_type = false) + const bool use_type = false, + const bool draft3_binary = false) { std::vector result; to_bjdata(j, result, use_size, use_type); @@ -23837,17 +23838,19 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec /// @brief create a BJData serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/ static void to_bjdata(const basic_json& j, detail::output_adapter o, - const bool use_size = false, const bool use_type = false) + const bool use_size = false, const bool use_type = false, + const bool draft3_binary = false) { - binary_writer(o).write_ubjson(j, use_size, use_type, true, true); + binary_writer(o).write_ubjson(j, use_size, use_type, true, true, draft3_binary); } /// @brief create a BJData serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/ static void to_bjdata(const basic_json& j, detail::output_adapter o, - const bool use_size = false, const bool use_type = false) + const bool use_size = false, const bool use_type = false, + const bool draft3_binary = false) { - binary_writer(o).write_ubjson(j, use_size, use_type, true, true); + binary_writer(o).write_ubjson(j, use_size, use_type, true, true, draft3_binary); } /// @brief create a BSON serialization of a given JSON value diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index 4671c32624..19042b5b4a 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -1529,6 +1529,11 @@ TEST_CASE("BJData") SECTION("binary") { + bool draft3; + SUBCASE("") { draft3 = false; } + SUBCASE("") { draft3 = true; } + CAPTURE(draft3); + SECTION("N = 0..127") { for (std::size_t N = 0; N <= 127; ++N) @@ -1542,8 +1547,11 @@ TEST_CASE("BJData") // create expected byte vector std::vector expected; expected.push_back(static_cast('[')); - expected.push_back(static_cast('$')); - expected.push_back(static_cast('B')); + if (draft3 || N != 0) + { + expected.push_back(static_cast('$')); + expected.push_back(static_cast(draft3 ? 'B' : 'U')); + } expected.push_back(static_cast('#')); expected.push_back(static_cast('i')); expected.push_back(static_cast(N)); @@ -1553,9 +1561,16 @@ TEST_CASE("BJData") } // compare result + size - const auto result = json::to_bjdata(j, true, true); + const auto result = json::to_bjdata(j, true, true, draft3); CHECK(result == expected); - CHECK(result.size() == N + 6); + if (!draft3 && N == 0) + { + CHECK(result.size() == N + 4); + } + else + { + CHECK(result.size() == N + 6); + } // check that no null byte is appended if (N > 0) @@ -1563,9 +1578,19 @@ TEST_CASE("BJData") CHECK(result.back() != '\x00'); } - // roundtrip - CHECK(json::from_bjdata(result) == j); - CHECK(json::from_bjdata(result, true, false) == j); + if (draft3) + { + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + else + { + // roundtrip only works to an array of numbers + json j_out = s; + CHECK(json::from_bjdata(result) == j_out); + CHECK(json::from_bjdata(result, true, false) == j_out); + } } } @@ -1583,7 +1608,7 @@ TEST_CASE("BJData") std::vector expected; expected.push_back(static_cast('[')); expected.push_back(static_cast('$')); - expected.push_back(static_cast('B')); + expected.push_back(static_cast(draft3 ? 'B' : 'U')); expected.push_back(static_cast('#')); expected.push_back(static_cast('U')); expected.push_back(static_cast(N)); @@ -1593,15 +1618,25 @@ TEST_CASE("BJData") } // compare result + size - const auto result = json::to_bjdata(j, true, true); + const auto result = json::to_bjdata(j, true, true, draft3); CHECK(result == expected); CHECK(result.size() == N + 6); // check that no null byte is appended CHECK(result.back() != '\x00'); - // roundtrip - CHECK(json::from_bjdata(result) == j); - CHECK(json::from_bjdata(result, true, false) == j); + if (draft3) + { + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + else + { + // roundtrip only works to an array of numbers + json j_out = s; + CHECK(json::from_bjdata(result) == j_out); + CHECK(json::from_bjdata(result, true, false) == j_out); + } } } @@ -1622,22 +1657,32 @@ TEST_CASE("BJData") std::vector expected(N + 7, 'x'); expected[0] = '['; expected[1] = '$'; - expected[2] = 'B'; + expected[2] = draft3 ? 'B' : 'U'; expected[3] = '#'; expected[4] = 'I'; expected[5] = static_cast(N & 0xFF); expected[6] = static_cast((N >> 8) & 0xFF); // compare result + size - const auto result = json::to_bjdata(j, true, true); + const auto result = json::to_bjdata(j, true, true, draft3); CHECK(result == expected); CHECK(result.size() == N + 7); // check that no null byte is appended CHECK(result.back() != '\x00'); - // roundtrip - CHECK(json::from_bjdata(result) == j); - CHECK(json::from_bjdata(result, true, false) == j); + if (draft3) + { + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + else + { + // roundtrip only works to an array of numbers + json j_out = s; + CHECK(json::from_bjdata(result) == j_out); + CHECK(json::from_bjdata(result, true, false) == j_out); + } } } @@ -1658,22 +1703,32 @@ TEST_CASE("BJData") std::vector expected(N + 7, 'x'); expected[0] = '['; expected[1] = '$'; - expected[2] = 'B'; + expected[2] = draft3 ? 'B' : 'U'; expected[3] = '#'; expected[4] = 'u'; expected[5] = static_cast(N & 0xFF); expected[6] = static_cast((N >> 8) & 0xFF); // compare result + size - const auto result = json::to_bjdata(j, true, true); + const auto result = json::to_bjdata(j, true, true, draft3); CHECK(result == expected); CHECK(result.size() == N + 7); // check that no null byte is appended CHECK(result.back() != '\x00'); - // roundtrip - CHECK(json::from_bjdata(result) == j); - CHECK(json::from_bjdata(result, true, false) == j); + if (draft3) + { + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + else + { + // roundtrip only works to an array of numbers + json j_out = s; + CHECK(json::from_bjdata(result) == j_out); + CHECK(json::from_bjdata(result, true, false) == j_out); + } } } @@ -1694,7 +1749,7 @@ TEST_CASE("BJData") std::vector expected(N + 9, 'x'); expected[0] = '['; expected[1] = '$'; - expected[2] = 'B'; + expected[2] = draft3 ? 'B' : 'U'; expected[3] = '#'; expected[4] = 'l'; expected[5] = static_cast(N & 0xFF); @@ -1703,15 +1758,25 @@ TEST_CASE("BJData") expected[8] = static_cast((N >> 24) & 0xFF); // compare result + size - const auto result = json::to_bjdata(j, true, true); + const auto result = json::to_bjdata(j, true, true, draft3); CHECK(result == expected); CHECK(result.size() == N + 9); // check that no null byte is appended CHECK(result.back() != '\x00'); - // roundtrip - CHECK(json::from_bjdata(result) == j); - CHECK(json::from_bjdata(result, true, false) == j); + if (draft3) + { + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + else + { + // roundtrip only works to an array of numbers + json j_out = s; + CHECK(json::from_bjdata(result) == j_out); + CHECK(json::from_bjdata(result, true, false) == j_out); + } } } @@ -1727,7 +1792,7 @@ TEST_CASE("BJData") expected.push_back(static_cast('[')); for (std::size_t i = 0; i < N; ++i) { - expected.push_back(static_cast('B')); + expected.push_back(static_cast(draft3 ? 'B' : 'U')); expected.push_back(static_cast(0x78)); } expected.push_back(static_cast(']')); @@ -1755,7 +1820,7 @@ TEST_CASE("BJData") for (size_t i = 0; i < N; ++i) { - expected.push_back(static_cast('B')); + expected.push_back(static_cast(draft3 ? 'B' : 'U')); expected.push_back(static_cast(0x78)); } @@ -2428,7 +2493,8 @@ TEST_CASE("BJData") CHECK(json::to_bjdata(json::from_bjdata(v_D), true, true) == v_D); CHECK(json::to_bjdata(json::from_bjdata(v_S), true, true) == v_S); CHECK(json::to_bjdata(json::from_bjdata(v_C), true, true) == v_S); // char is serialized to string - CHECK(json::to_bjdata(json::from_bjdata(v_B), true, true) == v_B); + CHECK(json::to_bjdata(json::from_bjdata(v_B), true, true, false) == v_U); // binary is serialized to uint8 (draft2) + CHECK(json::to_bjdata(json::from_bjdata(v_B), true, true, true) == v_B); } SECTION("optimized ndarray (type and vector-size as optimized 1D array)") @@ -3255,7 +3321,7 @@ TEST_CASE("Universal Binary JSON Specification Examples 1") }); json const j = {{"binary", json::binary(s)}, {"val", 123}}; std::vector const v = {'{', 'i', 6, 'b', 'i', 'n', 'a', 'r', 'y', '[', '$', 'B', '#', 'i', 4, 222, 173, 190, 239, 'i', 3, 'v', 'a', 'l', 'i', 123, '}'}; - //CHECK(json::to_bjdata(j) == v); + //CHECK(json::to_bjdata(j) == v); // 123 value gets encoded as uint8 CHECK(json::from_bjdata(v) == j); }