From 65ab41cf60c5f791514bc7ddca6dd4c53a1359fa Mon Sep 17 00:00:00 2001 From: Jerry Johns Date: Mon, 14 Mar 2022 06:34:49 -0700 Subject: [PATCH] TLV to JSON converter (#16086) This adds a TLV to JSON converter to provide a means to deal with TLV in a more human-readable format. This does not rely on any schema information and instead, just dumps out raw TLV present in cluster payloads into a JSON format. This relies on the jsoncpp third party library. Testing Added a unit-test. --- src/lib/support/jsontlv/BUILD.gn | 24 +++ src/lib/support/jsontlv/TlvJson.cpp | 242 +++++++++++++++++++++ src/lib/support/jsontlv/TlvJson.h | 38 ++++ src/lib/support/tests/BUILD.gn | 2 + src/lib/support/tests/TestTlvToJson.cpp | 272 ++++++++++++++++++++++++ 5 files changed, 578 insertions(+) create mode 100644 src/lib/support/jsontlv/BUILD.gn create mode 100644 src/lib/support/jsontlv/TlvJson.cpp create mode 100644 src/lib/support/jsontlv/TlvJson.h create mode 100644 src/lib/support/tests/TestTlvToJson.cpp diff --git a/src/lib/support/jsontlv/BUILD.gn b/src/lib/support/jsontlv/BUILD.gn new file mode 100644 index 00000000000000..97c92879050d3b --- /dev/null +++ b/src/lib/support/jsontlv/BUILD.gn @@ -0,0 +1,24 @@ +# Copyright (c) 2020 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("//build_overrides/chip.gni") + +static_library("jsontlv") { + public_deps = [ + "${chip_root}/src/lib/core", + "${chip_root}/third_party/jsoncpp", + ] + + sources = [ "TlvJson.cpp" ] +} diff --git a/src/lib/support/jsontlv/TlvJson.cpp b/src/lib/support/jsontlv/TlvJson.cpp new file mode 100644 index 00000000000000..0cc84c33d7d80b --- /dev/null +++ b/src/lib/support/jsontlv/TlvJson.cpp @@ -0,0 +1,242 @@ +/* + * + * Copyright (c) 2020 Project CHIP Authors + * Copyright (c) 2013-2017 Nest Labs, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "lib/support/CHIPMemString.h" +#include "lib/support/ScopedBuffer.h" +#include +#include +#include + +namespace { +/* + * Encapsulates the different types of keys permissible. + * + * Root Key = Key with a name of 'value'. This is the top-most key in a given JSON object generated from TLV. + * Struct Field = Key containing the 32-bit field ID of an item in a struct. + * Array Item = Key containing the 16-bit list index of an item in a list. + * + * In the latter two modes, the actual field ID/list index is encapsulated within the 'key' member. + * + */ +struct KeyContext +{ + enum KeyType + { + kRoot, + kStructField, + kArrayItem + }; + + KeyContext() = default; + + KeyContext(chip::FieldId fieldId) + { + keyType = kStructField; + key = fieldId; + } + + KeyContext(chip::ListIndex listIndex) + { + keyType = kArrayItem; + key = listIndex; + } + + KeyType keyType = kRoot; + unsigned int key = 0; +}; +} // namespace + +// +// For now, let's put a bound of the maximum length of a byte/char string to be the size of an IPv6 +// MTU. While this is smaller than that of the limit defined in the data model specification, +// strings by virtue of not being chunked are intrinsically limited in size to the size of the encompassing packet. +// +static constexpr uint16_t kMaxStringLen = 1280; + +namespace chip { + +/* + * This templated function inserts a key/value pair into the Json value object. + * The value is templated to be of type T and accepts any of the following primitive + * types: + * bool, uint*_t, int*_t, char *, float, double. + * + * This method uses the provided key context to deduce the type of element being added. + * + */ +template +void InsertKeyValue(Json::Value & json, const KeyContext & keyContext, T val) +{ + // + // This needs to accomodate either the string 'value', or a 32-bit integer. + // The size of the largest 32-bit integer key represented as a string is 11 characters long. + // Tack on 1 byte for the null character. + // + char keyBuf[12]; + + if (keyContext.keyType == KeyContext::kRoot) + { + Platform::CopyString(keyBuf, sizeof(keyBuf), "value"); + json[keyBuf] = val; + } + else if (keyContext.keyType == KeyContext::kStructField) + { + snprintf(keyBuf, sizeof(keyBuf), "%" PRIu32, keyContext.key); + json[keyBuf] = val; + } + else + { + json[keyContext.key] = val; + } +} + +std::string JsonToString(Json::Value & json) +{ + Json::StyledWriter writer; + return writer.write(json); +} + +CHIP_ERROR TlvToJson(TLV::TLVReader & reader, KeyContext context, Json::Value & parent) +{ + bool isStruct = false; + + switch (reader.GetType()) + { + case TLV::kTLVType_UnsignedInteger: { + uint64_t v; + ReturnErrorOnFailure(reader.Get(v)); + InsertKeyValue(parent, context, v); + break; + } + + case TLV::kTLVType_SignedInteger: { + int64_t v; + ReturnErrorOnFailure(reader.Get(v)); + InsertKeyValue(parent, context, v); + break; + } + + case TLV::kTLVType_Boolean: { + bool v; + ReturnErrorOnFailure(reader.Get(v)); + InsertKeyValue(parent, context, v); + break; + } + + case TLV::kTLVType_FloatingPointNumber: { + double v; + ReturnErrorOnFailure(reader.Get(v)); + InsertKeyValue(parent, context, v); + break; + } + + case TLV::kTLVType_ByteString: { + ByteSpan span; + + ReturnErrorOnFailure(reader.Get(span)); + VerifyOrReturnError(span.size() < kMaxStringLen, CHIP_ERROR_INVALID_TLV_ELEMENT); + + Platform::ScopedMemoryBuffer byteString; + byteString.Alloc(BASE64_ENCODED_LEN(span.size()) + 1); + VerifyOrReturnError(byteString.Get() != nullptr, CHIP_ERROR_NO_MEMORY); + + auto encodedLen = Base64Encode(span.data(), span.size(), byteString.Get()); + byteString.Get()[encodedLen] = '\0'; + + InsertKeyValue(parent, context, byteString.Get()); + break; + } + + case TLV::kTLVType_UTF8String: { + CharSpan span; + + ReturnErrorOnFailure(reader.Get(span)); + VerifyOrReturnError(span.size() < kMaxStringLen, CHIP_ERROR_INVALID_TLV_ELEMENT); + + Platform::ScopedMemoryString charString(span.data(), span.size()); + InsertKeyValue(parent, context, charString.Get()); + break; + } + + case TLV::kTLVType_Null: { + InsertKeyValue(parent, context, Json::Value()); + break; + } + + case TLV::kTLVType_Structure: + isStruct = true; + + // + // Fall-through to the case below since + // arrays and structs are handled similarly with + // just a small difference in terms of handling of field IDs vs. + // list indices of the elements in the respective collections. + // + + case TLV::kTLVType_Array: { + TLV::TLVType containerType; + + ReturnErrorOnFailure(reader.EnterContainer(containerType)); + + CHIP_ERROR err; + Json::Value value; + size_t listIndex = 0; + + while ((err = reader.Next()) == CHIP_NO_ERROR) + { + if (isStruct) + { + VerifyOrReturnError(TLV::IsContextTag(reader.GetTag()), CHIP_ERROR_INVALID_TLV_TAG); + KeyContext context2(static_cast(TLV::TagNumFromTag(reader.GetTag()))); + + // + // Recursively convert to JSON the encompassing item within the struct. + // + ReturnErrorOnFailure(TlvToJson(reader, context2, value)); + } + else + { + KeyContext context2(static_cast(listIndex++)); + + // + // Recursively convert to JSON the encompassing item within the array. + // + ReturnErrorOnFailure(TlvToJson(reader, context2, value)); + } + } + + VerifyOrReturnError(err == CHIP_END_OF_TLV, err); + ReturnErrorOnFailure(reader.ExitContainer(containerType)); + InsertKeyValue(parent, context, value); + break; + } + + default: + return CHIP_ERROR_INVALID_TLV_ELEMENT; + break; + } + + return CHIP_NO_ERROR; +} + +CHIP_ERROR TlvToJson(TLV::TLVReader & reader, Json::Value & root) +{ + KeyContext context; + return TlvToJson(reader, context, root); +} + +} // namespace chip diff --git a/src/lib/support/jsontlv/TlvJson.h b/src/lib/support/jsontlv/TlvJson.h new file mode 100644 index 00000000000000..85e2ce259dbce2 --- /dev/null +++ b/src/lib/support/jsontlv/TlvJson.h @@ -0,0 +1,38 @@ +/* + * + * Copyright (c) 2020 Project CHIP Authors + * Copyright (c) 2013-2017 Nest Labs, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +namespace chip { + +/* + * Given a TLVReader positioned at a particular cluster data payload, this function converts + * the TLV data into a JSON object representation. + * + * NOTE: This only accepts data model payloads for events/commands/attributes. It does not support + * arbitrary TLV conversion to JSON. + */ +CHIP_ERROR TlvToJson(TLV::TLVReader & reader, Json::Value & root); + +/* + * Converts a JSON object into string representation + */ +std::string JsonToString(Json::Value & json); + +} // namespace chip diff --git a/src/lib/support/tests/BUILD.gn b/src/lib/support/tests/BUILD.gn index e43146452192a1..a5d22ed00e064e 100644 --- a/src/lib/support/tests/BUILD.gn +++ b/src/lib/support/tests/BUILD.gn @@ -46,6 +46,7 @@ chip_test_suite("tests") { "TestStringBuilder.cpp", "TestThreadOperationalDataset.cpp", "TestTimeUtils.cpp", + "TestTlvToJson.cpp", "TestVariant.cpp", "TestZclString.cpp", ] @@ -68,6 +69,7 @@ chip_test_suite("tests") { public_deps = [ "${chip_root}/src/lib/core", + "${chip_root}/src/lib/support/jsontlv", "${chip_root}/src/platform", "${nlunit_test_root}:nlunit-test", ] diff --git a/src/lib/support/tests/TestTlvToJson.cpp b/src/lib/support/tests/TestTlvToJson.cpp new file mode 100644 index 00000000000000..6636e417c73d81 --- /dev/null +++ b/src/lib/support/tests/TestTlvToJson.cpp @@ -0,0 +1,272 @@ +/* + * + * Copyright (c) 2021 Project CHIP Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace chip::Encoding; +using namespace chip; +using namespace chip::app; + +System::TLVPacketBufferBackingStore gStore; +TLV::TLVWriter gWriter; +TLV::TLVReader gReader; +nlTestSuite * gSuite; + +void SetupBuf() +{ + System::PacketBufferHandle buf; + + buf = System::PacketBufferHandle::New(1024); + gStore.Init(std::move(buf)); + + gWriter.Init(gStore); + gReader.Init(gStore); +} + +CHIP_ERROR SetupReader() +{ + gReader.Init(gStore); + return gReader.Next(); +} + +bool Matches(const char * referenceString, Json::Value & generatedValue) +{ + Json::StyledWriter writer; + auto generatedStr = JsonToString(generatedValue); + + auto matches = (generatedStr == std::string(referenceString)); + + if (!matches) + { + printf("Didn't match!\n"); + printf("Reference:\n"); + printf("%s\n", referenceString); + + printf("Generated:\n"); + printf("%s\n", generatedStr.c_str()); + } + + return matches; + +#if 0 + // + // Converting the reference string to a JSON representation and comparing + // that against the generated JSON object would have been preferable. This avoids + // the need to have reference strings expressed precisely to match the generated string + // from the JSON converter, right down to the number of spaces,etc. This would have made + // the reference string less britle and coupled to the jsoncpp converter implementation. + // + // However, jsoncpp converter converts positive values in the JSON to a signed + // integer C type. This results in a mis-match with the generated JSON objects + // that are created from spec-compliant TLV that correctly represents them as unsigned + // integers in the JSON object. + // + // This mismatch nullifies this approach unfortunately. + // + // TODO: Investigate a way to compare using JSON objects. + // + Json::Reader reader; + Json::Value referenceValue; + + bool ret = reader.parse(referenceString, referenceValue); + if (ret != true) { + return ret; + } + + std::cout << generatedValue << "\n"; + std::cout << referenceValue << "\n"; + + int rett = generatedValue.compare(referenceValue); + printf("%d\n", rett); + return (rett == 0); +#endif +} + +template +void EncodeAndValidate(T val, const char * expectedJsonString) +{ + CHIP_ERROR err; + + SetupBuf(); + + err = DataModel::Encode(gWriter, TLV::AnonymousTag(), val); + NL_TEST_ASSERT(gSuite, err == CHIP_NO_ERROR); + + err = gWriter.Finalize(); + NL_TEST_ASSERT(gSuite, err == CHIP_NO_ERROR); + + err = SetupReader(); + NL_TEST_ASSERT(gSuite, err == CHIP_NO_ERROR); + + Json::Value d; + err = TlvToJson(gReader, d); + NL_TEST_ASSERT(gSuite, err == CHIP_NO_ERROR); + + bool matches = Matches(expectedJsonString, d); + NL_TEST_ASSERT(gSuite, matches); +} + +void TestConverter(nlTestSuite * inSuite, void * inContext) +{ + gSuite = inSuite; + + EncodeAndValidate(static_cast(30), + "{\n" + " \"value\" : 30\n" + "}\n"); + + EncodeAndValidate(static_cast(-30), + "{\n" + " \"value\" : -30\n" + "}\n"); + + EncodeAndValidate(false, + "{\n" + " \"value\" : false\n" + "}\n"); + + EncodeAndValidate(true, + "{\n" + " \"value\" : true\n" + "}\n"); + + EncodeAndValidate(1.0, + "{\n" + " \"value\" : 1.0\n" + "}\n"); + + const char charBuf[] = "hello"; + CharSpan charSpan(charBuf); + EncodeAndValidate(charSpan, + "{\n" + " \"value\" : \"hello\"\n" + "}\n"); + + // + // Validated using https://base64.guru/converter/encode/hex + // + const uint8_t byteBuf[] = { 0x01, 0x02, 0x03, 0x04, 0xff, 0xfe, 0x99, 0x88, 0xdd, 0xcd }; + ByteSpan byteSpan(byteBuf); + EncodeAndValidate(byteSpan, + "{\n" + " \"value\" : \"AQIDBP/+mYjdzQ==\"\n" + "}\n"); + + DataModel::Nullable nullValue; + EncodeAndValidate(nullValue, + "{\n" + " \"value\" : null\n" + "}\n"); + + Clusters::TestCluster::Structs::SimpleStruct::Type structVal; + structVal.a = 20; + structVal.b = true; + structVal.d = byteBuf; + structVal.e = charSpan; + structVal.g = 1.0; + structVal.h = 1.0; + + EncodeAndValidate(structVal, + "{\n" + " \"value\" : {\n" + " \"0\" : 20,\n" + " \"1\" : true,\n" + " \"2\" : 0,\n" + " \"3\" : \"AQIDBP/+mYjdzQ==\",\n" + " \"4\" : \"hello\",\n" + " \"5\" : 0,\n" + " \"6\" : 1.0,\n" + " \"7\" : 1.0\n" + " }\n" + "}\n"); + + uint8_t int8uListData[] = { 1, 2, 3, 4 }; + DataModel::List int8uList; + + int8uList = int8uListData; + + EncodeAndValidate(int8uList, + "{\n" + " \"value\" : [ 1, 2, 3, 4 ]\n" + "}\n"); + + Clusters::TestCluster::Structs::SimpleStruct::Type structListData[2] = { structVal, structVal }; + DataModel::List structList; + + structList = structListData; + + EncodeAndValidate(structList, + "{\n" + " \"value\" : [\n" + " {\n" + " \"0\" : 20,\n" + " \"1\" : true,\n" + " \"2\" : 0,\n" + " \"3\" : \"AQIDBP/+mYjdzQ==\",\n" + " \"4\" : \"hello\",\n" + " \"5\" : 0,\n" + " \"6\" : 1.0,\n" + " \"7\" : 1.0\n" + " },\n" + " {\n" + " \"0\" : 20,\n" + " \"1\" : true,\n" + " \"2\" : 0,\n" + " \"3\" : \"AQIDBP/+mYjdzQ==\",\n" + " \"4\" : \"hello\",\n" + " \"5\" : 0,\n" + " \"6\" : 1.0,\n" + " \"7\" : 1.0\n" + " }\n" + " ]\n" + "}\n"); +} + +int Initialize(void * apSuite) +{ + VerifyOrReturnError(chip::Platform::MemoryInit() == CHIP_NO_ERROR, FAILURE); + return SUCCESS; +} + +int Finalize(void * aContext) +{ + (void) gStore.Release(); + chip::Platform::MemoryShutdown(); + return SUCCESS; +} + +const nlTest sTests[] = { NL_TEST_DEF("TestConverter", TestConverter), NL_TEST_SENTINEL() }; + +} // namespace + +int TestTlvJson(void) +{ + nlTestSuite theSuite = { "TlvJson", sTests, Initialize, Finalize }; + nlTestRunner(&theSuite, nullptr); + return nlTestRunnerStats(&theSuite); +} + +CHIP_REGISTER_TEST_SUITE(TestTlvJson)