From 24febb54d02d8c0d7862b94b940481032ff5fea4 Mon Sep 17 00:00:00 2001 From: Rajan Date: Mon, 31 Jul 2023 11:51:10 -0400 Subject: [PATCH 1/6] adjust schema whitespace --- .../serialization/tests/autofb_schema_test.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vowpalwabbit/serialization/tests/autofb_schema_test.cc b/vowpalwabbit/serialization/tests/autofb_schema_test.cc index 7c307b42dd0..aec5ff99107 100644 --- a/vowpalwabbit/serialization/tests/autofb_schema_test.cc +++ b/vowpalwabbit/serialization/tests/autofb_schema_test.cc @@ -79,12 +79,12 @@ struct test_single TEST(Serialization, IndividualTypes) { - std::string schema_str = R"( - namespace test; - table test_single { - a:int32; - } - )"; + std::string schema_str = + R"(namespace test; + table test_single { + a:int32; + } + )"; type_registry registry; type_descriptor td = type_builder_ex>::register_type( From 6ea009df21be434c320d26baf63d6def47727d2a Mon Sep 17 00:00:00 2001 From: Rajan Date: Mon, 31 Jul 2023 16:11:00 -0400 Subject: [PATCH 2/6] fixed test: write_flatbuffer uses global type_registry. change test to use global type_registry --- .../serialization/tests/autofb_schema_test.cc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/vowpalwabbit/serialization/tests/autofb_schema_test.cc b/vowpalwabbit/serialization/tests/autofb_schema_test.cc index aec5ff99107..6036ac92dd5 100644 --- a/vowpalwabbit/serialization/tests/autofb_schema_test.cc +++ b/vowpalwabbit/serialization/tests/autofb_schema_test.cc @@ -77,7 +77,7 @@ struct test_single Prop a; }; -TEST(Serialization, IndividualTypes) +TEST(Serialization, IndividualType_SchemaGeneration) { std::string schema_str = R"(namespace test; @@ -95,7 +95,19 @@ TEST(Serialization, IndividualTypes) schema_builder builder("test", registry); fbs_data fbs = builder.build_idl(); EXPECT_EQ(normalize(fbs.text_data), normalize(schema_str)); +} +TEST(Serialization, IndividualType_Write_Read) +{ + // register with the global instance + auto& registry = type_registry::instance(); + //Register type + type_descriptor td = type_builder_ex>::register_type( + registry, "test_single" + ) + .with_property, &test_single::a>("a") + .descriptor(); + schema_builder builder("test", registry); // Serialize and deserialize test // Begin serialize From 6b96faf5ecaaf310cc7e1285a9f368aea8aa89b3 Mon Sep 17 00:00:00 2001 From: Rajan Date: Mon, 31 Jul 2023 18:05:34 -0400 Subject: [PATCH 3/6] schema generation tests for all basic types --- .../serialization/tests/autofb_schema_test.cc | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/vowpalwabbit/serialization/tests/autofb_schema_test.cc b/vowpalwabbit/serialization/tests/autofb_schema_test.cc index 6036ac92dd5..4dc17e537cd 100644 --- a/vowpalwabbit/serialization/tests/autofb_schema_test.cc +++ b/vowpalwabbit/serialization/tests/autofb_schema_test.cc @@ -77,26 +77,38 @@ struct test_single Prop a; }; -TEST(Serialization, IndividualType_SchemaGeneration) +template +auto test_single_type(const std::string& type_str) -> void { - std::string schema_str = - R"(namespace test; - table test_single { - a:int32; - } - )"; + std::string schema_str = "namespace test; table test_single { a:" + type_str + "; } "; type_registry registry; - type_descriptor td = type_builder_ex>::register_type( + type_descriptor td = type_builder_ex>::register_type( registry, "test_single" ) - .with_property, &test_single::a>("a") + .with_property, &test_single::a>("a") .descriptor(); schema_builder builder("test", registry); fbs_data fbs = builder.build_idl(); EXPECT_EQ(normalize(fbs.text_data), normalize(schema_str)); } +TEST(Serialization, IndividualType_SchemaGeneration) +{ + test_single_type("int8"); + test_single_type("int16"); + test_single_type("int32"); + test_single_type("int64"); + test_single_type("uint8"); + test_single_type("uint16"); + test_single_type("uint32"); + test_single_type("uint64"); + test_single_type("bool"); + test_single_type("float"); + test_single_type("double"); + test_single_type("string"); +} + TEST(Serialization, IndividualType_Write_Read) { // register with the global instance From 2ff28a763b1df03d22eb1908b6d77a2270032e6c Mon Sep 17 00:00:00 2001 From: Rajan Date: Mon, 31 Jul 2023 18:05:56 -0400 Subject: [PATCH 4/6] fix schme generation for int64 --- vowpalwabbit/serialization/src/autofb.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/vowpalwabbit/serialization/src/autofb.cc b/vowpalwabbit/serialization/src/autofb.cc index 921328d4ba4..61065b533cc 100644 --- a/vowpalwabbit/serialization/src/autofb.cc +++ b/vowpalwabbit/serialization/src/autofb.cc @@ -18,6 +18,7 @@ namespace autofb { typeid(std::int8_t), "int8" }, { typeid(std::int16_t), "int16" }, { typeid(std::int32_t), "int32" }, + { typeid(std::int64_t), "int64" }, { typeid(std::uint8_t), "uint8" }, { typeid(std::uint16_t), "uint16" }, { typeid(std::uint32_t), "uint32" }, From b3e9a4aa9e9f0047b56e0502988e66f871cfb39d Mon Sep 17 00:00:00 2001 From: Rajan Date: Mon, 31 Jul 2023 18:18:05 -0400 Subject: [PATCH 5/6] added deserialize test --- vowpalwabbit/serialization/tests/autofb_schema_test.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vowpalwabbit/serialization/tests/autofb_schema_test.cc b/vowpalwabbit/serialization/tests/autofb_schema_test.cc index 4dc17e537cd..ef7b30ec64b 100644 --- a/vowpalwabbit/serialization/tests/autofb_schema_test.cc +++ b/vowpalwabbit/serialization/tests/autofb_schema_test.cc @@ -126,7 +126,9 @@ TEST(Serialization, IndividualType_Write_Read) schema schema_var = builder.build(); serializer serializer(schema_var); flatbuffers::FlatBufferBuilder fbb; - test_single t{1}; + + constexpr int input_val = 1791; + test_single t{input_val}; auto erased = type>::erase(); ref a_ref(t); erased_lvalue_ref elv { erased, a_ref }; @@ -138,6 +140,9 @@ TEST(Serialization, IndividualType_Write_Read) uint8_t* buf = fbb.GetBufferPointer(); size_t size = fbb.GetSize(); + activation act = serializer.read_flatbuffer(buf, erased); + test_single out_struct = act.get>(); + EXPECT_EQ(input_val, out_struct.a); // End deserialize } From 1de6b445fbf40121eebf9b43f2e724cf894548b1 Mon Sep 17 00:00:00 2001 From: Rajan Date: Fri, 4 Aug 2023 14:55:27 -0400 Subject: [PATCH 6/6] Bug fix: erased_field_binder.try_bind was not binding to the right pointer. --- .../include/vw/serialization/type_builder.h | 3 +- .../include/vw/serialization/type_erase.h | 8 +-- vowpalwabbit/serialization/src/autofb.cc | 3 +- .../serialization/tests/autofb_schema_test.cc | 69 +++++++++++++++++++ 4 files changed, 76 insertions(+), 7 deletions(-) diff --git a/vowpalwabbit/serialization/include/vw/serialization/type_builder.h b/vowpalwabbit/serialization/include/vw/serialization/type_builder.h index 9ad4547e136..59679529ab2 100644 --- a/vowpalwabbit/serialization/include/vw/serialization/type_builder.h +++ b/vowpalwabbit/serialization/include/vw/serialization/type_builder.h @@ -38,7 +38,6 @@ namespace typesys if (typesys::is_instance::value) { // need to pull out the inner type of the property - std::cout << "type: " << typeid(typename P::value_type).name() << std::endl; // this is a scalar @@ -129,4 +128,4 @@ namespace typesys binding_builder(BaseT& pb) : BaseT(pb) {} }; -} \ No newline at end of file +} diff --git a/vowpalwabbit/serialization/include/vw/serialization/type_erase.h b/vowpalwabbit/serialization/include/vw/serialization/type_erase.h index 150a3ac93c7..523bd62b97b 100644 --- a/vowpalwabbit/serialization/include/vw/serialization/type_erase.h +++ b/vowpalwabbit/serialization/include/vw/serialization/type_erase.h @@ -235,9 +235,9 @@ struct ref } // void* ctor - // ref(void*& r) : _r(&r) - // { - // } + ref(void* r) : _r(r) + { + } //getter template @@ -409,4 +409,4 @@ struct vtype // TODO: Do we need to implement erase(v_array<>)? }; -} \ No newline at end of file +} diff --git a/vowpalwabbit/serialization/src/autofb.cc b/vowpalwabbit/serialization/src/autofb.cc index 61065b533cc..498607100dd 100644 --- a/vowpalwabbit/serialization/src/autofb.cc +++ b/vowpalwabbit/serialization/src/autofb.cc @@ -195,7 +195,8 @@ struct fbb_ReadElement_dispatcher void operator()(const flatbuffers::Table& table, const reflection::Field& field, typesys::erased_lvalue_ref& value) { - dt.dispatch(value._type, table, field, value); + const bool dispatched = dt.dispatch(value._type, table, field, value); + assert(dispatched); } private: diff --git a/vowpalwabbit/serialization/tests/autofb_schema_test.cc b/vowpalwabbit/serialization/tests/autofb_schema_test.cc index ef7b30ec64b..b2700e981af 100644 --- a/vowpalwabbit/serialization/tests/autofb_schema_test.cc +++ b/vowpalwabbit/serialization/tests/autofb_schema_test.cc @@ -109,6 +109,75 @@ TEST(Serialization, IndividualType_SchemaGeneration) test_single_type("string"); } +TEST(SerializationSupport, ErasedLValueAndErasedField) +{ + // This is the struct we want to manipulate + test_single t; + t.a = 1000; + + // erased ref to t.a + erased_lvalue_ref elv_t_a { + type>::erase(), + ref(t.a) + }; + + // erased ref to t + erased_lvalue_ref elv_t { + type>::erase(), + ref(t) + }; + + // set t.a using erased_lvalue_ref + auto& val = elv_t_a.get>(); + assert(val.val == 1000); + val.val = 2000; + assert(t.a == 2000); + + // field_ptr holds offset a given member variable for a given class + // In this case, we want to hold offset to 'a' for test_single + constexpr field_ptr, Prop> field_ptr_a {&test_single::a}; + + // Declare the pointer that will eventually point to t.a + erased_lvalue_ref* elv_field_ptr = nullptr; + + // binder that can be used to bind t.a to an erased_lvalue_ptr + const erased_field_binder erased_field_binder_a = field_ptr_a.erase_binder(); + assert(erased_field_binder_a.try_bind(elv_t, elv_field_ptr)); + + auto& val2 = elv_field_ptr->get>(); + + val2 = 3000; + assert(t.a == 3000); + delete elv_field_ptr; +} + +TEST(SerializationSupport, FieldPtr) +{ + test_single t; + + field_ptr_id,Prop,&test_single::a> ptr_id; + field_ptr, Prop> ptr = ptr_id(); + Prop& x = ptr.bind(t); + x = 1000; + assert(t.a == 1000); + + field_ptr, Prop> ptr2 {&test_single::a}; + erased_lvalue_ref elv_t { type>::erase(), ref(t) }; + Prop* y; + assert(ptr2.try_bind(elv_t,y)); + y->val = 2000; + assert(t.a == 2000); + + ///// Study this + erased_lvalue_ref* elv_ptr; // This will be a pointer to a Prop + erased_field_binder binder = ptr2.erase_binder(); // This is a binder for Prop + assert(binder.try_bind(elv_t, elv_ptr)); // This binds elv_ptr to t.a + elv_ptr->set(3000); + assert(t.a == 3000); + + delete elv_ptr; +} + TEST(Serialization, IndividualType_Write_Read) { // register with the global instance