Rajan/serialization tests

vowpalwabbit/serialization/include/vw/serialization/type_builder.h

@@ -21,7 +21,7 @@
 
     inline static type_builder_ex register_type(type_registry& tr)
     {
       const char* name = typeid(std::decay_t<C>).name();
       return register_type(tr, name);
     }
 
@@ -38,7 +38,6 @@
       if (typesys::is_instance<P, typesys::Prop>::value)
       {
         // need to pull out the inner type of the property
-
         std::cout << "type: " << typeid(typename P::value_type).name() << std::endl;
 
         // this is a scalar
@@ -103,7 +102,7 @@
       return *this;
     }
 
     template <typename BinderT, std::enable_if_t<is_instance<typename BinderT::BuilderT, binding_builder>::value>>
     typename BinderT::BuilderT& bind(BinderT binder)
     {
       return binder.template bind<C, P>(*this, pd);
@@ -129,4 +128,4 @@
     binding_builder(BaseT& pb) : BaseT(pb)
     {}
   };
-}
+}

vowpalwabbit/serialization/include/vw/serialization/type_erase.h

@@ -23,7 +23,7 @@
 {
 private:
   // must be created through type::erase()
   erased_type() = default;
 
 public:
   std::type_index tindex;
@@ -83,7 +83,7 @@
             >::type = true>
   inline static erased_type erase()
   {
     return erased_type{typeid(T), sizeof(T), &activator<T>::activate, nullptr};
   }
 
   using typed_init_f = T* (*)();
@@ -235,9 +235,9 @@
   }
 
   // void* ctor
-  // ref(void*& r) : _r(&r) 
-  // {
-  // }
+  ref(void* r) : _r(r) 
+  {
+  }
 
   //getter
   template <typename T>
@@ -409,4 +409,4 @@
   // TODO: Do we need to implement erase(v_array<>)?
 };
 
-}
+}

vowpalwabbit/serialization/src/autofb.cc

@@ -18,6 +18,7 @@
     { typeid(std::int8_t), "int8" },
     { typeid(std::int16_t), "int16" },
     { typeid(std::int32_t), "int32" },
+    { typeid(std::int64_t), "int64" },
     { typeid(std::uint8_t), "uint8" },
     { typeid(std::uint16_t), "uint16" },
     { typeid(std::uint32_t), "uint32" },
@@ -52,7 +53,7 @@
     // buffer representation is no longer conveniently put into a std::string. (We should probably be
     // using vectors instead, honestly, but this is based on the reflection examples.)
     static_assert(
         sizeof(std::remove_pointer_t<decltype(buf)>) == sizeof(char), "fb buffer type is not the same as char*");
 
     std::string result;
     result.resize(size);
@@ -194,7 +195,8 @@
 
   void operator()(const flatbuffers::Table& table, const reflection::Field& field, typesys::erased_lvalue_ref& value)
   {
-    dt.dispatch(value._type, table, field, value);
+    const bool dispatched = dt.dispatch(value._type, table, field, value);
+    assert(dispatched);
   }
 
 private:
@@ -331,7 +333,7 @@
 
   // pass1: depth-first build the offsets for complex object (non-builtins, or strings)
   std::for_each(ti.properties.begin(), ti.properties.end(), 
     [&offsets, &fbb, &descriptor, &schema, &object, &value](auto& it)
     {
       auto& eftype = it.eftype;
       const type_descriptor* maybe_ti = nullptr;
@@ -350,7 +352,7 @@
       // after caching the type lookup
       if (!eftype.is_vector() && 
           type_registry::instance().is_builtin(pti) && 
           !eftype.evalue.template is<std::string>()) 
       { return; }
 
       flatbuffers::uoffset_t offset;
@@ -392,7 +394,7 @@
   flatbuffers::uoffset_t table_offset = fbb.StartTable();
 
   std::for_each(ti.properties.begin(), ti.properties.end(), 
     [&offsets, &fbb, &schema, &object, &value](auto& it)
     {
       auto& eftype = it.eftype;
 
@@ -452,7 +454,7 @@
     {
       // this is a vector of scalars which are not strings
       uint8_t* buf;
       flatbuffers::uoffset_t result = fbb.CreateUninitializedVector(espan.size(), espan._type.size, &buf);
       espan.copy_to(reinterpret_cast<void*>(buf), espan.size() * espan._type.size);
 
       return result;
@@ -502,7 +504,7 @@
   using namespace typesys;
 
   std::for_each(ti.properties.begin(), ti.properties.end(), 
     [&schema, &table, &source, &target](auto& it)
     {
       auto& eftype = it.eftype;
 
@@ -553,7 +555,7 @@
           return;
         }
 
         if (eftype.evalue.template is<std::string>())
         {
           auto str = flatbuffers::GetFieldS(source, field);
           pvalue->set<std::string>(std::string(flatbuffers::GetString(str)));
@@ -603,7 +605,7 @@
 
   //print all of the names of the types in the schemas
   auto types = schema.objects();
   for ( size_t i = 0; i < types->Length(); i++ ) {
       const reflection::Object* type = types->Get( i );
       std::cout << "type: " << type->name()->str() << std::endl;
   }

vowpalwabbit/serialization/tests/autofb_schema_test.cc

@@ -77,32 +77,127 @@ struct test_single
   Prop<T> a;
 };
 
-TEST(Serialization, IndividualTypes)
+template<typename T>
+auto test_single_type(const std::string& type_str) -> void
 {
-  std::string schema_str = R"(
-  namespace test;
-  table test_single {
-    a:int32;
-  }
-  )";
+  std::string schema_str = "namespace test; table test_single { a:" + type_str + "; } ";
 
   type_registry registry;
-  type_descriptor td = type_builder_ex<test_single<int>>::register_type(
+  type_descriptor td = type_builder_ex<test_single<T>>::register_type(
                         registry, "test_single"
                       )
-                      .with_property<Prop<int>, &test_single<int>::a>("a")
+                      .with_property<Prop<T>, &test_single<T>::a>("a")
                       .descriptor();
   schema_builder builder("test", registry);
   fbs_data fbs = builder.build_idl();
   EXPECT_EQ(normalize(fbs.text_data), normalize(schema_str));
+}
+
+TEST(Serialization, IndividualType_SchemaGeneration)
+{
+  test_single_type<int8_t>("int8");
+  test_single_type<int16_t>("int16");
+  test_single_type<int32_t>("int32");
+  test_single_type<int64_t>("int64");
+  test_single_type<uint8_t>("uint8");
+  test_single_type<uint16_t>("uint16");
+  test_single_type<uint32_t>("uint32");
+  test_single_type<uint64_t>("uint64");
+  test_single_type<bool>("bool");
+  test_single_type<float>("float");
+  test_single_type<double>("double");
+  test_single_type<std::string>("string");
+}
+
+TEST(SerializationSupport, ErasedLValueAndErasedField)
+{
+  // This is the struct we want to manipulate
+  test_single<int> t;
+  t.a = 1000;
 
+  // erased ref to t.a
+  erased_lvalue_ref elv_t_a {
+    type<Prop<int>>::erase(),
+    ref(t.a)
+  };
+
+  // erased ref to t
+  erased_lvalue_ref elv_t {
+    type<test_single<int>>::erase(),
+    ref(t)
+  };
+
+  // set t.a using erased_lvalue_ref 
+  auto& val = elv_t_a.get<Prop<int>>();
+  assert(val.val == 1000);
+  val.val = 2000;
+  assert(t.a == 2000);
+
+  // field_ptr holds offset a given member variable for a given class
+  // In this case, we want to hold offset to 'a' for test_single<int>
+  constexpr field_ptr<test_single<int>, Prop<int>> field_ptr_a {&test_single<int>::a};
+
+  // Declare the pointer that will eventually point to t.a
+  erased_lvalue_ref* elv_field_ptr = nullptr;
+
+  // binder that can be used to bind t.a to an erased_lvalue_ptr 
+  const erased_field_binder erased_field_binder_a = field_ptr_a.erase_binder();
+  assert(erased_field_binder_a.try_bind(elv_t, elv_field_ptr));
+
+  auto& val2 = elv_field_ptr->get<Prop<int>>();
+
+  val2 = 3000;
+  assert(t.a == 3000);
+  delete elv_field_ptr;
+}
+
+TEST(SerializationSupport, FieldPtr)
+{
+  test_single<int> t;
+
+  field_ptr_id<test_single<int>,Prop<int>,&test_single<int>::a> ptr_id;
+  field_ptr<test_single<int>, Prop<int>> ptr = ptr_id();
+  Prop<int>& x = ptr.bind(t);
+  x = 1000;
+  assert(t.a == 1000);
+
+  field_ptr<test_single<int>, Prop<int>> ptr2 {&test_single<int>::a};
+  erased_lvalue_ref elv_t { type<test_single<int>>::erase(), ref(t) };
+  Prop<int>* y;
+  assert(ptr2.try_bind(elv_t,y));
+  y->val = 2000;
+  assert(t.a == 2000);
+
+  ///// Study this
+  erased_lvalue_ref* elv_ptr;  // This will be a pointer to a Prop<int>
+  erased_field_binder binder = ptr2.erase_binder();  // This is a binder for Prop<int>
+  assert(binder.try_bind(elv_t, elv_ptr));  // This binds elv_ptr to t.a
+  elv_ptr->set(3000);
+  assert(t.a == 3000);
+
+  delete elv_ptr;
+}
+
+TEST(Serialization, IndividualType_Write_Read)
+{
+  // register with the global instance
+  auto& registry = type_registry::instance();
+  //Register type
+  type_descriptor td = type_builder_ex<test_single<int>>::register_type(
+                        registry, "test_single"
+                      )
+                      .with_property<Prop<int>, &test_single<int>::a>("a")
+                      .descriptor();
+  schema_builder builder("test", registry);
   // Serialize and deserialize test
 
   // Begin serialize
   schema schema_var = builder.build();
   serializer serializer(schema_var);
   flatbuffers::FlatBufferBuilder fbb;
-  test_single<int> t{1};
+
+  constexpr int input_val = 1791;
+  test_single<int> t{input_val};
   auto erased = type<test_single<int>>::erase();
   ref a_ref(t); 
   erased_lvalue_ref elv { erased, a_ref };
@@ -114,6 +209,9 @@ TEST(Serialization, IndividualTypes)
   uint8_t* buf = fbb.GetBufferPointer();
   size_t size = fbb.GetSize();
 
+  activation act = serializer.read_flatbuffer(buf, erased);
+  test_single<int> out_struct = act.get<test_single<int>>();
+  EXPECT_EQ(input_val, out_struct.a);
   // End deserialize
 }