Skip to content

Commit

Permalink
Fix missing handling of dynamic arrays of enums, fix wrong JSON schem…
Browse files Browse the repository at this point in the history 
…a and data for enum arrays (static and dynamic) (#75)
  • Loading branch information
@tribal-tec
tribal-tec authored Dec 6, 2016
1 parent c7bede9 commit 9e51c26
Show file tree
Hide file tree
Showing 6 changed files with 177 additions and 34 deletions.
100 changes: 80 additions & 20 deletions bin/zerobufCxx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -534,47 +534,58 @@ def get_declaration(self):

def from_json(self):
fromJSON = 'if( ::zerobuf::hasJSONField( json, "{0}" ))'.format(self.name)
fromJSON += NEXTLINE + "{"
fromJSON += NEXTLINE + '{'
fromJSON += NEXTLINE + ' const Json::Value& field = ::zerobuf::getJSONField( json, "{0}" );'.format(self.name)

if self.value_type.is_zerobuf_type and not self.value_type.is_enum_type:
if self.value_type.is_zerobuf_type:
for i in range(0, self.nElems):
fromJSON += NEXTLINE + " ::zerobuf::fromJSON( ::zerobuf::getJSONField( field, {1} ), _{0}[{1}] );".\
fromJSON += NEXTLINE + ' ::zerobuf::fromJSON( ::zerobuf::getJSONField( field, {1} ), _{0}[{1}] );'.\
format(self.name, i)
else:
fromJSON += NEXTLINE + " {0}* array = ({0}*)get{1}();".\
fromJSON += NEXTLINE + ' {0}* array = ({0}*)get{1}();'.\
format(self.value_type.get_data_type(), self.cxxName)

if self.value_type.is_byte_type:
fromJSON += NEXTLINE + " const std::string& decoded = ::zerobuf::fromJSONBinary( field );"
fromJSON += NEXTLINE + " ::memcpy( array, decoded.data(), std::min( decoded.length(), size_t( {0}ull )));".format(self.nElems)
fromJSON += NEXTLINE + ' const std::string& decoded = ::zerobuf::fromJSONBinary( field );'
fromJSON += NEXTLINE + ' ::memcpy( array, decoded.data(), std::min( decoded.length(), size_t( {0}ull )));'.format(self.nElems)
elif self.value_type.is_enum_type:
for i in range(0, self.nElems):
# convert strings back to enum/int values
fromJSON += NEXTLINE + ' array[{0}] = {1}( string_to_{2}('\
'::zerobuf::fromJSON< std::string >( ::zerobuf::getJSONField( field, {0} ))));'.\
format(i, self.value_type.get_data_type(), self.value_type.type)
else:
for i in range(0, self.nElems):
fromJSON += NEXTLINE + " array[{0}] = ::zerobuf::fromJSON< {1} >( ::zerobuf::getJSONField( field, {0} ));".\
fromJSON += NEXTLINE + ' array[{0}] = ::zerobuf::fromJSON< {1} >( ::zerobuf::getJSONField( field, {0} ));'.\
format(i, self.value_type.get_data_type())

fromJSON += NEXTLINE + "}"
fromJSON += NEXTLINE + '}'
return fromJSON

def to_json(self):
toJSON = "{"
toJSON = '{'
toJSON += NEXTLINE + ' Json::Value& field = ::zerobuf::getJSONField( json, "{0}" );'.\
format(self.name)

if self.value_type.is_zerobuf_type and not self.value_type.is_enum_type:
if self.value_type.is_zerobuf_type:
for i in range(0, self.nElems):
toJSON += NEXTLINE + " ::zerobuf::toJSON( static_cast< const ::zerobuf::Zerobuf& >( _{0}[{1}] ), ::zerobuf::getJSONField( field, {1} ));".\
format(self.name, i)
toJSON += NEXTLINE + ' ::zerobuf::toJSON( static_cast< const ::zerobuf::Zerobuf& >( _{0}[{1}] ),'\
'::zerobuf::getJSONField( field, {1} ));'.format(self.name, i)
else:
toJSON += NEXTLINE + " const {0}* array = (const {0}*)get{1}();".\
toJSON += NEXTLINE + ' const {0}* array = (const {0}*)get{1}();'.\
format(self.value_type.get_data_type(), self.cxxName)

if self.value_type.is_byte_type:
toJSON += NEXTLINE + " ::zerobuf::toJSONBinary( array, {0}, field );".format(self.nElems)
toJSON += NEXTLINE + ' ::zerobuf::toJSONBinary( array, {0}, field );'.format(self.nElems)
elif self.value_type.is_enum_type:
for i in range(0, self.nElems):
# convert enum values to strings
toJSON += NEXTLINE + ' ::zerobuf::toJSON( to_string( {0}( array[{1}] )),'\
'::zerobuf::getJSONField( field, {1} ));'.format(self.value_type.type, i)
else:
for i in range(0, self.nElems):
toJSON += NEXTLINE + " ::zerobuf::toJSON( array[{0}], ::zerobuf::getJSONField( field, {0} ));".format(i)
toJSON += NEXTLINE + "}"
toJSON += NEXTLINE + ' ::zerobuf::toJSON( array[{0}], ::zerobuf::getJSONField( field, {0} ));'.format(i)
toJSON += NEXTLINE + '}'
return toJSON


Expand Down Expand Up @@ -867,6 +878,18 @@ def from_string(self):
'{0}{1}throw std::runtime_error( "{2}" );'
.format(NEXTLINE.join(strs), NEXTLINE, 'Cannot convert string to enum {0}'.format(self.name)), split=True)

def enum_to_string(self, namespaces):
""" Specialization for zerobuf::to_string() used in Vector.h """
return Function('template<> std::string',
'enum_to_string( const {0}::{1}& val )'.format(namespaces, self.name),
'return {0}::to_string( val );'.format(namespaces), split=True)

def string_to_enum(self, namespaces):
""" Specialization for zerobuf::string_to_enum() used in Vector.h """
return Function('template<> {0}::{1}'.format(namespaces, self.name),
'string_to_enum( const std::string& val )',
'return {0}::string_to_{1}( val );'.format(namespaces, self.name), split=True)

def ostream(self):
return Function('std::ostream&',
'operator << ( std::ostream& os, const {0}& val )'.format(self.name),
Expand All @@ -891,6 +914,16 @@ def write_implementation(self, file):
self.from_string.write_implementation(file)
self.ostream.write_implementation(file)

def write_string_conversion_declaration(self, file, namespaces):
""" Declarations for zerobuf::to_string() and zerobuf::string_to_enum() """
self.enum_to_string(namespaces).write_declaration(file)
self.string_to_enum(namespaces).write_declaration(file)

def write_string_conversion_implementation(self, file, namespaces):
""" Definitions for zerobuf::to_string() and zerobuf::string_to_enum() """
self.enum_to_string(namespaces).write_implementation(file)
self.string_to_enum(namespaces).write_implementation(file)


def _add_base64_string(property):
property['type'] = 'string'
Expand Down Expand Up @@ -927,9 +960,13 @@ def dynamic_member(self, name, cxxtype, fbs_type):
else:
property['type'] = 'array'
is_zerobuf_type = cxxtype in self.fbsFile.table_names
is_enum_type = cxxtype in self.fbsFile.enum_names
if is_zerobuf_type:
# array of ZeroBuf objects
property['items'] = self._table_schema(cxxtype)
elif is_enum_type:
# array of enums
property['items'] = self._enum_schema(cxxtype)
else:
# POD array
property['items'] = OrderedDict()
Expand All @@ -948,7 +985,7 @@ def fixed_size_member(self, name, cxxtype, fbs_type):
property['type'] = fbs_to_json_type(fbs_type)
self.properties[name] = property

def fixed_size_array(self, name, fbs_type, elem_count):
def fixed_size_array(self, name, cxxtype, fbs_type, elem_count):
property = OrderedDict()
self.properties[name] = property

Expand All @@ -957,11 +994,22 @@ def fixed_size_array(self, name, fbs_type, elem_count):
_add_base64_string(property)
else:
property['type'] = 'array'
property['items'] = OrderedDict()
property['items']['type'] = fbs_to_json_type(fbs_type)
property['minItems'] = elem_count
property['maxItems'] = elem_count

is_zerobuf_type = cxxtype in self.fbsFile.table_names
is_enum_type = cxxtype in self.fbsFile.enum_names
if is_zerobuf_type:
# array of ZeroBuf objects
property['items'] = self._table_schema(cxxtype)
elif is_enum_type:
# array of enums
property['items'] = self._enum_schema(cxxtype)
else:
# POD/enum array
property['items'] = OrderedDict()
property['items']['type'] = fbs_to_json_type(fbs_type)


class FbsTable():
"""An fbs Table (class) which can be written to a C++ implementation."""
Expand Down Expand Up @@ -1042,7 +1090,7 @@ def parse_members(self, fbsFile):
else:
elem_count = int(attrib[4])
member = FixedSizeArray(name, value_type, elem_count, self.name)
json_schema.fixed_size_array(name, fbs_type, elem_count)
json_schema.fixed_size_array(name, cxxtype, fbs_type, elem_count)
self.static_members.append(member)

self.all_members.append(member)
Expand Down Expand Up @@ -1518,6 +1566,12 @@ def write_declaration(self, header):

self.write_namespace_closing(header)

# Write enum string_conversions at the end of file in zerobuf namespace
header.write("namespace zerobuf\n{")
for enum in self.enums:
enum.write_string_conversion_declaration(header, '::'.join(self.namespace))
header.write("\n}\n\n")

def write_implementation(self, impl):
"""Write the C++ implementation file."""

Expand All @@ -1538,6 +1592,12 @@ def write_implementation(self, impl):
impl.write("\n")
self.write_namespace_closing(impl)

# Write enum string_conversions at the end of file in zerobuf namespace
impl.write("namespace zerobuf\n{\n")
for enum in self.enums:
enum.write_string_conversion_implementation(impl, '::'.join(self.namespace))
impl.write("\n}\n\n")


if __name__ == "__main__":
parser = argparse.ArgumentParser(
Expand Down
3 changes: 2 additions & 1 deletion tests/jsonSerialization.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,9 @@ const std::string expectedJson( "{\n"
" \"doublearray\" : [ 1, 1, 2, 3 ],\n"
" \"doubledynamic\" : [ 1, 1, 2, 3 ],\n"
" \"doublevalue\" : 42,\n"
" \"enumarray\" : [ \"FIRST\", \"SECOND\" ],\n"
" \"enumdynamic\" : [ \"SECOND\", \"THIRD_UNDERSCORE\" ],\n"
" \"enumeration\" : \"SECOND\",\n"
" \"enumerations\" : [ 0, 1 ],\n"
" \"falseBool\" : false,\n"
" \"floatarray\" : [ 1, 1, 2, 3 ],\n"
" \"floatdynamic\" : [ 1, 1, 2, 3 ],\n"
Expand Down
14 changes: 12 additions & 2 deletions tests/serialization.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,10 @@ test::TestSchema getTestObject()
object.setEnumeration( test::TestEnum::SECOND );
const std::vector<test::TestEnum> testEnums = { test::TestEnum::FIRST,
test::TestEnum::SECOND };
object.setEnumerations( testEnums );
object.setEnumarray( testEnums );

object.getEnumdynamic().push_back( test::TestEnum::SECOND );
object.getEnumdynamic().push_back( test::TestEnum::THIRD_UNDERSCORE );

int32_t intMagic = 42;
uint32_t uintMagic = 4200;
Expand Down Expand Up @@ -140,10 +143,17 @@ void checkTestObject( const test::TestSchema& object )

const std::vector<test::TestEnum> testEnums = { test::TestEnum::FIRST,
test::TestEnum::SECOND };
const std::vector<test::TestEnum>& result = object.getEnumerationsVector();
const std::vector<test::TestEnum>& result = object.getEnumarrayVector();
BOOST_CHECK_EQUAL_COLLECTIONS( testEnums.begin(), testEnums.end(),
result.begin(), result.end( ));

const std::vector<test::TestEnum> testEnumDynamic =
{ test::TestEnum::SECOND, test::TestEnum::THIRD_UNDERSCORE };

const std::vector<test::TestEnum>& result2 = object.getEnumdynamicVector();
BOOST_CHECK_EQUAL_COLLECTIONS( testEnumDynamic.begin(), testEnumDynamic.end(),
result2.begin(), result2.end( ));

checkTestObject( object.getNested( ));

// Test retrieved tables
Expand Down
3 changes: 2 additions & 1 deletion tests/testSchema.fbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,8 @@ table TestSchema {
boolvalue: bool;
stringvalue: string;
enumeration: TestEnum;
enumerations: [TestEnum:2];
enumarray: [TestEnum:2];
enumdynamic: [TestEnum];
nested: TestNested;
nestedarray: [TestNested:4];
nesteddynamic: [TestNested];
Expand Down
88 changes: 78 additions & 10 deletions zerobuf/Vector.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ class Vector

/** @return The pointer to the current allocation of the vector */
const T* data() const
{ return const_cast< const Allocator* >( _alloc )->template getDynamic< T >( _index ); }
{ return const_cast< const Allocator* >
( _alloc )->template getDynamic< T >( _index ); }

/** @return true if the two vectors of builtins are identical. */
bool operator == ( const Vector& rhs ) const;
Expand Down Expand Up @@ -81,33 +82,53 @@ class Vector
/** Insert a Zerobuf-derived element at the end of the vector. */
template< class Q = T > void
push_back( const typename std::enable_if<
std::is_base_of<Zerobuf,Q>::value, Q>::type&);
std::is_base_of<Zerobuf,Q>::value, Q>::type& );

/** @internal */
void reset( Allocator& alloc ) { _alloc = &alloc; _zerobufs.clear(); }

/** Remove unused memory from vector and all members. */
void compact( float ) { /* NOP: elements are static and clear frees */ }

/** Update this vector from its JSON representation. */
/** Update this vector of ZeroBufs from its JSON representation. */
template< class Q = T > void
fromJSON( const Json::Value& json, const typename std::enable_if<
std::is_base_of< Zerobuf, Q >::value, Q >::type* = nullptr );

/** Update this vector from its JSON representation. */
/** Update this vector of enums from its JSON representation. */
template< class Q = T > void
fromJSON( const Json::Value& json, const typename std::enable_if<
!std::is_base_of< Zerobuf, Q >::value, Q >::type* = nullptr );
std::is_enum< Q >::value, Q >::type* = nullptr );

/** @return the JSON representation of this vector. */
/** Update this vector of arithmetics from its JSON representation. */
template< class Q = T > void
fromJSON( const Json::Value& json, const typename std::enable_if<
std::is_arithmetic< Q >::value, Q >::type* = nullptr );

/** Update this vector of uint128_t from its JSON representation. */
template< class Q = T > void
fromJSON( const Json::Value& json, const typename std::enable_if<
std::is_same< Q, servus::uint128_t >::value, Q >::type* = nullptr );

/** @return the JSON representation of this vector of ZeroBufs. */
template< class Q = T > void
toJSON( Json::Value& json, const typename std::enable_if<
std::is_base_of< Zerobuf, Q >::value, Q >::type* = nullptr ) const;

/** @return the JSON representation of this vector. */
/** @return the JSON representation of this vector of enums. */
template< class Q = T > void
toJSON( Json::Value& json, const typename std::enable_if<
!std::is_base_of< Zerobuf, Q >::value, Q >::type* = nullptr ) const;
std::is_enum< Q >::value, Q >::type* = nullptr ) const;

/** @return the JSON representation of this vector of arithmetics. */
template< class Q = T > void
toJSON( Json::Value& json, const typename std::enable_if<
std::is_arithmetic< Q >::value, Q >::type* = nullptr ) const;

/** @return the JSON representation of this vector of uint128_t. */
template< class Q = T > void
toJSON( Json::Value& json, const typename std::enable_if<
std::is_same< Q, servus::uint128_t >::value, Q >::type* = nullptr ) const;

/** Update this vector from its JSON, base64-encoded representation. */
template< class Q = T > void
Expand Down Expand Up @@ -267,7 +288,33 @@ void Vector< T >::fromJSON( const Json::Value& json,

template< class T > template< class Q > inline
void Vector< T >::fromJSON( const Json::Value& json,
const typename std::enable_if<!std::is_base_of<Zerobuf,Q>::value, Q>::type*)
const typename std::enable_if<std::is_enum< Q>::value, Q>::type*)
{
const size_t size_ = getJSONSize( json );
T* array = reinterpret_cast< T* >(
_alloc->updateAllocation( _index, false /*no copy*/, size_*sizeof( T)));

for( size_t i = 0; i < size_; ++i )
array[i] = string_to_enum< T >( zerobuf::fromJSON< std::string >
( getJSONField( json, i )));
}

template< class T > template< class Q > inline
void Vector< T >::fromJSON( const Json::Value& json,
const typename std::enable_if<std::is_arithmetic< Q>::value, Q>::type*)
{
const size_t size_ = getJSONSize( json );
T* array = reinterpret_cast< T* >(
_alloc->updateAllocation( _index, false /*no copy*/, size_*sizeof( T)));

for( size_t i = 0; i < size_; ++i )
array[i] = zerobuf::fromJSON< T >( getJSONField( json, i ));
}

template< class T > template< class Q > inline
void Vector< T >::fromJSON( const Json::Value& json,
const typename std::enable_if< std::is_same< Q,
servus::uint128_t >::value, Q>::type*)
{
const size_t size_ = getJSONSize( json );
T* array = reinterpret_cast< T* >(
Expand All @@ -290,7 +337,28 @@ Vector< T >::toJSON( Json::Value& json, const typename std::enable_if<

template< class T > template< class Q > inline void
Vector< T >::toJSON( Json::Value& json, const typename std::enable_if<
!std::is_base_of< Zerobuf, Q >::value, Q >::type* ) const
std::is_enum< Q >::value, Q >::type* ) const
{
const size_t size_ = size();
zerobuf::emptyJSONArray( json ); // return [] instead of null if array is empty
for( size_t i = 0; i < size_; ++i )
zerobuf::toJSON( enum_to_string< T >( (*this)[i] ),
getJSONField( json, i ));
}

template< class T > template< class Q > inline void
Vector< T >::toJSON( Json::Value& json, const typename std::enable_if<
std::is_arithmetic< Q >::value, Q >::type* ) const
{
const size_t size_ = size();
zerobuf::emptyJSONArray( json ); // return [] instead of null if array is empty
for( size_t i = 0; i < size_; ++i )
zerobuf::toJSON( (*this)[i], getJSONField( json, i ));
}

template< class T > template< class Q > inline void
Vector< T >::toJSON( Json::Value& json, const typename std::enable_if<
std::is_same< Q, servus::uint128_t >::value, Q >::type* ) const
{
const size_t size_ = size();
zerobuf::emptyJSONArray( json ); // return [] instead of null if array is empty
Expand Down
3 changes: 3 additions & 0 deletions zerobuf/types.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ using servus::uint128_t;
typedef uint8_t byte_t; //!< alias type for base64 encoded fields

typedef servus::Serializable::Data Data;

template< typename T > std::string enum_to_string( const T& );
template< typename T > T string_to_enum( const std::string& );
}

namespace Json
Expand Down

0 comments on commit 9e51c26

Please sign in to comment.