diff --git a/.gitignore b/.gitignore index 46d1ae41947..5d7a88540b7 100755 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ tests/monsterdata_go_wire.mon CMakeLists.txt.user CMakeScripts/** CTestTestfile.cmake +FlatBuffers.cbp build/Xcode/FlatBuffers.xcodeproj/project.xcworkspace/** build/Xcode/FlatBuffers.xcodeproj/xcuserdata/** FlatBuffers.xcodeproj/ diff --git a/CMakeLists.txt b/CMakeLists.txt index d6a9704b2b4..2d48b72cd8b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,8 @@ set(FlatBuffers_Compiler_SRCS include/flatbuffers/hash.h include/flatbuffers/idl.h include/flatbuffers/util.h + include/flatbuffers/reflection.h + include/flatbuffers/reflection_generated.h src/idl_parser.cpp src/idl_gen_cpp.cpp src/idl_gen_general.cpp @@ -108,7 +110,16 @@ function(compile_flatbuffers_schema_to_cpp SRC_FBS) string(REGEX REPLACE "\\.fbs$" "_generated.h" GEN_HEADER ${SRC_FBS}) add_custom_command( OUTPUT ${GEN_HEADER} - COMMAND flatc -c --gen-mutable -o "${SRC_FBS_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FBS}" + COMMAND flatc -c --no-includes --gen-mutable -o "${SRC_FBS_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FBS}" + DEPENDS flatc) +endfunction() + +function(compile_flatbuffers_schema_to_binary SRC_FBS) + get_filename_component(SRC_FBS_DIR ${SRC_FBS} PATH) + string(REGEX REPLACE "\\.fbs$" ".bfbs" GEN_BINARY_SCHEMA ${SRC_FBS}) + add_custom_command( + OUTPUT ${GEN_BINARY_SCHEMA} + COMMAND flatc -b --schema -o "${SRC_FBS_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FBS}" DEPENDS flatc) endfunction() diff --git a/build/VS2010/flattests.vcxproj b/build/VS2010/flattests.vcxproj index 96d76b13ac0..447eebdf8d3 100755 --- a/build/VS2010/flattests.vcxproj +++ b/build/VS2010/flattests.vcxproj @@ -265,6 +265,7 @@ + @@ -276,4 +277,4 @@ - + \ No newline at end of file diff --git a/docs/html/index.html b/docs/html/index.html index 639c1dae415..291a2bdfde9 100644 --- a/docs/html/index.html +++ b/docs/html/index.html @@ -54,7 +54,7 @@

FlatBuffers is an efficient cross platform serialization library for C++, with support for Java, C# and Go. It was created at Google specifically for game development and other performance-critical applications.

-

It is available as open source under the Apache license, v2 (see LICENSE.txt).

+

It is available as Open Source on GitHub under the Apache license, v2 (see LICENSE.txt).

Why use FlatBuffers?

  • Access to serialized data without parsing/unpacking - What sets FlatBuffers apart is that it represents hierarchical data in a flat binary buffer in such a way that it can still be accessed directly without parsing/unpacking, while also still supporting data structure evolution (forwards/backwards compatibility).
    • @@ -96,8 +96,8 @@

    In-depth documentation

Online resources

diff --git a/docs/html/md__compiler.html b/docs/html/md__compiler.html index a372904d08f..9dd85c5a6a0 100644 --- a/docs/html/md__compiler.html +++ b/docs/html/md__compiler.html @@ -68,11 +68,14 @@
  • -o PATH : Output all generated files to PATH (either absolute, or relative to the current directory). If omitted, PATH will be the current directory. PATH should end in your systems path separator, e.g. / or \.
  • -I PATH : when encountering include statements, attempt to load the files from this path. Paths will be tried in the order given, and if all fail (or none are specified) it will try to load relative to the path of the schema file being parsed.
  • --strict-json : Require & generate strict JSON (field names are enclosed in quotes, no trailing commas in tables/vectors). By default, no quotes are required/generated, and trailing commas are allowed.
    • +
  • --defaults-json : Output fields whose value is equal to the default value when writing JSON text.
  • --no-prefix : Don't prefix enum values in generated C++ by their enum type.
    • -
  • --gen-includes : Generate include statements for included schemas the generated file depends on (C++).
    • +
  • --gen-includes : (deprecated), instead use:
    • +
  • --no-includes : Don't generate include statements for included schemas the generated file depends on (C++).
  • --gen-mutable : Generate additional non-const accessors for mutating FlatBuffers in-place.
  • --raw-binary : Allow binaries without a file_indentifier to be read. This may crash flatc given a mismatched schema.
    • -
  • --proto: Expect input files to be .proto files (protocol buffers). Output the corresponding .fbs file. Currently supports: package, message, enum. Does not support, but will skip without error: import, option. Does not support, will generate error: service, extend, extensions, oneof, group, custom options, nested declarations.
    • +
  • --proto: Expect input files to be .proto files (protocol buffers). Output the corresponding .fbs file. Currently supports: package, message, enum. Does not support, but will skip without error: import, option. Does not support, will generate error: service, extend, extensions, oneof, group, custom options, nested declarations.
    • +
  • --schema: Serialize schemas instead of JSON (use with -b). This will output a binary version of the specified schema that itself corresponds to the reflection/reflection.fbs schema. Loading this binary file is the basis for reflection functionality.
    • diff --git a/docs/html/md__cpp_usage.html b/docs/html/md__cpp_usage.html index 2217273def2..27cdd0b75bb 100644 --- a/docs/html/md__cpp_usage.html +++ b/docs/html/md__cpp_usage.html @@ -114,8 +114,14 @@

    Reading in C++

    We use the somewhat verbose term mutate instead of set to indicate that this is a special use case, not to be confused with the default way of constructing FlatBuffer data.

    After the above mutations, you can send on the FlatBuffer to a new recipient without any further work!

    Note that any mutate_ functions on tables return a bool, which is false if the field we're trying to set isn't present in the buffer. Fields are not present if they weren't set, or even if they happen to be equal to the default value. For example, in the creation code above we set the mana field to 150, which is the default value, so it was never stored in the buffer. Trying to call mutate_mana() on such data will return false, and the value won't actually be modified!

    -

    There's two ways around this. First, you can call ForceDefaults() on a FlatBufferBuilder to force all fields you set to actually be written. This of course increases the size of the buffer somewhat, but this may be acceptable for a mutable buffer.

    -

    Alternatively, you can use mutation functions that are able to insert fields and change the size of things. These functions are expensive however, since they need to resize the buffer and create new data.

    +

    One way to solve this is to call ForceDefaults() on a FlatBufferBuilder to force all fields you set to actually be written. This of course increases the size of the buffer somewhat, but this may be acceptable for a mutable buffer.

    +

    Alternatively, you can use the more powerful reflection functionality:

    +

    Reflection (& Resizing)

    +

    If the above ways of accessing a buffer are still too static for you, there is experimental support for reflection in FlatBuffers, allowing you to read and write data even if you don't know the exact format of a buffer, and even allows you to change sizes of strings and vectors in-place.

    +

    The way this works is very elegant, there is actually a FlatBuffer schema that describes schemas (!) which you can find in reflection/reflection.fbs. The compiler flatc can write out any schemas it has just parsed as a binary FlatBuffer, corresponding to this meta-schema.

    +

    Loading in one of these binary schemas at runtime allows you traverse any FlatBuffer data that corresponds to it without knowing the exact format. You can query what fields are present, and then read/write them after.

    +

    For convenient field manipulation, you can include the header flatbuffers/reflection.h which includes both the generated code from the meta schema, as well as a lot of helper functions.

    +

    And example of usage for the moment you can find in test.cpp/ReflectionTest().

    Storing maps / dictionaries in a FlatBuffer

    FlatBuffers doesn't support maps natively, but there is support to emulate their behavior with vectors and binary search, which means you can have fast lookups directly from a FlatBuffer without having to unpack your data into a std::map or similar.

    To use it:

      diff --git a/docs/source/Compiler.md b/docs/source/Compiler.md index 957baa5bbc2..643f19caae8 100755 --- a/docs/source/Compiler.md +++ b/docs/source/Compiler.md @@ -51,7 +51,8 @@ be generated for each file processed: - `--no-prefix` : Don't prefix enum values in generated C++ by their enum type. -- `--gen-includes` : Generate include statements for included schemas the +- `--gen-includes` : (deprecated), instead use: +- `--no-includes` : Don't generate include statements for included schemas the generated file depends on (C++). - `--gen-mutable` : Generate additional non-const accessors for mutating @@ -68,3 +69,8 @@ be generated for each file processed: Does not support, but will skip without error: `import`, `option`. Does not support, will generate error: `service`, `extend`, `extensions`, `oneof`, `group`, custom options, nested declarations. + +- `--schema`: Serialize schemas instead of JSON (use with -b). This will + output a binary version of the specified schema that itself corresponds + to the reflection/reflection.fbs schema. Loading this binary file is the + basis for reflection functionality. diff --git a/docs/source/CppUsage.md b/docs/source/CppUsage.md index 01576e20b4e..0786dcbf37f 100755 --- a/docs/source/CppUsage.md +++ b/docs/source/CppUsage.md @@ -203,14 +203,34 @@ to `150`, which is the default value, so it was never stored in the buffer. Trying to call mutate_mana() on such data will return false, and the value won't actually be modified! -There's two ways around this. First, you can call `ForceDefaults()` on a +One way to solve this is to call `ForceDefaults()` on a `FlatBufferBuilder` to force all fields you set to actually be written. This of course increases the size of the buffer somewhat, but this may be acceptable for a mutable buffer. -Alternatively, you can use mutation functions that are able to insert fields -and change the size of things. These functions are expensive however, since -they need to resize the buffer and create new data. +Alternatively, you can use the more powerful reflection functionality: + +### Reflection (& Resizing) + +If the above ways of accessing a buffer are still too static for you, there is +experimental support for reflection in FlatBuffers, allowing you to read and +write data even if you don't know the exact format of a buffer, and even allows +you to change sizes of strings and vectors in-place. + +The way this works is very elegant, there is actually a FlatBuffer schema that +describes schemas (!) which you can find in `reflection/reflection.fbs`. +The compiler `flatc` can write out any schemas it has just parsed as a binary +FlatBuffer, corresponding to this meta-schema. + +Loading in one of these binary schemas at runtime allows you traverse any +FlatBuffer data that corresponds to it without knowing the exact format. You +can query what fields are present, and then read/write them after. + +For convenient field manipulation, you can include the header +`flatbuffers/reflection.h` which includes both the generated code from the meta +schema, as well as a lot of helper functions. + +And example of usage for the moment you can find in `test.cpp/ReflectionTest()`. ### Storing maps / dictionaries in a FlatBuffer diff --git a/include/flatbuffers/flatbuffers.h b/include/flatbuffers/flatbuffers.h index 9502464a714..25cbad7bf38 100644 --- a/include/flatbuffers/flatbuffers.h +++ b/include/flatbuffers/flatbuffers.h @@ -754,7 +754,7 @@ class FlatBufferBuilder FLATBUFFERS_FINAL_CLASS { } template Offset>> CreateVectorOfSortedTables( - std::vector *v) { + std::vector> *v) { return CreateVectorOfSortedTables(v->data(), v->size()); } @@ -1017,6 +1017,8 @@ class Table { return true; } + uint8_t *GetVTable() { return data_ - ReadScalar(data_); } + bool CheckField(voffset_t field) const { return GetOptionalFieldOffset(field) != 0; } diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 016066f2765..d28fae2782e 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -25,6 +25,7 @@ #include "flatbuffers/flatbuffers.h" #include "flatbuffers/hash.h" +#include "flatbuffers/reflection.h" // This file defines the data types representing a parsed IDL (Interface // Definition Language) / schema file. @@ -126,6 +127,8 @@ struct Type { Type VectorType() const { return Type(element, struct_def, enum_def); } + Offset Serialize(FlatBufferBuilder *builder) const; + BaseType base_type; BaseType element; // only set if t == BASE_TYPE_VECTOR StructDef *struct_def; // only set if t or element == BASE_TYPE_STRUCT @@ -179,7 +182,8 @@ struct Namespace { // Base class for all definition types (fields, structs_, enums_). struct Definition { - Definition() : generated(false), defined_namespace(nullptr) {} + Definition() : generated(false), defined_namespace(nullptr), + serialized_location(0), index(-1) {} std::string name; std::string file; @@ -187,12 +191,19 @@ struct Definition { SymbolTable attributes; bool generated; // did we already output code for this definition? Namespace *defined_namespace; // Where it was defined. + + // For use with Serialize() + uoffset_t serialized_location; + int index; // Inside the vector it is stored. }; struct FieldDef : public Definition { FieldDef() : deprecated(false), required(false), key(false), padding(0), used(false) {} + Offset Serialize(FlatBufferBuilder *builder, uint16_t id) + const; + Value value; bool deprecated; // Field is allowed to be present in old data, but can't be // written in new data nor accessed in new code. @@ -218,6 +229,8 @@ struct StructDef : public Definition { if (fields.vec.size()) fields.vec.back()->padding = padding; } + Offset Serialize(FlatBufferBuilder *builder) const; + SymbolTable fields; bool fixed; // If it's struct, not a table. bool predecl; // If it's used before it was defined. @@ -243,6 +256,8 @@ struct EnumVal { EnumVal(const std::string &_name, int64_t _val) : name(_name), value(_val), struct_def(nullptr) {} + Offset Serialize(FlatBufferBuilder *builder) const; + std::string name; std::vector doc_comment; int64_t value; @@ -263,6 +278,8 @@ struct EnumDef : public Definition { return nullptr; } + Offset Serialize(FlatBufferBuilder *builder) const; + SymbolTable vals; bool is_union; Type underlying_type; @@ -271,7 +288,7 @@ struct EnumDef : public Definition { class Parser { public: Parser(bool strict_json = false, bool proto_mode = false) - : root_struct_def(nullptr), + : root_struct_def_(nullptr), source_(nullptr), cursor_(nullptr), line_(1), @@ -325,6 +342,10 @@ class Parser { std::set GetIncludedFilesRecursive( const std::string &file_name) const; + // Fills builder_ with a binary version of the schema parsed. + // See reflection/reflection.fbs + void Serialize(); + private: int64_t ParseHexNum(int nibbles); void Next(); @@ -363,7 +384,7 @@ class Parser { std::string error_; // User readable error_ if Parse() == false FlatBufferBuilder builder_; // any data contained in the file - StructDef *root_struct_def; + StructDef *root_struct_def_; std::string file_identifier_; std::string file_extension_; @@ -417,7 +438,7 @@ struct GeneratorOptions { output_default_scalars_in_json(false), indent_step(2), output_enum_identifiers(true), prefixed_enums(true), - include_dependence_headers(false), + include_dependence_headers(true), mutable_buffer(false), one_file(false), lang(GeneratorOptions::kJava) {} diff --git a/include/flatbuffers/reflection.h b/include/flatbuffers/reflection.h new file mode 100644 index 00000000000..475257a3101 --- /dev/null +++ b/include/flatbuffers/reflection.h @@ -0,0 +1,393 @@ +/* + * Copyright 2015 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_REFLECTION_H_ +#define FLATBUFFERS_REFLECTION_H_ + +#include "flatbuffers/util.h" + +// This is somewhat of a circular dependency because flatc (and thus this +// file) is needed to generate this header in the first place. +// Should normally not be a problem since it can be generated by the +// previous version of flatc whenever this code needs to change. +// See reflection/generate_code.sh +#include "flatbuffers/reflection_generated.h" + +// Helper functionality for reflection. + +namespace flatbuffers { + +inline size_t GetTypeSize(reflection::BaseType base_type) { + // This needs to correspond to the BaseType enum. + static size_t sizes[] = { 0, 1, 1, 1, 1, 2, 2, 4, 4, 8, 8, 4, 8, 4, 4, 4, 4 }; + return sizes[base_type]; +} + +// Get the root, regardless of what type it is. +inline Table *GetAnyRoot(uint8_t *flatbuf) { + return GetMutableRoot(flatbuf); +} +inline const Table *GetAnyRoot(const uint8_t *flatbuf) { + return GetRoot
      (flatbuf); +} + +// Get a field, if you know it's an integer, and its exact type. +template T GetFieldI(const Table *table, + const reflection::Field *field) { + assert(sizeof(T) == GetTypeSize(field->type()->base_type())); + return table->GetField(field->offset(), + static_cast(field->default_integer())); +} + +// Get a field, if you know it's floating point and its exact type. +template T GetFieldF(const Table *table, + const reflection::Field *field) { + assert(sizeof(T) == GetTypeSize(field->type()->base_type())); + return table->GetField(field->offset(), + static_cast(field->default_real())); +} + +// Get a field, if you know it's a string. +inline const String *GetFieldS(const Table *table, + const reflection::Field *field) { + assert(field->type()->base_type() == reflection::String); + return table->GetPointer(field->offset()); +} + +// Get a field, if you know it's a vector. +template const Vector *GetFieldV(const Table *table, + const reflection::Field *field) { + assert(field->type()->base_type() == reflection::Vector && + sizeof(T) == GetTypeSize(field->type()->element())); + return table->GetPointer *>(field->offset()); +} + +// Get any field as a 64bit int, regardless of what it is (bool/int/float/str). +inline int64_t GetAnyFieldI(const Table *table, + const reflection::Field *field) { +# define FLATBUFFERS_GET(C, T) \ + static_cast(GetField##C(table, field)) + switch (field->type()->base_type()) { + case reflection::UType: + case reflection::Bool: + case reflection::UByte: return FLATBUFFERS_GET(I, uint8_t); + case reflection::Byte: return FLATBUFFERS_GET(I, int8_t); + case reflection::Short: return FLATBUFFERS_GET(I, int16_t); + case reflection::UShort: return FLATBUFFERS_GET(I, uint16_t); + case reflection::Int: return FLATBUFFERS_GET(I, int32_t); + case reflection::UInt: return FLATBUFFERS_GET(I, uint32_t); + case reflection::Long: return FLATBUFFERS_GET(I, int64_t); + case reflection::ULong: return FLATBUFFERS_GET(I, uint64_t); + case reflection::Float: return FLATBUFFERS_GET(F, float); + case reflection::Double: return FLATBUFFERS_GET(F, double); + case reflection::String: return StringToInt( + GetFieldS(table, field)->c_str()); + default: return 0; + } +# undef FLATBUFFERS_GET +} + +// Get any field as a double, regardless of what it is (bool/int/float/str). +inline double GetAnyFieldF(const Table *table, + const reflection::Field *field) { + switch (field->type()->base_type()) { + case reflection::Float: return GetFieldF(table, field); + case reflection::Double: return GetFieldF(table, field); + case reflection::String: return strtod(GetFieldS(table, field)->c_str(), + nullptr); + default: return static_cast(GetAnyFieldI(table, field)); + } +} + +// Get any field as a string, regardless of what it is (bool/int/float/str). +inline std::string GetAnyFieldS(const Table *table, + const reflection::Field *field) { + switch (field->type()->base_type()) { + case reflection::Float: + case reflection::Double: return NumToString(GetAnyFieldF(table, field)); + case reflection::String: return GetFieldS(table, field)->c_str(); + // TODO: could return vector/table etc as JSON string. + default: return NumToString(GetAnyFieldI(table, field)); + } +} + +// Set any scalar field, if you know its exact type. +template bool SetField(Table *table, const reflection::Field *field, + T val) { + assert(sizeof(T) == GetTypeSize(field->type()->base_type())); + return table->SetField(field->offset(), val); +} + +// Set any field as a 64bit int, regardless of what it is (bool/int/float/str). +inline void SetAnyFieldI(Table *table, const reflection::Field *field, + int64_t val) { +# define FLATBUFFERS_SET(T) SetField(table, field, static_cast(val)) + switch (field->type()->base_type()) { + case reflection::UType: + case reflection::Bool: + case reflection::UByte: FLATBUFFERS_SET(uint8_t ); break; + case reflection::Byte: FLATBUFFERS_SET(int8_t ); break; + case reflection::Short: FLATBUFFERS_SET(int16_t ); break; + case reflection::UShort: FLATBUFFERS_SET(uint16_t ); break; + case reflection::Int: FLATBUFFERS_SET(int32_t ); break; + case reflection::UInt: FLATBUFFERS_SET(uint32_t ); break; + case reflection::Long: FLATBUFFERS_SET(int64_t ); break; + case reflection::ULong: FLATBUFFERS_SET(uint64_t ); break; + case reflection::Float: FLATBUFFERS_SET(float ); break; + case reflection::Double: FLATBUFFERS_SET(double ); break; + // TODO: support strings + default: break; + } +# undef FLATBUFFERS_SET +} + +// Set any field as a double, regardless of what it is (bool/int/float/str). +inline void SetAnyFieldF(Table *table, const reflection::Field *field, + double val) { + switch (field->type()->base_type()) { + case reflection::Float: SetField (table, field, + static_cast(val)); break; + case reflection::Double: SetField(table, field, val); break; + // TODO: support strings. + default: SetAnyFieldI(table, field, static_cast(val)); break; + } +} + +// Set any field as a string, regardless of what it is (bool/int/float/str). +inline void SetAnyFieldS(Table *table, const reflection::Field *field, + const char *val) { + switch (field->type()->base_type()) { + case reflection::Float: + case reflection::Double: SetAnyFieldF(table, field, strtod(val, nullptr)); + // TODO: support strings. + default: SetAnyFieldI(table, field, StringToInt(val)); break; + } +} + +// "smart" pointer for use with resizing vectors: turns a pointer inside +// a vector into a relative offset, such that it is not affected by resizes. +template class pointer_inside_vector { + public: + pointer_inside_vector(const T *ptr, const std::vector &vec) + : offset_(reinterpret_cast(ptr) - + reinterpret_cast(vec.data())), + vec_(vec) {} + + const T *operator*() const { + return reinterpret_cast( + reinterpret_cast(vec_.data()) + offset_); + } + const T *operator->() const { + return operator*(); + } + void operator=(const pointer_inside_vector &piv); + private: + size_t offset_; + const std::vector &vec_; +}; + +// Helper to create the above easily without specifying template args. +template pointer_inside_vector piv( + const T *ptr, const std::vector &vec) { + return pointer_inside_vector(ptr, vec); +} + +// Resize a FlatBuffer in-place by iterating through all offsets in the buffer +// and adjusting them by "delta" if they straddle the start offset. +// Once that is done, bytes can now be inserted/deleted safely. +// "delta" may be negative (shrinking). +// Unless "delta" is a multiple of the largest alignment, you'll create a small +// amount of garbage space in the buffer. +class ResizeContext { + public: + ResizeContext(const reflection::Schema &schema, uoffset_t start, int delta, + std::vector *flatbuf) + : schema_(schema), startptr_(flatbuf->data() + start), + delta_(delta), buf_(*flatbuf), + dag_check_(flatbuf->size() / sizeof(uoffset_t), false) { + auto mask = sizeof(largest_scalar_t) - 1; + delta_ = (delta_ + mask) & ~mask; + if (!delta_) return; // We can't shrink by less than largest_scalar_t. + // Now change all the offsets by delta_. + auto root = GetAnyRoot(buf_.data()); + Straddle(buf_.data(), root, buf_.data()); + ResizeTable(schema.root_table(), root); + // We can now add or remove bytes at start. + if (delta_ > 0) buf_.insert(buf_.begin() + start, delta_, 0); + else buf_.erase(buf_.begin() + start, buf_.begin() + start - delta_); + } + + // Check if the range between first (lower address) and second straddles + // the insertion point. If it does, change the offset at offsetloc (of + // type T, with direction D). + template void Straddle(void *first, void *second, + void *offsetloc) { + if (first <= startptr_ && second >= startptr_) { + WriteScalar(offsetloc, ReadScalar(offsetloc) + delta_ * D); + DagCheck(offsetloc) = true; + } + } + + // This returns a boolean that records if the corresponding offset location + // has been modified already. If so, we can't even read the corresponding + // offset, since it is pointing to a location that is illegal until the + // resize actually happens. + // This must be checked for every offset, since we can't know which offsets + // will straddle and which won't. + uint8_t &DagCheck(void *offsetloc) { + auto dag_idx = reinterpret_cast(offsetloc) - + reinterpret_cast(buf_.data()); + return dag_check_[dag_idx]; + } + + void ResizeTable(const reflection::Object *objectdef, Table *table) { + if (DagCheck(table)) + return; // Table already visited. + auto vtable = table->GetVTable(); + // Check if the vtable offset points beyond the insertion point. + Straddle(table, vtable, table); + // This direction shouldn't happen because vtables that sit before tables + // are always directly adjacent, but check just in case we ever change the + // way flatbuffers are built. + Straddle(vtable, table, table); + // Early out: since all fields inside the table must point forwards in + // memory, if the insertion point is before the table we can stop here. + auto tableloc = reinterpret_cast(table); + if (startptr_ <= tableloc) return; + // Check each field. + auto fielddefs = objectdef->fields(); + for (auto it = fielddefs->begin(); it != fielddefs->end(); ++it) { + auto fielddef = *it; + auto base_type = fielddef->type()->base_type(); + // Ignore scalars. + if (base_type <= reflection::Double) continue; + // Ignore fields that are not stored. + auto offset = table->GetOptionalFieldOffset(fielddef->offset()); + if (!offset) continue; + // Ignore structs. + auto subobjectdef = base_type == reflection::Obj ? + schema_.objects()->Get(fielddef->type()->index()) : nullptr; + if (subobjectdef && subobjectdef->is_struct()) continue; + // Get this fields' offset, and read it if safe. + auto offsetloc = tableloc + offset; + if (DagCheck(offsetloc)) + continue; // This offset already visited. + auto ref = offsetloc + ReadScalar(offsetloc); + Straddle(offsetloc, ref, offsetloc); + // Recurse. + switch (base_type) { + case reflection::Obj: { + ResizeTable(subobjectdef, reinterpret_cast
      (ref)); + break; + } + case reflection::Vector: { + if (fielddef->type()->element() != reflection::Obj) break; + auto vec = reinterpret_cast *>(ref); + auto elemobjectdef = + schema_.objects()->Get(fielddef->type()->index()); + if (elemobjectdef->is_struct()) break; + for (uoffset_t i = 0; i < vec->size(); i++) { + auto loc = vec->Data() + i * sizeof(uoffset_t); + if (DagCheck(loc)) + continue; // This offset already visited. + auto dest = loc + vec->Get(i); + Straddle(loc, dest ,loc); + ResizeTable(elemobjectdef, reinterpret_cast
      (dest)); + } + break; + } + case reflection::Union: { + auto enumdef = schema_.enums()->Get(fielddef->type()->index()); + // TODO: this is clumsy and slow, but no other way to find it? + auto type_field = fielddefs->LookupByKey( + (fielddef->name()->c_str() + std::string("_type")).c_str()); + assert(type_field); + auto union_type = GetFieldI(table, type_field); + auto enumval = enumdef->values()->LookupByKey(union_type); + ResizeTable(enumval->object(), reinterpret_cast
      (ref)); + break; + } + case reflection::String: + break; + default: + assert(false); + } + } + } + + void operator=(const ResizeContext &rc); + + private: + const reflection::Schema &schema_; + uint8_t *startptr_; + int delta_; + std::vector &buf_; + std::vector dag_check_; +}; + +// Changes the contents of a string inside a FlatBuffer. FlatBuffer must +// live inside a std::vector so we can resize the buffer if needed. +// "str" must live inside "flatbuf" and may be invalidated after this call. +inline void SetString(const reflection::Schema &schema, const std::string &val, + const String *str, std::vector *flatbuf) { + auto delta = static_cast(val.size()) - static_cast(str->Length()); + auto start = static_cast(reinterpret_cast(str) - + flatbuf->data() + + sizeof(uoffset_t)); + if (delta) { + // Different size, we must expand (or contract). + ResizeContext(schema, start, delta, flatbuf); + if (delta < 0) { + // Clear the old string, since we don't want parts of it remaining. + memset(flatbuf->data() + start, 0, str->Length()); + } + } + // Copy new data. Safe because we created the right amount of space. + memcpy(flatbuf->data() + start, val.c_str(), val.size() + 1); +} + +// Resizes a flatbuffers::Vector inside a FlatBuffer. FlatBuffer must +// live inside a std::vector so we can resize the buffer if needed. +// "vec" must live inside "flatbuf" and may be invalidated after this call. +template void ResizeVector(const reflection::Schema &schema, + uoffset_t newsize, T val, + const Vector *vec, + std::vector *flatbuf) { + auto delta_elem = static_cast(newsize) - static_cast(vec->size()); + auto delta_bytes = delta_elem * static_cast(sizeof(T)); + auto vec_start = reinterpret_cast(vec) - flatbuf->data(); + auto start = static_cast(vec_start + sizeof(uoffset_t) + + sizeof(T) * vec->size()); + if (delta_bytes) { + ResizeContext(schema, start, delta_bytes, flatbuf); + WriteScalar(flatbuf->data() + vec_start, newsize); // Length field. + // Set new elements to "val". + for (int i = 0; i < delta_elem; i++) { + auto loc = flatbuf->data() + start + i * sizeof(T); + auto is_scalar = std::is_scalar::value; + if (is_scalar) { + WriteScalar(loc, val); + } else { // struct + *reinterpret_cast(loc) = val; + } + } + } +} + + +} // namespace flatbuffers + +#endif // FLATBUFFERS_REFLECTION_H_ diff --git a/include/flatbuffers/reflection_generated.h b/include/flatbuffers/reflection_generated.h new file mode 100644 index 00000000000..dfcf3e1664d --- /dev/null +++ b/include/flatbuffers/reflection_generated.h @@ -0,0 +1,368 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +#ifndef FLATBUFFERS_GENERATED_REFLECTION_REFLECTION_H_ +#define FLATBUFFERS_GENERATED_REFLECTION_REFLECTION_H_ + +#include "flatbuffers/flatbuffers.h" + + +namespace reflection { + +struct Type; +struct EnumVal; +struct Enum; +struct Field; +struct Object; +struct Schema; + +enum BaseType { + None = 0, + UType = 1, + Bool = 2, + Byte = 3, + UByte = 4, + Short = 5, + UShort = 6, + Int = 7, + UInt = 8, + Long = 9, + ULong = 10, + Float = 11, + Double = 12, + String = 13, + Vector = 14, + Obj = 15, + Union = 16 +}; + +inline const char **EnumNamesBaseType() { + static const char *names[] = { "None", "UType", "Bool", "Byte", "UByte", "Short", "UShort", "Int", "UInt", "Long", "ULong", "Float", "Double", "String", "Vector", "Obj", "Union", nullptr }; + return names; +} + +inline const char *EnumNameBaseType(BaseType e) { return EnumNamesBaseType()[e]; } + +struct Type FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + BaseType base_type() const { return static_cast(GetField(4, 0)); } + BaseType element() const { return static_cast(GetField(6, 0)); } + int32_t index() const { return GetField(8, -1); } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyField(verifier, 4 /* base_type */) && + VerifyField(verifier, 6 /* element */) && + VerifyField(verifier, 8 /* index */) && + verifier.EndTable(); + } +}; + +struct TypeBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_base_type(BaseType base_type) { fbb_.AddElement(4, static_cast(base_type), 0); } + void add_element(BaseType element) { fbb_.AddElement(6, static_cast(element), 0); } + void add_index(int32_t index) { fbb_.AddElement(8, index, -1); } + TypeBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); } + TypeBuilder &operator=(const TypeBuilder &); + flatbuffers::Offset Finish() { + auto o = flatbuffers::Offset(fbb_.EndTable(start_, 3)); + return o; + } +}; + +inline flatbuffers::Offset CreateType(flatbuffers::FlatBufferBuilder &_fbb, + BaseType base_type = None, + BaseType element = None, + int32_t index = -1) { + TypeBuilder builder_(_fbb); + builder_.add_index(index); + builder_.add_element(element); + builder_.add_base_type(base_type); + return builder_.Finish(); +} + +struct EnumVal FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + const flatbuffers::String *name() const { return GetPointer(4); } + int64_t value() const { return GetField(6, 0); } + bool KeyCompareLessThan(const EnumVal *o) const { return value() < o->value(); } + int KeyCompareWithValue(int64_t val) const { return value() < val ? -1 : value() > val; } + const Object *object() const { return GetPointer(8); } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyFieldRequired(verifier, 4 /* name */) && + verifier.Verify(name()) && + VerifyField(verifier, 6 /* value */) && + VerifyField(verifier, 8 /* object */) && + verifier.VerifyTable(object()) && + verifier.EndTable(); + } +}; + +struct EnumValBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_name(flatbuffers::Offset name) { fbb_.AddOffset(4, name); } + void add_value(int64_t value) { fbb_.AddElement(6, value, 0); } + void add_object(flatbuffers::Offset object) { fbb_.AddOffset(8, object); } + EnumValBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); } + EnumValBuilder &operator=(const EnumValBuilder &); + flatbuffers::Offset Finish() { + auto o = flatbuffers::Offset(fbb_.EndTable(start_, 3)); + fbb_.Required(o, 4); // name + return o; + } +}; + +inline flatbuffers::Offset CreateEnumVal(flatbuffers::FlatBufferBuilder &_fbb, + flatbuffers::Offset name = 0, + int64_t value = 0, + flatbuffers::Offset object = 0) { + EnumValBuilder builder_(_fbb); + builder_.add_value(value); + builder_.add_object(object); + builder_.add_name(name); + return builder_.Finish(); +} + +struct Enum FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + const flatbuffers::String *name() const { return GetPointer(4); } + bool KeyCompareLessThan(const Enum *o) const { return *name() < *o->name(); } + int KeyCompareWithValue(const char *val) const { return strcmp(name()->c_str(), val); } + const flatbuffers::Vector> *values() const { return GetPointer> *>(6); } + uint8_t is_union() const { return GetField(8, 0); } + const Type *underlying_type() const { return GetPointer(10); } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyFieldRequired(verifier, 4 /* name */) && + verifier.Verify(name()) && + VerifyFieldRequired(verifier, 6 /* values */) && + verifier.Verify(values()) && + verifier.VerifyVectorOfTables(values()) && + VerifyField(verifier, 8 /* is_union */) && + VerifyFieldRequired(verifier, 10 /* underlying_type */) && + verifier.VerifyTable(underlying_type()) && + verifier.EndTable(); + } +}; + +struct EnumBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_name(flatbuffers::Offset name) { fbb_.AddOffset(4, name); } + void add_values(flatbuffers::Offset>> values) { fbb_.AddOffset(6, values); } + void add_is_union(uint8_t is_union) { fbb_.AddElement(8, is_union, 0); } + void add_underlying_type(flatbuffers::Offset underlying_type) { fbb_.AddOffset(10, underlying_type); } + EnumBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); } + EnumBuilder &operator=(const EnumBuilder &); + flatbuffers::Offset Finish() { + auto o = flatbuffers::Offset(fbb_.EndTable(start_, 4)); + fbb_.Required(o, 4); // name + fbb_.Required(o, 6); // values + fbb_.Required(o, 10); // underlying_type + return o; + } +}; + +inline flatbuffers::Offset CreateEnum(flatbuffers::FlatBufferBuilder &_fbb, + flatbuffers::Offset name = 0, + flatbuffers::Offset>> values = 0, + uint8_t is_union = 0, + flatbuffers::Offset underlying_type = 0) { + EnumBuilder builder_(_fbb); + builder_.add_underlying_type(underlying_type); + builder_.add_values(values); + builder_.add_name(name); + builder_.add_is_union(is_union); + return builder_.Finish(); +} + +struct Field FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + const flatbuffers::String *name() const { return GetPointer(4); } + bool KeyCompareLessThan(const Field *o) const { return *name() < *o->name(); } + int KeyCompareWithValue(const char *val) const { return strcmp(name()->c_str(), val); } + const Type *type() const { return GetPointer(6); } + uint16_t id() const { return GetField(8, 0); } + uint16_t offset() const { return GetField(10, 0); } + int64_t default_integer() const { return GetField(12, 0); } + double default_real() const { return GetField(14, 0.0); } + uint8_t deprecated() const { return GetField(16, 0); } + uint8_t required() const { return GetField(18, 0); } + uint8_t key() const { return GetField(20, 0); } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyFieldRequired(verifier, 4 /* name */) && + verifier.Verify(name()) && + VerifyFieldRequired(verifier, 6 /* type */) && + verifier.VerifyTable(type()) && + VerifyField(verifier, 8 /* id */) && + VerifyField(verifier, 10 /* offset */) && + VerifyField(verifier, 12 /* default_integer */) && + VerifyField(verifier, 14 /* default_real */) && + VerifyField(verifier, 16 /* deprecated */) && + VerifyField(verifier, 18 /* required */) && + VerifyField(verifier, 20 /* key */) && + verifier.EndTable(); + } +}; + +struct FieldBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_name(flatbuffers::Offset name) { fbb_.AddOffset(4, name); } + void add_type(flatbuffers::Offset type) { fbb_.AddOffset(6, type); } + void add_id(uint16_t id) { fbb_.AddElement(8, id, 0); } + void add_offset(uint16_t offset) { fbb_.AddElement(10, offset, 0); } + void add_default_integer(int64_t default_integer) { fbb_.AddElement(12, default_integer, 0); } + void add_default_real(double default_real) { fbb_.AddElement(14, default_real, 0.0); } + void add_deprecated(uint8_t deprecated) { fbb_.AddElement(16, deprecated, 0); } + void add_required(uint8_t required) { fbb_.AddElement(18, required, 0); } + void add_key(uint8_t key) { fbb_.AddElement(20, key, 0); } + FieldBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); } + FieldBuilder &operator=(const FieldBuilder &); + flatbuffers::Offset Finish() { + auto o = flatbuffers::Offset(fbb_.EndTable(start_, 9)); + fbb_.Required(o, 4); // name + fbb_.Required(o, 6); // type + return o; + } +}; + +inline flatbuffers::Offset CreateField(flatbuffers::FlatBufferBuilder &_fbb, + flatbuffers::Offset name = 0, + flatbuffers::Offset type = 0, + uint16_t id = 0, + uint16_t offset = 0, + int64_t default_integer = 0, + double default_real = 0.0, + uint8_t deprecated = 0, + uint8_t required = 0, + uint8_t key = 0) { + FieldBuilder builder_(_fbb); + builder_.add_default_real(default_real); + builder_.add_default_integer(default_integer); + builder_.add_type(type); + builder_.add_name(name); + builder_.add_offset(offset); + builder_.add_id(id); + builder_.add_key(key); + builder_.add_required(required); + builder_.add_deprecated(deprecated); + return builder_.Finish(); +} + +struct Object FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + const flatbuffers::String *name() const { return GetPointer(4); } + bool KeyCompareLessThan(const Object *o) const { return *name() < *o->name(); } + int KeyCompareWithValue(const char *val) const { return strcmp(name()->c_str(), val); } + const flatbuffers::Vector> *fields() const { return GetPointer> *>(6); } + uint8_t is_struct() const { return GetField(8, 0); } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyFieldRequired(verifier, 4 /* name */) && + verifier.Verify(name()) && + VerifyFieldRequired(verifier, 6 /* fields */) && + verifier.Verify(fields()) && + verifier.VerifyVectorOfTables(fields()) && + VerifyField(verifier, 8 /* is_struct */) && + verifier.EndTable(); + } +}; + +struct ObjectBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_name(flatbuffers::Offset name) { fbb_.AddOffset(4, name); } + void add_fields(flatbuffers::Offset>> fields) { fbb_.AddOffset(6, fields); } + void add_is_struct(uint8_t is_struct) { fbb_.AddElement(8, is_struct, 0); } + ObjectBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); } + ObjectBuilder &operator=(const ObjectBuilder &); + flatbuffers::Offset Finish() { + auto o = flatbuffers::Offset(fbb_.EndTable(start_, 3)); + fbb_.Required(o, 4); // name + fbb_.Required(o, 6); // fields + return o; + } +}; + +inline flatbuffers::Offset CreateObject(flatbuffers::FlatBufferBuilder &_fbb, + flatbuffers::Offset name = 0, + flatbuffers::Offset>> fields = 0, + uint8_t is_struct = 0) { + ObjectBuilder builder_(_fbb); + builder_.add_fields(fields); + builder_.add_name(name); + builder_.add_is_struct(is_struct); + return builder_.Finish(); +} + +struct Schema FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + const flatbuffers::Vector> *objects() const { return GetPointer> *>(4); } + const flatbuffers::Vector> *enums() const { return GetPointer> *>(6); } + const flatbuffers::String *file_ident() const { return GetPointer(8); } + const flatbuffers::String *file_ext() const { return GetPointer(10); } + const Object *root_table() const { return GetPointer(12); } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyFieldRequired(verifier, 4 /* objects */) && + verifier.Verify(objects()) && + verifier.VerifyVectorOfTables(objects()) && + VerifyFieldRequired(verifier, 6 /* enums */) && + verifier.Verify(enums()) && + verifier.VerifyVectorOfTables(enums()) && + VerifyField(verifier, 8 /* file_ident */) && + verifier.Verify(file_ident()) && + VerifyField(verifier, 10 /* file_ext */) && + verifier.Verify(file_ext()) && + VerifyField(verifier, 12 /* root_table */) && + verifier.VerifyTable(root_table()) && + verifier.EndTable(); + } +}; + +struct SchemaBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_objects(flatbuffers::Offset>> objects) { fbb_.AddOffset(4, objects); } + void add_enums(flatbuffers::Offset>> enums) { fbb_.AddOffset(6, enums); } + void add_file_ident(flatbuffers::Offset file_ident) { fbb_.AddOffset(8, file_ident); } + void add_file_ext(flatbuffers::Offset file_ext) { fbb_.AddOffset(10, file_ext); } + void add_root_table(flatbuffers::Offset root_table) { fbb_.AddOffset(12, root_table); } + SchemaBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); } + SchemaBuilder &operator=(const SchemaBuilder &); + flatbuffers::Offset Finish() { + auto o = flatbuffers::Offset(fbb_.EndTable(start_, 5)); + fbb_.Required(o, 4); // objects + fbb_.Required(o, 6); // enums + return o; + } +}; + +inline flatbuffers::Offset CreateSchema(flatbuffers::FlatBufferBuilder &_fbb, + flatbuffers::Offset>> objects = 0, + flatbuffers::Offset>> enums = 0, + flatbuffers::Offset file_ident = 0, + flatbuffers::Offset file_ext = 0, + flatbuffers::Offset root_table = 0) { + SchemaBuilder builder_(_fbb); + builder_.add_root_table(root_table); + builder_.add_file_ext(file_ext); + builder_.add_file_ident(file_ident); + builder_.add_enums(enums); + builder_.add_objects(objects); + return builder_.Finish(); +} + +inline const reflection::Schema *GetSchema(const void *buf) { return flatbuffers::GetRoot(buf); } + +inline bool VerifySchemaBuffer(flatbuffers::Verifier &verifier) { return verifier.VerifyBuffer(); } + +inline const char *SchemaIdentifier() { return "BFBS"; } + +inline bool SchemaBufferHasIdentifier(const void *buf) { return flatbuffers::BufferHasIdentifier(buf, SchemaIdentifier()); } + +inline const char *SchemaExtension() { return "bfbs"; } + +inline void FinishSchemaBuffer(flatbuffers::FlatBufferBuilder &fbb, flatbuffers::Offset root) { fbb.Finish(root, SchemaIdentifier()); } + +} // namespace reflection + +#endif // FLATBUFFERS_GENERATED_REFLECTION_REFLECTION_H_ diff --git a/reflection/generate_code.sh b/reflection/generate_code.sh new file mode 100644 index 00000000000..90f655d22f1 --- /dev/null +++ b/reflection/generate_code.sh @@ -0,0 +1 @@ +../flatc -c --no-prefix -o ../include/flatbuffers reflection.fbs diff --git a/reflection/reflection.fbs b/reflection/reflection.fbs new file mode 100644 index 00000000000..b2a262a3be2 --- /dev/null +++ b/reflection/reflection.fbs @@ -0,0 +1,79 @@ +// This schema defines objects that represent a parsed schema, like +// the binary version of a .fbs file. +// This could be used to operate on unknown FlatBuffers at runtime. +// It can even ... represent itself (!) + +namespace reflection; + +// These must correspond to the enum in idl.h. +enum BaseType : byte { + None, + UType, + Bool, + Byte, + UByte, + Short, + UShort, + Int, + UInt, + Long, + ULong, + Float, + Double, + String, + Vector, + Obj, // Used for tables & structs. + Union // +} + +table Type { + base_type:BaseType; + element:BaseType = None; // Only if base_type == Vector. + index:int = -1; // If base_type == Object, index into "objects" below. + // If base_type == Union, UnionType, or integral derived + // from an enum, index into "enums" below. +} + +table EnumVal { + name:string (required); + value:long (key); + object:Object; // Only if part of a union. +} + +table Enum { + name:string (required, key); + values:[EnumVal] (required); // In order of their values. + is_union:bool = false; + underlying_type:Type (required); +} + +table Field { + name:string (required, key); + type:Type (required); + id:ushort; + offset:ushort; // Offset into the vtable for tables, or into the struct. + default_integer:long = 0; + default_real:double = 0.0; + deprecated:bool = false; + required:bool = false; + key:bool = false; +} + +table Object { // Used for both tables and structs. + name:string (required, key); + fields:[Field] (required); // Sorted. + is_struct:bool = false; +} + +table Schema { + objects:[Object] (required); // Sorted. + enums:[Enum] (required); // Sorted. + file_ident:string; + file_ext:string; + root_table:Object; +} + +root_type Schema; + +file_identifier "BFBS"; +file_extension "bfbs"; diff --git a/src/flatc.cpp b/src/flatc.cpp index 87f0c72bf5e..7dcfec79fd7 100755 --- a/src/flatc.cpp +++ b/src/flatc.cpp @@ -90,13 +90,15 @@ static void Error(const std::string &err, bool usage, bool show_exe_name) { " --defaults-json Output fields whose value is the default when\n" " writing JSON\n" " --no-prefix Don\'t prefix enum values with the enum type in C++.\n" - " --gen-includes Generate include statements for included schemas the\n" - " generated file depends on (C++).\n" + " --gen-includes (deprecated), instead use:\n" + " --no-includes Don\'t generate include statements for included\n" + " schemas the generated file depends on (C++).\n" " --gen-mutable Generate accessors that can mutate buffers in-place.\n" " --gen-onefile Generate single output file for C#\n" " --raw-binary Allow binaries without file_indentifier to be read.\n" " This may crash flatc given a mismatched schema.\n" " --proto Input is a .proto, translate to .fbs.\n" + " --schema Serialize schemas instead of JSON (use with -b)\n" "FILEs may depend on declarations in earlier files.\n" "FILEs after the -- must be binary flatbuffer format files.\n" "Output files are named using the base file name of the input,\n" @@ -117,6 +119,7 @@ int main(int argc, const char *argv[]) { bool print_make_rules = false; bool proto_mode = false; bool raw_binary = false; + bool schema_binary = false; std::vector filenames; std::vector include_directories; size_t binary_files_from = std::numeric_limits::max(); @@ -140,7 +143,10 @@ int main(int argc, const char *argv[]) { } else if(arg == "--gen-mutable") { opts.mutable_buffer = true; } else if(arg == "--gen-includes") { - opts.include_dependence_headers = true; + // Deprecated, remove this option some time in the future. + printf("warning: --gen-includes is deprecated (it is now default)\n"); + } else if(arg == "--no-includes") { + opts.include_dependence_headers = false; } else if (arg == "--gen-onefile") { opts.one_file = true; } else if (arg == "--raw-binary") { @@ -150,6 +156,8 @@ int main(int argc, const char *argv[]) { } else if(arg == "--proto") { proto_mode = true; any_generator = true; + } else if(arg == "--schema") { + schema_binary = true; } else if(arg == "-M") { print_make_rules = true; } else { @@ -216,6 +224,10 @@ int main(int argc, const char *argv[]) { if (!parser.Parse(contents.c_str(), &include_directories[0], file_it->c_str())) Error(parser.error_, false, false); + if (schema_binary) { + parser.Serialize(); + parser.file_extension_ = reflection::SchemaExtension(); + } include_directories.pop_back(); include_directories.pop_back(); } diff --git a/src/idl_gen_cpp.cpp b/src/idl_gen_cpp.cpp index 23e3db40578..2cb15b56c33 100644 --- a/src/idl_gen_cpp.cpp +++ b/src/idl_gen_cpp.cpp @@ -701,8 +701,8 @@ std::string GenerateCPP(const Parser &parser, code += enum_code_post; // Generate convenient global helper functions: - if (parser.root_struct_def) { - auto &name = parser.root_struct_def->name; + if (parser.root_struct_def_) { + auto &name = parser.root_struct_def_->name; std::string qualified_name = parser.GetFullyQualifiedName(name); std::string cpp_qualified_name = TranslateNameSpace(qualified_name); diff --git a/src/idl_gen_general.cpp b/src/idl_gen_general.cpp index 00c50fe85b1..9a2132338eb 100644 --- a/src/idl_gen_general.cpp +++ b/src/idl_gen_general.cpp @@ -527,7 +527,7 @@ static void GenStruct(const LanguageParameters &lang, const Parser &parser, code += ") + _bb."; code += lang.get_bb_position; code += ", _bb)); }\n"; - if (parser.root_struct_def == &struct_def) { + if (parser.root_struct_def_ == &struct_def) { if (parser.file_identifier_.length()) { // Check if a buffer has the identifier. code += " public static "; @@ -842,7 +842,7 @@ static void GenStruct(const LanguageParameters &lang, const Parser &parser, } } code += " return o;\n }\n"; - if (parser.root_struct_def == &struct_def) { + if (parser.root_struct_def_ == &struct_def) { code += " public static void "; code += FunctionStart(lang, 'F') + "inish" + struct_def.name; code += "Buffer(FlatBufferBuilder builder, int offset) { "; @@ -878,11 +878,13 @@ static bool SaveClass(const LanguageParameters &lang, const Parser &parser, EnsureDirExists(namespace_dir); std::string code = "// automatically generated, do not modify\n\n"; - code += lang.namespace_ident + namespace_general + lang.namespace_begin; - code += "\n\n"; + if (!namespace_general.empty()) { + code += lang.namespace_ident + namespace_general + lang.namespace_begin; + code += "\n\n"; + } if (needs_includes) code += lang.includes; code += classcode; - code += lang.namespace_end; + if (!namespace_general.empty()) code += lang.namespace_end; auto filename = namespace_dir + defname + lang.file_extension; return SaveFile(filename.c_str(), code, false); } @@ -1007,7 +1009,7 @@ std::string BinaryMakeRule(const Parser &parser, std::string make_rule = BinaryFileName(parser, path, filebase) + ": " + file_name; auto included_files = parser.GetIncludedFilesRecursive( - parser.root_struct_def->file); + parser.root_struct_def_->file); for (auto it = included_files.begin(); it != included_files.end(); ++it) { make_rule += " " + *it; diff --git a/src/idl_gen_go.cpp b/src/idl_gen_go.cpp index 74e26b7a98e..6f256bd11b1 100644 --- a/src/idl_gen_go.cpp +++ b/src/idl_gen_go.cpp @@ -677,7 +677,7 @@ bool GenerateGo(const Parser &parser, for (auto it = parser.structs_.vec.begin(); it != parser.structs_.vec.end(); ++it) { std::string declcode; - go::GenStruct(**it, &declcode, parser.root_struct_def); + go::GenStruct(**it, &declcode, parser.root_struct_def_); if (!go::SaveType(parser, **it, declcode, path, true)) return false; } diff --git a/src/idl_gen_python.cpp b/src/idl_gen_python.cpp index 46bf40e5644..c03ac937bf2 100644 --- a/src/idl_gen_python.cpp +++ b/src/idl_gen_python.cpp @@ -651,7 +651,7 @@ bool GeneratePython(const Parser &parser, for (auto it = parser.structs_.vec.begin(); it != parser.structs_.vec.end(); ++it) { std::string declcode; - python::GenStruct(**it, &declcode, parser.root_struct_def); + python::GenStruct(**it, &declcode, parser.root_struct_def_); if (!python::SaveType(parser, **it, declcode, path, true)) return false; } diff --git a/src/idl_gen_text.cpp b/src/idl_gen_text.cpp index 848aac59544..138f7320a52 100644 --- a/src/idl_gen_text.cpp +++ b/src/idl_gen_text.cpp @@ -270,9 +270,9 @@ static void GenStruct(const StructDef &struct_def, const Table *table, void GenerateText(const Parser &parser, const void *flatbuffer, const GeneratorOptions &opts, std::string *_text) { std::string &text = *_text; - assert(parser.root_struct_def); // call SetRootType() + assert(parser.root_struct_def_); // call SetRootType() text.reserve(1024); // Reduce amount of inevitable reallocs. - GenStruct(*parser.root_struct_def, + GenStruct(*parser.root_struct_def_, GetRoot
      (flatbuffer), 0, opts, @@ -289,7 +289,7 @@ bool GenerateTextFile(const Parser &parser, const std::string &path, const std::string &file_name, const GeneratorOptions &opts) { - if (!parser.builder_.GetSize() || !parser.root_struct_def) return true; + if (!parser.builder_.GetSize() || !parser.root_struct_def_) return true; std::string text; GenerateText(parser, parser.builder_.GetBufferPointer(), opts, &text); @@ -302,12 +302,12 @@ std::string TextMakeRule(const Parser &parser, const std::string &path, const std::string &file_name, const GeneratorOptions & /*opts*/) { - if (!parser.builder_.GetSize() || !parser.root_struct_def) return ""; + if (!parser.builder_.GetSize() || !parser.root_struct_def_) return ""; std::string filebase = flatbuffers::StripPath( flatbuffers::StripExtension(file_name)); std::string make_rule = TextFileName(path, filebase) + ": " + file_name; auto included_files = parser.GetIncludedFilesRecursive( - parser.root_struct_def->file); + parser.root_struct_def_->file); for (auto it = included_files.begin(); it != included_files.end(); ++it) { make_rule += " " + *it; diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index 0d0f7defe19..2ed401b777a 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -17,8 +17,6 @@ #include #include -#include "flatbuffers/flatbuffers.h" -#include "flatbuffers/hash.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" @@ -39,6 +37,12 @@ const char kTypeSizes[] = { #undef FLATBUFFERS_TD }; +// The enums in the reflection schema should match the ones we use internally. +// Compare the last element to check if these go out of sync. +static_assert(BASE_TYPE_UNION == + static_cast(reflection::Union), + "enums don't match"); + static void Error(const std::string &msg) { throw msg; } @@ -198,7 +202,7 @@ void Parser::Next() { } // fall thru default: - if (isalpha(static_cast(c))) { + if (isalpha(static_cast(c)) || c == '_') { // Collect all chars of an identifier: const char *start = cursor_ - 1; while (isalnum(static_cast(*cursor_)) || @@ -981,8 +985,8 @@ void Parser::ParseDecl() { } bool Parser::SetRootType(const char *name) { - root_struct_def = structs_.Lookup(GetFullyQualifiedName(name)); - return root_struct_def != nullptr; + root_struct_def_ = structs_.Lookup(GetFullyQualifiedName(name)); + return root_struct_def_ != nullptr; } std::string Parser::GetFullyQualifiedName(const std::string &name) const { @@ -1200,11 +1204,11 @@ bool Parser::Parse(const char *source, const char **include_paths, } else if (token_ == kTokenNameSpace) { ParseNamespace(); } else if (token_ == '{') { - if (!root_struct_def) Error("no root type set to parse json with"); + if (!root_struct_def_) Error("no root type set to parse json with"); if (builder_.GetSize()) { Error("cannot have more than one json object in a file"); } - builder_.Finish(Offset
      (ParseTable(*root_struct_def)), + builder_.Finish(Offset
      (ParseTable(*root_struct_def_)), file_identifier_.length() ? file_identifier_.c_str() : nullptr); } else if (token_ == kTokenEnum) { ParseEnum(false); @@ -1216,7 +1220,7 @@ bool Parser::Parse(const char *source, const char **include_paths, Expect(kTokenIdentifier); if (!SetRootType(root_type.c_str())) Error("unknown root type: " + root_type); - if (root_struct_def->fixed) + if (root_struct_def_->fixed) Error("root type must be a table"); Expect(';'); } else if (token_ == kTokenFileIdentifier) { @@ -1302,4 +1306,105 @@ std::set Parser::GetIncludedFilesRecursive( return included_files; } +// Schema serialization functionality: + +template void AssignIndices(const std::vector &defvec) { + // Pre-sort these vectors, such that we can set the correct indices for them. + auto vec = defvec; + std::sort(vec.begin(), vec.end(), + [](const T *a, const T *b) { return a->name < b->name; }); + for (int i = 0; i < static_cast(vec.size()); i++) vec[i]->index = i; +} + +void Parser::Serialize() { + builder_.Clear(); + AssignIndices(structs_.vec); + AssignIndices(enums_.vec); + std::vector> object_offsets; + for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) { + auto offset = (*it)->Serialize(&builder_); + object_offsets.push_back(offset); + (*it)->serialized_location = offset.o; + } + std::vector> enum_offsets; + for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) { + auto offset = (*it)->Serialize(&builder_); + enum_offsets.push_back(offset); + (*it)->serialized_location = offset.o; + } + auto schema_offset = reflection::CreateSchema( + builder_, + builder_.CreateVectorOfSortedTables(&object_offsets), + builder_.CreateVectorOfSortedTables(&enum_offsets), + builder_.CreateString(file_identifier_), + builder_.CreateString(file_extension_), + root_struct_def_->serialized_location); + builder_.Finish(schema_offset, reflection::SchemaIdentifier()); +} + +Offset StructDef::Serialize(FlatBufferBuilder *builder) + const { + std::vector> field_offsets; + for (auto it = fields.vec.begin(); it != fields.vec.end(); ++it) { + field_offsets.push_back( + (*it)->Serialize(builder, + static_cast(it - fields.vec.begin()))); + } + return reflection::CreateObject(*builder, + builder->CreateString(name), + builder->CreateVectorOfSortedTables( + &field_offsets), + fixed); +} + +Offset FieldDef::Serialize(FlatBufferBuilder *builder, + uint16_t id) const { + return reflection::CreateField(*builder, + builder->CreateString(name), + value.type.Serialize(builder), + id, + value.offset, + IsInteger(value.type.base_type) + ? StringToInt(value.constant.c_str()) + : 0, + IsFloat(value.type.base_type) + ? strtod(value.constant.c_str(), nullptr) + : 0.0, + deprecated, + required, + key); + // TODO: value.constant is almost always "0", we could save quite a bit of + // space by sharing it. Same for common values of value.type. +} + +Offset EnumDef::Serialize(FlatBufferBuilder *builder) const { + std::vector> enumval_offsets; + for (auto it = vals.vec.begin(); it != vals.vec.end(); ++it) { + enumval_offsets.push_back((*it)->Serialize(builder)); + } + return reflection::CreateEnum(*builder, + builder->CreateString(name), + builder->CreateVector(enumval_offsets), + is_union, + underlying_type.Serialize(builder)); +} + +Offset EnumVal::Serialize(FlatBufferBuilder *builder) const + { + return reflection::CreateEnumVal(*builder, + builder->CreateString(name), + value, + struct_def + ? struct_def->serialized_location + : 0); +} + +Offset Type::Serialize(FlatBufferBuilder *builder) const { + return reflection::CreateType(*builder, + static_cast(base_type), + static_cast(element), + struct_def ? struct_def->index : + (enum_def ? enum_def->index : -1)); +} + } // namespace flatbuffers diff --git a/tests/generate_code.sh b/tests/generate_code.sh new file mode 100644 index 00000000000..c0ec1d78128 --- /dev/null +++ b/tests/generate_code.sh @@ -0,0 +1,2 @@ +../flatc -c -j -n -g -b --gen-mutable monster_test.fbs monsterdata_test.json +../flatc -b --schema monster_test.fbs diff --git a/tests/monster_test.bfbs b/tests/monster_test.bfbs new file mode 100644 index 00000000000..f72d036b5d0 Binary files /dev/null and b/tests/monster_test.bfbs differ diff --git a/tests/test.cpp b/tests/test.cpp index dc80e7156ff..9020125f8b8 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -14,6 +14,8 @@ * limitations under the License. */ +#define FLATBUFFERS_DEBUG_VERIFICATION_FAILURE 1 + #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" @@ -36,20 +38,33 @@ using namespace MyGame::Example; int testing_fails = 0; +void TestFail(const char *expval, const char *val, const char *exp, + const char *file, int line) { + TEST_OUTPUT_LINE("TEST FAILED: %s:%d, %s (%s) != %s", file, line, + exp, expval, val); + assert(0); + testing_fails++; +} + +void TestEqStr(const char *expval, const char *val, const char *exp, + const char *file, int line) { + if (strcmp(expval, val) != 0) { + TestFail(expval, val, exp, file, line); + } +} + template void TestEq(T expval, U val, const char *exp, const char *file, int line) { if (U(expval) != val) { - auto expval_str = flatbuffers::NumToString(expval); - auto val_str = flatbuffers::NumToString(val); - TEST_OUTPUT_LINE("TEST FAILED: %s:%d, %s (%s) != %s", file, line, - exp, expval_str.c_str(), val_str.c_str()); - assert(0); - testing_fails++; + TestFail(flatbuffers::NumToString(expval).c_str(), + flatbuffers::NumToString(val).c_str(), + exp, file, line); } } #define TEST_EQ(exp, val) TestEq(exp, val, #exp, __FILE__, __LINE__) #define TEST_NOTNULL(exp) TestEq(exp == NULL, false, #exp, __FILE__, __LINE__) +#define TEST_EQ_STR(exp, val) TestEqStr(exp, val, #exp, __FILE__, __LINE__) // Include simple random number generator to ensure results will be the // same cross platform. @@ -143,7 +158,7 @@ void AccessFlatBufferTest(const uint8_t *flatbuf, size_t length) { TEST_EQ(monster->hp(), 80); TEST_EQ(monster->mana(), 150); // default - TEST_EQ(strcmp(monster->name()->c_str(), "MyMonster"), 0); + TEST_EQ_STR(monster->name()->c_str(), "MyMonster"); // Can't access the following field, it is deprecated in the schema, // which means accessors are not generated: // monster.friendly() @@ -167,22 +182,22 @@ void AccessFlatBufferTest(const uint8_t *flatbuf, size_t length) { TEST_EQ(monster->test_type(), Any_Monster); // First make sure which it is. auto monster2 = reinterpret_cast(monster->test()); TEST_NOTNULL(monster2); - TEST_EQ(strcmp(monster2->name()->c_str(), "Fred"), 0); + TEST_EQ_STR(monster2->name()->c_str(), "Fred"); // Example of accessing a vector of strings: auto vecofstrings = monster->testarrayofstring(); TEST_EQ(vecofstrings->Length(), 2U); - TEST_EQ(strcmp(vecofstrings->Get(0)->c_str(), "bob"), 0); - TEST_EQ(strcmp(vecofstrings->Get(1)->c_str(), "fred"), 0); + TEST_EQ_STR(vecofstrings->Get(0)->c_str(), "bob"); + TEST_EQ_STR(vecofstrings->Get(1)->c_str(), "fred"); // Example of accessing a vector of tables: auto vecoftables = monster->testarrayoftables(); TEST_EQ(vecoftables->Length(), 3U); for (auto it = vecoftables->begin(); it != vecoftables->end(); ++it) TEST_EQ(strlen(it->name()->c_str()) >= 4, true); - TEST_EQ(strcmp(vecoftables->Get(0)->name()->c_str(), "Barney"), 0); - TEST_EQ(strcmp(vecoftables->Get(1)->name()->c_str(), "Fred"), 0); - TEST_EQ(strcmp(vecoftables->Get(2)->name()->c_str(), "Wilma"), 0); + TEST_EQ_STR(vecoftables->Get(0)->name()->c_str(), "Barney"); + TEST_EQ_STR(vecoftables->Get(1)->name()->c_str(), "Fred"); + TEST_EQ_STR(vecoftables->Get(2)->name()->c_str(), "Wilma"); TEST_NOTNULL(vecoftables->LookupByKey("Barney")); TEST_NOTNULL(vecoftables->LookupByKey("Fred")); TEST_NOTNULL(vecoftables->LookupByKey("Wilma")); @@ -275,6 +290,86 @@ void ParseAndGenerateTextTest() { } } +void ReflectionTest(uint8_t *flatbuf, size_t length) { + // Load a binary schema. + std::string bfbsfile; + TEST_EQ(flatbuffers::LoadFile( + "tests/monster_test.bfbs", true, &bfbsfile), true); + + // Verify it, just in case: + flatbuffers::Verifier verifier( + reinterpret_cast(bfbsfile.c_str()), bfbsfile.length()); + TEST_EQ(reflection::VerifySchemaBuffer(verifier), true); + + // Make sure the schema is what we expect it to be. + auto schema = reflection::GetSchema(bfbsfile.c_str()); + auto root_table = schema->root_table(); + TEST_EQ_STR(root_table->name()->c_str(), "Monster"); + auto fields = root_table->fields(); + auto hp_field = fields->LookupByKey("hp"); + TEST_NOTNULL(hp_field); + TEST_EQ_STR(hp_field->name()->c_str(), "hp"); + TEST_EQ(hp_field->id(), 2); + TEST_EQ(hp_field->type()->base_type(), reflection::Short); + + // Now use it to dynamically access a buffer. + auto root = flatbuffers::GetAnyRoot(flatbuf); + auto hp = flatbuffers::GetFieldI(root, hp_field); + TEST_EQ(hp, 80); + + // Rather than needing to know the type, we can also get the value of + // any field as an int64_t/double/string, regardless of what it actually is. + auto hp_int64 = flatbuffers::GetAnyFieldI(root, hp_field); + TEST_EQ(hp_int64, 80); + auto hp_double = flatbuffers::GetAnyFieldF(root, hp_field); + TEST_EQ(hp_double, 80.0); + auto hp_string = flatbuffers::GetAnyFieldS(root, hp_field); + TEST_EQ_STR(hp_string.c_str(), "80"); + + // We can also modify it. + flatbuffers::SetField(root, hp_field, 200); + hp = flatbuffers::GetFieldI(root, hp_field); + TEST_EQ(hp, 200); + + // We can also set fields generically: + flatbuffers::SetAnyFieldI(root, hp_field, 300); + hp_int64 = flatbuffers::GetAnyFieldI(root, hp_field); + TEST_EQ(hp_int64, 300); + flatbuffers::SetAnyFieldF(root, hp_field, 300.5); + hp_int64 = flatbuffers::GetAnyFieldI(root, hp_field); + TEST_EQ(hp_int64, 300); + flatbuffers::SetAnyFieldS(root, hp_field, "300"); + hp_int64 = flatbuffers::GetAnyFieldI(root, hp_field); + TEST_EQ(hp_int64, 300); + + // Reset it, for further tests. + flatbuffers::SetField(root, hp_field, 80); + + // More advanced functionality: changing the size of items in-line! + // First we put the FlatBuffer inside an std::vector. + std::vector resizingbuf(flatbuf, flatbuf + length); + // Find the field we want to modify. + auto name_field = fields->LookupByKey("name"); + // Get the root. + // This time we wrap the result from GetAnyRoot in a smartpointer that + // will keep rroot valid as resizingbuf resizes. + auto rroot = flatbuffers::piv(flatbuffers::GetAnyRoot(resizingbuf.data()), + resizingbuf); + SetString(*schema, "totally new string", GetFieldS(*rroot, name_field), + &resizingbuf); + // Here resizingbuf has changed, but rroot is still valid. + TEST_EQ_STR(GetFieldS(*rroot, name_field)->c_str(), "totally new string"); + // Now lets extend a vector by 100 elements (10 -> 110). + auto inventory_field = fields->LookupByKey("inventory"); + auto rinventory = flatbuffers::piv( + flatbuffers::GetFieldV(*rroot, inventory_field), + resizingbuf); + flatbuffers::ResizeVector(*schema, 110, 50, *rinventory, + &resizingbuf); + // rinventory still valid, so lets read from it. + TEST_EQ(rinventory->Get(10), 50); +} + // Parse a .proto schema, output as .fbs void ParseProtoTest() { // load the .proto and the golden file from disk @@ -291,6 +386,7 @@ void ParseProtoTest() { // Generate fbs. flatbuffers::GeneratorOptions opts; + opts.include_dependence_headers = false; auto fbs = flatbuffers::GenerateFBS(parser, "test", opts); // Ensure generated file is parsable. @@ -666,6 +762,7 @@ int main(int /*argc*/, const char * /*argv*/[]) { #ifndef FLATBUFFERS_NO_FILE_TESTS ParseAndGenerateTextTest(); + ReflectionTest(flatbuf.get(), rawbuf.length()); ParseProtoTest(); #endif