Merge pull request #2 from amoldeshpande/master

merge

.gitignore

@@ -42,6 +42,7 @@ tests/monsterdata_go_wire.mon
 CMakeLists.txt.user
 CMakeScripts/**
 CTestTestfile.cmake
+FlatBuffers.cbp
 build/Xcode/FlatBuffers.xcodeproj/project.xcworkspace/**
 build/Xcode/FlatBuffers.xcodeproj/xcuserdata/**
 FlatBuffers.xcodeproj/

CMakeLists.txt

@@ -20,6 +20,8 @@ set(FlatBuffers_Compiler_SRCS
   include/flatbuffers/hash.h
   include/flatbuffers/idl.h
   include/flatbuffers/util.h
+  include/flatbuffers/reflection.h
+  include/flatbuffers/reflection_generated.h
   src/idl_parser.cpp
   src/idl_gen_cpp.cpp
   src/idl_gen_general.cpp
@@ -108,7 +110,16 @@ function(compile_flatbuffers_schema_to_cpp SRC_FBS)
   string(REGEX REPLACE "\\.fbs$" "_generated.h" GEN_HEADER ${SRC_FBS})
   add_custom_command(
     OUTPUT ${GEN_HEADER}
-    COMMAND flatc -c --gen-mutable -o "${SRC_FBS_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FBS}"
+    COMMAND flatc -c --no-includes --gen-mutable -o "${SRC_FBS_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FBS}"
+    DEPENDS flatc)
+endfunction()
+
+function(compile_flatbuffers_schema_to_binary SRC_FBS)
+  get_filename_component(SRC_FBS_DIR ${SRC_FBS} PATH)
+  string(REGEX REPLACE "\\.fbs$" ".bfbs" GEN_BINARY_SCHEMA ${SRC_FBS})
+  add_custom_command(
+    OUTPUT ${GEN_BINARY_SCHEMA}
+    COMMAND flatc -b --schema -o "${SRC_FBS_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FBS}"
     DEPENDS flatc)
 endfunction()
 

build/VS2010/flattests.vcxproj

@@ -265,6 +265,7 @@
   <ItemGroup>
     <ClInclude Include="..\..\include\flatbuffers\flatbuffers.h" />
     <ClInclude Include="..\..\include\flatbuffers\idl.h" />
+    <ClInclude Include="..\..\include\flatbuffers\reflection.h" />
     <ClInclude Include="..\..\include\flatbuffers\util.h" />
     <ClInclude Include="..\..\tests\monster_test_generated.h" />
     <ClCompile Include="..\..\src\idl_gen_fbs.cpp" />
@@ -276,4 +277,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

docs/html/index.html

@@ -54,7 +54,7 @@
 </div><!--header-->
 <div class="contents">
 <div class="textblock"><p>FlatBuffers is an efficient cross platform serialization library for C++, with support for Java, C# and Go. It was created at Google specifically for game development and other performance-critical applications.</p>
-<p>It is available as open source under the Apache license, v2 (see LICENSE.txt).</p>
+<p>It is available as Open Source on <a href="http://github.com/google/flatbuffers">GitHub</a> under the Apache license, v2 (see LICENSE.txt).</p>
 <h2>Why use FlatBuffers?</h2>
 <ul>
 <li><b>Access to serialized data without parsing/unpacking</b> - What sets FlatBuffers apart is that it represents hierarchical data in a flat binary buffer in such a way that it can still be accessed directly without parsing/unpacking, while also still supporting data structure evolution (forwards/backwards compatibility).</li>
@@ -96,8 +96,8 @@ <h2>In-depth documentation</h2>
 </ul>
 <h2>Online resources</h2>
 <ul>
-<li><a href="http://github.com/google/flatbuffers">github repository</a></li>
-<li><a href="http://google.github.io/flatbuffers">landing page</a></li>
+<li><a href="http://github.com/google/flatbuffers">GitHub repository</a></li>
+<li><a href="http://google.github.io/flatbuffers">Landing page</a></li>
 <li><a href="http://group.google.com/group/flatbuffers">FlatBuffers Google Group</a></li>
 <li><a href="http://github.com/google/flatbuffers/issues">FlatBuffers Issues Tracker</a> </li>
 </ul>

docs/html/md__compiler.html

@@ -68,11 +68,14 @@
 <li><code>-o PATH</code> : Output all generated files to PATH (either absolute, or relative to the current directory). If omitted, PATH will be the current directory. PATH should end in your systems path separator, e.g. <code>/</code> or <code>\</code>.</li>
 <li><code>-I PATH</code> : when encountering <code>include</code> statements, attempt to load the files from this path. Paths will be tried in the order given, and if all fail (or none are specified) it will try to load relative to the path of the schema file being parsed.</li>
 <li><code>--strict-json</code> : Require &amp; generate strict JSON (field names are enclosed in quotes, no trailing commas in tables/vectors). By default, no quotes are required/generated, and trailing commas are allowed.</li>
+<li><code>--defaults-json</code> : Output fields whose value is equal to the default value when writing JSON text.</li>
 <li><code>--no-prefix</code> : Don't prefix enum values in generated C++ by their enum type.</li>
-<li><code>--gen-includes</code> : Generate include statements for included schemas the generated file depends on (C++).</li>
+<li><code>--gen-includes</code> : (deprecated), instead use:</li>
+<li><code>--no-includes</code> : Don't generate include statements for included schemas the generated file depends on (C++).</li>
 <li><code>--gen-mutable</code> : Generate additional non-const accessors for mutating FlatBuffers in-place.</li>
 <li><code>--raw-binary</code> : Allow binaries without a file_indentifier to be read. This may crash flatc given a mismatched schema.</li>
-<li><code>--proto</code>: Expect input files to be .proto files (protocol buffers). Output the corresponding .fbs file. Currently supports: <code>package</code>, <code>message</code>, <code>enum</code>. Does not support, but will skip without error: <code>import</code>, <code>option</code>. Does not support, will generate error: <code>service</code>, <code>extend</code>, <code>extensions</code>, <code>oneof</code>, <code>group</code>, custom options, nested declarations. </li>
+<li><code>--proto</code>: Expect input files to be .proto files (protocol buffers). Output the corresponding .fbs file. Currently supports: <code>package</code>, <code>message</code>, <code>enum</code>. Does not support, but will skip without error: <code>import</code>, <code>option</code>. Does not support, will generate error: <code>service</code>, <code>extend</code>, <code>extensions</code>, <code>oneof</code>, <code>group</code>, custom options, nested declarations.</li>
+<li><code>--schema</code>: Serialize schemas instead of JSON (use with -b). This will output a binary version of the specified schema that itself corresponds to the reflection/reflection.fbs schema. Loading this binary file is the basis for reflection functionality. </li>
 </ul>
 </div></div><!-- contents -->
 </div><!-- doc-content -->

docs/html/md__cpp_usage.html

@@ -114,8 +114,14 @@ <h3>Reading in C++</h3>
 </div><!-- fragment --><p>We use the somewhat verbose term <code>mutate</code> instead of <code>set</code> to indicate that this is a special use case, not to be confused with the default way of constructing FlatBuffer data.</p>
 <p>After the above mutations, you can send on the FlatBuffer to a new recipient without any further work!</p>
 <p>Note that any <code>mutate_</code> functions on tables return a bool, which is false if the field we're trying to set isn't present in the buffer. Fields are not present if they weren't set, or even if they happen to be equal to the default value. For example, in the creation code above we set the <code>mana</code> field to <code>150</code>, which is the default value, so it was never stored in the buffer. Trying to call mutate_mana() on such data will return false, and the value won't actually be modified!</p>
-<p>There's two ways around this. First, you can call <code>ForceDefaults()</code> on a <code>FlatBufferBuilder</code> to force all fields you set to actually be written. This of course increases the size of the buffer somewhat, but this may be acceptable for a mutable buffer.</p>
-<p>Alternatively, you can use mutation functions that are able to insert fields and change the size of things. These functions are expensive however, since they need to resize the buffer and create new data.</p>
+<p>One way to solve this is to call <code>ForceDefaults()</code> on a <code>FlatBufferBuilder</code> to force all fields you set to actually be written. This of course increases the size of the buffer somewhat, but this may be acceptable for a mutable buffer.</p>
+<p>Alternatively, you can use the more powerful reflection functionality:</p>
+<h3>Reflection (&amp; Resizing)</h3>
+<p>If the above ways of accessing a buffer are still too static for you, there is experimental support for reflection in FlatBuffers, allowing you to read and write data even if you don't know the exact format of a buffer, and even allows you to change sizes of strings and vectors in-place.</p>
+<p>The way this works is very elegant, there is actually a FlatBuffer schema that describes schemas (!) which you can find in <code>reflection/reflection.fbs</code>. The compiler <code>flatc</code> can write out any schemas it has just parsed as a binary FlatBuffer, corresponding to this meta-schema.</p>
+<p>Loading in one of these binary schemas at runtime allows you traverse any FlatBuffer data that corresponds to it without knowing the exact format. You can query what fields are present, and then read/write them after.</p>
+<p>For convenient field manipulation, you can include the header <code>flatbuffers/reflection.h</code> which includes both the generated code from the meta schema, as well as a lot of helper functions.</p>
+<p>And example of usage for the moment you can find in <code>test.cpp/ReflectionTest()</code>.</p>
 <h3>Storing maps / dictionaries in a FlatBuffer</h3>
 <p>FlatBuffers doesn't support maps natively, but there is support to emulate their behavior with vectors and binary search, which means you can have fast lookups directly from a FlatBuffer without having to unpack your data into a <code>std::map</code> or similar.</p>
 <p>To use it:</p><ul>

docs/source/Compiler.md

@@ -51,7 +51,8 @@ be generated for each file processed:
 -   `--no-prefix` : Don't prefix enum values in generated C++ by their enum
     type.
 
--   `--gen-includes` : Generate include statements for included schemas the
+-   `--gen-includes` : (deprecated), instead use:
+-   `--no-includes` : Don't generate include statements for included schemas the
     generated file depends on (C++).
 
 -   `--gen-mutable` : Generate additional non-const accessors for mutating
@@ -68,3 +69,8 @@ be generated for each file processed:
     Does not support, but will skip without error: `import`, `option`.
     Does not support, will generate error: `service`, `extend`, `extensions`,
     `oneof`, `group`, custom options, nested declarations.
+
+-   `--schema`: Serialize schemas instead of JSON (use with -b). This will
+    output a binary version of the specified schema that itself corresponds
+    to the reflection/reflection.fbs schema. Loading this binary file is the
+    basis for reflection functionality.

docs/source/CppUsage.md

@@ -203,14 +203,34 @@ to `150`, which is the default value, so it was never stored in the buffer.
 Trying to call mutate_mana() on such data will return false, and the value won't
 actually be modified!
 
-There's two ways around this. First, you can call `ForceDefaults()` on a
+One way to solve this is to call `ForceDefaults()` on a
 `FlatBufferBuilder` to force all fields you set to actually be written. This
 of course increases the size of the buffer somewhat, but this may be
 acceptable for a mutable buffer.
 
-Alternatively, you can use mutation functions that are able to insert fields
-and change the size of things. These functions are expensive however, since
-they need to resize the buffer and create new data.
+Alternatively, you can use the more powerful reflection functionality:
+
+### Reflection (& Resizing)
+
+If the above ways of accessing a buffer are still too static for you, there is
+experimental support for reflection in FlatBuffers, allowing you to read and
+write data even if you don't know the exact format of a buffer, and even allows
+you to change sizes of strings and vectors in-place.
+
+The way this works is very elegant, there is actually a FlatBuffer schema that
+describes schemas (!) which you can find in `reflection/reflection.fbs`.
+The compiler `flatc` can write out any schemas it has just parsed as a binary
+FlatBuffer, corresponding to this meta-schema.
+
+Loading in one of these binary schemas at runtime allows you traverse any
+FlatBuffer data that corresponds to it without knowing the exact format. You
+can query what fields are present, and then read/write them after.
+
+For convenient field manipulation, you can include the header
+`flatbuffers/reflection.h` which includes both the generated code from the meta
+schema, as well as a lot of helper functions.
+
+And example of usage for the moment you can find in `test.cpp/ReflectionTest()`.
 
 ### Storing maps / dictionaries in a FlatBuffer
 

include/flatbuffers/flatbuffers.h

@@ -754,7 +754,7 @@ class FlatBufferBuilder FLATBUFFERS_FINAL_CLASS {
   }
 
   template<typename T> Offset<Vector<Offset<T>>> CreateVectorOfSortedTables(
-                                                            std::vector<T> *v) {
+                                                    std::vector<Offset<T>> *v) {
     return CreateVectorOfSortedTables(v->data(), v->size());
   }
 
@@ -1017,6 +1017,8 @@ class Table {
     return true;
   }
 
+  uint8_t *GetVTable() { return data_ - ReadScalar<soffset_t>(data_); }
+
   bool CheckField(voffset_t field) const {
     return GetOptionalFieldOffset(field) != 0;
   }

include/flatbuffers/idl.h

@@ -25,6 +25,7 @@
 
 #include "flatbuffers/flatbuffers.h"
 #include "flatbuffers/hash.h"
+#include "flatbuffers/reflection.h"
 
 // This file defines the data types representing a parsed IDL (Interface
 // Definition Language) / schema file.
@@ -126,6 +127,8 @@ struct Type {
 
   Type VectorType() const { return Type(element, struct_def, enum_def); }
 
+  Offset<reflection::Type> Serialize(FlatBufferBuilder *builder) const;
+
   BaseType base_type;
   BaseType element;       // only set if t == BASE_TYPE_VECTOR
   StructDef *struct_def;  // only set if t or element == BASE_TYPE_STRUCT
@@ -179,20 +182,28 @@ struct Namespace {
 
 // Base class for all definition types (fields, structs_, enums_).
 struct Definition {
-  Definition() : generated(false), defined_namespace(nullptr) {}
+  Definition() : generated(false), defined_namespace(nullptr),
+                 serialized_location(0), index(-1) {}
 
   std::string name;
   std::string file;
   std::vector<std::string> doc_comment;
   SymbolTable<Value> attributes;
   bool generated;  // did we already output code for this definition?
   Namespace *defined_namespace;  // Where it was defined.
+
+  // For use with Serialize()
+  uoffset_t serialized_location;
+  int index;  // Inside the vector it is stored.
 };
 
 struct FieldDef : public Definition {
   FieldDef() : deprecated(false), required(false), key(false), padding(0),
                used(false) {}
 
+  Offset<reflection::Field> Serialize(FlatBufferBuilder *builder, uint16_t id)
+                                                                          const;
+
   Value value;
   bool deprecated; // Field is allowed to be present in old data, but can't be
                    // written in new data nor accessed in new code.
@@ -218,6 +229,8 @@ struct StructDef : public Definition {
     if (fields.vec.size()) fields.vec.back()->padding = padding;
   }
 
+  Offset<reflection::Object> Serialize(FlatBufferBuilder *builder) const;
+
   SymbolTable<FieldDef> fields;
   bool fixed;       // If it's struct, not a table.
   bool predecl;     // If it's used before it was defined.
@@ -243,6 +256,8 @@ struct EnumVal {
   EnumVal(const std::string &_name, int64_t _val)
     : name(_name), value(_val), struct_def(nullptr) {}
 
+  Offset<reflection::EnumVal> Serialize(FlatBufferBuilder *builder) const;
+
   std::string name;
   std::vector<std::string> doc_comment;
   int64_t value;
@@ -263,6 +278,8 @@ struct EnumDef : public Definition {
     return nullptr;
   }
 
+  Offset<reflection::Enum> Serialize(FlatBufferBuilder *builder) const;
+
   SymbolTable<EnumVal> vals;
   bool is_union;
   Type underlying_type;
@@ -271,7 +288,7 @@ struct EnumDef : public Definition {
 class Parser {
  public:
   Parser(bool strict_json = false, bool proto_mode = false)
-    : root_struct_def(nullptr),
+    : root_struct_def_(nullptr),
       source_(nullptr),
       cursor_(nullptr),
       line_(1),
@@ -325,6 +342,10 @@ class Parser {
   std::set<std::string> GetIncludedFilesRecursive(
       const std::string &file_name) const;
 
+  // Fills builder_ with a binary version of the schema parsed.
+  // See reflection/reflection.fbs
+  void Serialize();
+
  private:
   int64_t ParseHexNum(int nibbles);
   void Next();
@@ -363,7 +384,7 @@ class Parser {
   std::string error_;         // User readable error_ if Parse() == false
 
   FlatBufferBuilder builder_;  // any data contained in the file
-  StructDef *root_struct_def;
+  StructDef *root_struct_def_;
   std::string file_identifier_;
   std::string file_extension_;
 
@@ -417,7 +438,7 @@ struct GeneratorOptions {
                        output_default_scalars_in_json(false),
                        indent_step(2),
                        output_enum_identifiers(true), prefixed_enums(true),
-                       include_dependence_headers(false),
+                       include_dependence_headers(true),
                        mutable_buffer(false),
                        one_file(false),
                        lang(GeneratorOptions::kJava) {}