diff --git a/.gitignore b/.gitignore index c6bcf20d453..13a68f835a4 100755 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,9 @@ flattests flatsamplebinary flatsampletext snapshot.sh +tests/go_gen +CMakeLists.txt.user +CMakeScripts/** +build/Xcode/FlatBuffers.xcodeproj/project.xcworkspace/** +build/Xcode/FlatBuffers.xcodeproj/xcuserdata/** diff --git a/CMake/FindFlatBuffers.cmake b/CMake/FindFlatBuffers.cmake new file mode 100644 index 00000000000..0a54e2a200c --- /dev/null +++ b/CMake/FindFlatBuffers.cmake @@ -0,0 +1,56 @@ +# Copyright 2014 Stefan.Eilemann@epfl.ch +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Find the flatbuffers schema compiler +# +# Output Variables: +# * FLATBUFFERS_FLATC_EXECUTABLE the flatc compiler executable +# * FLATBUFFERS_FOUND +# +# Provides: +# * FLATBUFFERS_GENERATE_C_HEADERS(Name ) creates the C++ headers +# for the given flatbuffer schema files. +# Returns the header files in ${Name}_OUTPUTS + +find_program(FLATBUFFERS_FLATC_EXECUTABLE NAMES flatc) +find_path(FLATBUFFERS_INCLUDE_DIR NAMES flatbuffers/flatbuffers.h) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(flatbuffers + DEFAULT_MSG FLATBUFFERS_FLATC_EXECUTABLE FLATBUFFERS_INCLUDE_DIR) + +if(FLATBUFFERS_FOUND) + function(FLATBUFFERS_GENERATE_C_HEADERS Name) + set(FLATC_OUTPUTS) + foreach(FILE ${ARGN}) + get_filename_component(FLATC_OUTPUT ${FILE} NAME_WE) + set(FLATC_OUTPUT + "${CMAKE_CURRENT_BINARY_DIR}/${FLATC_OUTPUT}_generated.h") + list(APPEND FLATC_OUTPUTS ${FLATC_OUTPUT}) + + add_custom_command(OUTPUT ${FLATC_OUTPUT} + COMMAND ${FLATBUFFERS_FLATC_EXECUTABLE} + ARGS -c -o "${CMAKE_CURRENT_BINARY_DIR}/" ${FILE} + COMMENT "Building C++ header for ${FILE}" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + endforeach() + set(${Name}_OUTPUTS ${FLATC_OUTPUTS} PARENT_SCOPE) + endfunction() + + set(FLATBUFFERS_INCLUDE_DIRS ${FLATBUFFERS_INCLUDE_DIR}) + include_directories(${CMAKE_BINARY_DIR}) +else() + set(FLATBUFFERS_INCLUDE_DIR) +endif() diff --git a/CMakeLists.txt b/CMakeLists.txt index 7098d365bc6..2c88beffa08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,7 @@ set(FlatBuffers_Compiler_SRCS src/idl_parser.cpp src/idl_gen_cpp.cpp src/idl_gen_java.cpp + src/idl_gen_go.cpp src/idl_gen_text.cpp src/flatc.cpp ) @@ -63,14 +64,17 @@ set(CMAKE_BUILD_TYPE Debug) # source_group(Tests FILES ${FlatBuffers_Tests_SRCS}) if(APPLE) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -Wall") + set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -Wall -pedantic -Werror -Wextra") elseif(CMAKE_COMPILER_IS_GNUCXX OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x -Wall") + set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -std=c++0x -Wall -pedantic -Werror -Wextra") endif() if(FLATBUFFERS_CODE_COVERAGE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fprofile-arcs -ftest-coverage") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage") + set(CMAKE_EXE_LINKER_FLAGS + "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage") endif() include_directories(include) diff --git a/build/VS2010/flatc.vcxproj b/build/VS2010/flatc.vcxproj index d63edc48520..e9313cfeac6 100755 --- a/build/VS2010/flatc.vcxproj +++ b/build/VS2010/flatc.vcxproj @@ -91,11 +91,12 @@ NotUsing MultiThreadedDebugDLL true - Level3 + Level4 WIN32;_WINDOWS;_DEBUG;CMAKE_INTDIR="Debug";%(PreprocessorDefinitions) Debug $(IntDir) ../../Debug/flatc.pdb + true WIN32;_WINDOWS;_DEBUG;CMAKE_INTDIR=\"Debug\";%(PreprocessorDefinitions) @@ -180,13 +181,14 @@ NotUsing MultiThreadedDLL true - Level3 + Level4 WIN32;_WINDOWS;NDEBUG;CMAKE_INTDIR="Release";%(PreprocessorDefinitions) Release $(IntDir) ../../Release/flatc.pdb + true WIN32;_WINDOWS;NDEBUG;CMAKE_INTDIR=\"Release\";%(PreprocessorDefinitions) @@ -264,6 +266,9 @@ + + Level4 + diff --git a/build/VS2010/flatc.vcxproj.user b/build/VS2010/flatc.vcxproj.user index 53bc18a0853..b7bcc9a56b1 100755 --- a/build/VS2010/flatc.vcxproj.user +++ b/build/VS2010/flatc.vcxproj.user @@ -3,21 +3,22 @@ ..\..\tests WindowsLocalDebugger - -j -c -b -t monster_test.fbs monsterdata_test.golden + -j -c -g -b -t monster_test.fbs monsterdata_test.golden ..\.. WindowsLocalDebugger + -j -c -g -b -t monster_test.fbs monsterdata_test.golden - -j -c -b -t monster_test.fbs monsterdata_test.golden + -j -c -g -b -t monster_test.fbs monsterdata_test.golden ..\..\tests WindowsLocalDebugger - -j -c -b -t monster_test.fbs monsterdata_test.json + -j -c -g -b -t monster_test.fbs monsterdata_test.json ..\..\tests diff --git a/build/VS2010/flatsamplebinary.vcxproj b/build/VS2010/flatsamplebinary.vcxproj index dfa710c1e39..9d5fb107d2d 100755 --- a/build/VS2010/flatsamplebinary.vcxproj +++ b/build/VS2010/flatsamplebinary.vcxproj @@ -91,11 +91,12 @@ NotUsing MultiThreadedDebugDLL true - Level3 + Level4 WIN32;_WINDOWS;_DEBUG;CMAKE_INTDIR="Debug";%(PreprocessorDefinitions) Debug $(IntDir) ../../Debug/flatsamplebinary.pdb + true WIN32;_WINDOWS;_DEBUG;CMAKE_INTDIR=\"Debug\";%(PreprocessorDefinitions) @@ -180,13 +181,14 @@ NotUsing MultiThreadedDLL true - Level3 + Level4 WIN32;_WINDOWS;NDEBUG;CMAKE_INTDIR="Release";%(PreprocessorDefinitions) Release $(IntDir) ../../Release/flatsamplebinary.pdb + true WIN32;_WINDOWS;NDEBUG;CMAKE_INTDIR=\"Release\";%(PreprocessorDefinitions) diff --git a/build/VS2010/flatsampletext.vcxproj b/build/VS2010/flatsampletext.vcxproj index 5b81e6df8de..2519e50c8d6 100755 --- a/build/VS2010/flatsampletext.vcxproj +++ b/build/VS2010/flatsampletext.vcxproj @@ -91,11 +91,12 @@ NotUsing MultiThreadedDebugDLL true - Level3 + Level4 WIN32;_WINDOWS;_DEBUG;CMAKE_INTDIR="Debug";%(PreprocessorDefinitions) Debug $(IntDir) ../../Debug/flatsampletext.pdb + true WIN32;_WINDOWS;_DEBUG;CMAKE_INTDIR=\"Debug\";%(PreprocessorDefinitions) @@ -180,13 +181,14 @@ NotUsing MultiThreadedDLL true - Level3 + Level4 WIN32;_WINDOWS;NDEBUG;CMAKE_INTDIR="Release";%(PreprocessorDefinitions) Release $(IntDir) ../../Release/flatsampletext.pdb + true WIN32;_WINDOWS;NDEBUG;CMAKE_INTDIR=\"Release\";%(PreprocessorDefinitions) diff --git a/build/VS2010/flattests.vcxproj b/build/VS2010/flattests.vcxproj index 694a81cbe75..e4c2746c6d8 100755 --- a/build/VS2010/flattests.vcxproj +++ b/build/VS2010/flattests.vcxproj @@ -91,11 +91,12 @@ NotUsing MultiThreadedDebugDLL true - Level3 + Level4 WIN32;_WINDOWS;_DEBUG;CMAKE_INTDIR="Debug";%(PreprocessorDefinitions) Debug $(IntDir) ../../Debug/flattests.pdb + true WIN32;_WINDOWS;_DEBUG;CMAKE_INTDIR=\"Debug\";%(PreprocessorDefinitions) @@ -180,13 +181,14 @@ NotUsing MultiThreadedDLL true - Level3 + Level4 WIN32;_WINDOWS;NDEBUG;CMAKE_INTDIR="Release";%(PreprocessorDefinitions) Release $(IntDir) ../../Release/flattests.pdb + true WIN32;_WINDOWS;NDEBUG;CMAKE_INTDIR=\"Release\";%(PreprocessorDefinitions) @@ -225,7 +227,7 @@ NotUsing MultiThreadedDLL true - EnableAllWarnings + Level4 WIN32;_WINDOWS;NDEBUG;CMAKE_INTDIR="Release";%(PreprocessorDefinitions) diff --git a/build/Xcode/FlatBuffers.xcodeproj/project.pbxproj b/build/Xcode/FlatBuffers.xcodeproj/project.pbxproj index ead87f8d6fd..edf486ccd8a 100644 --- a/build/Xcode/FlatBuffers.xcodeproj/project.pbxproj +++ b/build/Xcode/FlatBuffers.xcodeproj/project.pbxproj @@ -11,6 +11,7 @@ 3343DD4ED370434BBA148FAB /* idl_gen_java.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3803689175184C7E8CB3EED0 /* idl_gen_java.cpp */; settings = {COMPILER_FLAGS = ""; }; }; 61823BBC53544106B6DBC38E /* idl_parser.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3709AC883348409592530AE6 /* idl_parser.cpp */; settings = {COMPILER_FLAGS = ""; }; }; 61FF3C34FBEC4819A1C30F92 /* sample_text.cpp in Sources */ = {isa = PBXBuildFile; fileRef = ECCEBFFA6977404F858F9739 /* sample_text.cpp */; settings = {COMPILER_FLAGS = ""; }; }; + 8C303C591975D6A700D7C1C5 /* idl_gen_go.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 8C303C581975D6A700D7C1C5 /* idl_gen_go.cpp */; }; A9C9A99F719A4ED58DC2D2FC /* idl_parser.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3709AC883348409592530AE6 /* idl_parser.cpp */; settings = {COMPILER_FLAGS = ""; }; }; AA9BACF55EB3456BA2F633BB /* flatc.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0DFD29781D8E490284B06504 /* flatc.cpp */; settings = {COMPILER_FLAGS = ""; }; }; AD71FEBEE4E846529002C1F0 /* idl_gen_text.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F6C5D81DBF864365B12E269D /* idl_gen_text.cpp */; settings = {COMPILER_FLAGS = ""; }; }; @@ -32,6 +33,7 @@ 423CA92401AE442B91546E63 /* CMakeLists.txt */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; name = CMakeLists.txt; path = /Users/wvo/flatbuffers_snapshot9/CMakeLists.txt; sourceTree = ""; }; 5EE44BFFAF8E43F485859145 /* sample_binary.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = sample_binary.cpp; path = samples/sample_binary.cpp; sourceTree = SOURCE_ROOT; }; 6AD24EEB3D024825A37741FF /* test.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = test.cpp; path = tests/test.cpp; sourceTree = SOURCE_ROOT; }; + 8C303C581975D6A700D7C1C5 /* idl_gen_go.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = idl_gen_go.cpp; path = src/idl_gen_go.cpp; sourceTree = ""; }; A13F25CDAD23435DA293690D /* flattests */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; path = flattests; sourceTree = BUILT_PRODUCTS_DIR; }; AB70F1FBA50E4120BCF37C8D /* monster_test_generated.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = monster_test_generated.h; path = tests/monster_test_generated.h; sourceTree = SOURCE_ROOT; }; AD3682C6E1DD4EABB822C0CC /* monster_generated.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = monster_generated.h; path = samples/monster_generated.h; sourceTree = SOURCE_ROOT; }; @@ -53,6 +55,7 @@ 28237E300FE042DEADA302D3 /* Source Files */ = { isa = PBXGroup; children = ( + 8C303C581975D6A700D7C1C5 /* idl_gen_go.cpp */, 0DFD29781D8E490284B06504 /* flatc.cpp */, CD90A7F6B2BE4D0384294DD1 /* idl_gen_cpp.cpp */, 3803689175184C7E8CB3EED0 /* idl_gen_java.cpp */, @@ -314,6 +317,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 8C303C591975D6A700D7C1C5 /* idl_gen_go.cpp in Sources */, AA9BACF55EB3456BA2F633BB /* flatc.cpp in Sources */, BE03D7B0C9584DD58B50ED34 /* idl_gen_cpp.cpp in Sources */, 3343DD4ED370434BBA148FAB /* idl_gen_java.cpp in Sources */, diff --git a/docs/html/index.html b/docs/html/index.html index f005a5658a1..ab9ec47bafe 100644 --- a/docs/html/index.html +++ b/docs/html/index.html @@ -53,7 +53,7 @@
FlatBuffers Documentation
-

FlatBuffers is an efficient cross platform serialization library in for C++ and Java. It was created at Google specifically for game development and other performance-critical applications.

+

FlatBuffers is an efficient cross platform serialization library for C++, with support for Java and Go. It was created at Google specifically for game development and other performance-critical applications.

It is available as open source under the Apache license, v2 (see LICENSE.txt).

Why use FlatBuffers?

    @@ -63,9 +63,9 @@

    Why use FlatBuffers?

  • Tiny code footprint - Small amounts of generated code, and just a single small header as the minimum dependency, which is very easy to integrate. Again, see the benchmark section for details.
  • Strongly typed - Errors happen at compile time rather than manually having to write repetitive and error prone run-time checks. Useful code can be generated for you.

  • Convenient to use - Generated C++ code allows for terse access & construction code. Then there's optional functionality for parsing schemas and JSON-like text representations at runtime efficiently if needed (faster and more memory efficient than other JSON parsers).

    -

    Java code supports object-reuse.

    +

    Java and Go code supports object-reuse.

    • -
  • Cross platform C++11/Java code with no dependencies - will work with any recent gcc/clang and VS2010. Comes with build files for the tests & samples (Android .mk files, and cmake for all other platforms).
    • +
  • Cross platform C++11/Java/Go code with no dependencies - will work with any recent gcc/clang and VS2010. Comes with build files for the tests & samples (Android .mk files, and cmake for all other platforms).

Why not use Protocol Buffers, or .. ?

Protocol Buffers is indeed relatively similar to FlatBuffers, with the primary difference being that FlatBuffers does not need a parsing/ unpacking step to a secondary representation before you can access data, often coupled with per-object memory allocation. The code is an order of magnitude bigger, too. Protocol Buffers has neither optional text import/export nor schema language features like unions.

@@ -76,7 +76,7 @@

Usage in brief

This section is a quick rundown of how to use this system. Subsequent sections provide a more in-depth usage guide.

  • Write a schema file that allows you to define the data structures you may want to serialize. Fields can have a scalar type (ints/floats of all sizes), or they can be a: string; array of any type; reference to yet another object; or, a set of possible objects (unions). Fields are optional and have defaults, so they don't need to be present for every object instance.
    • -
  • Use flatc (the FlatBuffer compiler) to generate a C++ header (or Java classes) with helper classes to access and construct serialized data. This header (say mydata_generated.h) only depends on flatbuffers.h, which defines the core functionality.
    • +
  • Use flatc (the FlatBuffer compiler) to generate a C++ header (or Java/Go classes) with helper classes to access and construct serialized data. This header (say mydata_generated.h) only depends on flatbuffers.h, which defines the core functionality.
  • Use the FlatBufferBuilder class to construct a flat binary buffer. The generated functions allow you to add objects to this buffer recursively, often as simply as making a single function call.
  • Store or send your buffer somewhere!
  • When reading it back, you can obtain the pointer to the root object from the binary buffer, and from there traverse it conveniently in-place with object->field().
    • @@ -88,6 +88,7 @@

    In-depth documentation

  • How to write a schema.
  • How to use the generated C++ code in your own programs.
  • How to use the generated Java code in your own programs.
    • +
  • How to use the generated Go code in your own programs.
  • Some benchmarks showing the advantage of using FlatBuffers.
  • A white paper explaining the "why" of FlatBuffers.
  • A description of the internals of FlatBuffers.
    • diff --git a/docs/html/md__benchmarks.html b/docs/html/md__benchmarks.html index 3a8a60190f3..e17ec0821a1 100644 --- a/docs/html/md__benchmarks.html +++ b/docs/html/md__benchmarks.html @@ -53,37 +53,36 @@
    Benchmarks
-

Comparing against other serialization solutions, running on Windows 7 64bit. We use the LITE runtime for Protocol Buffers (less code / lower overhead), and Rapid JSON, one of the fastest C++ JSON parsers around.

+

Comparing against other serialization solutions, running on Windows 7 64bit. We use the LITE runtime for Protocol Buffers (less code / lower overhead), Rapid JSON (one of the fastest C++ JSON parsers around), and pugixml, also one of the fastest XML parsers.

We compare against Flatbuffers with the binary wire format (as intended), and also with JSON as the wire format with the optional JSON parser (which, using a schema, parses JSON into a binary buffer that can then be accessed as before).

The benchmark object is a set of about 10 objects containing an array, 4 strings, and a large variety of int/float scalar values of all sizes, meant to be representative of game data, e.g. a scene format.

- + - + - + - + - + - + - + - + - + - +
FlatBuffers (binary) Protocol Buffers LITE Rapid JSON FlatBuffers (JSON)
FlatBuffers (binary) Protocol Buffers LITE Rapid JSON FlatBuffers (JSON) pugixml
Decode + Traverse + Dealloc (1 million times, seconds) 0.08 302 583 105
Decode + Traverse + Dealloc (1 million times, seconds) 0.08 302 583 105 196
Decode / Traverse / Dealloc (breakdown) 0 / 0.08 / 0 220 / 0.15 / 81 294 / 0.9 / 287 70 / 0.08 / 35
Decode / Traverse / Dealloc (breakdown) 0 / 0.08 / 0 220 / 0.15 / 81 294 / 0.9 / 287 70 / 0.08 / 35 41 / 3.9 / 150
Encode (1 million times, seconds) 3.2 185 650 169
Encode (1 million times, seconds) 3.2 185 650 169 273
Wire format size (normal / zlib, bytes) 344 / 220 228 / 174 1475 / 322 1029 / 298
Wire format size (normal / zlib, bytes) 344 / 220 228 / 174 1475 / 322 1029 / 298 1137 / 341
Memory needed to store decoded wire (bytes / blocks) 0 / 0 760 / 20 65689 / 40 328 / 1
Memory needed to store decoded wire (bytes / blocks) 0 / 0 760 / 20 65689 / 4 328 / 1 34194 / 3
Transient memory allocated during decode (KB) 0 1 131 4
Transient memory allocated during decode (KB) 0 1 131 4 34
Generated source code size (KB) 4 61 0 4
Generated source code size (KB) 4 61 0 4 0
Field access in handwritten traversal code accessors accessors manual error checking accessors
Field access in handwritten traversal code typed accessors typed accessors manual error checking typed accessors manual error checking
Library source code (KB) 15 some subset of 3800 87 43
Library source code (KB) 15 some subset of 3800 87 43 327

Some other serialization systems we compared against but did not benchmark (yet), in rough order of applicability:

  • Cap'n'Proto promises to reduce Protocol Buffers much like FlatBuffers does, though with a more complicated binary encoding and less flexibility (no optional fields to allow deprecating fields or serializing with missing fields for which defaults exist). It currently also isn't fully cross-platform portable (lack of VS support).
  • msgpack: has very minimal forwards/backwards compatability support when used with the typed C++ interface. Also lacks VS2010 support.
  • Thrift: very similar to Protocol Buffers, but appears to be less efficient, and have more dependencies.
    • -
  • XML: typically even slower than JSON, but has the advantage that it can be parsed with a schema to reduce error-checking boilerplate code.
  • YAML: a superset of JSON and otherwise very similar. Used by e.g. Unity.
  • C# comes with built-in serialization functionality, as used by Unity also. Being tied to the language, and having no automatic versioning support limits its applicability.
  • Project Anarchy (the free mobile engine by Havok) comes with a serialization system, that however does no automatic versioning (have to code around new fields manually), is very much tied to the rest of the engine, and works without a schema to generate code (tied to your C++ class definition).
    • diff --git a/docs/html/md__cpp_usage.html b/docs/html/md__cpp_usage.html index f02436ee053..9a74910f55c 100644 --- a/docs/html/md__cpp_usage.html +++ b/docs/html/md__cpp_usage.html @@ -74,7 +74,7 @@

    Writing in C++

    mb.add_inventory(inventory); auto mloc = mb.Finish();

    We start with a temporary helper class MonsterBuilder (which is defined in our generated code also), then call the various add_ methods to set fields, and Finish to complete the object. This is pretty much the same code as you find inside CreateMonster, except we're leaving out a few fields. Fields may also be added in any order, though orderings with fields of the same size adjacent to each other most efficient in size, due to alignment. You should not nest these Builder classes (serialize your data in pre-order).

    -

    Regardless of whether you used CreateMonster or MonsterBuilder, you now have an offset to the root of your data, and you can finish the buffer using: 

    fbb.Finish(mloc);
+
    Regardless of whether you used CreateMonster or MonsterBuilder, you now have an offset to the root of your data, and you can finish the buffer using: 
    FinishMonsterBuffer(fbb, mloc);
 
    The buffer is now ready to be stored somewhere, sent over the network, be compressed, or whatever you'd like to do with it. You can access the start of the buffer with fbb.GetBufferPointer(), and it's size from fbb.GetSize().
    
 
    samples/sample_binary.cpp is a complete code sample similar to the code above, that also includes the reading code below.
    
 
    Reading in C++
    
@@ -107,6 +107,7 @@ 
    Access of untrusted buffers
    
 
    Text & schema parsing
    
 
    Using binary buffers with the generated header provides a super low overhead use of FlatBuffer data. There are, however, times when you want to use text formats, for example because it interacts better with source control, or you want to give your users easy access to data.
    
 
    Another reason might be that you already have a lot of data in JSON format, or a tool that generates JSON, and if you can write a schema for it, this will provide you an easy way to use that data directly.
    
+
    (see the schema documentation for some specifics on the JSON format accepted).
    
 
    There are two ways to use text formats:
    
 
    Using the compiler as a conversion tool
    
 
    This is the preferred path, as it doesn't require you to add any new code to your program, and is maximally efficient since you can ship with binary data. The disadvantage is that it is an extra step for your users/developers to perform, though you might be able to automate it. 
    flatc -b myschema.fbs mydata.json
diff --git a/docs/html/md__go_usage.html b/docs/html/md__go_usage.html
new file mode 100644
index 00000000000..5a873b209aa
--- /dev/null
+++ b/docs/html/md__go_usage.html
@@ -0,0 +1,106 @@
+
+
+
+
+
+
+FlatBuffers: Use in Go
+
+
+
+
+
+
+
+
+
+
+
    
+
    
+
+ 
+ 
+  
+ 
+ 
+
    
+   
    FlatBuffers
+   
    
+  
    
+
    
+
+
+
    
+
    
+  
+  
    
+  
    
+
    
+
+
    
+
    
+  
    
+
    Use in Go 
      
    
+
    
+
    
+
    There's experimental support for reading FlatBuffers in Go. Generate code for Go with the -g option to flatc.
    
+
    See go_test.go for an example. You import the generated code, read a FlatBuffer binary file into a []byte, which you pass to the GetRootAsMonster function: 
    import (
+   example "MyGame/Example"
+   flatbuffers "github.com/google/flatbuffers/go"
+
+   io/ioutil
+)
+
+buf, err := ioutil.ReadFile("monster.dat")
+// handle err
+monster := example.GetRootAsMonster(buf, 0)
+
    Now you can access values like this: 
    hp := monster.Hp()
+pos := monster.Pos(nil)
+
    Note that whenever you access a new object like in the Pos example above, a new temporary accessor object gets created. If your code is very performance sensitive (you iterate through a lot of objects), you can replace nil with a pointer to a Vec3 object you've already created. This allows you to reuse it across many calls and reduce the amount of object allocation (and thus garbage collection) your program does.
    
+
    To access vectors you pass an extra index to the vector field accessor. Then a second method with the same name suffixed by Length let's you know the number of elements you can access: 
    for i := 0; i < monster.InventoryLength(); i++ {
+    monster.Inventory(i) // do something here
+}
+
    You can also construct these buffers in Go using the functions found in the generated code, and the FlatBufferBuilder class: 
    builder := flatbuffers.NewBuilder(0)
+
    Create strings: 
    str := builder.CreateString("MyMonster")
+
    Create a table with a struct contained therein: 
    example.MonsterStart(builder)
+example.MonsterAddPos(builder, example.CreateVec3(builder, 1.0, 2.0, 3.0, 3.0, 4, 5, 6))
+example.MonsterAddHp(builder, 80)
+example.MonsterAddName(builder, str)
+example.MonsterAddInventory(builder, inv)
+example.MonsterAddTest_Type(builder, 1)
+example.MonsterAddTest(builder, mon2)
+example.MonsterAddTest4(builder, test4s)
+mon := example.MonsterEnd(builder)
+
    Unlike C++, Go does not support table creation functions like 'createMonster()'. This is to create the buffer without using temporary object allocation (since the Vec3 is an inline component of Monster, it has to be created right where it is added, whereas the name and the inventory are not inline). Structs do have convenient methods that allow you to construct them in one call. These also have arguments for nested structs, e.g. if a struct has a field a and a nested struct field b (which has fields c and d), then the arguments will be a, c and d.
    
+
    Vectors also use this start/end pattern to allow vectors of both scalar types and structs: 
    example.MonsterStartInventoryVector(builder, 5)
+for i := 4; i >= 0; i-- {
+    builder.PrependByte(byte(i))
+}
+inv := builder.EndVector(5)
+
    The generated method 'StartInventoryVector' is provided as a convenience function which calls 'StartVector' with the correct element size of the vector type which in this case is 'ubyte' or 1 byte per vector element. You pass the number of elements you want to write. You write the elements backwards since the buffer is being constructed back to front.
    
+
    There are Prepend functions for all the scalar types. You use PrependUOffset for any previously constructed objects (such as other tables, strings, vectors). For structs, you use the appropriate create function in-line, as shown above in the Monster example.
    
+
    Text Parsing
    
+
    There currently is no support for parsing text (Schema's and JSON) directly from Go, though you could use the C++ parser through cgo. Please see the C++ documentation for more on text parsing. 
    
+
    
+
    
+
+
+
+
diff --git a/docs/html/md__grammar.html b/docs/html/md__grammar.html
index e5278c09b4f..8d4e525a157 100644
--- a/docs/html/md__grammar.html
+++ b/docs/html/md__grammar.html
@@ -53,7 +53,8 @@
 
    Formal Grammar of the schema language
-

schema = namespace_decl | type_decl | enum_decl | root_decl | object

+

schema = include* ( namespace_decl | type_decl | enum_decl | root_decl | object )*

+

include = include string_constant ;

namespace_decl = namespace ident ( . ident )* ;

type_decl = ( table | struct ) ident metadata { field_decl+ }

enum_decl = ( enum | union ) ident [ : type ] metadata { commasep( enumval_decl ) }

diff --git a/docs/html/md__internals.html b/docs/html/md__internals.html index 49f82874d83..b4da59ffcd0 100644 --- a/docs/html/md__internals.html +++ b/docs/html/md__internals.html @@ -56,7 +56,7 @@

This section is entirely optional for the use of FlatBuffers. In normal usage, you should never need the information contained herein. If you're interested however, it should give you more of an appreciation of why FlatBuffers is both efficient and convenient.

Format components

A FlatBuffer is a binary file and in-memory format consisting mostly of scalars of various sizes, all aligned to their own size. Each scalar is also always represented in little-endian format, as this corresponds to all commonly used CPUs today. FlatBuffers will also work on big-endian machines, but will be slightly slower because of additional byte-swap intrinsics.

-

On purpose, the format leaves a lot of details about where exactly things live in memory undefined, e.g. fields in a table can have any order, and objects to some extend can be stored in many orders. This is because the format doesn't need this information to be efficient, and it leaves room for optimization and extension (for example, fields can be packed in a way that is most compact). Instead, the format is defined in terms of offsets and adjacency only.

+

On purpose, the format leaves a lot of details about where exactly things live in memory undefined, e.g. fields in a table can have any order, and objects to some extend can be stored in many orders. This is because the format doesn't need this information to be efficient, and it leaves room for optimization and extension (for example, fields can be packed in a way that is most compact). Instead, the format is defined in terms of offsets and adjacency only. This may mean two different implementations may produce different binaries given the same input values, and this is perfectly valid.

Format identification

The format also doesn't contain information for format identification and versioning, which is also by design. FlatBuffers is a statically typed system, meaning the user of a buffer needs to know what kind of buffer it is. FlatBuffers can of course be wrapped inside other containers where needed, or you can use its union feature to dynamically identify multiple possible sub-objects stored. Additionally, it can be used together with the schema parser if full reflective capabilities are desired.

Versioning is something that is intrinsically part of the format (the optionality / extensibility of fields), so the format itself does not need a version number (it's a meta-format, in a sense). We're hoping that this format can accommodate all data needed. If format breaking changes are ever necessary, it would become a new kind of format rather than just a variation.

@@ -107,7 +107,7 @@

Code example

inline const char *EnumNameAny(int e) { return EnumNamesAny()[e]; }

Unions share a lot with enums. 

struct Vec3;
 struct Monster;
-

Predeclare all datatypes since there may be circular references. 

MANUALLY_ALIGNED_STRUCT(4) Vec3 {
+

Predeclare all data types since circular references between types are allowed (circular references between object are not, though). 

MANUALLY_ALIGNED_STRUCT(4) Vec3 {
  private:
   float x_;
   float y_;
diff --git a/docs/html/md__java_usage.html b/docs/html/md__java_usage.html
index 44e30e5065c..99b92304893 100644
--- a/docs/html/md__java_usage.html
+++ b/docs/html/md__java_usage.html
@@ -53,15 +53,15 @@
 
Use in Java
-

There's experimental support for reading FlatBuffers in Java. Generate code for Java with the -j option to flatc.

-

See javaTest.java for an example. Essentially, you read a FlatBuffer binary file into a byte[], which you then turn into a ByteBuffer, which you pass to the getRootAsMonster function: 

ByteBuffer bb = ByteBuffer.wrap(data);
+
FlatBuffers supports reading and writing binary FlatBuffers in Java. Generate code for Java with the -j option to flatc.

+
See javaTest.java for an example. Essentially, you read a FlatBuffer binary file into a byte[], which you then turn into a ByteBuffer, which you pass to the getRootAsMyRootType function: 
ByteBuffer bb = ByteBuffer.wrap(data);
 Monster monster = Monster.getRootAsMonster(bb);
 
Now you can access values much like C++: 
short hp = monster.hp();
 Vec3 pos = monster.pos();
 
Note that whenever you access a new object like in the pos example above, a new temporary accessor object gets created. If your code is very performance sensitive (you iterate through a lot of objects), there's a second pos() method to which you can pass a Vec3 object you've already created. This allows you to reuse it across many calls and reduce the amount of object allocation (and thus garbage collection) your program does.

 
Java does not support unsigned scalars. This means that any unsigned types you use in your schema will actually be represented as a signed value. This means all bits are still present, but may represent a negative value when used. For example, to read a byte b as an unsigned number, you can do: (short)(b & 0xFF)

 
Sadly the string accessors currently always create a new string when accessed, since FlatBuffer's UTF-8 strings can't be read in-place by Java.

-
Vector access is also a bit different from C++: you pass an extra index to the vector field accessor. Then a second method with the same name suffixed by _length let's you know the number of elements you can access: 
for (int i = 0; i < monster.inventory_length(); i++)
+
Vector access is also a bit different from C++: you pass an extra index to the vector field accessor. Then a second method with the same name suffixed by Length let's you know the number of elements you can access: 
for (int i = 0; i < monster.inventoryLength(); i++)
     monster.inventory(i); // do something here
 
You can also construct these buffers in Java using the static methods found in the generated code, and the FlatBufferBuilder class: 
FlatBufferBuilder fbb = new FlatBufferBuilder();
 
Create strings: 
int str = fbb.createString("MyMonster");
@@ -74,7 +74,10 @@
 Monster.addTest(fbb, mon2);
 Monster.addTest4(fbb, test4s);
 int mon = Monster.endMonster(fbb);
-
As you can see, the Java code for tables does not use a convenient createMonster call like the C++ code. This is to create the buffer without using temporary object allocation (since the Vec3 is an inline component of Monster, it has to be created right where it is added, whereas the name and the inventory are not inline). Structs do have convenient methods that even have arguments for nested structs.

+
As you can see, the Java code for tables does not use a convenient createMonster call like the C++ code. This is to create the buffer without using temporary object allocation.

+
It's important to understand that fields that are structs are inline (like Vec3 above), and MUST thus be created between the start and end calls of a table. Everything else (other tables, strings, vectors) MUST be created before the start of the table they are referenced in.

+
Structs do have convenient methods that even have arguments for nested structs.

+
As you can see, references to other objects (e.g. the string above) are simple ints, and thus do not have the type-safety of the Offset type in C++. Extra case must thus be taken that you set the right offset on the right field.

 
Vectors also use this start/end pattern to allow vectors of both scalar types and structs: 
Monster.startInventoryVector(fbb, 5);
 for (byte i = 4; i >=0; i--) fbb.addByte(i);
 int inv = fbb.endVector();
diff --git a/docs/html/md__schemas.html b/docs/html/md__schemas.html
index 6f1c6f614d8..6662ed3a378 100644
--- a/docs/html/md__schemas.html
+++ b/docs/html/md__schemas.html
@@ -111,9 +111,21 @@ 
Unions

 
Unions share a lot of properties with enums, but instead of new names for constants, you use names of tables. You can then declare a union field which can hold a reference to any of those types, and additionally a hidden field with the suffix _type is generated that holds the corresponding enum value, allowing you to know which type to cast to at runtime.

 
Namespaces

 
These will generate the corresponding namespace in C++ for all helper code, and packages in Java. You can use . to specify nested namespaces / packages.

+
Includes

+
You can include other schemas files in your current one, e.g.: 
include "mydefinitions.fbs"
+
This makes it easier to refer to types defined elsewhere. include automatically ensures each file is parsed just once, even when referred to more than once.

+
When using the flatc compiler to generate code for schema definitions, only definitions in the current file will be generated, not those from the included files (those you still generate separately).

 
Root type

-
This declares what you consider to be the root table (or struct) of the serialized data.

-
Comments & documentation

+
This declares what you consider to be the root table (or struct) of the serialized data. This is particular important for parsing JSON data, which doesn't include object type information.

+
File identification and extension

+
Typically, a FlatBuffer binary buffer is not self-describing, i.e. it needs you to know its schema to parse it correctly. But if you want to use a FlatBuffer as a file format, it would be convenient to be able to have a "magic number" in there, like most file formats have, to be able to do a sanity check to see if you're reading the kind of file you're expecting.

+
Now, you can always prefix a FlatBuffer with your own file header, but FlatBuffers has a built-in way to add an identifier to a FlatBuffer that takes up minimal space, and keeps the buffer compatible with buffers that don't have such an identifier.

+
You can specify in a schema, similar to root_type, that you intend for this type of FlatBuffer to be used as a file format: 
file_identifier "MYFI";
+
Identifiers must always be exactly 4 characters long. These 4 characters will end up as bytes at offsets 4-7 (inclusive) in the buffer.

+
For any schema that has such an identifier, flatc will automatically add the identifier to any binaries it generates (with -b), and generated calls like FinishMonsterBuffer also add the identifier. If you have specified an identifier and wish to generate a buffer without one, you can always still do so by calling FlatBufferBuilder::Finish explicitly.

+
After loading a buffer, you can use a call like MonsterBufferHasIdentifier to check if the identifier is present.

+
Additionally, by default flatc will output binary files as .bin. This declaration in the schema will change that to whatever you want: 
file_extension "ext";
+
Comments & documentation

 
May be written as in most C-based languages. Additionally, a triple comment (///) on a line by itself signals that a comment is documentation for whatever is declared on the line after it (table/struct/field/enum/union/element), and the comment is output in the corresponding C++ code. Multiple such lines per item are allowed.

 
Attributes

 
Attributes may be attached to a declaration, behind a field, or after the name of a table/struct/enum/union. These may either have a value or not. Some attributes like deprecated are understood by the compiler, others are simply ignored (like priority), but are available to query if you parse the schema at runtime. This is useful if you write your own code generators/editors etc., and you wish to add additional information specific to your tool (such as a help text).

@@ -123,7 +135,29 @@ 
Attributes

 
  • deprecated (on a field): do not generate accessors for this field anymore, code should stop using this data.
    • 
 
  • original_order (on a table): since elements in a table do not need to be stored in any particular order, they are often optimized for space by sorting them to size. This attribute stops that from happening.
    • 
 
  • force_align: size (on a struct): force the alignment of this struct to be something higher than what it is naturally aligned to. Causes these structs to be aligned to that amount inside a buffer, IF that buffer is allocated with that alignment (which is not necessarily the case for buffers accessed directly inside a FlatBufferBuilder).
    • 
+
  • bit_flags (on an enum): the values of this field indicate bits, meaning that any value N specified in the schema will end up representing 1<<N, or if you don't specify values at all, you'll get the sequence 1, 2, 4, 8, ...
    • 
 
+
    JSON Parsing
    
+
    The same parser that parses the schema declarations above is also able to parse JSON objects that conform to this schema. So, unlike other JSON parsers, this parser is strongly typed, and parses directly into a FlatBuffer (see the compiler documentation on how to do this from the command line, or the C++ documentation on how to do this at runtime).
    
+
    Besides needing a schema, there are a few other changes to how it parses JSON:
    
+
      
+
    • It accepts field names with and without quotes, like many JSON parsers already do. It outputs them without quotes as well, though can be made to output them using the strict_json flag.
      • 
+
    • If a field has an enum type, the parser will recognize symbolic enum values (with or without quotes) instead of numbers, e.g. field: EnumVal. If a field is of integral type, you can still use symbolic names, but values need to be prefixed with their type and need to be quoted, e.g. field: "Enum.EnumVal". For enums representing flags, you may place multiple inside a string separated by spaces to OR them, e.g. field: "EnumVal1 EnumVal2" or field: "Enum.EnumVal1 Enum.EnumVal2".
      • 
+
    
+
    When parsing JSON, it recognizes the following escape codes in strings:
    
+
      
+
    • \n - linefeed.
      • 
+
    • \t - tab.
      • 
+
    • \r - carriage return.
      • 
+
    • \b - backspace.
      • 
+
    • \f - form feed.
      • 
+
    • \" - double quote.
      • 
+
    • \\ - backslash.
      • 
+
    • \/ - forward slash.
      • 
+
    • \uXXXX - 16-bit unicode code point, converted to the equivalent UTF-8 representation.
      • 
+
    • \xXX - 8-bit binary hexadecimal number XX. This is the only one that is not in the JSON spec (see http://json.org/), but is needed to be able to encode arbitrary binary in strings to text and back without losing information (e.g. the byte 0xFF can't be represented in standard JSON).
      • 
+
    
+
    It also generates these escape codes back again when generating JSON from a binary representation.
    
 
    Gotchas
    
 
    Schemas and version control
    
 
    FlatBuffers relies on new field declarations being added at the end, and earlier declarations to not be removed, but be marked deprecated when needed. We think this is an improvement over the manual number assignment that happens in Protocol Buffers (and which is still an option using the id attribute mentioned above).
    
diff --git a/docs/html/navtree.js b/docs/html/navtree.js
index abdc1d0a20b..9ff8c25e10c 100644
--- a/docs/html/navtree.js
+++ b/docs/html/navtree.js
@@ -5,6 +5,7 @@ var NAVTREE =
     [ "Using the schema compiler", "md__compiler.html", null ],
     [ "Writing a schema", "md__schemas.html", null ],
     [ "Use in C++", "md__cpp_usage.html", null ],
+    [ "Use in Go", "md__go_usage.html", null ],
     [ "Use in Java", "md__java_usage.html", null ],
     [ "Benchmarks", "md__benchmarks.html", null ],
     [ "FlatBuffers white paper", "md__white_paper.html", null ],
diff --git a/docs/html/navtreeindex0.js b/docs/html/navtreeindex0.js
index 45204501305..7dda4761831 100644
--- a/docs/html/navtreeindex0.js
+++ b/docs/html/navtreeindex0.js
@@ -1,14 +1,15 @@
 var NAVTREEINDEX0 =
 {
 "index.html":[],
-"md__benchmarks.html":[5],
+"md__benchmarks.html":[6],
 "md__building.html":[0],
 "md__compiler.html":[1],
 "md__cpp_usage.html":[3],
-"md__grammar.html":[8],
-"md__internals.html":[7],
-"md__java_usage.html":[4],
+"md__go_usage.html":[4],
+"md__grammar.html":[9],
+"md__internals.html":[8],
+"md__java_usage.html":[5],
 "md__schemas.html":[2],
-"md__white_paper.html":[6],
+"md__white_paper.html":[7],
 "pages.html":[]
 };
diff --git a/docs/html/pages.html b/docs/html/pages.html
index 4d71a95e189..5810c72252a 100644
--- a/docs/html/pages.html
+++ b/docs/html/pages.html
@@ -59,11 +59,12 @@
  Using the schema compiler
  Writing a schema
  Use in C++
- Use in Java
- Benchmarks
- FlatBuffers white paper
- FlatBuffer Internals
- Formal Grammar of the schema language
+ Use in Go
+ Use in Java
+ Benchmarks
+ FlatBuffers white paper
+ FlatBuffer Internals
+ Formal Grammar of the schema language
    diff --git a/docs/source/Benchmarks.md b/docs/source/Benchmarks.md index 85305cb9118..6792bbbdc1f 100755 --- a/docs/source/Benchmarks.md +++ b/docs/source/Benchmarks.md @@ -2,7 +2,8 @@ Comparing against other serialization solutions, running on Windows 7 64bit. We use the LITE runtime for Protocol Buffers (less code / lower -overhead), and Rapid JSON, one of the fastest C++ JSON parsers around. +overhead), Rapid JSON (one of the fastest C++ JSON parsers around), +and pugixml, also one of the fastest XML parsers. We compare against Flatbuffers with the binary wire format (as intended), and also with JSON as the wire format with the optional JSON @@ -13,17 +14,17 @@ The benchmark object is a set of about 10 objects containing an array, 4 strings, and a large variety of int/float scalar values of all sizes, meant to be representative of game data, e.g. a scene format. -| | FlatBuffers (binary) | Protocol Buffers LITE | Rapid JSON | FlatBuffers (JSON) | -|--------------------------------------------------------|-----------------------|-----------------------|-----------------------|-----------------------| -| Decode + Traverse + Dealloc (1 million times, seconds) | 0.08 | 302 | 583 | 105 | -| Decode / Traverse / Dealloc (breakdown) | 0 / 0.08 / 0 | 220 / 0.15 / 81 | 294 / 0.9 / 287 | 70 / 0.08 / 35 | -| Encode (1 million times, seconds) | 3.2 | 185 | 650 | 169 | -| Wire format size (normal / zlib, bytes) | 344 / 220 | 228 / 174 | 1475 / 322 | 1029 / 298 | -| Memory needed to store decoded wire (bytes / blocks) | 0 / 0 | 760 / 20 | 65689 / 40 | 328 / 1 | -| Transient memory allocated during decode (KB) | 0 | 1 | 131 | 4 | -| Generated source code size (KB) | 4 | 61 | 0 | 4 | -| Field access in handwritten traversal code | accessors | accessors | manual error checking | accessors | -| Library source code (KB) | 15 | some subset of 3800 | 87 | 43 | +| | FlatBuffers (binary) | Protocol Buffers LITE | Rapid JSON | FlatBuffers (JSON) | pugixml | +|--------------------------------------------------------|-----------------------|-----------------------|-----------------------|-----------------------| ----------------------| +| Decode + Traverse + Dealloc (1 million times, seconds) | 0.08 | 302 | 583 | 105 | 196 | +| Decode / Traverse / Dealloc (breakdown) | 0 / 0.08 / 0 | 220 / 0.15 / 81 | 294 / 0.9 / 287 | 70 / 0.08 / 35 | 41 / 3.9 / 150 | +| Encode (1 million times, seconds) | 3.2 | 185 | 650 | 169 | 273 | +| Wire format size (normal / zlib, bytes) | 344 / 220 | 228 / 174 | 1475 / 322 | 1029 / 298 | 1137 / 341 | +| Memory needed to store decoded wire (bytes / blocks) | 0 / 0 | 760 / 20 | 65689 / 4 | 328 / 1 | 34194 / 3 | +| Transient memory allocated during decode (KB) | 0 | 1 | 131 | 4 | 34 | +| Generated source code size (KB) | 4 | 61 | 0 | 4 | 0 | +| Field access in handwritten traversal code | typed accessors | typed accessors | manual error checking | typed accessors | manual error checking | +| Library source code (KB) | 15 | some subset of 3800 | 87 | 43 | 327 | ### Some other serialization systems we compared against but did not benchmark (yet), in rough order of applicability: @@ -36,8 +37,6 @@ meant to be representative of game data, e.g. a scene format. with the typed C++ interface. Also lacks VS2010 support. - Thrift: very similar to Protocol Buffers, but appears to be less efficient, and have more dependencies. -- XML: typically even slower than JSON, but has the advantage that it can be - parsed with a schema to reduce error-checking boilerplate code. - YAML: a superset of JSON and otherwise very similar. Used by e.g. Unity. - C# comes with built-in serialization functionality, as used by Unity also. Being tied to the language, and having no automatic versioning support diff --git a/docs/source/Compiler.md b/docs/source/Compiler.md index e1be0a3f187..4fc930b7729 100755 --- a/docs/source/Compiler.md +++ b/docs/source/Compiler.md @@ -34,3 +34,6 @@ be generated for each file processed: - `-S` : Generate strict JSON (field names are enclosed in quotes). By default, no quotes are generated. + +- `-P` : Don't prefix enum values in generated C++ by their enum type. + diff --git a/docs/source/CppUsage.md b/docs/source/CppUsage.md index 5fe14cdadf4..c269e1bbc78 100755 --- a/docs/source/CppUsage.md +++ b/docs/source/CppUsage.md @@ -88,7 +88,7 @@ Regardless of whether you used `CreateMonster` or `MonsterBuilder`, you now have an offset to the root of your data, and you can finish the buffer using: - fbb.Finish(mloc); + FinishMonsterBuffer(fbb, mloc); The buffer is now ready to be stored somewhere, sent over the network, be compressed, or whatever you'd like to do with it. You can access the @@ -188,6 +188,11 @@ a full traversal (since any scalar data is not actually touched), and since it may cause the buffer to be brought into cache before reading, the actual overhead may be even lower than expected. +In specialized cases where a denial of service attack is possible, +the verifier has two additional constructor arguments that allow +you to limit the nesting depth and total amount of tables the +verifier may encounter before declaring the buffer malformed. + ## Text & schema parsing Using binary buffers with the generated header provides a super low @@ -199,6 +204,9 @@ Another reason might be that you already have a lot of data in JSON format, or a tool that generates JSON, and if you can write a schema for it, this will provide you an easy way to use that data directly. +(see the schema documentation for some specifics on the JSON format +accepted). + There are two ways to use text formats: ### Using the compiler as a conversion tool diff --git a/docs/source/FlatBuffers.md b/docs/source/FlatBuffers.md index 8f0401e41d3..fd3085551c7 100644 --- a/docs/source/FlatBuffers.md +++ b/docs/source/FlatBuffers.md @@ -1,8 +1,8 @@ # FlatBuffers -FlatBuffers is an efficient cross platform serialization library in for C++ and -Java. It was created at Google specifically for game development and other -performance-critical applications. +FlatBuffers is an efficient cross platform serialization library for C++, +with support for Java and Go. It was created at Google specifically for game +development and other performance-critical applications. It is available as open source under the Apache license, v2 (see LICENSE.txt). @@ -46,9 +46,9 @@ It is available as open source under the Apache license, v2 (see LICENSE.txt). needed (faster and more memory efficient than other JSON parsers). - Java code supports object-reuse. + Java and Go code supports object-reuse. -- **Cross platform C++11/Java code with no dependencies** - will work with +- **Cross platform C++11/Java/Go code with no dependencies** - will work with any recent gcc/clang and VS2010. Comes with build files for the tests & samples (Android .mk files, and cmake for all other platforms). @@ -87,7 +87,7 @@ sections provide a more in-depth usage guide. Fields are optional and have defaults, so they don't need to be present for every object instance. -- Use `flatc` (the FlatBuffer compiler) to generate a C++ header (or Java +- Use `flatc` (the FlatBuffer compiler) to generate a C++ header (or Java/Go classes) with helper classes to access and construct serialized data. This header (say `mydata_generated.h`) only depends on `flatbuffers.h`, which defines the core functionality. @@ -112,6 +112,8 @@ sections provide a more in-depth usage guide. programs. - How to [use the generated Java code](md__java_usage.html) in your own programs. +- How to [use the generated Go code](md__go_usage.html) in your own + programs. - Some [benchmarks](md__benchmarks.html) showing the advantage of using FlatBuffers. - A [white paper](md__white_paper.html) explaining the "why" of FlatBuffers. diff --git a/docs/source/GoUsage.md b/docs/source/GoUsage.md new file mode 100755 index 00000000000..0c2370b8ac6 --- /dev/null +++ b/docs/source/GoUsage.md @@ -0,0 +1,97 @@ +# Use in Go + +There's experimental support for reading FlatBuffers in Go. Generate code +for Go with the `-g` option to `flatc`. + +See `go_test.go` for an example. You import the generated code, read a +FlatBuffer binary file into a `[]byte`, which you pass to the +`GetRootAsMonster` function: + + import ( + example "MyGame/Example" + flatbuffers "github.com/google/flatbuffers/go" + + io/ioutil + ) + + buf, err := ioutil.ReadFile("monster.dat") + // handle err + monster := example.GetRootAsMonster(buf, 0) + +Now you can access values like this: + + hp := monster.Hp() + pos := monster.Pos(nil) + +Note that whenever you access a new object like in the `Pos` example above, +a new temporary accessor object gets created. If your code is very performance +sensitive (you iterate through a lot of objects), you can replace nil with a +pointer to a `Vec3` object you've already created. This allows +you to reuse it across many calls and reduce the amount of object allocation +(and thus garbage collection) your program does. + +To access vectors you pass an extra index to the +vector field accessor. Then a second method with the same name suffixed +by `Length` let's you know the number of elements you can access: + + for i := 0; i < monster.InventoryLength(); i++ { + monster.Inventory(i) // do something here + } + +You can also construct these buffers in Go using the functions found in the +generated code, and the FlatBufferBuilder class: + + builder := flatbuffers.NewBuilder(0) + +Create strings: + + str := builder.CreateString("MyMonster") + +Create a table with a struct contained therein: + + example.MonsterStart(builder) + example.MonsterAddPos(builder, example.CreateVec3(builder, 1.0, 2.0, 3.0, 3.0, 4, 5, 6)) + example.MonsterAddHp(builder, 80) + example.MonsterAddName(builder, str) + example.MonsterAddInventory(builder, inv) + example.MonsterAddTest_Type(builder, 1) + example.MonsterAddTest(builder, mon2) + example.MonsterAddTest4(builder, test4s) + mon := example.MonsterEnd(builder) + +Unlike C++, Go does not support table creation functions like 'createMonster()'. +This is to create the buffer without +using temporary object allocation (since the `Vec3` is an inline component of +`Monster`, it has to be created right where it is added, whereas the name and +the inventory are not inline). +Structs do have convenient methods that allow you to construct them in one call. +These also have arguments for nested structs, e.g. if a struct has a field `a` +and a nested struct field `b` (which has fields `c` and `d`), then the arguments +will be `a`, `c` and `d`. + +Vectors also use this start/end pattern to allow vectors of both scalar types +and structs: + + example.MonsterStartInventoryVector(builder, 5) + for i := 4; i >= 0; i-- { + builder.PrependByte(byte(i)) + } + inv := builder.EndVector(5) + +The generated method 'StartInventoryVector' is provided as a convenience +function which calls 'StartVector' with the correct element size of the vector +type which in this case is 'ubyte' or 1 byte per vector element. +You pass the number of elements you want to write. +You write the elements backwards since the buffer +is being constructed back to front. + +There are `Prepend` functions for all the scalar types. You use +`PrependUOffset` for any previously constructed objects (such as other tables, +strings, vectors). For structs, you use the appropriate `create` function +in-line, as shown above in the `Monster` example. + +## Text Parsing + +There currently is no support for parsing text (Schema's and JSON) directly +from Go, though you could use the C++ parser through cgo. Please see the +C++ documentation for more on text parsing. diff --git a/docs/source/Grammar.md b/docs/source/Grammar.md index 6242bfa6be8..33bd1709990 100755 --- a/docs/source/Grammar.md +++ b/docs/source/Grammar.md @@ -1,6 +1,9 @@ # Formal Grammar of the schema language -schema = namespace\_decl | type\_decl | enum\_decl | root\_decl | object +schema = include* + ( namespace\_decl | type\_decl | enum\_decl | root\_decl | object )* + +include = `include` string\_constant `;` namespace\_decl = `namespace` ident ( `.` ident )* `;` diff --git a/docs/source/Internals.md b/docs/source/Internals.md index b0fbbdc6999..bf4b58b4962 100755 --- a/docs/source/Internals.md +++ b/docs/source/Internals.md @@ -20,7 +20,9 @@ order, and objects to some extend can be stored in many orders. This is because the format doesn't need this information to be efficient, and it leaves room for optimization and extension (for example, fields can be packed in a way that is most compact). Instead, the format is defined in -terms of offsets and adjacency only. +terms of offsets and adjacency only. This may mean two different +implementations may produce different binaries given the same input +values, and this is perfectly valid. ### Format identification @@ -157,7 +159,8 @@ Unions share a lot with enums. struct Vec3; struct Monster; -Predeclare all datatypes since there may be circular references. +Predeclare all data types since circular references between types are allowed +(circular references between object are not, though). MANUALLY_ALIGNED_STRUCT(4) Vec3 { private: diff --git a/docs/source/JavaUsage.md b/docs/source/JavaUsage.md index f231a2f27d0..4e2fe8bc85f 100755 --- a/docs/source/JavaUsage.md +++ b/docs/source/JavaUsage.md @@ -1,11 +1,11 @@ # Use in Java -There's experimental support for reading FlatBuffers in Java. Generate code +FlatBuffers supports reading and writing binary FlatBuffers in Java. Generate code for Java with the `-j` option to `flatc`. See `javaTest.java` for an example. Essentially, you read a FlatBuffer binary file into a `byte[]`, which you then turn into a `ByteBuffer`, which you pass to -the `getRootAsMonster` function: +the `getRootAsMyRootType` function: ByteBuffer bb = ByteBuffer.wrap(data); Monster monster = Monster.getRootAsMonster(bb); @@ -33,9 +33,9 @@ since FlatBuffer's UTF-8 strings can't be read in-place by Java. Vector access is also a bit different from C++: you pass an extra index to the vector field accessor. Then a second method with the same name -suffixed by `_length` let's you know the number of elements you can access: +suffixed by `Length` let's you know the number of elements you can access: - for (int i = 0; i < monster.inventory_length(); i++) + for (int i = 0; i < monster.inventoryLength(); i++) monster.inventory(i); // do something here You can also construct these buffers in Java using the static methods found @@ -61,11 +61,19 @@ Create a table with a struct contained therein: As you can see, the Java code for tables does not use a convenient `createMonster` call like the C++ code. This is to create the buffer without -using temporary object allocation (since the `Vec3` is an inline component of -`Monster`, it has to be created right where it is added, whereas the name and -the inventory are not inline). +using temporary object allocation. + +It's important to understand that fields that are structs are inline (like +`Vec3` above), and MUST thus be created between the start and end calls of +a table. Everything else (other tables, strings, vectors) MUST be created +before the start of the table they are referenced in. + Structs do have convenient methods that even have arguments for nested structs. +As you can see, references to other objects (e.g. the string above) are simple +ints, and thus do not have the type-safety of the Offset type in C++. Extra +case must thus be taken that you set the right offset on the right field. + Vectors also use this start/end pattern to allow vectors of both scalar types and structs: diff --git a/docs/source/Schemas.md b/docs/source/Schemas.md index debab679438..f3afdfa6d76 100755 --- a/docs/source/Schemas.md +++ b/docs/source/Schemas.md @@ -141,10 +141,62 @@ These will generate the corresponding namespace in C++ for all helper code, and packages in Java. You can use `.` to specify nested namespaces / packages. +### Includes + +You can include other schemas files in your current one, e.g.: + + include "mydefinitions.fbs" + +This makes it easier to refer to types defined elsewhere. `include` +automatically ensures each file is parsed just once, even when referred to +more than once. + +When using the `flatc` compiler to generate code for schema definitions, +only definitions in the current file will be generated, not those from the +included files (those you still generate separately). + ### Root type This declares what you consider to be the root table (or struct) of the -serialized data. +serialized data. This is particular important for parsing JSON data, +which doesn't include object type information. + +### File identification and extension + +Typically, a FlatBuffer binary buffer is not self-describing, i.e. it +needs you to know its schema to parse it correctly. But if you +want to use a FlatBuffer as a file format, it would be convenient +to be able to have a "magic number" in there, like most file formats +have, to be able to do a sanity check to see if you're reading the +kind of file you're expecting. + +Now, you can always prefix a FlatBuffer with your own file header, +but FlatBuffers has a built-in way to add an identifier to a +FlatBuffer that takes up minimal space, and keeps the buffer +compatible with buffers that don't have such an identifier. + +You can specify in a schema, similar to `root_type`, that you intend +for this type of FlatBuffer to be used as a file format: + + file_identifier "MYFI"; + +Identifiers must always be exactly 4 characters long. These 4 characters +will end up as bytes at offsets 4-7 (inclusive) in the buffer. + +For any schema that has such an identifier, `flatc` will automatically +add the identifier to any binaries it generates (with `-b`), +and generated calls like `FinishMonsterBuffer` also add the identifier. +If you have specified an identifier and wish to generate a buffer +without one, you can always still do so by calling +`FlatBufferBuilder::Finish` explicitly. + +After loading a buffer, you can use a call like +`MonsterBufferHasIdentifier` to check if the identifier is present. + +Additionally, by default `flatc` will output binary files as `.bin`. +This declaration in the schema will change that to whatever you want: + + file_extension "ext"; ### Comments & documentation @@ -188,6 +240,53 @@ Current understood attributes: these structs to be aligned to that amount inside a buffer, IF that buffer is allocated with that alignment (which is not necessarily the case for buffers accessed directly inside a `FlatBufferBuilder`). +- `bit_flags` (on an enum): the values of this field indicate bits, + meaning that any value N specified in the schema will end up + representing 1< +// +// * N, where N is the number of fields in +// the schema for this type. Includes deprecated fields. +// Thus, a vtable is made of 2 + N elements, each SizeVOffsetT bytes wide. +// +// An object has the following format: +// +// + +func (b *Builder) WriteVtable() (n UOffsetT) { + // Prepend a zero scalar to the object. Later in this function we'll + // write an offset here that points to the object's vtable: + b.PrependSOffsetT(0) + + objectOffset := b.Offset() + existingVtable := UOffsetT(0) + + // Search backwards through existing vtables, because similar vtables + // are likely to have been recently appended. See + // BenchmarkVtableDeduplication for a case in which this heuristic + // saves about 30% of the time used in writing objects with duplicate + // tables. + for i := len(b.vtables) - 1; i >= 0; i-- { + // Find the other vtable, which is associated with `i`: + vt2Offset := b.vtables[i] + vt2Start := len(b.Bytes) - int(vt2Offset) + vt2Len := GetVOffsetT(b.Bytes[vt2Start:]) + + metadata := VtableMetadataFields * SizeVOffsetT + vt2End := vt2Start + int(vt2Len) + vt2 := b.Bytes[vt2Start+metadata : vt2End] + + // Compare the other vtable to the one under consideration. + // If they are equal, store the offset and break: + if vtableEqual(b.vtable, objectOffset, vt2) { + existingVtable = vt2Offset + break + } + } + + if existingVtable == 0 { + // Did not find a vtable, so write this one to the buffer. + + // Write out the current vtable in reverse , because + // serialization occurs in last-first order: + for i := len(b.vtable) - 1; i >= 0; i-- { + var off UOffsetT + if b.vtable[i] != 0 { + // Forward reference to field; + // use 32bit number to ensure no overflow: + off = objectOffset - b.vtable[i] + } + + b.PrependVOffsetT(VOffsetT(off)) + } + + // The two metadata fields are written last. + + // First, store the object bytesize: + objectSize := objectOffset - b.objectEnd + b.PrependVOffsetT(VOffsetT(objectSize)) + + // Second, store the vtable bytesize: + vBytes := (len(b.vtable) + VtableMetadataFields) * SizeVOffsetT + b.PrependVOffsetT(VOffsetT(vBytes)) + + // Next, write the offset to the new vtable in the + // already-allocated SOffsetT at the beginning of this object: + objectStart := SOffsetT(len(b.Bytes)) - SOffsetT(objectOffset) + WriteSOffsetT(b.Bytes[objectStart:], + SOffsetT(b.Offset())-SOffsetT(objectOffset)) + + // Finally, store this vtable in memory for future + // deduplication: + b.vtables = append(b.vtables, b.Offset()) + } else { + // Found a duplicate vtable. + + objectStart := SOffsetT(len(b.Bytes)) - SOffsetT(objectOffset) + b.head = UOffsetT(objectStart) + + // Write the offset to the found vtable in the + // already-allocated SOffsetT at the beginning of this object: + WriteSOffsetT(b.Bytes[b.head:], + SOffsetT(existingVtable)-SOffsetT(objectOffset)) + } + + b.vtable = nil + return objectOffset +} + +// EndObject writes data necessary to finish object construction. +func (b *Builder) EndObject() UOffsetT { + if b.vtable == nil { + panic("not in object") + } + return b.WriteVtable() +} + +// Doubles the size of the byteslice, and copies the old data towards the +// end of the new byteslice (since we build the buffer backwards). +func (b *Builder) growByteBuffer() { + if (len(b.Bytes) & 0xC0000000) != 0 { + panic("cannot grow buffer beyond 2 gigabytes") + } + newSize := len(b.Bytes) * 2 + if newSize == 0 { + newSize = 1 + } + bytes2 := make([]byte, newSize) + copy(bytes2[newSize-len(b.Bytes):], b.Bytes) + b.Bytes = bytes2 +} + +// Head gives the start of useful data in the underlying byte buffer. +// Note: unlike other functions, this value is interpreted as from the left. +func (b *Builder) Head() UOffsetT { + return b.head +} + +// Offset relative to the end of the buffer. +func (b *Builder) Offset() UOffsetT { + return UOffsetT(len(b.Bytes)) - b.head +} + +// Pad places zeros at the current offset. +func (b *Builder) Pad(n int) { + for i := 0; i < n; i++ { + b.PlaceByte(0) + } +} + +// Prep prepares to write an element of `size` after `additional_bytes` +// have been written, e.g. if you write a string, you need to align such +// the int length field is aligned to SizeInt32, and the string data follows it +// directly. +// If all you need to do is align, `additionalBytes` will be 0. +func (b *Builder) Prep(size, additionalBytes int) { + // Track the biggest thing we've ever aligned to. + if size > b.minalign { + b.minalign = size + } + // Find the amount of alignment needed such that `size` is properly + // aligned after `additionalBytes`: + alignSize := (^(len(b.Bytes) - int(b.Head()) + additionalBytes)) + 1 + alignSize &= (size - 1) + + // Reallocate the buffer if needed: + for int(b.head) < alignSize+size+additionalBytes { + oldBufSize := len(b.Bytes) + b.growByteBuffer() + b.head += UOffsetT(len(b.Bytes) - oldBufSize) + } + b.Pad(alignSize) +} + +// PrependSOffsetT prepends an SOffsetT, relative to where it will be written. +func (b *Builder) PrependSOffsetT(off SOffsetT) { + b.Prep(SizeSOffsetT, 0) // Ensure alignment is already done. + if !(UOffsetT(off) <= b.Offset()) { + panic("unreachable: off <= b.Offset()") + } + off2 := SOffsetT(b.Offset()) - off + SOffsetT(SizeSOffsetT) + b.PlaceSOffsetT(off2) +} + +// PrependUOffsetT prepends an UOffsetT, relative to where it will be written. +func (b *Builder) PrependUOffsetT(off UOffsetT) { + b.Prep(SizeUOffsetT, 0) // Ensure alignment is already done. + if !(off <= b.Offset()) { + panic("unreachable: off <= b.Offset()") + } + off2 := b.Offset() - off + UOffsetT(SizeUOffsetT) + b.PlaceUOffsetT(off2) +} + +// StartVector initializes bookkeeping for writing a new vector. +// +// A vector has the following format: +// +// +, where T is the type of elements of this vector. +func (b *Builder) StartVector(elemSize, numElems int) UOffsetT { + b.notNested() + b.Prep(SizeUint32, elemSize*numElems) + return b.Offset() +} + +// EndVector writes data necessary to finish vector construction. +func (b *Builder) EndVector(vectorNumElems int) UOffsetT { + // we already made space for this, so write without PrependUint32 + b.PlaceUOffsetT(UOffsetT(vectorNumElems)) + return b.Offset() +} + +// CreateString writes a null-terminated string as a vector. +func (b *Builder) CreateString(s string) UOffsetT { + b.Prep(int(SizeUOffsetT), (len(s)+1)*SizeByte) + b.PlaceByte(0) + + x := []byte(s) + l := UOffsetT(len(x)) + + b.head -= l + copy(b.Bytes[b.head:b.head+l], x) + + return b.EndVector(len(x)) +} + +func (b *Builder) notNested() { + // Check that no other objects are being built while making this + // object. If not, panic: + if b.vtable != nil { + panic("non-inline data write inside of object") + } +} + +func (b *Builder) nested(obj UOffsetT) { + // Structs are always stored inline, so need to be created right + // where they are used. You'll get this panic if you created it + // elsewhere: + if obj != b.Offset() { + panic("inline data write outside of object") + } +} + +// PrependBoolSlot prepends a bool onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependBoolSlot(o int, x, d bool) { + val := byte(0) + if x { + val = 1 + } + def := byte(0) + if d { + def = 1 + } + b.PrependByteSlot(o, val, def) +} + +// PrependByteSlot prepends a byte onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependByteSlot(o int, x, d byte) { + if x != d { + b.PrependByte(x) + b.Slot(o) + } +} + +// PrependUint8Slot prepends a uint8 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependUint8Slot(o int, x, d uint8) { + if x != d { + b.PrependUint8(x) + b.Slot(o) + } +} + +// PrependUint16Slot prepends a uint16 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependUint16Slot(o int, x, d uint16) { + if x != d { + b.PrependUint16(x) + b.Slot(o) + } +} + +// PrependUint32Slot prepends a uint32 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependUint32Slot(o int, x, d uint32) { + if x != d { + b.PrependUint32(x) + b.Slot(o) + } +} + +// PrependUint64Slot prepends a uint64 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependUint64Slot(o int, x, d uint64) { + if x != d { + b.PrependUint64(x) + b.Slot(o) + } +} + +// PrependInt8Slot prepends a int8 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependInt8Slot(o int, x, d int8) { + if x != d { + b.PrependInt8(x) + b.Slot(o) + } +} + +// PrependInt16Slot prepends a int16 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependInt16Slot(o int, x, d int16) { + if x != d { + b.PrependInt16(x) + b.Slot(o) + } +} + +// PrependInt32Slot prepends a int32 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependInt32Slot(o int, x, d int32) { + if x != d { + b.PrependInt32(x) + b.Slot(o) + } +} + +// PrependInt64Slot prepends a int64 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependInt64Slot(o int, x, d int64) { + if x != d { + b.PrependInt64(x) + b.Slot(o) + } +} + +// PrependFloat32Slot prepends a float32 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependFloat32Slot(o int, x, d float32) { + if x != d { + b.PrependFloat32(x) + b.Slot(o) + } +} + +// PrependFloat64Slot prepends a float64 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependFloat64Slot(o int, x, d float64) { + if x != d { + b.PrependFloat64(x) + b.Slot(o) + } +} + +// PrependUOffsetTSlot prepends an UOffsetT onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependUOffsetTSlot(o int, x, d UOffsetT) { + if x != d { + b.PrependUOffsetT(x) + b.Slot(o) + } +} + +// PrependStructSlot prepends a struct onto the object at vtable slot `o`. +// Structs are stored inline, so nothing additional is being added. +// In generated code, `d` is always 0. +func (b *Builder) PrependStructSlot(voffset int, x, d UOffsetT) { + if x != d { + b.nested(x) + b.Slot(voffset) + } +} + +// Slot sets the vtable key `voffset` to the current location in the buffer. +func (b *Builder) Slot(slotnum int) { + b.vtable[slotnum] = UOffsetT(b.Offset()) +} + +// Finish finalizes a buffer, pointing to the given `rootTable`. +func (b *Builder) Finish(rootTable UOffsetT) { + b.Prep(b.minalign, SizeUOffsetT) + b.PrependUOffsetT(rootTable) +} + +// vtableEqual compares an unwritten vtable to a written vtable. +func vtableEqual(a []UOffsetT, objectStart UOffsetT, b []byte) bool { + if len(a)*SizeVOffsetT != len(b) { + return false + } + + for i := 0; i < len(a); i++ { + x := GetVOffsetT(b[i*SizeVOffsetT : (i+1)*SizeVOffsetT]) + + // Skip vtable entries that indicate a default value. + if x == 0 && a[i] == 0 { + continue + } + + y := SOffsetT(objectStart) - SOffsetT(a[i]) + if SOffsetT(x) != y { + return false + } + } + return true +} + +// PrependBool prepends a bool to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependBool(x bool) { + b.Prep(SizeBool, 0) + b.PlaceBool(x) +} + +// PrependUint8 prepends a uint8 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependUint8(x uint8) { + b.Prep(SizeUint8, 0) + b.PlaceUint8(x) +} + +// PrependUint16 prepends a uint16 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependUint16(x uint16) { + b.Prep(SizeUint16, 0) + b.PlaceUint16(x) +} + +// PrependUint32 prepends a uint32 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependUint32(x uint32) { + b.Prep(SizeUint32, 0) + b.PlaceUint32(x) +} + +// PrependUint64 prepends a uint64 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependUint64(x uint64) { + b.Prep(SizeUint64, 0) + b.PlaceUint64(x) +} + +// PrependInt8 prepends a int8 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependInt8(x int8) { + b.Prep(SizeInt8, 0) + b.PlaceInt8(x) +} + +// PrependInt16 prepends a int16 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependInt16(x int16) { + b.Prep(SizeInt16, 0) + b.PlaceInt16(x) +} + +// PrependInt32 prepends a int32 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependInt32(x int32) { + b.Prep(SizeInt32, 0) + b.PlaceInt32(x) +} + +// PrependInt64 prepends a int64 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependInt64(x int64) { + b.Prep(SizeInt64, 0) + b.PlaceInt64(x) +} + +// PrependFloat32 prepends a float32 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependFloat32(x float32) { + b.Prep(SizeFloat32, 0) + b.PlaceFloat32(x) +} + +// PrependFloat64 prepends a float64 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependFloat64(x float64) { + b.Prep(SizeFloat64, 0) + b.PlaceFloat64(x) +} + +// PrependByte prepends a byte to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependByte(x byte) { + b.Prep(SizeByte, 0) + b.PlaceByte(x) +} + +// PrependVOffsetT prepends a VOffsetT to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependVOffsetT(x VOffsetT) { + b.Prep(SizeVOffsetT, 0) + b.PlaceVOffsetT(x) +} + +// PlaceBool prepends a bool to the Builder, without checking for space. +func (b *Builder) PlaceBool(x bool) { + b.head -= UOffsetT(SizeBool) + WriteBool(b.Bytes[b.head:], x) +} + +// PlaceUint8 prepends a uint8 to the Builder, without checking for space. +func (b *Builder) PlaceUint8(x uint8) { + b.head -= UOffsetT(SizeUint8) + WriteUint8(b.Bytes[b.head:], x) +} + +// PlaceUint16 prepends a uint16 to the Builder, without checking for space. +func (b *Builder) PlaceUint16(x uint16) { + b.head -= UOffsetT(SizeUint16) + WriteUint16(b.Bytes[b.head:], x) +} + +// PlaceUint32 prepends a uint32 to the Builder, without checking for space. +func (b *Builder) PlaceUint32(x uint32) { + b.head -= UOffsetT(SizeUint32) + WriteUint32(b.Bytes[b.head:], x) +} + +// PlaceUint64 prepends a uint64 to the Builder, without checking for space. +func (b *Builder) PlaceUint64(x uint64) { + b.head -= UOffsetT(SizeUint64) + WriteUint64(b.Bytes[b.head:], x) +} + +// PlaceInt8 prepends a int8 to the Builder, without checking for space. +func (b *Builder) PlaceInt8(x int8) { + b.head -= UOffsetT(SizeInt8) + WriteInt8(b.Bytes[b.head:], x) +} + +// PlaceInt16 prepends a int16 to the Builder, without checking for space. +func (b *Builder) PlaceInt16(x int16) { + b.head -= UOffsetT(SizeInt16) + WriteInt16(b.Bytes[b.head:], x) +} + +// PlaceInt32 prepends a int32 to the Builder, without checking for space. +func (b *Builder) PlaceInt32(x int32) { + b.head -= UOffsetT(SizeInt32) + WriteInt32(b.Bytes[b.head:], x) +} + +// PlaceInt64 prepends a int64 to the Builder, without checking for space. +func (b *Builder) PlaceInt64(x int64) { + b.head -= UOffsetT(SizeInt64) + WriteInt64(b.Bytes[b.head:], x) +} + +// PlaceFloat32 prepends a float32 to the Builder, without checking for space. +func (b *Builder) PlaceFloat32(x float32) { + b.head -= UOffsetT(SizeFloat32) + WriteFloat32(b.Bytes[b.head:], x) +} + +// PlaceFloat64 prepends a float64 to the Builder, without checking for space. +func (b *Builder) PlaceFloat64(x float64) { + b.head -= UOffsetT(SizeFloat64) + WriteFloat64(b.Bytes[b.head:], x) +} + +// PlaceByte prepends a byte to the Builder, without checking for space. +func (b *Builder) PlaceByte(x byte) { + b.head -= UOffsetT(SizeByte) + WriteByte(b.Bytes[b.head:], x) +} + +// PlaceVOffsetT prepends a VOffsetT to the Builder, without checking for space. +func (b *Builder) PlaceVOffsetT(x VOffsetT) { + b.head -= UOffsetT(SizeVOffsetT) + WriteVOffsetT(b.Bytes[b.head:], x) +} + +// PlaceSOffsetT prepends a SOffsetT to the Builder, without checking for space. +func (b *Builder) PlaceSOffsetT(x SOffsetT) { + b.head -= UOffsetT(SizeSOffsetT) + WriteSOffsetT(b.Bytes[b.head:], x) +} + +// PlaceUOffsetT prepends a UOffsetT to the Builder, without checking for space. +func (b *Builder) PlaceUOffsetT(x UOffsetT) { + b.head -= UOffsetT(SizeUOffsetT) + WriteUOffsetT(b.Bytes[b.head:], x) +} diff --git a/go/doc.go b/go/doc.go new file mode 100644 index 00000000000..694edc763d8 --- /dev/null +++ b/go/doc.go @@ -0,0 +1,3 @@ +// Package flatbuffers provides facilities to read and write flatbuffers +// objects. +package flatbuffers diff --git a/go/encode.go b/go/encode.go new file mode 100644 index 00000000000..48ff36ef016 --- /dev/null +++ b/go/encode.go @@ -0,0 +1,216 @@ +package flatbuffers + +import ( + "math" +) + +type ( + // A SOffsetT stores a signed offset into arbitrary data. + SOffsetT int32 + // A UOffsetT stores an unsigned offset into vector data. + UOffsetT uint32 + // A VOffsetT stores an unsigned offset in a vtable. + VOffsetT uint16 +) + +const ( + // VtableMetadataFields is the count of metadata fields in each vtable. + VtableMetadataFields = 2 +) + +// GetByte decodes a little-endian byte from a byte slice. +func GetByte(buf []byte) byte { + return byte(GetUint8(buf)) +} + +// GetBool decodes a little-endian bool from a byte slice. +func GetBool(buf []byte) bool { + return buf[0] == 1 +} + +// GetUint8 decodes a little-endian uint8 from a byte slice. +func GetUint8(buf []byte) (n uint8) { + n = uint8(buf[0]) + return +} + +// GetUint16 decodes a little-endian uint16 from a byte slice. +func GetUint16(buf []byte) (n uint16) { + n |= uint16(buf[0]) + n |= uint16(buf[1]) << 8 + return +} + +// GetUint32 decodes a little-endian uint32 from a byte slice. +func GetUint32(buf []byte) (n uint32) { + n |= uint32(buf[0]) + n |= uint32(buf[1]) << 8 + n |= uint32(buf[2]) << 16 + n |= uint32(buf[3]) << 24 + return +} + +// GetUint64 decodes a little-endian uint64 from a byte slice. +func GetUint64(buf []byte) (n uint64) { + n |= uint64(buf[0]) + n |= uint64(buf[1]) << 8 + n |= uint64(buf[2]) << 16 + n |= uint64(buf[3]) << 24 + n |= uint64(buf[4]) << 32 + n |= uint64(buf[5]) << 40 + n |= uint64(buf[6]) << 48 + n |= uint64(buf[7]) << 56 + return +} + +// GetInt8 decodes a little-endian int8 from a byte slice. +func GetInt8(buf []byte) (n int8) { + n = int8(buf[0]) + return +} + +// GetInt16 decodes a little-endian int16 from a byte slice. +func GetInt16(buf []byte) (n int16) { + n |= int16(buf[0]) + n |= int16(buf[1]) << 8 + return +} + +// GetInt32 decodes a little-endian int32 from a byte slice. +func GetInt32(buf []byte) (n int32) { + n |= int32(buf[0]) + n |= int32(buf[1]) << 8 + n |= int32(buf[2]) << 16 + n |= int32(buf[3]) << 24 + return +} + +// GetInt64 decodes a little-endian int64 from a byte slice. +func GetInt64(buf []byte) (n int64) { + n |= int64(buf[0]) + n |= int64(buf[1]) << 8 + n |= int64(buf[2]) << 16 + n |= int64(buf[3]) << 24 + n |= int64(buf[4]) << 32 + n |= int64(buf[5]) << 40 + n |= int64(buf[6]) << 48 + n |= int64(buf[7]) << 56 + return +} + +// GetFloat32 decodes a little-endian float32 from a byte slice. +func GetFloat32(buf []byte) float32 { + x := GetUint32(buf) + return math.Float32frombits(x) +} + +// GetFloat64 decodes a little-endian float64 from a byte slice. +func GetFloat64(buf []byte) float64 { + x := GetUint64(buf) + return math.Float64frombits(x) +} + +// GetUOffsetT decodes a little-endian UOffsetT from a byte slice. +func GetUOffsetT(buf []byte) UOffsetT { + return UOffsetT(GetInt32(buf)) +} + +// GetSOffsetT decodes a little-endian SOffsetT from a byte slice. +func GetSOffsetT(buf []byte) SOffsetT { + return SOffsetT(GetInt32(buf)) +} + +// GetVOffsetT decodes a little-endian VOffsetT from a byte slice. +func GetVOffsetT(buf []byte) VOffsetT { + return VOffsetT(GetUint16(buf)) +} + +// WriteByte encodes a little-endian uint8 into a byte slice. +func WriteByte(buf []byte, n byte) { + WriteUint8(buf, uint8(n)) +} + +// WriteBool encodes a little-endian bool into a byte slice. +func WriteBool(buf []byte, b bool) { + buf[0] = 0 + if b { + buf[0] = 1 + } +} + +// WriteUint8 encodes a little-endian uint8 into a byte slice. +func WriteUint8(buf []byte, n uint8) { + buf[0] = byte(n) +} + +// WriteUint16 encodes a little-endian uint16 into a byte slice. +func WriteUint16(buf []byte, n uint16) { + buf[0] = byte(n) + buf[1] = byte(n >> 8) +} + +// WriteUint32 encodes a little-endian uint32 into a byte slice. +func WriteUint32(buf []byte, n uint32) { + buf[0] = byte(n) + buf[1] = byte(n >> 8) + buf[2] = byte(n >> 16) + buf[3] = byte(n >> 24) +} + +// WriteUint64 encodes a little-endian uint64 into a byte slice. +func WriteUint64(buf []byte, n uint64) { + for i := uint(0); i < uint(SizeUint64); i++ { + buf[i] = byte(n >> (i * 8)) + } +} + +// WriteInt8 encodes a little-endian int8 into a byte slice. +func WriteInt8(buf []byte, n int8) { + buf[0] = byte(n) +} + +// WriteInt16 encodes a little-endian int16 into a byte slice. +func WriteInt16(buf []byte, n int16) { + buf[0] = byte(n) + buf[1] = byte(n >> 8) +} + +// WriteInt32 encodes a little-endian int32 into a byte slice. +func WriteInt32(buf []byte, n int32) { + buf[0] = byte(n) + buf[1] = byte(n >> 8) + buf[2] = byte(n >> 16) + buf[3] = byte(n >> 24) +} + +// WriteInt64 encodes a little-endian int64 into a byte slice. +func WriteInt64(buf []byte, n int64) { + for i := uint(0); i < uint(SizeInt64); i++ { + buf[i] = byte(n >> (i * 8)) + } +} + +// WriteFloat32 encodes a little-endian float32 into a byte slice. +func WriteFloat32(buf []byte, n float32) { + WriteUint32(buf, math.Float32bits(n)) +} + +// WriteFloat64 encodes a little-endian float64 into a byte slice. +func WriteFloat64(buf []byte, n float64) { + WriteUint64(buf, math.Float64bits(n)) +} + +// WriteVOffsetT encodes a little-endian VOffsetT into a byte slice. +func WriteVOffsetT(buf []byte, n VOffsetT) { + WriteUint16(buf, uint16(n)) +} + +// WriteSOffsetT encodes a little-endian SOffsetT into a byte slice. +func WriteSOffsetT(buf []byte, n SOffsetT) { + WriteInt32(buf, int32(n)) +} + +// WriteUOffsetT encodes a little-endian UOffsetT into a byte slice. +func WriteUOffsetT(buf []byte, n UOffsetT) { + WriteUint32(buf, uint32(n)) +} diff --git a/go/struct.go b/go/struct.go new file mode 100644 index 00000000000..11258f715d4 --- /dev/null +++ b/go/struct.go @@ -0,0 +1,8 @@ +package flatbuffers + +// Struct wraps a byte slice and provides read access to its data. +// +// Structs do not have a vtable. +type Struct struct { + Table +} diff --git a/go/table.go b/go/table.go new file mode 100644 index 00000000000..21c1121817d --- /dev/null +++ b/go/table.go @@ -0,0 +1,290 @@ +package flatbuffers + +// Table wraps a byte slice and provides read access to its data. +// +// The variable `Pos` indicates the root of the FlatBuffers object therein. +type Table struct { + Bytes []byte + Pos UOffsetT // Always < 1<<31. +} + +// Offset provides access into the Table's vtable. +// +// Deprecated fields are ignored by checking against the vtable's length. +func (t *Table) Offset(vtableOffset VOffsetT) VOffsetT { + vtable := UOffsetT(SOffsetT(t.Pos) - t.GetSOffsetT(t.Pos)) + if vtableOffset < t.GetVOffsetT(vtable) { + return t.GetVOffsetT(vtable + UOffsetT(vtableOffset)) + } + return 0 +} + +// Indirect retrieves the relative offset stored at `offset`. +func (t *Table) Indirect(off UOffsetT) UOffsetT { + return off + GetUOffsetT(t.Bytes[off:]) +} + +// String gets a string from data stored inside the flatbuffer. +func (t *Table) String(off UOffsetT) string { + off += t.Pos + off += GetUOffsetT(t.Bytes[off:]) + start := off + UOffsetT(SizeUOffsetT) + length := GetUOffsetT(t.Bytes[off:]) + return string(t.Bytes[start : start+length]) +} + +// VectorLen retrieves the length of the vector whose offset is stored at +// "off" in this object. +func (t *Table) VectorLen(off UOffsetT) int { + off += t.Pos + off += GetUOffsetT(t.Bytes[off:]) + return int(GetUOffsetT(t.Bytes[off:])) +} + +// Vector retrieves the start of data of the vector whose offset is stored +// at "off" in this object. +func (t *Table) Vector(off UOffsetT) UOffsetT { + off += t.Pos + x := off + GetUOffsetT(t.Bytes[off:]) + // data starts after metadata containing the vector length + x += UOffsetT(SizeUOffsetT) + return x +} + +// Union initializes any Table-derived type to point to the union at the given +// offset. +func (t *Table) Union(t2 *Table, off UOffsetT) { + off += t.Pos + t2.Pos = off + t.GetUOffsetT(off) + t2.Bytes = t.Bytes +} + +// GetBool retrieves a bool at the given offset. +func (t *Table) GetBool(off UOffsetT) bool { + return GetBool(t.Bytes[off:]) +} + +// GetByte retrieves a byte at the given offset. +func (t *Table) GetByte(off UOffsetT) byte { + return GetByte(t.Bytes[off:]) +} + +// GetUint8 retrieves a uint8 at the given offset. +func (t *Table) GetUint8(off UOffsetT) uint8 { + return GetUint8(t.Bytes[off:]) +} + +// GetUint16 retrieves a uint16 at the given offset. +func (t *Table) GetUint16(off UOffsetT) uint16 { + return GetUint16(t.Bytes[off:]) +} + +// GetUint32 retrieves a uint32 at the given offset. +func (t *Table) GetUint32(off UOffsetT) uint32 { + return GetUint32(t.Bytes[off:]) +} + +// GetUint64 retrieves a uint64 at the given offset. +func (t *Table) GetUint64(off UOffsetT) uint64 { + return GetUint64(t.Bytes[off:]) +} + +// GetInt8 retrieves a int8 at the given offset. +func (t *Table) GetInt8(off UOffsetT) int8 { + return GetInt8(t.Bytes[off:]) +} + +// GetInt16 retrieves a int16 at the given offset. +func (t *Table) GetInt16(off UOffsetT) int16 { + return GetInt16(t.Bytes[off:]) +} + +// GetInt32 retrieves a int32 at the given offset. +func (t *Table) GetInt32(off UOffsetT) int32 { + return GetInt32(t.Bytes[off:]) +} + +// GetInt64 retrieves a int64 at the given offset. +func (t *Table) GetInt64(off UOffsetT) int64 { + return GetInt64(t.Bytes[off:]) +} + +// GetFloat32 retrieves a float32 at the given offset. +func (t *Table) GetFloat32(off UOffsetT) float32 { + return GetFloat32(t.Bytes[off:]) +} + +// GetFloat64 retrieves a float64 at the given offset. +func (t *Table) GetFloat64(off UOffsetT) float64 { + return GetFloat64(t.Bytes[off:]) +} + +// GetUOffsetT retrieves a UOffsetT at the given offset. +func (t *Table) GetUOffsetT(off UOffsetT) UOffsetT { + return GetUOffsetT(t.Bytes[off:]) +} + +// GetVOffsetT retrieves a VOffsetT at the given offset. +func (t *Table) GetVOffsetT(off UOffsetT) VOffsetT { + return GetVOffsetT(t.Bytes[off:]) +} + +// GetSOffsetT retrieves a SOffsetT at the given offset. +func (t *Table) GetSOffsetT(off UOffsetT) SOffsetT { + return GetSOffsetT(t.Bytes[off:]) +} + +// GetBoolSlot retrieves the bool that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetBoolSlot(slot VOffsetT, d bool) bool { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetBool(t.Pos + UOffsetT(off)) +} + +// GetByteSlot retrieves the byte that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetByteSlot(slot VOffsetT, d byte) byte { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetByte(t.Pos + UOffsetT(off)) +} + +// GetInt8Slot retrieves the int8 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetInt8Slot(slot VOffsetT, d int8) int8 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetInt8(t.Pos + UOffsetT(off)) +} + +// GetUint8Slot retrieves the uint8 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetUint8Slot(slot VOffsetT, d uint8) uint8 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetUint8(t.Pos + UOffsetT(off)) +} + +// GetInt16Slot retrieves the int16 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetInt16Slot(slot VOffsetT, d int16) int16 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetInt16(t.Pos + UOffsetT(off)) +} + +// GetUint16Slot retrieves the uint16 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetUint16Slot(slot VOffsetT, d uint16) uint16 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetUint16(t.Pos + UOffsetT(off)) +} + +// GetInt32Slot retrieves the int32 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetInt32Slot(slot VOffsetT, d int32) int32 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetInt32(t.Pos + UOffsetT(off)) +} + +// GetUint32Slot retrieves the uint32 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetUint32Slot(slot VOffsetT, d uint32) uint32 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetUint32(t.Pos + UOffsetT(off)) +} + +// GetInt64Slot retrieves the int64 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetInt64Slot(slot VOffsetT, d int64) int64 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetInt64(t.Pos + UOffsetT(off)) +} + +// GetUint64Slot retrieves the uint64 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetUint64Slot(slot VOffsetT, d uint64) uint64 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetUint64(t.Pos + UOffsetT(off)) +} + +// GetFloat32Slot retrieves the float32 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetFloat32Slot(slot VOffsetT, d float32) float32 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetFloat32(t.Pos + UOffsetT(off)) +} + +// GetFloat64Slot retrieves the float64 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetFloat64Slot(slot VOffsetT, d float64) float64 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetFloat64(t.Pos + UOffsetT(off)) +} + +// GetVOffsetTSlot retrieves the VOffsetT that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetVOffsetTSlot(slot VOffsetT, d VOffsetT) VOffsetT { + off := t.Offset(slot) + if off == 0 { + return d + } + return VOffsetT(off) +} diff --git a/go/unsafe.go b/go/unsafe.go new file mode 100644 index 00000000000..e90ccb09d0d --- /dev/null +++ b/go/unsafe.go @@ -0,0 +1,45 @@ +package flatbuffers + +import "unsafe" + +var ( + // See http://golang.org/ref/spec#Numeric_types + + // SizeUint8 is the byte size of a uint8. + SizeUint8 = int(unsafe.Sizeof(uint8(0))) + // SizeUint16 is the byte size of a uint16. + SizeUint16 = int(unsafe.Sizeof(uint16(0))) + // SizeUint32 is the byte size of a uint32. + SizeUint32 = int(unsafe.Sizeof(uint32(0))) + // SizeUint64 is the byte size of a uint64. + SizeUint64 = int(unsafe.Sizeof(uint64(0))) + + // SizeInt8 is the byte size of a int8. + SizeInt8 = int(unsafe.Sizeof(int8(0))) + // SizeInt16 is the byte size of a int16. + SizeInt16 = int(unsafe.Sizeof(int16(0))) + // SizeInt32 is the byte size of a int32. + SizeInt32 = int(unsafe.Sizeof(int32(0))) + // SizeInt64 is the byte size of a int64. + SizeInt64 = int(unsafe.Sizeof(int64(0))) + + // SizeFloat32 is the byte size of a float32. + SizeFloat32 = int(unsafe.Sizeof(float32(0))) + // SizeFloat64 is the byte size of a float64. + SizeFloat64 = int(unsafe.Sizeof(float64(0))) + + // SizeByte is the byte size of a byte. + // The `byte` type is aliased (by Go definition) to uint8. + SizeByte = SizeUint8 + + // SizeBool is the byte size of a bool. + // The `bool` type is aliased (by flatbuffers convention) to uint8. + SizeBool = SizeUint8 + + // SizeSOffsetT is the byte size of an SOffsetT. + SizeSOffsetT = int(unsafe.Sizeof(SOffsetT(0))) + // SizeUOffsetT is the byte size of an UOffsetT. + SizeUOffsetT = int(unsafe.Sizeof(UOffsetT(0))) + // SizeVOffsetT is the byte size of an VOffsetT. + SizeVOffsetT = int(unsafe.Sizeof(VOffsetT(0))) +) diff --git a/include/flatbuffers/flatbuffers.h b/include/flatbuffers/flatbuffers.h index e141c788935..f3caf58c6a2 100644 --- a/include/flatbuffers/flatbuffers.h +++ b/include/flatbuffers/flatbuffers.h @@ -26,6 +26,7 @@ #else # include #endif +#include #include #include #include @@ -68,12 +69,6 @@ namespace flatbufferstd = std; #endif #endif // !defined(FLATBUFFERS_LITTLEENDIAN) -#if !defined(_MSC_VER) -#define FLATBUFFERS_WEAK __attribute__((weak)) -#else -#define FLATBUFFERS_WEAK __declspec(selectany) -#endif // WIN32 - #define FLATBUFFERS_VERSION_MAJOR 1 #define FLATBUFFERS_VERSION_MINOR 0 #define FLATBUFFERS_VERSION_REVISION 0 @@ -183,15 +178,14 @@ template struct IndirectHelper> { static const size_t element_stride = sizeof(uoffset_t); static return_type Read(const uint8_t *p, uoffset_t i) { p += i * sizeof(uoffset_t); - return EndianScalar(reinterpret_cast( - p + ReadScalar(p))); + return reinterpret_cast(p + ReadScalar(p)); } }; template struct IndirectHelper { - typedef const T &return_type; + typedef const T *return_type; static const size_t element_stride = sizeof(T); static return_type Read(const uint8_t *p, uoffset_t i) { - return *reinterpret_cast(p + i * sizeof(T)); + return reinterpret_cast(p + i * sizeof(T)); } }; @@ -233,12 +227,16 @@ struct VectorIterator : public return data_ != other.data_; } + ptrdiff_t operator-(const VectorIterator& other) const { + return (data_ - other.data_) / IndirectHelper::element_stride; + } + typename super_type::value_type operator *() const { return IndirectHelper::Read(data_, 0); } - typename super_type::pointer operator->() const { - return &IndirectHelper::Read(data_, 0); + typename super_type::value_type operator->() const { + return IndirectHelper::Read(data_, 0); } VectorIterator &operator++() { @@ -282,15 +280,16 @@ template class Vector { iterator end() { return iterator(Data(), length_); } const_iterator end() const { return const_iterator(Data(), length_); } + // The raw data in little endian format. Use with care. + const uint8_t *Data() const { + return reinterpret_cast(&length_ + 1); + } + protected: // This class is only used to access pre-existing data. Don't ever // try to construct these manually. Vector(); - const uint8_t *Data() const { - return reinterpret_cast(&length_ + 1); - } - uoffset_t length_; }; @@ -359,6 +358,10 @@ class vector_downward { void pop(size_t bytes_to_remove) { cur_ += bytes_to_remove; } private: + // You shouldn't really be copying instances of this class. + vector_downward(const vector_downward &); + vector_downward &operator=(const vector_downward &); + size_t reserved_; uint8_t *buf_; uint8_t *cur_; // Points at location between empty (below) and used (above). @@ -608,30 +611,45 @@ class FlatBufferBuilder { } template Offset> CreateVector(const std::vector &v){ - return CreateVector(&v[0], v.size()); + return CreateVector(v.data(), v.size()); } template Offset> CreateVectorOfStructs( const T *v, size_t len) { NotNested(); - StartVector(len, AlignOf()); + StartVector(len * sizeof(T) / AlignOf(), AlignOf()); PushBytes(reinterpret_cast(v), sizeof(T) * len); return Offset>(EndVector(len)); } template Offset> CreateVectorOfStructs( const std::vector &v) { - return CreateVectorOfStructs(&v[0], v.size()); + return CreateVectorOfStructs(v.data(), v.size()); } + static const size_t kFileIdentifierLength = 4; + // Finish serializing a buffer by writing the root offset. - template void Finish(Offset root) { + // If a file_identifier is given, the buffer will be prefix with a standard + // FlatBuffers file header. + template void Finish(Offset root, + const char *file_identifier = nullptr) { // This will cause the whole buffer to be aligned. - PreAlign(sizeof(uoffset_t), minalign_); + PreAlign(sizeof(uoffset_t) + (file_identifier ? kFileIdentifierLength : 0), + minalign_); + if (file_identifier) { + assert(strlen(file_identifier) == kFileIdentifierLength); + buf_.push(reinterpret_cast(file_identifier), + kFileIdentifierLength); + } PushElement(ReferTo(root.o)); // Location of root. } private: + // You shouldn't really be copying instances of this class. + FlatBufferBuilder(const FlatBufferBuilder &); + FlatBufferBuilder &operator=(const FlatBufferBuilder &); + struct FieldLoc { uoffset_t off; voffset_t id; @@ -656,17 +674,27 @@ template const T *GetRoot(const void *buf) { EndianScalar(*reinterpret_cast(buf))); } +// Helper to see if the identifier in a buffer has the expected value. +inline bool BufferHasIdentifier(const void *buf, const char *identifier) { + return strncmp(reinterpret_cast(buf) + 4, identifier, + FlatBufferBuilder::kFileIdentifierLength) == 0; +} + // Helper class to verify the integrity of a FlatBuffer class Verifier { public: - Verifier(const uint8_t *buf, size_t buf_len) - : buf_(buf), end_(buf + buf_len) + Verifier(const uint8_t *buf, size_t buf_len, size_t _max_depth = 64, + size_t _max_tables = 1000000) + : buf_(buf), end_(buf + buf_len), depth_(0), max_depth_(_max_depth), + num_tables_(0), max_tables_(_max_tables) {} // Verify any range within the buffer. bool Verify(const void *elem, size_t elem_len) const { bool ok = elem >= buf_ && elem <= end_ - elem_len; - assert(ok); + #ifdef FLATBUFFERS_DEBUG_VERIFICATION_FAILURE + assert(ok); + #endif return ok; } @@ -676,7 +704,7 @@ class Verifier { } // Verify a pointer (may be NULL) of a table type. - template bool VerifyTable(const T *table) const { + template bool VerifyTable(const T *table) { return !table || table->Verify(*this); } @@ -721,8 +749,7 @@ class Verifier { } // Special case for table contents, after the above has been called. - template bool VerifyVectorOfTables(const Vector> *vec) - const { + template bool VerifyVectorOfTables(const Vector> *vec) { if (vec) { for (uoffset_t i = 0; i < vec->Length(); i++) { if (!vec->Get(i)->Verify(*this)) return false; @@ -732,16 +759,40 @@ class Verifier { } // Verify this whole buffer, starting with root type T. - template bool VerifyBuffer() const { + template bool VerifyBuffer() { // Call T::Verify, which must be in the generated code for this type. return Verify(buf_) && reinterpret_cast(buf_ + ReadScalar(buf_))-> Verify(*this); } + // Called at the start of a table to increase counters measuring data + // structure depth and amount, and possibly bails out with false if + // limits set by the constructor have been hit. Needs to be balanced + // with EndTable(). + bool VerifyComplexity() { + depth_++; + num_tables_++; + bool too_complex = depth_ > max_depth_ || num_tables_ > max_tables_; + #ifdef FLATBUFFERS_DEBUG_VERIFICATION_FAILURE + assert(!too_complex); + #endif + return !too_complex; + } + + // Called at the end of a table to pop the depth count. + bool EndTable() { + depth_--; + return true; + } + private: const uint8_t *buf_; const uint8_t *end_; + size_t depth_; + size_t max_depth_; + size_t num_tables_; + size_t max_tables_; }; // "structs" are flat structures that do not have an offset table, thus @@ -816,12 +867,13 @@ class Table { // Verify the vtable of this table. // Call this once per table, followed by VerifyField once per field. - bool VerifyTable(const Verifier &verifier) const { + bool VerifyTableStart(Verifier &verifier) const { // Check the vtable offset. if (!verifier.Verify(data_)) return false; auto vtable = data_ - ReadScalar(data_); // Check the vtable size field, then check vtable fits in its entirety. - return verifier.Verify(vtable) && + return verifier.VerifyComplexity() && + verifier.Verify(vtable) && verifier.Verify(vtable, ReadScalar(vtable)); } @@ -870,14 +922,14 @@ inline int LookupEnum(const char **names, const char *name) { struct __declspec(align(alignment)) #define STRUCT_END(name, size) \ __pragma(pack()); \ - static_assert(sizeof(name) == size, "compiler breaks packing rules"); + static_assert(sizeof(name) == size, "compiler breaks packing rules") #elif defined(__GNUC__) || defined(__clang__) #define MANUALLY_ALIGNED_STRUCT(alignment) \ - _Pragma("pack(1)"); \ + _Pragma("pack(1)") \ struct __attribute__((aligned(alignment))) #define STRUCT_END(name, size) \ - _Pragma("pack()"); \ - static_assert(sizeof(name) == size, "compiler breaks packing rules"); + _Pragma("pack()") \ + static_assert(sizeof(name) == size, "compiler breaks packing rules") #else #error Unknown compiler, please define structure alignment macros #endif @@ -891,13 +943,18 @@ inline int LookupEnum(const char **names, const char *name) { // to measure popularity. You are free to remove it (of course) but we would // appreciate if you left it in. -extern volatile FLATBUFFERS_WEAK const char *flatbuffer_version_string; -volatile FLATBUFFERS_WEAK const char *flatbuffer_version_string = +// Weak linkage is culled by VS & doesn't work on cygwin. +#if !defined(_WIN32) && !defined(__CYGWIN__) + +extern volatile __attribute__((weak)) const char *flatbuffer_version_string; +volatile __attribute__((weak)) const char *flatbuffer_version_string = "FlatBuffers " FLATBUFFERS_STRING(FLATBUFFERS_VERSION_MAJOR) "." FLATBUFFERS_STRING(FLATBUFFERS_VERSION_MINOR) "." FLATBUFFERS_STRING(FLATBUFFERS_VERSION_REVISION); +#endif // !defined(_WIN32) && !defined(__CYGWIN__) + } // namespace flatbuffers #endif // FLATBUFFERS_H_ diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index d42c88218aa..4e0bc3f69dd 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -31,31 +31,31 @@ namespace flatbuffers { // Additionally, Parser::ParseType assumes bool..string is a contiguous range // of type tokens. #define FLATBUFFERS_GEN_TYPES_SCALAR(TD) \ - TD(NONE, "", uint8_t, byte ) \ - TD(UTYPE, "", uint8_t, byte ) /* begin scalars, ints */ \ - TD(BOOL, "bool", uint8_t, byte ) \ - TD(CHAR, "byte", int8_t, byte ) \ - TD(UCHAR, "ubyte", uint8_t, byte ) \ - TD(SHORT, "short", int16_t, short ) \ - TD(USHORT, "ushort", uint16_t, short ) \ - TD(INT, "int", int32_t, int ) \ - TD(UINT, "uint", uint32_t, int ) \ - TD(LONG, "long", int64_t, long ) \ - TD(ULONG, "ulong", uint64_t, long ) /* end ints */ \ - TD(FLOAT, "float", float, float ) /* begin floats */ \ - TD(DOUBLE, "double", double, double) /* end floats, scalars */ + TD(NONE, "", uint8_t, byte, byte) \ + TD(UTYPE, "", uint8_t, byte, byte) /* begin scalars, ints */ \ + TD(BOOL, "bool", uint8_t, byte, byte) \ + TD(CHAR, "byte", int8_t, byte, int8) \ + TD(UCHAR, "ubyte", uint8_t, byte, byte) \ + TD(SHORT, "short", int16_t, short, int16) \ + TD(USHORT, "ushort", uint16_t, short, uint16) \ + TD(INT, "int", int32_t, int, int32) \ + TD(UINT, "uint", uint32_t, int, uint32) \ + TD(LONG, "long", int64_t, long, int64) \ + TD(ULONG, "ulong", uint64_t, long, uint64) /* end ints */ \ + TD(FLOAT, "float", float, float, float32) /* begin floats */ \ + TD(DOUBLE, "double", double, double, float64) /* end floats, scalars */ #define FLATBUFFERS_GEN_TYPES_POINTER(TD) \ - TD(STRING, "string", Offset, int) \ - TD(VECTOR, "", Offset, int) \ - TD(STRUCT, "", Offset, int) \ - TD(UNION, "", Offset, int) + TD(STRING, "string", Offset, int, int) \ + TD(VECTOR, "", Offset, int, int) \ + TD(STRUCT, "", Offset, int, int) \ + TD(UNION, "", Offset, int, int) // using these macros, we can now write code dealing with types just once, e.g. /* switch (type) { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) \ case BASE_TYPE_ ## ENUM: \ // do something specific to CTYPE here FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) @@ -67,14 +67,17 @@ switch (type) { FLATBUFFERS_GEN_TYPES_SCALAR(TD) \ FLATBUFFERS_GEN_TYPES_POINTER(TD) -// Create an enum for all the types above +// Create an enum for all the types above. +#ifdef __GNUC__ +__extension__ // Stop GCC complaining about trailing comma with -Wpendantic. +#endif enum BaseType { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE) BASE_TYPE_ ## ENUM, + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) BASE_TYPE_ ## ENUM, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD }; -#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE) \ +#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) \ static_assert(sizeof(CTYPE) <= sizeof(largest_scalar_t), \ "define largest_scalar_t as " #CTYPE); FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) @@ -100,19 +103,21 @@ struct EnumDef; // Represents any type in the IDL, which is a combination of the BaseType // and additional information for vectors/structs_. struct Type { - explicit Type(BaseType _base_type = BASE_TYPE_NONE, StructDef *_sd = nullptr) + explicit Type(BaseType _base_type = BASE_TYPE_NONE, + StructDef *_sd = nullptr, EnumDef *_ed = nullptr) : base_type(_base_type), element(BASE_TYPE_NONE), struct_def(_sd), - enum_def(nullptr) + enum_def(_ed) {} - Type VectorType() const { return Type(element, struct_def); } + Type VectorType() const { return Type(element, struct_def, enum_def); } BaseType base_type; BaseType element; // only set if t == BASE_TYPE_VECTOR StructDef *struct_def; // only set if t or element == BASE_TYPE_STRUCT - EnumDef *enum_def; // only set if t == BASE_TYPE_UNION / BASE_TYPE_UTYPE + EnumDef *enum_def; // set if t == BASE_TYPE_UNION / BASE_TYPE_UTYPE, + // or for an integral type derived from an enum. }; // Represents a parsed scalar value, it's type, and field offset. @@ -154,6 +159,11 @@ template class SymbolTable { std::vector vec; // Used to iterate in order of insertion }; +// A name space, as set in the schema. +struct Namespace { + std::vector components; +}; + // Base class for all definition types (fields, structs_, enums_). struct Definition { Definition() : generated(false) {} @@ -178,7 +188,8 @@ struct StructDef : public Definition { predecl(true), sortbysize(true), minalign(1), - bytesize(0) + bytesize(0), + defined_namespace(nullptr) {} void PadLastField(size_t minalign) { @@ -193,6 +204,7 @@ struct StructDef : public Definition { bool sortbysize; // Whether fields come in the declaration or size order. size_t minalign; // What the whole object needs to be aligned to. size_t bytesize; // Size if fixed. + Namespace *defined_namespace; // Where it was defined. }; inline bool IsStruct(const Type &type) { @@ -208,23 +220,22 @@ inline size_t InlineAlignment(const Type &type) { } struct EnumVal { - EnumVal(const std::string &_name, int _val) + EnumVal(const std::string &_name, int64_t _val) : name(_name), value(_val), struct_def(nullptr) {} std::string name; std::string doc_comment; - int value; + int64_t value; StructDef *struct_def; // only set if this is a union }; struct EnumDef : public Definition { EnumDef() : is_union(false) {} - StructDef *ReverseLookup(int enum_idx) { - assert(is_union); + EnumVal *ReverseLookup(int enum_idx) { for (auto it = vals.vec.begin() + 1; it != vals.vec.end(); ++it) { if ((*it)->value == enum_idx) { - return (*it)->struct_def; + return *it; } } return nullptr; @@ -241,16 +252,31 @@ class Parser { root_struct_def(nullptr), source_(nullptr), cursor_(nullptr), - line_(1) {} + line_(1) { + // Just in case none are declared: + namespaces_.push_back(new Namespace()); + } + + ~Parser() { + for (auto it = namespaces_.begin(); it != namespaces_.end(); ++it) { + delete *it; + } + } // Parse the string containing either schema or JSON data, which will // populate the SymbolTable's or the FlatBufferBuilder above. - bool Parse(const char *_source); + // filepath indicates the file that _source was loaded from, it is + // used to resolve any include statements. + bool Parse(const char *_source, const char *filepath); // Set the root type. May override the one set in the schema. bool SetRootType(const char *name); + // Mark all definitions as already having code generated. + void MarkGenerated(); + private: + int64_t ParseHexNum(int nibbles); void Next(); bool IsNext(int t); void Expect(int t); @@ -267,6 +293,7 @@ class Parser { void ParseMetaData(Definition &def); bool TryTypedValue(int dtoken, bool check, Value &e, BaseType req); void ParseSingleValue(Value &e); + int64_t ParseIntegerFromString(Type &type); StructDef *LookupCreateStruct(const std::string &name); void ParseEnum(bool is_union); void ParseDecl(); @@ -274,11 +301,13 @@ class Parser { public: SymbolTable structs_; SymbolTable enums_; - std::vector name_space_; // As set in the schema. + std::vector namespaces_; std::string error_; // User readable error_ if Parse() == false FlatBufferBuilder builder_; // any data contained in the file StructDef *root_struct_def; + std::string file_identifier_; + std::string file_extension_; private: const char *source_, *cursor_; @@ -288,14 +317,35 @@ class Parser { std::vector> field_stack_; std::vector struct_stack_; + std::map included_files_; }; +// Utility functions for generators: + +// Convert an underscore_based_indentifier in to camelCase. +// Also uppercases the first character if first is true. +inline std::string MakeCamel(const std::string &in, bool first = true) { + std::string s; + for (size_t i = 0; i < in.length(); i++) { + if (!i && first) + s += static_cast(toupper(in[0])); + else if (in[i] == '_' && i + 1 < in.length()) + s += static_cast(toupper(in[++i])); + else + s += in[i]; + } + return s; +} + // Container of options that may apply to any of the source/text generators. struct GeneratorOptions { bool strict_json; int indent_step; + bool output_enum_identifiers; + bool prefixed_enums; - GeneratorOptions() : strict_json(false), indent_step(2) {} + GeneratorOptions() : strict_json(false), indent_step(2), + output_enum_identifiers(true), prefixed_enums(true) {} }; // Generate text (JSON) from a given FlatBuffer, and a given Parser @@ -312,12 +362,20 @@ extern void GenerateText(const Parser &parser, // Generate a C++ header from the definitions in the Parser object. // See idl_gen_cpp. extern std::string GenerateCPP(const Parser &parser, - const std::string &include_guard_ident); + const std::string &include_guard_ident, + const GeneratorOptions &opts); extern bool GenerateCPP(const Parser &parser, const std::string &path, const std::string &file_name, const GeneratorOptions &opts); +// Generate Go files from the definitions in the Parser object. +// See idl_gen_go.cpp. +extern bool GenerateGo(const Parser &parser, + const std::string &path, + const std::string &file_name, + const GeneratorOptions &opts); + // Generate Java files from the definitions in the Parser object. // See idl_gen_java.cpp. extern bool GenerateJava(const Parser &parser, diff --git a/include/flatbuffers/util.h b/include/flatbuffers/util.h index 0a3b0510e4e..93cf0cc527a 100644 --- a/include/flatbuffers/util.h +++ b/include/flatbuffers/util.h @@ -22,6 +22,11 @@ #include #include #include +#ifdef _WIN32 +#include +#else +#include +#endif namespace flatbuffers { @@ -44,13 +49,11 @@ template<> inline std::string NumToString(unsigned char t) { } // Convert an integer value to a hexadecimal string. -// The returned string length is the number of nibbles in -// the supplied value prefixed by 0 digits. For example, -// IntToStringHex(static_cast(0x23)) returns the -// string "00000023". -template std::string IntToStringHex(T i) { +// The returned string length is always xdigits long, prefixed by 0 digits. +// For example, IntToStringHex(0x23, 8) returns the string "00000023". +inline std::string IntToStringHex(int i, int xdigits) { std::stringstream ss; - ss << std::setw(sizeof(T) * 2) + ss << std::setw(xdigits) << std::setfill('0') << std::hex << std::uppercase @@ -59,11 +62,11 @@ template std::string IntToStringHex(T i) { } // Portable implementation of strtoull(). -inline int64_t StringToInt(const char *str) { +inline int64_t StringToInt(const char *str, int base = 10) { #ifdef _MSC_VER - return _strtoui64(str, nullptr, 10); + return _strtoui64(str, nullptr, base); #else - return strtoull(str, nullptr, 10); + return strtoull(str, nullptr, base); #endif } @@ -100,6 +103,101 @@ inline bool SaveFile(const char *name, const std::string &buf, bool binary) { return SaveFile(name, buf.c_str(), buf.size(), binary); } +// Functionality for minimalistic portable path handling: + +static const char kPosixPathSeparator = '/'; +#ifdef _WIN32 +static const char kPathSeparator = '\\'; +static const char *PathSeparatorSet = "\\/"; // Intentionally no ':' +#else +static const char kPathSeparator = kPosixPathSeparator; +static const char *PathSeparatorSet = "/"; +#endif // _WIN32 + +// Returns the path with the extension, if any, removed. +inline std::string StripExtension(const std::string &filepath) { + size_t i = filepath.find_last_of("."); + return i != std::string::npos ? filepath.substr(0, i) : filepath; +} + +// Return the last component of the path, after the last separator. +inline std::string StripPath(const std::string &filepath) { + size_t i = filepath.find_last_of(PathSeparatorSet); + return i != std::string::npos ? filepath.substr(i + 1) : filepath; +} + +// Strip the last component of the path + separator. +inline std::string StripFileName(const std::string &filepath) { + size_t i = filepath.find_last_of(PathSeparatorSet); + return i != std::string::npos ? filepath.substr(0, i) : ""; +} + +// This function ensure a directory exists, by recursively +// creating dirs for any parts of the path that don't exist yet. +inline void EnsureDirExists(const std::string &filepath) { + auto parent = StripFileName(filepath); + if (parent.length()) EnsureDirExists(parent); + #ifdef _WIN32 + _mkdir(filepath.c_str()); + #else + mkdir(filepath.c_str(), S_IRWXU|S_IRGRP|S_IXGRP); + #endif +} + +// To and from UTF-8 unicode conversion functions + +// Convert a unicode code point into a UTF-8 representation by appending it +// to a string. Returns the number of bytes generated. +inline int ToUTF8(uint32_t ucc, std::string *out) { + assert(!(ucc & 0x80000000)); // Top bit can't be set. + // 6 possible encodings: http://en.wikipedia.org/wiki/UTF-8 + for (int i = 0; i < 6; i++) { + // Max bits this encoding can represent. + uint32_t max_bits = 6 + i * 5 + static_cast(!i); + if (ucc < (1u << max_bits)) { // does it fit? + // Remaining bits not encoded in the first byte, store 6 bits each + uint32_t remain_bits = i * 6; + // Store first byte: + (*out) += static_cast((0xFE << (max_bits - remain_bits)) | + (ucc >> remain_bits)); + // Store remaining bytes: + for (int j = i - 1; j >= 0; j--) { + (*out) += static_cast(((ucc >> (j * 6)) & 0x3F) | 0x80); + } + return i + 1; // Return the number of bytes added. + } + } + assert(0); // Impossible to arrive here. + return -1; +} + +// Converts whatever prefix of the incoming string corresponds to a valid +// UTF-8 sequence into a unicode code. The incoming pointer will have been +// advanced past all bytes parsed. +// returns -1 upon corrupt UTF-8 encoding (ignore the incoming pointer in +// this case). +inline int FromUTF8(const char **in) { + int len = 0; + // Count leading 1 bits. + for (int mask = 0x80; mask >= 0x04; mask >>= 1) { + if (**in & mask) { + len++; + } else { + break; + } + } + if ((**in << len) & 0x80) return -1; // Bit after leading 1's must be 0. + if (!len) return *(*in)++; + // Grab initial bits of the code. + int ucc = *(*in)++ & ((1 << (7 - len)) - 1); + for (int i = 0; i < len - 1; i++) { + if ((**in & 0xC0) != 0x80) return -1; // Upper bits must 1 0. + ucc <<= 6; + ucc |= *(*in)++ & 0x3F; // Grab 6 more bits of the code. + } + return ucc; +} + } // namespace flatbuffers #endif // FLATBUFFERS_UTIL_H_ diff --git a/java/flatbuffers/FlatBufferBuilder.java b/java/flatbuffers/FlatBufferBuilder.java index ee0401780e2..a5ec1fb2909 100755 --- a/java/flatbuffers/FlatBufferBuilder.java +++ b/java/flatbuffers/FlatBufferBuilder.java @@ -123,10 +123,11 @@ public void addOffset(int off) { putInt(off); } - public void startVector(int elem_size, int num_elems) { + public void startVector(int elem_size, int num_elems, int alignment) { notNested(); vector_num_elems = num_elems; prep(SIZEOF_INT, elem_size * num_elems); + prep(alignment, elem_size * num_elems); // Just in case alignment > int. } public int endVector() { @@ -137,7 +138,7 @@ public int endVector() { public int createString(String s) { byte[] utf8 = s.getBytes(utf8charset); addByte((byte)0); - startVector(1, utf8.length); + startVector(1, utf8.length, 1); System.arraycopy(utf8, 0, bb.array(), space -= utf8.length, utf8.length); return endVector(); } diff --git a/java/flatbuffers/Table.java b/java/flatbuffers/Table.java index f2233fa5918..bfab3e0e611 100755 --- a/java/flatbuffers/Table.java +++ b/java/flatbuffers/Table.java @@ -41,7 +41,6 @@ protected int __indirect(int offset) { // Create a java String from UTF-8 data stored inside the flatbuffer. protected String __string(int offset) { - offset += bb_pos; offset += bb.getInt(offset); return new String(bb.array(), offset + SIZEOF_INT, bb.getInt(offset), Charset.forName("UTF-8")); } diff --git a/samples/monster_generated.h b/samples/monster_generated.h index 50011b486c3..b2a44e65feb 100755 --- a/samples/monster_generated.h +++ b/samples/monster_generated.h @@ -5,13 +5,17 @@ #include "flatbuffers/flatbuffers.h" + namespace MyGame { namespace Sample { +struct Vec3; +struct Monster; + enum { Color_Red = 0, Color_Green = 1, - Color_Blue = 2, + Color_Blue = 2 }; inline const char **EnumNamesColor() { @@ -23,7 +27,7 @@ inline const char *EnumNameColor(int e) { return EnumNamesColor()[e]; } enum { Any_NONE = 0, - Any_Monster = 1, + Any_Monster = 1 }; inline const char **EnumNamesAny() { @@ -33,10 +37,7 @@ inline const char **EnumNamesAny() { inline const char *EnumNameAny(int e) { return EnumNamesAny()[e]; } -bool VerifyAny(const flatbuffers::Verifier &verifier, const void *union_obj, uint8_t type); - -struct Vec3; -struct Monster; +bool VerifyAny(flatbuffers::Verifier &verifier, const void *union_obj, uint8_t type); MANUALLY_ALIGNED_STRUCT(4) Vec3 { private: @@ -46,7 +47,7 @@ MANUALLY_ALIGNED_STRUCT(4) Vec3 { public: Vec3(float x, float y, float z) - : x_(flatbuffers::EndianScalar(x)), y_(flatbuffers::EndianScalar(y)), z_(flatbuffers::EndianScalar(z)) {} + : x_(flatbuffers::EndianScalar(x)), y_(flatbuffers::EndianScalar(y)), z_(flatbuffers::EndianScalar(z)) { } float x() const { return flatbuffers::EndianScalar(x_); } float y() const { return flatbuffers::EndianScalar(y_); } @@ -61,16 +62,17 @@ struct Monster : private flatbuffers::Table { const flatbuffers::String *name() const { return GetPointer(10); } const flatbuffers::Vector *inventory() const { return GetPointer *>(14); } int8_t color() const { return GetField(16, 2); } - bool Verify(const flatbuffers::Verifier &verifier) const { - return VerifyTable(verifier) && - VerifyField(verifier, 4) && - VerifyField(verifier, 6) && - VerifyField(verifier, 8) && - VerifyField(verifier, 10) && + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyField(verifier, 4 /* pos */) && + VerifyField(verifier, 6 /* mana */) && + VerifyField(verifier, 8 /* hp */) && + VerifyField(verifier, 10 /* name */) && verifier.Verify(name()) && - VerifyField(verifier, 14) && + VerifyField(verifier, 14 /* inventory */) && verifier.Verify(inventory()) && - VerifyField(verifier, 16); + VerifyField(verifier, 16 /* color */) && + verifier.EndTable(); } }; @@ -88,7 +90,13 @@ struct MonsterBuilder { flatbuffers::Offset Finish() { return flatbuffers::Offset(fbb_.EndTable(start_, 7)); } }; -inline flatbuffers::Offset CreateMonster(flatbuffers::FlatBufferBuilder &_fbb, const Vec3 *pos, int16_t mana, int16_t hp, flatbuffers::Offset name, flatbuffers::Offset> inventory, int8_t color) { +inline flatbuffers::Offset CreateMonster(flatbuffers::FlatBufferBuilder &_fbb, + const Vec3 *pos = 0, + int16_t mana = 150, + int16_t hp = 100, + flatbuffers::Offset name = 0, + flatbuffers::Offset> inventory = 0, + int8_t color = 2) { MonsterBuilder builder_(_fbb); builder_.add_inventory(inventory); builder_.add_name(name); @@ -99,19 +107,21 @@ inline flatbuffers::Offset CreateMonster(flatbuffers::FlatBufferBuilder return builder_.Finish(); } -bool VerifyAny(const flatbuffers::Verifier &verifier, const void *union_obj, uint8_t type) { +bool VerifyAny(flatbuffers::Verifier &verifier, const void *union_obj, uint8_t type) { switch (type) { case Any_NONE: return true; - case Any_Monster: return reinterpret_cast(union_obj)->Verify(verifier); + case Any_Monster: return verifier.VerifyTable(reinterpret_cast(union_obj)); default: return false; } } inline const Monster *GetMonster(const void *buf) { return flatbuffers::GetRoot(buf); } -inline bool VerifyMonsterBuffer(const flatbuffers::Verifier &verifier) { return verifier.VerifyBuffer(); } +inline bool VerifyMonsterBuffer(flatbuffers::Verifier &verifier) { return verifier.VerifyBuffer(); } + +inline void FinishMonsterBuffer(flatbuffers::FlatBufferBuilder &fbb, flatbuffers::Offset root) { fbb.Finish(root); } -}; // namespace MyGame -}; // namespace Sample +} // namespace Sample +} // namespace MyGame #endif // FLATBUFFERS_GENERATED_MONSTER_MYGAME_SAMPLE_H_ diff --git a/samples/sample_binary.cpp b/samples/sample_binary.cpp index bb686174352..7c15b9d3cce 100755 --- a/samples/sample_binary.cpp +++ b/samples/sample_binary.cpp @@ -54,9 +54,11 @@ int main(int /*argc*/, const char * /*argv*/[]) { auto pos = monster->pos(); assert(pos); assert(pos->z() == 3); + (void)pos; auto inv = monster->inventory(); assert(inv); assert(inv->Get(9) == 9); + (void)inv; } diff --git a/samples/sample_text.cpp b/samples/sample_text.cpp index b1b7527ca7f..33cb58e153e 100755 --- a/samples/sample_text.cpp +++ b/samples/sample_text.cpp @@ -37,8 +37,8 @@ int main(int /*argc*/, const char * /*argv*/[]) { // parse schema first, so we can use it to parse the data after flatbuffers::Parser parser; - ok = parser.Parse(schemafile.c_str()) && - parser.Parse(jsonfile.c_str()); + ok = parser.Parse(schemafile.c_str(), "samples/") && + parser.Parse(jsonfile.c_str(), "samples/"); assert(ok); // here, parser.builder_ contains a binary buffer that is the parsed data. diff --git a/src/flatc.cpp b/src/flatc.cpp index 4327de7731c..2329a5dee16 100755 --- a/src/flatc.cpp +++ b/src/flatc.cpp @@ -27,9 +27,10 @@ bool GenerateBinary(const Parser &parser, const std::string &path, const std::string &file_name, const GeneratorOptions & /*opts*/) { + auto ext = parser.file_extension_.length() ? parser.file_extension_ : "bin"; return !parser.builder_.GetSize() || flatbuffers::SaveFile( - (path + file_name + ".bin").c_str(), + (path + file_name + "." + ext).c_str(), reinterpret_cast(parser.builder_.GetBufferPointer()), parser.builder_.GetSize(), true); @@ -71,6 +72,8 @@ const Generator generators[] = { "Generate text output for any data definitions" }, { flatbuffers::GenerateCPP, "c", "C++", "Generate C++ headers for tables/structs" }, + { flatbuffers::GenerateGo, "g", "Go", + "Generate Go files for tables/structs" }, { flatbuffers::GenerateJava, "j", "Java", "Generate Java classes for tables/structs" }, }; @@ -87,6 +90,7 @@ static void Error(const char *err, const char *obj, bool usage) { printf(" -%s %s.\n", generators[i].extension, generators[i].help); printf(" -o PATH Prefix PATH to all generated files.\n" " -S Strict JSON: add quotes to field names.\n" + " -P Don\'t prefix enum values with the enum name in C++.\n" "FILEs may depend on declarations in earlier files.\n" "FILEs after the -- must be binary flatbuffer format files.\n" "Output files are named using the base file name of the input," @@ -97,22 +101,6 @@ static void Error(const char *err, const char *obj, bool usage) { exit(1); } -std::string StripExtension(const std::string &filename) { - size_t i = filename.find_last_of("."); - return i != std::string::npos ? filename.substr(0, i) : filename; -} - -std::string StripPath(const std::string &filename) { - size_t i = filename.find_last_of( - #ifdef WIN32 - "\\:" - #else - "/" - #endif - ); - return i != std::string::npos ? filename.substr(i + 1) : filename; -} - int main(int argc, const char *argv[]) { program_name = argv[0]; flatbuffers::Parser parser; @@ -134,10 +122,17 @@ int main(int argc, const char *argv[]) { case 'o': if (++i >= argc) Error("missing path following", arg, true); output_path = argv[i]; + if (!(output_path.back() == flatbuffers::kPathSeparator || + output_path.back() == flatbuffers::kPosixPathSeparator)) { + output_path += flatbuffers::kPathSeparator; + } break; case 'S': opts.strict_json = true; break; + case 'P': + opts.prefixed_enums = false; + break; case '-': // Separator between text and binary input files. binary_files_from = filenames.size(); break; @@ -162,7 +157,7 @@ int main(int argc, const char *argv[]) { if (!any_generator) Error("no options: no output files generated.", - "specify one of -c -j -t -b etc.", true); + "specify one of -c -g -j -t -b etc.", true); // Now process the files: for (auto file_it = filenames.begin(); @@ -180,14 +175,16 @@ int main(int argc, const char *argv[]) { reinterpret_cast(contents.c_str()), contents.length()); } else { - if (!parser.Parse(contents.c_str())) - Error(parser.error_.c_str()); + if (!parser.Parse(contents.c_str(), file_it->c_str())) + Error((*file_it + ": " + parser.error_).c_str()); } - std::string filebase = StripPath(StripExtension(*file_it)); + std::string filebase = flatbuffers::StripPath( + flatbuffers::StripExtension(*file_it)); for (size_t i = 0; i < num_generators; ++i) { if (generator_enabled[i]) { + flatbuffers::EnsureDirExists(output_path); if (!generators[i].generate(parser, output_path, filebase, opts)) { Error((std::string("Unable to generate ") + generators[i].name + @@ -197,19 +194,10 @@ int main(int argc, const char *argv[]) { } } - // Since the Parser object retains definitions across files, we must - // ensure we only output code for these once, in the file they are first - // declared: - for (auto it = parser.enums_.vec.begin(); - it != parser.enums_.vec.end(); ++it) { - (*it)->generated = true; - } - for (auto it = parser.structs_.vec.begin(); - it != parser.structs_.vec.end(); ++it) { - (*it)->generated = true; - } + // We do not want to generate code for the definitions in this file + // in any files coming up next. + parser.MarkGenerated(); } return 0; } - diff --git a/src/idl_gen_cpp.cpp b/src/idl_gen_cpp.cpp index c711bb7f5b9..b7fe8e15d7c 100644 --- a/src/idl_gen_cpp.cpp +++ b/src/idl_gen_cpp.cpp @@ -26,25 +26,38 @@ namespace cpp { // Return a C++ type from the table in idl.h static std::string GenTypeBasic(const Type &type) { static const char *ctypename[] = { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE) #CTYPE, + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) #CTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD }; return ctypename[type.base_type]; } -static std::string GenTypeWire(const Type &type, const char *postfix); +static std::string GenTypeWire(const Parser &parser, const Type &type, + const char *postfix); // Return a C++ pointer type, specialized to the actual struct/table types, // and vector element types. -static std::string GenTypePointer(const Type &type) { +static std::string GenTypePointer(const Parser &parser, const Type &type) { switch (type.base_type) { case BASE_TYPE_STRING: return "flatbuffers::String"; case BASE_TYPE_VECTOR: - return "flatbuffers::Vector<" + GenTypeWire(type.VectorType(), "") + ">"; - case BASE_TYPE_STRUCT: - return type.struct_def->name; + return "flatbuffers::Vector<" + + GenTypeWire(parser, type.VectorType(), "") + ">"; + case BASE_TYPE_STRUCT: { + auto ns = type.struct_def->defined_namespace; + if (parser.namespaces_.back() != ns) { + std::string qualified_name; + for (auto it = ns->components.begin(); + it != ns->components.end(); ++it) { + qualified_name += *it + "::"; + } + return qualified_name + type.struct_def->name; + } else { + return type.struct_def->name; + } + } case BASE_TYPE_UNION: // fall through default: @@ -54,31 +67,33 @@ static std::string GenTypePointer(const Type &type) { // Return a C++ type for any type (scalar/pointer) specifically for // building a flatbuffer. -static std::string GenTypeWire(const Type &type, const char *postfix) { +static std::string GenTypeWire(const Parser &parser, const Type &type, + const char *postfix) { return IsScalar(type.base_type) ? GenTypeBasic(type) + postfix : IsStruct(type) - ? "const " + GenTypePointer(type) + " *" - : "flatbuffers::Offset<" + GenTypePointer(type) + ">" + postfix; + ? "const " + GenTypePointer(parser, type) + " *" + : "flatbuffers::Offset<" + GenTypePointer(parser, type) + ">" + postfix; } // Return a C++ type for any type (scalar/pointer) that reflects its // serialized size. -static std::string GenTypeSize(const Type &type) { +static std::string GenTypeSize(const Parser &parser, const Type &type) { return IsScalar(type.base_type) ? GenTypeBasic(type) : IsStruct(type) - ? GenTypePointer(type) + ? GenTypePointer(parser, type) : "flatbuffers::uoffset_t"; } // Return a C++ type for any type (scalar/pointer) specifically for // using a flatbuffer. -static std::string GenTypeGet(const Type &type, const char *afterbasic, - const char *beforeptr, const char *afterptr) { +static std::string GenTypeGet(const Parser &parser, const Type &type, + const char *afterbasic, const char *beforeptr, + const char *afterptr) { return IsScalar(type.base_type) ? GenTypeBasic(type) + afterbasic - : beforeptr + GenTypePointer(type) + afterptr; + : beforeptr + GenTypePointer(parser, type) + afterptr; } // Generate a documentation comment, if available. @@ -91,9 +106,16 @@ static void GenComment(const std::string &dc, } } +static std::string GenEnumVal(const EnumDef &enum_def, const EnumVal &enum_val, + const GeneratorOptions &opts) { + return opts.prefixed_enums ? enum_def.name + "_" + enum_val.name + : enum_val.name; +} + // Generate an enum declaration and an enum string lookup table. - static void GenEnum(EnumDef &enum_def, std::string *code_ptr, - std::string *code_ptr_post) { +static void GenEnum(EnumDef &enum_def, std::string *code_ptr, + std::string *code_ptr_post, + const GeneratorOptions &opts) { if (enum_def.generated) return; std::string &code = *code_ptr; std::string &code_post = *code_ptr_post; @@ -104,8 +126,9 @@ static void GenComment(const std::string &dc, ++it) { auto &ev = **it; GenComment(ev.doc_comment, code_ptr, " "); - code += " " + enum_def.name + "_" + ev.name + " = "; - code += NumToString(ev.value) + ",\n"; + code += " " + GenEnumVal(enum_def, ev, opts) + " = "; + code += NumToString(ev.value); + code += (it + 1) != enum_def.vals.vec.end() ? ",\n" : "\n"; } code += "};\n\n"; @@ -113,15 +136,15 @@ static void GenComment(const std::string &dc, // Problem is, if values are very sparse that could generate really big // tables. Ideally in that case we generate a map lookup instead, but for // the moment we simply don't output a table at all. - int range = enum_def.vals.vec.back()->value - - enum_def.vals.vec.front()->value + 1; + auto range = enum_def.vals.vec.back()->value - + enum_def.vals.vec.front()->value + 1; // Average distance between values above which we consider a table // "too sparse". Change at will. static const int kMaxSparseness = 5; - if (range / static_cast(enum_def.vals.vec.size()) < kMaxSparseness) { + if (range / static_cast(enum_def.vals.vec.size()) < kMaxSparseness) { code += "inline const char **EnumNames" + enum_def.name + "() {\n"; code += " static const char *names[] = { "; - int val = enum_def.vals.vec.front()->value; + auto val = enum_def.vals.vec.front()->value; for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); ++it) { @@ -132,7 +155,7 @@ static void GenComment(const std::string &dc, code += "inline const char *EnumName" + enum_def.name; code += "(int e) { return EnumNames" + enum_def.name + "()[e"; if (enum_def.vals.vec.front()->value) - code += " - " + enum_def.name + "_" + enum_def.vals.vec.front()->name; + code += " - " + GenEnumVal(enum_def, *enum_def.vals.vec.front(), opts); code += "]; }\n\n"; } @@ -143,7 +166,7 @@ static void GenComment(const std::string &dc, // has been corrupted, since the verifiers will simply fail when called // on the wrong type. auto signature = "bool Verify" + enum_def.name + - "(const flatbuffers::Verifier &verifier, " + + "(flatbuffers::Verifier &verifier, " + "const void *union_obj, uint8_t type)"; code += signature + ";\n\n"; code_post += signature + " {\n switch (type) {\n"; @@ -151,7 +174,7 @@ static void GenComment(const std::string &dc, it != enum_def.vals.vec.end(); ++it) { auto &ev = **it; - code_post += " case " + enum_def.name + "_" + ev.name; + code_post += " case " + GenEnumVal(enum_def, ev, opts); if (!ev.value) { code_post += ": return true;\n"; // "NONE" enum value. } else { @@ -164,7 +187,8 @@ static void GenComment(const std::string &dc, } // Generate an accessor struct, builder structs & function for a table. -static void GenTable(StructDef &struct_def, std::string *code_ptr) { +static void GenTable(const Parser &parser, StructDef &struct_def, + std::string *code_ptr) { if (struct_def.generated) return; std::string &code = *code_ptr; @@ -179,31 +203,39 @@ static void GenTable(StructDef &struct_def, std::string *code_ptr) { auto &field = **it; if (!field.deprecated) { // Deprecated fields won't be accessible. GenComment(field.doc_comment, code_ptr, " "); - code += " " + GenTypeGet(field.value.type, " ", "const ", " *"); + code += " " + GenTypeGet(parser, field.value.type, " ", "const ", " *"); code += field.name + "() const { return "; // Call a different accessor for pointers, that indirects. code += IsScalar(field.value.type.base_type) ? "GetField<" : (IsStruct(field.value.type) ? "GetStruct<" : "GetPointer<"); - code += GenTypeGet(field.value.type, "", "const ", " *") + ">("; + code += GenTypeGet(parser, field.value.type, "", "const ", " *") + ">("; code += NumToString(field.value.offset); // Default value as second arg for non-pointer types. if (IsScalar(field.value.type.base_type)) code += ", " + field.value.constant; code += "); }\n"; + auto nested = field.attributes.Lookup("nested_flatbuffer"); + if (nested) { + auto nested_root = parser.structs_.Lookup(nested->constant); + assert(nested_root); // Guaranteed to exist by parser. + code += " const " + nested_root->name + " *" + field.name; + code += "_nested_root() { return flatbuffers::GetRoot<"; + code += nested_root->name + ">(" + field.name + "()->Data()); }\n"; + } } } // Generate a verifier function that can check a buffer from an untrusted // source will never cause reads outside the buffer. - code += " bool Verify(const flatbuffers::Verifier &verifier) const {\n"; - code += " return VerifyTable(verifier)"; + code += " bool Verify(flatbuffers::Verifier &verifier) const {\n"; + code += " return VerifyTableStart(verifier)"; std::string prefix = " &&\n "; for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { auto &field = **it; if (!field.deprecated) { - code += prefix + "VerifyField<" + GenTypeSize(field.value.type); + code += prefix + "VerifyField<" + GenTypeSize(parser, field.value.type); code += ">(verifier, " + NumToString(field.value.offset); code += " /* " + field.name + " */)"; switch (field.value.type.base_type) { @@ -244,6 +276,7 @@ static void GenTable(StructDef &struct_def, std::string *code_ptr) { } } } + code += prefix + "verifier.EndTable()"; code += ";\n }\n"; code += "};\n\n"; @@ -258,9 +291,10 @@ static void GenTable(StructDef &struct_def, std::string *code_ptr) { auto &field = **it; if (!field.deprecated) { code += " void add_" + field.name + "("; - code += GenTypeWire(field.value.type, " ") + field.name + ") { fbb_.Add"; + code += GenTypeWire(parser, field.value.type, " ") + field.name; + code += ") { fbb_.Add"; if (IsScalar(field.value.type.base_type)) - code += "Element<" + GenTypeWire(field.value.type, "") + ">"; + code += "Element<" + GenTypeWire(parser, field.value.type, "") + ">"; else if (IsStruct(field.value.type)) code += "Struct"; else @@ -291,8 +325,8 @@ static void GenTable(StructDef &struct_def, std::string *code_ptr) { ++it) { auto &field = **it; if (!field.deprecated) { - code += ",\n " + GenTypeWire(field.value.type, " ") + field.name; - code += " = " + field.value.constant; + code += ",\n " + GenTypeWire(parser, field.value.type, " "); + code += field.name + " = " + field.value.constant; } } code += ") {\n " + struct_def.name + "Builder builder_(_fbb);\n"; @@ -314,7 +348,8 @@ static void GenTable(StructDef &struct_def, std::string *code_ptr) { } // Generate an accessor struct with constructor for a flatbuffers struct. -static void GenStruct(StructDef &struct_def, std::string *code_ptr) { +static void GenStruct(const Parser &parser, StructDef &struct_def, + std::string *code_ptr) { if (struct_def.generated) return; std::string &code = *code_ptr; @@ -331,7 +366,7 @@ static void GenStruct(StructDef &struct_def, std::string *code_ptr) { it != struct_def.fields.vec.end(); ++it) { auto &field = **it; - code += " " + GenTypeGet(field.value.type, " ", "", " "); + code += " " + GenTypeGet(parser, field.value.type, " ", "", " "); code += field.name + "_;\n"; if (field.padding) { for (int i = 0; i < 4; i++) @@ -349,7 +384,8 @@ static void GenStruct(StructDef &struct_def, std::string *code_ptr) { ++it) { auto &field = **it; if (it != struct_def.fields.vec.begin()) code += ", "; - code += GenTypeGet(field.value.type, " ", "const ", " &") + field.name; + code += GenTypeGet(parser, field.value.type, " ", "const ", " &"); + code += field.name; } code += ")\n : "; padding_id = 0; @@ -366,7 +402,16 @@ static void GenStruct(StructDef &struct_def, std::string *code_ptr) { if (field.padding) code += ", __padding" + NumToString(padding_id++) + "(0)"; } - code += " {}\n\n"; + code += " {"; + padding_id = 0; + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); + ++it) { + auto &field = **it; + if (field.padding) + code += " (void)__padding" + NumToString(padding_id++) + ";"; + } + code += " }\n\n"; // Generate accessor methods of the form: // type name() const { return flatbuffers::EndianScalar(name_); } @@ -375,7 +420,7 @@ static void GenStruct(StructDef &struct_def, std::string *code_ptr) { ++it) { auto &field = **it; GenComment(field.doc_comment, code_ptr, " "); - code += " " + GenTypeGet(field.value.type, " ", "const ", " &"); + code += " " + GenTypeGet(parser, field.value.type, " ", "const ", " &"); code += field.name + "() const { return "; if (IsScalar(field.value.type.base_type)) code += "flatbuffers::EndianScalar(" + field.name + "_)"; @@ -387,42 +432,81 @@ static void GenStruct(StructDef &struct_def, std::string *code_ptr) { code += NumToString(struct_def.bytesize) + ");\n\n"; } +void GenerateNestedNameSpaces(Namespace *ns, std::string *code_ptr) { + for (auto it = ns->components.begin(); it != ns->components.end(); ++it) { + *code_ptr += "namespace " + *it + " {\n"; + } +} + +void CloseNestedNameSpaces(Namespace *ns, std::string *code_ptr) { + for (auto it = ns->components.rbegin(); it != ns->components.rend(); ++it) { + *code_ptr += "} // namespace " + *it + "\n"; + } +} + } // namespace cpp // Iterate through all definitions we haven't generate code for (enums, structs, // and tables) and output them to a single file. -std::string GenerateCPP(const Parser &parser, const std::string &include_guard_ident) { +std::string GenerateCPP(const Parser &parser, + const std::string &include_guard_ident, + const GeneratorOptions &opts) { using namespace cpp; // Generate code for all the enum declarations. std::string enum_code, enum_code_post; for (auto it = parser.enums_.vec.begin(); it != parser.enums_.vec.end(); ++it) { - GenEnum(**it, &enum_code, &enum_code_post); + GenEnum(**it, &enum_code, &enum_code_post, opts); } // Generate forward declarations for all structs/tables, since they may // have circular references. - std::string forward_decl_code; + std::string forward_decl_code_same_namespace; + std::string forward_decl_code_other_namespace; + Namespace *cur_name_space = nullptr; for (auto it = parser.structs_.vec.begin(); it != parser.structs_.vec.end(); ++it) { - if (!(*it)->generated) - forward_decl_code += "struct " + (*it)->name + ";\n"; + auto &struct_def = **it; + auto decl = "struct " + struct_def.name + ";\n"; + if (struct_def.defined_namespace == parser.namespaces_.back()) { + forward_decl_code_same_namespace += decl; + } else { + // Wrap this decl in the correct namespace. Only open a namespace if + // the adjacent one is different. + // TODO: this could be done more intelligently, by sorting to + // namespace path and only opening/closing what is necessary, but that's + // quite a bit more complexity. + if (cur_name_space != struct_def.defined_namespace) { + if (cur_name_space) { + CloseNestedNameSpaces(cur_name_space, + &forward_decl_code_other_namespace); + } + GenerateNestedNameSpaces(struct_def.defined_namespace, + &forward_decl_code_other_namespace); + cur_name_space = struct_def.defined_namespace; + } + forward_decl_code_other_namespace += decl; + } + } + if (cur_name_space) { + CloseNestedNameSpaces(cur_name_space, + &forward_decl_code_other_namespace); } // Generate code for all structs, then all tables. std::string decl_code; for (auto it = parser.structs_.vec.begin(); it != parser.structs_.vec.end(); ++it) { - if ((**it).fixed) GenStruct(**it, &decl_code); + if ((**it).fixed) GenStruct(parser, **it, &decl_code); } for (auto it = parser.structs_.vec.begin(); it != parser.structs_.vec.end(); ++it) { - if (!(**it).fixed) GenTable(**it, &decl_code); + if (!(**it).fixed) GenTable(parser, **it, &decl_code); } // Only output file-level code if there were any declarations. - if (enum_code.length() || forward_decl_code.length() || decl_code.length()) { + if (enum_code.length() || decl_code.length()) { std::string code; code = "// automatically generated by the FlatBuffers compiler," " do not modify\n\n"; @@ -430,8 +514,9 @@ std::string GenerateCPP(const Parser &parser, const std::string &include_guard_i // Generate include guard. std::string include_guard = "FLATBUFFERS_GENERATED_" + include_guard_ident; include_guard += "_"; - for (auto it = parser.name_space_.begin(); - it != parser.name_space_.end(); ++it) { + auto name_space = parser.namespaces_.back(); + for (auto it = name_space->components.begin(); + it != name_space->components.end(); ++it) { include_guard += *it + "_"; } include_guard += "H_"; @@ -442,40 +527,54 @@ std::string GenerateCPP(const Parser &parser, const std::string &include_guard_i code += "#include \"flatbuffers/flatbuffers.h\"\n\n"; - // Generate nested namespaces. - for (auto it = parser.name_space_.begin(); - it != parser.name_space_.end(); ++it) { - code += "namespace " + *it + " {\n"; - } + code += forward_decl_code_other_namespace; + code += "\n"; - // Output the main declaration code from above. + GenerateNestedNameSpaces(name_space, &code); code += "\n"; - code += enum_code; - code += forward_decl_code; + + code += forward_decl_code_same_namespace; code += "\n"; + + // Output the main declaration code from above. + code += enum_code; code += decl_code; code += enum_code_post; - // Generate convenient root datatype accessor, and root verifier. + // Generate convenient global helper functions: if (parser.root_struct_def) { + // The root datatype accessor: code += "inline const " + parser.root_struct_def->name + " *Get"; code += parser.root_struct_def->name; code += "(const void *buf) { return flatbuffers::GetRoot<"; code += parser.root_struct_def->name + ">(buf); }\n\n"; + // The root verifier: code += "inline bool Verify"; code += parser.root_struct_def->name; - code += "Buffer(const flatbuffers::Verifier &verifier) { " + code += "Buffer(flatbuffers::Verifier &verifier) { " "return verifier.VerifyBuffer<"; code += parser.root_struct_def->name + ">(); }\n\n"; - } - // Close the namespaces. - for (auto it = parser.name_space_.begin(); - it != parser.name_space_.end(); ++it) { - code += "}; // namespace " + *it + "\n"; + // Finish a buffer with a given root object: + code += "inline void Finish" + parser.root_struct_def->name; + code += "Buffer(flatbuffers::FlatBufferBuilder &fbb, flatbuffers::Offset<"; + code += parser.root_struct_def->name + "> root) { fbb.Finish(root"; + if (parser.file_identifier_.length()) + code += ", \"" + parser.file_identifier_ + "\""; + code += "); }\n\n"; + + if (parser.file_identifier_.length()) { + // Check if a buffer has the identifier. + code += "inline bool " + parser.root_struct_def->name; + code += "BufferHasIdentifier(const void *buf) { return flatbuffers::"; + code += "BufferHasIdentifier(buf, \"" + parser.file_identifier_; + code += "\"); }\n\n"; + } } + CloseNestedNameSpaces(name_space, &code); + // Close the include guard. code += "\n#endif // " + include_guard + "\n"; @@ -488,8 +587,8 @@ std::string GenerateCPP(const Parser &parser, const std::string &include_guard_i bool GenerateCPP(const Parser &parser, const std::string &path, const std::string &file_name, - const GeneratorOptions & /*opts*/) { - auto code = GenerateCPP(parser, file_name); + const GeneratorOptions &opts) { + auto code = GenerateCPP(parser, file_name, opts); return !code.length() || SaveFile((path + file_name + "_generated.h").c_str(), code, false); } diff --git a/src/idl_gen_go.cpp b/src/idl_gen_go.cpp new file mode 100755 index 00000000000..95ebf371f27 --- /dev/null +++ b/src/idl_gen_go.cpp @@ -0,0 +1,680 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// independent from idl_parser, since this code is not needed for most clients + +#include + +#include "flatbuffers/flatbuffers.h" +#include "flatbuffers/idl.h" +#include "flatbuffers/util.h" + +#ifdef _WIN32 +#include +#define PATH_SEPARATOR "\\" +#define mkdir(n, m) _mkdir(n) +#else +#include +#define PATH_SEPARATOR "/" +#endif + +namespace flatbuffers { +namespace go { + +static std::string GenGetter(const Type &type); +static std::string GenMethod(const FieldDef &field); +static void GenStructBuilder(const StructDef &struct_def, + std::string *code_ptr); +static void GenReceiver(const StructDef &struct_def, std::string *code_ptr); +static std::string GenTypeBasic(const Type &type); +static std::string GenTypeGet(const Type &type); +static std::string TypeName(const FieldDef &field); + + +// Write a comment. +static void Comment(const std::string &dc, + std::string *code_ptr, + const char *prefix = "") { + std::string &code = *code_ptr; + if (dc.length()) { + code += std::string(prefix) + "///" + dc + "\n"; + } +} + +// Most field accessors need to retrieve and test the field offset first, +// this is the prefix code for that. +std::string OffsetPrefix(const FieldDef &field) { + return "{\n\to := flatbuffers.UOffsetT(rcv._tab.Offset(" + + NumToString(field.value.offset) + + "))\n\tif o != 0 {\n"; +} + +// Begin by declaring namespace and imports. +static void BeginFile(const std::string name_space_name, + const bool needs_imports, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += "// automatically generated, do not modify\n\n"; + code += "package " + name_space_name + "\n\n"; + if (needs_imports) { + code += "import (\n"; + code += "\tflatbuffers \"github.com/google/flatbuffers/go\"\n"; + code += ")\n"; + } +} + +// Begin a class declaration. +static void BeginClass(const StructDef &struct_def, std::string *code_ptr) { + std::string &code = *code_ptr; + + code += "type " + struct_def.name + " struct {\n\t"; + + // _ is reserved in flatbuffers field names, so no chance of name conflict: + code += "_tab "; + code += struct_def.fixed ? "flatbuffers.Struct" : "flatbuffers.Table"; + code += "\n}\n\n"; +} + +// Begin enum code with a class declaration. +static void BeginEnum(std::string *code_ptr) { + std::string &code = *code_ptr; + code += "const (\n"; +} + +// A single enum member. +static void EnumMember(const EnumDef &enum_def, const EnumVal ev, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += "\t"; + code += enum_def.name; + code += ev.name; + code += " = "; + code += NumToString(ev.value) + "\n"; +} + +// End enum code. +static void EndEnum(std::string *code_ptr) { + std::string &code = *code_ptr; + code += ")\n"; +} + +// Initialize a new struct or table from existing data. +static void NewRootTypeFromBuffer(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + + code += "func GetRootAs"; + code += struct_def.name; + code += "(buf []byte, offset flatbuffers.UOffsetT) "; + code += "*" + struct_def.name + ""; + code += " {\n"; + code += "\tn := flatbuffers.GetUOffsetT(buf[offset:])\n"; + code += "\tx := &" + struct_def.name + "{}\n"; + code += "\tx.Init(buf, n + offset)\n"; + code += "\treturn x\n"; + code += "}\n\n"; +} + +// Initialize an existing object with other data, to avoid an allocation. +static void InitializeExisting(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + + GenReceiver(struct_def, code_ptr); + code += " Init(buf []byte, i flatbuffers.UOffsetT) "; + code += "{\n"; + code += "\trcv._tab.Bytes = buf\n"; + code += "\trcv._tab.Pos = i\n"; + code += "}\n\n"; +} + +// Get the length of a vector. +static void GetVectorLen(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + + GenReceiver(struct_def, code_ptr); + code += " " + MakeCamel(field.name) + "Length("; + code += ") int " + OffsetPrefix(field); + code += "\t\treturn rcv._tab.VectorLen(o)\n\t}\n"; + code += "\treturn 0\n}\n\n"; +} + +// Get the value of a struct's scalar. +static void GetScalarFieldOfStruct(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + std::string getter = GenGetter(field.value.type); + GenReceiver(struct_def, code_ptr); + code += " " + MakeCamel(field.name); + code += "() " + TypeName(field) + " { return " + getter; + code += "(rcv._tab.Pos + flatbuffers.UOffsetT("; + code += NumToString(field.value.offset) + ")) }\n"; +} + +// Get the value of a table's scalar. +static void GetScalarFieldOfTable(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + std::string getter = GenGetter(field.value.type); + GenReceiver(struct_def, code_ptr); + code += " " + MakeCamel(field.name); + code += "() " + TypeName(field) + " "; + code += OffsetPrefix(field) + "\t\treturn " + getter; + code += "(o + rcv._tab.Pos)\n\t}\n"; + code += "\treturn " + field.value.constant + "\n"; + code += "}\n\n"; +} + +// Get a struct by initializing an existing struct. +// Specific to Struct. +static void GetStructFieldOfStruct(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + GenReceiver(struct_def, code_ptr); + code += " " + MakeCamel(field.name); + code += "(obj *" + TypeName(field); + code += ") *" + TypeName(field); + code += " {\n"; + code += "\tif obj == nil {\n"; + code += "\t\tobj = new(" + TypeName(field) + ")\n"; + code += "\t}\n"; + code += "\tobj.Init(rcv._tab.Bytes, rcv._tab.Pos + "; + code += NumToString(field.value.offset) + ")"; + code += "\n\treturn obj\n"; + code += "}\n"; +} + +// Get a struct by initializing an existing struct. +// Specific to Table. +static void GetStructFieldOfTable(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + GenReceiver(struct_def, code_ptr); + code += " " + MakeCamel(field.name); + code += "(obj *"; + code += TypeName(field); + code += ") *" + TypeName(field) + " " + OffsetPrefix(field); + if (field.value.type.struct_def->fixed) { + code += "\t\tx := o + rcv._tab.Pos\n"; + } else { + code += "\t\tx := rcv._tab.Indirect(o + rcv._tab.Pos)\n"; + } + code += "\t\tif obj == nil {\n"; + code += "\t\t\tobj = new(" + TypeName(field) + ")\n"; + code += "\t\t}\n"; + code += "\t\tobj.Init(rcv._tab.Bytes, x)\n"; + code += "\t\treturn obj\n\t}\n\treturn nil\n"; + code += "}\n\n"; +} + +// Get the value of a string. +static void GetStringField(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + GenReceiver(struct_def, code_ptr); + code += " " + MakeCamel(field.name); + code += "() " + TypeName(field) + " "; + code += OffsetPrefix(field) + "\t\treturn " + GenGetter(field.value.type); + code += "(o)\n\t}\n\treturn \"\"\n"; + code += "}\n\n"; +} + +// Get the value of a union from an object. +static void GetUnionField(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + GenReceiver(struct_def, code_ptr); + code += " " + MakeCamel(field.name) + "("; + code += "obj " + TypeName(field) + ") bool "; + code += OffsetPrefix(field); + code += "\t\t" + GenGetter(field.value.type); + code += "(obj, o)\n\t\treturn true\n\t}\n"; + code += "\treturn false\n"; + code += "}\n\n"; +} + +// Get the value of a vector's struct member. +static void GetMemberOfVectorOfStruct(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + auto vectortype = field.value.type.VectorType(); + + GenReceiver(struct_def, code_ptr); + code += " " + MakeCamel(field.name); + code += "(obj *" + TypeName(field); + code += ", j int) bool " + OffsetPrefix(field); + code += "\t\tx := rcv._tab.Vector(o)\n"; + code += "\t\tx += flatbuffers.UOffsetT(j) * "; + code += NumToString(InlineSize(vectortype)) + "\n"; + if (!(vectortype.struct_def->fixed)) { + code += "\t\tx = rcv._tab.Indirect(x)\n"; + } + code += "\tif obj == nil {\n"; + code += "\t\tobj = new(" + TypeName(field) + ")\n"; + code += "\t}\n"; + code += "\t\tobj.Init(rcv._tab.Bytes, x)\n"; + code += "\t\treturn true\n\t}\n"; + code += "\treturn false\n"; + code += "}\n\n"; +} + +// Get the value of a vector's non-struct member. Uses a named return +// argument to conveniently set the zero value for the result. +static void GetMemberOfVectorOfNonStruct(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + auto vectortype = field.value.type.VectorType(); + + GenReceiver(struct_def, code_ptr); + code += " " + MakeCamel(field.name); + code += "(j int) " + TypeName(field) + " "; + code += OffsetPrefix(field); + code += "\t\ta := rcv._tab.Vector(o)\n"; + code += "\t\treturn " + GenGetter(field.value.type) + "("; + code += "a + flatbuffers.UOffsetT(j * "; + code += NumToString(InlineSize(vectortype)) + "))\n"; + code += "\t}\n"; + if (vectortype.base_type == BASE_TYPE_STRING) { + code += "\treturn \"\"\n"; + } else { + code += "\treturn 0\n"; + } + code += "}\n\n"; +} + +// Begin the creator function signature. +static void BeginBuilderArgs(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + + code += "\n"; + code += "func Create" + struct_def.name; + code += "(builder *flatbuffers.Builder"; +} + +// Recursively generate arguments for a constructor, to deal with nested +// structs. +static void StructBuilderArgs(const StructDef &struct_def, + const char *nameprefix, + std::string *code_ptr) { + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); + ++it) { + auto &field = **it; + if (IsStruct(field.value.type)) { + // Generate arguments for a struct inside a struct. To ensure names + // don't clash, and to make it obvious these arguments are constructing + // a nested struct, prefix the name with the struct name. + StructBuilderArgs(*field.value.type.struct_def, + (field.value.type.struct_def->name + "_").c_str(), + code_ptr); + } else { + std::string &code = *code_ptr; + code += (std::string)", " + nameprefix; + code += MakeCamel(field.name, false); + code += " " + GenTypeBasic(field.value.type); + } + } +} + +// End the creator function signature. +static void EndBuilderArgs(std::string *code_ptr) { + std::string &code = *code_ptr; + code += ") flatbuffers.UOffsetT {\n"; +} + +// Recursively generate struct construction statements and instert manual +// padding. +static void StructBuilderBody(const StructDef &struct_def, + const char *nameprefix, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += " builder.Prep(" + NumToString(struct_def.minalign) + ", "; + code += NumToString(struct_def.bytesize) + ")\n"; + for (auto it = struct_def.fields.vec.rbegin(); + it != struct_def.fields.vec.rend(); + ++it) { + auto &field = **it; + if (field.padding) + code += " builder.Pad(" + NumToString(field.padding) + ")\n"; + if (IsStruct(field.value.type)) { + StructBuilderBody(*field.value.type.struct_def, + (field.value.type.struct_def->name + "_").c_str(), + code_ptr); + } else { + code += " builder.Prepend" + GenMethod(field) + "("; + code += nameprefix + MakeCamel(field.name, false) + ")\n"; + } + } +} + +static void EndBuilderBody(std::string *code_ptr) { + std::string &code = *code_ptr; + code += " return builder.Offset()\n"; + code += "}\n"; +} + +// Get the value of a table's starting offset. +static void GetStartOfTable(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += "func " + struct_def.name + "Start"; + code += "(builder *flatbuffers.Builder) { "; + code += "builder.StartObject("; + code += NumToString(struct_def.fields.vec.size()); + code += ") }\n"; +} + +// Set the value of a table's field. +static void BuildFieldOfTable(const StructDef &struct_def, + const FieldDef &field, + const size_t offset, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += "func " + struct_def.name + "Add" + MakeCamel(field.name); + code += "(builder *flatbuffers.Builder, "; + code += MakeCamel(field.name, false) + " "; + if (!IsScalar(field.value.type.base_type) && (!struct_def.fixed)) { + code += "flatbuffers.UOffsetT"; + } else { + code += GenTypeBasic(field.value.type); + } + code += ") "; + code += "{ builder.Prepend"; + code += GenMethod(field) + "Slot("; + code += NumToString(offset) + ", "; + if (!IsScalar(field.value.type.base_type) && (!struct_def.fixed)) { + code += "flatbuffers.UOffsetT"; + code += "("; + code += MakeCamel(field.name, false) + ")"; + } else { + code += MakeCamel(field.name, false); + } + code += ", " + field.value.constant; + code += ") }\n"; +} + +// Set the value of one of the members of a table's vector. +static void BuildVectorOfTable(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += "func " + struct_def.name + "Start"; + code += MakeCamel(field.name); + code += "Vector(builder *flatbuffers.Builder, numElems int) "; + code += "flatbuffers.UOffsetT { return builder.StartVector("; + code += NumToString(InlineSize(field.value.type.VectorType())); + code += ", numElems) }\n"; +} + +// Get the offset of the end of a table. +static void GetEndOffsetOnTable(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += "func " + struct_def.name + "End"; + code += "(builder *flatbuffers.Builder) flatbuffers.UOffsetT "; + code += "{ return builder.EndObject() }\n"; +} + +// Generate the receiver for function signatures. +static void GenReceiver(const StructDef &struct_def, std::string *code_ptr) { + std::string &code = *code_ptr; + code += "func (rcv *" + struct_def.name + ")"; +} + +// Generate a struct field, conditioned on its child type(s). +static void GenStructAccessor(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + Comment(field.doc_comment, code_ptr, ""); + if (IsScalar(field.value.type.base_type)) { + if (struct_def.fixed) { + GetScalarFieldOfStruct(struct_def, field, code_ptr); + } else { + GetScalarFieldOfTable(struct_def, field, code_ptr); + } + } else { + switch (field.value.type.base_type) { + case BASE_TYPE_STRUCT: + if (struct_def.fixed) { + GetStructFieldOfStruct(struct_def, field, code_ptr); + } else { + GetStructFieldOfTable(struct_def, field, code_ptr); + } + break; + case BASE_TYPE_STRING: + GetStringField(struct_def, field, code_ptr); + break; + case BASE_TYPE_VECTOR: { + auto vectortype = field.value.type.VectorType(); + if (vectortype.base_type == BASE_TYPE_STRUCT) { + GetMemberOfVectorOfStruct(struct_def, field, code_ptr); + } else { + GetMemberOfVectorOfNonStruct(struct_def, field, code_ptr); + } + break; + } + case BASE_TYPE_UNION: + GetUnionField(struct_def, field, code_ptr); + break; + default: + assert(0); + } + } + if (field.value.type.base_type == BASE_TYPE_VECTOR) { + GetVectorLen(struct_def, field, code_ptr); + } +} + +// Generate table constructors, conditioned on its members' types. +static void GenTableBuilders(const StructDef &struct_def, + std::string *code_ptr) { + GetStartOfTable(struct_def, code_ptr); + + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); + ++it) { + auto &field = **it; + if (field.deprecated) continue; + + auto offset = it - struct_def.fields.vec.begin(); + BuildFieldOfTable(struct_def, field, offset, code_ptr); + if (field.value.type.base_type == BASE_TYPE_VECTOR) { + BuildVectorOfTable(struct_def, field, code_ptr); + } + } + + GetEndOffsetOnTable(struct_def, code_ptr); +} + +// Generate struct or table methods. +static void GenStruct(const StructDef &struct_def, + std::string *code_ptr, + StructDef *root_struct_def) { + if (struct_def.generated) return; + + Comment(struct_def.doc_comment, code_ptr); + BeginClass(struct_def, code_ptr); + if (&struct_def == root_struct_def) { + // Generate a special accessor for the table that has been declared as + // the root type. + NewRootTypeFromBuffer(struct_def, code_ptr); + } + // Generate the Init method that sets the field in a pre-existing + // accessor object. This is to allow object reuse. + InitializeExisting(struct_def, code_ptr); + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); + ++it) { + auto &field = **it; + if (field.deprecated) continue; + + GenStructAccessor(struct_def, field, code_ptr); + } + + if (struct_def.fixed) { + // create a struct constructor function + GenStructBuilder(struct_def, code_ptr); + } else { + // Create a set of functions that allow table construction. + GenTableBuilders(struct_def, code_ptr); + } +} + +// Generate enum declarations. +static void GenEnum(const EnumDef &enum_def, std::string *code_ptr) { + if (enum_def.generated) return; + + Comment(enum_def.doc_comment, code_ptr); + BeginEnum(code_ptr); + for (auto it = enum_def.vals.vec.begin(); + it != enum_def.vals.vec.end(); + ++it) { + auto &ev = **it; + Comment(ev.doc_comment, code_ptr, " "); + EnumMember(enum_def, ev, code_ptr); + } + EndEnum(code_ptr); +} + +// Returns the function name that is able to read a value of the given type. +static std::string GenGetter(const Type &type) { + switch (type.base_type) { + case BASE_TYPE_STRING: return "rcv._tab.String"; + case BASE_TYPE_UNION: return "rcv._tab.Union"; + case BASE_TYPE_VECTOR: return GenGetter(type.VectorType()); + default: + return "rcv._tab.Get" + MakeCamel(GenTypeGet(type)); + } +} + +// Returns the method name for use with add/put calls. +static std::string GenMethod(const FieldDef &field) { + return IsScalar(field.value.type.base_type) + ? MakeCamel(GenTypeBasic(field.value.type)) + : (IsStruct(field.value.type) ? "Struct" : "UOffsetT"); +} + + +// Save out the generated code for a Go Table type. +static bool SaveType(const Parser &parser, const Definition &def, + const std::string &classcode, const std::string &path, + bool needs_imports) { + if (!classcode.length()) return true; + + std::string namespace_name; + std::string namespace_dir = path; + auto &namespaces = parser.namespaces_.back()->components; + for (auto it = namespaces.begin(); it != namespaces.end(); ++it) { + if (namespace_name.length()) { + namespace_name += "."; + namespace_dir += PATH_SEPARATOR; + } + namespace_name = *it; + namespace_dir += *it; + mkdir(namespace_dir.c_str(), S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH); + } + + + std::string code = ""; + BeginFile(namespace_name, needs_imports, &code); + code += classcode; + std::string filename = namespace_dir + PATH_SEPARATOR + def.name + ".go"; + return SaveFile(filename.c_str(), code, false); +} + +static std::string GenTypeBasic(const Type &type) { + static const char *ctypename[] = { + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) #GTYPE, + FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) + #undef FLATBUFFERS_TD + }; + return ctypename[type.base_type]; +} + +static std::string GenTypePointer(const Type &type) { + switch (type.base_type) { + case BASE_TYPE_STRING: + return "string"; + case BASE_TYPE_VECTOR: + return GenTypeGet(type.VectorType()); + case BASE_TYPE_STRUCT: + return type.struct_def->name; + case BASE_TYPE_UNION: + // fall through + default: + return "*flatbuffers.Table"; + } +} + +static std::string GenTypeGet(const Type &type) { + return IsScalar(type.base_type) + ? GenTypeBasic(type) + : GenTypePointer(type); +} + +static std::string TypeName(const FieldDef &field) { + return GenTypeGet(field.value.type); +} + +// Create a struct with a builder and the struct's arguments. +static void GenStructBuilder(const StructDef &struct_def, + std::string *code_ptr) { + BeginBuilderArgs(struct_def, code_ptr); + StructBuilderArgs(struct_def, "", code_ptr); + EndBuilderArgs(code_ptr); + + StructBuilderBody(struct_def, "", code_ptr); + EndBuilderBody(code_ptr); +} + +} // namespace go + +bool GenerateGo(const Parser &parser, + const std::string &path, + const std::string & /*file_name*/, + const GeneratorOptions & /*opts*/) { + for (auto it = parser.enums_.vec.begin(); + it != parser.enums_.vec.end(); ++it) { + std::string enumcode; + go::GenEnum(**it, &enumcode); + if (!go::SaveType(parser, **it, enumcode, path, false)) + return false; + } + + for (auto it = parser.structs_.vec.begin(); + it != parser.structs_.vec.end(); ++it) { + std::string declcode; + go::GenStruct(**it, &declcode, parser.root_struct_def); + if (!go::SaveType(parser, **it, declcode, path, true)) + return false; + } + + return true; +} + +} // namespace flatbuffers + diff --git a/src/idl_gen_java.cpp b/src/idl_gen_java.cpp index 35628cbab05..3009d4b0c6c 100755 --- a/src/idl_gen_java.cpp +++ b/src/idl_gen_java.cpp @@ -20,21 +20,12 @@ #include "flatbuffers/idl.h" #include "flatbuffers/util.h" -#ifdef _WIN32 -#include -#define PATH_SEPARATOR "\\" -#define mkdir(n, m) _mkdir(n) -#else -#include -#define PATH_SEPARATOR "/" -#endif - namespace flatbuffers { namespace java { static std::string GenTypeBasic(const Type &type) { static const char *ctypename[] = { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE) #JTYPE, + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) #JTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD }; @@ -73,21 +64,6 @@ static void GenComment(const std::string &dc, } } -// Convert an underscore_based_indentifier in to camelCase. -// Also uppercases the first character if first is true. -static std::string MakeCamel(const std::string &in, bool first = true) { - std::string s; - for (size_t i = 0; i < in.length(); i++) { - if (!i && first) - s += static_cast(toupper(in[0])); - else if (in[i] == '_' && i + 1 < in.length()) - s += static_cast(toupper(in[++i])); - else - s += in[i]; - } - return s; -} - static void GenEnum(EnumDef &enum_def, std::string *code_ptr) { std::string &code = *code_ptr; if (enum_def.generated) return; @@ -264,7 +240,7 @@ static void GenStruct(StructDef &struct_def, } break; case BASE_TYPE_STRING: - code += offset_prefix + getter +"(o) : null"; + code += offset_prefix + getter +"(o + bb_pos) : null"; break; case BASE_TYPE_VECTOR: { auto vectortype = field.value.type.VectorType(); @@ -337,8 +313,12 @@ static void GenStruct(StructDef &struct_def, code += " public static void start" + MakeCamel(field.name); code += "Vector(FlatBufferBuilder builder, int numElems) "; code += "{ builder.startVector("; - code += NumToString(InlineSize(field.value.type.VectorType())); - code += ", numElems); }\n"; + auto vector_type = field.value.type.VectorType(); + auto alignment = InlineAlignment(vector_type); + auto elem_size = InlineSize(vector_type); + code += NumToString(elem_size); + code += ", numElems, " + NumToString(alignment); + code += "); }\n"; } } code += " public static int end" + struct_def.name; @@ -354,27 +334,27 @@ static bool SaveClass(const Parser &parser, const Definition &def, bool needs_imports) { if (!classcode.length()) return true; - std::string name_space_java; - std::string name_space_dir = path; - for (auto it = parser.name_space_.begin(); - it != parser.name_space_.end(); ++it) { - if (name_space_java.length()) { - name_space_java += "."; - name_space_dir += PATH_SEPARATOR; + std::string namespace_java; + std::string namespace_dir = path; + auto &namespaces = parser.namespaces_.back()->components; + for (auto it = namespaces.begin(); it != namespaces.end(); ++it) { + if (namespace_java.length()) { + namespace_java += "."; + namespace_dir += kPathSeparator; } - name_space_java += *it; - name_space_dir += *it; - mkdir(name_space_dir.c_str(), S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH); + namespace_java += *it; + namespace_dir += *it; } + EnsureDirExists(namespace_dir); std::string code = "// automatically generated, do not modify\n\n"; - code += "package " + name_space_java + ";\n\n"; + code += "package " + namespace_java + ";\n\n"; if (needs_imports) { code += "import java.nio.*;\nimport java.lang.*;\nimport java.util.*;\n"; code += "import flatbuffers.*;\n\n"; } code += classcode; - auto filename = name_space_dir + PATH_SEPARATOR + def.name + ".java"; + auto filename = namespace_dir + kPathSeparator + def.name + ".java"; return SaveFile(filename.c_str(), code, false); } @@ -406,4 +386,3 @@ bool GenerateJava(const Parser &parser, } } // namespace flatbuffers - diff --git a/src/idl_gen_text.cpp b/src/idl_gen_text.cpp index 370607240b6..a4e36f8250c 100644 --- a/src/idl_gen_text.cpp +++ b/src/idl_gen_text.cpp @@ -28,18 +28,38 @@ static void GenStruct(const StructDef &struct_def, const Table *table, // If indentation is less than 0, that indicates we don't want any newlines // either. -const char *NewLine(int indent_step) { - return indent_step >= 0 ? "\n" : ""; +const char *NewLine(const GeneratorOptions &opts) { + return opts.indent_step >= 0 ? "\n" : ""; +} + +int Indent(const GeneratorOptions &opts) { + return std::max(opts.indent_step, 0); +} + +// Output an identifier with or without quotes depending on strictness. +void OutputIdentifier(const std::string &name, const GeneratorOptions &opts, + std::string *_text) { + std::string &text = *_text; + if (opts.strict_json) text += "\""; + text += name; + if (opts.strict_json) text += "\""; } // Print (and its template specialization below for pointers) generate text // for a single FlatBuffer value into JSON format. // The general case for scalars: -template void Print(T val, Type /*type*/, int /*indent*/, +template void Print(T val, Type type, int /*indent*/, StructDef * /*union_sd*/, - const GeneratorOptions & /*opts*/, + const GeneratorOptions &opts, std::string *_text) { std::string &text = *_text; + if (type.enum_def && opts.output_enum_identifiers) { + auto enum_val = type.enum_def->ReverseLookup(static_cast(val)); + if (enum_val) { + OutputIdentifier(enum_val->name, opts, _text); + return; + } + } text += NumToString(val); } @@ -49,21 +69,21 @@ template void PrintVector(const Vector &v, Type type, std::string *_text) { std::string &text = *_text; text += "["; - text += NewLine(opts.indent_step); + text += NewLine(opts); for (uoffset_t i = 0; i < v.Length(); i++) { if (i) { text += ","; - text += NewLine(opts.indent_step); + text += NewLine(opts); } - text.append(indent + opts.indent_step, ' '); + text.append(indent + Indent(opts), ' '); if (IsStruct(type)) Print(v.GetStructFromOffset(i * type.struct_def->bytesize), type, - indent + opts.indent_step, nullptr, opts, _text); + indent + Indent(opts), nullptr, opts, _text); else - Print(v.Get(i), type, indent + opts.indent_step, nullptr, + Print(v.Get(i), type, indent + Indent(opts), nullptr, opts, _text); } - text += NewLine(opts.indent_step); + text += NewLine(opts); text.append(indent, ' '); text += "]"; } @@ -77,14 +97,29 @@ static void EscapeString(const String &s, std::string *_text) { case '\n': text += "\\n"; break; case '\t': text += "\\t"; break; case '\r': text += "\\r"; break; + case '\b': text += "\\b"; break; + case '\f': text += "\\f"; break; case '\"': text += "\\\""; break; case '\\': text += "\\\\"; break; default: if (c >= ' ' && c <= '~') { text += c; } else { - auto u = static_cast(c); - text += "\\x" + IntToStringHex(u); + // Not printable ASCII data. Let's see if it's valid UTF-8 first: + const char *utf8 = s.c_str() + i; + int ucc = FromUTF8(&utf8); + if (ucc >= 0x80 && ucc <= 0xFFFF) { + // Parses as Unicode within JSON's \uXXXX range, so use that. + text += "\\u"; + text += IntToStringHex(ucc, 4); + // Skip past characters recognized. + i = static_cast(utf8 - s.c_str() - 1); + } else { + // It's either unprintable ASCII, arbitrary binary, or Unicode data + // that doesn't fit \uXXXX, so use \xXX escape code instead. + text += "\\x"; + text += IntToStringHex(static_cast(c), 2); + } } break; } @@ -124,7 +159,7 @@ template<> void Print(const void *val, type = type.VectorType(); // Call PrintVector above specifically for each element type: switch (type.base_type) { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) \ case BASE_TYPE_ ## ENUM: \ PrintVector( \ *reinterpret_cast *>(val), \ @@ -174,7 +209,6 @@ static void GenStruct(const StructDef &struct_def, const Table *table, std::string *_text) { std::string &text = *_text; text += "{"; - text += NewLine(opts.indent_step); int fieldout = 0; StructDef *union_sd = nullptr; for (auto it = struct_def.fields.vec.begin(); @@ -185,37 +219,37 @@ static void GenStruct(const StructDef &struct_def, const Table *table, // The field is present. if (fieldout++) { text += ","; - text += NewLine(opts.indent_step); } - text.append(indent + opts.indent_step, ' '); - if (opts.strict_json) text += "\""; - text += fd.name; - if (opts.strict_json) text += "\""; + text += NewLine(opts); + text.append(indent + Indent(opts), ' '); + OutputIdentifier(fd.name, opts, _text); text += ": "; switch (fd.value.type.base_type) { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) \ case BASE_TYPE_ ## ENUM: \ GenField(fd, table, struct_def.fixed, \ - opts, indent + opts.indent_step, _text); \ + opts, indent + Indent(opts), _text); \ break; FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD) #undef FLATBUFFERS_TD // Generate drop-thru case statements for all pointer types: - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) \ case BASE_TYPE_ ## ENUM: FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD) #undef FLATBUFFERS_TD - GenFieldOffset(fd, table, struct_def.fixed, indent + opts.indent_step, + GenFieldOffset(fd, table, struct_def.fixed, indent + Indent(opts), union_sd, opts, _text); break; } if (fd.value.type.base_type == BASE_TYPE_UTYPE) { - union_sd = fd.value.type.enum_def->ReverseLookup( + auto enum_val = fd.value.type.enum_def->ReverseLookup( table->GetField(fd.value.offset, 0)); + assert(enum_val); + union_sd = enum_val->struct_def; } } } - text += NewLine(opts.indent_step); + text += NewLine(opts); text.append(indent, ' '); text += "}"; } @@ -231,7 +265,7 @@ void GenerateText(const Parser &parser, const void *flatbuffer, 0, opts, _text); - text += NewLine(opts.indent_step); + text += NewLine(opts); } } // namespace flatbuffers diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index 4a372b8e749..bf550dfd408 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -23,14 +23,14 @@ namespace flatbuffers { const char *const kTypeNames[] = { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE) IDLTYPE, + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) IDLTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD nullptr }; const char kTypeSizes[] = { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE) sizeof(CTYPE), + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) sizeof(CTYPE), FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD }; @@ -81,12 +81,18 @@ template<> inline Offset atot>(const char *s) { TD(Enum, 263, "enum") \ TD(Union, 264, "union") \ TD(NameSpace, 265, "namespace") \ - TD(RootType, 266, "root_type") + TD(RootType, 266, "root_type") \ + TD(FileIdentifier, 267, "file_identifier") \ + TD(FileExtension, 268, "file_extension") \ + TD(Include, 269, "include") +#ifdef __GNUC__ +__extension__ // Stop GCC complaining about trailing comma with -Wpendantic. +#endif enum { #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) kToken ## NAME = VALUE, FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN) #undef FLATBUFFERS_TOKEN - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE) kToken ## ENUM, + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) kToken ## ENUM, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD }; @@ -96,7 +102,7 @@ static std::string TokenToString(int t) { #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) STRING, FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN) #undef FLATBUFFERS_TOKEN - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE) IDLTYPE, + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) IDLTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD }; @@ -109,6 +115,17 @@ static std::string TokenToString(int t) { } } +// Parses exactly nibbles worth of hex digits into a number, or error. +int64_t Parser::ParseHexNum(int nibbles) { + for (int i = 0; i < nibbles; i++) + if (!isxdigit(cursor_[i])) + Error("escape code must be followed by " + NumToString(nibbles) + + " hex digits"); + auto val = StringToInt(cursor_, 16); + cursor_ += nibbles; + return val; +} + void Parser::Next() { doc_comment_.clear(); bool seen_newline = false; @@ -136,8 +153,21 @@ void Parser::Next() { case 'n': attribute_ += '\n'; cursor_++; break; case 't': attribute_ += '\t'; cursor_++; break; case 'r': attribute_ += '\r'; cursor_++; break; + case 'b': attribute_ += '\b'; cursor_++; break; + case 'f': attribute_ += '\f'; cursor_++; break; case '\"': attribute_ += '\"'; cursor_++; break; case '\\': attribute_ += '\\'; cursor_++; break; + case '/': attribute_ += '/'; cursor_++; break; + case 'x': { // Not in the JSON standard + cursor_++; + attribute_ += static_cast(ParseHexNum(2)); + break; + } + case 'u': { + cursor_++; + ToUTF8(static_cast(ParseHexNum(4)), &attribute_); + break; + } default: Error("unknown escape code in string constant"); break; } } else { // printable chars + UTF-8 bytes @@ -170,7 +200,7 @@ void Parser::Next() { attribute_.clear(); attribute_.append(start, cursor_); // First, see if it is a type keyword from the table of types: - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) \ if (attribute_ == IDLTYPE) { \ token_ = kToken ## ENUM; \ return; \ @@ -191,6 +221,15 @@ void Parser::Next() { if (attribute_ == "union") { token_ = kTokenUnion; return; } if (attribute_ == "namespace") { token_ = kTokenNameSpace; return; } if (attribute_ == "root_type") { token_ = kTokenRootType; return; } + if (attribute_ == "include") { token_ = kTokenInclude; return; } + if (attribute_ == "file_identifier") { + token_ = kTokenFileIdentifier; + return; + } + if (attribute_ == "file_extension") { + token_ = kTokenFileExtension; + return; + } // If not, it is a user-defined identifier: token_ = kTokenIdentifier; return; @@ -268,7 +307,7 @@ void Parser::ParseType(Type &type) { // union element. Error("vector of union types not supported (wrap in table first)."); } - type = Type(BASE_TYPE_VECTOR, subtype.struct_def); + type = Type(BASE_TYPE_VECTOR, subtype.struct_def, subtype.enum_def); type.element = subtype.base_type; Expect(']'); return; @@ -333,6 +372,17 @@ void Parser::ParseField(StructDef &struct_def) { field.deprecated = field.attributes.Lookup("deprecated") != nullptr; if (field.deprecated && struct_def.fixed) Error("can't deprecate fields in a struct"); + auto nested = field.attributes.Lookup("nested_flatbuffer"); + if (nested) { + if (nested->type.base_type != BASE_TYPE_STRING) + Error("nested_flatbuffer attribute must be a string (the root type)"); + if (field.value.type.base_type != BASE_TYPE_VECTOR || + field.value.type.element != BASE_TYPE_UCHAR) + Error("nested_flatbuffer attribute may only apply to a vector of ubyte"); + // This will cause an error if the root type of the nested flatbuffer + // wasn't defined elsewhere. + LookupCreateStruct(nested->constant); + } if (typefield) { // If this field is a union, and it has a manually assigned id, @@ -359,9 +409,9 @@ void Parser::ParseAnyValue(Value &val, FieldDef *field) { Error("missing type field before this union value: " + field->name); auto enum_idx = atot( field_stack_.back().first.constant.c_str()); - auto struct_def = val.type.enum_def->ReverseLookup(enum_idx); - if (!struct_def) Error("illegal type id for: " + field->name); - val.constant = NumToString(ParseTable(*struct_def)); + auto enum_val = val.type.enum_def->ReverseLookup(enum_idx); + if (!enum_val) Error("illegal type id for: " + field->name); + val.constant = NumToString(ParseTable(*enum_val->struct_def)); break; } case BASE_TYPE_STRUCT: @@ -396,7 +446,7 @@ void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) { uoffset_t Parser::ParseTable(const StructDef &struct_def) { Expect('{'); size_t fieldn = 0; - for (;;) { + if (!IsNext('}')) for (;;) { std::string name = attribute_; if (!IsNext(kTokenStringConstant)) Expect(kTokenIdentifier); auto field = struct_def.fields.Lookup(name); @@ -429,16 +479,20 @@ uoffset_t Parser::ParseTable(const StructDef &struct_def) { auto field = it->second; if (!struct_def.sortbysize || size == SizeOf(value.type.base_type)) { switch (value.type.base_type) { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) \ case BASE_TYPE_ ## ENUM: \ builder_.Pad(field->padding); \ - builder_.AddElement(value.offset, \ + if (struct_def.fixed) { \ + builder_.PushElement(atot(value.constant.c_str())); \ + } else { \ + builder_.AddElement(value.offset, \ atot( value.constant.c_str()), \ atot(field->value.constant.c_str())); \ + } \ break; FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD); #undef FLATBUFFERS_TD - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) \ case BASE_TYPE_ ## ENUM: \ builder_.Pad(field->padding); \ if (IsStruct(field->value.type)) { \ @@ -487,12 +541,13 @@ uoffset_t Parser::ParseVector(const Type &type) { } Next(); - builder_.StartVector(count * InlineSize(type), InlineAlignment((type))); + builder_.StartVector(count * InlineSize(type) / InlineAlignment(type), + InlineAlignment(type)); for (int i = 0; i < count; i++) { // start at the back, since we're building the data backwards. auto &val = field_stack_.back().first; switch (val.type.base_type) { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE) \ + #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE) \ case BASE_TYPE_ ## ENUM: \ if (IsStruct(val.type)) SerializeStruct(*val.type.struct_def, val); \ else builder_.PushElement(atot(val.constant.c_str())); \ @@ -545,8 +600,52 @@ bool Parser::TryTypedValue(int dtoken, return match; } +int64_t Parser::ParseIntegerFromString(Type &type) { + int64_t result = 0; + // Parse one or more enum identifiers, separated by spaces. + const char *next = attribute_.c_str(); + do { + const char *divider = strchr(next, ' '); + std::string word; + if (divider) { + word = std::string(next, divider); + next = divider + strspn(divider, " "); + } else { + word = next; + next += word.length(); + } + if (type.enum_def) { // The field has an enum type + auto enum_val = type.enum_def->vals.Lookup(word); + if (!enum_val) + Error("unknown enum value: " + word + + ", for enum: " + type.enum_def->name); + result |= enum_val->value; + } else { // No enum type, probably integral field. + if (!IsInteger(type.base_type)) + Error("not a valid value for this field: " + word); + // TODO: could check if its a valid number constant here. + const char *dot = strchr(word.c_str(), '.'); + if (!dot) Error("enum values need to be qualified by an enum type"); + std::string enum_def_str(word.c_str(), dot); + std::string enum_val_str(dot + 1, word.c_str() + word.length()); + auto enum_def = enums_.Lookup(enum_def_str); + if (!enum_def) Error("unknown enum: " + enum_def_str); + auto enum_val = enum_def->vals.Lookup(enum_val_str); + if (!enum_val) Error("unknown enum value: " + enum_val_str); + result |= enum_val->value; + } + } while(*next); + return result; +} + void Parser::ParseSingleValue(Value &e) { - if (TryTypedValue(kTokenIntegerConstant, + // First check if this could be a string/identifier enum value: + if (e.type.base_type != BASE_TYPE_STRING && + e.type.base_type != BASE_TYPE_NONE && + (token_ == kTokenIdentifier || token_ == kTokenStringConstant)) { + e.constant = NumToString(ParseIntegerFromString(e.type)); + Next(); + } else if (TryTypedValue(kTokenIntegerConstant, IsScalar(e.type.base_type), e, BASE_TYPE_INT) || @@ -558,19 +657,6 @@ void Parser::ParseSingleValue(Value &e) { e.type.base_type == BASE_TYPE_STRING, e, BASE_TYPE_STRING)) { - } else if (token_ == kTokenIdentifier) { - for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) { - auto ev = (*it)->vals.Lookup(attribute_); - if (ev) { - attribute_ = NumToString(ev->value); - TryTypedValue(kTokenIdentifier, - IsInteger(e.type.base_type), - e, - BASE_TYPE_INT); - return; - } - } - Error("not valid enum value: " + attribute_); } else { Error("cannot parse value starting with: " + TokenToString(token_)); } @@ -585,6 +671,7 @@ StructDef *Parser::LookupCreateStruct(const std::string &name) { structs_.Add(name, struct_def); struct_def->name = name; struct_def->predecl = true; + struct_def->defined_namespace = namespaces_.back(); } return struct_def; } @@ -611,6 +698,8 @@ void Parser::ParseEnum(bool is_union) { ParseType(enum_def.underlying_type); if (!IsInteger(enum_def.underlying_type.base_type)) Error("underlying enum type must be integral"); + // Make this type refer back to the enum it was derived from. + enum_def.underlying_type.enum_def = &enum_def; } ParseMetaData(enum_def); Expect('{'); @@ -620,10 +709,10 @@ void Parser::ParseEnum(bool is_union) { std::string dc = doc_comment_; Expect(kTokenIdentifier); auto prevsize = enum_def.vals.vec.size(); - auto &ev = *new EnumVal(name, static_cast( - enum_def.vals.vec.size() - ? enum_def.vals.vec.back()->value + 1 - : 0)); + auto value = enum_def.vals.vec.size() + ? enum_def.vals.vec.back()->value + 1 + : 0; + auto &ev = *new EnumVal(name, value); if (enum_def.vals.Add(name, &ev)) Error("enum value already exists: " + name); ev.doc_comment = dc; @@ -636,8 +725,17 @@ void Parser::ParseEnum(bool is_union) { if (prevsize && enum_def.vals.vec[prevsize - 1]->value >= ev.value) Error("enum values must be specified in ascending order"); } - } while (IsNext(',')); + } while (IsNext(',') && token_ != '}'); Expect('}'); + if (enum_def.attributes.Lookup("bit_flags")) { + for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); + ++it) { + if (static_cast((*it)->value) >= + SizeOf(enum_def.underlying_type.base_type) * 8) + Error("bit flag out of range of underlying integral type"); + (*it)->value = 1LL << (*it)->value; + } + } } void Parser::ParseDecl() { @@ -702,6 +800,31 @@ void Parser::ParseDecl() { } } } + // Check that no identifiers clash with auto generated fields. + // This is not an ideal situation, but should occur very infrequently, + // and allows us to keep using very readable names for type & length fields + // without inducing compile errors. + auto CheckClash = [&fields, &struct_def](const char *suffix, + BaseType basetype) { + auto len = strlen(suffix); + for (auto it = fields.begin(); it != fields.end(); ++it) { + auto &name = (*it)->name; + if (name.length() > len && + name.compare(name.length() - len, len, suffix) == 0 && + (*it)->value.type.base_type != BASE_TYPE_UTYPE) { + auto field = struct_def.fields.Lookup( + name.substr(0, name.length() - len)); + if (field && field->value.type.base_type == basetype) + Error("Field " + name + + " would clash with generated functions for field " + + field->name); + } + } + }; + CheckClash("_type", BASE_TYPE_UNION); + CheckClash("Type", BASE_TYPE_UNION); + CheckClash("_length", BASE_TYPE_VECTOR); + CheckClash("Length", BASE_TYPE_VECTOR); Expect('}'); } @@ -710,19 +833,66 @@ bool Parser::SetRootType(const char *name) { return root_struct_def != nullptr; } -bool Parser::Parse(const char *source) { +void Parser::MarkGenerated() { + // Since the Parser object retains definitions across files, we must + // ensure we only output code for definitions once, in the file they are first + // declared. This function marks all existing definitions as having already + // been generated. + for (auto it = enums_.vec.begin(); + it != enums_.vec.end(); ++it) { + (*it)->generated = true; + } + for (auto it = structs_.vec.begin(); + it != structs_.vec.end(); ++it) { + (*it)->generated = true; + } +} + +bool Parser::Parse(const char *source, const char *filepath) { + included_files_[filepath] = true; + // This is the starting point to reset to if we interrupted our parsing + // to deal with an include: + restart_parse_after_include: source_ = cursor_ = source; line_ = 1; error_.clear(); builder_.Clear(); try { Next(); + // Includes must come first: + while (IsNext(kTokenInclude)) { + auto name = attribute_; + Expect(kTokenStringConstant); + auto path = StripFileName(filepath); + if (path.length()) name = path + kPathSeparator + name; + if (included_files_.find(name) == included_files_.end()) { + // We found an include file that we have not parsed yet. + // Load it and parse it. + std::string contents; + if (!LoadFile(name.c_str(), true, &contents)) + Error("unable to load include file: " + name); + Parse(contents.c_str(), name.c_str()); + // Any errors, we're done. + if (error_.length()) return false; + // We do not want to output code for any included files: + MarkGenerated(); + // This is the easiest way to continue this file after an include: + // instead of saving and restoring all the state, we simply start the + // file anew. This will cause it to encounter the same include statement + // again, but this time it will skip it, because it was entered into + // included_files_. + goto restart_parse_after_include; + } + Expect(';'); + } + // Now parse all other kinds of declarations: while (token_ != kTokenEof) { if (token_ == kTokenNameSpace) { Next(); - name_space_.clear(); + auto ns = new Namespace(); + namespaces_.push_back(ns); for (;;) { - name_space_.push_back(attribute_); + ns->components.push_back(attribute_); Expect(kTokenIdentifier); if (!IsNext('.')) break; } @@ -741,11 +911,28 @@ bool Parser::Parse(const char *source) { Next(); auto root_type = attribute_; Expect(kTokenIdentifier); - Expect(';'); if (!SetRootType(root_type.c_str())) Error("unknown root type: " + root_type); if (root_struct_def->fixed) Error("root type must be a table"); + Expect(';'); + } else if (token_ == kTokenFileIdentifier) { + Next(); + file_identifier_ = attribute_; + Expect(kTokenStringConstant); + if (file_identifier_.length() != + FlatBufferBuilder::kFileIdentifierLength) + Error("file_identifier must be exactly " + + NumToString(FlatBufferBuilder::kFileIdentifierLength) + + " characters"); + Expect(';'); + } else if (token_ == kTokenFileExtension) { + Next(); + file_extension_ = attribute_; + Expect(kTokenStringConstant); + Expect(';'); + } else if(token_ == kTokenInclude) { + Error("includes must come before declarations"); } else { ParseDecl(); } diff --git a/tests/GoTest.sh b/tests/GoTest.sh new file mode 100755 index 00000000000..2e7b5c7a111 --- /dev/null +++ b/tests/GoTest.sh @@ -0,0 +1,52 @@ +#!/bin/bash -eu +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pushd "$(dirname $0)" >/dev/null +test_dir="$(pwd)" +go_path=${test_dir}/go_gen +go_src=${go_path}/src + +# Emit Go code for the example schema in the test dir: +../flatc -g monster_test.fbs + +# Go requires a particular layout of files in order to link multiple packages. +# Copy flatbuffer Go files to their own package directories to compile the +# test binary: +mkdir -p ${go_src}/MyGame/Example +mkdir -p ${go_src}/github.com/google/flatbuffers/go +mkdir -p ${go_src}/flatbuffers_test + +cp -u MyGame/Example/*.go ./go_gen/src/MyGame/Example/ +cp -u ../go/* ./go_gen/src/github.com/google/flatbuffers/go +cp -u ./go_test.go ./go_gen/src/flatbuffers_test/ + +# Run tests with necessary flags. +# Developers may wish to see more detail by appending the verbosity flag +# -test.v to arguments for this command, as in: +# go -test -test.v ... +# Developers may also wish to run benchmarks, which may be achieved with the +# flag -test.bench and the wildcard regexp ".": +# go -test -test.bench=. ... +GOPATH=${go_path} go test flatbuffers_test \ + --test.coverpkg=github.com/google/flatbuffers/go \ + --cpp_data=${test_dir}/monsterdata_test.bin \ + --out_data=${test_dir}/monsterdata_go_wire.bin \ + --fuzz=true \ + --fuzz_fields=4 \ + --fuzz_objects=10000 + +rm -rf ${go_path}/{pkg,src} + +echo "OK: Go tests passed." diff --git a/tests/JavaTest.java b/tests/JavaTest.java index dd6cbcaeb82..902362fe6d5 100755 --- a/tests/JavaTest.java +++ b/tests/JavaTest.java @@ -52,6 +52,8 @@ public static void main(String[] args) { // We set up the same values as monsterdata.json: int str = fbb.createString("MyMonster"); + int test1 = fbb.createString("test1"); + int test2 = fbb.createString("test2"); Monster.startInventoryVector(fbb, 5); for (byte i = 4; i >=0; i--) fbb.addByte(i); @@ -66,6 +68,11 @@ public static void main(String[] args) { Test.createTest(fbb, (short)30, (byte)40); int test4 = fbb.endVector(); + Monster.startTestarrayofstringVector(fbb, 2); + fbb.addOffset(test2); + fbb.addOffset(test1); + int testArrayOfString = fbb.endVector(); + Monster.startMonster(fbb); Monster.addPos(fbb, Vec3.createVec3(fbb, 1.0f, 2.0f, 3.0f, 3.0, (byte)4, (short)5, (byte)6)); @@ -75,6 +82,7 @@ public static void main(String[] args) { Monster.addTestType(fbb, (byte)1); Monster.addTest(fbb, mon2); Monster.addTest4(fbb, test4); + Monster.addTestarrayofstring(fbb, testArrayOfString); int mon = Monster.endMonster(fbb); fbb.finish(mon); @@ -135,6 +143,10 @@ static void TestBuffer(ByteBuffer bb, int start) { Test test_1 = monster.test4(1); TestEq(monster.test4Length(), 2); TestEq(test_0.a() + test_0.b() + test_1.a() + test_1.b(), 100); + + TestEq(monster.testarrayofstringLength(), 2); + TestEq(monster.testarrayofstring(0),"test1"); + TestEq(monster.testarrayofstring(1),"test2"); } static void TestEq(T a, T b) { diff --git a/tests/MyGame/Example/Any.go b/tests/MyGame/Example/Any.go new file mode 100644 index 00000000000..0039bb8c066 --- /dev/null +++ b/tests/MyGame/Example/Any.go @@ -0,0 +1,8 @@ +// automatically generated, do not modify + +package Example + +const ( + AnyNONE = 0 + AnyMonster = 1 +) diff --git a/tests/MyGame/Example/Color.go b/tests/MyGame/Example/Color.go new file mode 100644 index 00000000000..289a7b063cb --- /dev/null +++ b/tests/MyGame/Example/Color.go @@ -0,0 +1,9 @@ +// automatically generated, do not modify + +package Example + +const ( + ColorRed = 1 + ColorGreen = 2 + ColorBlue = 8 +) diff --git a/tests/MyGame/Example/Color.java b/tests/MyGame/Example/Color.java index 44738ea4f0f..8144e41981c 100755 --- a/tests/MyGame/Example/Color.java +++ b/tests/MyGame/Example/Color.java @@ -3,8 +3,8 @@ package MyGame.Example; public class Color { - public static final byte Red = 0; - public static final byte Green = 1; - public static final byte Blue = 2; + public static final byte Red = 1; + public static final byte Green = 2; + public static final byte Blue = 8; }; diff --git a/tests/MyGame/Example/Monster.go b/tests/MyGame/Example/Monster.go new file mode 100644 index 00000000000..2e35ab6c8a8 --- /dev/null +++ b/tests/MyGame/Example/Monster.go @@ -0,0 +1,215 @@ +// automatically generated, do not modify + +package Example + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) +type Monster struct { + _tab flatbuffers.Table +} + +func GetRootAsMonster(buf []byte, offset flatbuffers.UOffsetT) *Monster { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Monster{} + x.Init(buf, n + offset) + return x +} + +func (rcv *Monster) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Monster) Pos(obj *Vec3) *Vec3 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + x := o + rcv._tab.Pos + if obj == nil { + obj = new(Vec3) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +func (rcv *Monster) Mana() int16 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 150 +} + +func (rcv *Monster) Hp() int16 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 100 +} + +func (rcv *Monster) Name() string { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + return rcv._tab.String(o) + } + return "" +} + +func (rcv *Monster) Inventory(j int) byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) + if o != 0 { + a := rcv._tab.Vector(o) + return rcv._tab.GetByte(a + flatbuffers.UOffsetT(j * 1)) + } + return 0 +} + +func (rcv *Monster) InventoryLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func (rcv *Monster) Color() int8 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(16)) + if o != 0 { + return rcv._tab.GetInt8(o + rcv._tab.Pos) + } + return 8 +} + +func (rcv *Monster) TestType() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(18)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Monster) Test(obj *flatbuffers.Table) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(20)) + if o != 0 { + rcv._tab.Union(obj, o) + return true + } + return false +} + +func (rcv *Monster) Test4(obj *Test, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(22)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 4 + if obj == nil { + obj = new(Test) + } + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Monster) Test4Length() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(22)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func (rcv *Monster) Testarrayofstring(j int) string { + o := flatbuffers.UOffsetT(rcv._tab.Offset(24)) + if o != 0 { + a := rcv._tab.Vector(o) + return rcv._tab.String(a + flatbuffers.UOffsetT(j * 4)) + } + return "" +} + +func (rcv *Monster) TestarrayofstringLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(24)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +/// an example documentation comment: this will end up in the generated code multiline too +func (rcv *Monster) Testarrayoftables(obj *Monster, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(26)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 4 + x = rcv._tab.Indirect(x) + if obj == nil { + obj = new(Monster) + } + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Monster) TestarrayoftablesLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(26)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func (rcv *Monster) Enemy(obj *Monster) *Monster { + o := flatbuffers.UOffsetT(rcv._tab.Offset(28)) + if o != 0 { + x := rcv._tab.Indirect(o + rcv._tab.Pos) + if obj == nil { + obj = new(Monster) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +func (rcv *Monster) Testnestedflatbuffer(j int) byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(30)) + if o != 0 { + a := rcv._tab.Vector(o) + return rcv._tab.GetByte(a + flatbuffers.UOffsetT(j * 1)) + } + return 0 +} + +func (rcv *Monster) TestnestedflatbufferLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(30)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func MonsterStart(builder *flatbuffers.Builder) { builder.StartObject(14) } +func MonsterAddPos(builder *flatbuffers.Builder, pos flatbuffers.UOffsetT) { builder.PrependStructSlot(0, flatbuffers.UOffsetT(pos), 0) } +func MonsterAddMana(builder *flatbuffers.Builder, mana int16) { builder.PrependInt16Slot(1, mana, 150) } +func MonsterAddHp(builder *flatbuffers.Builder, hp int16) { builder.PrependInt16Slot(2, hp, 100) } +func MonsterAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(name), 0) } +func MonsterAddInventory(builder *flatbuffers.Builder, inventory flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(5, flatbuffers.UOffsetT(inventory), 0) } +func MonsterStartInventoryVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(1, numElems) } +func MonsterAddColor(builder *flatbuffers.Builder, color int8) { builder.PrependInt8Slot(6, color, 8) } +func MonsterAddTestType(builder *flatbuffers.Builder, testType byte) { builder.PrependByteSlot(7, testType, 0) } +func MonsterAddTest(builder *flatbuffers.Builder, test flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(8, flatbuffers.UOffsetT(test), 0) } +func MonsterAddTest4(builder *flatbuffers.Builder, test4 flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(9, flatbuffers.UOffsetT(test4), 0) } +func MonsterStartTest4Vector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(4, numElems) } +func MonsterAddTestarrayofstring(builder *flatbuffers.Builder, testarrayofstring flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(10, flatbuffers.UOffsetT(testarrayofstring), 0) } +func MonsterStartTestarrayofstringVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(4, numElems) } +func MonsterAddTestarrayoftables(builder *flatbuffers.Builder, testarrayoftables flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(11, flatbuffers.UOffsetT(testarrayoftables), 0) } +func MonsterStartTestarrayoftablesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(4, numElems) } +func MonsterAddEnemy(builder *flatbuffers.Builder, enemy flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(12, flatbuffers.UOffsetT(enemy), 0) } +func MonsterAddTestnestedflatbuffer(builder *flatbuffers.Builder, testnestedflatbuffer flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(13, flatbuffers.UOffsetT(testnestedflatbuffer), 0) } +func MonsterStartTestnestedflatbufferVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(1, numElems) } +func MonsterEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } diff --git a/tests/MyGame/Example/Monster.java b/tests/MyGame/Example/Monster.java index 439dd7a4066..68da9d89466 100755 --- a/tests/MyGame/Example/Monster.java +++ b/tests/MyGame/Example/Monster.java @@ -14,10 +14,10 @@ public class Monster extends Table { public Vec3 pos(Vec3 obj) { int o = __offset(4); return o != 0 ? obj.__init(o + bb_pos, bb) : null; } public short mana() { int o = __offset(6); return o != 0 ? bb.getShort(o + bb_pos) : 150; } public short hp() { int o = __offset(8); return o != 0 ? bb.getShort(o + bb_pos) : 100; } - public String name() { int o = __offset(10); return o != 0 ? __string(o) : null; } + public String name() { int o = __offset(10); return o != 0 ? __string(o + bb_pos) : null; } public byte inventory(int j) { int o = __offset(14); return o != 0 ? bb.get(__vector(o) + j * 1) : 0; } public int inventoryLength() { int o = __offset(14); return o != 0 ? __vector_len(o) : 0; } - public byte color() { int o = __offset(16); return o != 0 ? bb.get(o + bb_pos) : 2; } + public byte color() { int o = __offset(16); return o != 0 ? bb.get(o + bb_pos) : 8; } public byte testType() { int o = __offset(18); return o != 0 ? bb.get(o + bb_pos) : 0; } public Table test(Table obj) { int o = __offset(20); return o != 0 ? __union(obj, o) : null; } public Test test4(int j) { return test4(new Test(), j); } @@ -31,24 +31,31 @@ public class Monster extends Table { public int testarrayoftablesLength() { int o = __offset(26); return o != 0 ? __vector_len(o) : 0; } public Monster enemy() { return enemy(new Monster()); } public Monster enemy(Monster obj) { int o = __offset(28); return o != 0 ? obj.__init(__indirect(o + bb_pos), bb) : null; } + public byte testnestedflatbuffer(int j) { int o = __offset(30); return o != 0 ? bb.get(__vector(o) + j * 1) : 0; } + public int testnestedflatbufferLength() { int o = __offset(30); return o != 0 ? __vector_len(o) : 0; } + public Monster testempty() { return testempty(new Monster()); } + public Monster testempty(Monster obj) { int o = __offset(32); return o != 0 ? obj.__init(__indirect(o + bb_pos), bb) : null; } - public static void startMonster(FlatBufferBuilder builder) { builder.startObject(13); } + public static void startMonster(FlatBufferBuilder builder) { builder.startObject(15); } public static void addPos(FlatBufferBuilder builder, int posOffset) { builder.addStruct(0, posOffset, 0); } public static void addMana(FlatBufferBuilder builder, short mana) { builder.addShort(1, mana, 150); } public static void addHp(FlatBufferBuilder builder, short hp) { builder.addShort(2, hp, 100); } public static void addName(FlatBufferBuilder builder, int nameOffset) { builder.addOffset(3, nameOffset, 0); } public static void addInventory(FlatBufferBuilder builder, int inventoryOffset) { builder.addOffset(5, inventoryOffset, 0); } - public static void startInventoryVector(FlatBufferBuilder builder, int numElems) { builder.startVector(1, numElems); } - public static void addColor(FlatBufferBuilder builder, byte color) { builder.addByte(6, color, 2); } + public static void startInventoryVector(FlatBufferBuilder builder, int numElems) { builder.startVector(1, numElems, 1); } + public static void addColor(FlatBufferBuilder builder, byte color) { builder.addByte(6, color, 8); } public static void addTestType(FlatBufferBuilder builder, byte testType) { builder.addByte(7, testType, 0); } public static void addTest(FlatBufferBuilder builder, int testOffset) { builder.addOffset(8, testOffset, 0); } public static void addTest4(FlatBufferBuilder builder, int test4Offset) { builder.addOffset(9, test4Offset, 0); } - public static void startTest4Vector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems); } + public static void startTest4Vector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 2); } public static void addTestarrayofstring(FlatBufferBuilder builder, int testarrayofstringOffset) { builder.addOffset(10, testarrayofstringOffset, 0); } - public static void startTestarrayofstringVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems); } + public static void startTestarrayofstringVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 4); } public static void addTestarrayoftables(FlatBufferBuilder builder, int testarrayoftablesOffset) { builder.addOffset(11, testarrayoftablesOffset, 0); } - public static void startTestarrayoftablesVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems); } + public static void startTestarrayoftablesVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 4); } public static void addEnemy(FlatBufferBuilder builder, int enemyOffset) { builder.addOffset(12, enemyOffset, 0); } + public static void addTestnestedflatbuffer(FlatBufferBuilder builder, int testnestedflatbufferOffset) { builder.addOffset(13, testnestedflatbufferOffset, 0); } + public static void startTestnestedflatbufferVector(FlatBufferBuilder builder, int numElems) { builder.startVector(1, numElems, 1); } + public static void addTestempty(FlatBufferBuilder builder, int testemptyOffset) { builder.addOffset(14, testemptyOffset, 0); } public static int endMonster(FlatBufferBuilder builder) { return builder.endObject(); } }; diff --git a/tests/MyGame/Example/Test.go b/tests/MyGame/Example/Test.go new file mode 100644 index 00000000000..dc4de8eec9a --- /dev/null +++ b/tests/MyGame/Example/Test.go @@ -0,0 +1,26 @@ +// automatically generated, do not modify + +package Example + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) +type Test struct { + _tab flatbuffers.Struct +} + +func (rcv *Test) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Test) A() int16 { return rcv._tab.GetInt16(rcv._tab.Pos + flatbuffers.UOffsetT(0)) } +func (rcv *Test) B() int8 { return rcv._tab.GetInt8(rcv._tab.Pos + flatbuffers.UOffsetT(2)) } + +func CreateTest(builder *flatbuffers.Builder, a int16, b int8) flatbuffers.UOffsetT { + builder.Prep(2, 4) + builder.Pad(1) + builder.PrependInt8(b) + builder.PrependInt16(a) + return builder.Offset() +} diff --git a/tests/MyGame/Example/Vec3.go b/tests/MyGame/Example/Vec3.go new file mode 100644 index 00000000000..d2bab2c9bc1 --- /dev/null +++ b/tests/MyGame/Example/Vec3.go @@ -0,0 +1,45 @@ +// automatically generated, do not modify + +package Example + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) +type Vec3 struct { + _tab flatbuffers.Struct +} + +func (rcv *Vec3) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Vec3) X() float32 { return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(0)) } +func (rcv *Vec3) Y() float32 { return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(4)) } +func (rcv *Vec3) Z() float32 { return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(8)) } +func (rcv *Vec3) Test1() float64 { return rcv._tab.GetFloat64(rcv._tab.Pos + flatbuffers.UOffsetT(16)) } +func (rcv *Vec3) Test2() int8 { return rcv._tab.GetInt8(rcv._tab.Pos + flatbuffers.UOffsetT(24)) } +func (rcv *Vec3) Test3(obj *Test) *Test { + if obj == nil { + obj = new(Test) + } + obj.Init(rcv._tab.Bytes, rcv._tab.Pos + 26) + return obj +} + +func CreateVec3(builder *flatbuffers.Builder, x float32, y float32, z float32, test1 float64, test2 int8, Test_a int16, Test_b int8) flatbuffers.UOffsetT { + builder.Prep(16, 32) + builder.Pad(2) + builder.Prep(2, 4) + builder.Pad(1) + builder.PrependInt8(Test_b) + builder.PrependInt16(Test_a) + builder.Pad(1) + builder.PrependInt8(test2) + builder.PrependFloat64(test1) + builder.Pad(4) + builder.PrependFloat32(z) + builder.PrependFloat32(y) + builder.PrependFloat32(x) + return builder.Offset() +} diff --git a/tests/go_test.go b/tests/go_test.go new file mode 100644 index 00000000000..5ca2ef7b104 --- /dev/null +++ b/tests/go_test.go @@ -0,0 +1,1104 @@ +package main + +import ( + example "MyGame/Example" // refers to generated code + "bytes" + "flag" + "fmt" + flatbuffers "github.com/google/flatbuffers/go" + "io/ioutil" + "os" + "reflect" + "sort" + "testing" +) + +var ( + cppData, javaData, outData string + fuzz bool + fuzzFields, fuzzObjects int +) + +func init() { + flag.StringVar(&cppData, "cpp_data", "", + "location of monsterdata_test.bin to verify against (required)") + flag.StringVar(&javaData, "java_data", "", + "location of monsterdata_java_wire.bin to verify against (optional)") + flag.StringVar(&outData, "out_data", "", + "location to write generated Go data") + flag.BoolVar(&fuzz, "fuzz", false, "perform fuzzing") + flag.IntVar(&fuzzFields, "fuzz_fields", 4, "fields per fuzzer object") + flag.IntVar(&fuzzObjects, "fuzz_objects", 10000, + "number of fuzzer objects (higher is slower and more thorough") + flag.Parse() + + if cppData == "" { + fmt.Fprintf(os.Stderr, "cpp_data argument is required\n") + os.Exit(1) + } +} + +// Store specific byte patterns in these variables for the fuzzer. These +// values are taken verbatim from the C++ function FuzzTest1. +var ( + overflowingInt32Val = flatbuffers.GetInt32([]byte{0x83, 0x33, 0x33, 0x33}) + overflowingInt64Val = flatbuffers.GetInt64([]byte{0x84, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44}) +) + +// TestAll runs all checks, failing if any errors occur. +func TestAll(t *testing.T) { + // Verify that the Go FlatBuffers runtime library generates the + // expected bytes (does not use any schema): + CheckByteLayout(t.Fatalf) + + // Verify that using the generated Go code builds a buffer without + // returning errors: + generated, off := CheckGeneratedBuild(t.Fatalf) + + // Verify that the buffer generated by Go code is readable by the + // generated Go code: + CheckReadBuffer(generated, off, t.Fatalf) + + // Verify that the buffer generated by C++ code is readable by the + // generated Go code: + monsterDataCpp, err := ioutil.ReadFile(cppData) + if err != nil { + t.Fatal(err) + } + CheckReadBuffer(monsterDataCpp, 0, t.Fatalf) + + // Verify that vtables are deduplicated when written: + CheckVtableDeduplication(t.Fatalf) + + // Verify that the Go code used in FlatBuffers documentation passes + // some sanity checks: + CheckDocExample(generated, off, t.Fatalf) + + // If the filename of the FlatBuffers file generated by the Java test + // is given, check that Go code can read it, and that Go code + // generates an identical buffer when used to create the example data: + if javaData != "" { + monsterDataJava, err := ioutil.ReadFile(javaData) + if err != nil { + t.Fatal(err) + } + CheckReadBuffer(monsterDataJava, 0, t.Fatalf) + CheckByteEquality(generated[off:], monsterDataJava, t.Fatalf) + } + + // Verify that various fuzzing scenarios produce a valid FlatBuffer. + if fuzz { + checkFuzz(fuzzFields, fuzzObjects, t.Fatalf) + } + + // Write the generated buffer out to a file: + err = ioutil.WriteFile(outData, generated[off:], os.FileMode(0644)) + if err != nil { + t.Fatal(err) + } +} + +// CheckReadBuffer checks that the given buffer is evaluated correctly +// as the example Monster. +func CheckReadBuffer(buf []byte, offset flatbuffers.UOffsetT, fail func(string, ...interface{})) { + monster := example.GetRootAsMonster(buf, offset) + + if got := monster.Hp(); 80 != got { + fail(FailString("hp", 80, got)) + } + + // default + if got := monster.Mana(); 150 != got { + fail(FailString("mana", 150, got)) + } + + if got := monster.Name(); "MyMonster" != got { + fail(FailString("name", "MyMonster", got)) + } + + // initialize a Vec3 from Pos() + vec := new(example.Vec3) + vec = monster.Pos(vec) + if vec == nil { + fail("vec3 initialization failed") + } + + // check that new allocs equal given ones: + vec2 := monster.Pos(nil) + if !reflect.DeepEqual(vec, vec2) { + fail("fresh allocation failed") + } + + // verify the properties of the Vec3 + if got := vec.X(); float32(1.0) != got { + fail(FailString("Pos.X", float32(1.0), got)) + } + + if got := vec.Y(); float32(2.0) != got { + fail(FailString("Pos.Y", float32(2.0), got)) + } + + if got := vec.Z(); float32(3.0) != got { + fail(FailString("Pos.Z", float32(3.0), got)) + } + + if got := vec.Test1(); float64(3.0) != got { + fail(FailString("Pos.Test1", float64(3.0), got)) + } + + if got := vec.Test2(); int8(4) != got { + fail(FailString("Pos.Test2", int8(4), got)) + } + + // initialize a Test from Test3(...) + t := new(example.Test) + t = vec.Test3(t) + if t == nil { + fail("vec.Test3(&t) failed") + } + + // check that new allocs equal given ones: + t2 := vec.Test3(nil) + if !reflect.DeepEqual(t, t2) { + fail("fresh allocation failed") + } + + // verify the properties of the Test + if got := t.A(); int16(5) != got { + fail(FailString("t.A()", int16(5), got)) + } + + if got := t.B(); int8(6) != got { + fail(FailString("t.B()", int8(6), got)) + } + + if got := monster.TestType(); example.AnyMonster != got { + fail(FailString("monster.TestType()", example.AnyMonster, got)) + } + + if unionType := monster.TestType(); unionType != example.AnyMonster { + fail("monster.TestType()") + } + + // initialize a Table from a union field Test(...) + var table2 flatbuffers.Table + if ok := monster.Test(&table2); !ok { + fail("monster.Test(&monster2) failed") + } + + // initialize a Monster from the Table from the union + var monster2 example.Monster + monster2.Init(table2.Bytes, table2.Pos) + + if got := monster2.Hp(); int16(20) != got { + fail(FailString("monster2.Hp()", int16(20), got)) + } + + if got := monster.InventoryLength(); 5 != got { + fail(FailString("monster.InventoryLength", 5, got)) + } + + invsum := 0 + l := monster.InventoryLength() + for i := 0; i < l; i++ { + v := monster.Inventory(i) + invsum += int(v) + } + if invsum != 10 { + fail(FailString("monster inventory sum", 10, invsum)) + } + + if got := monster.Test4Length(); 2 != got { + fail(FailString("monster.Test4Length()", 2, got)) + } + + var test0 example.Test + ok := monster.Test4(&test0, 0) + if !ok { + fail(FailString("monster.Test4(&test0, 0)", true, ok)) + } + + var test1 example.Test + ok = monster.Test4(&test1, 1) + if !ok { + fail(FailString("monster.Test4(&test1, 1)", true, ok)) + } + + // the position of test0 and test1 are swapped in monsterdata_java_wire + // and monsterdata_test_wire, so ignore ordering + v0 := test0.A() + v1 := test0.B() + v2 := test1.A() + v3 := test1.B() + sum := int(v0) + int(v1) + int(v2) + int(v3) + + if 100 != sum { + fail(FailString("test0 and test1 sum", 100, sum)) + } +} + +// Low level stress/fuzz test: serialize/deserialize a variety of +// different kinds of data in different combinations +func checkFuzz(fuzzFields, fuzzObjects int, fail func(string, ...interface{})) { + + // Values we're testing against: chosen to ensure no bits get chopped + // off anywhere, and also be different from eachother. + boolVal := true + int8Val := int8(-127) // 0x81 + uint8Val := uint8(0xFF) + int16Val := int16(-32222) // 0x8222 + uint16Val := uint16(0xFEEE) + int32Val := int32(overflowingInt32Val) + uint32Val := uint32(0xFDDDDDDD) + int64Val := int64(overflowingInt64Val) + uint64Val := uint64(0xFCCCCCCCCCCCCCCC) + float32Val := float32(3.14159) + float64Val := float64(3.14159265359) + + testValuesMax := 11 // hardcoded to the number of scalar types + + builder := flatbuffers.NewBuilder(0) + l := NewLCG() + + objects := make([]flatbuffers.UOffsetT, fuzzObjects) + + // Generate fuzzObjects random objects each consisting of + // fuzzFields fields, each of a random type. + for i := 0; i < fuzzObjects; i++ { + builder.StartObject(fuzzFields) + + for f := 0; f < fuzzFields; f++ { + choice := l.Next() % uint32(testValuesMax) + switch choice { + case 0: + builder.PrependBoolSlot(int(f), boolVal, false) + case 1: + builder.PrependInt8Slot(int(f), int8Val, 0) + case 2: + builder.PrependUint8Slot(int(f), uint8Val, 0) + case 3: + builder.PrependInt16Slot(int(f), int16Val, 0) + case 4: + builder.PrependUint16Slot(int(f), uint16Val, 0) + case 5: + builder.PrependInt32Slot(int(f), int32Val, 0) + case 6: + builder.PrependUint32Slot(int(f), uint32Val, 0) + case 7: + builder.PrependInt64Slot(int(f), int64Val, 0) + case 8: + builder.PrependUint64Slot(int(f), uint64Val, 0) + case 9: + builder.PrependFloat32Slot(int(f), float32Val, 0) + case 10: + builder.PrependFloat64Slot(int(f), float64Val, 0) + } + } + + off := builder.EndObject() + + // store the offset from the end of the builder buffer, + // since it will keep growing: + objects[i] = off + } + + // Do some bookkeeping to generate stats on fuzzes: + stats := map[string]int{} + check := func(desc string, want, got interface{}) { + stats[desc]++ + if want != got { + fail("%s want %v got %v", desc, want, got) + } + } + + l = NewLCG() // Reset. + + // Test that all objects we generated are readable and return the + // expected values. We generate random objects in the same order + // so this is deterministic. + for i := 0; i < fuzzObjects; i++ { + + table := &flatbuffers.Table{ + Bytes: builder.Bytes, + Pos: flatbuffers.UOffsetT(len(builder.Bytes)) - objects[i], + } + + for j := 0; j < fuzzFields; j++ { + f := flatbuffers.VOffsetT((flatbuffers.VtableMetadataFields + j) * flatbuffers.SizeVOffsetT) + choice := int(l.Next()) % testValuesMax + + switch choice { + case 0: + check("bool", boolVal, table.GetBoolSlot(f, false)) + case 1: + check("int8", int8Val, table.GetInt8Slot(f, 0)) + case 2: + check("uint8", uint8Val, table.GetUint8Slot(f, 0)) + case 3: + check("int16", int16Val, table.GetInt16Slot(f, 0)) + case 4: + check("uint16", uint16Val, table.GetUint16Slot(f, 0)) + case 5: + check("int32", int32Val, table.GetInt32Slot(f, 0)) + case 6: + check("uint32", uint32Val, table.GetUint32Slot(f, 0)) + case 7: + check("int64", int64Val, table.GetInt64Slot(f, 0)) + case 8: + check("uint64", uint64Val, table.GetUint64Slot(f, 0)) + case 9: + check("float32", float32Val, table.GetFloat32Slot(f, 0)) + case 10: + check("float64", float64Val, table.GetFloat64Slot(f, 0)) + } + } + } + + // If enough checks were made, verify that all scalar types were used: + if fuzzFields*fuzzObjects >= testValuesMax { + if len(stats) != testValuesMax { + fail("fuzzing failed to test all scalar types") + } + } + + // Print some counts, if needed: + if testing.Verbose() { + if fuzzFields == 0 || fuzzObjects == 0 { + fmt.Printf("fuzz\tfields: %d\tobjects: %d\t[none]\t%d\n", + fuzzFields, fuzzObjects, 0) + } else { + keys := make([]string, 0, len(stats)) + for k := range stats { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + fmt.Printf("fuzz\tfields: %d\tobjects: %d\t%s\t%d\n", + fuzzFields, fuzzObjects, k, stats[k]) + } + } + } + + return +} + +// FailString makes a message for when expectations differ from reality. +func FailString(name string, want, got interface{}) string { + return fmt.Sprintf("bad %s: want %#v got %#v", name, want, got) +} + +// CheckByteLayout verifies the bytes of a Builder in various scenarios. +func CheckByteLayout(fail func(string, ...interface{})) { + var b *flatbuffers.Builder + + var i int + check := func(want []byte) { + i++ + got := b.Bytes[b.Head():] + if !bytes.Equal(want, got) { + fail("case %d: want\n%v\nbut got\n%v\n", i, want, got) + } + } + + // test 1: numbers + + b = flatbuffers.NewBuilder(0) + check([]byte{}) + b.PrependBool(true) + check([]byte{1}) + b.PrependInt8(-127) + check([]byte{129, 1}) + b.PrependUint8(255) + check([]byte{255, 129, 1}) + b.PrependInt16(-32222) + check([]byte{0x22, 0x82, 0, 255, 129, 1}) // first pad + b.PrependUint16(0xFEEE) + check([]byte{0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1}) // no pad this time + b.PrependInt32(-53687092) + check([]byte{204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1}) + b.PrependUint32(0x98765432) + check([]byte{0x32, 0x54, 0x76, 0x98, 204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1}) + + // test 1b: numbers 2 + + b = flatbuffers.NewBuilder(0) + b.PrependUint64(0x1122334455667788) + check([]byte{0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11}) + + // test 2: 1xbyte vector + + b = flatbuffers.NewBuilder(0) + check([]byte{}) + b.StartVector(flatbuffers.SizeByte, 1) + check([]byte{0, 0, 0}) // align to 4bytes + b.PrependByte(1) + check([]byte{1, 0, 0, 0}) + b.EndVector(1) + check([]byte{1, 0, 0, 0, 1, 0, 0, 0}) // padding + + // test 3: 2xbyte vector + + b = flatbuffers.NewBuilder(0) + b.StartVector(flatbuffers.SizeByte, 2) + check([]byte{0, 0}) // align to 4bytes + b.PrependByte(1) + check([]byte{1, 0, 0}) + b.PrependByte(2) + check([]byte{2, 1, 0, 0}) + b.EndVector(2) + check([]byte{2, 0, 0, 0, 2, 1, 0, 0}) // padding + + // test 4: 1xuint16 vector + + b = flatbuffers.NewBuilder(0) + b.StartVector(flatbuffers.SizeUint16, 1) + check([]byte{0, 0}) // align to 4bytes + b.PrependUint16(1) + check([]byte{1, 0, 0, 0}) + b.EndVector(1) + check([]byte{1, 0, 0, 0, 1, 0, 0, 0}) // padding + + // test 5: 2xuint16 vector + + b = flatbuffers.NewBuilder(0) + b.StartVector(flatbuffers.SizeUint16, 2) + check([]byte{}) // align to 4bytes + b.PrependUint16(0xABCD) + check([]byte{0xCD, 0xAB}) + b.PrependUint16(0xDCBA) + check([]byte{0xBA, 0xDC, 0xCD, 0xAB}) + b.EndVector(2) + check([]byte{2, 0, 0, 0, 0xBA, 0xDC, 0xCD, 0xAB}) + + // test 6: CreateString + + b = flatbuffers.NewBuilder(0) + b.CreateString("foo") + check([]byte{3, 0, 0, 0, 'f', 'o', 'o', 0}) // 0-terminated, no pad + b.CreateString("moop") + check([]byte{4, 0, 0, 0, 'm', 'o', 'o', 'p', 0, 0, 0, 0, // 0-terminated, 3-byte pad + 3, 0, 0, 0, 'f', 'o', 'o', 0}) + + // test 7: empty vtable + b = flatbuffers.NewBuilder(0) + b.StartObject(0) + check([]byte{}) + b.EndObject() + check([]byte{4, 0, 4, 0, 4, 0, 0, 0}) + + // test 8: vtable with one true bool + b = flatbuffers.NewBuilder(0) + check([]byte{}) + b.StartObject(1) + check([]byte{}) + b.PrependBoolSlot(0, true, false) + b.EndObject() + check([]byte{ + 6, 0, // vtable bytes + 8, 0, // length of object including vtable offset + 7, 0, // start of bool value + 6, 0, 0, 0, // offset for start of vtable (int32) + 0, 0, 0, // padded to 4 bytes + 1, // bool value + }) + + // test 9: vtable with one default bool + b = flatbuffers.NewBuilder(0) + check([]byte{}) + b.StartObject(1) + check([]byte{}) + b.PrependBoolSlot(0, false, false) + b.EndObject() + check([]byte{ + 6, 0, // vtable bytes + 4, 0, // end of object from here + 0, 0, // entry 1 is zero + 6, 0, 0, 0, // offset for start of vtable (int32) + }) + + // test 10: vtable with one int16 + b = flatbuffers.NewBuilder(0) + b.StartObject(1) + b.PrependInt16Slot(0, 0x789A, 0) + b.EndObject() + check([]byte{ + 6, 0, // vtable bytes + 8, 0, // end of object from here + 6, 0, // offset to value + 6, 0, 0, 0, // offset for start of vtable (int32) + 0, 0, // padding to 4 bytes + 0x9A, 0x78, + }) + + // test 11: vtable with two int16 + b = flatbuffers.NewBuilder(0) + b.StartObject(2) + b.PrependInt16Slot(0, 0x3456, 0) + b.PrependInt16Slot(1, 0x789A, 0) + b.EndObject() + check([]byte{ + 8, 0, // vtable bytes + 8, 0, // end of object from here + 6, 0, // offset to value 0 + 4, 0, // offset to value 1 + 8, 0, 0, 0, // offset for start of vtable (int32) + 0x9A, 0x78, // value 1 + 0x56, 0x34, // value 0 + }) + + // test 12: vtable with int16 and bool + b = flatbuffers.NewBuilder(0) + b.StartObject(2) + b.PrependInt16Slot(0, 0x3456, 0) + b.PrependBoolSlot(1, true, false) + b.EndObject() + check([]byte{ + 8, 0, // vtable bytes + 8, 0, // end of object from here + 6, 0, // offset to value 0 + 5, 0, // offset to value 1 + 8, 0, 0, 0, // offset for start of vtable (int32) + 0, // padding + 1, // value 1 + 0x56, 0x34, // value 0 + }) + + // test 12: vtable with empty vector + b = flatbuffers.NewBuilder(0) + b.StartVector(flatbuffers.SizeByte, 0) + vecend := b.EndVector(0) + b.StartObject(1) + b.PrependUOffsetTSlot(0, vecend, 0) + b.EndObject() + check([]byte{ + 6, 0, // vtable bytes + 8, 0, + 4, 0, // offset to vector offset + 6, 0, 0, 0, // offset for start of vtable (int32) + 4, 0, 0, 0, + 0, 0, 0, 0, // length of vector (not in struct) + }) + + // test 12b: vtable with empty vector of byte and some scalars + b = flatbuffers.NewBuilder(0) + b.StartVector(flatbuffers.SizeByte, 0) + vecend = b.EndVector(0) + b.StartObject(2) + b.PrependInt16Slot(0, 55, 0) + b.PrependUOffsetTSlot(1, vecend, 0) + b.EndObject() + check([]byte{ + 8, 0, // vtable bytes + 12, 0, + 10, 0, // offset to value 0 + 4, 0, // offset to vector offset + 8, 0, 0, 0, // vtable loc + 8, 0, 0, 0, // value 1 + 0, 0, 55, 0, // value 0 + + 0, 0, 0, 0, // length of vector (not in struct) + }) + + // test 13: vtable with 1 int16 and 2-vector of int16 + b = flatbuffers.NewBuilder(0) + b.StartVector(flatbuffers.SizeInt16, 2) + b.PrependInt16(0x1234) + b.PrependInt16(0x5678) + vecend = b.EndVector(2) + b.StartObject(2) + b.PrependUOffsetTSlot(1, vecend, 0) + b.PrependInt16Slot(0, 55, 0) + b.EndObject() + check([]byte{ + 8, 0, // vtable bytes + 12, 0, // length of object + 6, 0, // start of value 0 from end of vtable + 8, 0, // start of value 1 from end of buffer + 8, 0, 0, 0, // offset for start of vtable (int32) + 0, 0, // padding + 55, 0, // value 0 + 4, 0, 0, 0, // vector position from here + 2, 0, 0, 0, // length of vector (uint32) + 0x78, 0x56, // vector value 1 + 0x34, 0x12, // vector value 0 + }) + + // test 14: vtable with 1 struct of 1 int8, 1 int16, 1 int32 + b = flatbuffers.NewBuilder(0) + b.StartObject(1) + b.Prep(4+4+4, 0) + b.PrependInt8(55) + b.Pad(3) + b.PrependInt16(0x1234) + b.Pad(2) + b.PrependInt32(0x12345678) + structStart := b.Offset() + b.PrependStructSlot(0, structStart, 0) + b.EndObject() + check([]byte{ + 6, 0, // vtable bytes + 16, 0, // end of object from here + 4, 0, // start of struct from here + 6, 0, 0, 0, // offset for start of vtable (int32) + 0x78, 0x56, 0x34, 0x12, // value 2 + 0, 0, // padding + 0x34, 0x12, // value 1 + 0, 0, 0, // padding + 55, // value 0 + }) + + // test 15: vtable with 1 vector of 2 struct of 2 int8 + b = flatbuffers.NewBuilder(0) + b.StartVector(flatbuffers.SizeInt8*2, 2) + b.PrependInt8(33) + b.PrependInt8(44) + b.PrependInt8(55) + b.PrependInt8(66) + vecend = b.EndVector(2) + b.StartObject(1) + b.PrependUOffsetTSlot(0, vecend, 0) + b.EndObject() + check([]byte{ + 6, 0, // vtable bytes + 8, 0, + 4, 0, // offset of vector offset + 6, 0, 0, 0, // offset for start of vtable (int32) + 4, 0, 0, 0, // vector start offset + + 2, 0, 0, 0, // vector length + 66, // vector value 1,1 + 55, // vector value 1,0 + 44, // vector value 0,1 + 33, // vector value 0,0 + }) + + // test 16: table with some elements + b = flatbuffers.NewBuilder(0) + b.StartObject(2) + b.PrependInt8Slot(0, 33, 0) + b.PrependInt16Slot(1, 66, 0) + off := b.EndObject() + b.Finish(off) + + check([]byte{ + 12, 0, 0, 0, // root of table: points to vtable offset + + 8, 0, // vtable bytes + 8, 0, // end of object from here + 7, 0, // start of value 0 + 4, 0, // start of value 1 + + 8, 0, 0, 0, // offset for start of vtable (int32) + + 66, 0, // value 1 + 0, // padding + 33, // value 0 + }) + + // test 17: one unfinished table and one finished table + b = flatbuffers.NewBuilder(0) + b.StartObject(2) + b.PrependInt8Slot(0, 33, 0) + b.PrependInt8Slot(1, 44, 0) + off = b.EndObject() + b.Finish(off) + + b.StartObject(3) + b.PrependInt8Slot(0, 55, 0) + b.PrependInt8Slot(1, 66, 0) + b.PrependInt8Slot(2, 77, 0) + off = b.EndObject() + b.Finish(off) + + check([]byte{ + 16, 0, 0, 0, // root of table: points to object + 0, 0, // padding + + 10, 0, // vtable bytes + 8, 0, // size of object + 7, 0, // start of value 0 + 6, 0, // start of value 1 + 5, 0, // start of value 2 + 10, 0, 0, 0, // offset for start of vtable (int32) + 0, // padding + 77, // value 2 + 66, // value 1 + 55, // value 0 + + 12, 0, 0, 0, // root of table: points to object + + 8, 0, // vtable bytes + 8, 0, // size of object + 7, 0, // start of value 0 + 6, 0, // start of value 1 + 8, 0, 0, 0, // offset for start of vtable (int32) + 0, 0, // padding + 44, // value 1 + 33, // value 0 + }) + + // test 18: a bunch of bools + b = flatbuffers.NewBuilder(0) + b.StartObject(8) + b.PrependBoolSlot(0, true, false) + b.PrependBoolSlot(1, true, false) + b.PrependBoolSlot(2, true, false) + b.PrependBoolSlot(3, true, false) + b.PrependBoolSlot(4, true, false) + b.PrependBoolSlot(5, true, false) + b.PrependBoolSlot(6, true, false) + b.PrependBoolSlot(7, true, false) + off = b.EndObject() + b.Finish(off) + + check([]byte{ + 24, 0, 0, 0, // root of table: points to vtable offset + + 20, 0, // vtable bytes + 12, 0, // size of object + 11, 0, // start of value 0 + 10, 0, // start of value 1 + 9, 0, // start of value 2 + 8, 0, // start of value 3 + 7, 0, // start of value 4 + 6, 0, // start of value 5 + 5, 0, // start of value 6 + 4, 0, // start of value 7 + 20, 0, 0, 0, // vtable offset + + 1, // value 7 + 1, // value 6 + 1, // value 5 + 1, // value 4 + 1, // value 3 + 1, // value 2 + 1, // value 1 + 1, // value 0 + }) + + // test 19: three bools + b = flatbuffers.NewBuilder(0) + b.StartObject(3) + b.PrependBoolSlot(0, true, false) + b.PrependBoolSlot(1, true, false) + b.PrependBoolSlot(2, true, false) + off = b.EndObject() + b.Finish(off) + + check([]byte{ + 16, 0, 0, 0, // root of table: points to vtable offset + + 0, 0, // padding + + 10, 0, // vtable bytes + 8, 0, // size of object + 7, 0, // start of value 0 + 6, 0, // start of value 1 + 5, 0, // start of value 2 + 10, 0, 0, 0, // vtable offset from here + + 0, // padding + 1, // value 2 + 1, // value 1 + 1, // value 0 + }) + + // test 20: some floats + b = flatbuffers.NewBuilder(0) + b.StartObject(1) + b.PrependFloat32Slot(0, 1.0, 0.0) + off = b.EndObject() + + check([]byte{ + 6, 0, // vtable bytes + 8, 0, // size of object + 4, 0, // start of value 0 + 6, 0, 0, 0, // vtable offset + + 0, 0, 128, 63, // value 0 + }) +} + +// CheckManualBuild builds a Monster manually. +func CheckManualBuild(fail func(string, ...interface{})) ([]byte, flatbuffers.UOffsetT) { + b := flatbuffers.NewBuilder(0) + str := b.CreateString("MyMonster") + + b.StartVector(1, 5) + b.PrependByte(4) + b.PrependByte(3) + b.PrependByte(2) + b.PrependByte(1) + b.PrependByte(0) + inv := b.EndVector(5) + + b.StartObject(13) + b.PrependInt16Slot(2, 20, 100) + mon2 := b.EndObject() + + // Test4Vector + b.StartVector(4, 2) + + // Test 0 + b.Prep(2, 4) + b.Pad(1) + b.PlaceInt8(20) + b.PlaceInt16(10) + + // Test 1 + b.Prep(2, 4) + b.Pad(1) + b.PlaceInt8(40) + b.PlaceInt16(30) + + // end testvector + test4 := b.EndVector(2) + + b.StartObject(13) + + // a vec3 + b.Prep(16, 32) + b.Pad(2) + b.Prep(2, 4) + b.Pad(1) + b.PlaceByte(6) + b.PlaceInt16(5) + b.Pad(1) + b.PlaceByte(4) + b.PlaceFloat64(3.0) + b.Pad(4) + b.PlaceFloat32(3.0) + b.PlaceFloat32(2.0) + b.PlaceFloat32(1.0) + vec3Loc := b.Offset() + // end vec3 + + b.PrependStructSlot(0, vec3Loc, 0) // vec3. noop + b.PrependInt16Slot(2, 80, 100) // hp + b.PrependUOffsetTSlot(3, str, 0) + b.PrependUOffsetTSlot(5, inv, 0) // inventory + b.PrependByteSlot(7, 1, 0) + b.PrependUOffsetTSlot(8, mon2, 0) + b.PrependUOffsetTSlot(9, test4, 0) + mon := b.EndObject() + + b.Finish(mon) + + return b.Bytes, b.Head() +} + +// CheckGeneratedBuild uses generated code to build the example Monster. +func CheckGeneratedBuild(fail func(string, ...interface{})) ([]byte, flatbuffers.UOffsetT) { + b := flatbuffers.NewBuilder(0) + str := b.CreateString("MyMonster") + example.MonsterStartInventoryVector(b, 5) + b.PrependByte(4) + b.PrependByte(3) + b.PrependByte(2) + b.PrependByte(1) + b.PrependByte(0) + inv := b.EndVector(5) + + example.MonsterStart(b) + example.MonsterAddHp(b, int16(20)) + mon2 := example.MonsterEnd(b) + + example.MonsterStartTest4Vector(b, 2) + example.CreateTest(b, 10, 20) + example.CreateTest(b, 30, 40) + test4 := b.EndVector(2) + + example.MonsterStart(b) + + pos := example.CreateVec3(b, 1.0, 2.0, 3.0, 3.0, 4, 5, 6) + example.MonsterAddPos(b, pos) + + example.MonsterAddHp(b, 80) + example.MonsterAddName(b, str) + example.MonsterAddInventory(b, inv) + example.MonsterAddTestType(b, 1) + example.MonsterAddTest(b, mon2) + example.MonsterAddTest4(b, test4) + mon := example.MonsterEnd(b) + + b.Finish(mon) + + return b.Bytes, b.Head() +} + +// CheckVtableDeduplication verifies that vtables are deduplicated. +func CheckVtableDeduplication(fail func(string, ...interface{})) { + b := flatbuffers.NewBuilder(0) + + b.StartObject(4) + b.PrependByteSlot(0, 0, 0) + b.PrependByteSlot(1, 11, 0) + b.PrependByteSlot(2, 22, 0) + b.PrependInt16Slot(3, 33, 0) + obj0 := b.EndObject() + + b.StartObject(4) + b.PrependByteSlot(0, 0, 0) + b.PrependByteSlot(1, 44, 0) + b.PrependByteSlot(2, 55, 0) + b.PrependInt16Slot(3, 66, 0) + obj1 := b.EndObject() + + b.StartObject(4) + b.PrependByteSlot(0, 0, 0) + b.PrependByteSlot(1, 77, 0) + b.PrependByteSlot(2, 88, 0) + b.PrependInt16Slot(3, 99, 0) + obj2 := b.EndObject() + + got := b.Bytes[b.Head():] + + want := []byte{ + 240, 255, 255, 255, // == -12. offset to dedupped vtable. + 99, 0, + 88, + 77, + 248, 255, 255, 255, // == -8. offset to dedupped vtable. + 66, 0, + 55, + 44, + 12, 0, + 8, 0, + 0, 0, + 7, 0, + 6, 0, + 4, 0, + 12, 0, 0, 0, + 33, 0, + 22, + 11, + } + + if !bytes.Equal(want, got) { + fail("testVtableDeduplication want:\n%d %v\nbut got:\n%d %v\n", + len(want), want, len(got), got) + } + + table0 := &flatbuffers.Table{b.Bytes, flatbuffers.UOffsetT(len(b.Bytes)) - obj0} + table1 := &flatbuffers.Table{b.Bytes, flatbuffers.UOffsetT(len(b.Bytes)) - obj1} + table2 := &flatbuffers.Table{b.Bytes, flatbuffers.UOffsetT(len(b.Bytes)) - obj2} + + testTable := func(tab *flatbuffers.Table, a flatbuffers.VOffsetT, b, c, d byte) { + // vtable size + if got := tab.GetVOffsetTSlot(0, 0); 12 != got { + fail("failed 0, 0: %d", got) + } + // object size + if got := tab.GetVOffsetTSlot(2, 0); 8 != got { + fail("failed 2, 0: %d", got) + } + // default value + if got := tab.GetVOffsetTSlot(4, 0); a != got { + fail("failed 4, 0: %d", got) + } + if got := tab.GetByteSlot(6, 0); b != got { + fail("failed 6, 0: %d", got) + } + if val := tab.GetByteSlot(8, 0); c != val { + fail("failed 8, 0: %d", got) + } + if got := tab.GetByteSlot(10, 0); d != got { + fail("failed 10, 0: %d", got) + } + } + + testTable(table0, 0, 11, 22, 33) + testTable(table1, 0, 44, 55, 66) + testTable(table2, 0, 77, 88, 99) +} + +// CheckDocExample checks that the code given in FlatBuffers documentation +// is syntactically correct. +func CheckDocExample(buf []byte, off flatbuffers.UOffsetT, fail func(string, ...interface{})) { + monster := example.GetRootAsMonster(buf, off) + _ = monster.Hp() + _ = monster.Pos(nil) + for i := 0; i < monster.InventoryLength(); i++ { + _ = monster.Inventory(i) // do something here + } + + builder := flatbuffers.NewBuilder(0) + + example.MonsterStartInventoryVector(builder, 5) + for i := 4; i >= 0; i-- { + builder.PrependByte(byte(i)) + } + inv := builder.EndVector(5) + + str := builder.CreateString("MyMonster") + example.MonsterStart(builder) + example.MonsterAddPos(builder, example.CreateVec3(builder, 1.0, 2.0, 3.0, 3.0, 4, 5, 6)) + example.MonsterAddHp(builder, 80) + example.MonsterAddName(builder, str) + example.MonsterAddInventory(builder, inv) + example.MonsterAddTestType(builder, 1) + // example.MonsterAddTest(builder, mon2) + // example.MonsterAddTest4(builder, test4s) + _ = example.MonsterEnd(builder) +} + +// Include simple random number generator to ensure results will be the +// same cross platform. +// http://en.wikipedia.org/wiki/Park%E2%80%93Miller_random_number_generator +type LCG uint32 + +const InitialLCGSeed = 48271 + +func NewLCG() *LCG { + n := uint32(InitialLCGSeed) + l := LCG(n) + return &l +} + +func (lcg *LCG) Reset() { + *lcg = InitialLCGSeed +} + +func (lcg *LCG) Next() uint32 { + n := uint32((uint64(*lcg) * uint64(279470273)) % uint64(4294967291)) + *lcg = LCG(n) + return n +} + +// CheckByteEquality verifies that two byte buffers are the same. +func CheckByteEquality(a, b []byte, fail func(string, ...interface{})) { + if !bytes.Equal(a, b) { + fail("objects are not byte-wise equal") + } +} + +// BenchmarkVtableDeduplication measures the speed of vtable deduplication +// by creating prePop vtables, then populating b.N objects with a +// different single vtable. +// +// When b.N is large (as in long benchmarks), memory usage may be high. +func BenchmarkVtableDeduplication(b *testing.B) { + prePop := 10 + builder := flatbuffers.NewBuilder(0) + + // pre-populate some vtables: + for i := 0; i < prePop; i++ { + builder.StartObject(i) + for j := 0; j < i; j++ { + builder.PrependInt16Slot(j, int16(j), 0) + } + builder.EndObject() + } + + // benchmark deduplication of a new vtable: + b.ResetTimer() + for i := 0; i < b.N; i++ { + lim := prePop + + builder.StartObject(lim) + for j := 0; j < lim; j++ { + builder.PrependInt16Slot(j, int16(j), 0) + } + builder.EndObject() + } +} diff --git a/tests/include_test1.fbs b/tests/include_test1.fbs new file mode 100644 index 00000000000..11aebe81f37 --- /dev/null +++ b/tests/include_test1.fbs @@ -0,0 +1,5 @@ +include "include_test2.fbs"; +include "include_test2.fbs"; // should be skipped +include "include_test1.fbs"; // should be skipped + + diff --git a/tests/include_test2.fbs b/tests/include_test2.fbs new file mode 100644 index 00000000000..d22c0d9b4b9 --- /dev/null +++ b/tests/include_test2.fbs @@ -0,0 +1,9 @@ +include "include_test2.fbs"; // should be skipped + +namespace MyGame.OtherNameSpace; + +enum FromInclude:long { IncludeVal } + +struct Unused {} + + diff --git a/tests/monster_test.fbs b/tests/monster_test.fbs index cf191c4107d..9f3920d3ba4 100755 --- a/tests/monster_test.fbs +++ b/tests/monster_test.fbs @@ -1,8 +1,10 @@ // example IDL file +include "include_test1.fbs"; + namespace MyGame.Example; -enum Color:byte { Red = 0, Green, Blue = 2 } +enum Color:byte (bit_flags) { Red = 0, Green, Blue = 3, } union Any { Monster } // TODO: add more elements @@ -32,6 +34,11 @@ table Monster { enemy:Monster (id:12); test:Any (id: 8); test4:[Test] (id: 9); + testnestedflatbuffer:[ubyte] (id:13, nested_flatbuffer: "Monster"); + testempty:Monster (id:14); } root_type Monster; + +file_identifier "MONS"; +file_extension "mon"; diff --git a/tests/monster_test_generated.h b/tests/monster_test_generated.h index b8ad91c40b6..8c45a40b7ff 100755 --- a/tests/monster_test_generated.h +++ b/tests/monster_test_generated.h @@ -5,25 +5,35 @@ #include "flatbuffers/flatbuffers.h" +namespace MyGame { +namespace OtherNameSpace { +struct Unused; +} // namespace OtherNameSpace +} // namespace MyGame + namespace MyGame { namespace Example { +struct Test; +struct Vec3; +struct Monster; + enum { - Color_Red = 0, - Color_Green = 1, - Color_Blue = 2, + Color_Red = 1, + Color_Green = 2, + Color_Blue = 8 }; inline const char **EnumNamesColor() { - static const char *names[] = { "Red", "Green", "Blue", nullptr }; + static const char *names[] = { "Red", "Green", "", "", "", "", "", "Blue", nullptr }; return names; } -inline const char *EnumNameColor(int e) { return EnumNamesColor()[e]; } +inline const char *EnumNameColor(int e) { return EnumNamesColor()[e - Color_Red]; } enum { Any_NONE = 0, - Any_Monster = 1, + Any_Monster = 1 }; inline const char **EnumNamesAny() { @@ -33,11 +43,7 @@ inline const char **EnumNamesAny() { inline const char *EnumNameAny(int e) { return EnumNamesAny()[e]; } -bool VerifyAny(const flatbuffers::Verifier &verifier, const void *union_obj, uint8_t type); - -struct Test; -struct Vec3; -struct Monster; +bool VerifyAny(flatbuffers::Verifier &verifier, const void *union_obj, uint8_t type); MANUALLY_ALIGNED_STRUCT(2) Test { private: @@ -47,7 +53,7 @@ MANUALLY_ALIGNED_STRUCT(2) Test { public: Test(int16_t a, int8_t b) - : a_(flatbuffers::EndianScalar(a)), b_(flatbuffers::EndianScalar(b)), __padding0(0) {} + : a_(flatbuffers::EndianScalar(a)), b_(flatbuffers::EndianScalar(b)), __padding0(0) { (void)__padding0; } int16_t a() const { return flatbuffers::EndianScalar(a_); } int8_t b() const { return flatbuffers::EndianScalar(b_); } @@ -68,7 +74,7 @@ MANUALLY_ALIGNED_STRUCT(16) Vec3 { public: Vec3(float x, float y, float z, double test1, int8_t test2, const Test &test3) - : x_(flatbuffers::EndianScalar(x)), y_(flatbuffers::EndianScalar(y)), z_(flatbuffers::EndianScalar(z)), __padding0(0), test1_(flatbuffers::EndianScalar(test1)), test2_(flatbuffers::EndianScalar(test2)), __padding1(0), test3_(test3), __padding2(0) {} + : x_(flatbuffers::EndianScalar(x)), y_(flatbuffers::EndianScalar(y)), z_(flatbuffers::EndianScalar(z)), __padding0(0), test1_(flatbuffers::EndianScalar(test1)), test2_(flatbuffers::EndianScalar(test2)), __padding1(0), test3_(test3), __padding2(0) { (void)__padding0; (void)__padding1; (void)__padding2; } float x() const { return flatbuffers::EndianScalar(x_); } float y() const { return flatbuffers::EndianScalar(y_); } @@ -85,7 +91,7 @@ struct Monster : private flatbuffers::Table { int16_t hp() const { return GetField(8, 100); } const flatbuffers::String *name() const { return GetPointer(10); } const flatbuffers::Vector *inventory() const { return GetPointer *>(14); } - int8_t color() const { return GetField(16, 2); } + int8_t color() const { return GetField(16, 8); } uint8_t test_type() const { return GetField(18, 0); } const void *test() const { return GetPointer(20); } const flatbuffers::Vector *test4() const { return GetPointer *>(22); } @@ -93,8 +99,11 @@ struct Monster : private flatbuffers::Table { /// an example documentation comment: this will end up in the generated code multiline too const flatbuffers::Vector> *testarrayoftables() const { return GetPointer> *>(26); } const Monster *enemy() const { return GetPointer(28); } - bool Verify(const flatbuffers::Verifier &verifier) const { - return VerifyTable(verifier) && + const flatbuffers::Vector *testnestedflatbuffer() const { return GetPointer *>(30); } + const Monster *testnestedflatbuffer_nested_root() { return flatbuffers::GetRoot(testnestedflatbuffer()->Data()); } + const Monster *testempty() const { return GetPointer(32); } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && VerifyField(verifier, 4 /* pos */) && VerifyField(verifier, 6 /* mana */) && VerifyField(verifier, 8 /* hp */) && @@ -115,7 +124,12 @@ struct Monster : private flatbuffers::Table { verifier.Verify(testarrayoftables()) && verifier.VerifyVectorOfTables(testarrayoftables()) && VerifyField(verifier, 28 /* enemy */) && - verifier.VerifyTable(enemy()); + verifier.VerifyTable(enemy()) && + VerifyField(verifier, 30 /* testnestedflatbuffer */) && + verifier.Verify(testnestedflatbuffer()) && + VerifyField(verifier, 32 /* testempty */) && + verifier.VerifyTable(testempty()) && + verifier.EndTable(); } }; @@ -127,16 +141,18 @@ struct MonsterBuilder { void add_hp(int16_t hp) { fbb_.AddElement(8, hp, 100); } void add_name(flatbuffers::Offset name) { fbb_.AddOffset(10, name); } void add_inventory(flatbuffers::Offset> inventory) { fbb_.AddOffset(14, inventory); } - void add_color(int8_t color) { fbb_.AddElement(16, color, 2); } + void add_color(int8_t color) { fbb_.AddElement(16, color, 8); } void add_test_type(uint8_t test_type) { fbb_.AddElement(18, test_type, 0); } void add_test(flatbuffers::Offset test) { fbb_.AddOffset(20, test); } void add_test4(flatbuffers::Offset> test4) { fbb_.AddOffset(22, test4); } void add_testarrayofstring(flatbuffers::Offset>> testarrayofstring) { fbb_.AddOffset(24, testarrayofstring); } void add_testarrayoftables(flatbuffers::Offset>> testarrayoftables) { fbb_.AddOffset(26, testarrayoftables); } void add_enemy(flatbuffers::Offset enemy) { fbb_.AddOffset(28, enemy); } + void add_testnestedflatbuffer(flatbuffers::Offset> testnestedflatbuffer) { fbb_.AddOffset(30, testnestedflatbuffer); } + void add_testempty(flatbuffers::Offset testempty) { fbb_.AddOffset(32, testempty); } MonsterBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); } MonsterBuilder &operator=(const MonsterBuilder &); - flatbuffers::Offset Finish() { return flatbuffers::Offset(fbb_.EndTable(start_, 13)); } + flatbuffers::Offset Finish() { return flatbuffers::Offset(fbb_.EndTable(start_, 15)); } }; inline flatbuffers::Offset CreateMonster(flatbuffers::FlatBufferBuilder &_fbb, @@ -145,14 +161,18 @@ inline flatbuffers::Offset CreateMonster(flatbuffers::FlatBufferBuilder int16_t hp = 100, flatbuffers::Offset name = 0, flatbuffers::Offset> inventory = 0, - int8_t color = 2, + int8_t color = 8, uint8_t test_type = 0, flatbuffers::Offset test = 0, flatbuffers::Offset> test4 = 0, flatbuffers::Offset>> testarrayofstring = 0, flatbuffers::Offset>> testarrayoftables = 0, - flatbuffers::Offset enemy = 0) { + flatbuffers::Offset enemy = 0, + flatbuffers::Offset> testnestedflatbuffer = 0, + flatbuffers::Offset testempty = 0) { MonsterBuilder builder_(_fbb); + builder_.add_testempty(testempty); + builder_.add_testnestedflatbuffer(testnestedflatbuffer); builder_.add_enemy(enemy); builder_.add_testarrayoftables(testarrayoftables); builder_.add_testarrayofstring(testarrayofstring); @@ -168,7 +188,7 @@ inline flatbuffers::Offset CreateMonster(flatbuffers::FlatBufferBuilder return builder_.Finish(); } -bool VerifyAny(const flatbuffers::Verifier &verifier, const void *union_obj, uint8_t type) { +bool VerifyAny(flatbuffers::Verifier &verifier, const void *union_obj, uint8_t type) { switch (type) { case Any_NONE: return true; case Any_Monster: return verifier.VerifyTable(reinterpret_cast(union_obj)); @@ -178,9 +198,13 @@ bool VerifyAny(const flatbuffers::Verifier &verifier, const void *union_obj, uin inline const Monster *GetMonster(const void *buf) { return flatbuffers::GetRoot(buf); } -inline bool VerifyMonsterBuffer(const flatbuffers::Verifier &verifier) { return verifier.VerifyBuffer(); } +inline bool VerifyMonsterBuffer(flatbuffers::Verifier &verifier) { return verifier.VerifyBuffer(); } + +inline void FinishMonsterBuffer(flatbuffers::FlatBufferBuilder &fbb, flatbuffers::Offset root) { fbb.Finish(root, "MONS"); } + +inline bool MonsterBufferHasIdentifier(const void *buf) { return flatbuffers::BufferHasIdentifier(buf, "MONS"); } -}; // namespace MyGame -}; // namespace Example +} // namespace Example +} // namespace MyGame #endif // FLATBUFFERS_GENERATED_MONSTER_TEST_MYGAME_EXAMPLE_H_ diff --git a/tests/monsterdata_test.bin b/tests/monsterdata_test.bin index f09bf5081d6..952a96bef5f 100644 Binary files a/tests/monsterdata_test.bin and b/tests/monsterdata_test.bin differ diff --git a/tests/monsterdata_test.golden b/tests/monsterdata_test.golden index 26d53041f59..ca63f42851e 100644 --- a/tests/monsterdata_test.golden +++ b/tests/monsterdata_test.golden @@ -19,7 +19,7 @@ 3, 4 ], - test_type: 1, + test_type: Monster, test: { hp: 20 }, @@ -32,5 +32,11 @@ a: 30, b: 40 } - ] + ], + testarrayofstring: [ + "test1", + "test2" + ], + testempty: { + } } diff --git a/tests/monsterdata_test.json b/tests/monsterdata_test.json index 26d53041f59..c905fe4e518 100755 --- a/tests/monsterdata_test.json +++ b/tests/monsterdata_test.json @@ -19,7 +19,7 @@ 3, 4 ], - test_type: 1, + test_type: Monster, test: { hp: 20 }, diff --git a/tests/test.cpp b/tests/test.cpp index b859a5e880f..edb971aab9b 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -93,7 +93,7 @@ std::string CreateFlatBufferTest() { Any_Monster, mloc2.Union(), // Store a union. testv, vecofstrings, vecoftables, 0); - builder.Finish(mloc); + FinishMonsterBuffer(builder, mloc); #ifdef FLATBUFFERS_TEST_VERBOSE // print byte data for debugging: @@ -116,6 +116,8 @@ void AccessFlatBufferTest(const std::string &flatbuf) { flatbuf.length()); TEST_EQ(VerifyMonsterBuffer(verifier), true); + TEST_EQ(MonsterBufferHasIdentifier(flatbuf.c_str()), true); + // Access the buffer from the root. auto monster = GetMonster(flatbuf.c_str()); @@ -135,8 +137,8 @@ void AccessFlatBufferTest(const std::string &flatbuf) { auto inventory = monster->inventory(); TEST_NOTNULL(inventory); unsigned char inv_data[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; - for (flatbuffers::uoffset_t i = 0; i < inventory->Length(); i++) - TEST_EQ(inventory->Get(i), inv_data[i]); + for (auto it = inventory->begin(); it != inventory->end(); ++it) + TEST_EQ(*it, inv_data[it - inventory->begin()]); // Example of accessing a union: TEST_EQ(monster->test_type(), Any_Monster); // First make sure which it is. @@ -153,7 +155,8 @@ void AccessFlatBufferTest(const std::string &flatbuf) { // Example of accessing a vector of tables: auto vecoftables = monster->testarrayoftables(); TEST_EQ(vecoftables->Length(), 1U); - TEST_EQ(vecoftables->Get(0)->hp(), 20); + for (auto it = vecoftables->begin(); it != vecoftables->end(); ++it) + TEST_EQ(it->hp(), 20); // Since Flatbuffers uses explicit mechanisms to override the default // compiler alignment, double check that the compiler indeed obeys them: @@ -163,12 +166,16 @@ void AccessFlatBufferTest(const std::string &flatbuf) { auto tests = monster->test4(); TEST_NOTNULL(tests); - auto &test_0 = tests->Get(0); - auto &test_1 = tests->Get(1); - TEST_EQ(test_0.a(), 10); - TEST_EQ(test_0.b(), 20); - TEST_EQ(test_1.a(), 30); - TEST_EQ(test_1.b(), 40); + auto test_0 = tests->Get(0); + auto test_1 = tests->Get(1); + TEST_EQ(test_0->a(), 10); + TEST_EQ(test_0->b(), 20); + TEST_EQ(test_1->a(), 30); + TEST_EQ(test_1->b(), 40); + for (auto it = tests->begin(); it != tests->end(); ++it) { + TEST_EQ(it->a() == 10 || it->a() == 30, true); // Just testing iterators. + } + } // example of parsing text straight into a buffer, and generating @@ -185,8 +192,8 @@ void ParseAndGenerateTextTest() { // parse schema first, so we can use it to parse the data after flatbuffers::Parser parser; - TEST_EQ(parser.Parse(schemafile.c_str()), true); - TEST_EQ(parser.Parse(jsonfile.c_str()), true); + TEST_EQ(parser.Parse(schemafile.c_str(), "tests/"), true); + TEST_EQ(parser.Parse(jsonfile.c_str(), "tests/"), true); // here, parser.builder_ contains a binary buffer that is the parsed data. @@ -399,12 +406,12 @@ void FuzzTest2() { // Parse the schema, parse the generated data, then generate text back // from the binary and compare against the original. - TEST_EQ(parser.Parse(schema.c_str()), true); + TEST_EQ(parser.Parse(schema.c_str(), ""), true); const std::string &json = definitions[num_definitions - 1].instances[0] + "\n"; - TEST_EQ(parser.Parse(json.c_str()), true); + TEST_EQ(parser.Parse(json.c_str(), ""), true); std::string jsongen; flatbuffers::GeneratorOptions opts; @@ -436,7 +443,7 @@ void FuzzTest2() { // Test that parser errors are actually generated. void TestError(const char *src, const char *error_substr) { flatbuffers::Parser parser; - TEST_EQ(parser.Parse(src), false); // Must signal error + TEST_EQ(parser.Parse(src, ""), false); // Must signal error // Must be the error we're expecting TEST_NOTNULL(strstr(parser.error_.c_str(), error_substr)); } @@ -465,12 +472,14 @@ void ErrorTest() { TestError("table X { Y:int; } root_type X; { Z:", "unknown field"); TestError("struct X { Y:int; Z:int; } table W { V:X; } root_type W; " "{ V:{ Y:1 } }", "incomplete"); - TestError("table X { Y:byte; } root_type X; { Y:U }", "valid enum"); + TestError("enum E:byte { A } table X { Y:E; } root_type X; { Y:U }", + "unknown enum value"); TestError("table X { Y:byte; } root_type X; { Y:; }", "starting"); TestError("enum X:byte { Y } enum X {", "enum already"); TestError("enum X:float {}", "underlying"); TestError("enum X:byte { Y, Y }", "value already"); TestError("enum X:byte { Y=2, Z=1 }", "ascending"); + TestError("enum X:byte (bit_flags) { Y=8 }", "bit flag out"); TestError("table X { Y:int; } table X {", "datatype already"); TestError("struct X (force_align: 7) { Y:int; }", "force_align"); TestError("{}", "no root"); @@ -479,23 +488,49 @@ void ErrorTest() { TestError("struct X { Y:int; } root_type X;", "a table"); TestError("union X { Y }", "referenced"); TestError("union Z { X } struct X { Y:int; }", "only tables"); + TestError("table X { Y:[int]; YLength:int; }", "clash"); } // Additional parser testing not covered elsewhere. -void TokenTest() { +void ScientificTest() { flatbuffers::Parser parser; // Simple schema. - TEST_EQ(parser.Parse("table X { Y:float; } root_type X;"), true); + TEST_EQ(parser.Parse("table X { Y:float; } root_type X;", ""), true); // Test scientific notation numbers. - TEST_EQ(parser.Parse("{ Y:0.0314159e+2 }"), true); + TEST_EQ(parser.Parse("{ Y:0.0314159e+2 }", ""), true); auto root = flatbuffers::GetRoot(parser.builder_.GetBufferPointer()); // root will point to the table, which is a 32bit vtable offset followed // by a float: TEST_EQ(fabs(root[1] - 3.14159) < 0.001, true); } +void EnumStringsTest() { + flatbuffers::Parser parser1; + TEST_EQ(parser1.Parse("enum E:byte { A, B, C } table T { F:[E]; }" + "root_type T;" + "{ F:[ A, B, \"C\", \"A B C\" ] }", ""), true); + flatbuffers::Parser parser2; + TEST_EQ(parser2.Parse("enum E:byte { A, B, C } table T { F:[int]; }" + "root_type T;" + "{ F:[ \"E.C\", \"E.A E.B E.C\" ] }", ""), true); +} + +void UnicodeTest() { + flatbuffers::Parser parser; + TEST_EQ(parser.Parse("table T { F:string; }" + "root_type T;" + "{ F:\"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC" + "\\u5225\\u30B5\\u30A4\\u30C8\\x01\\x80\" }", ""), true); + std::string jsongen; + flatbuffers::GeneratorOptions opts; + opts.indent_step = -1; + GenerateText(parser, parser.builder_.GetBufferPointer(), opts, &jsongen); + TEST_EQ(jsongen == "{F: \"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC" + "\\u5225\\u30B5\\u30A4\\u30C8\\x01\\x80\"}", true); +} + int main(int /*argc*/, const char * /*argv*/[]) { // Run our various test suites: @@ -510,7 +545,9 @@ int main(int /*argc*/, const char * /*argv*/[]) { FuzzTest2(); ErrorTest(); - TokenTest(); + ScientificTest(); + EnumStringsTest(); + UnicodeTest(); if (!testing_fails) { TEST_OUTPUT_LINE("ALL TESTS PASSED");