From c9198dbbb8422306cb2d3b05b232ee114317e3f6 Mon Sep 17 00:00:00 2001 From: Shuhei Tanuma Date: Thu, 17 Dec 2015 11:35:31 +0900 Subject: [PATCH 1/5] (PHP) fixes getting indirect table, also fixes getInt method on 32bit machine. --- php/ByteBuffer.php | 9 +++++++-- src/idl_gen_php.cpp | 8 +++++++- tests/MyGame/Example/Monster.php | 4 ++-- tests/generate_code.bat | 1 + tests/generate_code.sh | 1 + tests/monsterdata_indirect.json | 6 ++++++ tests/monsterdata_indirect.mon | Bin 0 -> 156 bytes tests/phpTest.php | 13 ++++++++++++- 8 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 tests/monsterdata_indirect.json create mode 100644 tests/monsterdata_indirect.mon diff --git a/php/ByteBuffer.php b/php/ByteBuffer.php index 9ab9717aff0..9929a7df199 100644 --- a/php/ByteBuffer.php +++ b/php/ByteBuffer.php @@ -399,8 +399,13 @@ public function getInt($index) $sign = $index + (ByteBuffer::isLittleEndian() ? 3 : 0); $issigned = isset($this->_buffer[$sign]) && ord($this->_buffer[$sign]) & 0x80; - // 4294967296 = 1 << 32 = Maximum unsigned 32-bit int - return $issigned ? $result - 4294967296 : $result; + if (PHP_INT_SIZE > 4) { + // 4294967296 = 1 << 32 = Maximum unsigned 32-bit int + return $issigned ? $result - 4294967296 : $result; + } else { + // 32bit / Windows treated number as signed integer. + return $result; + } } /** diff --git a/src/idl_gen_php.cpp b/src/idl_gen_php.cpp index 2aa7e222e8a..2488b1d18fb 100644 --- a/src/idl_gen_php.cpp +++ b/src/idl_gen_php.cpp @@ -249,7 +249,13 @@ namespace php { NumToString(field.value.offset) + ");\n"; code += Indent + Indent; - code += "return $o != 0 ? $obj->init($o + $this->bb_pos, $this->bb) : "; + code += "return $o != 0 ? $obj->init("; + if (field.value.type.struct_def->fixed) + { + code += "$o + $this->bb_pos, $this->bb) : "; + } else { + code += "$this->__indirect($o + $this->bb_pos), $this->bb) : "; + } code += GenDefaultValue(field.value) + ";\n"; code += Indent + "}\n\n"; } diff --git a/tests/MyGame/Example/Monster.php b/tests/MyGame/Example/Monster.php index 94a0df7d989..6c1d34a0bdd 100644 --- a/tests/MyGame/Example/Monster.php +++ b/tests/MyGame/Example/Monster.php @@ -188,7 +188,7 @@ public function getEnemy() { $obj = new Monster(); $o = $this->__offset(28); - return $o != 0 ? $obj->init($o + $this->bb_pos, $this->bb) : 0; + return $o != 0 ? $obj->init($this->__indirect($o + $this->bb_pos), $this->bb) : 0; } /** @@ -214,7 +214,7 @@ public function getTestempty() { $obj = new Stat(); $o = $this->__offset(32); - return $o != 0 ? $obj->init($o + $this->bb_pos, $this->bb) : 0; + return $o != 0 ? $obj->init($this->__indirect($o + $this->bb_pos), $this->bb) : 0; } /** diff --git a/tests/generate_code.bat b/tests/generate_code.bat index 75ac1433e55..c46d10af62e 100644 --- a/tests/generate_code.bat +++ b/tests/generate_code.bat @@ -1,2 +1,3 @@ ..\flatc.exe -c -j -n -g -b -p --php -s --gen-mutable --no-includes monster_test.fbs monsterdata_test.json ..\flatc.exe -b --schema monster_test.fbs +..\flatc.exe -b .\monster_test.fbs .\monsterdata_indirect.json \ No newline at end of file diff --git a/tests/generate_code.sh b/tests/generate_code.sh index 91661f30de2..de736da5c9e 100644 --- a/tests/generate_code.sh +++ b/tests/generate_code.sh @@ -1,2 +1,3 @@ ../flatc --cpp --java --csharp --go --binary --python --js --php --gen-mutable --no-includes monster_test.fbs monsterdata_test.json ../flatc --binary --schema monster_test.fbs +../flatc --binary monster_test.fbs monsterdata_indirect.json \ No newline at end of file diff --git a/tests/monsterdata_indirect.json b/tests/monsterdata_indirect.json new file mode 100644 index 00000000000..297c1721fc5 --- /dev/null +++ b/tests/monsterdata_indirect.json @@ -0,0 +1,6 @@ +{ + "name": "Gob", + "enemy": { + "name": "Awk" + } +} \ No newline at end of file diff --git a/tests/monsterdata_indirect.mon b/tests/monsterdata_indirect.mon new file mode 100644 index 0000000000000000000000000000000000000000..f6262f32d7ef8ca17f248065de056846ffb8b24f GIT binary patch literal 156 zcmZ=@U|{g|_X}oVFk|2YvO$0aLLreHNNiN58IT(Q#5Q1IGX|JSOfyjRA@e|HfYdVs Ov155Qn0C)kVgLYoBLz(W literal 0 HcmV?d00001 diff --git a/tests/phpTest.php b/tests/phpTest.php index e91e47a1d7b..027780cc1a4 100644 --- a/tests/phpTest.php +++ b/tests/phpTest.php @@ -74,6 +74,9 @@ function main() fuzzTest1($assert); // testUnicode($assert); + + testIndirectBuffer($assert); + echo 'FlatBuffers php test: completed successfully' . PHP_EOL; } @@ -587,7 +590,15 @@ function testByteBuffer(Assert $assert) { $assert->Equal(0x0D0C0B0A, $uut->readLittleEndian(0, 4, true)); } - +function testIndirectBuffer(Assert $assert) +{ + $js = json_decode(file_get_contents('monsterdata_indirect.json'), true); + $data = file_get_contents('monsterdata_indirect.mon'); + $bb = Google\FlatBuffers\ByteBuffer::wrap($data); + $mons = \MyGame\Example\Monster::getRootAsMonster($bb); + $assert->Equal($js["name"], $mons->getName()); + $assert->Equal($js["enemy"]["name"], $mons->getEnemy()->getName()); +} class Assert { public function ok($result, $message = "") { if (!$result){ From 13d0594b4c8cd6677ddb43ecdbd2d7d3300b6208 Mon Sep 17 00:00:00 2001 From: Nalinichandra Penke Date: Tue, 22 Dec 2015 00:02:19 -0800 Subject: [PATCH 2/5] Fix #2775: Add parser option to skip unknown JSON fields --- include/flatbuffers/idl.h | 7 ++- src/idl_parser.cpp | 103 ++++++++++++++++++++++++++++++++------ tests/test.cpp | 21 ++++++++ 3 files changed, 116 insertions(+), 15 deletions(-) diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 118a74dd826..134c46f5a40 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -320,6 +320,7 @@ struct IDLOptions { bool one_file; bool proto_mode; bool generate_all; + bool skip_unexpected_fields_in_json; // Possible options for the more general generator below. enum Language { kJava, kCSharp, kGo, kMAX }; @@ -337,6 +338,7 @@ struct IDLOptions { one_file(false), proto_mode(false), generate_all(false), + skip_unexpected_fields_in_json(false), lang(IDLOptions::kJava) {} }; @@ -477,7 +479,10 @@ class Parser { CHECKED_ERROR ParseProtoDecl(); CHECKED_ERROR ParseProtoCurliesOrIdent(); CHECKED_ERROR ParseTypeFromProtoType(Type *type); - + CHECKED_ERROR SkipAnyJsonValue(); + CHECKED_ERROR SkipJsonObject(); + CHECKED_ERROR SkipJsonArray(); + CHECKED_ERROR SkipJsonString(); CHECKED_ERROR DoParse(const char *_source, const char **include_paths, const char *source_filename); diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index 6a013c90f27..d84ffddd60c 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -673,24 +673,33 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier); } auto field = struct_def.fields.Lookup(name); - if (!field) return Error("unknown field: " + name); - EXPECT(':'); - Value val = field->value; - ECHECK(ParseAnyValue(val, field, fieldn)); - size_t i = field_stack_.size(); - // Hardcoded insertion-sort with error-check. - // If fields are specified in order, then this loop exits immediately. - for (; i > field_stack_.size() - fieldn; i--) { - auto existing_field = field_stack_[i - 1].second; - if (existing_field == field) - return Error("field set more than once: " + field->name); - if (existing_field->value.offset < field->value.offset) break; + if (!field) { + if (!opts.skip_unexpected_fields_in_json) { + return Error("unknown field: " + name); + } else { + EXPECT(':'); + ECHECK(SkipAnyJsonValue()); + } + } else { + EXPECT(':'); + Value val = field->value; + ECHECK(ParseAnyValue(val, field, fieldn)); + size_t i = field_stack_.size(); + // Hardcoded insertion-sort with error-check. + // If fields are specified in order, then this loop exits immediately. + for (; i > field_stack_.size() - fieldn; i--) { + auto existing_field = field_stack_[i - 1].second; + if (existing_field == field) + return Error("field set more than once: " + field->name); + if (existing_field->value.offset < field->value.offset) break; + } + field_stack_.insert(field_stack_.begin() + i, std::make_pair(val, field)); + fieldn++; } - field_stack_.insert(field_stack_.begin() + i, std::make_pair(val, field)); - fieldn++; if (Is('}')) { NEXT(); break; } EXPECT(','); } + if (struct_def.fixed && fieldn != struct_def.fields.vec.size()) return Error("struct: wrong number of initializers: " + struct_def.name); @@ -1480,6 +1489,72 @@ CheckedError Parser::ParseTypeFromProtoType(Type *type) { return NoError(); } +CheckedError Parser::SkipAnyJsonValue() { + switch (token_) { + case '{': + ECHECK(SkipJsonObject()); + break; + case kTokenStringConstant: + ECHECK(SkipJsonString()); + break; + case '[': + ECHECK(SkipJsonArray()); + break; + case kTokenIntegerConstant: + EXPECT(kTokenIntegerConstant); + break; + case kTokenFloatConstant: + EXPECT(kTokenFloatConstant); + break; + default: + return Error(std::string("Unexpected token:") + std::string(1, token_)); + } + return NoError(); +} + +CheckedError Parser::SkipJsonObject() { + EXPECT('{'); + size_t fieldn = 0; + + while (true) { + if ((!opts.strict_json || !fieldn) && Is('}')) break; + + if (!Is(kTokenStringConstant)) + EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier); + + EXPECT(':'); + ECHECK(SkipAnyJsonValue()); + fieldn++; + + if (Is('}')) break; + EXPECT(','); + } + + NEXT(); + return NoError(); +} + +CheckedError Parser::SkipJsonArray() { + EXPECT('['); + + while (true) { + if (Is(']')) break; + + ECHECK(SkipAnyJsonValue()); + + if (Is(']')) break; + EXPECT(','); + } + + NEXT(); + return NoError(); +} + +CheckedError Parser::SkipJsonString() { + EXPECT(kTokenStringConstant); + return NoError(); +} + bool Parser::Parse(const char *source, const char **include_paths, const char *source_filename) { return !DoParse(source, include_paths, source_filename).Check(); diff --git a/tests/test.cpp b/tests/test.cpp index d009438ea78..83cdaef94a5 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -813,6 +813,26 @@ void UnicodeTest() { "\\u5225\\u30B5\\u30A4\\u30C8\\x01\\x80\"}", true); } +void UnknownFieldsTest() { + flatbuffers::IDLOptions opts; + opts.skip_unexpected_fields_in_json = true; + flatbuffers::Parser parser(opts); + + TEST_EQ(parser.Parse("table T { str:string; i:int;}" + "root_type T;" + "{ str:\"test\"," + "unknown_int:10," + "unknown_float:1.0," + "unknown_array: [ 1, 2, 3, 4]," + "unknown_object: { i: 10 }," + "i:10}"), true); + + std::string jsongen; + parser.opts.indent_step = -1; + GenerateText(parser, parser.builder_.GetBufferPointer(), &jsongen); + TEST_EQ(jsongen == "{str: \"test\",i: 10}", true); +} + int main(int /*argc*/, const char * /*argv*/[]) { // Run our various test suites: @@ -837,6 +857,7 @@ int main(int /*argc*/, const char * /*argv*/[]) { ScientificTest(); EnumStringsTest(); UnicodeTest(); + UnknownFieldsTest(); if (!testing_fails) { TEST_OUTPUT_LINE("ALL TESTS PASSED"); From f243109524eb7a62cd2556e2ffc7e70920427973 Mon Sep 17 00:00:00 2001 From: Nalinichandra Penke Date: Tue, 22 Dec 2015 00:13:22 -0800 Subject: [PATCH 3/5] Fix #2775: Add option to flatc to skip unknown fields in JSON --- src/flatc.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/flatc.cpp b/src/flatc.cpp index ead3a7c2973..48d98ce41ae 100644 --- a/src/flatc.cpp +++ b/src/flatc.cpp @@ -100,12 +100,15 @@ static void Error(const std::string &err, bool usage, bool show_exe_name) { " no trailing commas in tables/vectors.\n" " --defaults-json Output fields whose value is the default when\n" " writing JSON\n" + " --unknown-json Allow fields in JSON that are not defined in the\n" + " schema. These fields will be discared when generating\n" + " binaries.\n" " --no-prefix Don\'t prefix enum values with the enum type in C++.\n" " --scoped-enums Use C++11 style scoped and strongly typed enums.\n" " also implies --no-prefix.\n" " --gen-includes (deprecated), this is the default behavior.\n" " If the original behavior is required (no include\n" - " statements) use --no-includes.\n" + " statements) use --no-includes.\n" " --no-includes Don\'t generate include statements for included\n" " schemas the generated file depends on (C++).\n" " --gen-mutable Generate accessors that can mutate buffers in-place.\n" @@ -155,6 +158,8 @@ int main(int argc, const char *argv[]) { opts.skip_js_exports = true; } else if(arg == "--defaults-json") { opts.output_default_scalars_in_json = true; + } else if (arg == "--unknown-json") { + opts.skip_unexpected_fields_in_json = true; } else if(arg == "--no-prefix") { opts.prefixed_enums = false; } else if(arg == "--scoped-enums") { From 7fbd9b8de4d59ba63ae4c7651a1f3d0e308532ed Mon Sep 17 00:00:00 2001 From: Shuhei Tanuma Date: Thu, 24 Dec 2015 15:35:28 +0900 Subject: [PATCH 4/5] (PHP) improve indirect buffer test --- tests/generate_code.bat | 1 - tests/generate_code.sh | 1 - tests/monsterdata_indirect.json | 6 ------ tests/monsterdata_indirect.mon | Bin 156 -> 0 bytes tests/monsterdata_test.json | 3 +++ tests/monsterdata_test.mon | Bin 352 -> 384 bytes tests/phpTest.php | 26 ++++++++++++++++++++------ 7 files changed, 23 insertions(+), 14 deletions(-) delete mode 100644 tests/monsterdata_indirect.json delete mode 100644 tests/monsterdata_indirect.mon diff --git a/tests/generate_code.bat b/tests/generate_code.bat index c46d10af62e..75ac1433e55 100644 --- a/tests/generate_code.bat +++ b/tests/generate_code.bat @@ -1,3 +1,2 @@ ..\flatc.exe -c -j -n -g -b -p --php -s --gen-mutable --no-includes monster_test.fbs monsterdata_test.json ..\flatc.exe -b --schema monster_test.fbs -..\flatc.exe -b .\monster_test.fbs .\monsterdata_indirect.json \ No newline at end of file diff --git a/tests/generate_code.sh b/tests/generate_code.sh index de736da5c9e..91661f30de2 100644 --- a/tests/generate_code.sh +++ b/tests/generate_code.sh @@ -1,3 +1,2 @@ ../flatc --cpp --java --csharp --go --binary --python --js --php --gen-mutable --no-includes monster_test.fbs monsterdata_test.json ../flatc --binary --schema monster_test.fbs -../flatc --binary monster_test.fbs monsterdata_indirect.json \ No newline at end of file diff --git a/tests/monsterdata_indirect.json b/tests/monsterdata_indirect.json deleted file mode 100644 index 297c1721fc5..00000000000 --- a/tests/monsterdata_indirect.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "Gob", - "enemy": { - "name": "Awk" - } -} \ No newline at end of file diff --git a/tests/monsterdata_indirect.mon b/tests/monsterdata_indirect.mon deleted file mode 100644 index f6262f32d7ef8ca17f248065de056846ffb8b24f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 156 zcmZ=@U|{g|_X}oVFk|2YvO$0aLLreHNNiN58IT(Q#5Q1IGX|JSOfyjRA@e|HfYdVs Ov155Qn0C)kVgLYoBLz(W diff --git a/tests/monsterdata_test.json b/tests/monsterdata_test.json index 7ed39a8de15..89278755edd 100755 --- a/tests/monsterdata_test.json +++ b/tests/monsterdata_test.json @@ -37,6 +37,9 @@ "test1", "test2" ], + enemy: { + name: "Fred" + }, testarrayofbools:[ true, false, true ], diff --git a/tests/monsterdata_test.mon b/tests/monsterdata_test.mon index eff1e66ca2a5b60af52b6deeffcf0eaba91dd00d..69d1d7538e406a7d46bc5d85bbb565ea95e960ca 100644 GIT binary patch delta 145 zcmaFB)WAGJ#JGilgMop8jll#+S^zOCgAIcNgA0)50mLBS!w|rbz>vWZ!Vtkwz)-;u zGtu2p7bNioh@SxQ86a)};us(XX>>eUc^8BWmo)Z)u*Sr>%JM*Q24)~;1j1ea|NmzJ QvO(CbC^ZF0O*}pm06JwJZ~y=R delta 98 zcmZo*e!w(A#JGZigMop8jllp&ngB5?g9U>Pg9DHS11=063=s?o3_c723>gds3?UQU l4FzuiWlsR{1|ZG=Vuy(ng&9>QE>&h{2J#pg7$-iT2>_pL56%Dp diff --git a/tests/phpTest.php b/tests/phpTest.php index 027780cc1a4..ab41121ced1 100644 --- a/tests/phpTest.php +++ b/tests/phpTest.php @@ -30,6 +30,10 @@ function main() // We set up the same values as monsterdata.json: $str = $fbb->createString("MyMonster"); + $name = $fbb->createString('Fred'); + \MyGame\Example\Monster::startMonster($fbb); + \MyGame\Example\Monster::addName($fbb, $name); + $enemy = \MyGame\Example\Monster::endMonster($fbb); $inv = \MyGame\Example\Monster::CreateInventoryVector($fbb, array(0, 1, 2, 3, 4)); @@ -62,6 +66,7 @@ function main() \MyGame\Example\Monster::AddTest($fbb, $mon2); \MyGame\Example\Monster::AddTest4($fbb, $test4); \MyGame\Example\Monster::AddTestarrayofstring($fbb, $testArrayOfString); + \MyGame\Example\Monster::AddEnemy($fbb, $enemy); \MyGame\Example\Monster::AddTestbool($fbb, false); $mon = \MyGame\Example\Monster::EndMonster($fbb); @@ -75,7 +80,7 @@ function main() // testUnicode($assert); - testIndirectBuffer($assert); + testAnyBuffer($assert); echo 'FlatBuffers php test: completed successfully' . PHP_EOL; } @@ -135,6 +140,10 @@ function test_buffer(Assert $assert, Google\FlatBuffers\ByteBuffer $bb) { $assert->strictEqual($monster->GetTestarrayofstringLength(), 2); $assert->strictEqual($monster->GetTestarrayofstring(0), 'test1'); $assert->strictEqual($monster->GetTestarrayofstring(1), 'test2'); + + $fred = $monster->getEnemy(); + $assert->Equal('Fred', $fred->getName()); + $assert->strictEqual($monster->GetTestbool(), false); } @@ -590,15 +599,20 @@ function testByteBuffer(Assert $assert) { $assert->Equal(0x0D0C0B0A, $uut->readLittleEndian(0, 4, true)); } -function testIndirectBuffer(Assert $assert) + +function testAnyBuffer(Assert $assert) { - $js = json_decode(file_get_contents('monsterdata_indirect.json'), true); - $data = file_get_contents('monsterdata_indirect.mon'); +// PHP needs double quote. for now, use Fred directly +// $js = json_decode(file_get_contents('monsterdata_test.json'), true); + $data = file_get_contents('monsterdata_test.mon'); $bb = Google\FlatBuffers\ByteBuffer::wrap($data); $mons = \MyGame\Example\Monster::getRootAsMonster($bb); - $assert->Equal($js["name"], $mons->getName()); - $assert->Equal($js["enemy"]["name"], $mons->getEnemy()->getName()); + $indirect_monster = new \MyGame\Example\Monster(); + $assert->Equal($mons->getTestType(), \MyGame\Example\Any::Monster); + $mons->getTest($indirect_monster); + $assert->Equal("Fred", $indirect_monster->getName()); } + class Assert { public function ok($result, $message = "") { if (!$result){ From 4691558ee3b9c245899deca4d5d243b4a3c9855f Mon Sep 17 00:00:00 2001 From: Shuhei Tanuma Date: Thu, 24 Dec 2015 15:50:03 +0900 Subject: [PATCH 5/5] (PHP) remove duplicate test --- tests/phpTest.php | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tests/phpTest.php b/tests/phpTest.php index ab41121ced1..0afc0af7fe0 100644 --- a/tests/phpTest.php +++ b/tests/phpTest.php @@ -79,9 +79,6 @@ function main() fuzzTest1($assert); // testUnicode($assert); - - testAnyBuffer($assert); - echo 'FlatBuffers php test: completed successfully' . PHP_EOL; } @@ -600,19 +597,6 @@ function testByteBuffer(Assert $assert) { } -function testAnyBuffer(Assert $assert) -{ -// PHP needs double quote. for now, use Fred directly -// $js = json_decode(file_get_contents('monsterdata_test.json'), true); - $data = file_get_contents('monsterdata_test.mon'); - $bb = Google\FlatBuffers\ByteBuffer::wrap($data); - $mons = \MyGame\Example\Monster::getRootAsMonster($bb); - $indirect_monster = new \MyGame\Example\Monster(); - $assert->Equal($mons->getTestType(), \MyGame\Example\Any::Monster); - $mons->getTest($indirect_monster); - $assert->Equal("Fred", $indirect_monster->getName()); -} - class Assert { public function ok($result, $message = "") { if (!$result){