From ef88e389d7f2d14e61b6b0f0c827d21521ee12b9 Mon Sep 17 00:00:00 2001 From: Okapist Date: Fri, 5 Aug 2022 21:15:20 +0300 Subject: [PATCH] Splitter for big java methods Signed-off-by: Okapist --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- .gitignore | 18 + BUILD.bazel | 2 +- CHANGES.txt | 73 +- CMakeLists.txt | 79 +- Makefile.am | 9 +- Protobuf-C++.podspec | 2 +- Protobuf.podspec | 2 +- README.md | 7 + .../protobuf/ProtoCaliperBenchmark.java | 5 +- cmake/README.md | 69 +- cmake/conformance.cmake | 15 +- cmake/examples.cmake | 4 +- cmake/extract_includes.bat.in | 144 -- cmake/install.cmake | 30 +- cmake/tests.cmake | 51 +- configure.ac | 2 +- conformance/binary_json_conformance_suite.cc | 6 + conformance/failure_list_php.txt | 2 + conformance/failure_list_php_c.txt | 3 + conformance/failure_list_ruby.txt | 3 + csharp/Google.Protobuf.Tools.nuspec | 2 +- csharp/README.md | 46 +- csharp/generate_protos.sh | 3 +- csharp/protos/unittest_issues.proto | 18 +- .../MapUnittestProto3.cs | 74 +- .../TestMessagesProto2.cs | 38 +- .../TestMessagesProto3.cs | 38 +- .../Unittest.cs | 4 +- .../UnittestIssues.cs | 618 ++++++- .../UnittestWellKnownTypes.cs | 36 +- .../GeneratedMessageTest.cs | 40 + .../JsonFormatterSettingsTest.cs | 111 ++ .../Google.Protobuf.Test/JsonFormatterTest.cs | 208 ++- .../ParsingPrimitivesTest.cs | 63 + .../WritingPrimitivesTest.cs | 61 + csharp/src/Google.Protobuf.Test/testprotos.pb | Bin 345377 -> 346547 bytes .../Google.Protobuf/Collections/MapField.cs | 15 + csharp/src/Google.Protobuf/Compiler/Plugin.cs | 1490 +++++++++++++++++ .../Google.Protobuf/Google.Protobuf.csproj | 2 +- csharp/src/Google.Protobuf/JsonFormatter.cs | 237 ++- csharp/src/Google.Protobuf/MessageParser.cs | 7 + .../Google.Protobuf/WellKnownTypes/Struct.cs | 2 +- docs/third_party.md | 1 + java/README.md | 6 +- java/bom/pom.xml | 2 +- java/core/pom.xml | 2 +- .../com/google/protobuf/BinaryWriter.java | 40 +- .../java/com/google/protobuf/Descriptors.java | 21 + .../google/protobuf/GeneratedMessageLite.java | 2 +- .../com/google/protobuf/DescriptorsTest.java | 15 + .../protobuf/LiteEqualsAndHashTest.java | 5 +- java/kotlin-lite/pom.xml | 2 +- java/kotlin/pom.xml | 2 +- .../kotlin/com/google/protobuf/Proto3Test.kt | 5 + java/lite.md | 2 +- java/lite/pom.xml | 2 +- java/pom.xml | 2 +- java/protoc/pom.xml | 4 +- java/util/pom.xml | 2 +- kokoro/linux/bazel_distcheck/build.sh | 61 - kokoro/linux/bazel_distcheck/common.cfg | 9 - kokoro/linux/bazel_distcheck/continuous.cfg | 5 - kokoro/linux/bazel_distcheck/presubmit.cfg | 5 - kokoro/linux/cmake/build.sh | 26 + kokoro/linux/cmake/continuous.cfg | 11 + kokoro/linux/cmake/presubmit.cfg | 11 + kokoro/linux/cmake_distcheck/build.sh | 60 - kokoro/linux/cmake_distcheck/presubmit.cfg | 5 - kokoro/linux/cmake_install/build.sh | 30 + kokoro/linux/cmake_install/continuous.cfg | 11 + kokoro/linux/cmake_install/presubmit.cfg | 11 + kokoro/linux/cmake_ninja/build.sh | 26 + kokoro/linux/cmake_ninja/continuous.cfg | 11 + kokoro/linux/cmake_ninja/presubmit.cfg | 11 + kokoro/linux/cmake_shared/build.sh | 26 + kokoro/linux/cmake_shared/continuous.cfg | 11 + kokoro/linux/cmake_shared/presubmit.cfg | 11 + kokoro/linux/cpp_distcheck/build.sh | 25 - kokoro/linux/dist_install/build.sh | 15 - kokoro/linux/dist_install/continuous.cfg | 5 - kokoro/macos/cpp_distcheck/build.sh | 11 - kokoro/macos/cpp_distcheck/continuous.cfg | 5 - kokoro/macos/cpp_distcheck/presubmit.cfg | 5 - kokoro/macos/php7.0_mac/build.sh | 12 - kokoro/macos/php7.3_mac/build.sh | 12 - kokoro/macos/php74/build.sh | 20 + .../{php7.0_mac => php74}/continuous.cfg | 2 +- .../macos/{php7.3_mac => php74}/presubmit.cfg | 2 +- kokoro/macos/php80/build.sh | 20 + .../{php7.3_mac => php80}/continuous.cfg | 2 +- .../macos/{php7.0_mac => php80}/presubmit.cfg | 2 +- kokoro/macos/prepare_build_macos_rc | 36 +- kokoro/release/ruby/macos/build_artifacts.sh | 3 - kokoro/release/ruby/macos/ruby/ruby_build.sh | 1 - .../ruby/macos/ruby/ruby_build_environment.sh | 10 +- kokoro/windows/bazel/build.bat | 4 + kokoro/windows/bazel/continuous.cfg | 5 + kokoro/windows/bazel/presubmit.cfg | 5 + kokoro/windows/cmake/build.bat | 4 + kokoro/windows/cmake/continuous.cfg | 5 + kokoro/windows/cmake/presubmit.cfg | 5 + kokoro/windows/cmake_install/build.bat | 4 + .../cmake_install}/continuous.cfg | 2 +- .../cmake_install}/presubmit.cfg | 2 +- kokoro/windows/cmake_nmake/build.bat | 4 + .../cmake_nmake}/continuous.cfg | 2 +- .../cmake_nmake}/presubmit.cfg | 2 +- kokoro/windows/cmake_shared/build.bat | 4 + kokoro/windows/cmake_shared/continuous.cfg | 5 + kokoro/windows/cmake_shared/presubmit.cfg | 5 + objectivec/GPBApi.pbobjc.h | 8 +- objectivec/GPBFieldMask.pbobjc.h | 2 +- objectivec/GPBStruct.pbobjc.h | 4 +- objectivec/GPBType.pbobjc.h | 14 +- objectivec/GPBUtilities.m | 2 +- objectivec/generate_well_known_types.sh | 1 + php/BUILD.bazel | 2 + php/composer.json | 5 +- php/ext/google/protobuf/def.c | 65 +- php/ext/google/protobuf/names.c | 90 +- php/ext/google/protobuf/names.h | 3 +- php/ext/google/protobuf/package.xml | 44 +- php/ext/google/protobuf/protobuf.c | 14 +- php/ext/google/protobuf/protobuf.h | 2 +- .../Google/Protobuf/Internal/Descriptor.php | 16 +- .../Protobuf/Internal/DescriptorPool.php | 1 + .../Protobuf/Internal/EnumDescriptor.php | 3 +- php/src/Google/Protobuf/Internal/GPBUtil.php | 58 +- php/tests/DescriptorsTest.php | 20 +- php/tests/GeneratedClassTest.php | 19 + php/tests/PreviouslyGeneratedClassTest.php | 18 + .../TestPreviouslyUnreservedMessage.php | 28 + .../generated_previous/Previous/readonly.php | 31 + php/tests/proto/test_descriptors.proto | 1 + php/tests/proto/test_php_namespace.proto | 2 + .../proto/test_reserved_enum_lower.proto | 1 + .../proto/test_reserved_enum_upper.proto | 1 + .../test_reserved_enum_value_lower.proto | 1 + .../test_reserved_enum_value_upper.proto | 1 + .../proto/test_reserved_message_lower.proto | 1 + .../proto/test_reserved_message_upper.proto | 1 + .../test_previously_unreserved_message.proto | 5 + protobuf_deps.bzl | 4 +- protobuf_version.bzl | 6 +- protoc-artifacts/pom.xml | 4 +- python/google/protobuf/__init__.py | 2 +- python/google/protobuf/descriptor.py | 8 +- .../protobuf/internal/descriptor_test.py | 16 + .../google/protobuf/internal/message_test.py | 33 + python/google/protobuf/message.py | 3 +- python/release.sh | 22 +- ruby/README.md | 2 +- ruby/google-protobuf.gemspec | 2 +- ruby/pom.xml | 4 +- src/Makefile.am | 3 +- src/file_lists.cmake | 1 + src/google/protobuf/any.pb.h | 2 +- src/google/protobuf/api.pb.h | 2 +- src/google/protobuf/arena.cc | 2 +- src/google/protobuf/arena.h | 34 +- src/google/protobuf/arena_unittest.cc | 56 +- src/google/protobuf/arenastring.cc | 7 +- src/google/protobuf/arenaz_sampler.cc | 93 +- src/google/protobuf/arenaz_sampler.h | 51 +- src/google/protobuf/arenaz_sampler_test.cc | 246 ++- .../compiler/command_line_interface.cc | 17 +- src/google/protobuf/compiler/cpp/field.cc | 1 - src/google/protobuf/compiler/cpp/field.h | 1 - src/google/protobuf/compiler/cpp/file.cc | 8 +- src/google/protobuf/compiler/cpp/helpers.cc | 11 +- src/google/protobuf/compiler/cpp/helpers.h | 5 + src/google/protobuf/compiler/cpp/message.cc | 105 +- src/google/protobuf/compiler/cpp/options.h | 2 +- .../compiler/cpp/parse_function_generator.cc | 46 +- .../csharp/csharp_generator_unittest.cc | 16 + .../compiler/csharp/csharp_helpers.cc | 43 +- .../compiler/csharp/csharp_map_field.cc | 2 +- .../protobuf/compiler/java/enum_field.cc | 12 + .../protobuf/compiler/java/enum_field_lite.cc | 12 + src/google/protobuf/compiler/java/file.cc | 5 +- src/google/protobuf/compiler/java/generator.h | 5 + src/google/protobuf/compiler/java/helpers.cc | 18 + src/google/protobuf/compiler/java/helpers.h | 7 + src/google/protobuf/compiler/java/message.cc | 329 +++- src/google/protobuf/compiler/java/message.h | 2 + .../protobuf/compiler/java/message_builder.cc | 125 +- .../compiler/java/message_serialization.h | 61 +- .../java/message_serialization_unittest.cc | 124 ++ .../java/message_serialization_unittest.proto | 56 + src/google/protobuf/compiler/main.cc | 9 + .../compiler/objectivec/objectivec_field.cc | 2 +- .../protobuf/compiler/php/php_generator.cc | 168 +- src/google/protobuf/compiler/plugin.pb.h | 2 +- src/google/protobuf/compiler/plugin.proto | 3 - .../protobuf/compiler/python/generator.cc | 146 +- .../protobuf/compiler/python/generator.h | 7 +- .../protobuf/compiler/python/pyi_generator.cc | 214 +-- .../protobuf/compiler/python/pyi_generator.h | 32 +- src/google/protobuf/compiler/subprocess.cc | 2 +- src/google/protobuf/descriptor.h | 6 +- src/google/protobuf/descriptor.pb.h | 2 +- src/google/protobuf/duration.pb.h | 2 +- src/google/protobuf/empty.pb.h | 2 +- src/google/protobuf/field_mask.pb.h | 2 +- .../protobuf/generated_message_reflection.cc | 1 + .../protobuf/generated_message_tctable_impl.h | 162 +- .../generated_message_tctable_lite.cc | 73 +- src/google/protobuf/io/coded_stream.h | 4 +- src/google/protobuf/io/printer.h | 16 +- src/google/protobuf/io/tokenizer.cc | 2 +- src/google/protobuf/io/tokenizer_unittest.cc | 3 +- src/google/protobuf/map.h | 60 +- src/google/protobuf/map_test.cc | 2 + src/google/protobuf/map_test.inc | 94 ++ src/google/protobuf/message.cc | 6 +- src/google/protobuf/message.h | 9 +- src/google/protobuf/message_lite.cc | 8 +- src/google/protobuf/message_unittest.inc | 98 +- src/google/protobuf/metadata_lite.h | 13 +- src/google/protobuf/parse_context.h | 1 + src/google/protobuf/port_def.inc | 103 +- src/google/protobuf/port_undef.inc | 5 + src/google/protobuf/repeated_field.h | 71 +- .../protobuf/repeated_field_unittest.cc | 54 + src/google/protobuf/repeated_ptr_field.h | 2 - src/google/protobuf/source_context.pb.h | 2 +- src/google/protobuf/struct.pb.h | 2 +- src/google/protobuf/stubs/common.h | 2 +- src/google/protobuf/timestamp.pb.h | 2 +- src/google/protobuf/type.pb.h | 2 +- src/google/protobuf/unittest.proto | 33 + .../protobuf/util/json_format_proto3.proto | 8 + src/google/protobuf/util/json_util.cc | 6 + src/google/protobuf/util/json_util.h | 13 + src/google/protobuf/util/json_util_test.cc | 410 ++++- .../protobuf/util/message_differencer.h | 6 +- src/google/protobuf/wire_format_lite.h | 1 - src/google/protobuf/wrappers.pb.h | 2 +- tests.sh | 80 +- version.json | 20 +- 242 files changed, 6642 insertions(+), 1758 deletions(-) delete mode 100644 cmake/extract_includes.bat.in create mode 100644 csharp/src/Google.Protobuf.Test/JsonFormatterSettingsTest.cs create mode 100644 csharp/src/Google.Protobuf.Test/ParsingPrimitivesTest.cs create mode 100644 csharp/src/Google.Protobuf.Test/WritingPrimitivesTest.cs create mode 100644 csharp/src/Google.Protobuf/Compiler/Plugin.cs delete mode 100755 kokoro/linux/bazel_distcheck/build.sh delete mode 100644 kokoro/linux/bazel_distcheck/common.cfg delete mode 100644 kokoro/linux/bazel_distcheck/continuous.cfg delete mode 100644 kokoro/linux/bazel_distcheck/presubmit.cfg create mode 100755 kokoro/linux/cmake/build.sh create mode 100644 kokoro/linux/cmake/continuous.cfg create mode 100644 kokoro/linux/cmake/presubmit.cfg delete mode 100755 kokoro/linux/cmake_distcheck/build.sh delete mode 100644 kokoro/linux/cmake_distcheck/presubmit.cfg create mode 100755 kokoro/linux/cmake_install/build.sh create mode 100644 kokoro/linux/cmake_install/continuous.cfg create mode 100644 kokoro/linux/cmake_install/presubmit.cfg create mode 100755 kokoro/linux/cmake_ninja/build.sh create mode 100644 kokoro/linux/cmake_ninja/continuous.cfg create mode 100644 kokoro/linux/cmake_ninja/presubmit.cfg create mode 100755 kokoro/linux/cmake_shared/build.sh create mode 100644 kokoro/linux/cmake_shared/continuous.cfg create mode 100644 kokoro/linux/cmake_shared/presubmit.cfg delete mode 100755 kokoro/linux/cpp_distcheck/build.sh delete mode 100755 kokoro/linux/dist_install/build.sh delete mode 100644 kokoro/linux/dist_install/continuous.cfg delete mode 100755 kokoro/macos/cpp_distcheck/build.sh delete mode 100644 kokoro/macos/cpp_distcheck/continuous.cfg delete mode 100644 kokoro/macos/cpp_distcheck/presubmit.cfg delete mode 100755 kokoro/macos/php7.0_mac/build.sh delete mode 100755 kokoro/macos/php7.3_mac/build.sh create mode 100755 kokoro/macos/php74/build.sh rename kokoro/macos/{php7.0_mac => php74}/continuous.cfg (65%) rename kokoro/macos/{php7.3_mac => php74}/presubmit.cfg (65%) create mode 100755 kokoro/macos/php80/build.sh rename kokoro/macos/{php7.3_mac => php80}/continuous.cfg (65%) rename kokoro/macos/{php7.0_mac => php80}/presubmit.cfg (65%) create mode 100644 kokoro/windows/bazel/build.bat create mode 100644 kokoro/windows/bazel/continuous.cfg create mode 100644 kokoro/windows/bazel/presubmit.cfg create mode 100644 kokoro/windows/cmake/build.bat create mode 100644 kokoro/windows/cmake/continuous.cfg create mode 100644 kokoro/windows/cmake/presubmit.cfg create mode 100644 kokoro/windows/cmake_install/build.bat rename kokoro/{linux/cmake_distcheck => windows/cmake_install}/continuous.cfg (63%) rename kokoro/{linux/dist_install => windows/cmake_install}/presubmit.cfg (63%) create mode 100644 kokoro/windows/cmake_nmake/build.bat rename kokoro/{linux/cpp_distcheck => windows/cmake_nmake}/continuous.cfg (64%) rename kokoro/{linux/cpp_distcheck => windows/cmake_nmake}/presubmit.cfg (64%) create mode 100644 kokoro/windows/cmake_shared/build.bat create mode 100644 kokoro/windows/cmake_shared/continuous.cfg create mode 100644 kokoro/windows/cmake_shared/presubmit.cfg create mode 100644 php/tests/PreviouslyGeneratedClassTest.php create mode 100644 php/tests/generated_previous/GPBMetadata/ProtoPrevious/TestPreviouslyUnreservedMessage.php create mode 100644 php/tests/generated_previous/Previous/readonly.php create mode 100644 php/tests/proto_previous/test_previously_unreserved_message.proto create mode 100644 src/google/protobuf/compiler/java/message_serialization_unittest.cc create mode 100644 src/google/protobuf/compiler/java/message_serialization_unittest.proto diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index f9411a6062fe..572a5d2268a9 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -2,7 +2,7 @@ name: Bug report about: Create a report to help us improve title: '' -labels: '' +labels: 'untriaged' assignees: '' --- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 28b9bc97ce81..ada9ed7724c5 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: Feature request about: Suggest an idea for this project title: '' -labels: '' +labels: 'untriaged' assignees: '' --- diff --git a/.gitignore b/.gitignore index d8b0bd3842a3..cc881f3f2c86 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,23 @@ m4/ltversion.m4 m4/lt~obsolete.m4 autom4te.cache +# CMake-generated files +.ninja_deps +.ninja_logs +cmake/protobuf/*.cmake +cmake_install.cmake +CMakeCache.txt +CTestTestfile.cmake +CMakeFiles/* +Testing/Temporary/* + +/core +/protoc +/test_plugin +/tests +/lite-test +/protoc-*.* + # downloaded files /gmock @@ -41,6 +58,7 @@ stamp-h1 *.la src/.libs *.so +*.a .dirstamp diff --git a/BUILD.bazel b/BUILD.bazel index 1a305e7f1933..a14476a553d2 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -266,7 +266,7 @@ alias( alias( name = "well_known_types_py_pb2", actual = "//python:well_known_types_py_pb2", - visibility = ["@upb//:__subpackages__"], + visibility = ["//visibility:public"], ) alias( diff --git a/CHANGES.txt b/CHANGES.txt index fc30d1ae2215..d8ed3e99bbc8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,19 +1,64 @@ 2022-07-01 Unreleased version - * Add TransferAllMessages() util function - * Break ZeroCopyOutputByteSink into its own file. - * Explicitly instantiate InternalMetadata members at UnknownFieldSet. - * Support kdoc for Kotlin - * Delete unused function: IsProto3Field. + + C++ + * cpp_generated_lib_linked support is removed in protoc + * Reduced .pb.o object file size slightly by explicitly instantiating + InternalMetadata templates in the runtime. * Add C++20 keywords guarded by PROTOBUF_FUTURE_CPP20_KEYWORDS - * Escape Kotlin keywords in package names in proto generated code - * Fix StrAppend crashing on empty strings. - -2022-06-27 Unreleased version - * Handle reflection for message splitting. - * make metadata fields lazy. - * Extend visibility of plugin library to upb - * Modernize conformance_cpp.cc. - * Don't request 64-byte alignment unless the toolchain supports it. + * Fixed crash in ThreadLocalStorage for pre-C++17 compilers on 32-bit ARM. + * Clarified that JSON API non-OK statuses are not a stable API. + * Added a default implementation of MessageDifferencer::Reporter methods. + * proto2::MapPair is now an alias to std::pair. + * Hide C++ RepeatedField::UnsafeArenaSwap + + Kotlin + * Suppress deprecation warnings in Kotlin generated code. + * Kotlin generated code comments now use kdoc format instead of javadoc. + * Escape keywords in package names in proto generated code + * Add Kotlin enum int value getters and setters + + Java + * Performance improvement for repeated use of FieldMaskUtil#merge by caching + constructed FieldMaskTrees. + * Optimized Java proto serialization gencode for protos having many extension ranges with few fields in between. + + Python + * Changes ordering of printed fields in .pyi files from lexicographic to the same ordering found in the proto descriptor. + + Compiler + * Print full path name of source .proto file on error + + +2022-07-25 version 21.4 (C++/Java/Python/PHP/Objective-C/C#/Ruby) + + C++ + * Reduce the required alignment of ArenaString from 8 to 4 (#10298) + + +2022-07-19 version 21.3 (C++/Java/Python/PHP/Objective-C/C#/Ruby) + C++ + * Add header search paths to Protobuf-C++.podspec (#10024) + * Fixed Visual Studio constinit errors (#10232) + * Fix #9947: make the ABI compatible between debug and non-debug builds (#10271) + + UPB + * Allow empty package names (fixes behavior regression in 4.21.0) + * Fix a SEGV bug when comparing a non-materialized sub-message (#10208) + * Fix several bugs in descriptor mapping containers (eg. descriptor.services_by_name) + * for x in mapping now yields keys rather than values, to match Python conventions and the behavior of the old library. + * Lookup operations now correctly reject unhashable types as map keys. + * We implement repr() to use the same format as dict. + * Fix maps to use the ScalarMapContainer class when appropriate + * Fix bug when parsing an unknown value in a proto2 enum extension (protocolbuffers/upb#717) + + PHP + * Add "readonly" as a keyword for PHP and add previous classnames to descriptor pool (#10041) + + Python + * Make //:protobuf_python and //:well_known_types_py_pb2 public (#10118) + + Bazel + * Add back a filegroup for :well_known_protos (#10061) 2022-06-27 version 21.2 (C++/Java/Python/PHP/Objective-C/C#/Ruby) C++ diff --git a/CMakeLists.txt b/CMakeLists.txt index 04cb3303ac6a..2ed5ca11414d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,9 +57,11 @@ endif() option(protobuf_BUILD_TESTS "Build tests" ON) option(protobuf_BUILD_CONFORMANCE "Build conformance tests" OFF) option(protobuf_BUILD_EXAMPLES "Build examples" OFF) +option(protobuf_BUILD_PROTOBUF_BINARIES "Build protobuf libraries and protoc compiler" ON) option(protobuf_BUILD_PROTOC_BINARIES "Build libprotoc and protoc compiler" ON) option(protobuf_BUILD_LIBPROTOC "Build libprotoc" OFF) option(protobuf_DISABLE_RTTI "Remove runtime type information in the binaries" OFF) +option(protobuf_TEST_XML_OUTDIR "Output directory for XML logs from tests." "") if (BUILD_SHARED_LIBS) set(protobuf_BUILD_SHARED_LIBS_DEFAULT ON) else (BUILD_SHARED_LIBS) @@ -81,6 +83,9 @@ include(${protobuf_SOURCE_DIR}/cmake/protobuf-options.cmake) if (protobuf_BUILD_PROTOC_BINARIES OR protobuf_BUILD_TESTS) set(protobuf_BUILD_LIBPROTOC ON) endif () +if (NOT protobuf_BUILD_PROTOBUF_BINARIES) + set(protobuf_INSTALL OFF) +endif() # Path to main configure script set(protobuf_CONFIGURE_SCRIPT "${protobuf_SOURCE_DIR}/configure.ac") @@ -242,12 +247,12 @@ endif (protobuf_BUILD_SHARED_LIBS) if (MSVC) if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") # Build with multiple processes - add_definitions(/MP) + add_compile_options(/MP) endif() # Set source file and execution character sets to UTF-8 - add_definitions(/utf-8) + add_compile_options(/utf-8) # MSVC warning suppressions - add_definitions( + add_compile_options( /wd4065 # switch statement contains 'default' but no 'case' labels /wd4244 # 'conversion' conversion from 'type1' to 'type2', possible loss of data /wd4251 # 'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2' @@ -262,23 +267,16 @@ if (MSVC) /wd4996 # The compiler encountered a deprecated declaration. ) # Allow big object - add_definitions(/bigobj) + add_compile_options(/bigobj) string(REPLACE "/" "\\" PROTOBUF_SOURCE_WIN32_PATH ${protobuf_SOURCE_DIR}) string(REPLACE "/" "\\" PROTOBUF_BINARY_WIN32_PATH ${protobuf_BINARY_DIR}) string(REPLACE "." "," protobuf_RC_FILEVERSION "${protobuf_VERSION}") - configure_file(${protobuf_SOURCE_DIR}/cmake/extract_includes.bat.in extract_includes.bat) # Suppress linker warnings about files with no symbols defined. - set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} /ignore:4221") + string(APPEND CMAKE_STATIC_LINKER_FLAGS " /ignore:4221") - if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - # Configure Resource Compiler - enable_language(RC) - # use English language (0x409) in resource compiler - set(rc_flags "/l0x409") - # fix rc.exe invocations because of usage of add_definitions() - set(CMAKE_RC_COMPILE_OBJECT " ${rc_flags} /fo ") - endif() + # use English language (0x409) in resource compiler + string(APPEND CMAKE_RC_FLAGS " -l0x409") # Generate the version.rc file used elsewhere. configure_file(${protobuf_SOURCE_DIR}/cmake/version.rc.in ${CMAKE_CURRENT_BINARY_DIR}/version.rc @ONLY) @@ -304,28 +302,49 @@ if (protobuf_UNICODE) add_definitions(-DUNICODE -D_UNICODE) endif (protobuf_UNICODE) -include(${protobuf_SOURCE_DIR}/cmake/libprotobuf-lite.cmake) -include(${protobuf_SOURCE_DIR}/cmake/libprotobuf.cmake) -if (protobuf_BUILD_LIBPROTOC) - include(${protobuf_SOURCE_DIR}/cmake/libprotoc.cmake) -endif (protobuf_BUILD_LIBPROTOC) -if (protobuf_BUILD_PROTOC_BINARIES) - include(${protobuf_SOURCE_DIR}/cmake/protoc.cmake) - if (NOT DEFINED protobuf_PROTOC_EXE) - set(protobuf_PROTOC_EXE protoc) - endif (NOT DEFINED protobuf_PROTOC_EXE) -endif (protobuf_BUILD_PROTOC_BINARIES) +if (protobuf_BUILD_PROTOBUF_BINARIES) + include(${protobuf_SOURCE_DIR}/cmake/libprotobuf-lite.cmake) + if (NOT DEFINED protobuf_LIB_PROTOBUF_LITE) + set(protobuf_LIB_PROTOBUF_LITE libprotobuf-lite) + endif () + include(${protobuf_SOURCE_DIR}/cmake/libprotobuf.cmake) + if (NOT DEFINED protobuf_LIB_PROTOBUF) + set(protobuf_LIB_PROTOBUF libprotobuf) + endif () + if (protobuf_BUILD_LIBPROTOC) + include(${protobuf_SOURCE_DIR}/cmake/libprotoc.cmake) + if (NOT DEFINED protobuf_LIB_PROTOC) + set(protobuf_LIB_PROTOC libprotoc) + endif () + endif () + if (protobuf_BUILD_PROTOC_BINARIES) + include(${protobuf_SOURCE_DIR}/cmake/protoc.cmake) + if (NOT DEFINED protobuf_PROTOC_EXE) + set(protobuf_PROTOC_EXE protoc) + endif () + endif () +else () + find_package(protobuf) + if (protobuf_FOUND) + set(protobuf_PROTOC_EXE protobuf::protoc) + set(protobuf_LIB_PROTOC protobuf::libprotoc) + set(protobuf_LIB_PROTOBUF protobuf::libprotobuf) + set(protobuf_LIB_PROTOBUF_LITE protobuf::libprotobuf-lite) + message(STATUS "CMake installation of Protobuf found.") + endif () +endif () -# Ensure we have a protoc executable if we need one +# Ensure we have a protoc executable and protobuf libraries if we need one if (protobuf_BUILD_TESTS OR protobuf_BUILD_CONFORMANCE OR protobuf_BUILD_EXAMPLES) if (NOT DEFINED protobuf_PROTOC_EXE) - find_program(protobuf_PROTOC_EXE protoc) - if (NOT protobuf_PROTOC_EXE) - message(FATAL "Build requires 'protoc' but binary not found and not building protoc.") - endif () + find_program(protobuf_PROTOC_EXE protoc REQUIRED) + message(STATUS "Found system ${protobuf_PROTOC_EXE}.") endif () if(protobuf_VERBOSE) message(STATUS "Using protoc : ${protobuf_PROTOC_EXE}") + message(STATUS "Using libprotobuf : ${protobuf_LIB_PROTOBUF}") + message(STATUS "Using libprotobuf-lite : ${protobuf_LIB_PROTOBUF_LITE}") + message(STATUS "Using libprotoc : ${protobuf_LIB_PROTOC}") endif(protobuf_VERBOSE) endif () diff --git a/Makefile.am b/Makefile.am index c2a45f06444f..acc72c27ef41 100644 --- a/Makefile.am +++ b/Makefile.am @@ -131,11 +131,13 @@ csharp_EXTRA_DIST= \ csharp/src/Google.Protobuf.Test/GeneratedMessageTest.Proto2.cs \ csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj \ csharp/src/Google.Protobuf.Test/IssuesTest.cs \ + csharp/src/Google.Protobuf.Test/JsonFormatterSettingsTest.cs \ csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs \ csharp/src/Google.Protobuf.Test/JsonParserTest.cs \ csharp/src/Google.Protobuf.Test/JsonTokenizerTest.cs \ csharp/src/Google.Protobuf.Test/LegacyGeneratedCodeTest.cs \ csharp/src/Google.Protobuf.Test/MessageParsingHelpers.cs \ + csharp/src/Google.Protobuf.Test/ParsingPrimitivesTest.cs \ csharp/src/Google.Protobuf.Test/Proto3OptionalTest.cs \ csharp/src/Google.Protobuf.Test/ReadOnlySequenceFactory.cs \ csharp/src/Google.Protobuf.Test/RefStructCompatibilityTest.cs \ @@ -174,6 +176,7 @@ csharp_EXTRA_DIST= \ csharp/src/Google.Protobuf.Test/WellKnownTypes/FieldMaskTest.cs \ csharp/src/Google.Protobuf.Test/WellKnownTypes/TimestampTest.cs \ csharp/src/Google.Protobuf.Test/WellKnownTypes/WrappersTest.cs \ + csharp/src/Google.Protobuf.Test/WritingPrimitivesTest.cs \ csharp/src/Google.Protobuf.Test/UnknownFieldSetTest.cs \ csharp/src/Google.Protobuf.Test/testprotos.pb \ csharp/src/Google.Protobuf.sln \ @@ -193,6 +196,7 @@ csharp_EXTRA_DIST= \ csharp/src/Google.Protobuf/Compatibility/RequiresUnreferencedCodeAttribute.cs \ csharp/src/Google.Protobuf/Compatibility/TypeExtensions.cs \ csharp/src/Google.Protobuf/Compatibility/UnconditionalSuppressMessageAttribute.cs \ + csharp/src/Google.Protobuf/Compiler/Plugin.cs \ csharp/src/Google.Protobuf/Extension.cs \ csharp/src/Google.Protobuf/ExtensionRegistry.cs \ csharp/src/Google.Protobuf/ExtensionSet.cs \ @@ -957,6 +961,8 @@ php_EXTRA_DIST= \ php/tests/EncodeDecodeTest.php \ php/tests/force_c_ext.php \ php/tests/gdb_test.sh \ + php/tests/generated_previous/GPBMetadata/ProtoPrevious/TestPreviouslyUnreservedMessage.php \ + php/tests/generated_previous/Previous/readonly.php \ php/tests/GeneratedClassTest.php \ php/tests/GeneratedPhpdocTest.php \ php/tests/GeneratedServiceTest.php \ @@ -966,6 +972,7 @@ php_EXTRA_DIST= \ php/tests/multirequest.php \ php/tests/multirequest.sh \ php/tests/PhpImplementationTest.php \ + php/tests/PreviouslyGeneratedClassTest.php \ php/tests/proto/empty/echo.proto \ php/tests/proto/test.proto \ php/tests/proto/test_descriptors.proto \ @@ -984,6 +991,7 @@ php_EXTRA_DIST= \ php/tests/proto/test_service.proto \ php/tests/proto/test_service_namespace.proto \ php/tests/proto/test_wrapper_type_setters.proto \ + php/tests/proto_previous/test_previously_unreserved_message.proto \ php/tests/test_base.php \ php/tests/test_util.php \ php/tests/valgrind.supp \ @@ -1213,7 +1221,6 @@ EXTRA_DIST = $(@DIST_LANG@_EXTRA_DIST) \ cmake/README.md \ cmake/conformance.cmake \ cmake/examples.cmake \ - cmake/extract_includes.bat.in \ cmake/install.cmake \ cmake/libprotobuf-lite.cmake \ cmake/libprotobuf.cmake \ diff --git a/Protobuf-C++.podspec b/Protobuf-C++.podspec index c8a516bb0563..858a861a5b64 100644 --- a/Protobuf-C++.podspec +++ b/Protobuf-C++.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'Protobuf-C++' - s.version = '3.21.2' + s.version = '3.21.4' s.summary = 'Protocol Buffers v3 runtime library for C++.' s.homepage = 'https://github.com/google/protobuf' s.license = 'BSD-3-Clause' diff --git a/Protobuf.podspec b/Protobuf.podspec index 85f1dc6fa133..e41f39a7043b 100644 --- a/Protobuf.podspec +++ b/Protobuf.podspec @@ -5,7 +5,7 @@ # dependent projects use the :git notation to refer to the library. Pod::Spec.new do |s| s.name = 'Protobuf' - s.version = '3.21.2' + s.version = '3.21.4' s.summary = 'Protocol Buffers v.3 runtime library for Objective-C.' s.homepage = 'https://github.com/protocolbuffers/protobuf' s.license = 'BSD-3-Clause' diff --git a/README.md b/README.md index c2e6575f90ef..ae631588a760 100644 --- a/README.md +++ b/README.md @@ -82,3 +82,10 @@ The complete documentation for Protocol Buffers is available via the web at: https://developers.google.com/protocol-buffers/ + +Developer Community +------------------- + +To be alerted to upcoming changes in Protocol Buffers and connect with protobuf developers and users, +[join the Google Group](https://groups.google.com/g/protobuf). + diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java index adf5a2abbf07..061981eacfb3 100644 --- a/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java +++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java @@ -12,9 +12,8 @@ import java.util.ArrayList; import java.util.List; -/** - * Basic benchmarks for Java protobuf parsing. - */ +/** Basic benchmarks for Java protobuf parsing. */ +@SuppressWarnings("CheckReturnValue") public class ProtoCaliperBenchmark { public enum BenchmarkMessageType { GOOGLE_MESSAGE1_PROTO3 { diff --git a/cmake/README.md b/cmake/README.md index ce3e68023498..4a4259e45fe0 100644 --- a/cmake/README.md +++ b/cmake/README.md @@ -1,15 +1,19 @@ -This directory contains *CMake* files that can be used to build protobuf -with *MSVC* on *Windows*. You can build the project from *Command Prompt* -and using an *Visual Studio* IDE. +This directory contains *CMake* files that can be used to build protobuf. -You need to have [CMake](http://www.cmake.org), [Visual Studio](https://www.visualstudio.com) -and optionally [Git](http://git-scm.com) installed on your computer before proceeding. +You need to have [CMake](http://www.cmake.org) and +[Git](http://git-scm.com) installed on your computer before proceeding. -Most of the instructions will be given to the *Сommand Prompt*, but the same -actions can be performed using appropriate GUI tools. +Most of the instructions will be given using CMake's command-line interface, but +the same actions can be performed using appropriate GUI tools. -Environment Setup -================= +# Windows Builds + +On Windows, you can build the project from *Command Prompt* and using an +*Visual Studio* IDE. You will also need to have +[Visual Studio](https://www.visualstudio.com) installed on your computer before +proceeding. + +## Environment Setup Open the appropriate *Command Prompt* from the *Start* menu. @@ -42,8 +46,7 @@ Optionally, you will want to download [ninja](https://ninja-build.org/) and add Good. Now you are ready to continue. -Getting Sources -=============== +## Getting Sources You can get the latest stable source packages from the release page: @@ -76,8 +79,7 @@ C:\Path\to\src\protobuf> git submodule update --init --recursive Good. Now you are ready for *CMake* configuration. -CMake Configuration -=================== +## CMake Configuration *CMake* supports a lot of different [generators](http://www.cmake.org/cmake/help/latest/manual/cmake-generators.7.html) @@ -137,8 +139,7 @@ The *Visual Studio* generator is multi-configuration: it will generate a single It will generate *Visual Studio* solution file *protobuf.sln* in current directory. -Unit Tests ----------- +### Unit Tests Unit tests are being built along with the rest of protobuf. The unit tests require Google Mock (now a part of Google Test). @@ -178,8 +179,7 @@ For example: -Dprotobuf_BUILD_TESTS=OFF ^ C:\Path\to\src\protobuf -Compiling -========= +## Compiling The standard way to compile a *CMake* project is `cmake --build `. @@ -206,8 +206,7 @@ If you prefer to use the IDE: And wait for the compilation to finish. -Testing -======= +## Testing To run unit-tests, first you must compile protobuf as described above. Then run: @@ -259,8 +258,7 @@ Note that the tests must be run from the source folder. If all tests are passed, safely continue. -Installing -========== +## Installing To install protobuf to the *install* folder you've specified in the configuration step, you need to build the `install` target: @@ -292,8 +290,7 @@ compiling a debug build of your application, you may need to link against a debug build of libprotobufd.lib with "d" postfix. Similarly, release builds should link against release libprotobuf.lib library. -DLLs vs. static linking -======================= +## DLLs vs. static linking Static linking is now the default for the Protocol Buffer libraries. Due to issues with Win32's use of a separate heap for each DLL, as well as binary @@ -318,8 +315,7 @@ recommend that you do NOT expose protocol buffer objects in your library's public interface, and that you statically link protocol buffers into your library. -ZLib support -============ +## ZLib support If you want to include GzipInputStream and GzipOutputStream (google/protobuf/io/gzip_stream.h) in libprotobuf, you will need to do a few @@ -369,8 +365,7 @@ If you already have ZLIB library and headers at some other location on your syst Build and testing protobuf as usual. -Notes on Compiler Warnings -========================== +## Notes on Compiler Warnings The following warnings have been disabled while building the protobuf libraries and compiler. You may have to disable some of them in your own project as @@ -397,3 +392,23 @@ unique, so there should be no problem with this, but MSVC prints warning nevertheless. So, we disable it. Unfortunately, this warning will also be produced when compiling code which merely uses protocol buffers, meaning you may have to disable it in your code too. + +# Linux Builds + +Building with CMake works very similarly on Linux. Instead of Visual Studio, +you will need to have gcc or clang installed to handle the C++ builds. CMake +will generate Makefiles by default, but can also be configured to use Ninja. To +build Protobuf, you will need to run (from the source directory): + + cmake . + cmake --build . --parallel 10 + +Protobuf can be tested and installed with CMake: + + ctest --verbose + sudo cmake --install . + +or directly with the generated Makefiles: + + make VERBOSE=1 test + sudo make install diff --git a/cmake/conformance.cmake b/cmake/conformance.cmake index d6c435ac33ce..338b2757a500 100644 --- a/cmake/conformance.cmake +++ b/cmake/conformance.cmake @@ -24,6 +24,9 @@ add_executable(conformance_test_runner ${protobuf_SOURCE_DIR}/conformance/conformance.pb.cc ${protobuf_SOURCE_DIR}/conformance/conformance_test.cc ${protobuf_SOURCE_DIR}/conformance/conformance_test_runner.cc + ${protobuf_SOURCE_DIR}/conformance/conformance_test_main.cc + ${protobuf_SOURCE_DIR}/conformance/text_format_conformance_suite.cc + ${protobuf_SOURCE_DIR}/conformance/text_format_conformance_suite.h ${protobuf_SOURCE_DIR}/conformance/third_party/jsoncpp/json.h ${protobuf_SOURCE_DIR}/conformance/third_party/jsoncpp/jsoncpp.cpp ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.pb.cc @@ -45,5 +48,13 @@ target_include_directories( conformance_cpp PUBLIC ${protobuf_SOURCE_DIR}/conformance) -target_link_libraries(conformance_test_runner libprotobuf) -target_link_libraries(conformance_cpp libprotobuf) +target_link_libraries(conformance_test_runner ${protobuf_LIB_PROTOBUF}) +target_link_libraries(conformance_cpp ${protobuf_LIB_PROTOBUF}) + +add_test(NAME conformance_cpp_test + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/conformance_test_runner + --failure_list ${protobuf_SOURCE_DIR}/conformance/failure_list_cpp.txt + --text_format_failure_list ${protobuf_SOURCE_DIR}/conformance/text_format_failure_list_cpp.txt + --output_dir ${protobuf_TEST_XML_OUTDIR} + ${CMAKE_CURRENT_BINARY_DIR}/conformance_cpp + DEPENDS conformance_test_runner conformance_cpp) diff --git a/cmake/examples.cmake b/cmake/examples.cmake index 3b83d2b4cf08..07cfad2b9fda 100644 --- a/cmake/examples.cmake +++ b/cmake/examples.cmake @@ -30,7 +30,7 @@ endfunction() # Add examples as an external project. # sub_directory cannot be used because the find_package(protobuf) call would cause failures with redefined targets. add_examples_build(examples "-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}") -add_dependencies(examples libprotobuf protoc) +add_dependencies(examples ${protobuf_LIB_PROTOBUF} ${protobuf_PROTOC_EXE}) option(protobuf_BUILD_EXAMPLES_MULTITEST "Build Examples in multiple configurations. Useful for testing." OFF) mark_as_advanced(protobuf_BUILD_EXAMPLES_MULTITEST) @@ -42,7 +42,7 @@ if(protobuf_BUILD_EXAMPLES_MULTITEST) "-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}" "-Dprotobuf_MODULE_COMPATIBLE:BOOL=TRUE" ) - add_dependencies(examples-legacy libprotobuf protoc) + add_dependencies(examples-legacy ${protobuf_LIB_PROTOBUF} ${protobuf_PROTOC_EXE}) #Build using the installed library. add_examples_build(examples-installed diff --git a/cmake/extract_includes.bat.in b/cmake/extract_includes.bat.in deleted file mode 100644 index 129282921904..000000000000 --- a/cmake/extract_includes.bat.in +++ /dev/null @@ -1,144 +0,0 @@ -mkdir include -mkdir include\google -mkdir include\google\protobuf -mkdir include\google\protobuf\compiler -mkdir include\google\protobuf\compiler\cpp -mkdir include\google\protobuf\compiler\csharp -mkdir include\google\protobuf\compiler\java -mkdir include\google\protobuf\compiler\objectivec -mkdir include\google\protobuf\compiler\php -mkdir include\google\protobuf\compiler\python -mkdir include\google\protobuf\compiler\ruby -mkdir include\google\protobuf\io -mkdir include\google\protobuf\stubs -mkdir include\google\protobuf\util -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\any.h" include\google\protobuf\any.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\any.pb.h" include\google\protobuf\any.pb.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\api.pb.h" include\google\protobuf\api.pb.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\arena.h" include\google\protobuf\arena.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\arena_impl.h" include\google\protobuf\arena_impl.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\arenastring.h" include\google\protobuf\arenastring.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\arenaz_sampler.h" include\google\protobuf\arenaz_sampler.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\code_generator.h" include\google\protobuf\compiler\code_generator.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\command_line_interface.h" include\google\protobuf\compiler\command_line_interface.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\file.h" include\google\protobuf\compiler\cpp\file.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\cpp_generator.h" include\google\protobuf\compiler\cpp\cpp_generator.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\generator.h" include\google\protobuf\compiler\cpp\generator.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\helpers.h" include\google\protobuf\compiler\cpp\helpers.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\names.h" include\google\protobuf\compiler\cpp\names.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_doc_comment.h" include\google\protobuf\compiler\csharp\csharp_doc_comment.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_generator.h" include\google\protobuf\compiler\csharp\csharp_generator.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_names.h" include\google\protobuf\compiler\csharp\csharp_names.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_options.h" include\google\protobuf\compiler\csharp\csharp_options.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\importer.h" include\google\protobuf\compiler\importer.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\generator.h" include\google\protobuf\compiler\java\generator.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\java_generator.h" include\google\protobuf\compiler\java\java_generator.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\kotlin_generator.h" include\google\protobuf\compiler\java\kotlin_generator.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\names.h" include\google\protobuf\compiler\java\names.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\objectivec\objectivec_generator.h" include\google\protobuf\compiler\objectivec\objectivec_generator.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\objectivec\objectivec_helpers.h" include\google\protobuf\compiler\objectivec\objectivec_helpers.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\parser.h" include\google\protobuf\compiler\parser.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\php\php_generator.h" include\google\protobuf\compiler\php\php_generator.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\plugin.h" include\google\protobuf\compiler\plugin.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\plugin.pb.h" include\google\protobuf\compiler\plugin.pb.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\python\generator.h" include\google\protobuf\compiler\python\generator.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\python\pyi_generator.h" include\google\protobuf\compiler\python\pyi_generator.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\python\python_generator.h" include\google\protobuf\compiler\python\python_generator.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\ruby\ruby_generator.h" include\google\protobuf\compiler\ruby\ruby_generator.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\descriptor.h" include\google\protobuf\descriptor.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\descriptor.pb.h" include\google\protobuf\descriptor.pb.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\descriptor_database.h" include\google\protobuf\descriptor_database.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\duration.pb.h" include\google\protobuf\duration.pb.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\dynamic_message.h" include\google\protobuf\dynamic_message.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\empty.pb.h" include\google\protobuf\empty.pb.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\endian.h" include\google\protobuf\endian.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\explicitly_constructed.h" include\google\protobuf\explicitly_constructed.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\extension_set.h" include\google\protobuf\extension_set.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\extension_set_inl.h" include\google\protobuf\extension_set_inl.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\field_access_listener.h" include\google\protobuf\field_access_listener.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\field_mask.pb.h" include\google\protobuf\field_mask.pb.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_enum_reflection.h" include\google\protobuf\generated_enum_reflection.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_enum_util.h" include\google\protobuf\generated_enum_util.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_bases.h" include\google\protobuf\generated_message_bases.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_reflection.h" include\google\protobuf\generated_message_reflection.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_tctable_decl.h" include\google\protobuf\generated_message_tctable_decl.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_tctable_impl.h" include\google\protobuf\generated_message_tctable_impl.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_util.h" include\google\protobuf\generated_message_util.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\has_bits.h" include\google\protobuf\has_bits.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\implicit_weak_message.h" include\google\protobuf\implicit_weak_message.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\inlined_string_field.h" include\google\protobuf\inlined_string_field.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\coded_stream.h" include\google\protobuf\io\coded_stream.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\gzip_stream.h" include\google\protobuf\io\gzip_stream.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\io_win32.h" include\google\protobuf\io\io_win32.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\printer.h" include\google\protobuf\io\printer.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\strtod.h" include\google\protobuf\io\strtod.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\tokenizer.h" include\google\protobuf\io\tokenizer.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\zero_copy_stream.h" include\google\protobuf\io\zero_copy_stream.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\zero_copy_stream_impl.h" include\google\protobuf\io\zero_copy_stream_impl.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\zero_copy_stream_impl_lite.h" include\google\protobuf\io\zero_copy_stream_impl_lite.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map.h" include\google\protobuf\map.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_entry.h" include\google\protobuf\map_entry.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_entry_lite.h" include\google\protobuf\map_entry_lite.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_field.h" include\google\protobuf\map_field.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_field_inl.h" include\google\protobuf\map_field_inl.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_field_lite.h" include\google\protobuf\map_field_lite.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\map_type_handler.h" include\google\protobuf\map_type_handler.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\message.h" include\google\protobuf\message.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\message_lite.h" include\google\protobuf\message_lite.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\metadata.h" include\google\protobuf\metadata.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\metadata_lite.h" include\google\protobuf\metadata_lite.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\parse_context.h" include\google\protobuf\parse_context.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\port.h" include\google\protobuf\port.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\port_def.inc" include\google\protobuf\port_def.inc -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\port_undef.inc" include\google\protobuf\port_undef.inc -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\reflection.h" include\google\protobuf\reflection.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\reflection_ops.h" include\google\protobuf\reflection_ops.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\repeated_field.h" include\google\protobuf\repeated_field.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\repeated_ptr_field.h" include\google\protobuf\repeated_ptr_field.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\service.h" include\google\protobuf\service.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\source_context.pb.h" include\google\protobuf\source_context.pb.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\struct.pb.h" include\google\protobuf\struct.pb.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\bytestream.h" include\google\protobuf\stubs\bytestream.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\callback.h" include\google\protobuf\stubs\callback.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\casts.h" include\google\protobuf\stubs\casts.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\common.h" include\google\protobuf\stubs\common.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\hash.h" include\google\protobuf\stubs\hash.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\logging.h" include\google\protobuf\stubs\logging.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\macros.h" include\google\protobuf\stubs\macros.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\map_util.h" include\google\protobuf\stubs\map_util.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\mutex.h" include\google\protobuf\stubs\mutex.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\once.h" include\google\protobuf\stubs\once.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\platform_macros.h" include\google\protobuf\stubs\platform_macros.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\port.h" include\google\protobuf\stubs\port.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\status.h" include\google\protobuf\stubs\status.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\stl_util.h" include\google\protobuf\stubs\stl_util.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\stringpiece.h" include\google\protobuf\stubs\stringpiece.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\strutil.h" include\google\protobuf\stubs\strutil.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\template_util.h" include\google\protobuf\stubs\template_util.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\text_format.h" include\google\protobuf\text_format.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\timestamp.pb.h" include\google\protobuf\timestamp.pb.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\type.pb.h" include\google\protobuf\type.pb.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\unknown_field_set.h" include\google\protobuf\unknown_field_set.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\delimited_message_util.h" include\google\protobuf\util\delimited_message_util.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\field_comparator.h" include\google\protobuf\util\field_comparator.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\field_mask_util.h" include\google\protobuf\util\field_mask_util.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\json_util.h" include\google\protobuf\util\json_util.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\message_differencer.h" include\google\protobuf\util\message_differencer.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\time_util.h" include\google\protobuf\util\time_util.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\type_resolver.h" include\google\protobuf\util\type_resolver.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\type_resolver_util.h" include\google\protobuf\util\type_resolver_util.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wire_format.h" include\google\protobuf\wire_format.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wire_format_lite.h" include\google\protobuf\wire_format_lite.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wrappers.pb.h" include\google\protobuf\wrappers.pb.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\any.proto" include\google\protobuf\any.proto -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\api.proto" include\google\protobuf\api.proto -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\plugin.proto" include\google\protobuf\compiler\plugin.proto -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\descriptor.proto" include\google\protobuf\descriptor.proto -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\duration.proto" include\google\protobuf\duration.proto -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\empty.proto" include\google\protobuf\empty.proto -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\field_mask.proto" include\google\protobuf\field_mask.proto -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\source_context.proto" include\google\protobuf\source_context.proto -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\struct.proto" include\google\protobuf\struct.proto -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\timestamp.proto" include\google\protobuf\timestamp.proto -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\type.proto" include\google\protobuf\type.proto -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wrappers.proto" include\google\protobuf\wrappers.proto diff --git a/cmake/install.cmake b/cmake/install.cmake index 2da817054426..cf24e307c1e9 100644 --- a/cmake/install.cmake +++ b/cmake/install.cmake @@ -43,25 +43,23 @@ endif (protobuf_BUILD_PROTOC_BINARIES) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/protobuf.pc ${CMAKE_CURRENT_BINARY_DIR}/protobuf-lite.pc DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") -file(STRINGS ${protobuf_SOURCE_DIR}/cmake/extract_includes.bat.in _extract_strings - REGEX "^copy") -foreach(_extract_string ${_extract_strings}) - string(REGEX REPLACE "^.* .+ include\\\\(.+)$" "\\1" - _header ${_extract_string}) - string(REPLACE "\\" "/" _header ${_header}) +include(${protobuf_SOURCE_DIR}/src/file_lists.cmake) +set(protobuf_HEADERS + ${libprotobuf_hdrs} + ${libprotoc_hdrs} + ${wkt_protos_files} + ${descriptor_proto_proto_srcs} + ${plugin_proto_proto_srcs} +) +foreach(_header ${protobuf_HEADERS}) + string(REPLACE "${protobuf_SOURCE_DIR}/src" "" _header ${_header}) get_filename_component(_extract_from "${protobuf_SOURCE_DIR}/src/${_header}" ABSOLUTE) get_filename_component(_extract_name ${_header} NAME) get_filename_component(_extract_to "${CMAKE_INSTALL_INCLUDEDIR}/${_header}" DIRECTORY) - if(EXISTS "${_extract_from}") - install(FILES "${_extract_from}" - DESTINATION "${_extract_to}" - COMPONENT protobuf-headers - RENAME "${_extract_name}") - else() - message(AUTHOR_WARNING "The file \"${_extract_from}\" is listed in " - "\"${protobuf_SOURCE_DIR}/cmake/extract_includes.bat.in\" " - "but there not exists. The file will not be installed.") - endif() + install(FILES "${_extract_from}" + DESTINATION "${_extract_to}" + COMPONENT protobuf-headers + RENAME "${_extract_name}") endforeach() # Internal function for parsing auto tools scripts diff --git a/cmake/tests.cmake b/cmake/tests.cmake index be1d429ab85c..6a1d1c09cc34 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -1,6 +1,7 @@ option(protobuf_USE_EXTERNAL_GTEST "Use external Google Test (i.e. not the one in third_party/googletest)" OFF) -option(protobuf_TEST_XML_OUTDIR "Output directory for XML logs from tests." "") +option(protobuf_REMOVE_INSTALLED_HEADERS + "Remove local headers so that installed ones are used instead" OFF) option(protobuf_ABSOLUTE_TEST_PLUGIN_PATH "Using absolute test_plugin path in tests" ON) @@ -77,10 +78,9 @@ endforeach(proto_file) add_library(protobuf-lite-test-common STATIC ${lite_test_util_srcs} ${lite_test_proto_files}) -target_link_libraries(protobuf-lite-test-common libprotobuf-lite GTest::gmock) +target_link_libraries(protobuf-lite-test-common ${protobuf_LIB_PROTOBUF_LITE} GTest::gmock) set(common_test_files - ${lite_test_util_srcs} ${test_util_hdrs} ${test_util_srcs} ${mock_code_generator_srcs} @@ -89,7 +89,7 @@ set(common_test_files add_library(protobuf-test-common STATIC ${common_test_files} ${tests_proto_files}) -target_link_libraries(protobuf-test-common libprotobuf GTest::gmock) +target_link_libraries(protobuf-test-common ${protobuf_LIB_PROTOBUF} GTest::gmock) set(tests_files ${protobuf_test_files} @@ -130,7 +130,7 @@ if (MSVC) /wd4146 # unary minus operator applied to unsigned type, result still unsigned ) endif() -target_link_libraries(tests protobuf-lite-test-common protobuf-test-common libprotoc libprotobuf GTest::gmock_main) +target_link_libraries(tests protobuf-lite-test-common protobuf-test-common ${protobuf_LIB_PROTOC} ${protobuf_LIB_PROTOBUF} GTest::gmock_main) set(test_plugin_files ${test_plugin_files} @@ -139,19 +139,50 @@ set(test_plugin_files ) add_executable(test_plugin ${test_plugin_files}) -target_link_libraries(test_plugin libprotoc libprotobuf GTest::gmock) +target_link_libraries(test_plugin ${protobuf_LIB_PROTOC} ${protobuf_LIB_PROTOBUF} GTest::gmock) add_executable(lite-test ${protobuf_lite_test_files}) -target_link_libraries(lite-test protobuf-lite-test-common libprotobuf-lite GTest::gmock_main) +target_link_libraries(lite-test protobuf-lite-test-common ${protobuf_LIB_PROTOBUF_LITE} GTest::gmock_main) add_test(NAME lite-test COMMAND lite-test ${protobuf_GTEST_ARGS}) add_custom_target(check COMMAND tests - DEPENDS tests test_plugin + DEPENDS tests lite-test test_plugin WORKING_DIRECTORY ${protobuf_SOURCE_DIR}) add_test(NAME check - COMMAND tests ${protobuf_GTEST_ARGS} - WORKING_DIRECTORY "${protobuf_SOURCE_DIR}") + COMMAND tests ${protobuf_GTEST_ARGS}) + +# For test purposes, remove headers that should already be installed. This +# prevents accidental conflicts and also version skew (since local headers take +# precedence over installed headers). +add_custom_target(save-installed-headers) +add_custom_target(remove-installed-headers) +add_custom_target(restore-installed-headers) + +# Explicitly skip the bootstrapping headers as it's directly used in tests +set(_installed_hdrs ${libprotobuf_hdrs} ${libprotoc_hdrs}) +list(REMOVE_ITEM _installed_hdrs + "${protobuf_SOURCE_DIR}/src/google/protobuf/descriptor.pb.h" + "${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/plugin.pb.h") + +foreach(_hdr ${_installed_hdrs}) + string(REPLACE "${protobuf_SOURCE_DIR}/src" "" _file ${_hdr}) + set(_tmp_file "${CMAKE_BINARY_DIR}/tmp-install-test/${_file}") + add_custom_command(TARGET remove-installed-headers PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E remove -f "${_hdr}") + add_custom_command(TARGET save-installed-headers PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E + copy "${_hdr}" "${_tmp_file}" || true) + add_custom_command(TARGET restore-installed-headers PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E + copy "${_tmp_file}" "${_hdr}") +endforeach() + +add_dependencies(remove-installed-headers save-installed-headers) +if(protobuf_REMOVE_INSTALLED_HEADERS) + add_dependencies(protobuf-lite-test-common remove-installed-headers) + add_dependencies(protobuf-test-common remove-installed-headers) +endif() diff --git a/configure.ac b/configure.ac index c55a6a2c2f95..64ddb3dc84d4 100644 --- a/configure.ac +++ b/configure.ac @@ -17,7 +17,7 @@ AC_PREREQ(2.59) # In the SVN trunk, the version should always be the next anticipated release # version with the "-pre" suffix. (We used to use "-SNAPSHOT" but this pushed # the size of one file name in the dist tarfile over the 99-char limit.) -AC_INIT([Protocol Buffers],[3.21.2],[protobuf@googlegroups.com],[protobuf]) +AC_INIT([Protocol Buffers],[3.21.4],[protobuf@googlegroups.com],[protobuf]) AM_MAINTAINER_MODE([enable]) diff --git a/conformance/binary_json_conformance_suite.cc b/conformance/binary_json_conformance_suite.cc index 9b1548df55fe..3719a527aab7 100644 --- a/conformance/binary_json_conformance_suite.cc +++ b/conformance/binary_json_conformance_suite.cc @@ -2724,6 +2724,12 @@ void BinaryAndJsonConformanceSuite::RunJsonTestsForWrapperTypes() { "DurationNull", REQUIRED, R"({"optionalDuration": null})", ""); + RunValidJsonTest("DurationNegativeSeconds", REQUIRED, + R"({"optionalDuration": "-5s"})", + "optional_duration: {seconds: -5 nanos: 0}"); + RunValidJsonTest("DurationNegativeNanos", REQUIRED, + R"({"optionalDuration": "-0.5s"})", + "optional_duration: {seconds: 0 nanos: -500000000}"); ExpectParseFailureForJson( "DurationMissingS", REQUIRED, diff --git a/conformance/failure_list_php.txt b/conformance/failure_list_php.txt index 667f80ca37e1..3232047aa759 100644 --- a/conformance/failure_list_php.txt +++ b/conformance/failure_list_php.txt @@ -8,6 +8,8 @@ Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter Recommended.Proto3.ProtobufInput.ValidDataOneofBinary.MESSAGE.Merge.ProtobufOutput Required.Proto2.JsonInput.StoresDefaultPrimitive.Validator Required.Proto3.JsonInput.DoubleFieldTooSmall +Required.Proto3.JsonInput.DurationNegativeNanos.JsonOutput +Required.Proto3.JsonInput.DurationNegativeNanos.ProtobufOutput Required.Proto3.JsonInput.FloatFieldTooLarge Required.Proto3.JsonInput.FloatFieldTooSmall Required.Proto3.JsonInput.Int32FieldNotInteger diff --git a/conformance/failure_list_php_c.txt b/conformance/failure_list_php_c.txt index 63c7e8a024cc..52ca4544b6c7 100644 --- a/conformance/failure_list_php_c.txt +++ b/conformance/failure_list_php_c.txt @@ -1,2 +1,5 @@ Recommended.Proto2.JsonInput.FieldNameExtension.Validator Required.Proto2.JsonInput.StoresDefaultPrimitive.Validator +Required.Proto3.JsonInput.DurationNegativeNanos.JsonOutput +Required.Proto3.JsonInput.DurationNegativeNanos.ProtobufOutput +Required.Proto3.JsonInput.DurationNegativeSeconds.JsonOutput diff --git a/conformance/failure_list_ruby.txt b/conformance/failure_list_ruby.txt index 4938202ad70b..ff230a26a9c3 100644 --- a/conformance/failure_list_ruby.txt +++ b/conformance/failure_list_ruby.txt @@ -56,3 +56,6 @@ Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.UnpackedOu Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.UnpackedOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT64.PackedInput.UnpackedOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT64.UnpackedInput.UnpackedOutput.ProtobufOutput +Required.Proto3.JsonInput.DurationNegativeNanos.JsonOutput +Required.Proto3.JsonInput.DurationNegativeNanos.ProtobufOutput +Required.Proto3.JsonInput.DurationNegativeSeconds.JsonOutput diff --git a/csharp/Google.Protobuf.Tools.nuspec b/csharp/Google.Protobuf.Tools.nuspec index ea0559770ea4..90033cfae821 100644 --- a/csharp/Google.Protobuf.Tools.nuspec +++ b/csharp/Google.Protobuf.Tools.nuspec @@ -5,7 +5,7 @@ Google Protocol Buffers tools Tools for Protocol Buffers - Google's data interchange format. See project site for more info. - 3.21.2 + 3.21.4 Google Inc. protobuf-packages https://github.com/protocolbuffers/protobuf/blob/main/LICENSE diff --git a/csharp/README.md b/csharp/README.md index df4849956d6c..02abb7cee0e0 100644 --- a/csharp/README.md +++ b/csharp/README.md @@ -16,18 +16,19 @@ To generate C# files from your `.proto` files, invoke `protoc` with the Supported platforms =================== -The runtime library is built as a portable class library, supporting: +The runtime library is built as a class library, supporting targets of: -- .NET 4.5 -- Windows 8 -- Windows Phone Silverlight 8 -- Windows Phone 8.1 -- .NET Core -- .NET 5 +- .NET 4.5+ (`net45`) +- .NET Standard 1.1 and 2.0 (`netstandard1.1` and `netstandard2.0`) +- .NET 5+ (`net50`) You should be able to use Protocol Buffers in Visual Studio 2012 and all later versions. This includes all code generated by `protoc`, -which only uses features from C# 3 and earlier. +which only uses features from C# 3 and earlier. When compiling generated +code with old compilers (before C# 7.2) you need to define the +`GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE` symbol in your project +so that the generated classes don't implement `IBufferMessage`, which uses +`ref struct` types. Building ======== @@ -42,12 +43,6 @@ in its implementation and runs tests under .NET 6. These features have no impact when using the compiled code - they're only relevant when building the `Google.Protobuf` assembly. -In order to run and debug the AddressBook example in the IDE, you must -install the optional component, ".Net Core 1.0 - 1.1 development tools -for Web" (as it's labelled in current versions of the VS2017 -installer), above and beyond the main .NET Core cross-platform -development feature. - Testing ======= @@ -57,20 +52,11 @@ run using the Visual Studio Test Explorer or `dotnet test`. .NET 3.5 ======== -We don't officially support .NET 3.5. However, there has been some effort -to make enabling .NET 3.5 support relatively painless in case you require it. -There's no guarantee that this will continue in the future, so rely on .NET -3.5 support at your peril. - -To enable .NET 3.5 support, you must edit the `TargetFrameworks` elements of -[src/Google.Protobuf/Google.Protobuf.csproj](src/Google.Protobuf/Google.Protobuf.csproj) -(and [src/Google.Protobuf.Test/Google.Protobuf.Test.csproj](src/Google.Protobuf.Test/Google.Protobuf.Test.csproj) -if you want to run the unit tests): - -Open the .csproj file in a text editor and simply add `net35` to the list of -target frameworks, noting that the `TargetFrameworks` element appears twice in -the file (once in the first `PropertyGroup` element, and again in the second -`PropertyGroup` element, i.e., the one with the conditional). +We don't support .NET 3.5. It *used* to be feasible to build this library +targeting .NET 3.5, but a number of changes requiring newer runtime/framework +features have been added over time. While it would no doubt be *possible* to +rework the current implementation to allow most of the functionality to be built +in .NET 3.5, this would create an undue maintenance burden. History of C# protobufs ======================= @@ -82,9 +68,9 @@ and maintained by Google. All the development will be done in open, under this r The previous project differs from this project in a number of ways: -- The old code only supported proto2; the new code only supports +- The old code only supported proto2; the new code initially only supported proto3 (so no unknown fields, no required/optional distinction, no -extensions) +extensions); since then proto2 support has been added - The old code was based on immutable message types and builders for them - The old code did not support maps or `oneof` diff --git a/csharp/generate_protos.sh b/csharp/generate_protos.sh index b021de2acad4..62ba1a199464 100755 --- a/csharp/generate_protos.sh +++ b/csharp/generate_protos.sh @@ -40,7 +40,8 @@ $PROTOC -Isrc --csharp_out=csharp/src/Google.Protobuf \ src/google/protobuf/struct.proto \ src/google/protobuf/timestamp.proto \ src/google/protobuf/type.proto \ - src/google/protobuf/wrappers.proto + src/google/protobuf/wrappers.proto \ + src/google/protobuf/compiler/plugin.proto # Test protos # Note that this deliberately does *not* include old_extensions1.proto diff --git a/csharp/protos/unittest_issues.proto b/csharp/protos/unittest_issues.proto index f46c20e4da1f..45df7a19a08a 100644 --- a/csharp/protos/unittest_issues.proto +++ b/csharp/protos/unittest_issues.proto @@ -169,4 +169,20 @@ message OneofWithNoneName { string x = 1; string y = 2; } -} \ No newline at end of file +} + +// Issue 8810 +message DisambiguateCommonMembers { + int32 disambiguate_common_members = 1; + int32 types = 2; + int32 descriptor = 3; + int32 equals = 4; + int32 to_string = 5; + int32 get_hash_code = 6; + int32 write_to = 7; + int32 clone = 8; + int32 calculate_size = 9; + int32 merge_from = 10; + int32 on_construction = 11; + int32 parser = 12; +} diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/MapUnittestProto3.cs b/csharp/src/Google.Protobuf.Test.TestProtos/MapUnittestProto3.cs index 57e59a9ac336..d3284a40f66f 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/MapUnittestProto3.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/MapUnittestProto3.cs @@ -580,23 +580,23 @@ public void MergeFrom(TestMap other) { if (other == null) { return; } - mapInt32Int32_.Add(other.mapInt32Int32_); - mapInt64Int64_.Add(other.mapInt64Int64_); - mapUint32Uint32_.Add(other.mapUint32Uint32_); - mapUint64Uint64_.Add(other.mapUint64Uint64_); - mapSint32Sint32_.Add(other.mapSint32Sint32_); - mapSint64Sint64_.Add(other.mapSint64Sint64_); - mapFixed32Fixed32_.Add(other.mapFixed32Fixed32_); - mapFixed64Fixed64_.Add(other.mapFixed64Fixed64_); - mapSfixed32Sfixed32_.Add(other.mapSfixed32Sfixed32_); - mapSfixed64Sfixed64_.Add(other.mapSfixed64Sfixed64_); - mapInt32Float_.Add(other.mapInt32Float_); - mapInt32Double_.Add(other.mapInt32Double_); - mapBoolBool_.Add(other.mapBoolBool_); - mapStringString_.Add(other.mapStringString_); - mapInt32Bytes_.Add(other.mapInt32Bytes_); - mapInt32Enum_.Add(other.mapInt32Enum_); - mapInt32ForeignMessage_.Add(other.mapInt32ForeignMessage_); + mapInt32Int32_.MergeFrom(other.mapInt32Int32_); + mapInt64Int64_.MergeFrom(other.mapInt64Int64_); + mapUint32Uint32_.MergeFrom(other.mapUint32Uint32_); + mapUint64Uint64_.MergeFrom(other.mapUint64Uint64_); + mapSint32Sint32_.MergeFrom(other.mapSint32Sint32_); + mapSint64Sint64_.MergeFrom(other.mapSint64Sint64_); + mapFixed32Fixed32_.MergeFrom(other.mapFixed32Fixed32_); + mapFixed64Fixed64_.MergeFrom(other.mapFixed64Fixed64_); + mapSfixed32Sfixed32_.MergeFrom(other.mapSfixed32Sfixed32_); + mapSfixed64Sfixed64_.MergeFrom(other.mapSfixed64Sfixed64_); + mapInt32Float_.MergeFrom(other.mapInt32Float_); + mapInt32Double_.MergeFrom(other.mapInt32Double_); + mapBoolBool_.MergeFrom(other.mapBoolBool_); + mapStringString_.MergeFrom(other.mapStringString_); + mapInt32Bytes_.MergeFrom(other.mapInt32Bytes_); + mapInt32Enum_.MergeFrom(other.mapInt32Enum_); + mapInt32ForeignMessage_.MergeFrom(other.mapInt32ForeignMessage_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -1100,7 +1100,7 @@ public void MergeFrom(TestMessageMap other) { if (other == null) { return; } - mapInt32Message_.Add(other.mapInt32Message_); + mapInt32Message_.MergeFrom(other.mapInt32Message_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -1298,8 +1298,8 @@ public void MergeFrom(TestSameTypeMap other) { if (other == null) { return; } - map1_.Add(other.map1_); - map2_.Add(other.map2_); + map1_.MergeFrom(other.map1_); + map2_.MergeFrom(other.map2_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -1723,21 +1723,21 @@ public void MergeFrom(TestArenaMap other) { if (other == null) { return; } - mapInt32Int32_.Add(other.mapInt32Int32_); - mapInt64Int64_.Add(other.mapInt64Int64_); - mapUint32Uint32_.Add(other.mapUint32Uint32_); - mapUint64Uint64_.Add(other.mapUint64Uint64_); - mapSint32Sint32_.Add(other.mapSint32Sint32_); - mapSint64Sint64_.Add(other.mapSint64Sint64_); - mapFixed32Fixed32_.Add(other.mapFixed32Fixed32_); - mapFixed64Fixed64_.Add(other.mapFixed64Fixed64_); - mapSfixed32Sfixed32_.Add(other.mapSfixed32Sfixed32_); - mapSfixed64Sfixed64_.Add(other.mapSfixed64Sfixed64_); - mapInt32Float_.Add(other.mapInt32Float_); - mapInt32Double_.Add(other.mapInt32Double_); - mapBoolBool_.Add(other.mapBoolBool_); - mapInt32Enum_.Add(other.mapInt32Enum_); - mapInt32ForeignMessage_.Add(other.mapInt32ForeignMessage_); + mapInt32Int32_.MergeFrom(other.mapInt32Int32_); + mapInt64Int64_.MergeFrom(other.mapInt64Int64_); + mapUint32Uint32_.MergeFrom(other.mapUint32Uint32_); + mapUint64Uint64_.MergeFrom(other.mapUint64Uint64_); + mapSint32Sint32_.MergeFrom(other.mapSint32Sint32_); + mapSint64Sint64_.MergeFrom(other.mapSint64Sint64_); + mapFixed32Fixed32_.MergeFrom(other.mapFixed32Fixed32_); + mapFixed64Fixed64_.MergeFrom(other.mapFixed64Fixed64_); + mapSfixed32Sfixed32_.MergeFrom(other.mapSfixed32Sfixed32_); + mapSfixed64Sfixed64_.MergeFrom(other.mapSfixed64Sfixed64_); + mapInt32Float_.MergeFrom(other.mapInt32Float_); + mapInt32Double_.MergeFrom(other.mapInt32Double_); + mapBoolBool_.MergeFrom(other.mapBoolBool_); + mapInt32Enum_.MergeFrom(other.mapInt32Enum_); + mapInt32ForeignMessage_.MergeFrom(other.mapInt32ForeignMessage_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -2031,7 +2031,7 @@ public void MergeFrom(MessageContainingEnumCalledType other) { if (other == null) { return; } - type_.Add(other.type_); + type_.MergeFrom(other.type_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -2224,7 +2224,7 @@ public void MergeFrom(MessageContainingMapCalledEntry other) { if (other == null) { return; } - entry_.Add(other.entry_); + entry_.MergeFrom(other.entry_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto2.cs b/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto2.cs index 08b1aaf80c84..d5734552c04b 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto2.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto2.cs @@ -4344,25 +4344,25 @@ public void MergeFrom(TestAllTypesProto2 other) { unpackedDouble_.Add(other.unpackedDouble_); unpackedBool_.Add(other.unpackedBool_); unpackedNestedEnum_.Add(other.unpackedNestedEnum_); - mapInt32Int32_.Add(other.mapInt32Int32_); - mapInt64Int64_.Add(other.mapInt64Int64_); - mapUint32Uint32_.Add(other.mapUint32Uint32_); - mapUint64Uint64_.Add(other.mapUint64Uint64_); - mapSint32Sint32_.Add(other.mapSint32Sint32_); - mapSint64Sint64_.Add(other.mapSint64Sint64_); - mapFixed32Fixed32_.Add(other.mapFixed32Fixed32_); - mapFixed64Fixed64_.Add(other.mapFixed64Fixed64_); - mapSfixed32Sfixed32_.Add(other.mapSfixed32Sfixed32_); - mapSfixed64Sfixed64_.Add(other.mapSfixed64Sfixed64_); - mapInt32Float_.Add(other.mapInt32Float_); - mapInt32Double_.Add(other.mapInt32Double_); - mapBoolBool_.Add(other.mapBoolBool_); - mapStringString_.Add(other.mapStringString_); - mapStringBytes_.Add(other.mapStringBytes_); - mapStringNestedMessage_.Add(other.mapStringNestedMessage_); - mapStringForeignMessage_.Add(other.mapStringForeignMessage_); - mapStringNestedEnum_.Add(other.mapStringNestedEnum_); - mapStringForeignEnum_.Add(other.mapStringForeignEnum_); + mapInt32Int32_.MergeFrom(other.mapInt32Int32_); + mapInt64Int64_.MergeFrom(other.mapInt64Int64_); + mapUint32Uint32_.MergeFrom(other.mapUint32Uint32_); + mapUint64Uint64_.MergeFrom(other.mapUint64Uint64_); + mapSint32Sint32_.MergeFrom(other.mapSint32Sint32_); + mapSint64Sint64_.MergeFrom(other.mapSint64Sint64_); + mapFixed32Fixed32_.MergeFrom(other.mapFixed32Fixed32_); + mapFixed64Fixed64_.MergeFrom(other.mapFixed64Fixed64_); + mapSfixed32Sfixed32_.MergeFrom(other.mapSfixed32Sfixed32_); + mapSfixed64Sfixed64_.MergeFrom(other.mapSfixed64Sfixed64_); + mapInt32Float_.MergeFrom(other.mapInt32Float_); + mapInt32Double_.MergeFrom(other.mapInt32Double_); + mapBoolBool_.MergeFrom(other.mapBoolBool_); + mapStringString_.MergeFrom(other.mapStringString_); + mapStringBytes_.MergeFrom(other.mapStringBytes_); + mapStringNestedMessage_.MergeFrom(other.mapStringNestedMessage_); + mapStringForeignMessage_.MergeFrom(other.mapStringForeignMessage_); + mapStringNestedEnum_.MergeFrom(other.mapStringNestedEnum_); + mapStringForeignEnum_.MergeFrom(other.mapStringForeignEnum_); if (other.HasData) { if (!HasData) { Data = new global::ProtobufTestMessages.Proto2.TestAllTypesProto2.Types.Data(); diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto3.cs b/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto3.cs index 520216fdf548..74e2a5768065 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto3.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto3.cs @@ -3753,25 +3753,25 @@ public void MergeFrom(TestAllTypesProto3 other) { unpackedDouble_.Add(other.unpackedDouble_); unpackedBool_.Add(other.unpackedBool_); unpackedNestedEnum_.Add(other.unpackedNestedEnum_); - mapInt32Int32_.Add(other.mapInt32Int32_); - mapInt64Int64_.Add(other.mapInt64Int64_); - mapUint32Uint32_.Add(other.mapUint32Uint32_); - mapUint64Uint64_.Add(other.mapUint64Uint64_); - mapSint32Sint32_.Add(other.mapSint32Sint32_); - mapSint64Sint64_.Add(other.mapSint64Sint64_); - mapFixed32Fixed32_.Add(other.mapFixed32Fixed32_); - mapFixed64Fixed64_.Add(other.mapFixed64Fixed64_); - mapSfixed32Sfixed32_.Add(other.mapSfixed32Sfixed32_); - mapSfixed64Sfixed64_.Add(other.mapSfixed64Sfixed64_); - mapInt32Float_.Add(other.mapInt32Float_); - mapInt32Double_.Add(other.mapInt32Double_); - mapBoolBool_.Add(other.mapBoolBool_); - mapStringString_.Add(other.mapStringString_); - mapStringBytes_.Add(other.mapStringBytes_); - mapStringNestedMessage_.Add(other.mapStringNestedMessage_); - mapStringForeignMessage_.Add(other.mapStringForeignMessage_); - mapStringNestedEnum_.Add(other.mapStringNestedEnum_); - mapStringForeignEnum_.Add(other.mapStringForeignEnum_); + mapInt32Int32_.MergeFrom(other.mapInt32Int32_); + mapInt64Int64_.MergeFrom(other.mapInt64Int64_); + mapUint32Uint32_.MergeFrom(other.mapUint32Uint32_); + mapUint64Uint64_.MergeFrom(other.mapUint64Uint64_); + mapSint32Sint32_.MergeFrom(other.mapSint32Sint32_); + mapSint64Sint64_.MergeFrom(other.mapSint64Sint64_); + mapFixed32Fixed32_.MergeFrom(other.mapFixed32Fixed32_); + mapFixed64Fixed64_.MergeFrom(other.mapFixed64Fixed64_); + mapSfixed32Sfixed32_.MergeFrom(other.mapSfixed32Sfixed32_); + mapSfixed64Sfixed64_.MergeFrom(other.mapSfixed64Sfixed64_); + mapInt32Float_.MergeFrom(other.mapInt32Float_); + mapInt32Double_.MergeFrom(other.mapInt32Double_); + mapBoolBool_.MergeFrom(other.mapBoolBool_); + mapStringString_.MergeFrom(other.mapStringString_); + mapStringBytes_.MergeFrom(other.mapStringBytes_); + mapStringNestedMessage_.MergeFrom(other.mapStringNestedMessage_); + mapStringForeignMessage_.MergeFrom(other.mapStringForeignMessage_); + mapStringNestedEnum_.MergeFrom(other.mapStringNestedEnum_); + mapStringForeignEnum_.MergeFrom(other.mapStringForeignEnum_); if (other.optionalBoolWrapper_ != null) { if (optionalBoolWrapper_ == null || other.OptionalBoolWrapper != false) { OptionalBoolWrapper = other.OptionalBoolWrapper; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/Unittest.cs b/csharp/src/Google.Protobuf.Test.TestProtos/Unittest.cs index c1f43ce0f80c..7f1aca1b92aa 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/Unittest.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/Unittest.cs @@ -24112,7 +24112,7 @@ public void MergeFrom(TestRequiredMap other) { if (other == null) { return; } - foo_.Add(other.foo_); + foo_.MergeFrom(other.foo_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -30708,7 +30708,7 @@ public void MergeFrom(TestHugeFieldNumbers other) { } OptionalGroup.MergeFrom(other.OptionalGroup); } - stringStringMap_.Add(other.stringStringMap_); + stringStringMap_.MergeFrom(other.stringStringMap_); switch (other.OneofFieldCase) { case OneofFieldOneofCase.OneofUint32: OneofUint32 = other.OneofUint32; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssues.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssues.cs index 3fe2dd68abc7..d65a65662f2a 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssues.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssues.cs @@ -56,11 +56,17 @@ static UnittestIssuesReflection() { "dWxhcl9maWVsZBgBIAEoCRIbCg5vcHRpb25hbF9maWVsZBgCIAEoCUgAiAEB", "QhEKD19vcHRpb25hbF9maWVsZCI5ChJPbmVvZldpdGhOb25lRmllbGQSCwoB", "eBgBIAEoCUgAEg4KBG5vbmUYAiABKAlIAEIGCgR0ZXN0IjUKEU9uZW9mV2l0", - "aE5vbmVOYW1lEgsKAXgYASABKAlIABILCgF5GAIgASgJSABCBgoEbm9uZSpV", - "CgxOZWdhdGl2ZUVudW0SFgoSTkVHQVRJVkVfRU5VTV9aRVJPEAASFgoJRml2", - "ZUJlbG93EPv//////////wESFQoITWludXNPbmUQ////////////ASouCg5E", - "ZXByZWNhdGVkRW51bRITCg9ERVBSRUNBVEVEX1pFUk8QABIHCgNvbmUQAUId", - "qgIaVW5pdFRlc3QuSXNzdWVzLlRlc3RQcm90b3NiBnByb3RvMw==")); + "aE5vbmVOYW1lEgsKAXgYASABKAlIABILCgF5GAIgASgJSABCBgoEbm9uZSKT", + "AgoZRGlzYW1iaWd1YXRlQ29tbW9uTWVtYmVycxIjChtkaXNhbWJpZ3VhdGVf", + "Y29tbW9uX21lbWJlcnMYASABKAUSDQoFdHlwZXMYAiABKAUSEgoKZGVzY3Jp", + "cHRvchgDIAEoBRIOCgZlcXVhbHMYBCABKAUSEQoJdG9fc3RyaW5nGAUgASgF", + "EhUKDWdldF9oYXNoX2NvZGUYBiABKAUSEAoId3JpdGVfdG8YByABKAUSDQoF", + "Y2xvbmUYCCABKAUSFgoOY2FsY3VsYXRlX3NpemUYCSABKAUSEgoKbWVyZ2Vf", + "ZnJvbRgKIAEoBRIXCg9vbl9jb25zdHJ1Y3Rpb24YCyABKAUSDgoGcGFyc2Vy", + "GAwgASgFKlUKDE5lZ2F0aXZlRW51bRIWChJORUdBVElWRV9FTlVNX1pFUk8Q", + "ABIWCglGaXZlQmVsb3cQ+///////////ARIVCghNaW51c09uZRD/////////", + "//8BKi4KDkRlcHJlY2F0ZWRFbnVtEhMKD0RFUFJFQ0FURURfWkVSTxAAEgcK", + "A29uZRABQh2qAhpVbml0VGVzdC5Jc3N1ZXMuVGVzdFByb3Rvc2IGcHJvdG8z")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { global::Google.Protobuf.WellKnownTypes.StructReflection.Descriptor, }, new pbr::GeneratedClrTypeInfo(new[] {typeof(global::UnitTest.Issues.TestProtos.NegativeEnum), typeof(global::UnitTest.Issues.TestProtos.DeprecatedEnum), }, null, new pbr::GeneratedClrTypeInfo[] { @@ -77,7 +83,8 @@ static UnittestIssuesReflection() { new pbr::GeneratedClrTypeInfo(typeof(global::UnitTest.Issues.TestProtos.NullValueNotInOneof), global::UnitTest.Issues.TestProtos.NullValueNotInOneof.Parser, new[]{ "NullValue" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::UnitTest.Issues.TestProtos.MixedRegularAndOptional), global::UnitTest.Issues.TestProtos.MixedRegularAndOptional.Parser, new[]{ "RegularField", "OptionalField" }, new[]{ "OptionalField" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::UnitTest.Issues.TestProtos.OneofWithNoneField), global::UnitTest.Issues.TestProtos.OneofWithNoneField.Parser, new[]{ "X", "None" }, new[]{ "Test" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::UnitTest.Issues.TestProtos.OneofWithNoneName), global::UnitTest.Issues.TestProtos.OneofWithNoneName.Parser, new[]{ "X", "Y" }, new[]{ "None" }, null, null, null) + new pbr::GeneratedClrTypeInfo(typeof(global::UnitTest.Issues.TestProtos.OneofWithNoneName), global::UnitTest.Issues.TestProtos.OneofWithNoneName.Parser, new[]{ "X", "Y" }, new[]{ "None" }, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::UnitTest.Issues.TestProtos.DisambiguateCommonMembers), global::UnitTest.Issues.TestProtos.DisambiguateCommonMembers.Parser, new[]{ "DisambiguateCommonMembers_", "Types_", "Descriptor_", "Equals_", "ToString_", "GetHashCode_", "WriteTo_", "Clone_", "CalculateSize_", "MergeFrom_", "OnConstruction_", "Parser_" }, null, null, null, null) })); } #endregion @@ -4347,6 +4354,605 @@ public void MergeFrom(pb::CodedInputStream input) { } + /// + /// Issue 8810 + /// + public sealed partial class DisambiguateCommonMembers : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new DisambiguateCommonMembers()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::UnitTest.Issues.TestProtos.UnittestIssuesReflection.Descriptor.MessageTypes[14]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public DisambiguateCommonMembers() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public DisambiguateCommonMembers(DisambiguateCommonMembers other) : this() { + disambiguateCommonMembers_ = other.disambiguateCommonMembers_; + types_ = other.types_; + descriptor_ = other.descriptor_; + equals_ = other.equals_; + toString_ = other.toString_; + getHashCode_ = other.getHashCode_; + writeTo_ = other.writeTo_; + clone_ = other.clone_; + calculateSize_ = other.calculateSize_; + mergeFrom_ = other.mergeFrom_; + onConstruction_ = other.onConstruction_; + parser_ = other.parser_; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public DisambiguateCommonMembers Clone() { + return new DisambiguateCommonMembers(this); + } + + /// Field number for the "disambiguate_common_members" field. + public const int DisambiguateCommonMembers_FieldNumber = 1; + private int disambiguateCommonMembers_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int DisambiguateCommonMembers_ { + get { return disambiguateCommonMembers_; } + set { + disambiguateCommonMembers_ = value; + } + } + + /// Field number for the "types" field. + public const int Types_FieldNumber = 2; + private int types_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int Types_ { + get { return types_; } + set { + types_ = value; + } + } + + /// Field number for the "descriptor" field. + public const int Descriptor_FieldNumber = 3; + private int descriptor_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int Descriptor_ { + get { return descriptor_; } + set { + descriptor_ = value; + } + } + + /// Field number for the "equals" field. + public const int Equals_FieldNumber = 4; + private int equals_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int Equals_ { + get { return equals_; } + set { + equals_ = value; + } + } + + /// Field number for the "to_string" field. + public const int ToString_FieldNumber = 5; + private int toString_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int ToString_ { + get { return toString_; } + set { + toString_ = value; + } + } + + /// Field number for the "get_hash_code" field. + public const int GetHashCode_FieldNumber = 6; + private int getHashCode_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int GetHashCode_ { + get { return getHashCode_; } + set { + getHashCode_ = value; + } + } + + /// Field number for the "write_to" field. + public const int WriteTo_FieldNumber = 7; + private int writeTo_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int WriteTo_ { + get { return writeTo_; } + set { + writeTo_ = value; + } + } + + /// Field number for the "clone" field. + public const int Clone_FieldNumber = 8; + private int clone_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int Clone_ { + get { return clone_; } + set { + clone_ = value; + } + } + + /// Field number for the "calculate_size" field. + public const int CalculateSize_FieldNumber = 9; + private int calculateSize_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize_ { + get { return calculateSize_; } + set { + calculateSize_ = value; + } + } + + /// Field number for the "merge_from" field. + public const int MergeFrom_FieldNumber = 10; + private int mergeFrom_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int MergeFrom_ { + get { return mergeFrom_; } + set { + mergeFrom_ = value; + } + } + + /// Field number for the "on_construction" field. + public const int OnConstruction_FieldNumber = 11; + private int onConstruction_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int OnConstruction_ { + get { return onConstruction_; } + set { + onConstruction_ = value; + } + } + + /// Field number for the "parser" field. + public const int Parser_FieldNumber = 12; + private int parser_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int Parser_ { + get { return parser_; } + set { + parser_ = value; + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as DisambiguateCommonMembers); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(DisambiguateCommonMembers other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (DisambiguateCommonMembers_ != other.DisambiguateCommonMembers_) return false; + if (Types_ != other.Types_) return false; + if (Descriptor_ != other.Descriptor_) return false; + if (Equals_ != other.Equals_) return false; + if (ToString_ != other.ToString_) return false; + if (GetHashCode_ != other.GetHashCode_) return false; + if (WriteTo_ != other.WriteTo_) return false; + if (Clone_ != other.Clone_) return false; + if (CalculateSize_ != other.CalculateSize_) return false; + if (MergeFrom_ != other.MergeFrom_) return false; + if (OnConstruction_ != other.OnConstruction_) return false; + if (Parser_ != other.Parser_) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (DisambiguateCommonMembers_ != 0) hash ^= DisambiguateCommonMembers_.GetHashCode(); + if (Types_ != 0) hash ^= Types_.GetHashCode(); + if (Descriptor_ != 0) hash ^= Descriptor_.GetHashCode(); + if (Equals_ != 0) hash ^= Equals_.GetHashCode(); + if (ToString_ != 0) hash ^= ToString_.GetHashCode(); + if (GetHashCode_ != 0) hash ^= GetHashCode_.GetHashCode(); + if (WriteTo_ != 0) hash ^= WriteTo_.GetHashCode(); + if (Clone_ != 0) hash ^= Clone_.GetHashCode(); + if (CalculateSize_ != 0) hash ^= CalculateSize_.GetHashCode(); + if (MergeFrom_ != 0) hash ^= MergeFrom_.GetHashCode(); + if (OnConstruction_ != 0) hash ^= OnConstruction_.GetHashCode(); + if (Parser_ != 0) hash ^= Parser_.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (DisambiguateCommonMembers_ != 0) { + output.WriteRawTag(8); + output.WriteInt32(DisambiguateCommonMembers_); + } + if (Types_ != 0) { + output.WriteRawTag(16); + output.WriteInt32(Types_); + } + if (Descriptor_ != 0) { + output.WriteRawTag(24); + output.WriteInt32(Descriptor_); + } + if (Equals_ != 0) { + output.WriteRawTag(32); + output.WriteInt32(Equals_); + } + if (ToString_ != 0) { + output.WriteRawTag(40); + output.WriteInt32(ToString_); + } + if (GetHashCode_ != 0) { + output.WriteRawTag(48); + output.WriteInt32(GetHashCode_); + } + if (WriteTo_ != 0) { + output.WriteRawTag(56); + output.WriteInt32(WriteTo_); + } + if (Clone_ != 0) { + output.WriteRawTag(64); + output.WriteInt32(Clone_); + } + if (CalculateSize_ != 0) { + output.WriteRawTag(72); + output.WriteInt32(CalculateSize_); + } + if (MergeFrom_ != 0) { + output.WriteRawTag(80); + output.WriteInt32(MergeFrom_); + } + if (OnConstruction_ != 0) { + output.WriteRawTag(88); + output.WriteInt32(OnConstruction_); + } + if (Parser_ != 0) { + output.WriteRawTag(96); + output.WriteInt32(Parser_); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (DisambiguateCommonMembers_ != 0) { + output.WriteRawTag(8); + output.WriteInt32(DisambiguateCommonMembers_); + } + if (Types_ != 0) { + output.WriteRawTag(16); + output.WriteInt32(Types_); + } + if (Descriptor_ != 0) { + output.WriteRawTag(24); + output.WriteInt32(Descriptor_); + } + if (Equals_ != 0) { + output.WriteRawTag(32); + output.WriteInt32(Equals_); + } + if (ToString_ != 0) { + output.WriteRawTag(40); + output.WriteInt32(ToString_); + } + if (GetHashCode_ != 0) { + output.WriteRawTag(48); + output.WriteInt32(GetHashCode_); + } + if (WriteTo_ != 0) { + output.WriteRawTag(56); + output.WriteInt32(WriteTo_); + } + if (Clone_ != 0) { + output.WriteRawTag(64); + output.WriteInt32(Clone_); + } + if (CalculateSize_ != 0) { + output.WriteRawTag(72); + output.WriteInt32(CalculateSize_); + } + if (MergeFrom_ != 0) { + output.WriteRawTag(80); + output.WriteInt32(MergeFrom_); + } + if (OnConstruction_ != 0) { + output.WriteRawTag(88); + output.WriteInt32(OnConstruction_); + } + if (Parser_ != 0) { + output.WriteRawTag(96); + output.WriteInt32(Parser_); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (DisambiguateCommonMembers_ != 0) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(DisambiguateCommonMembers_); + } + if (Types_ != 0) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(Types_); + } + if (Descriptor_ != 0) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(Descriptor_); + } + if (Equals_ != 0) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(Equals_); + } + if (ToString_ != 0) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(ToString_); + } + if (GetHashCode_ != 0) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(GetHashCode_); + } + if (WriteTo_ != 0) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(WriteTo_); + } + if (Clone_ != 0) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(Clone_); + } + if (CalculateSize_ != 0) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(CalculateSize_); + } + if (MergeFrom_ != 0) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(MergeFrom_); + } + if (OnConstruction_ != 0) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(OnConstruction_); + } + if (Parser_ != 0) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(Parser_); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(DisambiguateCommonMembers other) { + if (other == null) { + return; + } + if (other.DisambiguateCommonMembers_ != 0) { + DisambiguateCommonMembers_ = other.DisambiguateCommonMembers_; + } + if (other.Types_ != 0) { + Types_ = other.Types_; + } + if (other.Descriptor_ != 0) { + Descriptor_ = other.Descriptor_; + } + if (other.Equals_ != 0) { + Equals_ = other.Equals_; + } + if (other.ToString_ != 0) { + ToString_ = other.ToString_; + } + if (other.GetHashCode_ != 0) { + GetHashCode_ = other.GetHashCode_; + } + if (other.WriteTo_ != 0) { + WriteTo_ = other.WriteTo_; + } + if (other.Clone_ != 0) { + Clone_ = other.Clone_; + } + if (other.CalculateSize_ != 0) { + CalculateSize_ = other.CalculateSize_; + } + if (other.MergeFrom_ != 0) { + MergeFrom_ = other.MergeFrom_; + } + if (other.OnConstruction_ != 0) { + OnConstruction_ = other.OnConstruction_; + } + if (other.Parser_ != 0) { + Parser_ = other.Parser_; + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 8: { + DisambiguateCommonMembers_ = input.ReadInt32(); + break; + } + case 16: { + Types_ = input.ReadInt32(); + break; + } + case 24: { + Descriptor_ = input.ReadInt32(); + break; + } + case 32: { + Equals_ = input.ReadInt32(); + break; + } + case 40: { + ToString_ = input.ReadInt32(); + break; + } + case 48: { + GetHashCode_ = input.ReadInt32(); + break; + } + case 56: { + WriteTo_ = input.ReadInt32(); + break; + } + case 64: { + Clone_ = input.ReadInt32(); + break; + } + case 72: { + CalculateSize_ = input.ReadInt32(); + break; + } + case 80: { + MergeFrom_ = input.ReadInt32(); + break; + } + case 88: { + OnConstruction_ = input.ReadInt32(); + break; + } + case 96: { + Parser_ = input.ReadInt32(); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 8: { + DisambiguateCommonMembers_ = input.ReadInt32(); + break; + } + case 16: { + Types_ = input.ReadInt32(); + break; + } + case 24: { + Descriptor_ = input.ReadInt32(); + break; + } + case 32: { + Equals_ = input.ReadInt32(); + break; + } + case 40: { + ToString_ = input.ReadInt32(); + break; + } + case 48: { + GetHashCode_ = input.ReadInt32(); + break; + } + case 56: { + WriteTo_ = input.ReadInt32(); + break; + } + case 64: { + Clone_ = input.ReadInt32(); + break; + } + case 72: { + CalculateSize_ = input.ReadInt32(); + break; + } + case 80: { + MergeFrom_ = input.ReadInt32(); + break; + } + case 88: { + OnConstruction_ = input.ReadInt32(); + break; + } + case 96: { + Parser_ = input.ReadInt32(); + break; + } + } + } + } + #endif + + } + #endregion } diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestWellKnownTypes.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestWellKnownTypes.cs index 3ec8d3556fbd..50b9046a3b1d 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestWellKnownTypes.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestWellKnownTypes.cs @@ -3258,24 +3258,24 @@ public void MergeFrom(MapWellKnownTypes other) { if (other == null) { return; } - anyField_.Add(other.anyField_); - apiField_.Add(other.apiField_); - durationField_.Add(other.durationField_); - emptyField_.Add(other.emptyField_); - fieldMaskField_.Add(other.fieldMaskField_); - sourceContextField_.Add(other.sourceContextField_); - structField_.Add(other.structField_); - timestampField_.Add(other.timestampField_); - typeField_.Add(other.typeField_); - doubleField_.Add(other.doubleField_); - floatField_.Add(other.floatField_); - int64Field_.Add(other.int64Field_); - uint64Field_.Add(other.uint64Field_); - int32Field_.Add(other.int32Field_); - uint32Field_.Add(other.uint32Field_); - boolField_.Add(other.boolField_); - stringField_.Add(other.stringField_); - bytesField_.Add(other.bytesField_); + anyField_.MergeFrom(other.anyField_); + apiField_.MergeFrom(other.apiField_); + durationField_.MergeFrom(other.durationField_); + emptyField_.MergeFrom(other.emptyField_); + fieldMaskField_.MergeFrom(other.fieldMaskField_); + sourceContextField_.MergeFrom(other.sourceContextField_); + structField_.MergeFrom(other.structField_); + timestampField_.MergeFrom(other.timestampField_); + typeField_.MergeFrom(other.typeField_); + doubleField_.MergeFrom(other.doubleField_); + floatField_.MergeFrom(other.floatField_); + int64Field_.MergeFrom(other.int64Field_); + uint64Field_.MergeFrom(other.uint64Field_); + int32Field_.MergeFrom(other.int32Field_); + uint32Field_.MergeFrom(other.uint32Field_); + boolField_.MergeFrom(other.boolField_); + stringField_.MergeFrom(other.stringField_); + bytesField_.MergeFrom(other.bytesField_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } diff --git a/csharp/src/Google.Protobuf.Test/GeneratedMessageTest.cs b/csharp/src/Google.Protobuf.Test/GeneratedMessageTest.cs index 838729195686..17c52491401e 100644 --- a/csharp/src/Google.Protobuf.Test/GeneratedMessageTest.cs +++ b/csharp/src/Google.Protobuf.Test/GeneratedMessageTest.cs @@ -36,6 +36,7 @@ using NUnit.Framework; using System.Linq; using Google.Protobuf.WellKnownTypes; +using Google.Protobuf.Collections; namespace Google.Protobuf { @@ -795,5 +796,44 @@ public void NaNComparisons() EqualityTester.AssertInequality(message1, message2); EqualityTester.AssertEquality(message1, message3); } + + [Test] + [TestCase(false)] + [TestCase(true)] + public void MapFieldMerging(bool direct) + { + var message1 = new TestMap + { + MapStringString = + { + { "x1", "y1" }, + { "common", "message1" } + } + }; + var message2 = new TestMap + { + MapStringString = + { + { "x2", "y2" }, + { "common", "message2" } + } + }; + if (direct) + { + message1.MergeFrom(message2); + } + else + { + message1.MergeFrom(message2.ToByteArray()); + } + + var expected = new MapField + { + { "x1", "y1" }, + { "x2", "y2" }, + { "common", "message2" } + }; + Assert.AreEqual(expected, message1.MapStringString); + } } } \ No newline at end of file diff --git a/csharp/src/Google.Protobuf.Test/JsonFormatterSettingsTest.cs b/csharp/src/Google.Protobuf.Test/JsonFormatterSettingsTest.cs new file mode 100644 index 000000000000..f7ea97c135a2 --- /dev/null +++ b/csharp/src/Google.Protobuf.Test/JsonFormatterSettingsTest.cs @@ -0,0 +1,111 @@ +#region Copyright notice and license + +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#endregion + +using Google.Protobuf.Reflection; +using NUnit.Framework; + +// For WrapInQuotes + +namespace Google.Protobuf +{ + public class JsonFormatterSettingsTest + { + [Test] + public void WithIndentation() + { + var settings = JsonFormatter.Settings.Default.WithIndentation("\t"); + Assert.AreEqual("\t", settings.Indentation); + } + + [Test] + public void WithTypeRegistry() + { + var typeRegistry = TypeRegistry.Empty; + var settings = JsonFormatter.Settings.Default.WithTypeRegistry(typeRegistry); + Assert.AreEqual(typeRegistry, settings.TypeRegistry); + } + + [Test] + public void WithFormatDefaultValues() + { + var settingsWith = JsonFormatter.Settings.Default.WithFormatDefaultValues(true); + Assert.AreEqual(true, settingsWith.FormatDefaultValues); + + var settingsWithout = JsonFormatter.Settings.Default.WithFormatDefaultValues(false); + Assert.AreEqual(false, settingsWithout.FormatDefaultValues); + } + + [Test] + public void WithFormatEnumsAsIntegers() + { + var settingsWith = JsonFormatter.Settings.Default.WithFormatEnumsAsIntegers(true); + Assert.AreEqual(true, settingsWith.FormatEnumsAsIntegers); + + var settingsWithout = JsonFormatter.Settings.Default.WithFormatEnumsAsIntegers(false); + Assert.AreEqual(false, settingsWithout.FormatEnumsAsIntegers); + } + + [Test] + public void WithMethodsPreserveExistingSettings() + { + var typeRegistry = TypeRegistry.Empty; + var baseSettings = JsonFormatter.Settings.Default + .WithIndentation("\t") + .WithFormatDefaultValues(true) + .WithFormatEnumsAsIntegers(true) + .WithTypeRegistry(typeRegistry) + .WithPreserveProtoFieldNames(true); + + var settings1 = baseSettings.WithIndentation("\t"); + var settings2 = baseSettings.WithFormatDefaultValues(true); + var settings3 = baseSettings.WithFormatEnumsAsIntegers(true); + var settings4 = baseSettings.WithTypeRegistry(typeRegistry); + var settings5 = baseSettings.WithPreserveProtoFieldNames(true); + + AssertAreEqual(baseSettings, settings1); + AssertAreEqual(baseSettings, settings2); + AssertAreEqual(baseSettings, settings3); + AssertAreEqual(baseSettings, settings4); + AssertAreEqual(baseSettings, settings5); + } + + private static void AssertAreEqual(JsonFormatter.Settings settings, JsonFormatter.Settings other) + { + Assert.AreEqual(settings.Indentation, other.Indentation); + Assert.AreEqual(settings.FormatDefaultValues, other.FormatDefaultValues); + Assert.AreEqual(settings.FormatEnumsAsIntegers, other.FormatEnumsAsIntegers); + Assert.AreEqual(settings.TypeRegistry, other.TypeRegistry); + } + } +} diff --git a/csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs b/csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs index 714c78cbeb7c..f4dfde243590 100644 --- a/csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs +++ b/csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs @@ -674,6 +674,200 @@ public void WriteValue_List() AssertWriteValue(value, "[ 1, 2, 3 ]"); } + [Test] + public void WriteValueWithIndentation_EmptyMessage() + { + var value = new TestEmptyMessage(); + + AssertWriteValue(value, "{}", JsonFormatter.Settings.Default.WithIndentation()); + } + + [Test] + public void WriteValueWithIndentation_NestedTestAllTypes() + { + var value = new NestedTestAllTypes + { + Payload = new TestAllTypes + { + SingleBool = true, + SingleInt32 = 100, + SingleString = "multiple fields", + RepeatedString = { "string1", "string2" }, + }, + Child = new NestedTestAllTypes + { + Payload = new TestAllTypes + { + SingleString = "single field", + }, + }, + RepeatedChild = + { + new NestedTestAllTypes { Payload = new TestAllTypes { SingleString = "child 1", RepeatedString = { "string" } } }, + new NestedTestAllTypes { Payload = new TestAllTypes { SingleString = "child 2" } }, + }, + }; + + const string expectedJson = @" +{ + 'child': { + 'payload': { + 'singleString': 'single field' + } + }, + 'payload': { + 'singleInt32': 100, + 'singleBool': true, + 'singleString': 'multiple fields', + 'repeatedString': [ + 'string1', + 'string2' + ] + }, + 'repeatedChild': [ + { + 'payload': { + 'singleString': 'child 1', + 'repeatedString': [ + 'string' + ] + } + }, + { + 'payload': { + 'singleString': 'child 2' + } + } + ] +}"; + AssertWriteValue(value, expectedJson, JsonFormatter.Settings.Default.WithIndentation()); + } + + [Test] + public void WriteValueWithIndentation_WellKnownTypes() + { + var value = new TestWellKnownTypes + { + StructField = new Struct + { + Fields = + { + { "string", Value.ForString("foo") }, + { "numbers", Value.ForList(Value.ForNumber(1), Value.ForNumber(2), Value.ForNumber(3)) }, + { "emptyList", Value.ForList() }, + { "emptyStruct", Value.ForStruct(new Struct()) }, + }, + }, + }; + + const string expectedJson = @" +{ + 'structField': { + 'string': 'foo', + 'numbers': [ + 1, + 2, + 3 + ], + 'emptyList': [], + 'emptyStruct': {} + } +}"; + AssertWriteValue(value, expectedJson, JsonFormatter.Settings.Default.WithIndentation()); + } + + [Test] + public void WriteValueWithIndentation_StructSingleField() + { + var value = new Struct { Fields = { { "structField1", Value.ForString("structFieldValue1") } } }; + + const string expectedJson = @" +{ + 'structField1': 'structFieldValue1' +}"; + AssertWriteValue(value, expectedJson, JsonFormatter.Settings.Default.WithIndentation()); + } + + [Test] + public void WriteValueWithIndentation_StructMultipleFields() + { + var value = new Struct + { + Fields = + { + { "structField1", Value.ForString("structFieldValue1") }, + { "structField2", Value.ForString("structFieldValue2") }, + { "structField3", Value.ForString("structFieldValue3") }, + }, + }; + + const string expectedJson = @" +{ + 'structField1': 'structFieldValue1', + 'structField2': 'structFieldValue2', + 'structField3': 'structFieldValue3' +}"; + AssertWriteValue(value, expectedJson, JsonFormatter.Settings.Default.WithIndentation()); + } + + [Test] + public void FormatWithIndentation_EmbeddedMessage() + { + var value = new TestAllTypes { SingleInt32 = 100, SingleInt64 = 3210987654321L }; + var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithIndentation()); + var valueJson = formatter.Format(value, indentationLevel: 1); + + var actualJson = $@" +{{ + ""data"": {valueJson} +}}"; + const string expectedJson = @" +{ + 'data': { + 'singleInt32': 100, + 'singleInt64': '3210987654321' + } +}"; + AssertJson(expectedJson, actualJson.Trim()); + } + + [Test] + public void WriteValueWithIndentation_Map() + { + var value = new TestMap + { + MapStringString = + { + { "key1", "value1" }, + { "key2", "value2" }, + }, + }; + + const string expectedJson = @" +{ + 'mapStringString': { + 'key1': 'value1', + 'key2': 'value2' + } +}"; + + AssertWriteValue(value, expectedJson, JsonFormatter.Settings.Default.WithIndentation()); + } + + [Test] + public void WriteValueWithIndentation_List() + { + var value = new RepeatedField { 1, 2, 3 }; + AssertWriteValue(value, "[\n 1,\n 2,\n 3\n]", JsonFormatter.Settings.Default.WithIndentation()); + } + + [Test] + public void WriteValueWithIndentation_CustomIndentation() + { + var value = new RepeatedField { 1, 2, 3 }; + AssertWriteValue(value, "[\n\t1,\n\t2,\n\t3\n]", JsonFormatter.Settings.Default.WithIndentation("\t")); + } + [Test] public void Proto2_DefaultValuesWritten() { @@ -683,7 +877,7 @@ public void Proto2_DefaultValuesWritten() private static void AssertWriteValue(object value, string expectedJson, JsonFormatter.Settings settings = null) { - var writer = new StringWriter(); + var writer = new StringWriter { NewLine = "\n" }; new JsonFormatter(settings ?? JsonFormatter.Settings.Default).WriteValue(writer, value); string actual = writer.ToString(); AssertJson(expectedJson, actual); @@ -691,13 +885,17 @@ private static void AssertWriteValue(object value, string expectedJson, JsonForm /// /// Checks that the actual JSON is the same as the expected JSON - but after replacing - /// all apostrophes in the expected JSON with double quotes. This basically makes the tests easier - /// to read. + /// all apostrophes in the expected JSON with double quotes, trimming leading whitespace and normalizing new lines. + /// This basically makes the tests easier to read. /// + /// + /// Line endings are normalized because indented JSON strings are generated with system-specific line endings, + /// while line endings in the test cases are hard-coded, but may be converted during source checkout, depending + /// on git settings, causing unpredictability in the test results otherwise. private static void AssertJson(string expectedJsonWithApostrophes, string actualJson) { - var expectedJson = expectedJsonWithApostrophes.Replace("'", "\""); - Assert.AreEqual(expectedJson, actualJson); + var expectedJson = expectedJsonWithApostrophes.Replace("'", "\"").Replace("\r\n", "\n").TrimStart(); + Assert.AreEqual(expectedJson, actualJson.Replace("\r\n", "\n")); } } } diff --git a/csharp/src/Google.Protobuf.Test/ParsingPrimitivesTest.cs b/csharp/src/Google.Protobuf.Test/ParsingPrimitivesTest.cs new file mode 100644 index 000000000000..4d0aa9e015f3 --- /dev/null +++ b/csharp/src/Google.Protobuf.Test/ParsingPrimitivesTest.cs @@ -0,0 +1,63 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2022 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +using NUnit.Framework; +using System; +using System.Linq; + +namespace Google.Protobuf.Test; + +internal class ParsingPrimitivesTest +{ + // Note: test cases use integers rather than bytes as they're easier + // to specify in attributes. + + [Test] + [TestCase("\ufffd", 255)] + [TestCase("A\ufffd", 65, 255)] + [TestCase("A\ufffd\ufffdB", 65, 255, 255, 66)] + // Overlong form of "space" + [TestCase("\ufffd\ufffd", 0xc0, 0xa0)] + public void ReadRawString_NonUtf8(string expectedText, params int[] bytes) + { + var context = CreateContext(bytes); + string text = ParsingPrimitives.ReadRawString(ref context.buffer, ref context.state, bytes.Length); + Assert.AreEqual(expectedText, text); + } + + private static ParseContext CreateContext(int[] bytes) + { + byte[] actualBytes = bytes.Select(b => (byte) b).ToArray(); + ParseContext.Initialize(actualBytes.AsSpan(), out var context); + return context; + } +} diff --git a/csharp/src/Google.Protobuf.Test/WritingPrimitivesTest.cs b/csharp/src/Google.Protobuf.Test/WritingPrimitivesTest.cs new file mode 100644 index 000000000000..069df343fd77 --- /dev/null +++ b/csharp/src/Google.Protobuf.Test/WritingPrimitivesTest.cs @@ -0,0 +1,61 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2022 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Google.Protobuf.Test; + +internal class WritingPrimitivesTest +{ + [Test] + public void WriteRawString_IllFormedUnicodeString() + { + // See https://codeblog.jonskeet.uk/2014/11/07/when-is-a-string-not-a-string/ + char c1 = '\u0058'; + char c2 = '\ud800'; + char c3 = '\u0059'; + string text = new string(new[] { c1, c2, c3 }); + Span buffer = new byte[10]; + WriteContext.Initialize(ref buffer, out var context); + WritingPrimitives.WriteString(ref context.buffer, ref context.state, text); + + // The high surrogate is written out in a "raw" form, surrounded by the ASCII + // characters. + byte[] expectedBytes = { 0x5, 0x58, 0xef, 0xbf, 0xbd, 0x59 }; + Assert.AreEqual(expectedBytes, buffer.Slice(0, context.state.position).ToArray()); + } +} diff --git a/csharp/src/Google.Protobuf.Test/testprotos.pb b/csharp/src/Google.Protobuf.Test/testprotos.pb index 5f41bc4da3705d59ff536bc23bf35549564b1afb..bdfe1cce5a5d996252548abe5d1b636be71a6c25 100644 GIT binary patch delta 1228 zcmY+?J#W)M7zgk?=d*pjywF^e)=8;U%1b-(B9=}FsR+TqLfNAr+vLx~{?F~d zmOuZ+;FU*yTn|?LX1AVr%V7|N?Pp%F>P4}5z-QJjm9>U!Yr(j7VGb6}=atJV|N9a* zlQ$hNcFE?i-D+yceZJDQB=Xx$*OV1Y!WEgt zJkK}1q_$p<*YSFRU6;E z+erMd?PldpA#C4RzBC^F^iDmBy~xeU1(i0x_rEz`Z*yCinapWF7~CnPyvhkJ2x|v2 zrvTO5+|xMjdUH!l_wI6@@f@eLKr({v0$JpFZjO>Eqpc(E?gG26qcMuN z78J#@2Slh6koF)Y+JdZEUe{=QFi}*N2)YMkT9pJXMokIQJz)CI1nC}>rxeAs4`fnN z11UyLQv>Zju!64Tc2Oux1?>ZwQYA%;QCo_%53HzLqG&K?v;%M}%36XR04XbLp~a}Jh4uj0gl>@@z+_2LThPOcL$;A()YL|L2B+^X%_2R4;^eB;iNYkj a&wqb1`1`H14_jw|7s1OIZk^HB@BRT^yV?o> delta 57 zcmdlyS#)8S=!Si3n4Xw#KCtF5+vKFJ%Q?4caIp)qGBB*(EVNBuq*?E5yWUwwAZ7w$ L=IwfCSz2oV(6JYP diff --git a/csharp/src/Google.Protobuf/Collections/MapField.cs b/csharp/src/Google.Protobuf/Collections/MapField.cs index f0124ee12b1a..09afb75cdb11 100644 --- a/csharp/src/Google.Protobuf/Collections/MapField.cs +++ b/csharp/src/Google.Protobuf/Collections/MapField.cs @@ -237,6 +237,21 @@ public void Add(IDictionary entries) } } + /// + /// Adds the specified entries to the map, replacing any existing entries with the same keys. + /// The keys and values are not automatically cloned. + /// + /// This method primarily exists to be called from MergeFrom methods in generated classes for messages. + /// The entries to add to the map. + public void MergeFrom(IDictionary entries) + { + ProtoPreconditions.CheckNotNull(entries, nameof(entries)); + foreach (var pair in entries) + { + this[pair.Key] = pair.Value; + } + } + /// /// Returns an enumerator that iterates through the collection. /// diff --git a/csharp/src/Google.Protobuf/Compiler/Plugin.cs b/csharp/src/Google.Protobuf/Compiler/Plugin.cs new file mode 100644 index 000000000000..f3a6c171de21 --- /dev/null +++ b/csharp/src/Google.Protobuf/Compiler/Plugin.cs @@ -0,0 +1,1490 @@ +// +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: google/protobuf/compiler/plugin.proto +// +#pragma warning disable 1591, 0612, 3021, 8981 +#region Designer generated code + +using pb = global::Google.Protobuf; +using pbc = global::Google.Protobuf.Collections; +using pbr = global::Google.Protobuf.Reflection; +using scg = global::System.Collections.Generic; +namespace Google.Protobuf.Compiler { + + /// Holder for reflection information generated from google/protobuf/compiler/plugin.proto + public static partial class PluginReflection { + + #region Descriptor + /// File descriptor for google/protobuf/compiler/plugin.proto + public static pbr::FileDescriptor Descriptor { + get { return descriptor; } + } + private static pbr::FileDescriptor descriptor; + + static PluginReflection() { + byte[] descriptorData = global::System.Convert.FromBase64String( + string.Concat( + "CiVnb29nbGUvcHJvdG9idWYvY29tcGlsZXIvcGx1Z2luLnByb3RvEhhnb29n", + "bGUucHJvdG9idWYuY29tcGlsZXIaIGdvb2dsZS9wcm90b2J1Zi9kZXNjcmlw", + "dG9yLnByb3RvIkYKB1ZlcnNpb24SDQoFbWFqb3IYASABKAUSDQoFbWlub3IY", + "AiABKAUSDQoFcGF0Y2gYAyABKAUSDgoGc3VmZml4GAQgASgJIroBChRDb2Rl", + "R2VuZXJhdG9yUmVxdWVzdBIYChBmaWxlX3RvX2dlbmVyYXRlGAEgAygJEhEK", + "CXBhcmFtZXRlchgCIAEoCRI4Cgpwcm90b19maWxlGA8gAygLMiQuZ29vZ2xl", + "LnByb3RvYnVmLkZpbGVEZXNjcmlwdG9yUHJvdG8SOwoQY29tcGlsZXJfdmVy", + "c2lvbhgDIAEoCzIhLmdvb2dsZS5wcm90b2J1Zi5jb21waWxlci5WZXJzaW9u", + "IsECChVDb2RlR2VuZXJhdG9yUmVzcG9uc2USDQoFZXJyb3IYASABKAkSGgoS", + "c3VwcG9ydGVkX2ZlYXR1cmVzGAIgASgEEkIKBGZpbGUYDyADKAsyNC5nb29n", + "bGUucHJvdG9idWYuY29tcGlsZXIuQ29kZUdlbmVyYXRvclJlc3BvbnNlLkZp", + "bGUafwoERmlsZRIMCgRuYW1lGAEgASgJEhcKD2luc2VydGlvbl9wb2ludBgC", + "IAEoCRIPCgdjb250ZW50GA8gASgJEj8KE2dlbmVyYXRlZF9jb2RlX2luZm8Y", + "ECABKAsyIi5nb29nbGUucHJvdG9idWYuR2VuZXJhdGVkQ29kZUluZm8iOAoH", + "RmVhdHVyZRIQCgxGRUFUVVJFX05PTkUQABIbChdGRUFUVVJFX1BST1RPM19P", + "UFRJT05BTBABQlcKHGNvbS5nb29nbGUucHJvdG9idWYuY29tcGlsZXJCDFBs", + "dWdpblByb3Rvc1opZ29vZ2xlLmdvbGFuZy5vcmcvcHJvdG9idWYvdHlwZXMv", + "cGx1Z2lucGI=")); + descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, + new pbr::FileDescriptor[] { global::Google.Protobuf.Reflection.DescriptorReflection.Descriptor, }, + new pbr::GeneratedClrTypeInfo(null, null, new pbr::GeneratedClrTypeInfo[] { + new pbr::GeneratedClrTypeInfo(typeof(global::Google.Protobuf.Compiler.Version), global::Google.Protobuf.Compiler.Version.Parser, new[]{ "Major", "Minor", "Patch", "Suffix" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Google.Protobuf.Compiler.CodeGeneratorRequest), global::Google.Protobuf.Compiler.CodeGeneratorRequest.Parser, new[]{ "FileToGenerate", "Parameter", "ProtoFile", "CompilerVersion" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Google.Protobuf.Compiler.CodeGeneratorResponse), global::Google.Protobuf.Compiler.CodeGeneratorResponse.Parser, new[]{ "Error", "SupportedFeatures", "File" }, null, new[]{ typeof(global::Google.Protobuf.Compiler.CodeGeneratorResponse.Types.Feature) }, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Google.Protobuf.Compiler.CodeGeneratorResponse.Types.File), global::Google.Protobuf.Compiler.CodeGeneratorResponse.Types.File.Parser, new[]{ "Name", "InsertionPoint", "Content", "GeneratedCodeInfo" }, null, null, null, null)}) + })); + } + #endregion + + } + #region Messages + /// + /// The version number of protocol compiler. + /// + public sealed partial class Version : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new Version()); + private pb::UnknownFieldSet _unknownFields; + private int _hasBits0; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::Google.Protobuf.Compiler.PluginReflection.Descriptor.MessageTypes[0]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public Version() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public Version(Version other) : this() { + _hasBits0 = other._hasBits0; + major_ = other.major_; + minor_ = other.minor_; + patch_ = other.patch_; + suffix_ = other.suffix_; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public Version Clone() { + return new Version(this); + } + + /// Field number for the "major" field. + public const int MajorFieldNumber = 1; + private readonly static int MajorDefaultValue = 0; + + private int major_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int Major { + get { if ((_hasBits0 & 1) != 0) { return major_; } else { return MajorDefaultValue; } } + set { + _hasBits0 |= 1; + major_ = value; + } + } + /// Gets whether the "major" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasMajor { + get { return (_hasBits0 & 1) != 0; } + } + /// Clears the value of the "major" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearMajor() { + _hasBits0 &= ~1; + } + + /// Field number for the "minor" field. + public const int MinorFieldNumber = 2; + private readonly static int MinorDefaultValue = 0; + + private int minor_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int Minor { + get { if ((_hasBits0 & 2) != 0) { return minor_; } else { return MinorDefaultValue; } } + set { + _hasBits0 |= 2; + minor_ = value; + } + } + /// Gets whether the "minor" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasMinor { + get { return (_hasBits0 & 2) != 0; } + } + /// Clears the value of the "minor" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearMinor() { + _hasBits0 &= ~2; + } + + /// Field number for the "patch" field. + public const int PatchFieldNumber = 3; + private readonly static int PatchDefaultValue = 0; + + private int patch_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int Patch { + get { if ((_hasBits0 & 4) != 0) { return patch_; } else { return PatchDefaultValue; } } + set { + _hasBits0 |= 4; + patch_ = value; + } + } + /// Gets whether the "patch" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasPatch { + get { return (_hasBits0 & 4) != 0; } + } + /// Clears the value of the "patch" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearPatch() { + _hasBits0 &= ~4; + } + + /// Field number for the "suffix" field. + public const int SuffixFieldNumber = 4; + private readonly static string SuffixDefaultValue = ""; + + private string suffix_; + /// + /// A suffix for alpha, beta or rc release, e.g., "alpha-1", "rc2". It should + /// be empty for mainline stable releases. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string Suffix { + get { return suffix_ ?? SuffixDefaultValue; } + set { + suffix_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "suffix" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasSuffix { + get { return suffix_ != null; } + } + /// Clears the value of the "suffix" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearSuffix() { + suffix_ = null; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as Version); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(Version other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (Major != other.Major) return false; + if (Minor != other.Minor) return false; + if (Patch != other.Patch) return false; + if (Suffix != other.Suffix) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (HasMajor) hash ^= Major.GetHashCode(); + if (HasMinor) hash ^= Minor.GetHashCode(); + if (HasPatch) hash ^= Patch.GetHashCode(); + if (HasSuffix) hash ^= Suffix.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (HasMajor) { + output.WriteRawTag(8); + output.WriteInt32(Major); + } + if (HasMinor) { + output.WriteRawTag(16); + output.WriteInt32(Minor); + } + if (HasPatch) { + output.WriteRawTag(24); + output.WriteInt32(Patch); + } + if (HasSuffix) { + output.WriteRawTag(34); + output.WriteString(Suffix); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (HasMajor) { + output.WriteRawTag(8); + output.WriteInt32(Major); + } + if (HasMinor) { + output.WriteRawTag(16); + output.WriteInt32(Minor); + } + if (HasPatch) { + output.WriteRawTag(24); + output.WriteInt32(Patch); + } + if (HasSuffix) { + output.WriteRawTag(34); + output.WriteString(Suffix); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (HasMajor) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(Major); + } + if (HasMinor) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(Minor); + } + if (HasPatch) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(Patch); + } + if (HasSuffix) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Suffix); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(Version other) { + if (other == null) { + return; + } + if (other.HasMajor) { + Major = other.Major; + } + if (other.HasMinor) { + Minor = other.Minor; + } + if (other.HasPatch) { + Patch = other.Patch; + } + if (other.HasSuffix) { + Suffix = other.Suffix; + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 8: { + Major = input.ReadInt32(); + break; + } + case 16: { + Minor = input.ReadInt32(); + break; + } + case 24: { + Patch = input.ReadInt32(); + break; + } + case 34: { + Suffix = input.ReadString(); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 8: { + Major = input.ReadInt32(); + break; + } + case 16: { + Minor = input.ReadInt32(); + break; + } + case 24: { + Patch = input.ReadInt32(); + break; + } + case 34: { + Suffix = input.ReadString(); + break; + } + } + } + } + #endif + + } + + /// + /// An encoded CodeGeneratorRequest is written to the plugin's stdin. + /// + public sealed partial class CodeGeneratorRequest : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new CodeGeneratorRequest()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::Google.Protobuf.Compiler.PluginReflection.Descriptor.MessageTypes[1]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public CodeGeneratorRequest() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public CodeGeneratorRequest(CodeGeneratorRequest other) : this() { + fileToGenerate_ = other.fileToGenerate_.Clone(); + parameter_ = other.parameter_; + protoFile_ = other.protoFile_.Clone(); + compilerVersion_ = other.compilerVersion_ != null ? other.compilerVersion_.Clone() : null; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public CodeGeneratorRequest Clone() { + return new CodeGeneratorRequest(this); + } + + /// Field number for the "file_to_generate" field. + public const int FileToGenerateFieldNumber = 1; + private static readonly pb::FieldCodec _repeated_fileToGenerate_codec + = pb::FieldCodec.ForString(10); + private readonly pbc::RepeatedField fileToGenerate_ = new pbc::RepeatedField(); + /// + /// The .proto files that were explicitly listed on the command-line. The + /// code generator should generate code only for these files. Each file's + /// descriptor will be included in proto_file, below. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public pbc::RepeatedField FileToGenerate { + get { return fileToGenerate_; } + } + + /// Field number for the "parameter" field. + public const int ParameterFieldNumber = 2; + private readonly static string ParameterDefaultValue = ""; + + private string parameter_; + /// + /// The generator parameter passed on the command-line. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string Parameter { + get { return parameter_ ?? ParameterDefaultValue; } + set { + parameter_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "parameter" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasParameter { + get { return parameter_ != null; } + } + /// Clears the value of the "parameter" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearParameter() { + parameter_ = null; + } + + /// Field number for the "proto_file" field. + public const int ProtoFileFieldNumber = 15; + private static readonly pb::FieldCodec _repeated_protoFile_codec + = pb::FieldCodec.ForMessage(122, global::Google.Protobuf.Reflection.FileDescriptorProto.Parser); + private readonly pbc::RepeatedField protoFile_ = new pbc::RepeatedField(); + /// + /// FileDescriptorProtos for all files in files_to_generate and everything + /// they import. The files will appear in topological order, so each file + /// appears before any file that imports it. + /// + /// protoc guarantees that all proto_files will be written after + /// the fields above, even though this is not technically guaranteed by the + /// protobuf wire format. This theoretically could allow a plugin to stream + /// in the FileDescriptorProtos and handle them one by one rather than read + /// the entire set into memory at once. However, as of this writing, this + /// is not similarly optimized on protoc's end -- it will store all fields in + /// memory at once before sending them to the plugin. + /// + /// Type names of fields and extensions in the FileDescriptorProto are always + /// fully qualified. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public pbc::RepeatedField ProtoFile { + get { return protoFile_; } + } + + /// Field number for the "compiler_version" field. + public const int CompilerVersionFieldNumber = 3; + private global::Google.Protobuf.Compiler.Version compilerVersion_; + /// + /// The version number of protocol compiler. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::Google.Protobuf.Compiler.Version CompilerVersion { + get { return compilerVersion_; } + set { + compilerVersion_ = value; + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as CodeGeneratorRequest); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(CodeGeneratorRequest other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if(!fileToGenerate_.Equals(other.fileToGenerate_)) return false; + if (Parameter != other.Parameter) return false; + if(!protoFile_.Equals(other.protoFile_)) return false; + if (!object.Equals(CompilerVersion, other.CompilerVersion)) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + hash ^= fileToGenerate_.GetHashCode(); + if (HasParameter) hash ^= Parameter.GetHashCode(); + hash ^= protoFile_.GetHashCode(); + if (compilerVersion_ != null) hash ^= CompilerVersion.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + fileToGenerate_.WriteTo(output, _repeated_fileToGenerate_codec); + if (HasParameter) { + output.WriteRawTag(18); + output.WriteString(Parameter); + } + if (compilerVersion_ != null) { + output.WriteRawTag(26); + output.WriteMessage(CompilerVersion); + } + protoFile_.WriteTo(output, _repeated_protoFile_codec); + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + fileToGenerate_.WriteTo(ref output, _repeated_fileToGenerate_codec); + if (HasParameter) { + output.WriteRawTag(18); + output.WriteString(Parameter); + } + if (compilerVersion_ != null) { + output.WriteRawTag(26); + output.WriteMessage(CompilerVersion); + } + protoFile_.WriteTo(ref output, _repeated_protoFile_codec); + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + size += fileToGenerate_.CalculateSize(_repeated_fileToGenerate_codec); + if (HasParameter) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Parameter); + } + size += protoFile_.CalculateSize(_repeated_protoFile_codec); + if (compilerVersion_ != null) { + size += 1 + pb::CodedOutputStream.ComputeMessageSize(CompilerVersion); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(CodeGeneratorRequest other) { + if (other == null) { + return; + } + fileToGenerate_.Add(other.fileToGenerate_); + if (other.HasParameter) { + Parameter = other.Parameter; + } + protoFile_.Add(other.protoFile_); + if (other.compilerVersion_ != null) { + if (compilerVersion_ == null) { + CompilerVersion = new global::Google.Protobuf.Compiler.Version(); + } + CompilerVersion.MergeFrom(other.CompilerVersion); + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + fileToGenerate_.AddEntriesFrom(input, _repeated_fileToGenerate_codec); + break; + } + case 18: { + Parameter = input.ReadString(); + break; + } + case 26: { + if (compilerVersion_ == null) { + CompilerVersion = new global::Google.Protobuf.Compiler.Version(); + } + input.ReadMessage(CompilerVersion); + break; + } + case 122: { + protoFile_.AddEntriesFrom(input, _repeated_protoFile_codec); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 10: { + fileToGenerate_.AddEntriesFrom(ref input, _repeated_fileToGenerate_codec); + break; + } + case 18: { + Parameter = input.ReadString(); + break; + } + case 26: { + if (compilerVersion_ == null) { + CompilerVersion = new global::Google.Protobuf.Compiler.Version(); + } + input.ReadMessage(CompilerVersion); + break; + } + case 122: { + protoFile_.AddEntriesFrom(ref input, _repeated_protoFile_codec); + break; + } + } + } + } + #endif + + } + + /// + /// The plugin writes an encoded CodeGeneratorResponse to stdout. + /// + public sealed partial class CodeGeneratorResponse : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new CodeGeneratorResponse()); + private pb::UnknownFieldSet _unknownFields; + private int _hasBits0; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::Google.Protobuf.Compiler.PluginReflection.Descriptor.MessageTypes[2]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public CodeGeneratorResponse() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public CodeGeneratorResponse(CodeGeneratorResponse other) : this() { + _hasBits0 = other._hasBits0; + error_ = other.error_; + supportedFeatures_ = other.supportedFeatures_; + file_ = other.file_.Clone(); + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public CodeGeneratorResponse Clone() { + return new CodeGeneratorResponse(this); + } + + /// Field number for the "error" field. + public const int ErrorFieldNumber = 1; + private readonly static string ErrorDefaultValue = ""; + + private string error_; + /// + /// Error message. If non-empty, code generation failed. The plugin process + /// should exit with status code zero even if it reports an error in this way. + /// + /// This should be used to indicate errors in .proto files which prevent the + /// code generator from generating correct code. Errors which indicate a + /// problem in protoc itself -- such as the input CodeGeneratorRequest being + /// unparseable -- should be reported by writing a message to stderr and + /// exiting with a non-zero status code. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string Error { + get { return error_ ?? ErrorDefaultValue; } + set { + error_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "error" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasError { + get { return error_ != null; } + } + /// Clears the value of the "error" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearError() { + error_ = null; + } + + /// Field number for the "supported_features" field. + public const int SupportedFeaturesFieldNumber = 2; + private readonly static ulong SupportedFeaturesDefaultValue = 0UL; + + private ulong supportedFeatures_; + /// + /// A bitmask of supported features that the code generator supports. + /// This is a bitwise "or" of values from the Feature enum. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public ulong SupportedFeatures { + get { if ((_hasBits0 & 1) != 0) { return supportedFeatures_; } else { return SupportedFeaturesDefaultValue; } } + set { + _hasBits0 |= 1; + supportedFeatures_ = value; + } + } + /// Gets whether the "supported_features" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasSupportedFeatures { + get { return (_hasBits0 & 1) != 0; } + } + /// Clears the value of the "supported_features" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearSupportedFeatures() { + _hasBits0 &= ~1; + } + + /// Field number for the "file" field. + public const int FileFieldNumber = 15; + private static readonly pb::FieldCodec _repeated_file_codec + = pb::FieldCodec.ForMessage(122, global::Google.Protobuf.Compiler.CodeGeneratorResponse.Types.File.Parser); + private readonly pbc::RepeatedField file_ = new pbc::RepeatedField(); + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public pbc::RepeatedField File { + get { return file_; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as CodeGeneratorResponse); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(CodeGeneratorResponse other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (Error != other.Error) return false; + if (SupportedFeatures != other.SupportedFeatures) return false; + if(!file_.Equals(other.file_)) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (HasError) hash ^= Error.GetHashCode(); + if (HasSupportedFeatures) hash ^= SupportedFeatures.GetHashCode(); + hash ^= file_.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (HasError) { + output.WriteRawTag(10); + output.WriteString(Error); + } + if (HasSupportedFeatures) { + output.WriteRawTag(16); + output.WriteUInt64(SupportedFeatures); + } + file_.WriteTo(output, _repeated_file_codec); + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (HasError) { + output.WriteRawTag(10); + output.WriteString(Error); + } + if (HasSupportedFeatures) { + output.WriteRawTag(16); + output.WriteUInt64(SupportedFeatures); + } + file_.WriteTo(ref output, _repeated_file_codec); + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (HasError) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Error); + } + if (HasSupportedFeatures) { + size += 1 + pb::CodedOutputStream.ComputeUInt64Size(SupportedFeatures); + } + size += file_.CalculateSize(_repeated_file_codec); + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(CodeGeneratorResponse other) { + if (other == null) { + return; + } + if (other.HasError) { + Error = other.Error; + } + if (other.HasSupportedFeatures) { + SupportedFeatures = other.SupportedFeatures; + } + file_.Add(other.file_); + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + Error = input.ReadString(); + break; + } + case 16: { + SupportedFeatures = input.ReadUInt64(); + break; + } + case 122: { + file_.AddEntriesFrom(input, _repeated_file_codec); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 10: { + Error = input.ReadString(); + break; + } + case 16: { + SupportedFeatures = input.ReadUInt64(); + break; + } + case 122: { + file_.AddEntriesFrom(ref input, _repeated_file_codec); + break; + } + } + } + } + #endif + + #region Nested types + /// Container for nested types declared in the CodeGeneratorResponse message type. + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static partial class Types { + /// + /// Sync with code_generator.h. + /// + public enum Feature { + [pbr::OriginalName("FEATURE_NONE")] None = 0, + [pbr::OriginalName("FEATURE_PROTO3_OPTIONAL")] Proto3Optional = 1, + } + + /// + /// Represents a single generated file. + /// + public sealed partial class File : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new File()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::Google.Protobuf.Compiler.CodeGeneratorResponse.Descriptor.NestedTypes[0]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public File() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public File(File other) : this() { + name_ = other.name_; + insertionPoint_ = other.insertionPoint_; + content_ = other.content_; + generatedCodeInfo_ = other.generatedCodeInfo_ != null ? other.generatedCodeInfo_.Clone() : null; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public File Clone() { + return new File(this); + } + + /// Field number for the "name" field. + public const int NameFieldNumber = 1; + private readonly static string NameDefaultValue = ""; + + private string name_; + /// + /// The file name, relative to the output directory. The name must not + /// contain "." or ".." components and must be relative, not be absolute (so, + /// the file cannot lie outside the output directory). "/" must be used as + /// the path separator, not "\". + /// + /// If the name is omitted, the content will be appended to the previous + /// file. This allows the generator to break large files into small chunks, + /// and allows the generated text to be streamed back to protoc so that large + /// files need not reside completely in memory at one time. Note that as of + /// this writing protoc does not optimize for this -- it will read the entire + /// CodeGeneratorResponse before writing files to disk. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string Name { + get { return name_ ?? NameDefaultValue; } + set { + name_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "name" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasName { + get { return name_ != null; } + } + /// Clears the value of the "name" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearName() { + name_ = null; + } + + /// Field number for the "insertion_point" field. + public const int InsertionPointFieldNumber = 2; + private readonly static string InsertionPointDefaultValue = ""; + + private string insertionPoint_; + /// + /// If non-empty, indicates that the named file should already exist, and the + /// content here is to be inserted into that file at a defined insertion + /// point. This feature allows a code generator to extend the output + /// produced by another code generator. The original generator may provide + /// insertion points by placing special annotations in the file that look + /// like: + /// @@protoc_insertion_point(NAME) + /// The annotation can have arbitrary text before and after it on the line, + /// which allows it to be placed in a comment. NAME should be replaced with + /// an identifier naming the point -- this is what other generators will use + /// as the insertion_point. Code inserted at this point will be placed + /// immediately above the line containing the insertion point (thus multiple + /// insertions to the same point will come out in the order they were added). + /// The double-@ is intended to make it unlikely that the generated code + /// could contain things that look like insertion points by accident. + /// + /// For example, the C++ code generator places the following line in the + /// .pb.h files that it generates: + /// // @@protoc_insertion_point(namespace_scope) + /// This line appears within the scope of the file's package namespace, but + /// outside of any particular class. Another plugin can then specify the + /// insertion_point "namespace_scope" to generate additional classes or + /// other declarations that should be placed in this scope. + /// + /// Note that if the line containing the insertion point begins with + /// whitespace, the same whitespace will be added to every line of the + /// inserted text. This is useful for languages like Python, where + /// indentation matters. In these languages, the insertion point comment + /// should be indented the same amount as any inserted code will need to be + /// in order to work correctly in that context. + /// + /// The code generator that generates the initial file and the one which + /// inserts into it must both run as part of a single invocation of protoc. + /// Code generators are executed in the order in which they appear on the + /// command line. + /// + /// If |insertion_point| is present, |name| must also be present. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string InsertionPoint { + get { return insertionPoint_ ?? InsertionPointDefaultValue; } + set { + insertionPoint_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "insertion_point" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasInsertionPoint { + get { return insertionPoint_ != null; } + } + /// Clears the value of the "insertion_point" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearInsertionPoint() { + insertionPoint_ = null; + } + + /// Field number for the "content" field. + public const int ContentFieldNumber = 15; + private readonly static string ContentDefaultValue = ""; + + private string content_; + /// + /// The file contents. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string Content { + get { return content_ ?? ContentDefaultValue; } + set { + content_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "content" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasContent { + get { return content_ != null; } + } + /// Clears the value of the "content" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearContent() { + content_ = null; + } + + /// Field number for the "generated_code_info" field. + public const int GeneratedCodeInfoFieldNumber = 16; + private global::Google.Protobuf.Reflection.GeneratedCodeInfo generatedCodeInfo_; + /// + /// Information describing the file content being inserted. If an insertion + /// point is used, this information will be appropriately offset and inserted + /// into the code generation metadata for the generated files. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::Google.Protobuf.Reflection.GeneratedCodeInfo GeneratedCodeInfo { + get { return generatedCodeInfo_; } + set { + generatedCodeInfo_ = value; + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as File); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(File other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (Name != other.Name) return false; + if (InsertionPoint != other.InsertionPoint) return false; + if (Content != other.Content) return false; + if (!object.Equals(GeneratedCodeInfo, other.GeneratedCodeInfo)) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (HasName) hash ^= Name.GetHashCode(); + if (HasInsertionPoint) hash ^= InsertionPoint.GetHashCode(); + if (HasContent) hash ^= Content.GetHashCode(); + if (generatedCodeInfo_ != null) hash ^= GeneratedCodeInfo.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (HasName) { + output.WriteRawTag(10); + output.WriteString(Name); + } + if (HasInsertionPoint) { + output.WriteRawTag(18); + output.WriteString(InsertionPoint); + } + if (HasContent) { + output.WriteRawTag(122); + output.WriteString(Content); + } + if (generatedCodeInfo_ != null) { + output.WriteRawTag(130, 1); + output.WriteMessage(GeneratedCodeInfo); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (HasName) { + output.WriteRawTag(10); + output.WriteString(Name); + } + if (HasInsertionPoint) { + output.WriteRawTag(18); + output.WriteString(InsertionPoint); + } + if (HasContent) { + output.WriteRawTag(122); + output.WriteString(Content); + } + if (generatedCodeInfo_ != null) { + output.WriteRawTag(130, 1); + output.WriteMessage(GeneratedCodeInfo); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (HasName) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Name); + } + if (HasInsertionPoint) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(InsertionPoint); + } + if (HasContent) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Content); + } + if (generatedCodeInfo_ != null) { + size += 2 + pb::CodedOutputStream.ComputeMessageSize(GeneratedCodeInfo); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(File other) { + if (other == null) { + return; + } + if (other.HasName) { + Name = other.Name; + } + if (other.HasInsertionPoint) { + InsertionPoint = other.InsertionPoint; + } + if (other.HasContent) { + Content = other.Content; + } + if (other.generatedCodeInfo_ != null) { + if (generatedCodeInfo_ == null) { + GeneratedCodeInfo = new global::Google.Protobuf.Reflection.GeneratedCodeInfo(); + } + GeneratedCodeInfo.MergeFrom(other.GeneratedCodeInfo); + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + Name = input.ReadString(); + break; + } + case 18: { + InsertionPoint = input.ReadString(); + break; + } + case 122: { + Content = input.ReadString(); + break; + } + case 130: { + if (generatedCodeInfo_ == null) { + GeneratedCodeInfo = new global::Google.Protobuf.Reflection.GeneratedCodeInfo(); + } + input.ReadMessage(GeneratedCodeInfo); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 10: { + Name = input.ReadString(); + break; + } + case 18: { + InsertionPoint = input.ReadString(); + break; + } + case 122: { + Content = input.ReadString(); + break; + } + case 130: { + if (generatedCodeInfo_ == null) { + GeneratedCodeInfo = new global::Google.Protobuf.Reflection.GeneratedCodeInfo(); + } + input.ReadMessage(GeneratedCodeInfo); + break; + } + } + } + } + #endif + + } + + } + #endregion + + } + + #endregion + +} + +#endregion Designer generated code diff --git a/csharp/src/Google.Protobuf/Google.Protobuf.csproj b/csharp/src/Google.Protobuf/Google.Protobuf.csproj index 37d86ce04304..c95ecad94e86 100644 --- a/csharp/src/Google.Protobuf/Google.Protobuf.csproj +++ b/csharp/src/Google.Protobuf/Google.Protobuf.csproj @@ -4,7 +4,7 @@ C# runtime library for Protocol Buffers - Google's data interchange format. Copyright 2015, Google Inc. Google Protocol Buffers - 3.21.2 + 3.21.4 10.0 Google Inc. netstandard1.1;netstandard2.0;net45;net50 diff --git a/csharp/src/Google.Protobuf/JsonFormatter.cs b/csharp/src/Google.Protobuf/JsonFormatter.cs index 2ef10ee7e924..9cecb6a2d228 100644 --- a/csharp/src/Google.Protobuf/JsonFormatter.cs +++ b/csharp/src/Google.Protobuf/JsonFormatter.cs @@ -63,7 +63,12 @@ public sealed class JsonFormatter internal const string AnyDiagnosticValueField = "@value"; internal const string AnyWellKnownTypeValueField = "value"; private const string NameValueSeparator = ": "; - private const string PropertySeparator = ", "; + private const string ValueSeparator = ", "; + private const string MultilineValueSeparator = ","; + private const char ObjectOpenBracket = '{'; + private const char ObjectCloseBracket = '}'; + private const char ListBracketOpen = '['; + private const char ListBracketClose = ']'; /// /// Returns a formatter using the default settings. @@ -140,11 +145,26 @@ public JsonFormatter(Settings settings) /// Formats the specified message as JSON. /// /// The message to format. + /// This method delegates to Format(IMessage, int) with indentationLevel = 0. /// The formatted message. - public string Format(IMessage message) + public string Format(IMessage message) => Format(message, indentationLevel: 0); + + /// + /// Formats the specified message as JSON. + /// + /// The message to format. + /// Indentation level to start at. + /// To keep consistent indentation when embedding a message inside another JSON string, set . E.g: + /// + /// var response = $@"{{ + /// ""data"": { Format(message, indentationLevel: 1) } + /// }}" + /// + /// The formatted message. + public string Format(IMessage message, int indentationLevel) { var writer = new StringWriter(); - Format(message, writer); + Format(message, writer, indentationLevel); return writer.ToString(); } @@ -153,19 +173,29 @@ public string Format(IMessage message) /// /// The message to format. /// The TextWriter to write the formatted message to. + /// This method delegates to Format(IMessage, TextWriter, int) with indentationLevel = 0. /// The formatted message. - public void Format(IMessage message, TextWriter writer) + public void Format(IMessage message, TextWriter writer) => Format(message, writer, indentationLevel: 0); + + /// + /// Formats the specified message as JSON. When is not null, start indenting at the specified . + /// + /// The message to format. + /// The TextWriter to write the formatted message to. + /// Indentation level to start at. + /// To keep consistent indentation when embedding a message inside another JSON string, set . + public void Format(IMessage message, TextWriter writer, int indentationLevel) { ProtoPreconditions.CheckNotNull(message, nameof(message)); ProtoPreconditions.CheckNotNull(writer, nameof(writer)); if (message.Descriptor.IsWellKnownType) { - WriteWellKnownTypeValue(writer, message.Descriptor, message); + WriteWellKnownTypeValue(writer, message.Descriptor, message, indentationLevel); } else { - WriteMessage(writer, message); + WriteMessage(writer, message, indentationLevel); } } @@ -192,7 +222,7 @@ public static string ToDiagnosticString(IMessage message) return diagnosticFormatter.Format(message); } - private void WriteMessage(TextWriter writer, IMessage message) + private void WriteMessage(TextWriter writer, IMessage message, int indentationLevel) { if (message == null) { @@ -207,12 +237,13 @@ private void WriteMessage(TextWriter writer, IMessage message) return; } } - writer.Write("{ "); - bool writtenFields = WriteMessageFields(writer, message, false); - writer.Write(writtenFields ? " }" : "}"); + + WriteBracketOpen(writer, ObjectOpenBracket); + bool writtenFields = WriteMessageFields(writer, message, false, indentationLevel + 1); + WriteBracketClose(writer, ObjectCloseBracket, writtenFields, indentationLevel); } - private bool WriteMessageFields(TextWriter writer, IMessage message, bool assumeFirstFieldWritten) + private bool WriteMessageFields(TextWriter writer, IMessage message, bool assumeFirstFieldWritten, int indentationLevel) { var fields = message.Descriptor.Fields; bool first = !assumeFirstFieldWritten; @@ -226,10 +257,8 @@ private bool WriteMessageFields(TextWriter writer, IMessage message, bool assume continue; } - if (!first) - { - writer.Write(PropertySeparator); - } + MaybeWriteValueSeparator(writer, first); + MaybeWriteValueWhitespace(writer, indentationLevel); if (settings.PreserveProtoFieldNames) { @@ -240,13 +269,23 @@ private bool WriteMessageFields(TextWriter writer, IMessage message, bool assume WriteString(writer, accessor.Descriptor.JsonName); } writer.Write(NameValueSeparator); - WriteValue(writer, value); + WriteValue(writer, value, indentationLevel); first = false; } return !first; } + private void MaybeWriteValueSeparator(TextWriter writer, bool first) + { + if (first) + { + return; + } + + writer.Write(settings.Indentation == null ? ValueSeparator : MultilineValueSeparator); + } + /// /// Determines whether or not a field value should be serialized according to the field, /// its value in the message, and the settings of this formatter. @@ -342,7 +381,19 @@ private static bool IsDefaultValue(FieldDescriptor descriptor, object value) /// /// The writer to write the value to. Must not be null. /// The value to write. May be null. - public void WriteValue(TextWriter writer, object value) + /// Delegates to WriteValue(TextWriter, object, int) with indentationLevel = 0. + public void WriteValue(TextWriter writer, object value) => WriteValue(writer, value, 0); + + /// + /// Writes a single value to the given writer as JSON. Only types understood by + /// Protocol Buffers can be written in this way. This method is only exposed for + /// advanced use cases; most users should be using + /// or . + /// + /// The writer to write the value to. Must not be null. + /// The value to write. May be null. + /// The current indentationLevel. Not used when is null. + public void WriteValue(TextWriter writer, object value, int indentationLevel) { if (value == null || value is NullValue) { @@ -365,11 +416,11 @@ public void WriteValue(TextWriter writer, object value) } else if (value is IDictionary dictionary) { - WriteDictionary(writer, dictionary); + WriteDictionary(writer, dictionary, indentationLevel); } else if (value is IList list) { - WriteList(writer, list); + WriteList(writer, list, indentationLevel); } else if (value is int || value is uint) { @@ -418,7 +469,7 @@ public void WriteValue(TextWriter writer, object value) } else if (value is IMessage message) { - Format(message, writer); + Format(message, writer, indentationLevel); } else { @@ -432,7 +483,7 @@ public void WriteValue(TextWriter writer, object value) /// values are using the embedded well-known types, in order to allow for dynamic messages /// in the future. /// - private void WriteWellKnownTypeValue(TextWriter writer, MessageDescriptor descriptor, object value) + private void WriteWellKnownTypeValue(TextWriter writer, MessageDescriptor descriptor, object value, int indentationLevel) { // Currently, we can never actually get here, because null values are always handled by the caller. But if we *could*, // this would do the right thing. @@ -472,26 +523,26 @@ private void WriteWellKnownTypeValue(TextWriter writer, MessageDescriptor descri } if (descriptor.FullName == Struct.Descriptor.FullName) { - WriteStruct(writer, (IMessage)value); + WriteStruct(writer, (IMessage)value, indentationLevel); return; } if (descriptor.FullName == ListValue.Descriptor.FullName) { var fieldAccessor = descriptor.Fields[ListValue.ValuesFieldNumber].Accessor; - WriteList(writer, (IList)fieldAccessor.GetValue((IMessage)value)); + WriteList(writer, (IList)fieldAccessor.GetValue((IMessage)value), indentationLevel); return; } if (descriptor.FullName == Value.Descriptor.FullName) { - WriteStructFieldValue(writer, (IMessage)value); + WriteStructFieldValue(writer, (IMessage)value, indentationLevel); return; } if (descriptor.FullName == Any.Descriptor.FullName) { - WriteAny(writer, (IMessage)value); + WriteAny(writer, (IMessage)value, indentationLevel); return; } - WriteMessage(writer, (IMessage)value); + WriteMessage(writer, (IMessage)value, indentationLevel); } private void WriteTimestamp(TextWriter writer, IMessage value) @@ -519,7 +570,7 @@ private void WriteFieldMask(TextWriter writer, IMessage value) writer.Write(FieldMask.ToJson(paths, DiagnosticOnly)); } - private void WriteAny(TextWriter writer, IMessage value) + private void WriteAny(TextWriter writer, IMessage value, int indentationLevel) { if (DiagnosticOnly) { @@ -536,23 +587,23 @@ private void WriteAny(TextWriter writer, IMessage value) throw new InvalidOperationException($"Type registry has no descriptor for type name '{typeName}'"); } IMessage message = descriptor.Parser.ParseFrom(data); - writer.Write("{ "); + WriteBracketOpen(writer, ObjectOpenBracket); WriteString(writer, AnyTypeUrlField); writer.Write(NameValueSeparator); WriteString(writer, typeUrl); if (descriptor.IsWellKnownType) { - writer.Write(PropertySeparator); + writer.Write(ValueSeparator); WriteString(writer, AnyWellKnownTypeValueField); writer.Write(NameValueSeparator); - WriteWellKnownTypeValue(writer, descriptor, message); + WriteWellKnownTypeValue(writer, descriptor, message, indentationLevel); } else { - WriteMessageFields(writer, message, true); + WriteMessageFields(writer, message, true, indentationLevel); } - writer.Write(" }"); + WriteBracketClose(writer, ObjectCloseBracket, true, indentationLevel); } private void WriteDiagnosticOnlyAny(TextWriter writer, IMessage value) @@ -563,7 +614,7 @@ private void WriteDiagnosticOnlyAny(TextWriter writer, IMessage value) WriteString(writer, AnyTypeUrlField); writer.Write(NameValueSeparator); WriteString(writer, typeUrl); - writer.Write(PropertySeparator); + writer.Write(ValueSeparator); WriteString(writer, AnyDiagnosticValueField); writer.Write(NameValueSeparator); writer.Write('"'); @@ -572,9 +623,9 @@ private void WriteDiagnosticOnlyAny(TextWriter writer, IMessage value) writer.Write(" }"); } - private void WriteStruct(TextWriter writer, IMessage message) + private void WriteStruct(TextWriter writer, IMessage message, int indentationLevel) { - writer.Write("{ "); + WriteBracketOpen(writer, ObjectOpenBracket); IDictionary fields = (IDictionary) message.Descriptor.Fields[Struct.FieldsFieldNumber].Accessor.GetValue(message); bool first = true; foreach (DictionaryEntry entry in fields) @@ -586,19 +637,17 @@ private void WriteStruct(TextWriter writer, IMessage message) throw new InvalidOperationException("Struct fields cannot have an empty key or a null value."); } - if (!first) - { - writer.Write(PropertySeparator); - } + MaybeWriteValueSeparator(writer, first); + MaybeWriteValueWhitespace(writer, indentationLevel + 1); WriteString(writer, key); writer.Write(NameValueSeparator); - WriteStructFieldValue(writer, value); + WriteStructFieldValue(writer, value, indentationLevel + 1); first = false; } - writer.Write(first ? "}" : " }"); + WriteBracketClose(writer, ObjectCloseBracket, !first, indentationLevel); } - private void WriteStructFieldValue(TextWriter writer, IMessage message) + private void WriteStructFieldValue(TextWriter writer, IMessage message, int indentationLevel) { var specifiedField = message.Descriptor.Oneofs[0].Accessor.GetCaseFieldDescriptor(message); if (specifiedField == null) @@ -619,7 +668,7 @@ private void WriteStructFieldValue(TextWriter writer, IMessage message) case Value.ListValueFieldNumber: // Structs and ListValues are nested messages, and already well-known types. var nestedMessage = (IMessage) specifiedField.Accessor.GetValue(message); - WriteWellKnownTypeValue(writer, nestedMessage.Descriptor, nestedMessage); + WriteWellKnownTypeValue(writer, nestedMessage.Descriptor, nestedMessage, indentationLevel); return; case Value.NullValueFieldNumber: WriteNull(writer); @@ -629,33 +678,30 @@ private void WriteStructFieldValue(TextWriter writer, IMessage message) } } - internal void WriteList(TextWriter writer, IList list) + internal void WriteList(TextWriter writer, IList list, int indentationLevel = 0) { - writer.Write("[ "); + WriteBracketOpen(writer, ListBracketOpen); + bool first = true; foreach (var value in list) { - if (!first) - { - writer.Write(PropertySeparator); - } - WriteValue(writer, value); + MaybeWriteValueSeparator(writer, first); + MaybeWriteValueWhitespace(writer, indentationLevel + 1); + WriteValue(writer, value, indentationLevel + 1); first = false; } - writer.Write(first ? "]" : " ]"); + + WriteBracketClose(writer, ListBracketClose, !first, indentationLevel); } - internal void WriteDictionary(TextWriter writer, IDictionary dictionary) + internal void WriteDictionary(TextWriter writer, IDictionary dictionary, int indentationLevel = 0) { - writer.Write("{ "); + WriteBracketOpen(writer, ObjectOpenBracket); + bool first = true; // This will box each pair. Could use IDictionaryEnumerator, but that's ugly in terms of disposal. foreach (DictionaryEntry pair in dictionary) { - if (!first) - { - writer.Write(PropertySeparator); - } string keyText; if (pair.Key is string s) { @@ -677,12 +723,16 @@ internal void WriteDictionary(TextWriter writer, IDictionary dictionary) } throw new ArgumentException("Unhandled dictionary key type: " + pair.Key.GetType()); } + + MaybeWriteValueSeparator(writer, first); + MaybeWriteValueWhitespace(writer, indentationLevel + 1); WriteString(writer, keyText); writer.Write(NameValueSeparator); WriteValue(writer, pair.Value); first = false; } - writer.Write(first ? "}" : " }"); + + WriteBracketClose(writer, ObjectCloseBracket, !first, indentationLevel); } /// @@ -766,6 +816,49 @@ private static void HexEncodeUtf16CodeUnit(TextWriter writer, char c) writer.Write(Hex[(c >> 0) & 0xf]); } + private void WriteBracketOpen(TextWriter writer, char openChar) + { + writer.Write(openChar); + if (settings.Indentation == null) + { + writer.Write(' '); + } + } + + private void WriteBracketClose(TextWriter writer, char closeChar, bool hasFields, int indentationLevel) + { + if (hasFields) + { + if (settings.Indentation != null) + { + writer.WriteLine(); + WriteIndentation(writer, indentationLevel); + } + else + { + writer.Write(" "); + } + } + + writer.Write(closeChar); + } + + private void MaybeWriteValueWhitespace(TextWriter writer, int indentationLevel) + { + if (settings.Indentation != null) { + writer.WriteLine(); + WriteIndentation(writer, indentationLevel); + } + } + + private void WriteIndentation(TextWriter writer, int indentationLevel) + { + for (int i = 0; i < indentationLevel; i++) + { + writer.Write(settings.Indentation); + } + } + /// /// Settings controlling JSON formatting. /// @@ -806,6 +899,10 @@ static Settings() /// public bool PreserveProtoFieldNames { get; } + /// + /// Indentation string, used for formatting. Setting null disables indentation. + /// + public string Indentation { get; } /// /// Creates a new object with the specified formatting of default values @@ -833,40 +930,54 @@ public Settings(bool formatDefaultValues, TypeRegistry typeRegistry) : this(form /// The to use when formatting messages. TypeRegistry.Empty will be used if it is null. /// true to format the enums as integers; false to format enums as enum names. /// true to preserve proto field names; false to convert them to lowerCamelCase. + /// The indentation string to use for multi-line formatting. null to disable multi-line format. private Settings(bool formatDefaultValues, TypeRegistry typeRegistry, bool formatEnumsAsIntegers, - bool preserveProtoFieldNames) + bool preserveProtoFieldNames, + string indentation = null) { FormatDefaultValues = formatDefaultValues; TypeRegistry = typeRegistry ?? TypeRegistry.Empty; FormatEnumsAsIntegers = formatEnumsAsIntegers; PreserveProtoFieldNames = preserveProtoFieldNames; + Indentation = indentation; } /// /// Creates a new object with the specified formatting of default values and the current settings. /// /// true if default values (0, empty strings etc) should be formatted; false otherwise. - public Settings WithFormatDefaultValues(bool formatDefaultValues) => new Settings(formatDefaultValues, TypeRegistry, FormatEnumsAsIntegers, PreserveProtoFieldNames); + public Settings WithFormatDefaultValues(bool formatDefaultValues) => new Settings(formatDefaultValues, TypeRegistry, FormatEnumsAsIntegers, PreserveProtoFieldNames, Indentation); /// /// Creates a new object with the specified type registry and the current settings. /// /// The to use when formatting messages. - public Settings WithTypeRegistry(TypeRegistry typeRegistry) => new Settings(FormatDefaultValues, typeRegistry, FormatEnumsAsIntegers, PreserveProtoFieldNames); + public Settings WithTypeRegistry(TypeRegistry typeRegistry) => new Settings(FormatDefaultValues, typeRegistry, FormatEnumsAsIntegers, PreserveProtoFieldNames, Indentation); /// /// Creates a new object with the specified enums formatting option and the current settings. /// /// true to format the enums as integers; false to format enums as enum names. - public Settings WithFormatEnumsAsIntegers(bool formatEnumsAsIntegers) => new Settings(FormatDefaultValues, TypeRegistry, formatEnumsAsIntegers, PreserveProtoFieldNames); + public Settings WithFormatEnumsAsIntegers(bool formatEnumsAsIntegers) => new Settings(FormatDefaultValues, TypeRegistry, formatEnumsAsIntegers, PreserveProtoFieldNames, Indentation); /// /// Creates a new object with the specified field name formatting option and the current settings. /// /// true to preserve proto field names; false to convert them to lowerCamelCase. - public Settings WithPreserveProtoFieldNames(bool preserveProtoFieldNames) => new Settings(FormatDefaultValues, TypeRegistry, FormatEnumsAsIntegers, preserveProtoFieldNames); + public Settings WithPreserveProtoFieldNames(bool preserveProtoFieldNames) => new Settings(FormatDefaultValues, TypeRegistry, FormatEnumsAsIntegers, preserveProtoFieldNames, Indentation); + + /// + /// Creates a new object with the specified indentation and the current settings. + /// + /// The string to output for each level of indentation (nesting). The default is two spaces per level. Use null to disable indentation entirely. + /// A non-null value for will insert additional line-breaks to the JSON output. + /// Each line will contain either a single value, or braces. The default line-break is determined by , + /// which is "\n" on Unix platforms, and "\r\n" on Windows. If seems to produce empty lines, + /// you need to pass a that uses a "\n" newline. See . + /// + public Settings WithIndentation(string indentation = " ") => new Settings(FormatDefaultValues, TypeRegistry, FormatEnumsAsIntegers, PreserveProtoFieldNames, indentation); } // Effectively a cache of mapping from enum values to the original name as specified in the proto file, diff --git a/csharp/src/Google.Protobuf/MessageParser.cs b/csharp/src/Google.Protobuf/MessageParser.cs index 66907d46fa02..57102920ef99 100644 --- a/csharp/src/Google.Protobuf/MessageParser.cs +++ b/csharp/src/Google.Protobuf/MessageParser.cs @@ -171,6 +171,10 @@ public IMessage ParseFrom(CodedInputStream input) /// /// Parses a message from the given JSON. /// + /// This method always uses the default JSON parser; it is not affected by . + /// To ignore unknown fields when parsing JSON, create a using a + /// with set to true and call directly. + /// /// The JSON to parse. /// The parsed message. /// The JSON does not comply with RFC 7159 @@ -203,6 +207,9 @@ internal void MergeFrom(IMessage message, CodedInputStream codedInput) /// /// Creates a new message parser which optionally discards unknown fields when parsing. /// + /// Note that this does not affect the behavior of + /// at all. To ignore unknown fields when parsing JSON, create a using a + /// with set to true and call directly. /// Whether or not to discard unknown fields when parsing. /// A newly configured message parser. public MessageParser WithDiscardUnknownFields(bool discardUnknownFields) => diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Struct.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Struct.cs index 8c1eec53b4b6..aa25686dee03 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Struct.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Struct.cs @@ -212,7 +212,7 @@ public void MergeFrom(Struct other) { if (other == null) { return; } - fields_.Add(other.fields_); + fields_.MergeFrom(other.fields_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } diff --git a/docs/third_party.md b/docs/third_party.md index d8478e9adaa6..d9ff970227a0 100644 --- a/docs/third_party.md +++ b/docs/third_party.md @@ -203,3 +203,4 @@ There are miscellaneous other things you may find useful as a Protocol Buffers d * [vim-protolint: A protobuf linter for Vim](https://github.com/yoheimuta/vim-protolint) * [super-linter: Protocol Buffer lint as GitHub Action](https://github.com/github/super-linter) * [protoc-gen-fieldmask - A plugin to generate static type fieldmask paths](https://github.com/idodod/protoc-gen-fieldmask) +* [protoc-gen-bq-schema - A protoc plugin to generate BigQuery schema files](https://github.com/GoogleCloudPlatform/protoc-gen-bq-schema) diff --git a/java/README.md b/java/README.md index 7e71e667160f..5e3ded418f61 100644 --- a/java/README.md +++ b/java/README.md @@ -23,7 +23,7 @@ If you are using Maven, use the following: com.google.protobuf protobuf-java - 3.21.2 + 3.21.4 ``` @@ -37,7 +37,7 @@ protobuf-java-util package: com.google.protobuf protobuf-java-util - 3.21.2 + 3.21.4 ``` @@ -45,7 +45,7 @@ protobuf-java-util package: If you are using Gradle, add the following to your `build.gradle` file's dependencies: ``` - implementation 'com.google.protobuf:protobuf-java:3.21.2' + implementation 'com.google.protobuf:protobuf-java:3.21.4' ``` Again, be sure to check that the version number matches (or is newer than) the version number of protoc that you are using. diff --git a/java/bom/pom.xml b/java/bom/pom.xml index b4c3465e1180..4cccfb35456a 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-bom - 3.21.2 + 3.21.4 pom Protocol Buffers [BOM] diff --git a/java/core/pom.xml b/java/core/pom.xml index 845ff8856acf..d5eacd6a109c 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.21.2 + 3.21.4 protobuf-java diff --git a/java/core/src/main/java/com/google/protobuf/BinaryWriter.java b/java/core/src/main/java/com/google/protobuf/BinaryWriter.java index cf394e337167..66cf51d24c4a 100644 --- a/java/core/src/main/java/com/google/protobuf/BinaryWriter.java +++ b/java/core/src/main/java/com/google/protobuf/BinaryWriter.java @@ -209,7 +209,7 @@ public final void writeInt32List(int fieldNumber, List list, boolean pa } } - private final void writeInt32List_Internal(int fieldNumber, List list, boolean packed) + private void writeInt32List_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT64_SIZE)); @@ -227,7 +227,7 @@ private final void writeInt32List_Internal(int fieldNumber, List list, } } - private final void writeInt32List_Internal(int fieldNumber, IntArrayList list, boolean packed) + private void writeInt32List_Internal(int fieldNumber, IntArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT64_SIZE)); @@ -255,7 +255,7 @@ public final void writeFixed32List(int fieldNumber, List list, boolean } } - private final void writeFixed32List_Internal(int fieldNumber, List list, boolean packed) + private void writeFixed32List_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * FIXED32_SIZE)); @@ -273,7 +273,7 @@ private final void writeFixed32List_Internal(int fieldNumber, List list } } - private final void writeFixed32List_Internal(int fieldNumber, IntArrayList list, boolean packed) + private void writeFixed32List_Internal(int fieldNumber, IntArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * FIXED32_SIZE)); @@ -307,7 +307,7 @@ public final void writeUInt64List(int fieldNumber, List list, boolean pack } } - private final void writeUInt64List_Internal(int fieldNumber, List list, boolean packed) + private void writeUInt64List_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT64_SIZE)); @@ -325,7 +325,7 @@ private final void writeUInt64List_Internal(int fieldNumber, List list, bo } } - private final void writeUInt64List_Internal(int fieldNumber, LongArrayList list, boolean packed) + private void writeUInt64List_Internal(int fieldNumber, LongArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT64_SIZE)); @@ -353,7 +353,7 @@ public final void writeFixed64List(int fieldNumber, List list, boolean pac } } - private final void writeFixed64List_Internal(int fieldNumber, List list, boolean packed) + private void writeFixed64List_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * FIXED64_SIZE)); @@ -371,7 +371,7 @@ private final void writeFixed64List_Internal(int fieldNumber, List list, b } } - private final void writeFixed64List_Internal(int fieldNumber, LongArrayList list, boolean packed) + private void writeFixed64List_Internal(int fieldNumber, LongArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * FIXED64_SIZE)); @@ -399,7 +399,7 @@ public final void writeFloatList(int fieldNumber, List list, boolean pack } } - private final void writeFloatList_Internal(int fieldNumber, List list, boolean packed) + private void writeFloatList_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * FIXED32_SIZE)); @@ -417,7 +417,7 @@ private final void writeFloatList_Internal(int fieldNumber, List list, bo } } - private final void writeFloatList_Internal(int fieldNumber, FloatArrayList list, boolean packed) + private void writeFloatList_Internal(int fieldNumber, FloatArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * FIXED32_SIZE)); @@ -445,7 +445,7 @@ public final void writeDoubleList(int fieldNumber, List list, boolean pa } } - private final void writeDoubleList_Internal(int fieldNumber, List list, boolean packed) + private void writeDoubleList_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * FIXED64_SIZE)); @@ -463,7 +463,7 @@ private final void writeDoubleList_Internal(int fieldNumber, List list, } } - private final void writeDoubleList_Internal(int fieldNumber, DoubleArrayList list, boolean packed) + private void writeDoubleList_Internal(int fieldNumber, DoubleArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * FIXED64_SIZE)); @@ -497,7 +497,7 @@ public final void writeBoolList(int fieldNumber, List list, boolean pac } } - private final void writeBoolList_Internal(int fieldNumber, List list, boolean packed) + private void writeBoolList_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + list.size()); @@ -515,7 +515,7 @@ private final void writeBoolList_Internal(int fieldNumber, List list, b } } - private final void writeBoolList_Internal(int fieldNumber, BooleanArrayList list, boolean packed) + private void writeBoolList_Internal(int fieldNumber, BooleanArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + list.size()); @@ -572,7 +572,7 @@ public final void writeUInt32List(int fieldNumber, List list, boolean p } } - private final void writeUInt32List_Internal(int fieldNumber, List list, boolean packed) + private void writeUInt32List_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT32_SIZE)); @@ -590,7 +590,7 @@ private final void writeUInt32List_Internal(int fieldNumber, List list, } } - private final void writeUInt32List_Internal(int fieldNumber, IntArrayList list, boolean packed) + private void writeUInt32List_Internal(int fieldNumber, IntArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT32_SIZE)); @@ -630,7 +630,7 @@ public final void writeSInt32List(int fieldNumber, List list, boolean p } } - private final void writeSInt32List_Internal(int fieldNumber, List list, boolean packed) + private void writeSInt32List_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT32_SIZE)); @@ -648,7 +648,7 @@ private final void writeSInt32List_Internal(int fieldNumber, List list, } } - private final void writeSInt32List_Internal(int fieldNumber, IntArrayList list, boolean packed) + private void writeSInt32List_Internal(int fieldNumber, IntArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT32_SIZE)); @@ -759,7 +759,7 @@ static final void writeMapEntryField( } } - private final void writeSInt64List_Internal(int fieldNumber, List list, boolean packed) + private void writeSInt64List_Internal(int fieldNumber, List list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT64_SIZE)); @@ -777,7 +777,7 @@ private final void writeSInt64List_Internal(int fieldNumber, List list, bo } } - private final void writeSInt64List_Internal(int fieldNumber, LongArrayList list, boolean packed) + private void writeSInt64List_Internal(int fieldNumber, LongArrayList list, boolean packed) throws IOException { if (packed) { requireSpace((MAX_VARINT32_SIZE * 2) + (list.size() * MAX_VARINT64_SIZE)); diff --git a/java/core/src/main/java/com/google/protobuf/Descriptors.java b/java/core/src/main/java/com/google/protobuf/Descriptors.java index 8ba1f9b07fbc..91c9b1977ff2 100644 --- a/java/core/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/core/src/main/java/com/google/protobuf/Descriptors.java @@ -1788,6 +1788,27 @@ public List getValues() { return Collections.unmodifiableList(Arrays.asList(values)); } + /** Determines if the given field number is reserved. */ + public boolean isReservedNumber(final int number) { + for (final EnumDescriptorProto.EnumReservedRange range : proto.getReservedRangeList()) { + if (range.getStart() <= number && number <= range.getEnd()) { + return true; + } + } + return false; + } + + /** Determines if the given field name is reserved. */ + public boolean isReservedName(final String name) { + checkNotNull(name); + for (final String reservedName : proto.getReservedNameList()) { + if (reservedName.equals(name)) { + return true; + } + } + return false; + } + /** * Find an enum value by name. * diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java index e212ab5fe099..0c160003d3af 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java @@ -137,7 +137,7 @@ public boolean equals( // any unnecessary intermediary allocations while reducing the generated code size. /** Lazily initializes unknown fields. */ - private final void ensureUnknownFieldsInitialized() { + private void ensureUnknownFieldsInitialized() { if (unknownFields == UnknownFieldSetLite.getDefaultInstance()) { unknownFields = UnknownFieldSetLite.newInstance(); } diff --git a/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java b/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java index 49a71dc545ee..7988e7cce6c8 100644 --- a/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java +++ b/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java @@ -61,6 +61,7 @@ import protobuf_unittest.UnittestProto.TestJsonName; import protobuf_unittest.UnittestProto.TestMultipleExtensionRanges; import protobuf_unittest.UnittestProto.TestRequired; +import protobuf_unittest.UnittestProto.TestReservedEnumFields; import protobuf_unittest.UnittestProto.TestReservedFields; import protobuf_unittest.UnittestProto.TestService; import java.util.Collections; @@ -796,6 +797,20 @@ public void testReservedFields() { assertThat(d.isReservedName("baz")).isTrue(); } + @Test + public void testReservedEnumFields() { + EnumDescriptor d = TestReservedEnumFields.getDescriptor(); + assertThat(d.isReservedNumber(2)).isTrue(); + assertThat(d.isReservedNumber(8)).isFalse(); + assertThat(d.isReservedNumber(9)).isTrue(); + assertThat(d.isReservedNumber(10)).isTrue(); + assertThat(d.isReservedNumber(11)).isTrue(); + assertThat(d.isReservedNumber(12)).isFalse(); + assertThat(d.isReservedName("foo")).isFalse(); + assertThat(d.isReservedName("bar")).isTrue(); + assertThat(d.isReservedName("baz")).isTrue(); + } + @Test public void testToString() { assertThat( diff --git a/java/core/src/test/java/com/google/protobuf/LiteEqualsAndHashTest.java b/java/core/src/test/java/com/google/protobuf/LiteEqualsAndHashTest.java index 1270ef06ee4d..9ea7f93ffdb1 100644 --- a/java/core/src/test/java/com/google/protobuf/LiteEqualsAndHashTest.java +++ b/java/core/src/test/java/com/google/protobuf/LiteEqualsAndHashTest.java @@ -30,14 +30,13 @@ package com.google.protobuf; +import junit.framework.TestCase; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Bar; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.BarPrime; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Foo; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.TestOneofEquals; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.TestRecursiveOneof; -import junit.framework.TestCase; - /** * Test generate equal and hash methods for the lite runtime. * @@ -120,6 +119,6 @@ private void assertEqualsAndHashCodeAreFalse(Object o1, Object o2) { public void testRecursiveHashcode() { // This tests that we don't infinite loop. - TestRecursiveOneof.getDefaultInstance().hashCode(); + int unused = TestRecursiveOneof.getDefaultInstance().hashCode(); } } diff --git a/java/kotlin-lite/pom.xml b/java/kotlin-lite/pom.xml index 3fec5ec45960..2dc430907ed8 100644 --- a/java/kotlin-lite/pom.xml +++ b/java/kotlin-lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.21.2 + 3.21.4 protobuf-kotlin-lite diff --git a/java/kotlin/pom.xml b/java/kotlin/pom.xml index 43debafe5062..7a350f785399 100644 --- a/java/kotlin/pom.xml +++ b/java/kotlin/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.21.2 + 3.21.4 protobuf-kotlin diff --git a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt index 1645cfbad106..8ce00946dc6d 100644 --- a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt +++ b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt @@ -66,6 +66,11 @@ class Proto3Test { assertThat(optionalNestedMessage).isEqualTo(TestAllTypesKt.nestedMessage { bb = 118 }) optionalNestedEnum = NestedEnum.BAZ assertThat(optionalNestedEnum).isEqualTo(NestedEnum.BAZ) + assertThat(optionalNestedEnumValue).isEqualTo(3) + optionalNestedEnumValue = 1 + assertThat(optionalNestedEnumValue).isEqualTo(1) + assertThat(optionalNestedEnum).isEqualTo(NestedEnum.FOO) + oneofUint32 = 601 assertThat(oneofUint32).isEqualTo(601) } diff --git a/java/lite.md b/java/lite.md index 1ad001984791..d4b4fc87b2d5 100644 --- a/java/lite.md +++ b/java/lite.md @@ -29,7 +29,7 @@ protobuf Java Lite runtime. If you are using Maven, include the following: com.google.protobuf protobuf-javalite - 3.21.2 + 3.21.4 ``` diff --git a/java/lite/pom.xml b/java/lite/pom.xml index 7c635ec33f12..20381c65ba99 100644 --- a/java/lite/pom.xml +++ b/java/lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.21.2 + 3.21.4 protobuf-javalite diff --git a/java/pom.xml b/java/pom.xml index 34475a455a66..99abcf86a994 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.21.2 + 3.21.4 pom Protocol Buffers [Parent] diff --git a/java/protoc/pom.xml b/java/protoc/pom.xml index aa6bc588a7d0..b2cd0d551928 100644 --- a/java/protoc/pom.xml +++ b/java/protoc/pom.xml @@ -4,11 +4,11 @@ com.google google - 1 + 5 com.google.protobuf protoc - 3.21.2 + 3.21.3 pom Protobuf Compiler diff --git a/java/util/pom.xml b/java/util/pom.xml index 53f90ce6dcd1..f634047c6df2 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.21.2 + 3.21.4 protobuf-java-util diff --git a/kokoro/linux/bazel_distcheck/build.sh b/kokoro/linux/bazel_distcheck/build.sh deleted file mode 100755 index a50b175db20b..000000000000 --- a/kokoro/linux/bazel_distcheck/build.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -# -# Build file to set up and run tests using bazel-build dist archive -# -# Note that the builds use WORKSPACE to fetch external sources, not -# git submodules. - -set -eu - -use_bazel.sh 5.0.0 || true -bazel version - -# Change to repo root -cd $(dirname $0)/../../.. - -# Get kokoro scripts from repo root by default. -: ${SCRIPT_ROOT:=$(pwd)} -source ${SCRIPT_ROOT}/kokoro/common/pyenv.sh - -# Build distribution archive -echo "============================================================" -echo -e "[[ $(date) ]] Building distribution archive...\n" -${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh build //pkg:dist_all_tar -DIST_ARCHIVE=$(readlink $(bazel info bazel-bin)/pkg/dist_all_tar.tar.gz) -bazel shutdown - -# Extract the dist archive. -echo "============================================================" -echo -e "[[ $(date) ]] Extracting distribution archive...\n" - -# Construct temp directory for running the dist build. -# If you want to run locally and keep the build dir, create a directory -# and pass it in the DIST_WORK_ROOT env var. -if [[ -z ${DIST_WORK_ROOT:-} ]]; then - : ${DIST_WORK_ROOT:=$(mktemp -d)} - function dist_cleanup() { - (( $BASH_SUBSHELL == 0 )) && rm -rf ${DIST_WORK_ROOT} - } - trap dist_cleanup EXIT -fi - -DIST_WORKSPACE=${DIST_WORK_ROOT}/protobuf -mkdir -p ${DIST_WORKSPACE} -tar -C ${DIST_WORKSPACE} --strip-components=1 -axf bazel-bin/pkg/dist_all_tar.tar.gz - -echo "============================================================" -echo -e "[[ $(date) ]] Building extracted archive...\n" - -cd ${DIST_WORKSPACE} - -bazel_args=( - test - --keep_going - --test_output=errors - -- - //... - -//objectivec/... # only works on macOS - -//csharp/... # release builds require external dependencies - @com_google_protobuf_examples//... -) -${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh "${bazel_args[@]}" diff --git a/kokoro/linux/bazel_distcheck/common.cfg b/kokoro/linux/bazel_distcheck/common.cfg deleted file mode 100644 index 6b1848816db1..000000000000 --- a/kokoro/linux/bazel_distcheck/common.cfg +++ /dev/null @@ -1,9 +0,0 @@ -# Common config shared by presubmit and continuous. - -bazel_setting: { - project_id: "protobuf-build" - bes_backend_address: "buildeventservice.googleapis.com" - foundry_backend_address: "remotebuildexecution.googleapis.com" - upsalite_frontend_address: "https://source.cloud.google.com" - local_execution: true -} diff --git a/kokoro/linux/bazel_distcheck/continuous.cfg b/kokoro/linux/bazel_distcheck/continuous.cfg deleted file mode 100644 index 4ea8b2100666..000000000000 --- a/kokoro/linux/bazel_distcheck/continuous.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/bazel_distcheck/build.sh" -timeout_mins: 15 diff --git a/kokoro/linux/bazel_distcheck/presubmit.cfg b/kokoro/linux/bazel_distcheck/presubmit.cfg deleted file mode 100644 index 4ea8b2100666..000000000000 --- a/kokoro/linux/bazel_distcheck/presubmit.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/bazel_distcheck/build.sh" -timeout_mins: 15 diff --git a/kokoro/linux/cmake/build.sh b/kokoro/linux/cmake/build.sh new file mode 100755 index 000000000000..523253da2596 --- /dev/null +++ b/kokoro/linux/cmake/build.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Build file to set up and run tests using CMake + +set -eux + +# Change to repo root +cd $(dirname $0)/../../.. +GIT_REPO_ROOT=`pwd` + +CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671 + +# Update git submodules +git submodule update --init --recursive + +tmpfile=$(mktemp -u) + +docker run \ + --cidfile $tmpfile \ + -v $GIT_REPO_ROOT:/workspace \ + $CONTAINER_IMAGE \ + /test.sh -Dprotobuf_BUILD_CONFORMANCE=ON + +# Save logs for Kokoro +docker cp \ + `cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR diff --git a/kokoro/linux/cmake/continuous.cfg b/kokoro/linux/cmake/continuous.cfg new file mode 100644 index 000000000000..f03bd3945f1a --- /dev/null +++ b/kokoro/linux/cmake/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/cmake/build.sh" +timeout_mins: 1440 + +action { + define_artifacts { + regex: "**/sponge_log.*" + } +} diff --git a/kokoro/linux/cmake/presubmit.cfg b/kokoro/linux/cmake/presubmit.cfg new file mode 100644 index 000000000000..f03bd3945f1a --- /dev/null +++ b/kokoro/linux/cmake/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/cmake/build.sh" +timeout_mins: 1440 + +action { + define_artifacts { + regex: "**/sponge_log.*" + } +} diff --git a/kokoro/linux/cmake_distcheck/build.sh b/kokoro/linux/cmake_distcheck/build.sh deleted file mode 100755 index 116e40b86de6..000000000000 --- a/kokoro/linux/cmake_distcheck/build.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# -# Build file to set up and run tests based on distribution archive - -set -eux - -# Change to repo root -cd $(dirname $0)/../../.. - -# -# Update git submodules -# -git submodule update --init --recursive - -# -# Build distribution archive -# -# TODO: this should use Bazel-built dist archives. -date ; ./autogen.sh -date ; ./configure -date ; make dist -date - -DIST_ARCHIVE=( $(ls protobuf-*.tar.gz) ) -if (( ${#DIST_ARCHIVE[@]} != 1 )); then - echo >&2 "Distribution archive not found. ${#DIST_ARCHIVE[@]} matches:" - echo >&2 "${DIST_ARCHIVE[@]}" - exit 1 -fi - -# -# Check for all expected files -# -kokoro/common/check_missing_dist_files.sh ${DIST_ARCHIVE} - -# -# Extract to a temporary directory -# -if [[ -z ${DIST_WORK_ROOT:-} ]]; then - # If you want to preserve the extracted sources, set the DIST_WORK_ROOT - # environment variable to an existing directory that should be used. - DIST_WORK_ROOT=$(mktemp -d) - function cleanup_work_root() { - echo "Cleaning up temporary directory ${DIST_WORK_ROOT}..." - rm -rf ${DIST_WORK_ROOT} - } - trap cleanup_work_root EXIT -fi - -tar -C ${DIST_WORK_ROOT} --strip-components=1 -axf ${DIST_ARCHIVE} - -# -# Run tests using extracted sources -# -SOURCE_DIR=${DIST_WORK_ROOT} \ -CMAKE_GENERATOR=Ninja \ -CTEST_PARALLEL_LEVEL=$(nproc) \ -kokoro/common/cmake.sh - -echo "PASS" diff --git a/kokoro/linux/cmake_distcheck/presubmit.cfg b/kokoro/linux/cmake_distcheck/presubmit.cfg deleted file mode 100644 index 6ef4c890b402..000000000000 --- a/kokoro/linux/cmake_distcheck/presubmit.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/cmake_distcheck/build.sh" -timeout_mins: 1440 diff --git a/kokoro/linux/cmake_install/build.sh b/kokoro/linux/cmake_install/build.sh new file mode 100755 index 000000000000..7fdf267f74bf --- /dev/null +++ b/kokoro/linux/cmake_install/build.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Build file to build, install, and test using CMake. + +set -eux + +# Change to repo root +cd $(dirname $0)/../../.. +GIT_REPO_ROOT=`pwd` + +CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671 + +# Update git submodules +git submodule update --init --recursive + +tmpfile=$(mktemp -u) + +docker run \ + --cidfile $tmpfile \ + -v $GIT_REPO_ROOT:/workspace \ + $CONTAINER_IMAGE \ + "/install.sh && /test.sh \ + -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON \ + -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF \ + -Dprotobuf_BUILD_CONFORMANCE=ON" + + +# Save logs for Kokoro +docker cp \ + `cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR diff --git a/kokoro/linux/cmake_install/continuous.cfg b/kokoro/linux/cmake_install/continuous.cfg new file mode 100644 index 000000000000..f1ae0b351f8d --- /dev/null +++ b/kokoro/linux/cmake_install/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/cmake_install/build.sh" +timeout_mins: 1440 + +action { + define_artifacts { + regex: "**/sponge_log.*" + } +} diff --git a/kokoro/linux/cmake_install/presubmit.cfg b/kokoro/linux/cmake_install/presubmit.cfg new file mode 100644 index 000000000000..f1ae0b351f8d --- /dev/null +++ b/kokoro/linux/cmake_install/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/cmake_install/build.sh" +timeout_mins: 1440 + +action { + define_artifacts { + regex: "**/sponge_log.*" + } +} diff --git a/kokoro/linux/cmake_ninja/build.sh b/kokoro/linux/cmake_ninja/build.sh new file mode 100755 index 000000000000..21cc01e56ac0 --- /dev/null +++ b/kokoro/linux/cmake_ninja/build.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Build file to set up and run tests using CMake with the Ninja generator. + +set -eux + +# Change to repo root +cd $(dirname $0)/../../.. +GIT_REPO_ROOT=`pwd` + +CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671 + +# Update git submodules +git submodule update --init --recursive + +tmpfile=$(mktemp -u) + +docker run \ + --cidfile $tmpfile \ + -v $GIT_REPO_ROOT:/workspace \ + $CONTAINER_IMAGE \ + /test.sh -G Ninja -Dprotobuf_BUILD_CONFORMANCE=ON + +# Save logs for Kokoro +docker cp \ + `cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR diff --git a/kokoro/linux/cmake_ninja/continuous.cfg b/kokoro/linux/cmake_ninja/continuous.cfg new file mode 100644 index 000000000000..144fc90a822c --- /dev/null +++ b/kokoro/linux/cmake_ninja/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/cmake_ninja/build.sh" +timeout_mins: 1440 + +action { + define_artifacts { + regex: "**/sponge_log.*" + } +} diff --git a/kokoro/linux/cmake_ninja/presubmit.cfg b/kokoro/linux/cmake_ninja/presubmit.cfg new file mode 100644 index 000000000000..144fc90a822c --- /dev/null +++ b/kokoro/linux/cmake_ninja/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/cmake_ninja/build.sh" +timeout_mins: 1440 + +action { + define_artifacts { + regex: "**/sponge_log.*" + } +} diff --git a/kokoro/linux/cmake_shared/build.sh b/kokoro/linux/cmake_shared/build.sh new file mode 100755 index 000000000000..1fda3dfca49d --- /dev/null +++ b/kokoro/linux/cmake_shared/build.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Build file to set up and run tests via CMake using shared libraries + +set -eux + +# Change to repo root +cd $(dirname $0)/../../.. +GIT_REPO_ROOT=`pwd` + +CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671 + +# Update git submodules +git submodule update --init --recursive + +tmpfile=$(mktemp -u) + +docker run \ + --cidfile $tmpfile \ + -v $GIT_REPO_ROOT:/workspace \ + $CONTAINER_IMAGE \ + /test.sh -Dprotobuf_BUILD_CONFORMANCE=ON -Dprotobuf_BUILD_SHARED_LIBS=ON + +# Save logs for Kokoro +docker cp \ + `cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR diff --git a/kokoro/linux/cmake_shared/continuous.cfg b/kokoro/linux/cmake_shared/continuous.cfg new file mode 100644 index 000000000000..f03bd3945f1a --- /dev/null +++ b/kokoro/linux/cmake_shared/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/cmake/build.sh" +timeout_mins: 1440 + +action { + define_artifacts { + regex: "**/sponge_log.*" + } +} diff --git a/kokoro/linux/cmake_shared/presubmit.cfg b/kokoro/linux/cmake_shared/presubmit.cfg new file mode 100644 index 000000000000..f03bd3945f1a --- /dev/null +++ b/kokoro/linux/cmake_shared/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/cmake/build.sh" +timeout_mins: 1440 + +action { + define_artifacts { + regex: "**/sponge_log.*" + } +} diff --git a/kokoro/linux/cpp_distcheck/build.sh b/kokoro/linux/cpp_distcheck/build.sh deleted file mode 100755 index a28843e9cba1..000000000000 --- a/kokoro/linux/cpp_distcheck/build.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# -# Build file to set up and run tests - -set -ex # exit immediately on error - -# Change to repo root -cd $(dirname $0)/../../.. - -./tests.sh cpp_distcheck - -# Run tests under release docker image. -DOCKER_IMAGE_NAME=protobuf/protoc_$(sha1sum protoc-artifacts/Dockerfile | cut -f1 -d " ") -until docker pull $DOCKER_IMAGE_NAME; do sleep 10; done - -docker run -v $(pwd):/var/local/protobuf --rm $DOCKER_IMAGE_NAME \ - bash -l /var/local/protobuf/tests.sh cpp || FAILED="true" - -# This directory is owned by root. We need to delete it, because otherwise -# Kokoro will attempt to rsync it and fail with a permission error. -rm -rf src/core - -if [ "$FAILED" = "true" ]; then - exit 1 -fi diff --git a/kokoro/linux/dist_install/build.sh b/kokoro/linux/dist_install/build.sh deleted file mode 100755 index c456ee81faa0..000000000000 --- a/kokoro/linux/dist_install/build.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# -# Build file to set up and run tests - -set -ex # exit immediately on error - -# Change to repo root -cd $(dirname $0)/../../.. - -export DOCKERHUB_ORGANIZATION=protobuftesting -export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/java_stretch -export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh -export OUTPUT_DIR=testoutput -export TEST_SET="dist_install" -./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/dist_install/continuous.cfg b/kokoro/linux/dist_install/continuous.cfg deleted file mode 100644 index b1e0b2013d06..000000000000 --- a/kokoro/linux/dist_install/continuous.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/dist_install/build.sh" -timeout_mins: 1440 diff --git a/kokoro/macos/cpp_distcheck/build.sh b/kokoro/macos/cpp_distcheck/build.sh deleted file mode 100755 index d729b63db196..000000000000 --- a/kokoro/macos/cpp_distcheck/build.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# -# Build file to set up and run tests - -# Change to repo root -cd $(dirname $0)/../../.. - -# Prepare worker environment to run tests -source kokoro/macos/prepare_build_macos_rc - -./tests.sh cpp_distcheck diff --git a/kokoro/macos/cpp_distcheck/continuous.cfg b/kokoro/macos/cpp_distcheck/continuous.cfg deleted file mode 100644 index 89441bcca0b7..000000000000 --- a/kokoro/macos/cpp_distcheck/continuous.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/macos/cpp_distcheck/build.sh" -timeout_mins: 1440 diff --git a/kokoro/macos/cpp_distcheck/presubmit.cfg b/kokoro/macos/cpp_distcheck/presubmit.cfg deleted file mode 100644 index 89441bcca0b7..000000000000 --- a/kokoro/macos/cpp_distcheck/presubmit.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/macos/cpp_distcheck/build.sh" -timeout_mins: 1440 diff --git a/kokoro/macos/php7.0_mac/build.sh b/kokoro/macos/php7.0_mac/build.sh deleted file mode 100755 index c6717e071b7f..000000000000 --- a/kokoro/macos/php7.0_mac/build.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# -# Build file to set up and run tests - -# Change to repo root -cd $(dirname $0)/../../.. - -# Prepare worker environment to run tests -source kokoro/macos/prepare_build_macos_rc - -# TODO(mkruskal) Re-enable this once we can get a working PHP 7.0 installed. -#./tests.sh php7.0_mac diff --git a/kokoro/macos/php7.3_mac/build.sh b/kokoro/macos/php7.3_mac/build.sh deleted file mode 100755 index 2688ddbf65f0..000000000000 --- a/kokoro/macos/php7.3_mac/build.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# -# Build file to set up and run tests - -# Change to repo root -cd $(dirname $0)/../../.. - -# Prepare worker environment to run tests -source kokoro/macos/prepare_build_macos_rc - -# TODO(mkruskal) Re-enable this once we can get a working PHP 7.0 installed. -#./tests.sh php7.3_mac diff --git a/kokoro/macos/php74/build.sh b/kokoro/macos/php74/build.sh new file mode 100755 index 000000000000..ff3965756c77 --- /dev/null +++ b/kokoro/macos/php74/build.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Build file to set up and run tests + +# Change to repo root +cd $(dirname $0)/../../.. + +# Prepare worker environment to run tests +source kokoro/macos/prepare_build_macos_rc + +# Install Dependencies +brew install coreutils php@7.4 + +# Configure path +PHP_FOLDER=$(find $HOMEBREW_PREFIX -type d -regex ".*php.*/7.4.[0-9]*") +test ! -z "$PHP_FOLDER" +export PATH="$PHP_FOLDER/bin:$PATH" + +# Test +./tests.sh php_mac diff --git a/kokoro/macos/php7.0_mac/continuous.cfg b/kokoro/macos/php74/continuous.cfg similarity index 65% rename from kokoro/macos/php7.0_mac/continuous.cfg rename to kokoro/macos/php74/continuous.cfg index c2c18119207d..cf7e80beb8bd 100644 --- a/kokoro/macos/php7.0_mac/continuous.cfg +++ b/kokoro/macos/php74/continuous.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/macos/php7.0_mac/build.sh" +build_file: "protobuf/kokoro/macos/php74/build.sh" timeout_mins: 1440 diff --git a/kokoro/macos/php7.3_mac/presubmit.cfg b/kokoro/macos/php74/presubmit.cfg similarity index 65% rename from kokoro/macos/php7.3_mac/presubmit.cfg rename to kokoro/macos/php74/presubmit.cfg index 9a717451d373..cf7e80beb8bd 100644 --- a/kokoro/macos/php7.3_mac/presubmit.cfg +++ b/kokoro/macos/php74/presubmit.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/macos/php7.3_mac/build.sh" +build_file: "protobuf/kokoro/macos/php74/build.sh" timeout_mins: 1440 diff --git a/kokoro/macos/php80/build.sh b/kokoro/macos/php80/build.sh new file mode 100755 index 000000000000..84e2c464ef78 --- /dev/null +++ b/kokoro/macos/php80/build.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Build file to set up and run tests + +# Change to repo root +cd $(dirname $0)/../../.. + +# Prepare worker environment to run tests +source kokoro/macos/prepare_build_macos_rc + +# Install Dependencies +brew install coreutils php@8.0 + +# Configure path +PHP_FOLDER=$(find $HOMEBREW_PREFIX -type d -regex ".*php.*/8.0.[0-9]*") +test ! -z "$PHP_FOLDER" +export PATH="$PHP_FOLDER/bin:$PATH" + +# Test +./tests.sh php_mac diff --git a/kokoro/macos/php7.3_mac/continuous.cfg b/kokoro/macos/php80/continuous.cfg similarity index 65% rename from kokoro/macos/php7.3_mac/continuous.cfg rename to kokoro/macos/php80/continuous.cfg index 9a717451d373..ded43e62c57f 100644 --- a/kokoro/macos/php7.3_mac/continuous.cfg +++ b/kokoro/macos/php80/continuous.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/macos/php7.3_mac/build.sh" +build_file: "protobuf/kokoro/macos/php80/build.sh" timeout_mins: 1440 diff --git a/kokoro/macos/php7.0_mac/presubmit.cfg b/kokoro/macos/php80/presubmit.cfg similarity index 65% rename from kokoro/macos/php7.0_mac/presubmit.cfg rename to kokoro/macos/php80/presubmit.cfg index c2c18119207d..ded43e62c57f 100644 --- a/kokoro/macos/php7.0_mac/presubmit.cfg +++ b/kokoro/macos/php80/presubmit.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/macos/php7.0_mac/build.sh" +build_file: "protobuf/kokoro/macos/php80/build.sh" timeout_mins: 1440 diff --git a/kokoro/macos/prepare_build_macos_rc b/kokoro/macos/prepare_build_macos_rc index 8e0a87edbbf8..7ec2832b8596 100755 --- a/kokoro/macos/prepare_build_macos_rc +++ b/kokoro/macos/prepare_build_macos_rc @@ -4,20 +4,16 @@ set -eux +export HOMEBREW_PREFIX=$(brew --prefix) + ## # Select Xcode version - -# Remember to update the Xcode version when Xcode_11.3.app is not available. -# If xcode is not available, it will probably encounter the failure for -# "autom4te: need GNU m4 1.4 or later: /usr/bin/m4" -# go/kokoro/userdocs/macos/selecting_xcode.md for more information. -export DEVELOPER_DIR=/Applications/Xcode_11.3.app/Contents/Developer +export DEVELOPER_DIR=/Applications/Xcode_13.3.1.app/Contents/Developer +sudo xcode-select -s "${DEVELOPER_DIR}" ## -# Select C/C++ compilers - -export CC=gcc -export CXX=g++ +# Use Python 2 by default (for googletest) +pyenv global 2.7.18 ## # Install Tox @@ -27,14 +23,18 @@ if [[ "${KOKORO_INSTALL_TOX:-}" == "yes" ]] ; then fi ## -# Install RVM - +# Setup RVM if [[ "${KOKORO_INSTALL_RVM:-}" == "yes" ]] ; then - curl -sSL https://rvm.io/mpapis.asc | gpg --import - - curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - + git config --global --add safe.directory $HOMEBREW_PREFIX/Library/Taps/homebrew/homebrew-cask + git config --global --add safe.directory $HOMEBREW_PREFIX/Library/Taps/homebrew/homebrew-core + git config --global --add safe.directory $HOMEBREW_PREFIX/Library/Taps/homebrew/homebrew-services + sudo chown -R $(whoami) $HOME/.rvm/ +fi - # Old OpenSSL versions cannot handle the SSL certificate used by - # https://get.rvm.io, so as a workaround we download RVM directly from - # GitHub. See this issue for details: https://github.com/rvm/rvm/issues/5133 - curl -sSL https://raw.githubusercontent.com/rvm/rvm/master/binscripts/rvm-installer | bash -s master --ruby +# "Install" valgrind if it doesn't exist +## +if [ ! -x "$(command -v valgrind)" ]; then + echo "#! /bin/bash" > valgrind + chmod ug+x valgrind + sudo mv valgrind /usr/local/bin/valgrind fi diff --git a/kokoro/release/ruby/macos/build_artifacts.sh b/kokoro/release/ruby/macos/build_artifacts.sh index c68b63cc43d7..a109d45b6166 100755 --- a/kokoro/release/ruby/macos/build_artifacts.sh +++ b/kokoro/release/ruby/macos/build_artifacts.sh @@ -12,8 +12,5 @@ export ARTIFACT_DIR=$(pwd)/artifacts # ruby environment bash kokoro/release/ruby/macos/ruby/ruby_build_environment.sh -gem install rubygems-update -update_rubygems - # build artifacts bash kokoro/release/ruby/macos/ruby/ruby_build.sh diff --git a/kokoro/release/ruby/macos/ruby/ruby_build.sh b/kokoro/release/ruby/macos/ruby/ruby_build.sh index 55773b21a9e5..bbfc631197e8 100755 --- a/kokoro/release/ruby/macos/ruby/ruby_build.sh +++ b/kokoro/release/ruby/macos/ruby/ruby_build.sh @@ -3,7 +3,6 @@ set -ex # Build protoc -use_bazel.sh 5.1.1 bazel build //:protoc export PROTOC=$PWD/bazel-bin/protoc diff --git a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh index 2a9cb1687b09..ae973fce4350 100755 --- a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh +++ b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh @@ -4,13 +4,9 @@ set -ex set +ex # rvm script is very verbose and exits with errorcode -curl -sSL https://rvm.io/mpapis.asc | gpg --import - -curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - - -# Old OpenSSL versions cannot handle the SSL certificate used by -# https://get.rvm.io, so as a workaround we download RVM directly from -# GitHub. See this issue for details: https://github.com/rvm/rvm/issues/5133 -curl -sSL https://raw.githubusercontent.com/rvm/rvm/master/binscripts/rvm-installer | bash -s master --ruby +# Fix permissions +sudo chown -R $(whoami) $HOME/.rvm/ +sudo chown -R $(whoami) /Library/Ruby/ source $HOME/.rvm/scripts/rvm set -e # rvm commands are very verbose diff --git a/kokoro/windows/bazel/build.bat b/kokoro/windows/bazel/build.bat new file mode 100644 index 000000000000..52b83f466685 --- /dev/null +++ b/kokoro/windows/bazel/build.bat @@ -0,0 +1,4 @@ +@rem enter repo root +cd /d %~dp0\..\..\.. + +@rem TODO(mkruskal) Implement tests diff --git a/kokoro/windows/bazel/continuous.cfg b/kokoro/windows/bazel/continuous.cfg new file mode 100644 index 000000000000..37e89e068b3a --- /dev/null +++ b/kokoro/windows/bazel/continuous.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/windows/cmake/build.bat" +timeout_mins: 1440 diff --git a/kokoro/windows/bazel/presubmit.cfg b/kokoro/windows/bazel/presubmit.cfg new file mode 100644 index 000000000000..37e89e068b3a --- /dev/null +++ b/kokoro/windows/bazel/presubmit.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/windows/cmake/build.bat" +timeout_mins: 1440 diff --git a/kokoro/windows/cmake/build.bat b/kokoro/windows/cmake/build.bat new file mode 100644 index 000000000000..52b83f466685 --- /dev/null +++ b/kokoro/windows/cmake/build.bat @@ -0,0 +1,4 @@ +@rem enter repo root +cd /d %~dp0\..\..\.. + +@rem TODO(mkruskal) Implement tests diff --git a/kokoro/windows/cmake/continuous.cfg b/kokoro/windows/cmake/continuous.cfg new file mode 100644 index 000000000000..37e89e068b3a --- /dev/null +++ b/kokoro/windows/cmake/continuous.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/windows/cmake/build.bat" +timeout_mins: 1440 diff --git a/kokoro/windows/cmake/presubmit.cfg b/kokoro/windows/cmake/presubmit.cfg new file mode 100644 index 000000000000..37e89e068b3a --- /dev/null +++ b/kokoro/windows/cmake/presubmit.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/windows/cmake/build.bat" +timeout_mins: 1440 diff --git a/kokoro/windows/cmake_install/build.bat b/kokoro/windows/cmake_install/build.bat new file mode 100644 index 000000000000..52b83f466685 --- /dev/null +++ b/kokoro/windows/cmake_install/build.bat @@ -0,0 +1,4 @@ +@rem enter repo root +cd /d %~dp0\..\..\.. + +@rem TODO(mkruskal) Implement tests diff --git a/kokoro/linux/cmake_distcheck/continuous.cfg b/kokoro/windows/cmake_install/continuous.cfg similarity index 63% rename from kokoro/linux/cmake_distcheck/continuous.cfg rename to kokoro/windows/cmake_install/continuous.cfg index 6ef4c890b402..2efc0dced2f7 100644 --- a/kokoro/linux/cmake_distcheck/continuous.cfg +++ b/kokoro/windows/cmake_install/continuous.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/linux/cmake_distcheck/build.sh" +build_file: "protobuf/kokoro/windows/cmake_install/build.bat" timeout_mins: 1440 diff --git a/kokoro/linux/dist_install/presubmit.cfg b/kokoro/windows/cmake_install/presubmit.cfg similarity index 63% rename from kokoro/linux/dist_install/presubmit.cfg rename to kokoro/windows/cmake_install/presubmit.cfg index b1e0b2013d06..2efc0dced2f7 100644 --- a/kokoro/linux/dist_install/presubmit.cfg +++ b/kokoro/windows/cmake_install/presubmit.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/linux/dist_install/build.sh" +build_file: "protobuf/kokoro/windows/cmake_install/build.bat" timeout_mins: 1440 diff --git a/kokoro/windows/cmake_nmake/build.bat b/kokoro/windows/cmake_nmake/build.bat new file mode 100644 index 000000000000..52b83f466685 --- /dev/null +++ b/kokoro/windows/cmake_nmake/build.bat @@ -0,0 +1,4 @@ +@rem enter repo root +cd /d %~dp0\..\..\.. + +@rem TODO(mkruskal) Implement tests diff --git a/kokoro/linux/cpp_distcheck/continuous.cfg b/kokoro/windows/cmake_nmake/continuous.cfg similarity index 64% rename from kokoro/linux/cpp_distcheck/continuous.cfg rename to kokoro/windows/cmake_nmake/continuous.cfg index 4289f6a715f4..3c279fe42161 100644 --- a/kokoro/linux/cpp_distcheck/continuous.cfg +++ b/kokoro/windows/cmake_nmake/continuous.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/linux/cpp_distcheck/build.sh" +build_file: "protobuf/kokoro/windows/cmake_nmake/build.bat" timeout_mins: 1440 diff --git a/kokoro/linux/cpp_distcheck/presubmit.cfg b/kokoro/windows/cmake_nmake/presubmit.cfg similarity index 64% rename from kokoro/linux/cpp_distcheck/presubmit.cfg rename to kokoro/windows/cmake_nmake/presubmit.cfg index 4289f6a715f4..3c279fe42161 100644 --- a/kokoro/linux/cpp_distcheck/presubmit.cfg +++ b/kokoro/windows/cmake_nmake/presubmit.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/linux/cpp_distcheck/build.sh" +build_file: "protobuf/kokoro/windows/cmake_nmake/build.bat" timeout_mins: 1440 diff --git a/kokoro/windows/cmake_shared/build.bat b/kokoro/windows/cmake_shared/build.bat new file mode 100644 index 000000000000..52b83f466685 --- /dev/null +++ b/kokoro/windows/cmake_shared/build.bat @@ -0,0 +1,4 @@ +@rem enter repo root +cd /d %~dp0\..\..\.. + +@rem TODO(mkruskal) Implement tests diff --git a/kokoro/windows/cmake_shared/continuous.cfg b/kokoro/windows/cmake_shared/continuous.cfg new file mode 100644 index 000000000000..37e89e068b3a --- /dev/null +++ b/kokoro/windows/cmake_shared/continuous.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/windows/cmake/build.bat" +timeout_mins: 1440 diff --git a/kokoro/windows/cmake_shared/presubmit.cfg b/kokoro/windows/cmake_shared/presubmit.cfg new file mode 100644 index 000000000000..37e89e068b3a --- /dev/null +++ b/kokoro/windows/cmake_shared/presubmit.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/windows/cmake/build.bat" +timeout_mins: 1440 diff --git a/objectivec/GPBApi.pbobjc.h b/objectivec/GPBApi.pbobjc.h index 871e90885a61..1848aa6a911e 100644 --- a/objectivec/GPBApi.pbobjc.h +++ b/objectivec/GPBApi.pbobjc.h @@ -74,12 +74,12 @@ GPB_FINAL @interface GPBApi : GPBMessage /** The methods of this interface, in unspecified order. */ @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *methodsArray; -/** The number of items in @c methodsArray without causing the array to be created. */ +/** The number of items in @c methodsArray without causing the container to be created. */ @property(nonatomic, readonly) NSUInteger methodsArray_Count; /** Any metadata attached to the interface. */ @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *optionsArray; -/** The number of items in @c optionsArray without causing the array to be created. */ +/** The number of items in @c optionsArray without causing the container to be created. */ @property(nonatomic, readonly) NSUInteger optionsArray_Count; /** @@ -115,7 +115,7 @@ GPB_FINAL @interface GPBApi : GPBMessage /** Included interfaces. See [Mixin][]. */ @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *mixinsArray; -/** The number of items in @c mixinsArray without causing the array to be created. */ +/** The number of items in @c mixinsArray without causing the container to be created. */ @property(nonatomic, readonly) NSUInteger mixinsArray_Count; /** The source syntax of the service. */ @@ -169,7 +169,7 @@ GPB_FINAL @interface GPBMethod : GPBMessage /** Any metadata attached to the method. */ @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *optionsArray; -/** The number of items in @c optionsArray without causing the array to be created. */ +/** The number of items in @c optionsArray without causing the container to be created. */ @property(nonatomic, readonly) NSUInteger optionsArray_Count; /** The source syntax of this method. */ diff --git a/objectivec/GPBFieldMask.pbobjc.h b/objectivec/GPBFieldMask.pbobjc.h index c4667b44db26..b7eccff21ca6 100644 --- a/objectivec/GPBFieldMask.pbobjc.h +++ b/objectivec/GPBFieldMask.pbobjc.h @@ -247,7 +247,7 @@ GPB_FINAL @interface GPBFieldMask : GPBMessage /** The set of field mask paths. */ @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *pathsArray; -/** The number of items in @c pathsArray without causing the array to be created. */ +/** The number of items in @c pathsArray without causing the container to be created. */ @property(nonatomic, readonly) NSUInteger pathsArray_Count; @end diff --git a/objectivec/GPBStruct.pbobjc.h b/objectivec/GPBStruct.pbobjc.h index dd6ab28f0e81..ff4eefde52dd 100644 --- a/objectivec/GPBStruct.pbobjc.h +++ b/objectivec/GPBStruct.pbobjc.h @@ -87,7 +87,7 @@ GPB_FINAL @interface GPBStruct : GPBMessage /** Unordered map of dynamically typed values. */ @property(nonatomic, readwrite, strong, null_resettable) NSMutableDictionary *fields; -/** The number of items in @c fields without causing the array to be created. */ +/** The number of items in @c fields without causing the container to be created. */ @property(nonatomic, readonly) NSUInteger fields_Count; @end @@ -178,7 +178,7 @@ GPB_FINAL @interface GPBListValue : GPBMessage /** Repeated field of dynamically typed values. */ @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *valuesArray; -/** The number of items in @c valuesArray without causing the array to be created. */ +/** The number of items in @c valuesArray without causing the container to be created. */ @property(nonatomic, readonly) NSUInteger valuesArray_Count; @end diff --git a/objectivec/GPBType.pbobjc.h b/objectivec/GPBType.pbobjc.h index b0230501cf3c..969219f03f40 100644 --- a/objectivec/GPBType.pbobjc.h +++ b/objectivec/GPBType.pbobjc.h @@ -195,17 +195,17 @@ GPB_FINAL @interface GPBType : GPBMessage /** The list of fields. */ @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *fieldsArray; -/** The number of items in @c fieldsArray without causing the array to be created. */ +/** The number of items in @c fieldsArray without causing the container to be created. */ @property(nonatomic, readonly) NSUInteger fieldsArray_Count; /** The list of types appearing in `oneof` definitions in this type. */ @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *oneofsArray; -/** The number of items in @c oneofsArray without causing the array to be created. */ +/** The number of items in @c oneofsArray without causing the container to be created. */ @property(nonatomic, readonly) NSUInteger oneofsArray_Count; /** The protocol buffer options. */ @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *optionsArray; -/** The number of items in @c optionsArray without causing the array to be created. */ +/** The number of items in @c optionsArray without causing the container to be created. */ @property(nonatomic, readonly) NSUInteger optionsArray_Count; /** The source context. */ @@ -279,7 +279,7 @@ GPB_FINAL @interface GPBField : GPBMessage /** The protocol buffer options. */ @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *optionsArray; -/** The number of items in @c optionsArray without causing the array to be created. */ +/** The number of items in @c optionsArray without causing the container to be created. */ @property(nonatomic, readonly) NSUInteger optionsArray_Count; /** The field JSON name. */ @@ -334,12 +334,12 @@ GPB_FINAL @interface GPBEnum : GPBMessage /** Enum value definitions. */ @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *enumvalueArray; -/** The number of items in @c enumvalueArray without causing the array to be created. */ +/** The number of items in @c enumvalueArray without causing the container to be created. */ @property(nonatomic, readonly) NSUInteger enumvalueArray_Count; /** Protocol buffer options. */ @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *optionsArray; -/** The number of items in @c optionsArray without causing the array to be created. */ +/** The number of items in @c optionsArray without causing the container to be created. */ @property(nonatomic, readonly) NSUInteger optionsArray_Count; /** The source context. */ @@ -385,7 +385,7 @@ GPB_FINAL @interface GPBEnumValue : GPBMessage /** Protocol buffer options. */ @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *optionsArray; -/** The number of items in @c optionsArray without causing the array to be created. */ +/** The number of items in @c optionsArray without causing the container to be created. */ @property(nonatomic, readonly) NSUInteger optionsArray_Count; @end diff --git a/objectivec/GPBUtilities.m b/objectivec/GPBUtilities.m index b72d0e2de3b0..cd0910c5776f 100644 --- a/objectivec/GPBUtilities.m +++ b/objectivec/GPBUtilities.m @@ -2248,7 +2248,7 @@ void GPBMaybeClearOneof(GPBMessage *self, GPBOneofDescriptor *oneof, NSCAssert([[self descriptor] oneofWithName:oneof.name] == oneof, @"OneofDescriptor %@ doesn't appear to be for %@ messages.", oneof.name, [self class]); - GPBFieldDescriptor *firstField = oneof->fields_[0]; + GPBFieldDescriptor *firstField __unused = oneof->fields_[0]; NSCAssert(firstField->description_->hasIndex == oneofHasIndex, @"Internal error, oneofHasIndex (%d) doesn't match (%d).", firstField->description_->hasIndex, oneofHasIndex); diff --git a/objectivec/generate_well_known_types.sh b/objectivec/generate_well_known_types.sh index 1b9de6ec55ee..e6c9b044d145 100755 --- a/objectivec/generate_well_known_types.sh +++ b/objectivec/generate_well_known_types.sh @@ -73,6 +73,7 @@ for PROTO_FILE in "${RUNTIME_PROTO_FILES[@]}"; do if ! diff "${ObjCDir}/GPB${OBJC_NAME}${EXT}" "${TMP_DIR}/${DIR}/${OBJC_NAME}${EXT}" > /dev/null 2>&1 ; then if [[ "${CHECK_ONLY}" == 1 ]] ; then echo "ERROR: The WKTs need to be regenerated! Run $0" + diff -u "${ObjCDir}/GPB${OBJC_NAME}${EXT}" "${TMP_DIR}/${DIR}/${OBJC_NAME}${EXT}" exit 1 fi diff --git a/php/BUILD.bazel b/php/BUILD.bazel index 346095957452..dfb0976fcadb 100644 --- a/php/BUILD.bazel +++ b/php/BUILD.bazel @@ -12,7 +12,9 @@ pkg_files( "src/Google/Protobuf/**/*.php", "tests/*.php", "tests/*.sh", + "tests/generated_previous/**/*.php", "tests/proto/**/*.proto", + "tests/proto_previous/*.proto", ]) + [ "BUILD.bazel", "README.md", diff --git a/php/composer.json b/php/composer.json index 756704f8fcaa..436142ff7f27 100644 --- a/php/composer.json +++ b/php/composer.json @@ -20,7 +20,10 @@ "autoload-dev": { "psr-4": { "": "tmp" - } + }, + "classmap": [ + "tests/generated_previous" + ] }, "scripts": { "test_c": "./generate_test_protos.sh && ./tests/compile_extension.sh && php -dextension=ext/google/protobuf/modules/protobuf.so vendor/bin/phpunit --bootstrap tests/force_c_ext.php tests", diff --git a/php/ext/google/protobuf/def.c b/php/ext/google/protobuf/def.c index dfb96f283f69..8c3266eb07f5 100644 --- a/php/ext/google/protobuf/def.c +++ b/php/ext/google/protobuf/def.c @@ -162,7 +162,7 @@ static void EnumDescriptor_FromEnumDef(zval *val, const upb_EnumDef *m) { ZVAL_NULL(val); } else { char *classname = - GetPhpClassname(upb_EnumDef_File(m), upb_EnumDef_FullName(m)); + GetPhpClassname(upb_EnumDef_File(m), upb_EnumDef_FullName(m), false); zend_string *str = zend_string_init(classname, strlen(classname), 0); zend_class_entry *ce = zend_lookup_class(str); // May autoload the class. @@ -456,6 +456,44 @@ PHP_METHOD(FieldDescriptor, getEnumType) { RETURN_COPY_VALUE(&ret); } +/* + * FieldDescriptor::getContainingOneof() + * + * Returns the OneofDescriptor for this field, or null if it is not inside + * a oneof. + */ +PHP_METHOD(FieldDescriptor, getContainingOneof) { + FieldDescriptor *intern = (FieldDescriptor*)Z_OBJ_P(getThis()); + const upb_OneofDef *o = upb_FieldDef_ContainingOneof(intern->fielddef); + zval ret; + + if (!o) { + RETURN_NULL(); + } + + OneofDescriptor_FromOneofDef(&ret, o); + RETURN_COPY_VALUE(&ret); +} + +/* + * FieldDescriptor::getRealContainingOneof() + * + * Returns the non-synthetic OneofDescriptor for this field, or null if it is + * not inside a oneof. + */ +PHP_METHOD(FieldDescriptor, getRealContainingOneof) { + FieldDescriptor *intern = (FieldDescriptor*)Z_OBJ_P(getThis()); + const upb_OneofDef *o = upb_FieldDef_RealContainingOneof(intern->fielddef); + zval ret; + + if (!o) { + RETURN_NULL(); + } + + OneofDescriptor_FromOneofDef(&ret, o); + RETURN_COPY_VALUE(&ret); +} + /* * FieldDescriptor::getMessageType() * @@ -482,6 +520,8 @@ static zend_function_entry FieldDescriptor_methods[] = { PHP_ME(FieldDescriptor, getType, arginfo_void, ZEND_ACC_PUBLIC) PHP_ME(FieldDescriptor, isMap, arginfo_void, ZEND_ACC_PUBLIC) PHP_ME(FieldDescriptor, getEnumType, arginfo_void, ZEND_ACC_PUBLIC) + PHP_ME(FieldDescriptor, getContainingOneof, arginfo_void, ZEND_ACC_PUBLIC) + PHP_ME(FieldDescriptor, getRealContainingOneof, arginfo_void, ZEND_ACC_PUBLIC) PHP_ME(FieldDescriptor, getMessageType, arginfo_void, ZEND_ACC_PUBLIC) ZEND_FE_END }; @@ -499,19 +539,24 @@ static void Descriptor_destructor(zend_object* obj) { } static zend_class_entry *Descriptor_GetGeneratedClass(const upb_MessageDef *m) { - char *classname = - GetPhpClassname(upb_MessageDef_File(m), upb_MessageDef_FullName(m)); - zend_string *str = zend_string_init(classname, strlen(classname), 0); - zend_class_entry *ce = zend_lookup_class(str); // May autoload the class. + for (int i = 0; i < 2; ++i) { + char *classname = + GetPhpClassname(upb_MessageDef_File(m), upb_MessageDef_FullName(m), (bool)i); + zend_string *str = zend_string_init(classname, strlen(classname), 0); + zend_class_entry *ce = zend_lookup_class(str); // May autoload the class. - zend_string_release (str); + zend_string_release (str); + free(classname); - if (!ce) { - zend_error(E_ERROR, "Couldn't load generated class %s", classname); + if (ce) { + return ce; + } } - free(classname); - return ce; + char *classname = + GetPhpClassname(upb_MessageDef_File(m), upb_MessageDef_FullName(m), false); + zend_error(E_ERROR, "Couldn't load generated class %s", classname); + return NULL; } void Descriptor_FromMessageDef(zval *val, const upb_MessageDef *m) { diff --git a/php/ext/google/protobuf/names.c b/php/ext/google/protobuf/names.c index 5d7b68aaf5dd..e359d01728d9 100644 --- a/php/ext/google/protobuf/names.c +++ b/php/ext/google/protobuf/names.c @@ -82,12 +82,15 @@ const char *const kReservedNames[] = { "global", "goto", "insteadof", "interface", "isset", "list", "match", "namespace", "new", "object", "or", "parent", "print", "private", "protected", - "public", "require", "require_once", "return", "self", - "static", "switch", "throw", "trait", "try", - "unset", "use", "var", "while", "xor", - "yield", "int", "float", "bool", "string", - "true", "false", "null", "void", "iterable", - NULL}; + "public", "readonly", "require", "require_once", "return", + "self", "static", "switch", "throw", "trait", + "try", "unset", "use", "var", "while", + "xor", "yield", "int", "float", "bool", + "string", "true", "false", "null", "void", + "iterable", NULL}; + +const char *const kPreviouslyUnreservedNames[] = { + "readonly", NULL}; bool is_reserved_name(const char* name) { int i; @@ -99,6 +102,15 @@ bool is_reserved_name(const char* name) { return false; } +bool is_previously_unreserved_name(const char* name) { + for (int i = 0; kPreviouslyUnreservedNames[i]; i++) { + if (strcmp(kPreviouslyUnreservedNames[i], name) == 0) { + return true; + } + } + return false; +} + static char nolocale_tolower(char ch) { if (ch >= 'A' && ch <= 'Z') { return ch - ('A' - 'a'); @@ -115,17 +127,22 @@ static char nolocale_toupper(char ch) { } } -static bool is_reserved(const char *segment, int length) { +static char *strdup_nolocale_lower(char *str, int length) { + char* lower = malloc(length + 1); + lower[length] = '\0'; + for(int i = 0; i < length; ++i) { + lower[i] = nolocale_tolower(str[i]); + } + return lower; +} + +static bool is_reserved(const char *segment, int length, bool previous) { bool result; - char* lower = calloc(1, length + 1); - memcpy(lower, segment, length); - int i = 0; - while(lower[i]) { - lower[i] = nolocale_tolower(lower[i]); - i++; - } - lower[length] = 0; + char* lower = strdup_nolocale_lower(segment, length); result = is_reserved_name(lower); + if (result && previous && is_previously_unreserved_name(lower)) { + result = false; + } free(lower); return result; } @@ -133,11 +150,12 @@ static bool is_reserved(const char *segment, int length) { static void fill_prefix(const char *segment, int length, const char *prefix_given, const char *package_name, - stringsink *classname) { + stringsink *classname, + bool previous) { if (prefix_given != NULL && strcmp(prefix_given, "") != 0) { stringsink_string(classname, prefix_given, strlen(prefix_given)); } else { - if (is_reserved(segment, length)) { + if (is_reserved(segment, length, previous)) { if (package_name != NULL && strcmp("google.protobuf", package_name) == 0) { stringsink_string(classname, "GPB", 3); @@ -160,7 +178,7 @@ static void fill_segment(const char *segment, int length, } static void fill_namespace(const char *package, const char *php_namespace, - stringsink *classname) { + stringsink *classname, bool previous) { if (php_namespace != NULL) { if (strlen(php_namespace) != 0) { stringsink_string(classname, php_namespace, strlen(php_namespace)); @@ -174,7 +192,7 @@ static void fill_namespace(const char *package, const char *php_namespace, while (j < package_len && package[j] != '.') { j++; } - fill_prefix(package + i, j - i, "", package, classname); + fill_prefix(package + i, j - i, "", package, classname, previous); fill_segment(package + i, j - i, classname, true); stringsink_string(classname, "\\", 1); i = j + 1; @@ -185,7 +203,8 @@ static void fill_namespace(const char *package, const char *php_namespace, static void fill_classname(const char *fullname, const char *package, const char *prefix, - stringsink *classname) { + stringsink *classname, + bool previous) { int classname_start = 0; if (package != NULL) { size_t package_len = strlen(package); @@ -199,7 +218,7 @@ static void fill_classname(const char *fullname, while (j < fullname_len && fullname[j] != '.') { j++; } - fill_prefix(fullname + i, j - i, prefix, package, classname); + fill_prefix(fullname + i, j - i, prefix, package, classname, previous); fill_segment(fullname + i, j - i, classname, false); if (j != fullname_len) { stringsink_string(classname, "\\", 1); @@ -215,7 +234,7 @@ char *str_view_dup(upb_StringView str) { return ret; } -char *GetPhpClassname(const upb_FileDef *file, const char *fullname) { +char *GetPhpClassname(const upb_FileDef *file, const char *fullname, bool previous) { // Prepend '.' to package name to make it absolute. In the 5 additional // bytes allocated, one for '.', one for trailing 0, and 3 for 'GPB' if // given message is google.protobuf.Empty. @@ -234,8 +253,8 @@ char *GetPhpClassname(const upb_FileDef *file, const char *fullname) { stringsink namesink; stringsink_init(&namesink); - fill_namespace(package, php_namespace, &namesink); - fill_classname(fullname, package, prefix, &namesink); + fill_namespace(package, php_namespace, &namesink, previous); + fill_classname(fullname, package, prefix, &namesink, previous); stringsink_string(&namesink, "\0", 1); ret = strdup(namesink.ptr); stringsink_uninit(&namesink); @@ -243,3 +262,26 @@ char *GetPhpClassname(const upb_FileDef *file, const char *fullname) { free(prefix); return ret; } + +bool IsPreviouslyUnreservedClassName(const char* fullname) { + const char *classname = strrchr(fullname, '\\'); + if (classname) { + classname += 1; + } else { + classname = fullname; + } + if (strncmp(classname, "PB", 2) != 0) { + return false; + } + classname += 2; + int length = strlen(classname); + char* lower = strdup_nolocale_lower(classname, length); + for (int j = 0; kPreviouslyUnreservedNames[j]; j++) { + if (strcmp(kPreviouslyUnreservedNames[j], lower) == 0) { + free(lower); + return true; + } + } + free(lower); + return false; +} diff --git a/php/ext/google/protobuf/names.h b/php/ext/google/protobuf/names.h index 86af799ac0c2..cc42dc85a0d5 100644 --- a/php/ext/google/protobuf/names.h +++ b/php/ext/google/protobuf/names.h @@ -35,6 +35,7 @@ // Translates a protobuf symbol name (eg. foo.bar.Baz) into a PHP class name // (eg. \Foo\Bar\Baz). -char *GetPhpClassname(const upb_FileDef *file, const char *fullname); +char *GetPhpClassname(const upb_FileDef *file, const char *fullname, bool previous); +bool IsPreviouslyUnreservedClassName(const char* fullname); #endif // PHP_PROTOBUF_NAMES_H_ diff --git a/php/ext/google/protobuf/package.xml b/php/ext/google/protobuf/package.xml index bf3d90eb8688..c2917142652a 100644 --- a/php/ext/google/protobuf/package.xml +++ b/php/ext/google/protobuf/package.xml @@ -5,16 +5,16 @@ Google's language-neutral, platform-neutral, extensible mechanism for serializing structured data. https://developers.google.com/protocol-buffers/ - Bo Yang - stanleycheung - protobuf-opensource@google.com + Protobuf Team + protobufpackages + protobuf-packages@google.com yes - 2022-06-23 - + 2022-07-25 + - 3.21.2 - 3.21.2 + 3.21.4 + 3.21.4 stable @@ -1358,5 +1358,35 @@ G A release. + + + 3.21.3 + 3.21.3 + + + stable + stable + + 2022-07-21 + + BSD-3-Clause + + + + + + 3.21.4 + 3.21.4 + + + stable + stable + + 2022-07-25 + + BSD-3-Clause + + + diff --git a/php/ext/google/protobuf/protobuf.c b/php/ext/google/protobuf/protobuf.c index c786b6eca010..a5aba23f476b 100644 --- a/php/ext/google/protobuf/protobuf.c +++ b/php/ext/google/protobuf/protobuf.c @@ -242,13 +242,19 @@ bool ObjCache_Get(const void *upb_obj, zval *val) { // ----------------------------------------------------------------------------- void NameMap_AddMessage(const upb_MessageDef *m) { - char *k = GetPhpClassname(upb_MessageDef_File(m), upb_MessageDef_FullName(m)); - zend_hash_str_add_ptr(&PROTOBUF_G(name_msg_cache), k, strlen(k), (void*)m); - free(k); + for (int i = 0; i < 2; ++i) { + char *k = GetPhpClassname(upb_MessageDef_File(m), upb_MessageDef_FullName(m), (bool)i); + zend_hash_str_add_ptr(&PROTOBUF_G(name_msg_cache), k, strlen(k), (void*)m); + if (!IsPreviouslyUnreservedClassName(k)) { + free(k); + return; + } + free(k); + } } void NameMap_AddEnum(const upb_EnumDef *e) { - char *k = GetPhpClassname(upb_EnumDef_File(e), upb_EnumDef_FullName(e)); + char *k = GetPhpClassname(upb_EnumDef_File(e), upb_EnumDef_FullName(e), false); zend_hash_str_add_ptr(&PROTOBUF_G(name_enum_cache), k, strlen(k), (void*)e); free(k); } diff --git a/php/ext/google/protobuf/protobuf.h b/php/ext/google/protobuf/protobuf.h index c5dac329093d..e63e9e83039f 100644 --- a/php/ext/google/protobuf/protobuf.h +++ b/php/ext/google/protobuf/protobuf.h @@ -127,7 +127,7 @@ ZEND_BEGIN_ARG_INFO_EX(arginfo_setter, 0, 0, 1) ZEND_ARG_INFO(0, value) ZEND_END_ARG_INFO() -#define PHP_PROTOBUF_VERSION "3.21.2" +#define PHP_PROTOBUF_VERSION "3.21.4" // ptr -> PHP object cache. This is a weak map that caches lazily-created // wrapper objects around upb types: diff --git a/php/src/Google/Protobuf/Internal/Descriptor.php b/php/src/Google/Protobuf/Internal/Descriptor.php index a7f80d534e01..51a34d6bc9f3 100644 --- a/php/src/Google/Protobuf/Internal/Descriptor.php +++ b/php/src/Google/Protobuf/Internal/Descriptor.php @@ -45,6 +45,7 @@ class Descriptor private $enum_type = []; private $klass; private $legacy_klass; + private $previous_klass; private $options; private $oneof_decl = []; @@ -162,6 +163,16 @@ public function getLegacyClass() return $this->legacy_klass; } + public function setPreviouslyUnreservedClass($klass) + { + $this->previous_klass = $klass; + } + + public function getPreviouslyUnreservedClass() + { + return $this->previous_klass; + } + public function setOptions($options) { $this->options = $options; @@ -179,6 +190,7 @@ public static function buildFromProto($proto, $file_proto, $containing) $message_name_without_package = ""; $classname = ""; $legacy_classname = ""; + $previous_classname = ""; $fullname = ""; GPBUtil::getFullClassName( $proto, @@ -187,10 +199,12 @@ public static function buildFromProto($proto, $file_proto, $containing) $message_name_without_package, $classname, $legacy_classname, - $fullname); + $fullname, + $previous_classname); $desc->setFullName($fullname); $desc->setClass($classname); $desc->setLegacyClass($legacy_classname); + $desc->setPreviouslyUnreservedClass($previous_classname); $desc->setOptions($proto->getOptions()); foreach ($proto->getField() as $field_proto) { diff --git a/php/src/Google/Protobuf/Internal/DescriptorPool.php b/php/src/Google/Protobuf/Internal/DescriptorPool.php index 1468a02380d9..1be00e2ffdfd 100644 --- a/php/src/Google/Protobuf/Internal/DescriptorPool.php +++ b/php/src/Google/Protobuf/Internal/DescriptorPool.php @@ -96,6 +96,7 @@ public function addDescriptor($descriptor) $descriptor->getClass(); $this->class_to_desc[$descriptor->getClass()] = $descriptor; $this->class_to_desc[$descriptor->getLegacyClass()] = $descriptor; + $this->class_to_desc[$descriptor->getPreviouslyUnreservedClass()] = $descriptor; foreach ($descriptor->getNestedType() as $nested_type) { $this->addDescriptor($nested_type); } diff --git a/php/src/Google/Protobuf/Internal/EnumDescriptor.php b/php/src/Google/Protobuf/Internal/EnumDescriptor.php index 7af4f84012e5..383f53b13fa0 100644 --- a/php/src/Google/Protobuf/Internal/EnumDescriptor.php +++ b/php/src/Google/Protobuf/Internal/EnumDescriptor.php @@ -101,7 +101,8 @@ public static function buildFromProto($proto, $file_proto, $containing) $enum_name_without_package, $classname, $legacy_classname, - $fullname); + $fullname, + $unused_previous_classname); $desc->setFullName($fullname); $desc->setClass($classname); $desc->setLegacyClass($legacy_classname); diff --git a/php/src/Google/Protobuf/Internal/GPBUtil.php b/php/src/Google/Protobuf/Internal/GPBUtil.php index 4b152839ec5e..ffea90083738 100644 --- a/php/src/Google/Protobuf/Internal/GPBUtil.php +++ b/php/src/Google/Protobuf/Internal/GPBUtil.php @@ -285,11 +285,12 @@ public static function getClassNamePrefix( "include"=>0, "include_once"=>0, "instanceof"=>0, "insteadof"=>0, "interface"=>0, "isset"=>0, "list"=>0, "match"=>0, "namespace"=>0, "new"=>0, "or"=>0, "parent"=>0, "print"=>0, "private"=>0, - "protected"=>0,"public"=>0, "require"=>0, "require_once"=>0, - "return"=>0, "self"=>0, "static"=>0, "switch"=>0, "throw"=>0, - "trait"=>0, "try"=>0,"unset"=>0, "use"=>0, "var"=>0, "while"=>0, - "xor"=>0, "yield"=>0, "int"=>0, "float"=>0, "bool"=>0, "string"=>0, - "true"=>0, "false"=>0, "null"=>0, "void"=>0, "iterable"=>0 + "protected"=>0,"public"=>0, "readonly" => 0,"require"=>0, + "require_once"=>0,"return"=>0, "self"=>0, "static"=>0, "switch"=>0, + "throw"=>0,"trait"=>0, "try"=>0,"unset"=>0, "use"=>0, "var"=>0, + "while"=>0,"xor"=>0, "yield"=>0, "int"=>0, "float"=>0, "bool"=>0, + "string"=>0,"true"=>0, "false"=>0, "null"=>0, "void"=>0, + "iterable"=>0 ); if (array_key_exists(strtolower($classname), $reserved_words)) { @@ -303,6 +304,27 @@ public static function getClassNamePrefix( return ""; } + private static function getPreviouslyUnreservedClassNamePrefix( + $classname, + $file_proto) + { + $previously_unreserved_words = array( + "readonly"=>0 + ); + + if (array_key_exists(strtolower($classname), $previously_unreserved_words)) { + $option = $file_proto->getOptions(); + $prefix = is_null($option) ? "" : $option->getPhpClassPrefix(); + if ($prefix !== "") { + return $prefix; + } + + return ""; + } + + return self::getClassNamePrefix($classname, $file_proto); + } + public static function getLegacyClassNameWithoutPackage( $name, $file_proto) @@ -322,6 +344,17 @@ public static function getClassNameWithoutPackage( return implode('\\', $parts); } + private static function getPreviouslyUnreservedClassNameWithoutPackage( + $name, + $file_proto) + { + $parts = explode('.', $name); + foreach ($parts as $i => $part) { + $parts[$i] = static::getPreviouslyUnreservedClassNamePrefix($parts[$i], $file_proto) . $parts[$i]; + } + return implode('\\', $parts); + } + public static function getFullClassName( $proto, $containing, @@ -329,7 +362,8 @@ public static function getFullClassName( &$message_name_without_package, &$classname, &$legacy_classname, - &$fullname) + &$fullname, + &$previous_classname) { // Full name needs to start with '.'. $message_name_without_package = $proto->getName(); @@ -350,6 +384,9 @@ public static function getFullClassName( $legacy_class_name_without_package = static::getLegacyClassNameWithoutPackage( $message_name_without_package, $file_proto); + $previous_class_name_without_package = + static::getPreviouslyUnreservedClassNameWithoutPackage( + $message_name_without_package, $file_proto); $option = $file_proto->getOptions(); if (!is_null($option) && $option->hasPhpNamespace()) { @@ -358,10 +395,13 @@ public static function getFullClassName( $classname = $namespace . "\\" . $class_name_without_package; $legacy_classname = $namespace . "\\" . $legacy_class_name_without_package; + $previous_classname = + $namespace . "\\" . $previous_class_name_without_package; return; } else { $classname = $class_name_without_package; $legacy_classname = $legacy_class_name_without_package; + $previous_classname = $previous_class_name_without_package; return; } } @@ -369,6 +409,7 @@ public static function getFullClassName( if ($package === "") { $classname = $class_name_without_package; $legacy_classname = $legacy_class_name_without_package; + $previous_classname = $previous_class_name_without_package; } else { $parts = array_map('ucwords', explode('.', $package)); foreach ($parts as $i => $part) { @@ -381,6 +422,11 @@ public static function getFullClassName( $legacy_classname = implode('\\', array_map('ucwords', explode('.', $package))). "\\".$legacy_class_name_without_package; + $previous_classname = + implode('\\', array_map('ucwords', explode('.', $package))). + "\\".self::getPreviouslyUnreservedClassNamePrefix( + $previous_class_name_without_package, $file_proto). + $previous_class_name_without_package; } } diff --git a/php/tests/DescriptorsTest.php b/php/tests/DescriptorsTest.php index 4f17a49571f6..336d97942845 100644 --- a/php/tests/DescriptorsTest.php +++ b/php/tests/DescriptorsTest.php @@ -83,10 +83,10 @@ public function testDescriptor() $this->assertSame($class, $desc->getClass()); $this->assertInstanceOf('\Google\Protobuf\FieldDescriptor', $desc->getField(0)); - $this->assertSame(7, $desc->getFieldCount()); + $this->assertSame(8, $desc->getFieldCount()); $this->assertInstanceOf('\Google\Protobuf\OneofDescriptor', $desc->getOneofDecl(0)); - $this->assertSame(1, $desc->getOneofDeclCount()); + $this->assertSame(2, $desc->getOneofDeclCount()); } public function testDescriptorForIncludedMessage() @@ -180,6 +180,7 @@ public function testFieldDescriptor() $this->assertSame(self::GPBTYPE_MESSAGE, $fieldDesc->getType()); $this->assertInstanceOf('\Google\Protobuf\Descriptor', $fieldDesc->getMessageType()); $this->assertFalse($fieldDesc->isMap()); + $this->assertNull($fieldDesc->getContainingOneof()); // Oneof int field // Tested further in testOneofDescriptor() @@ -189,6 +190,21 @@ public function testFieldDescriptor() $this->assertSame(self::GPBLABEL_OPTIONAL, $fieldDesc->getLabel()); $this->assertSame(self::GPBTYPE_INT32, $fieldDesc->getType()); $this->assertFalse($fieldDesc->isMap()); + $this->assertSame($fieldDesc->getContainingOneof(), $fieldDesc->getRealContainingOneof()); + + $oneofDesc = $fieldDesc->getContainingOneof(); + $this->assertSame('my_oneof', $oneofDesc->getName()); + + // Proto3 optional it field. + // Tested further in testOneofDescriptor() + $fieldDesc = $fieldDescMap[52]; + $this->assertSame('proto3_optional_int32', $fieldDesc->getName()); + $this->assertSame(52, $fieldDesc->getNumber()); + $this->assertSame(self::GPBLABEL_OPTIONAL, $fieldDesc->getLabel()); + $this->assertSame(self::GPBTYPE_INT32, $fieldDesc->getType()); + $this->assertFalse($fieldDesc->isMap()); + $this->assertNull($fieldDesc->getRealContainingOneof()); + $this->assertNotNull($fieldDesc->getContainingOneof()); // Map int-enum field $fieldDesc = $fieldDescMap[71]; diff --git a/php/tests/GeneratedClassTest.php b/php/tests/GeneratedClassTest.php index 8a89973ce804..b42649dc18c1 100644 --- a/php/tests/GeneratedClassTest.php +++ b/php/tests/GeneratedClassTest.php @@ -334,6 +334,19 @@ public function testLegacyTypehintWithNestedEnums() $this->legacyEnum(new TestLegacyMessage\NestedEnum); } + public function testLegacyReadOnlyMessage() + { + $this->assertTrue(class_exists('\Upper\READONLY')); + $this->assertTrue(class_exists('\Lower\readonly')); + $this->assertTrue(class_exists('\Php\Test\TestNamespace\PBEmpty\ReadOnly')); + } + + public function testLegacyReadOnlyEnum() + { + $this->assertTrue(class_exists('\Upper_enum\READONLY')); + $this->assertTrue(class_exists('\Lower_enum\readonly')); + } + private function legacyEnum(TestLegacyMessage_NestedEnum $enum) { // If we made it here without a PHP Fatal error, the typehint worked @@ -943,6 +956,7 @@ public function testPrefixForReservedWords() $m = new \Lower\PBprivate(); $m = new \Lower\PBprotected(); $m = new \Lower\PBpublic(); + $m = new \Lower\PBreadonly(); $m = new \Lower\PBrequire(); $m = new \Lower\PBrequire_once(); $m = new \Lower\PBreturn(); @@ -1023,6 +1037,7 @@ public function testPrefixForReservedWords() $m = new \Upper\PBPRIVATE(); $m = new \Upper\PBPROTECTED(); $m = new \Upper\PBPUBLIC(); + $m = new \Upper\PBREADONLY(); $m = new \Upper\PBREQUIRE(); $m = new \Upper\PBREQUIRE_ONCE(); $m = new \Upper\PBRETURN(); @@ -1104,6 +1119,7 @@ public function testPrefixForReservedWords() $m = new \Lower_enum\PBprotected(); $m = new \Lower_enum\PBpublic(); $m = new \Lower_enum\PBrequire(); + $m = new \Lower_enum\PBreadonly(); $m = new \Lower_enum\PBrequire_once(); $m = new \Lower_enum\PBreturn(); $m = new \Lower_enum\PBself(); @@ -1183,6 +1199,7 @@ public function testPrefixForReservedWords() $m = new \Upper_enum\PBPRIVATE(); $m = new \Upper_enum\PBPROTECTED(); $m = new \Upper_enum\PBPUBLIC(); + $m = new \Upper_enum\PBREADONLY(); $m = new \Upper_enum\PBREQUIRE(); $m = new \Upper_enum\PBREQUIRE_ONCE(); $m = new \Upper_enum\PBRETURN(); @@ -1287,6 +1304,7 @@ public function testPrefixForReservedWords() $m = \Lower_enum_value\NotAllowed::iterable; $m = \Lower_enum_value\NotAllowed::parent; $m = \Lower_enum_value\NotAllowed::self; + $m = \Lower_enum_value\NotAllowed::readonly; $m = \Upper_enum_value\NotAllowed::PBABSTRACT; $m = \Upper_enum_value\NotAllowed::PBAND; @@ -1367,6 +1385,7 @@ public function testPrefixForReservedWords() $m = \Upper_enum_value\NotAllowed::ITERABLE; $m = \Upper_enum_value\NotAllowed::PARENT; $m = \Upper_enum_value\NotAllowed::SELF; + $m = \Upper_enum_value\NotAllowed::READONLY; $this->assertTrue(true); } diff --git a/php/tests/PreviouslyGeneratedClassTest.php b/php/tests/PreviouslyGeneratedClassTest.php new file mode 100644 index 000000000000..077c84ff1359 --- /dev/null +++ b/php/tests/PreviouslyGeneratedClassTest.php @@ -0,0 +1,18 @@ +assertTrue(true); + } +} diff --git a/php/tests/generated_previous/GPBMetadata/ProtoPrevious/TestPreviouslyUnreservedMessage.php b/php/tests/generated_previous/GPBMetadata/ProtoPrevious/TestPreviouslyUnreservedMessage.php new file mode 100644 index 000000000000..d2120f9b6b4d --- /dev/null +++ b/php/tests/generated_previous/GPBMetadata/ProtoPrevious/TestPreviouslyUnreservedMessage.php @@ -0,0 +1,28 @@ +internalAddGeneratedFile( + ' +W +7proto_previous/test_previously_unreserved_message.protoprevious" + +readonlybproto3' + , true); + + static::$is_initialized = true; + } +} + diff --git a/php/tests/generated_previous/Previous/readonly.php b/php/tests/generated_previous/Previous/readonly.php new file mode 100644 index 000000000000..013f29354c8c --- /dev/null +++ b/php/tests/generated_previous/Previous/readonly.php @@ -0,0 +1,31 @@ +previous.readonly + */ +class readonly extends \Google\Protobuf\Internal\Message +{ + + /** + * Constructor. + * + * @param array $data { + * Optional. Data for populating the Message object. + * + * } + */ + public function __construct($data = NULL) { + \GPBMetadata\ProtoPrevious\TestPreviouslyUnreservedMessage::initOnce(); + parent::__construct($data); + } + +} + diff --git a/php/tests/proto/test_descriptors.proto b/php/tests/proto/test_descriptors.proto index d42aec7cec4f..b0956c6edd52 100644 --- a/php/tests/proto/test_descriptors.proto +++ b/php/tests/proto/test_descriptors.proto @@ -14,6 +14,7 @@ message TestDescriptorsMessage { oneof my_oneof { int32 oneof_int32 = 51; } + optional int32 proto3_optional_int32 = 52; map map_int32_enum = 71; diff --git a/php/tests/proto/test_php_namespace.proto b/php/tests/proto/test_php_namespace.proto index 61085bf75a93..eb4a5237978d 100644 --- a/php/tests/proto/test_php_namespace.proto +++ b/php/tests/proto/test_php_namespace.proto @@ -27,5 +27,7 @@ message TestNamespace { enum NestedEnum { ZERO = 0; }; + // Test previously unreserved name + message ReadOnly {} } } diff --git a/php/tests/proto/test_reserved_enum_lower.proto b/php/tests/proto/test_reserved_enum_lower.proto index f8557d250fea..1f96ac6fe020 100644 --- a/php/tests/proto/test_reserved_enum_lower.proto +++ b/php/tests/proto/test_reserved_enum_lower.proto @@ -57,6 +57,7 @@ enum print { ZERO51 = 0; } enum private { ZERO52 = 0; } enum protected { ZERO53 = 0; } enum public { ZERO54 = 0; } +enum readonly { ZERO80 = 0; } enum require { ZERO55 = 0; } enum require_once { ZERO56 = 0; } enum return { ZERO57 = 0; } diff --git a/php/tests/proto/test_reserved_enum_upper.proto b/php/tests/proto/test_reserved_enum_upper.proto index 8d382ab31e5d..c5e7e99fd5dd 100644 --- a/php/tests/proto/test_reserved_enum_upper.proto +++ b/php/tests/proto/test_reserved_enum_upper.proto @@ -57,6 +57,7 @@ enum PRINT { ZERO51 = 0; } enum PRIVATE { ZERO52 = 0; } enum PROTECTED { ZERO53 = 0; } enum PUBLIC { ZERO54 = 0; } +enum READONLY { ZERO80 = 0; } enum REQUIRE { ZERO55 = 0; } enum REQUIRE_ONCE { ZERO56 = 0; } enum RETURN { ZERO57 = 0; } diff --git a/php/tests/proto/test_reserved_enum_value_lower.proto b/php/tests/proto/test_reserved_enum_value_lower.proto index ca5a7c7352a0..86c6877f7d88 100644 --- a/php/tests/proto/test_reserved_enum_value_lower.proto +++ b/php/tests/proto/test_reserved_enum_value_lower.proto @@ -58,6 +58,7 @@ enum NotAllowed { private = 51; protected = 52; public = 53; + readonly = 79; require = 54; require_once = 55; return = 56; diff --git a/php/tests/proto/test_reserved_enum_value_upper.proto b/php/tests/proto/test_reserved_enum_value_upper.proto index 6b4040d5e4ab..ac0beda7d903 100644 --- a/php/tests/proto/test_reserved_enum_value_upper.proto +++ b/php/tests/proto/test_reserved_enum_value_upper.proto @@ -58,6 +58,7 @@ enum NotAllowed { PRIVATE = 51; PROTECTED = 52; PUBLIC = 53; + READONLY = 79; REQUIRE = 54; REQUIRE_ONCE = 55; RETURN = 56; diff --git a/php/tests/proto/test_reserved_message_lower.proto b/php/tests/proto/test_reserved_message_lower.proto index 2390a87dd696..551ed7a408d4 100644 --- a/php/tests/proto/test_reserved_message_lower.proto +++ b/php/tests/proto/test_reserved_message_lower.proto @@ -57,6 +57,7 @@ message print {} message private {} message protected {} message public {} +message readonly {} message require {} message require_once {} message return {} diff --git a/php/tests/proto/test_reserved_message_upper.proto b/php/tests/proto/test_reserved_message_upper.proto index 9f5533022347..96995c99178a 100644 --- a/php/tests/proto/test_reserved_message_upper.proto +++ b/php/tests/proto/test_reserved_message_upper.proto @@ -57,6 +57,7 @@ message PRINT {} message PRIVATE {} message PROTECTED {} message PUBLIC {} +message READONLY {} message REQUIRE {} message REQUIRE_ONCE {} message RETURN {} diff --git a/php/tests/proto_previous/test_previously_unreserved_message.proto b/php/tests/proto_previous/test_previously_unreserved_message.proto new file mode 100644 index 000000000000..1b4b0b7978e2 --- /dev/null +++ b/php/tests/proto_previous/test_previously_unreserved_message.proto @@ -0,0 +1,5 @@ +syntax = "proto3"; + +package previous; + +message readonly {} \ No newline at end of file diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index e9526431b012..0d2611f16f7c 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -114,6 +114,6 @@ def protobuf_deps(): _github_archive( name = "upb", repo = "https://github.com/protocolbuffers/upb", - commit = "04cb5af6b67c80db61f0aee76dcb6d233e51795c", - sha256 = "62d3519a7b65d6695e011f2733bfc5d7c6ab77f2bd83cdd2dca449da2e739c7f", + commit = "17b6451684ffcf6e77d10a5def9bf19af57eccd3", + sha256 = "655c30a01c8ab56680c154baded548c5df8f726305d3338d0885cbb1f700ec10", ) diff --git a/protobuf_version.bzl b/protobuf_version.bzl index b6c35f38aec7..edac50366c6f 100644 --- a/protobuf_version.bzl +++ b/protobuf_version.bzl @@ -1,3 +1,3 @@ -PROTOC_VERSION = '21.2' -PROTOBUF_JAVA_VERSION = '3.21.2' -PROTOBUF_PYTHON_VERSION = '4.21.2' +PROTOC_VERSION = '21.4' +PROTOBUF_JAVA_VERSION = '3.21.4' +PROTOBUF_PYTHON_VERSION = '4.21.4' diff --git a/protoc-artifacts/pom.xml b/protoc-artifacts/pom.xml index bce8121b4666..d816174f0850 100644 --- a/protoc-artifacts/pom.xml +++ b/protoc-artifacts/pom.xml @@ -4,11 +4,11 @@ com.google google - 1 + 5 com.google.protobuf protoc - 3.21.2 + 3.21.4 pom Protobuf Compiler diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py index 2c3e06cc1b92..b01d870dfa7b 100644 --- a/python/google/protobuf/__init__.py +++ b/python/google/protobuf/__init__.py @@ -30,4 +30,4 @@ # Copyright 2007 Google Inc. All Rights Reserved. -__version__ = '4.21.2' +__version__ = '4.21.4' diff --git a/python/google/protobuf/descriptor.py b/python/google/protobuf/descriptor.py index fa7a5b96d59c..997f6db6f50e 100644 --- a/python/google/protobuf/descriptor.py +++ b/python/google/protobuf/descriptor.py @@ -1021,13 +1021,7 @@ def __new__(cls, name, package, options=None, # FileDescriptor() is called from various places, not only from generated # files, to register dynamic proto files and messages. # pylint: disable=g-explicit-bool-comparison - if serialized_pb == b'': - # Cpp generated code must be linked in if serialized_pb is '' - try: - return _message.default_pool.FindFileByName(name) - except KeyError: - raise RuntimeError('Please link in cpp generated lib for %s' % (name)) - elif serialized_pb: + if serialized_pb: return _message.default_pool.AddSerializedFile(serialized_pb) else: return super(FileDescriptor, cls).__new__(cls) diff --git a/python/google/protobuf/internal/descriptor_test.py b/python/google/protobuf/internal/descriptor_test.py index 39a19c2c2a31..3af2bdcf566c 100644 --- a/python/google/protobuf/internal/descriptor_test.py +++ b/python/google/protobuf/internal/descriptor_test.py @@ -118,6 +118,22 @@ def setUp(self): def GetDescriptorPool(self): return symbol_database.Default().pool + def testMissingPackage(self): + file_proto = descriptor_pb2.FileDescriptorProto( + name='some/filename/some.proto') + serialized = file_proto.SerializeToString() + pool = descriptor_pool.DescriptorPool() + file_descriptor = pool.AddSerializedFile(serialized) + self.assertEqual('', file_descriptor.package) + + def testEmptyPackage(self): + file_proto = descriptor_pb2.FileDescriptorProto( + name='some/filename/some.proto', package='') + serialized = file_proto.SerializeToString() + pool = descriptor_pool.DescriptorPool() + file_descriptor = pool.AddSerializedFile(serialized) + self.assertEqual('', file_descriptor.package) + def testFindMethodByName(self): service_descriptor = (unittest_custom_options_pb2. TestServiceWithCustomOptions.DESCRIPTOR) diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index 3671c31a66d9..0a6c0b1ab213 100644 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py @@ -1768,6 +1768,19 @@ def testScalarMapDefaults(self): with self.assertRaises(TypeError): 123 in msg.map_string_string + def testScalarMapComparison(self): + msg1 = map_unittest_pb2.TestMap() + msg2 = map_unittest_pb2.TestMap() + + self.assertEqual(msg1.map_int32_int32, msg2.map_int32_int32) + + def testMessageMapComparison(self): + msg1 = map_unittest_pb2.TestMap() + msg2 = map_unittest_pb2.TestMap() + + self.assertEqual(msg1.map_int32_foreign_message, + msg2.map_int32_foreign_message) + def testMapGet(self): # Need to test that get() properly returns the default, even though the dict # has defaultdict-like semantics. @@ -2455,6 +2468,26 @@ def testSurrogatesInPython3(self): with self.assertRaises(ValueError): unittest_proto3_arena_pb2.TestAllTypes(optional_string=u'\ud801\ud801') + def testCrashNullAA(self): + self.assertEqual( + unittest_proto3_arena_pb2.TestAllTypes.NestedMessage(), + unittest_proto3_arena_pb2.TestAllTypes.NestedMessage()) + + def testCrashNullAB(self): + self.assertEqual( + unittest_proto3_arena_pb2.TestAllTypes.NestedMessage(), + unittest_proto3_arena_pb2.TestAllTypes().optional_nested_message) + + def testCrashNullBA(self): + self.assertEqual( + unittest_proto3_arena_pb2.TestAllTypes().optional_nested_message, + unittest_proto3_arena_pb2.TestAllTypes.NestedMessage()) + + def testCrashNullBB(self): + self.assertEqual( + unittest_proto3_arena_pb2.TestAllTypes().optional_nested_message, + unittest_proto3_arena_pb2.TestAllTypes().optional_nested_message) + diff --git a/python/google/protobuf/message.py b/python/google/protobuf/message.py index 76c6802f7096..0fe6a4f93cde 100644 --- a/python/google/protobuf/message.py +++ b/python/google/protobuf/message.py @@ -74,7 +74,8 @@ class Message(object): __slots__ = [] - #: The :class:`google.protobuf.descriptor.Descriptor` for this message type. + #: The :class:`google.protobuf.Descriptor` + # for this message type. DESCRIPTOR = None def __deepcopy__(self, memo=None): diff --git a/python/release.sh b/python/release.sh index 15a70db31f8f..87fcf8cfeea4 100755 --- a/python/release.sh +++ b/python/release.sh @@ -19,11 +19,24 @@ function run_install_test() { chmod +x test-venv/bin/protoc source test-venv/bin/activate - pip install -i ${PYPI} protobuf==${VERSION} --no-cache-dir + (pip install -i ${PYPI} protobuf==${VERSION} --no-cache-dir) || (retry_pip_install ${PYPI} ${VERSION}) deactivate rm -fr test-venv } +function retry_pip_install() { + local PYPI=$1 + local VERSION=$2 + + read -p "pip install failed, possibly due to delay between upload and availability on pip. Retry? [y/n]" -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi + + (pip install -i ${PYPI} protobuf==${VERSION} --no-cache-dir) || (retry_pip_install ${PYPI} ${VERSION}) +} + [ $# -lt 1 ] && { echo "Usage: $0 VERSION [" @@ -86,13 +99,16 @@ python3 setup.py test python3 setup.py sdist twine upload --skip-existing -r testpypi -u protobuf-wheel-test dist/* -# Test locally with different python versions. +# Sleep to allow time for distribution to be available on pip. +sleep 5m + +# Test locally. run_install_test ${TESTING_VERSION} python3 https://test.pypi.org/simple # Deploy egg/wheel packages to testing PyPI and test again. python3 setup.py clean build bdist_wheel twine upload --skip-existing -r testpypi -u protobuf-wheel-test dist/* - +sleep 5m run_install_test ${TESTING_VERSION} python3 https://test.pypi.org/simple echo "All install tests have passed using testing PyPI." diff --git a/ruby/README.md b/ruby/README.md index 42a1ffaad414..be8d6bc8b1cf 100644 --- a/ruby/README.md +++ b/ruby/README.md @@ -9,7 +9,7 @@ install protoc as well to have Ruby code generation functionality. Installation from Gem --------------------- -In Gemfile (Please check a version of Protocol Buffers you needed [RubyGems](https://rubygems.org/gems/google-protobuf)): +In Gemfile (Please check which version of Protocol Buffers you need: [RubyGems](https://rubygems.org/gems/google-protobuf)): gem 'google-protobuf' diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index e7aa377519ae..e43665d44734 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "google-protobuf" - s.version = "3.21.2" + s.version = "3.21.4" git_tag = "v#{s.version.to_s.sub('.rc.', '-rc')}" # Converts X.Y.Z.rc.N to vX.Y.Z-rcN, used for the git tag s.licenses = ["BSD-3-Clause"] s.summary = "Protocol Buffers" diff --git a/ruby/pom.xml b/ruby/pom.xml index f1bdbf1f1f06..42641727728a 100644 --- a/ruby/pom.xml +++ b/ruby/pom.xml @@ -9,7 +9,7 @@ com.google.protobuf.jruby protobuf-jruby - 3.21.2 + 3.21.4 Protocol Buffer JRuby native extension Protocol Buffers are a way of encoding structured data in an efficient yet @@ -76,7 +76,7 @@ com.google.protobuf protobuf-java-util - 3.21.2 + 3.21.4 org.jruby diff --git a/src/Makefile.am b/src/Makefile.am index 9875b1c5753e..15f7aa42a32d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -18,7 +18,7 @@ else PTHREAD_DEF = endif -PROTOBUF_VERSION = 32:2:0 +PROTOBUF_VERSION = 32:4:0 if GCC # Turn on all warnings except for sign comparison (we ignore sign comparison @@ -774,6 +774,7 @@ protobuf_test_SOURCES = \ google/protobuf/compiler/csharp/csharp_generator_unittest.cc \ google/protobuf/compiler/importer_unittest.cc \ google/protobuf/compiler/java/doc_comment_unittest.cc \ + google/protobuf/compiler/java/message_serialization_unittest.cc \ google/protobuf/compiler/java/plugin_unittest.cc \ google/protobuf/compiler/mock_code_generator.cc \ google/protobuf/compiler/mock_code_generator.h \ diff --git a/src/file_lists.cmake b/src/file_lists.cmake index 32f10b2705ed..67092cc39e55 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -771,6 +771,7 @@ set(compiler_test_files ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/csharp/csharp_generator_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/importer_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/doc_comment_unittest.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_serialization_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/plugin_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/objectivec_helpers_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/parser_unittest.cc diff --git a/src/google/protobuf/any.pb.h b/src/google/protobuf/any.pb.h index 2368446bba62..0a9ee260b263 100644 --- a/src/google/protobuf/any.pb.h +++ b/src/google/protobuf/any.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/api.pb.h b/src/google/protobuf/api.pb.h index ac805601aeaa..49032949b529 100644 --- a/src/google/protobuf/api.pb.h +++ b/src/google/protobuf/api.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/arena.cc b/src/google/protobuf/arena.cc index 9200cb2d043a..0c1bcf922735 100644 --- a/src/google/protobuf/arena.cc +++ b/src/google/protobuf/arena.cc @@ -382,7 +382,7 @@ void ThreadSafeArena::Init() { void ThreadSafeArena::SetInitialBlock(void* mem, size_t size) { SerialArena* serial = SerialArena::New({mem, size}, &thread_cache(), arena_stats_.MutableStats()); - serial->set_next(NULL); + serial->set_next(nullptr); threads_.store(serial, std::memory_order_relaxed); CacheSerialArena(serial); } diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h index b6297808225f..59e02e0963d2 100644 --- a/src/google/protobuf/arena.h +++ b/src/google/protobuf/arena.h @@ -120,7 +120,7 @@ struct ArenaOptions { // here. size_t max_block_size; - // An initial block of memory for the arena to use, or NULL for none. If + // An initial block of memory for the arena to use, or nullptr for none. If // provided, the block must live at least as long as the arena itself. The // creator of the Arena retains ownership of the block after the Arena is // destroyed. @@ -143,7 +143,7 @@ struct ArenaOptions { ArenaOptions() : start_block_size(internal::AllocationPolicy::kDefaultStartBlockSize), max_block_size(internal::AllocationPolicy::kDefaultMaxBlockSize), - initial_block(NULL), + initial_block(nullptr), initial_block_size(0), block_alloc(nullptr), block_dealloc(nullptr), @@ -180,7 +180,7 @@ struct ArenaOptions { #if PROTOBUF_RTTI #define RTTI_TYPE_ID(type) (&typeid(type)) #else -#define RTTI_TYPE_ID(type) (NULL) +#define RTTI_TYPE_ID(type) (nullptr) #endif // Arena allocator. Arena allocation replaces ordinary (heap-based) allocation @@ -210,7 +210,7 @@ struct ArenaOptions { // with `args` (without `arena`), called when a T is allocated on the heap; // and a constructor callable with `Arena* arena, Args&&... args`, called when // a T is allocated on an arena. If the second constructor is called with a -// NULL arena pointer, it must be equivalent to invoking the first +// null arena pointer, it must be equivalent to invoking the first // (`args`-only) constructor. // // - The type T must have a particular type trait: a nested type @@ -220,7 +220,7 @@ struct ArenaOptions { // // - The type T *may* have the type trait |DestructorSkippable_|. If this type // trait is present in the type, then its destructor will not be called if and -// only if it was passed a non-NULL arena pointer. If this type trait is not +// only if it was passed a non-null arena pointer. If this type trait is not // present on the type, then its destructor is always called when the // containing arena is destroyed. // @@ -263,9 +263,9 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { void Init(const ArenaOptions&) {} // API to create proto2 message objects on the arena. If the arena passed in - // is NULL, then a heap allocated object is returned. Type T must be a message - // defined in a .proto file with cc_enable_arenas set to true, otherwise a - // compilation error will occur. + // is nullptr, then a heap allocated object is returned. Type T must be a + // message defined in a .proto file with cc_enable_arenas set to true, + // otherwise a compilation error will occur. // // RepeatedField and RepeatedPtrField may also be instantiated directly on an // arena with this method. @@ -342,7 +342,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { "CreateArray requires a trivially destructible type"); GOOGLE_CHECK_LE(num_elements, std::numeric_limits::max() / sizeof(T)) << "Requested size is too large to fit into size_t."; - if (arena == NULL) { + if (arena == nullptr) { return static_cast(::operator new[](num_elements * sizeof(T))); } else { return arena->CreateInternalRawArray(num_elements); @@ -386,7 +386,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { // arena-allocated memory. template PROTOBUF_ALWAYS_INLINE void OwnDestructor(T* object) { - if (object != NULL) { + if (object != nullptr) { impl_.AddCleanup(object, &internal::cleanup::arena_destruct_object); } } @@ -401,9 +401,9 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { } // Retrieves the arena associated with |value| if |value| is an arena-capable - // message, or NULL otherwise. If possible, the call resolves at compile time. - // Note that we can often devirtualize calls to `value->GetArena()` so usually - // calling this method is unnecessary. + // message, or nullptr otherwise. If possible, the call resolves at compile + // time. Note that we can often devirtualize calls to `value->GetArena()` so + // usually calling this method is unnecessary. template PROTOBUF_ALWAYS_INLINE static Arena* GetArena(const T* value) { return GetArenaInternal(value); @@ -568,7 +568,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { static_assert( InternalHelper::is_arena_constructable::value, "CreateMessage can only construct types that are ArenaConstructable"); - if (arena == NULL) { + if (arena == nullptr) { return new T(nullptr, static_cast(args)...); } else { return arena->DoCreateMessage(static_cast(args)...); @@ -583,7 +583,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { static_assert( InternalHelper::is_arena_constructable::value, "CreateMessage can only construct types that are ArenaConstructable"); - if (arena == NULL) { + if (arena == nullptr) { // Generated arena constructor T(Arena*) is protected. Call via // InternalHelper. return InternalHelper::New(); @@ -730,13 +730,13 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { // using the virtual destructor instead. template PROTOBUF_ALWAYS_INLINE void OwnInternal(T* object, std::true_type) { - if (object != NULL) { + if (object != nullptr) { impl_.AddCleanup(object, &internal::arena_delete_object); } } template PROTOBUF_ALWAYS_INLINE void OwnInternal(T* object, std::false_type) { - if (object != NULL) { + if (object != nullptr) { impl_.AddCleanup(object, &internal::arena_delete_object); } } diff --git a/src/google/protobuf/arena_unittest.cc b/src/google/protobuf/arena_unittest.cc index 6c88505d8d4b..1466376ed7b9 100644 --- a/src/google/protobuf/arena_unittest.cc +++ b/src/google/protobuf/arena_unittest.cc @@ -84,10 +84,10 @@ class Notifier { class SimpleDataType { public: - SimpleDataType() : notifier_(NULL) {} + SimpleDataType() : notifier_(nullptr) {} void SetNotifier(Notifier* notifier) { notifier_ = notifier; } virtual ~SimpleDataType() { - if (notifier_ != NULL) { + if (notifier_ != nullptr) { notifier_->Notify(); } }; @@ -171,17 +171,17 @@ TEST(ArenaTest, DestructorSkippable) { TEST(ArenaTest, BasicCreate) { Arena arena; - EXPECT_TRUE(Arena::Create(&arena) != NULL); - EXPECT_TRUE(Arena::Create(&arena) != NULL); - EXPECT_TRUE(Arena::Create(&arena) != NULL); - EXPECT_TRUE(Arena::Create(&arena) != NULL); - EXPECT_TRUE(Arena::Create(&arena) != NULL); + EXPECT_TRUE(Arena::Create(&arena) != nullptr); + EXPECT_TRUE(Arena::Create(&arena) != nullptr); + EXPECT_TRUE(Arena::Create(&arena) != nullptr); + EXPECT_TRUE(Arena::Create(&arena) != nullptr); + EXPECT_TRUE(Arena::Create(&arena) != nullptr); arena.Own(new int32_t); arena.Own(new int64_t); arena.Own(new float); arena.Own(new double); arena.Own(new std::string); - arena.Own(NULL); + arena.Own(nullptr); Notifier notifier; SimpleDataType* data = Arena::Create(&arena); data->SetNotifier(¬ifier); @@ -196,7 +196,7 @@ TEST(ArenaTest, CreateAndConstCopy) { Arena arena; const std::string s("foo"); const std::string* s_copy = Arena::Create(&arena, s); - EXPECT_TRUE(s_copy != NULL); + EXPECT_TRUE(s_copy != nullptr); EXPECT_EQ("foo", s); EXPECT_EQ("foo", *s_copy); } @@ -205,7 +205,7 @@ TEST(ArenaTest, CreateAndNonConstCopy) { Arena arena; std::string s("foo"); const std::string* s_copy = Arena::Create(&arena, s); - EXPECT_TRUE(s_copy != NULL); + EXPECT_TRUE(s_copy != nullptr); EXPECT_EQ("foo", s); EXPECT_EQ("foo", *s_copy); } @@ -214,7 +214,7 @@ TEST(ArenaTest, CreateAndMove) { Arena arena; std::string s("foo"); const std::string* s_move = Arena::Create(&arena, std::move(s)); - EXPECT_TRUE(s_move != NULL); + EXPECT_TRUE(s_move != nullptr); EXPECT_TRUE(s.empty()); // NOLINT EXPECT_EQ("foo", *s_move); } @@ -226,7 +226,7 @@ TEST(ArenaTest, CreateWithFourConstructorArguments) { const MustBeConstructedWithOneThroughFour* new_object = Arena::Create(&arena, 1, "2", three, &four); - EXPECT_TRUE(new_object != NULL); + EXPECT_TRUE(new_object != nullptr); ASSERT_EQ(1, new_object->one_); ASSERT_STREQ("2", new_object->two_); ASSERT_EQ("3", new_object->three_); @@ -242,7 +242,7 @@ TEST(ArenaTest, CreateWithEightConstructorArguments) { const MustBeConstructedWithOneThroughEight* new_object = Arena::Create( &arena, 1, "2", three, &four, 5, "6", seven, eight); - EXPECT_TRUE(new_object != NULL); + EXPECT_TRUE(new_object != nullptr); ASSERT_EQ(1, new_object->one_); ASSERT_STREQ("2", new_object->two_); ASSERT_EQ("3", new_object->three_); @@ -504,7 +504,7 @@ TEST(ArenaTest, ReleaseMessage) { TestAllTypes::NestedMessage* released_null = arena_message->release_optional_nested_message(); - EXPECT_EQ(NULL, released_null); + EXPECT_EQ(nullptr, released_null); } TEST(ArenaTest, SetAllocatedString) { @@ -605,8 +605,8 @@ TEST(ArenaTest, SwapBetweenArenaAndNonArenaUsingReflection) { } TEST(ArenaTest, ReleaseFromArenaMessageMakesCopy) { - TestAllTypes::NestedMessage* nested_msg = NULL; - std::string* nested_string = NULL; + TestAllTypes::NestedMessage* nested_msg = nullptr; + std::string* nested_string = nullptr; { Arena arena; TestAllTypes* arena_message = Arena::CreateMessage(&arena); @@ -623,7 +623,7 @@ TEST(ArenaTest, ReleaseFromArenaMessageMakesCopy) { #if PROTOBUF_RTTI TEST(ArenaTest, ReleaseFromArenaMessageUsingReflectionMakesCopy) { - TestAllTypes::NestedMessage* nested_msg = NULL; + TestAllTypes::NestedMessage* nested_msg = nullptr; // Note: no string: reflection API only supports releasing submessages. { Arena arena; @@ -1098,7 +1098,7 @@ TEST(ArenaTest, ArenaOneofReflection) { EXPECT_TRUE(refl->HasOneof(*message, oneof)); submsg = refl->ReleaseMessage(message, msg_field); EXPECT_FALSE(refl->HasOneof(*message, oneof)); - EXPECT_TRUE(submsg->GetArena() == NULL); + EXPECT_TRUE(submsg->GetArena() == nullptr); delete submsg; } @@ -1111,7 +1111,7 @@ void TestSwapRepeatedField(Arena* arena1, Arena* arena2) { TestAllTypes* t = Arena::CreateMessage(arena1); t->set_optional_string("field1"); t->set_optional_int32(i); - if (arena1 != NULL) { + if (arena1 != nullptr) { field1.UnsafeArenaAddAllocated(t); } else { field1.AddAllocated(t); @@ -1121,7 +1121,7 @@ void TestSwapRepeatedField(Arena* arena1, Arena* arena2) { TestAllTypes* t = Arena::CreateMessage(arena2); t->set_optional_string("field2"); t->set_optional_int32(i); - if (arena2 != NULL) { + if (arena2 != nullptr) { field2.UnsafeArenaAddAllocated(t); } else { field2.AddAllocated(t); @@ -1154,12 +1154,12 @@ TEST(ArenaTest, SwapRepeatedFieldWithDifferentArenas) { TEST(ArenaTest, SwapRepeatedFieldWithNoArenaOnRightHandSide) { Arena arena; - TestSwapRepeatedField(&arena, NULL); + TestSwapRepeatedField(&arena, nullptr); } TEST(ArenaTest, SwapRepeatedFieldWithNoArenaOnLeftHandSide) { Arena arena; - TestSwapRepeatedField(NULL, &arena); + TestSwapRepeatedField(nullptr, &arena); } TEST(ArenaTest, ExtensionsOnArena) { @@ -1218,11 +1218,11 @@ TEST(ArenaTest, RepeatedFieldOnArena) { TestAllTypes* extracted_messages[5]; // ExtractSubrange should copy to the heap. repeated_message.ExtractSubrange(0, 5, extracted_messages); - EXPECT_EQ(NULL, extracted_messages[0]->GetArena()); + EXPECT_EQ(nullptr, extracted_messages[0]->GetArena()); // We need to free the heap-allocated messages to prevent a leak. for (int i = 0; i < 5; i++) { delete extracted_messages[i]; - extracted_messages[i] = NULL; + extracted_messages[i] = nullptr; } } @@ -1342,7 +1342,7 @@ TEST(ArenaTest, MessageLiteOnArena) { { MessageLite* generic_message = prototype->New(&arena); - EXPECT_TRUE(generic_message != NULL); + EXPECT_TRUE(generic_message != nullptr); EXPECT_EQ(&arena, generic_message->GetArena()); EXPECT_TRUE(generic_message->ParseFromString(serialized)); TestAllTypes* deserialized = static_cast(generic_message); @@ -1419,7 +1419,7 @@ TEST(ArenaTest, BlockSizeDoubling) { ASSERT_GT(arena.SpaceAllocated(), first_block_size); auto second_block_size = (arena.SpaceAllocated() - first_block_size); - EXPECT_EQ(second_block_size, 2*first_block_size); + EXPECT_GE(second_block_size, 2*first_block_size); } TEST(ArenaTest, Alignment) { @@ -1463,8 +1463,8 @@ TEST(ArenaTest, GetArenaShouldReturnTheArenaForArenaAllocatedMessages) { TEST(ArenaTest, GetArenaShouldReturnNullForNonArenaAllocatedMessages) { ArenaMessage message; const ArenaMessage* const_pointer_to_message = &message; - EXPECT_EQ(NULL, Arena::GetArena(&message)); - EXPECT_EQ(NULL, Arena::GetArena(const_pointer_to_message)); + EXPECT_EQ(nullptr, Arena::GetArena(&message)); + EXPECT_EQ(nullptr, Arena::GetArena(const_pointer_to_message)); } TEST(ArenaTest, GetArenaShouldReturnNullForNonArenaCompatibleTypes) { diff --git a/src/google/protobuf/arenastring.cc b/src/google/protobuf/arenastring.cc index 03b1438b44b0..8d7e6ff48768 100644 --- a/src/google/protobuf/arenastring.cc +++ b/src/google/protobuf/arenastring.cc @@ -50,7 +50,8 @@ namespace internal { namespace { -// Enforce that allocated data aligns to at least 8 bytes, and that +// TaggedStringPtr::Flags uses the lower 2 bits as tags. +// Enforce that allocated data aligns to at least 4 bytes, and that // the alignment of the global const string value does as well. // The alignment guaranteed by `new std::string` depends on both: // - new align = __STDCPP_DEFAULT_NEW_ALIGNMENT__ / max_align_t @@ -64,8 +65,8 @@ constexpr size_t kNewAlign = alignof(std::max_align_t); #endif constexpr size_t kStringAlign = alignof(std::string); -static_assert((kStringAlign > kNewAlign ? kStringAlign : kNewAlign) >= 8, ""); -static_assert(alignof(ExplicitlyConstructedArenaString) >= 8, ""); +static_assert((kStringAlign > kNewAlign ? kStringAlign : kNewAlign) >= 4, ""); +static_assert(alignof(ExplicitlyConstructedArenaString) >= 4, ""); } // namespace diff --git a/src/google/protobuf/arenaz_sampler.cc b/src/google/protobuf/arenaz_sampler.cc index 6ad3a3650448..ef675182f9fc 100644 --- a/src/google/protobuf/arenaz_sampler.cc +++ b/src/google/protobuf/arenaz_sampler.cc @@ -33,6 +33,7 @@ #include #include #include +#include // Must be included last. @@ -56,23 +57,34 @@ namespace { PROTOBUF_CONSTINIT std::atomic g_arenaz_enabled{true}; PROTOBUF_CONSTINIT std::atomic g_arenaz_sample_parameter{1 << 10}; +PROTOBUF_CONSTINIT std::atomic + g_arenaz_config_listener{nullptr}; PROTOBUF_THREAD_LOCAL absl::profiling_internal::ExponentialBiased g_exponential_biased_generator; +void TriggerThreadSafeArenazConfigListener() { + auto* listener = g_arenaz_config_listener.load(std::memory_order_acquire); + if (listener != nullptr) listener(); +} + } // namespace PROTOBUF_THREAD_LOCAL SamplingState global_sampling_state = { - .next_sample = int64_t{1} << 10, .sample_stride = int64_t{1} << 10}; + /*next_sample=*/0, /*sample_stride=*/0}; ThreadSafeArenaStats::ThreadSafeArenaStats() { PrepareForSampling(0); } ThreadSafeArenaStats::~ThreadSafeArenaStats() = default; -void ThreadSafeArenaStats::PrepareForSampling(int64_t stride) { +void ThreadSafeArenaStats::BlockStats::PrepareForSampling() { num_allocations.store(0, std::memory_order_relaxed); - bytes_used.store(0, std::memory_order_relaxed); bytes_allocated.store(0, std::memory_order_relaxed); + bytes_used.store(0, std::memory_order_relaxed); bytes_wasted.store(0, std::memory_order_relaxed); - max_bytes_allocated.store(0, std::memory_order_relaxed); +} + +void ThreadSafeArenaStats::PrepareForSampling(int64_t stride) { + for (auto& blockstats : block_histogram) blockstats.PrepareForSampling(); + max_block_size.store(0, std::memory_order_relaxed); thread_ids.store(0, std::memory_order_relaxed); weight = stride; // The inliner makes hardcoded skip_count difficult (especially when combined @@ -81,12 +93,44 @@ void ThreadSafeArenaStats::PrepareForSampling(int64_t stride) { depth = absl::GetStackTrace(stack, kMaxStackDepth, /* skip_count= */ 0); } +size_t ThreadSafeArenaStats::FindBin(size_t bytes) { + if (bytes <= kMaxSizeForBinZero) return 0; + if (bytes <= kMaxSizeForPenultimateBin) { + // absl::bit_width() returns one plus the base-2 logarithm of x, with any + // fractional part discarded. + return absl::bit_width(absl::bit_ceil(bytes)) - kLogMaxSizeForBinZero - 1; + } + return kBlockHistogramBins - 1; +} + +std::pair ThreadSafeArenaStats::MinMaxBlockSizeForBin( + size_t bin) { + ABSL_ASSERT(bin < kBlockHistogramBins); + if (bin == 0) return {1, kMaxSizeForBinZero}; + if (bin < kBlockHistogramBins - 1) { + return {(1 << (kLogMaxSizeForBinZero + bin - 1)) + 1, + 1 << (kLogMaxSizeForBinZero + bin)}; + } + return {kMaxSizeForPenultimateBin + 1, std::numeric_limits::max()}; +} + void RecordAllocateSlow(ThreadSafeArenaStats* info, size_t used, size_t allocated, size_t wasted) { - info->num_allocations.fetch_add(1, std::memory_order_relaxed); - info->bytes_used.fetch_add(used, std::memory_order_relaxed); - info->bytes_allocated.fetch_add(allocated, std::memory_order_relaxed); - info->bytes_wasted.fetch_add(wasted, std::memory_order_relaxed); + // Update the allocated bytes for the current block. + ThreadSafeArenaStats::BlockStats& curr = + info->block_histogram[ThreadSafeArenaStats::FindBin(allocated)]; + curr.bytes_allocated.fetch_add(allocated, std::memory_order_relaxed); + curr.num_allocations.fetch_add(1, std::memory_order_relaxed); + + // Update the used and wasted bytes for the previous block. + ThreadSafeArenaStats::BlockStats& prev = + info->block_histogram[ThreadSafeArenaStats::FindBin(used + wasted)]; + prev.bytes_used.fetch_add(used, std::memory_order_relaxed); + prev.bytes_wasted.fetch_add(wasted, std::memory_order_relaxed); + + if (info->max_block_size.load(std::memory_order_relaxed) < allocated) { + info->max_block_size.store(allocated, std::memory_order_relaxed); + } const uint64_t tid = 1ULL << (GetCachedTID() % 63); info->thread_ids.fetch_or(tid, std::memory_order_relaxed); } @@ -115,11 +159,29 @@ ThreadSafeArenaStats* SampleSlow(SamplingState& sampling_state) { return GlobalThreadSafeArenazSampler().Register(old_stride); } +void SetThreadSafeArenazConfigListener(ThreadSafeArenazConfigListener l) { + g_arenaz_config_listener.store(l, std::memory_order_release); +} + +bool IsThreadSafeArenazEnabled() { + return g_arenaz_enabled.load(std::memory_order_acquire); +} + void SetThreadSafeArenazEnabled(bool enabled) { + SetThreadSafeArenazEnabledInternal(enabled); + TriggerThreadSafeArenazConfigListener(); +} + +void SetThreadSafeArenazEnabledInternal(bool enabled) { g_arenaz_enabled.store(enabled, std::memory_order_release); } void SetThreadSafeArenazSampleParameter(int32_t rate) { + SetThreadSafeArenazSampleParameterInternal(rate); + TriggerThreadSafeArenazConfigListener(); +} + +void SetThreadSafeArenazSampleParameterInternal(int32_t rate) { if (rate > 0) { g_arenaz_sample_parameter.store(rate, std::memory_order_release); } else { @@ -133,6 +195,11 @@ int32_t ThreadSafeArenazSampleParameter() { } void SetThreadSafeArenazMaxSamples(int32_t max) { + SetThreadSafeArenazMaxSamplesInternal(max); + TriggerThreadSafeArenazConfigListener(); +} + +void SetThreadSafeArenazMaxSamplesInternal(int32_t max) { if (max > 0) { GlobalThreadSafeArenazSampler().SetMaxSamples(max); } else { @@ -141,6 +208,10 @@ void SetThreadSafeArenazMaxSamples(int32_t max) { } } +size_t ThreadSafeArenazMaxSamples() { + return GlobalThreadSafeArenazSampler().GetMaxSamples(); +} + void SetThreadSafeArenazGlobalNextSample(int64_t next_sample) { if (next_sample >= 0) { global_sampling_state.next_sample = next_sample; @@ -157,10 +228,16 @@ ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) { return nullptr; } +void SetThreadSafeArenazConfigListener(ThreadSafeArenazConfigListener) {} void SetThreadSafeArenazEnabled(bool enabled) {} +void SetThreadSafeArenazEnabledInternal(bool enabled) {} +bool IsThreadSafeArenazEnabled() { return false; } void SetThreadSafeArenazSampleParameter(int32_t rate) {} +void SetThreadSafeArenazSampleParameterInternal(int32_t rate) {} int32_t ThreadSafeArenazSampleParameter() { return 0; } void SetThreadSafeArenazMaxSamples(int32_t max) {} +void SetThreadSafeArenazMaxSamplesInternal(int32_t max) {} +size_t ThreadSafeArenazMaxSamples() { return 0; } void SetThreadSafeArenazGlobalNextSample(int64_t next_sample) {} #endif // defined(PROTOBUF_ARENAZ_SAMPLE) diff --git a/src/google/protobuf/arenaz_sampler.h b/src/google/protobuf/arenaz_sampler.h index 4a04c6a95354..c608bcd77d4f 100644 --- a/src/google/protobuf/arenaz_sampler.h +++ b/src/google/protobuf/arenaz_sampler.h @@ -31,9 +31,11 @@ #ifndef GOOGLE_PROTOBUF_SRC_GOOGLE_PROTOBUF_ARENAZ_SAMPLER_H__ #define GOOGLE_PROTOBUF_SRC_GOOGLE_PROTOBUF_ARENAZ_SAMPLER_H__ +#include #include #include #include +#include // Must be included last. @@ -65,12 +67,30 @@ struct ThreadSafeArenaStats // These fields are mutated by the various Record* APIs and need to be // thread-safe. - std::atomic num_allocations; - std::atomic bytes_used; - std::atomic bytes_allocated; - std::atomic bytes_wasted; - // Records the largest size an arena ever had. - std::atomic max_bytes_allocated; + struct BlockStats { + std::atomic num_allocations; + std::atomic bytes_allocated; + std::atomic bytes_used; + std::atomic bytes_wasted; + + void PrepareForSampling(); + }; + + // block_histogram is a kBlockHistogramBins sized histogram. The zeroth bin + // stores info about blocks of size \in [1, 1 << kLogMaxSizeForBinZero]. Bin + // i, where i > 0, stores info for blocks of size \in (max_size_bin (i-1), + // 1 << (kLogMaxSizeForBinZero + i)]. The final bin stores info about blocks + // of size \in [kMaxSizeForPenultimateBin + 1, + // std::numeric_limits::max()]. + static constexpr size_t kBlockHistogramBins = 15; + static constexpr size_t kLogMaxSizeForBinZero = 7; + static constexpr size_t kMaxSizeForBinZero = (1 << kLogMaxSizeForBinZero); + static constexpr size_t kMaxSizeForPenultimateBin = + 1 << (kLogMaxSizeForBinZero + kBlockHistogramBins - 2); + std::array block_histogram; + + // Records the largest block allocated for the arena. + std::atomic max_block_size; // Bit `i` is set to 1 indicates that a thread with `tid % 63 = i` accessed // the underlying arena. We use `% 63` as a rudimentary hash to ensure some // bit mixing for thread-ids; `% 64` would only grab the low bits and might @@ -90,6 +110,13 @@ struct ThreadSafeArenaStats if (PROTOBUF_PREDICT_TRUE(info == nullptr)) return; RecordAllocateSlow(info, used, allocated, wasted); } + + // Returns the bin for the provided size. + static size_t FindBin(size_t bytes); + + // Returns the min and max bytes that can be stored in the histogram for + // blocks in the provided bin. + static std::pair MinMaxBlockSizeForBin(size_t bin); }; struct SamplingState { @@ -199,17 +226,29 @@ inline ThreadSafeArenaStatsHandle Sample() { // Returns a global Sampler. ThreadSafeArenazSampler& GlobalThreadSafeArenazSampler(); +using ThreadSafeArenazConfigListener = void (*)(); +void SetThreadSafeArenazConfigListener(ThreadSafeArenazConfigListener l); + // Enables or disables sampling for thread safe arenas. void SetThreadSafeArenazEnabled(bool enabled); +void SetThreadSafeArenazEnabledInternal(bool enabled); + +// Returns true if sampling is on, false otherwise. +bool IsThreadSafeArenazEnabled(); // Sets the rate at which thread safe arena will be sampled. void SetThreadSafeArenazSampleParameter(int32_t rate); +void SetThreadSafeArenazSampleParameterInternal(int32_t rate); // Returns the rate at which thread safe arena will be sampled. int32_t ThreadSafeArenazSampleParameter(); // Sets a soft max for the number of samples that will be kept. void SetThreadSafeArenazMaxSamples(int32_t max); +void SetThreadSafeArenazMaxSamplesInternal(int32_t max); + +// Returns the max number of samples that will be kept. +size_t ThreadSafeArenazMaxSamples(); // Sets the current value for when arenas should be next sampled. void SetThreadSafeArenazGlobalNextSample(int64_t next_sample); diff --git a/src/google/protobuf/arenaz_sampler_test.cc b/src/google/protobuf/arenaz_sampler_test.cc index 7bac100ee12d..67570e87a46a 100644 --- a/src/google/protobuf/arenaz_sampler_test.cc +++ b/src/google/protobuf/arenaz_sampler_test.cc @@ -30,6 +30,8 @@ #include +#include +#include #include #include #include @@ -37,7 +39,6 @@ #include #include -#include // Must be included last. @@ -57,10 +58,17 @@ class ThreadSafeArenaStatsHandlePeer { return h->info_; } }; + std::vector GetBytesAllocated(ThreadSafeArenazSampler* s) { std::vector res; s->Iterate([&](const ThreadSafeArenaStats& info) { - res.push_back(info.bytes_allocated.load(std::memory_order_acquire)); + for (const auto& block_stats : info.block_histogram) { + size_t bytes_allocated = + block_stats.bytes_allocated.load(std::memory_order_acquire); + if (bytes_allocated != 0) { + res.push_back(bytes_allocated); + } + } }); return res; } @@ -69,7 +77,8 @@ ThreadSafeArenaStats* Register(ThreadSafeArenazSampler* s, size_t size, int64_t stride) { auto* info = s->Register(stride); assert(info != nullptr); - info->bytes_allocated.store(size); + info->block_histogram[0].bytes_allocated.store(size, + std::memory_order_relaxed); return info; } @@ -85,46 +94,115 @@ TEST(ThreadSafeArenaStatsTest, PrepareForSampling) { MutexLock l(&info.init_mu); info.PrepareForSampling(kTestStride); - EXPECT_EQ(info.num_allocations.load(), 0); - EXPECT_EQ(info.bytes_used.load(), 0); - EXPECT_EQ(info.bytes_allocated.load(), 0); - EXPECT_EQ(info.bytes_wasted.load(), 0); - EXPECT_EQ(info.max_bytes_allocated.load(), 0); + for (const auto& block_stats : info.block_histogram) { + EXPECT_EQ(block_stats.num_allocations.load(std::memory_order_relaxed), 0); + EXPECT_EQ(block_stats.bytes_used.load(std::memory_order_relaxed), 0); + EXPECT_EQ(block_stats.bytes_allocated.load(std::memory_order_relaxed), 0); + EXPECT_EQ(block_stats.bytes_wasted.load(std::memory_order_relaxed), 0); + } + EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 0); EXPECT_EQ(info.weight, kTestStride); - info.num_allocations.store(1, std::memory_order_relaxed); - info.bytes_used.store(1, std::memory_order_relaxed); - info.bytes_allocated.store(1, std::memory_order_relaxed); - info.bytes_wasted.store(1, std::memory_order_relaxed); - info.max_bytes_allocated.store(1, std::memory_order_relaxed); + for (auto& block_stats : info.block_histogram) { + block_stats.num_allocations.store(1, std::memory_order_relaxed); + block_stats.bytes_used.store(1, std::memory_order_relaxed); + block_stats.bytes_allocated.store(1, std::memory_order_relaxed); + block_stats.bytes_wasted.store(1, std::memory_order_relaxed); + } + info.max_block_size.store(1, std::memory_order_relaxed); info.PrepareForSampling(2 * kTestStride); - EXPECT_EQ(info.num_allocations.load(), 0); - EXPECT_EQ(info.bytes_used.load(), 0); - EXPECT_EQ(info.bytes_allocated.load(), 0); - EXPECT_EQ(info.bytes_wasted.load(), 0); - EXPECT_EQ(info.max_bytes_allocated.load(), 0); + for (auto& block_stats : info.block_histogram) { + EXPECT_EQ(block_stats.num_allocations.load(std::memory_order_relaxed), 0); + EXPECT_EQ(block_stats.bytes_used.load(std::memory_order_relaxed), 0); + EXPECT_EQ(block_stats.bytes_allocated.load(std::memory_order_relaxed), 0); + EXPECT_EQ(block_stats.bytes_wasted.load(std::memory_order_relaxed), 0); + } + EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 0); EXPECT_EQ(info.weight, 2 * kTestStride); } +TEST(ThreadSafeArenaStatsTest, FindBin) { + size_t current_bin = 0; + size_t bytes = 1; + while (current_bin < ThreadSafeArenaStats::kBlockHistogramBins - 1) { + size_t next_bin = ThreadSafeArenaStats::FindBin(bytes); + if (next_bin != current_bin) { + // Test the bins increase linearly. + EXPECT_EQ(next_bin, current_bin + 1); + // Test the bins change only at values of the form 2^k + 1. + EXPECT_EQ(absl::popcount(bytes - 1), 1); + current_bin = next_bin; + } + ++bytes; + } +} + +TEST(ThreadSafeArenaStatsTest, MinMaxBlockSizeForBin) { + std::pair current_limits = + ThreadSafeArenaStats::MinMaxBlockSizeForBin(0); + EXPECT_EQ(current_limits.first, 1); + EXPECT_LT(current_limits.first, current_limits.second); + for (size_t i = 1; i < ThreadSafeArenaStats::kBlockHistogramBins; ++i) { + std::pair next_limits = + ThreadSafeArenaStats::MinMaxBlockSizeForBin(i); + EXPECT_LT(next_limits.first, next_limits.second); + // Test the limits do not have gaps. + EXPECT_EQ(next_limits.first, current_limits.second + 1); + if (i != ThreadSafeArenaStats::kBlockHistogramBins - 1) { + EXPECT_EQ(next_limits.second, 2 * current_limits.second); + } + current_limits = next_limits; + } + // Test the limits cover the entire range possible. + EXPECT_EQ(current_limits.second, std::numeric_limits::max()); +} + TEST(ThreadSafeArenaStatsTest, RecordAllocateSlow) { + ThreadSafeArenaStats info; + constexpr int64_t kTestStride = 458; + MutexLock l(&info.init_mu); + info.PrepareForSampling(kTestStride); + RecordAllocateSlow(&info, /*requested=*/0, /*allocated=*/128, /*wasted=*/0); + EXPECT_EQ( + info.block_histogram[0].num_allocations.load(std::memory_order_relaxed), + 1); + EXPECT_EQ(info.block_histogram[0].bytes_used.load(std::memory_order_relaxed), + 0); + EXPECT_EQ( + info.block_histogram[0].bytes_allocated.load(std::memory_order_relaxed), + 128); + EXPECT_EQ( + info.block_histogram[0].bytes_wasted.load(std::memory_order_relaxed), 0); + EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 128); + RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/256, + /*wasted=*/28); + EXPECT_EQ(info.block_histogram[0].bytes_used.load(std::memory_order_relaxed), + 100); + EXPECT_EQ( + info.block_histogram[0].bytes_wasted.load(std::memory_order_relaxed), 28); + EXPECT_EQ( + info.block_histogram[1].num_allocations.load(std::memory_order_relaxed), + 1); + EXPECT_EQ( + info.block_histogram[1].bytes_allocated.load(std::memory_order_relaxed), + 256); + EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 256); +} + +TEST(ThreadSafeArenaStatsTest, RecordAllocateSlowMaxBlockSizeTest) { ThreadSafeArenaStats info; constexpr int64_t kTestStride = 458; MutexLock l(&info.init_mu); info.PrepareForSampling(kTestStride); RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128, /*wasted=*/0); - EXPECT_EQ(info.num_allocations.load(), 1); - EXPECT_EQ(info.bytes_used.load(), 100); - EXPECT_EQ(info.bytes_allocated.load(), 128); - EXPECT_EQ(info.bytes_wasted.load(), 0); - EXPECT_EQ(info.max_bytes_allocated.load(), 0); + EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 128); RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/256, /*wasted=*/28); - EXPECT_EQ(info.num_allocations.load(), 2); - EXPECT_EQ(info.bytes_used.load(), 200); - EXPECT_EQ(info.bytes_allocated.load(), 384); - EXPECT_EQ(info.bytes_wasted.load(), 28); - EXPECT_EQ(info.max_bytes_allocated.load(), 0); + EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 256); + RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128, + /*wasted=*/28); + EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 256); } TEST(ThreadSafeArenazSamplerTest, SamplingCorrectness) { @@ -197,12 +275,15 @@ TEST(ThreadSafeArenazSamplerTest, Handle) { constexpr int64_t kTestStride = 17; ThreadSafeArenaStatsHandle h(sampler.Register(kTestStride)); auto* info = ThreadSafeArenaStatsHandlePeer::GetInfo(&h); - info->bytes_allocated.store(0x12345678, std::memory_order_relaxed); + info->block_histogram[0].bytes_allocated.store(0x12345678, + std::memory_order_relaxed); bool found = false; sampler.Iterate([&](const ThreadSafeArenaStats& h) { if (&h == info) { - EXPECT_EQ(h.bytes_allocated.load(), 0x12345678); + EXPECT_EQ( + h.block_histogram[0].bytes_allocated.load(std::memory_order_relaxed), + 0x12345678); EXPECT_EQ(h.weight, kTestStride); found = true; } @@ -215,7 +296,8 @@ TEST(ThreadSafeArenazSamplerTest, Handle) { if (&h == info) { // this will only happen if some other thread has resurrected the info // the old handle was using. - if (h.bytes_allocated.load() == 0x12345678) { + if (h.block_histogram[0].bytes_allocated.load( + std::memory_order_relaxed) == 0x12345678) { found = true; } } @@ -231,7 +313,7 @@ TEST(ThreadSafeArenazSamplerTest, Registration) { auto* info2 = Register(&sampler, 2, kTestStride); EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 2)); - info1->bytes_allocated.store(3); + info1->block_histogram[0].bytes_allocated.store(3, std::memory_order_relaxed); EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(3, 2)); sampler.Unregister(info1); @@ -243,18 +325,18 @@ TEST(ThreadSafeArenazSamplerTest, Unregistration) { std::vector infos; constexpr int64_t kTestStride = 200; for (size_t i = 0; i < 3; ++i) { - infos.push_back(Register(&sampler, i, kTestStride)); + infos.push_back(Register(&sampler, i + 1, kTestStride)); } - EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 1, 2)); + EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 2, 3)); sampler.Unregister(infos[1]); - EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2)); + EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 3)); infos.push_back(Register(&sampler, 3, kTestStride)); infos.push_back(Register(&sampler, 4, kTestStride)); - EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2, 3, 4)); + EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 3, 3, 4)); sampler.Unregister(infos[3]); - EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2, 4)); + EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 3, 4)); sampler.Unregister(infos[0]); sampler.Unregister(infos[2]); @@ -360,6 +442,7 @@ TEST(ThreadSafeArenazSamplerTest, MultiThread) { SetThreadSafeArenazEnabled(true); // Setting 1 as the parameter value means one in every two arenas would be // sampled, on average. + int32_t oldparam = ThreadSafeArenazSampleParameter(); SetThreadSafeArenazSampleParameter(1); SetThreadSafeArenazGlobalNextSample(0); auto& sampler = GlobalThreadSafeArenazSampler(); @@ -387,6 +470,95 @@ TEST(ThreadSafeArenazSamplerTest, MultiThread) { } } EXPECT_GT(count, 0); + SetThreadSafeArenazSampleParameter(oldparam); +} + +class SampleFirstArenaThread : public Thread { + protected: + void Run() override { + google::protobuf::Arena arena; + google::protobuf::ArenaSafeUniquePtr< + protobuf_test_messages::proto2::TestAllTypesProto2> + message = google::protobuf::MakeArenaSafeUnique< + protobuf_test_messages::proto2::TestAllTypesProto2>(&arena); + GOOGLE_CHECK(message != nullptr); + arena_created_.Notify(); + samples_counted_.WaitForNotification(); + } + + public: + explicit SampleFirstArenaThread(const thread::Options& options) + : Thread(options, "SampleFirstArenaThread") {} + + absl::Notification arena_created_; + absl::Notification samples_counted_; +}; + +// Test that the first arena created on a thread may and may not be chosen for +// sampling. +TEST(ThreadSafeArenazSamplerTest, SampleFirstArena) { + SetThreadSafeArenazEnabled(true); + auto& sampler = GlobalThreadSafeArenazSampler(); + + enum class SampleResult { + kSampled, + kUnsampled, + kSpoiled, + }; + + auto count_samples = [&]() { + int count = 0; + sampler.Iterate([&](const ThreadSafeArenaStats& h) { ++count; }); + return count; + }; + + auto run_sample_experiment = [&]() { + int before = count_samples(); + thread::Options options; + options.set_joinable(true); + SampleFirstArenaThread t(options); + t.Start(); + t.arena_created_.WaitForNotification(); + int during = count_samples(); + t.samples_counted_.Notify(); + t.Join(); + int after = count_samples(); + + // If we didn't get back where we were, some other thread may have + // created an arena and produced an invalid experiment run. + if (before != after) return SampleResult::kSpoiled; + + switch (during - before) { + case 1: + return SampleResult::kSampled; + case 0: + return SampleResult::kUnsampled; + default: + return SampleResult::kSpoiled; + } + }; + + constexpr int kTrials = 10000; + bool sampled = false; + bool unsampled = false; + for (int i = 0; i < kTrials; ++i) { + switch (run_sample_experiment()) { + case SampleResult::kSampled: + sampled = true; + break; + case SampleResult::kUnsampled: + unsampled = true; + break; + default: + break; + } + + // This is the success criteria for the entire test. At some point + // we sampled the first arena and at some point we did not. + if (sampled && unsampled) return; + } + EXPECT_TRUE(sampled); + EXPECT_TRUE(unsampled); } #endif // defined(PROTOBUF_ARENAZ_SAMPLE) diff --git a/src/google/protobuf/compiler/command_line_interface.cc b/src/google/protobuf/compiler/command_line_interface.cc index a688bcfb0623..f5cc9a91737f 100644 --- a/src/google/protobuf/compiler/command_line_interface.cc +++ b/src/google/protobuf/compiler/command_line_interface.cc @@ -339,9 +339,12 @@ class CommandLineInterface::ErrorPrinter void AddErrorOrWarning(const std::string& filename, int line, int column, const std::string& message, const std::string& type, std::ostream& out) { - // Print full path when running under MSVS std::string dfile; - if (format_ == CommandLineInterface::ERROR_FORMAT_MSVS && + if ( +#ifndef PROTOBUF_OPENSOURCE + // Print full path when running under MSVS + format_ == CommandLineInterface::ERROR_FORMAT_MSVS && +#endif // !PROTOBUF_OPENSOURCE tree_ != nullptr && tree_->VirtualFileToDiskFile(filename, &dfile)) { out << dfile; } else { @@ -398,7 +401,6 @@ class CommandLineInterface::GeneratorContextImpl : public GeneratorContext { // Get name of all output files. void GetOutputFilenames(std::vector* output_filenames); - // implements GeneratorContext -------------------------------------- io::ZeroCopyOutputStream* Open(const std::string& filename) override; io::ZeroCopyOutputStream* OpenForAppend(const std::string& filename) override; @@ -963,6 +965,7 @@ PopulateSingleSimpleDescriptorDatabase(const std::string& descriptor_set_name); int CommandLineInterface::Run(int argc, const char* const argv[]) { Clear(); + switch (ParseArguments(argc, argv)) { case PARSE_ARGUMENT_DONE_AND_EXIT: return 0; @@ -1076,7 +1079,6 @@ int CommandLineInterface::Run(int argc, const char* const argv[]) { } } - // Write all output to disk. for (const auto& pair : output_directories) { const std::string& location = pair.first; GeneratorContextImpl* directory = pair.second.get(); @@ -1151,7 +1153,6 @@ int CommandLineInterface::Run(int argc, const char* const argv[]) { // Do not add a default case. } } - return 0; } @@ -1680,8 +1681,7 @@ CommandLineInterface::InterpretArgument(const std::string& name, })) { case google::protobuf::io::win32::ExpandWildcardsResult::kSuccess: break; - case google::protobuf::io::win32::ExpandWildcardsResult:: - kErrorNoMatchingFile: + case google::protobuf::io::win32::ExpandWildcardsResult::kErrorNoMatchingFile: // Path does not exist, is not a file, or it's longer than MAX_PATH and // long path handling is disabled. std::cerr << "Invalid file name pattern or missing input file \"" @@ -2124,7 +2124,8 @@ bool CommandLineInterface::EnforceProto3OptionalSupport( << codegen_name << " hasn't been updated to support optional fields in " "proto3. Please ask the owner of this code generator to " - "support proto3 optional."; + "support proto3 optional." + << std::endl; return false; } } diff --git a/src/google/protobuf/compiler/cpp/field.cc b/src/google/protobuf/compiler/cpp/field.cc index cf4f14e8bfa3..90d20848b3af 100644 --- a/src/google/protobuf/compiler/cpp/field.cc +++ b/src/google/protobuf/compiler/cpp/field.cc @@ -330,7 +330,6 @@ void FieldGenerator::GenerateCopyConstructorCode(io::Printer* printer) const { } } - void SetCommonOneofFieldVariables( const FieldDescriptor* descriptor, std::map* variables) { diff --git a/src/google/protobuf/compiler/cpp/field.h b/src/google/protobuf/compiler/cpp/field.h index 3903e7986278..3fcbda371c59 100644 --- a/src/google/protobuf/compiler/cpp/field.h +++ b/src/google/protobuf/compiler/cpp/field.h @@ -208,7 +208,6 @@ class FieldGenerator { virtual bool IsInlined() const { return false; } - virtual ArenaDtorNeeds NeedsArenaDestructor() const { return ArenaDtorNeeds::kNone; } diff --git a/src/google/protobuf/compiler/cpp/file.cc b/src/google/protobuf/compiler/cpp/file.cc index 838e0ab9b414..502d8c007e67 100644 --- a/src/google/protobuf/compiler/cpp/file.cc +++ b/src/google/protobuf/compiler/cpp/file.cc @@ -495,12 +495,10 @@ void FileGenerator::GenerateSourceDefaultInstance(int idx, generator->GenerateInitDefaultSplitInstance(printer); format( "} {}\n" - " ~$1$() {}\n" " union {\n" - " $2$ _instance;\n" + " $1$ _instance;\n" " };\n" "};\n", - DefaultInstanceType(generator->descriptor_, options_, /*split=*/true), StrCat(generator->classname_, "::Impl_::Split")); // NO_DESTROY is not necessary for correctness. The empty destructor is // enough. However, the empty destructor fails to be elided in some @@ -508,7 +506,7 @@ void FileGenerator::GenerateSourceDefaultInstance(int idx, // there just to improve performance and binary size in these builds. format( "PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT " - "PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 $1$ $2$;\n", + "PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 const $1$ $2$;\n", DefaultInstanceType(generator->descriptor_, options_, /*split=*/true), DefaultInstanceName(generator->descriptor_, options_, /*split=*/true)); } @@ -999,7 +997,7 @@ class FileGenerator::ForwardDeclarations { const Descriptor* class_desc = p.second; format( "struct $1$;\n" - "$dllexport_decl $extern $1$ $2$;\n", + "$dllexport_decl $extern const $1$ $2$;\n", DefaultInstanceType(class_desc, options, /*split=*/true), DefaultInstanceName(class_desc, options, /*split=*/true)); } diff --git a/src/google/protobuf/compiler/cpp/helpers.cc b/src/google/protobuf/compiler/cpp/helpers.cc index a8c5899180df..d182c8607f17 100644 --- a/src/google/protobuf/compiler/cpp/helpers.cc +++ b/src/google/protobuf/compiler/cpp/helpers.cc @@ -176,7 +176,6 @@ static const char* const kKeywordList[] = { #endif // !PROTOBUF_FUTURE_BREAKING_CHANGES }; - static std::unordered_set* MakeKeywordsMap() { auto* result = new std::unordered_set(); for (const auto keyword : kKeywordList) { @@ -525,7 +524,6 @@ std::string FieldName(const FieldDescriptor* field) { return result; } - std::string FieldMemberName(const FieldDescriptor* field, bool split) { StringPiece prefix = IsMapEntryMessage(field->containing_type()) ? "" : "_impl_."; @@ -876,8 +874,6 @@ std::string SafeFunctionName(const Descriptor* descriptor, bool IsProfileDriven(const Options& options) { return options.access_info_map != nullptr; } - - bool IsStringInlined(const FieldDescriptor* descriptor, const Options& options) { (void)descriptor; @@ -925,6 +921,13 @@ bool HasLazyFields(const FileDescriptor* file, const Options& options, bool ShouldSplit(const Descriptor*, const Options&) { return false; } bool ShouldSplit(const FieldDescriptor*, const Options&) { return false; } +bool ShouldForceAllocationOnConstruction(const Descriptor* desc, + const Options& options) { + (void)desc; + (void)options; + return false; +} + static bool HasRepeatedFields(const Descriptor* descriptor) { for (int i = 0; i < descriptor->field_count(); ++i) { if (descriptor->field(i)->label() == FieldDescriptor::LABEL_REPEATED) { diff --git a/src/google/protobuf/compiler/cpp/helpers.h b/src/google/protobuf/compiler/cpp/helpers.h index 6befbaa0cb0e..21a488e2a5f2 100644 --- a/src/google/protobuf/compiler/cpp/helpers.h +++ b/src/google/protobuf/compiler/cpp/helpers.h @@ -379,6 +379,11 @@ bool ShouldSplit(const Descriptor* desc, const Options& options); // Is the given field being split out? bool ShouldSplit(const FieldDescriptor* field, const Options& options); +// Should we generate code that force creating an allocation in the constructor +// of the given message? +bool ShouldForceAllocationOnConstruction(const Descriptor* desc, + const Options& options); + inline bool IsFieldUsed(const FieldDescriptor* /* field */, const Options& /* options */) { return true; diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index 04bb5d149298..50b86f7b2b08 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -828,7 +828,6 @@ void MessageGenerator::GenerateFieldAccessorDeclarations(io::Printer* printer) { // Generate type-specific accessor declarations. field_generators_.get(field).GenerateAccessorDeclarations(printer); - format("\n"); } @@ -1238,41 +1237,41 @@ void MessageGenerator::GenerateFieldAccessorDefinitions(io::Printer* printer) { Formatter::SaveState saver(&format); format.AddMap(vars); - // Generate has_$name$() or $name$_size(). - if (field->is_repeated()) { - if (IsFieldStripped(field, options_)) { - format( - "inline int $classname$::$name$_size() const { " - "__builtin_trap(); }\n"); - } else { - format( - "inline int $classname$::_internal_$name$_size() const {\n" - " return $field$$1$.size();\n" - "}\n" - "inline int $classname$::$name$_size() const {\n" - "$annotate_size$" - " return _internal_$name$_size();\n" - "}\n", - IsImplicitWeakField(field, options_, scc_analyzer_) && - field->message_type() - ? ".weak" - : ""); - } - } else if (field->real_containing_oneof()) { - format.Set("field_name", UnderscoresToCamelCase(field->name(), true)); - format.Set("oneof_name", field->containing_oneof()->name()); - format.Set("oneof_index", - StrCat(field->containing_oneof()->index())); - GenerateOneofMemberHasBits(field, format); + + // Generate has_$name$() or $name$_size(). + if (field->is_repeated()) { + if (IsFieldStripped(field, options_)) { + format( + "inline int $classname$::$name$_size() const { " + "__builtin_trap(); }\n"); } else { - // Singular field. - GenerateSingularFieldHasBits(field, format); + format( + "inline int $classname$::_internal_$name$_size() const {\n" + " return $field$$1$.size();\n" + "}\n" + "inline int $classname$::$name$_size() const {\n" + "$annotate_size$" + " return _internal_$name$_size();\n" + "}\n", + IsImplicitWeakField(field, options_, scc_analyzer_) && + field->message_type() + ? ".weak" + : ""); } - - if (!IsCrossFileMaybeMap(field)) { - GenerateFieldClear(field, true, format); + } else if (field->real_containing_oneof()) { + format.Set("field_name", UnderscoresToCamelCase(field->name(), true)); + format.Set("oneof_name", field->containing_oneof()->name()); + format.Set("oneof_index", + StrCat(field->containing_oneof()->index())); + GenerateOneofMemberHasBits(field, format); + } else { + // Singular field. + GenerateSingularFieldHasBits(field, format); } + if (!IsCrossFileMaybeMap(field)) { + GenerateFieldClear(field, true, format); + } // Generate type-specific accessors. if (!IsFieldStripped(field, options_)) { field_generators_.get(field).GenerateInlineAccessorDefinitions(printer); @@ -1760,7 +1759,7 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* printer) { format( "private:\n" "inline bool IsSplitMessageDefault() const {\n" - " return $split$ == reinterpret_cast(&$1$);\n" + " return $split$ == reinterpret_cast(&$1$);\n" "}\n" "PROTOBUF_NOINLINE void PrepareSplitMessageForWrite();\n" "public:\n", @@ -1928,6 +1927,8 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* printer) { " typedef void InternalArenaConstructable_;\n" " typedef void DestructorSkippable_;\n" "};\n" + "static_assert(std::is_trivially_copy_constructible::value);\n" + "static_assert(std::is_trivially_destructible::value);\n" "Split* _split_;\n"); } @@ -2204,11 +2205,12 @@ void MessageGenerator::GenerateClassMethods(io::Printer* printer) { " if (IsSplitMessageDefault()) {\n" " void* chunk = " "::PROTOBUF_NAMESPACE_ID::internal::CreateSplitMessageGeneric(" - "GetArenaForAllocation(), &$1$, sizeof(Impl_::Split));\n" + "GetArenaForAllocation(), &$1$, sizeof(Impl_::Split), this, &$2$);\n" " $split$ = reinterpret_cast(chunk);\n" " }\n" "}\n", - DefaultInstanceName(descriptor_, options_, /*split=*/true)); + DefaultInstanceName(descriptor_, options_, /*split=*/true), + DefaultInstanceName(descriptor_, options_, /*split=*/false)); } GenerateVerify(printer); @@ -2421,8 +2423,15 @@ void MessageGenerator::GenerateSharedConstructorCode(io::Printer* printer) { } if (ShouldSplit(descriptor_, options_)) { put_sep(); - format("decltype($split$){reinterpret_cast(&$1$)}", - DefaultInstanceName(descriptor_, options_, /*split=*/true)); + // We can't assign the default split to this->split without the const_cast + // because the former is a const. The const_cast is safe because we don't + // intend to modify the default split through this pointer, and we also + // expect the default split to be in the rodata section which is protected + // from mutation. + format( + "decltype($split$){const_cast" + "(reinterpret_cast(&$1$))}", + DefaultInstanceName(descriptor_, options_, /*split=*/true)); } for (auto oneof : OneOfRange(descriptor_)) { put_sep(); @@ -2483,6 +2492,13 @@ void MessageGenerator::GenerateSharedConstructorCode(io::Printer* printer) { field_generators_.get(field).GenerateConstructorCode(printer); } + if (ShouldForceAllocationOnConstruction(descriptor_, options_)) { + format( + "#ifdef PROTOBUF_FORCE_ALLOCATION_ON_CONSTRUCTION\n" + "$mutable_unknown_fields$;\n" + "#endif // PROTOBUF_FORCE_ALLOCATION_ON_CONSTRUCTION\n"); + } + for (auto oneof : OneOfRange(descriptor_)) { format("clear_has_$1$();\n", oneof->name()); } @@ -2674,7 +2690,7 @@ void MessageGenerator::GenerateConstexprConstructor(io::Printer* printer) { } if (ShouldSplit(descriptor_, options_)) { put_sep(); - format("/*decltype($split$)*/&$1$._instance", + format("/*decltype($split$)*/const_cast(&$1$._instance)", DefaultInstanceName(descriptor_, options_, /*split=*/true)); } @@ -2722,6 +2738,13 @@ void MessageGenerator::GenerateCopyConstructorBody(io::Printer* printer) const { " static_cast(reinterpret_cast(&$last$) -\n" " reinterpret_cast(&$first$)) + sizeof($last$));\n"; + if (ShouldForceAllocationOnConstruction(descriptor_, options_)) { + format( + "#ifdef PROTOBUF_FORCE_ALLOCATION_ON_CONSTRUCTION\n" + "$mutable_unknown_fields$;\n" + "#endif // PROTOBUF_FORCE_ALLOCATION_ON_CONSTRUCTION\n"); + } + for (size_t i = 0; i < optimized_order_.size(); ++i) { const FieldDescriptor* field = optimized_order_[i]; if (ShouldSplit(field, options_)) { @@ -2852,8 +2875,10 @@ void MessageGenerator::GenerateStructors(io::Printer* printer) { } if (ShouldSplit(descriptor_, options_)) { put_sep(); - format("decltype($split$){reinterpret_cast(&$1$)}", - DefaultInstanceName(descriptor_, options_, /*split=*/true)); + format( + "decltype($split$){const_cast" + "(reinterpret_cast(&$1$))}", + DefaultInstanceName(descriptor_, options_, /*split=*/true)); } for (auto oneof : OneOfRange(descriptor_)) { put_sep(); diff --git a/src/google/protobuf/compiler/cpp/options.h b/src/google/protobuf/compiler/cpp/options.h index 88d4b3097a87..7aab66913131 100644 --- a/src/google/protobuf/compiler/cpp/options.h +++ b/src/google/protobuf/compiler/cpp/options.h @@ -85,7 +85,7 @@ struct Options { bool profile_driven_inline_string = true; bool message_owned_arena_trial = false; bool force_split = false; - bool profile_driven_split = false; + bool profile_driven_split = true; #ifdef PROTOBUF_STABLE_EXPERIMENTS bool force_eagerly_verified_lazy = true; bool force_inline_string = true; diff --git a/src/google/protobuf/compiler/cpp/parse_function_generator.cc b/src/google/protobuf/compiler/cpp/parse_function_generator.cc index 8045f480fd87..30b0fd44fe80 100644 --- a/src/google/protobuf/compiler/cpp/parse_function_generator.cc +++ b/src/google/protobuf/compiler/cpp/parse_function_generator.cc @@ -83,7 +83,8 @@ int TagSize(uint32_t field_number) { return 2; } -void PopulateFastFieldEntry(const TailCallTableInfo::FieldEntryInfo& entry, +void PopulateFastFieldEntry(const Descriptor* descriptor, + const TailCallTableInfo::FieldEntryInfo& entry, const Options& options, TailCallTableInfo::FastFieldInfo& info); @@ -158,6 +159,7 @@ bool IsFieldEligibleForFastParsing( } std::vector SplitFastFieldsForSize( + const Descriptor* descriptor, const std::vector& field_entries, int table_size_log2, const Options& options, MessageSCCAnalyzer* scc_analyzer) { @@ -200,7 +202,7 @@ std::vector SplitFastFieldsForSize( GOOGLE_CHECK(info.func_name.empty()) << info.func_name; info.field = field; info.coded_tag = tag; - PopulateFastFieldEntry(entry, options, info); + PopulateFastFieldEntry(descriptor, entry, options, info); // If this field does not have presence, then it can set an out-of-bounds // bit (tailcall parsing uses a uint64_t for hasbits, but only stores 32). info.hasbit_idx = HasHasbit(field) ? entry.hasbit_idx : 63; @@ -412,8 +414,8 @@ TailCallTableInfo::TailCallTableInfo( int num_fast_fields = -1; for (int try_size_log2 : {0, 1, 2, 3, 4, 5}) { size_t try_size = 1 << try_size_log2; - auto split_fields = SplitFastFieldsForSize(field_entries, try_size_log2, - options, scc_analyzer); + auto split_fields = SplitFastFieldsForSize( + descriptor, field_entries, try_size_log2, options, scc_analyzer); GOOGLE_CHECK_EQ(split_fields.size(), try_size); int try_num_fast_fields = 0; for (const auto& info : split_fields) { @@ -1667,11 +1669,12 @@ void ParseFunctionGenerator::GenerateFieldSwitch( namespace { -void PopulateFastFieldEntry(const TailCallTableInfo::FieldEntryInfo& entry, +void PopulateFastFieldEntry(const Descriptor* descriptor, + const TailCallTableInfo::FieldEntryInfo& entry, const Options& options, TailCallTableInfo::FastFieldInfo& info) { const FieldDescriptor* field = entry.field; - std::string name = "::_pbi::TcParser::Fast"; + std::string name; uint8_t aux_idx = static_cast(entry.aux_idx); switch (field->type()) { @@ -1784,7 +1787,36 @@ void PopulateFastFieldEntry(const TailCallTableInfo::FieldEntryInfo& entry, // Append the tag length. Fast parsing only handles 1- or 2-byte tags. name.append(TagSize(field->number()) == 1 ? "1" : "2"); - info.func_name = std::move(name); + if (name == "V8S1") { + info.func_name = StrCat( + "::_pbi::TcParser::SingularVarintNoZag1()"); + } else if (name == "V32S1") { + info.func_name = StrCat( + "::_pbi::TcParser::SingularVarintNoZag1()"); + } else if (name == "V64S1") { + info.func_name = StrCat( + "::_pbi::TcParser::SingularVarintNoZag1()"); + } else { + info.func_name = StrCat("::_pbi::TcParser::Fast", name); + } info.aux_idx = aux_idx; } diff --git a/src/google/protobuf/compiler/csharp/csharp_generator_unittest.cc b/src/google/protobuf/compiler/csharp/csharp_generator_unittest.cc index e21eff17ba88..56b920dbdf06 100644 --- a/src/google/protobuf/compiler/csharp/csharp_generator_unittest.cc +++ b/src/google/protobuf/compiler/csharp/csharp_generator_unittest.cc @@ -74,6 +74,22 @@ TEST(DescriptorProtoHelpers, IsDescriptorOptionMessage) { EXPECT_FALSE(IsDescriptorOptionMessage(DescriptorProto::descriptor())); } +TEST(CSharpIdentifiers, UnderscoresToCamelCase) { + EXPECT_EQ("FooBar", UnderscoresToCamelCase("Foo_Bar", true)); + EXPECT_EQ("fooBar", UnderscoresToCamelCase("FooBar", false)); + EXPECT_EQ("foo123", UnderscoresToCamelCase("foo_123", false)); + // remove leading underscores + EXPECT_EQ("Foo123", UnderscoresToCamelCase("_Foo_123", true)); + // this one has slight unexpected output as it capitalises the first + // letter after consuming the underscores, but this was the existing + // behaviour so I have not changed it + EXPECT_EQ("FooBar", UnderscoresToCamelCase("___fooBar", false)); + // leave a leading underscore for identifiers that would otherwise + // be invalid because they would start with a digit + EXPECT_EQ("_123Foo", UnderscoresToCamelCase("_123_foo", true)); + EXPECT_EQ("_123Foo", UnderscoresToCamelCase("___123_foo", true)); +} + } // namespace } // namespace csharp } // namespace compiler diff --git a/src/google/protobuf/compiler/csharp/csharp_helpers.cc b/src/google/protobuf/compiler/csharp/csharp_helpers.cc index 42d952721e11..1b58b956691b 100644 --- a/src/google/protobuf/compiler/csharp/csharp_helpers.cc +++ b/src/google/protobuf/compiler/csharp/csharp_helpers.cc @@ -144,6 +144,7 @@ std::string UnderscoresToCamelCase(const std::string& input, bool cap_next_letter, bool preserve_period) { std::string result; + // Note: I distrust ctype.h due to locales. for (int i = 0; i < input.size(); i++) { if ('a' <= input[i] && input[i] <= 'z') { @@ -177,6 +178,23 @@ std::string UnderscoresToCamelCase(const std::string& input, if (input.size() > 0 && input[input.size() - 1] == '#') { result += '_'; } + + // https://github.com/protocolbuffers/protobuf/issues/8101 + // To avoid generating invalid identifiers - if the input string + // starts with _ (or multiple underscores then digit) then + // we need to preserve the underscore as an identifier cannot start + // with a digit. + // This check is being done after the loop rather than before + // to handle the case where there are multiple underscores before the + // first digit. We let them all be consumed so we can see if we would + // start with a digit. + // Note: not preserving leading underscores for all otherwise valid identifiers + // so as to not break anything that relies on the existing behaviour + if (result.size() > 0 && ('0' <= result[0] && result[0] <= '9') + && input.size() > 0 && input[0] == '_') + { + result.insert(0, 1, '_'); + } return result; } @@ -379,15 +397,30 @@ std::string GetFieldConstantName(const FieldDescriptor* field) { } std::string GetPropertyName(const FieldDescriptor* descriptor) { + // Names of members declared or overridden in the message. + static const auto& reserved_member_names = *new std::unordered_set({ + "Types", + "Descriptor", + "Equals", + "ToString", + "GetHashCode", + "WriteTo", + "Clone", + "CalculateSize", + "MergeFrom", + "OnConstruction", + "Parser" + }); + // TODO(jtattermusch): consider introducing csharp_property_name field option std::string property_name = UnderscoresToPascalCase(GetFieldName(descriptor)); - // Avoid either our own type name or reserved names. Note that not all names - // are reserved - a field called to_string, write_to etc would still cause a problem. + // Avoid either our own type name or reserved names. // There are various ways of ending up with naming collisions, but we try to avoid obvious - // ones. + // ones. In particular, we avoid the names of all the members we generate. + // Note that we *don't* add an underscore for MemberwiseClone or GetType. Those generate + // warnings, but not errors; changing the name now could be a breaking change. if (property_name == descriptor->containing_type()->name() - || property_name == "Types" - || property_name == "Descriptor") { + || reserved_member_names.find(property_name) != reserved_member_names.end()) { property_name += "_"; } return property_name; diff --git a/src/google/protobuf/compiler/csharp/csharp_map_field.cc b/src/google/protobuf/compiler/csharp/csharp_map_field.cc index a13b995da8c9..9efd3d524f26 100644 --- a/src/google/protobuf/compiler/csharp/csharp_map_field.cc +++ b/src/google/protobuf/compiler/csharp/csharp_map_field.cc @@ -90,7 +90,7 @@ void MapFieldGenerator::GenerateMembers(io::Printer* printer) { void MapFieldGenerator::GenerateMergingCode(io::Printer* printer) { printer->Print( variables_, - "$name$_.Add(other.$name$_);\n"); + "$name$_.MergeFrom(other.$name$_);\n"); } void MapFieldGenerator::GenerateParsingCode(io::Printer* printer) { diff --git a/src/google/protobuf/compiler/java/enum_field.cc b/src/google/protobuf/compiler/java/enum_field.cc index b82d147716cd..aa3a51f65e5e 100644 --- a/src/google/protobuf/compiler/java/enum_field.cc +++ b/src/google/protobuf/compiler/java/enum_field.cc @@ -281,6 +281,18 @@ void ImmutableEnumFieldGenerator::GenerateKotlinDslMembers( " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" " }\n"); + if (SupportUnknownEnumValue(descriptor_->file())) { + printer->Print( + variables_, + "$kt_deprecation$ var $kt_name$Value: kotlin.Int\n" + " @JvmName(\"${$get$kt_capitalized_name$Value$}$\")\n" + " get() = $kt_dsl_builder$.${$get$capitalized_name$Value$}$()\n" + " @JvmName(\"${$set$kt_capitalized_name$Value$}$\")\n" + " set(value) {\n" + " $kt_dsl_builder$.${$set$capitalized_name$Value$}$(value)\n" + " }\n"); + } + WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, /* builder */ false, /* kdoc */ true); printer->Print(variables_, diff --git a/src/google/protobuf/compiler/java/enum_field_lite.cc b/src/google/protobuf/compiler/java/enum_field_lite.cc index caa215b7da78..a2568247148f 100644 --- a/src/google/protobuf/compiler/java/enum_field_lite.cc +++ b/src/google/protobuf/compiler/java/enum_field_lite.cc @@ -296,6 +296,18 @@ void ImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" " }\n"); + if (SupportUnknownEnumValue(descriptor_->file())) { + printer->Print( + variables_, + "$kt_deprecation$ var $kt_name$Value: kotlin.Int\n" + " @JvmName(\"${$get$kt_capitalized_name$Value$}$\")\n" + " get() = $kt_dsl_builder$.${$get$capitalized_name$Value$}$()\n" + " @JvmName(\"${$set$kt_capitalized_name$Value$}$\")\n" + " set(value) {\n" + " $kt_dsl_builder$.${$set$capitalized_name$Value$}$(value)\n" + " }\n"); + } + WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, /* builder */ false, /* kdoc */ true); printer->Print(variables_, diff --git a/src/google/protobuf/compiler/java/file.cc b/src/google/protobuf/compiler/java/file.cc index bd3647d7b870..c4665182d4a0 100644 --- a/src/google/protobuf/compiler/java/file.cc +++ b/src/google/protobuf/compiler/java/file.cc @@ -704,10 +704,13 @@ void FileGenerator::GenerateKotlinSiblings( options_.annotate_code ? &annotation_collector : nullptr); printer.Print( - "//Generated by the protocol buffer compiler. DO NOT EDIT!\n" + "// Generated by the protocol buffer compiler. DO NOT EDIT!\n" "// source: $filename$\n" "\n", "filename", descriptor->file()->name()); + printer.Print( + "// Generated files should ignore deprecation warnings\n" + "@file:Suppress(\"DEPRECATION\")"); if (!java_package_.empty()) { printer.Print( "package $package$;\n" diff --git a/src/google/protobuf/compiler/java/generator.h b/src/google/protobuf/compiler/java/generator.h index bbc71700c66f..39d2445e7dd4 100644 --- a/src/google/protobuf/compiler/java/generator.h +++ b/src/google/protobuf/compiler/java/generator.h @@ -63,7 +63,12 @@ class PROTOC_EXPORT JavaGenerator : public CodeGenerator { uint64_t GetSupportedFeatures() const override; + void set_opensource_runtime(bool opensource) { + opensource_runtime_ = opensource; + } + private: + bool opensource_runtime_ = true; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(JavaGenerator); }; diff --git a/src/google/protobuf/compiler/java/helpers.cc b/src/google/protobuf/compiler/java/helpers.cc index 4593991f3088..98c0a79b00f6 100644 --- a/src/google/protobuf/compiler/java/helpers.cc +++ b/src/google/protobuf/compiler/java/helpers.cc @@ -34,6 +34,8 @@ #include +#include "message.h" + #include #include #include @@ -1120,6 +1122,22 @@ void EscapeUtf16ToString(uint16_t code, std::string* output) { } } +void MaybeSplitJavaMethod(io::Printer *printer, int* fields_in_function, int* method_num, + const char *chain_statement, const char *method_decl, + std::map& variables) { + + if ((*fields_in_function) > kMaxFieldsInMethod) { + variables["method_num"] = StrCat(*method_num); + printer->Print(variables, chain_statement); + printer->Outdent(); + printer->Print("}\n"); + printer->Print(variables, method_decl); + printer->Indent(); + *fields_in_function = 0; + ++(*method_num); + } +} + } // namespace java } // namespace compiler } // namespace protobuf diff --git a/src/google/protobuf/compiler/java/helpers.h b/src/google/protobuf/compiler/java/helpers.h index f4554324ede6..67f32e34052d 100644 --- a/src/google/protobuf/compiler/java/helpers.h +++ b/src/google/protobuf/compiler/java/helpers.h @@ -465,6 +465,13 @@ void EscapeUtf16ToString(uint16_t code, std::string* output); // bits 4-7: unused int GetExperimentalJavaFieldType(const FieldDescriptor* field); +// Split method if fields count > kMaxFields +// JIT compile only 8k bytes of bytecode with default settings +// variables - changes inside method +void MaybeSplitJavaMethod(io::Printer *printer, int* fields_in_function, int* method_num, + const char *chain_statement, const char *method_decl, + std::map& variables); + // To get the total number of entries need to be built for experimental runtime // and the first field number that are not in the table part std::pair GetTableDrivenNumberOfEntriesAndLookUpStartFieldNumber( diff --git a/src/google/protobuf/compiler/java/message.cc b/src/google/protobuf/compiler/java/message.cc index c1d8238a6654..feaef12f298b 100644 --- a/src/google/protobuf/compiler/java/message.cc +++ b/src/google/protobuf/compiler/java/message.cc @@ -621,9 +621,12 @@ void ImmutableMessageGenerator::GenerateMessageSerializationMethods( "ver", GeneratedCodeVersionSuffix()); } } + std::map variables; + variables["classname"] = name_resolver_->GetImmutableClassName(descriptor_); + variables["ver"] = GeneratedCodeVersionSuffix(); GenerateSerializeFieldsAndExtensions(printer, field_generators_, descriptor_, - sorted_fields.get()); + sorted_fields.get(), variables); if (descriptor_->options().message_set_wire_format()) { printer->Print("unknownFields.writeAsMessageSetTo(output);\n"); @@ -644,8 +647,17 @@ void ImmutableMessageGenerator::GenerateMessageSerializationMethods( printer->Print("size = 0;\n"); + int fields_in_function = 0; + int method_num = 1; + std::map variables_serialized_size; for (int i = 0; i < descriptor_->field_count(); i++) { field_generators_.get(sorted_fields[i]).GenerateSerializedSizeCode(printer); + variables_serialized_size["method_num"] = StrCat(method_num); + MaybeSplitJavaMethod(printer, &fields_in_function, &method_num, + "return _getSerializedSize_autosplit_$method_num$(size);\n", + "private int _getSerializedSize_autosplit_$method_num$(int size) {\n", + variables_serialized_size); + fields_in_function++; } if (descriptor_->extension_range_count() > 0) { @@ -982,7 +994,7 @@ void ImmutableMessageGenerator::GenerateEqualsAndHashCode( printer->Print( "@java.lang.Override\n" "public boolean equals("); - printer->Print("final java.lang.Object obj) {\n"); + printer->Print("java.lang.Object obj) {\n"); printer->Indent(); printer->Print( "if (obj == this) {\n" @@ -997,6 +1009,10 @@ void ImmutableMessageGenerator::GenerateEqualsAndHashCode( "\n", "classname", name_resolver_->GetImmutableClassName(descriptor_)); + int fields_in_function = 0; + int method_num = 1; + std::map variables; + variables["classname"] = name_resolver_->GetImmutableClassName(descriptor_); for (int i = 0; i < descriptor_->field_count(); i++) { const FieldDescriptor* field = descriptor_->field(i); if (!IsRealOneof(field)) { @@ -1014,6 +1030,11 @@ void ImmutableMessageGenerator::GenerateEqualsAndHashCode( printer->Outdent(); printer->Print("}\n"); } + MaybeSplitJavaMethod(printer, &fields_in_function, &method_num, + "return _equals_autosplit_$method_num$(other);\n", + "private boolean _equals_autosplit_$method_num$($classname$ other) {\n", + variables); + fields_in_function++; } } @@ -1078,6 +1099,9 @@ void ImmutableMessageGenerator::GenerateEqualsAndHashCode( printer->Print("hash = (19 * hash) + getDescriptor().hashCode();\n"); } + int fields_in_function_hashcode = 0; + int method_num_hashcode = 1; + std::map variables_hashcode; // hashCode non-oneofs. for (int i = 0; i < descriptor_->field_count(); i++) { const FieldDescriptor* field = descriptor_->field(i); @@ -1094,6 +1118,12 @@ void ImmutableMessageGenerator::GenerateEqualsAndHashCode( printer->Print("}\n"); } } + variables_hashcode["method_num"] = StrCat(method_num_hashcode); + MaybeSplitJavaMethod(printer, &fields_in_function_hashcode, &method_num_hashcode, + "return _hashCode_autosplit_$method_num$(hash);\n", + "private int _hashCode_autosplit_$method_num$(int hash) {\n", + variables_hashcode); + fields_in_function_hashcode++; } // hashCode oneofs. @@ -1149,9 +1179,6 @@ void ImmutableMessageGenerator::GenerateExtensionRegistrationCode( // =================================================================== void ImmutableMessageGenerator::GenerateParsingConstructor( io::Printer* printer) { - std::unique_ptr sorted_fields( - SortFieldsByNumber(descriptor_)); - printer->Print( "private $classname$(\n" " com.google.protobuf.CodedInputStream input,\n" @@ -1191,67 +1218,86 @@ void ImmutableMessageGenerator::GenerateParsingConstructor( "boolean done = false;\n" "while (!done) {\n"); printer->Indent(); + printer->Print("int tag = input.readTag();\n"); - printer->Print( - "int tag = input.readTag();\n" - "switch (tag) {\n"); - printer->Indent(); - - printer->Print( - "case 0:\n" // zero signals EOF / limit reached - " done = true;\n" - " break;\n"); + if(descriptor_->field_count() <= kMaxFieldsInMethod) { //small proto render all in one method + std::unique_ptr sorted_fields( + SortFieldsByNumber(descriptor_)); - for (int i = 0; i < descriptor_->field_count(); i++) { - const FieldDescriptor* field = sorted_fields[i]; - uint32_t tag = WireFormatLite::MakeTag( - field->number(), WireFormat::WireTypeForFieldType(field->type())); - - printer->Print("case $tag$: {\n", "tag", - StrCat(static_cast(tag))); + printer->Print("switch (tag) {\n"); printer->Indent(); - field_generators_.get(field).GenerateParsingCode(printer); - - printer->Outdent(); printer->Print( - " break;\n" - "}\n"); + "case 0:\n" // zero signals EOF / limit reached + " done = true;\n" + " break;\n"); + + for (int i = 0; i < descriptor_->field_count(); i++) { + const FieldDescriptor* field = sorted_fields[i]; + uint32_t tag = WireFormatLite::MakeTag( + field->number(), WireFormat::WireTypeForFieldType(field->type())); - if (field->is_packable()) { - // To make packed = true wire compatible, we generate parsing code from a - // packed version of this field regardless of field->options().packed(). - uint32_t packed_tag = WireFormatLite::MakeTag( - field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED); printer->Print("case $tag$: {\n", "tag", - StrCat(static_cast(packed_tag))); + StrCat(static_cast(tag))); printer->Indent(); - field_generators_.get(field).GenerateParsingCodeFromPacked(printer); + field_generators_.get(field).GenerateParsingCode(printer); printer->Outdent(); printer->Print( - " break;\n" - "}\n"); + " break;\n" + "}\n"); + + if (field->is_packable()) { + // To make packed = true wire compatible, we generate parsing code from a + // packed version of this field regardless of field->options().packed(). + uint32_t packed_tag = WireFormatLite::MakeTag( + field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED); + printer->Print("case $tag$: {\n", "tag", + StrCat(static_cast(packed_tag))); + printer->Indent(); + + field_generators_.get(field).GenerateParsingCodeFromPacked(printer); + + printer->Outdent(); + printer->Print( + " break;\n" + "}\n"); + } } - } - printer->Print( - "default: {\n" - " if (!parseUnknownField(\n" - " input, unknownFields, extensionRegistry, tag)) {\n" - " done = true;\n" // it's an endgroup tag - " }\n" - " break;\n" - "}\n"); + printer->Print( + "default: {\n" + " if (!parseUnknownField(\n" + " input, unknownFields, extensionRegistry, tag)) {\n" + " done = true;\n" // it's an endgroup tag + " }\n" + " break;\n" + "}\n"); - printer->Outdent(); - printer->Outdent(); - printer->Print( - " }\n" // switch (tag) - "}\n"); // while (!done) + printer->Outdent(); + printer->Outdent(); + printer->Print( + " }\n"); // switch (tag) + } else { // big proto. split methods + //mutable fields processing + for (int i = 0; i < totalBuilderInts; i++) { + printer->Print( + "long parse_mutable_fields_$bit_field_name$result = parse_mutable_fields_$bit_field_num$(input, extensionRegistry, unknownFields, tag, mutable_$bit_field_name$);\n", + "bit_field_name", GetBitFieldName(i), + "bit_field_num", StrCat(i)); + printer->Print("if (parse_mutable_fields_$bit_field_name$result != java.lang.Long.MAX_VALUE) {\n" + " mutable_$bit_field_name$ = (int) parse_mutable_fields_$bit_field_name$result;\n" + " continue;\n" + "}\n", + "bit_field_name", GetBitFieldName(i)); + } + //immutable fields split inside + printer->Print("done = parse_immutable_fields_0(input, extensionRegistry, unknownFields, tag);\n"); + } printer->Outdent(); + printer->Print("}\n"); // while (!done) printer->Print( "} catch (com.google.protobuf.InvalidProtocolBufferException e) {\n" " throw e.setUnfinishedMessage(this);\n" @@ -1266,7 +1312,7 @@ void ImmutableMessageGenerator::GenerateParsingConstructor( // Make repeated field list immutable. for (int i = 0; i < descriptor_->field_count(); i++) { - const FieldDescriptor* field = sorted_fields[i]; + const FieldDescriptor* field = descriptor_->field(i); field_generators_.get(field).GenerateParsingDoneCode(printer); } @@ -1280,7 +1326,179 @@ void ImmutableMessageGenerator::GenerateParsingConstructor( printer->Outdent(); printer->Print( " }\n" // finally - "}\n"); + "}\n" + "\n"); + + if(descriptor_->field_count() > kMaxFieldsInMethod) { //need to generate separate functions + //mutable functions generator + int current_mutable_byte_num = -1; + int total_mutable_bits = 0; + for (int i = 0; i < descriptor_->field_count(); i++) { + const ImmutableFieldGenerator &field_generator = + field_generators_.get(descriptor_->field(i)); + if (field_generator.GetNumBitsForBuilder() == 0) { + continue; + } + + total_mutable_bits += field_generator.GetNumBitsForBuilder(); + if (((total_mutable_bits + 31) / 32) - 1 > current_mutable_byte_num) { //start other mutable byte. need to generate functions split + + if (current_mutable_byte_num >= 0) { //end split function + printer->Print("}\n"); + printer->Print("return java.lang.Long.MAX_VALUE;\n"); + printer->Outdent(); + printer->Print("}\n"); + } + current_mutable_byte_num = ((total_mutable_bits + 31) / 32) - 1; + + printer->Print("private long parse_mutable_fields_$bit_field_num$(com.google.protobuf.CodedInputStream input,\n" + " com.google.protobuf.ExtensionRegistryLite extensionRegistry,\n" + " com.google.protobuf.UnknownFieldSet.Builder unknownFields,\n" + " int tag,\n" + " int mutable_$bit_field_name$) throws java.io.IOException {\n", + "bit_field_name", GetBitFieldName(current_mutable_byte_num), + "bit_field_num", StrCat(current_mutable_byte_num)); + printer->Indent(); + + printer->Print("switch (tag) {\n"); + printer->Indent(); + } + + const FieldDescriptor *field = descriptor_->field(i); + uint32_t tag = WireFormatLite::MakeTag( + field->number(), WireFormat::WireTypeForFieldType(field->type())); + + printer->Print("case $tag$: {\n", "tag", + StrCat(static_cast(tag))); + printer->Indent(); + field_generators_.get(field).GenerateParsingCode(printer); + + printer->Print( + " return mutable_$bit_field_name$;\n" + "}\n", + "bit_field_name", GetBitFieldName(current_mutable_byte_num)); + + if (field->is_packable()) { + // To make packed = true wire compatible, we generate parsing code from a + // packed version of this field regardless of field->options().packed(). + uint32_t packed_tag = WireFormatLite::MakeTag( + field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED); + printer->Print("case $tag$: {\n", "tag", + StrCat(static_cast(packed_tag))); + + field_generators_.get(field).GenerateParsingCodeFromPacked(printer); + + printer->Print( + " return mutable_$bit_field_name$;\n" + "}\n", + "bit_field_name", GetBitFieldName(current_mutable_byte_num)); + } + printer->Outdent(); + } + + //all mutable code generated. need to close function + if (total_mutable_bits > 0) { + printer->Outdent(); + printer->Print("}\n"); + printer->Print("return java.lang.Long.MAX_VALUE;\n"); + printer->Outdent(); + printer->Print("}\n"); + } + + //immutable functions generate + int total_fields_in_function = 0; + int function_num = 0; + + printer->Print("private boolean parse_immutable_fields_$func_num$(com.google.protobuf.CodedInputStream input,\n" + " com.google.protobuf.ExtensionRegistryLite extensionRegistry,\n" + " com.google.protobuf.UnknownFieldSet.Builder unknownFields,\n" + " int tag) throws java.io.IOException {\n", + "func_num", StrCat(function_num)); + ++function_num; + + printer->Print("switch (tag) {\n"); + printer->Indent(); + + printer->Print( + "case 0:\n" // zero signals EOF / limit reached + " return true;\n"); + + for (int i = 0; i < descriptor_->field_count(); ++i) { + const ImmutableFieldGenerator &field_generator = + field_generators_.get(descriptor_->field(i)); + if (field_generator.GetNumBitsForBuilder() > 0) { + continue; + } + + if (total_fields_in_function > kMaxFieldsInMethod) { //split functions + printer->Outdent(); + printer->Print("}\n"); + printer->Print("return parse_immutable_fields_$func_num$(input, extensionRegistry, unknownFields, tag);\n", + "func_num", StrCat(function_num)); + + printer->Outdent(); + printer->Print("}\n"); + + printer->Print("private boolean parse_immutable_fields_$func_num$(com.google.protobuf.CodedInputStream input,\n" + " com.google.protobuf.ExtensionRegistryLite extensionRegistry,\n" + " com.google.protobuf.UnknownFieldSet.Builder unknownFields,\n" + " int tag) throws java.io.IOException {\n", + "func_num", StrCat(function_num)); + ++function_num; + total_fields_in_function = 0; + printer->Indent(); + + printer->Print("switch (tag) {\n"); + printer->Indent(); + } + + const FieldDescriptor *field = descriptor_->field(i); + uint32_t tag = WireFormatLite::MakeTag( + field->number(), WireFormat::WireTypeForFieldType(field->type())); + + printer->Print("case $tag$: {\n", "tag", + StrCat(static_cast(tag))); + printer->Indent(); + field_generators_.get(field).GenerateParsingCode(printer); + + printer->Print( + " return false;\n" + "}\n", + "bit_field_name", GetBitFieldName(current_mutable_byte_num)); + + if (field->is_packable()) { + // To make packed = true wire compatible, we generate parsing code from a + // packed version of this field regardless of field->options().packed(). + uint32_t packed_tag = WireFormatLite::MakeTag( + field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED); + printer->Print("case $tag$: {\n", "tag", + StrCat(static_cast(packed_tag))); + + field_generators_.get(field).GenerateParsingCodeFromPacked(printer); + + printer->Print( + " return mutable_$bit_field_name$;\n" + "}\n", + "bit_field_name", GetBitFieldName(current_mutable_byte_num)); + } + printer->Outdent(); + ++total_fields_in_function; + } + + printer->Print( + "default: {\n" + " if (!parseUnknownField(\n" + " input, unknownFields, extensionRegistry, tag)) {\n" + " return true;\n" // it's an endgroup tag + " }\n" + " return false;\n" + "}\n"); + + printer->Outdent(); + printer->Print("}\n"); + printer->Outdent(); + printer->Print("}\n"); + } } // =================================================================== @@ -1343,10 +1561,19 @@ void ImmutableMessageGenerator::GenerateParser(io::Printer* printer) { // =================================================================== void ImmutableMessageGenerator::GenerateInitializers(io::Printer* printer) { + int fields_in_function = 0; + int method_num = 1; + std::map variables; for (int i = 0; i < descriptor_->field_count(); i++) { if (!IsRealOneof(descriptor_->field(i))) { field_generators_.get(descriptor_->field(i)) .GenerateInitializationCode(printer); + + MaybeSplitJavaMethod(printer, &fields_in_function, &method_num, + "_initializer_autosplit_$method_num$();\n", + "private void _initializer_autosplit_$method_num$() {\n", + variables); + fields_in_function++; } } } diff --git a/src/google/protobuf/compiler/java/message.h b/src/google/protobuf/compiler/java/message.h index 45da8ff6464f..7124da5381c2 100644 --- a/src/google/protobuf/compiler/java/message.h +++ b/src/google/protobuf/compiler/java/message.h @@ -60,6 +60,8 @@ namespace compiler { namespace java { static const int kMaxStaticSize = 1 << 15; // aka 32k +static const int kMaxFieldsInMethod = 1 << 6; // aka 64 +//static const int kMaxFieldsInMethod = 1 << 3; // aka 4, for testing class MessageGenerator { public: diff --git a/src/google/protobuf/compiler/java/message_builder.cc b/src/google/protobuf/compiler/java/message_builder.cc index 2366c0c8d524..c07735a3b509 100644 --- a/src/google/protobuf/compiler/java/message_builder.cc +++ b/src/google/protobuf/compiler/java/message_builder.cc @@ -34,6 +34,8 @@ #include +#include "message.h" + #include #include #include @@ -333,10 +335,18 @@ void MessageBuilderGenerator::GenerateCommonBuilderMethods( printer->Indent(); + int fields_in_function = 0; + int method_num = 1; + std::map variables; for (int i = 0; i < descriptor_->field_count(); i++) { if (!IsRealOneof(descriptor_->field(i))) { field_generators_.get(descriptor_->field(i)) .GenerateBuilderClearCode(printer); + MaybeSplitJavaMethod(printer, &fields_in_function, &method_num, + "return _clear_autosplit_$method_num$();\n", + "private Builder _clear_autosplit_$method_num$() {\n", + variables); + fields_in_function++; } } @@ -392,40 +402,92 @@ void MessageBuilderGenerator::GenerateCommonBuilderMethods( "classname", name_resolver_->GetImmutableClassName(descriptor_)); printer->Indent(); + printer->Print("buildFields(result);\n" + "onBuilt();\n"); + printer->Outdent(); + + printer->Print( + " return result;\n" + "}\n" + "\n", + "classname", name_resolver_->GetImmutableClassName(descriptor_)); + + printer->Print("private void buildFields($classname$ result) {\n", + "classname", name_resolver_->GetImmutableClassName(descriptor_)); + printer->Indent(); + + int function_num = -1; + int function_start_field = 0; + + std::map builderBitFields; + std::map messageBitFields; int totalBuilderBits = 0; int totalMessageBits = 0; for (int i = 0; i < descriptor_->field_count(); i++) { - const ImmutableFieldGenerator& field = - field_generators_.get(descriptor_->field(i)); + const ImmutableFieldGenerator &field = + field_generators_.get(descriptor_->field(i)); totalBuilderBits += field.GetNumBitsForBuilder(); totalMessageBits += field.GetNumBitsForMessage(); - } - int totalBuilderInts = (totalBuilderBits + 31) / 32; - int totalMessageInts = (totalMessageBits + 31) / 32; - - // Local vars for from and to bit fields to avoid accessing the builder and - // message over and over for these fields. Seems to provide a slight - // perforamance improvement in micro benchmark and this is also what proto1 - // code does. - for (int i = 0; i < totalBuilderInts; i++) { - printer->Print("int from_$bit_field_name$ = $bit_field_name$;\n", - "bit_field_name", GetBitFieldName(i)); - } - for (int i = 0; i < totalMessageInts; i++) { - printer->Print("int to_$bit_field_name$ = 0;\n", "bit_field_name", - GetBitFieldName(i)); + + builderBitFields[i] = (totalBuilderBits + 31) / 32 - 1; + messageBitFields[i] = (totalMessageBits + 31) / 32 - 1; } - // Output generation code for each field. for (int i = 0; i < descriptor_->field_count(); i++) { + if(i / kMaxFieldsInMethod > function_num) { + function_num = i / kMaxFieldsInMethod; + + if(function_num > 0) { + std::set writtenOut; + for (int j = function_start_field; j < std::min(function_start_field + kMaxFieldsInMethod, descriptor_->field_count()); j++) { + int messageBitToWrite = std::max(messageBitFields[j], 0); + if (totalMessageBits > 0 && writtenOut.find(messageBitToWrite) == writtenOut.end()) { + printer->Print("result.$bit_field_name$ = to_$bit_field_name$;\n", + "bit_field_name", GetBitFieldName(messageBitToWrite)); + writtenOut.insert(messageBitToWrite); + } + } + printer->Print("buildFields_$num$(result);\n", + "num", StrCat(function_num)); + printer->Outdent(); + printer->Print("}\n"); + printer->Print("private void buildFields_$num$($classname$ result) {\n", + "classname", name_resolver_->GetImmutableClassName(descriptor_), + "num", StrCat(function_num)); + printer->Indent(); + } + function_start_field = i; + + std::set writtenBuilder; + std::set writtenMessage; + for (int j = function_start_field; j < std::min(function_start_field + kMaxFieldsInMethod, descriptor_->field_count()); j++) { + int builderBitToWrite = std::max(builderBitFields[j], 0); + if(totalBuilderBits> 0 && writtenBuilder.find(builderBitToWrite) == writtenBuilder.end()) { + printer->Print("int from_$bit_field_name$ = $bit_field_name$;\n", + "bit_field_name", GetBitFieldName(builderBitToWrite)); + writtenBuilder.insert(builderBitToWrite); + } + + int messageBitToWrite = std::max(messageBitFields[j], 0); + if(totalMessageBits> 0 && writtenMessage.find(messageBitToWrite) == writtenMessage.end()) { + printer->Print("int to_$bit_field_name$ = result.$bit_field_name$;\n", "bit_field_name", + GetBitFieldName(messageBitToWrite)); + writtenMessage.insert(messageBitToWrite); + } + } + } field_generators_.get(descriptor_->field(i)).GenerateBuildingCode(printer); } - // Copy the bit field results to the generated message - for (int i = 0; i < totalMessageInts; i++) { - printer->Print("result.$bit_field_name$ = to_$bit_field_name$;\n", - "bit_field_name", GetBitFieldName(i)); + std::set writtenOut; + for (int j = function_start_field; j < std::min(function_start_field + kMaxFieldsInMethod, descriptor_->field_count()); j++) { + int messageBitToWrite = std::max(messageBitFields[j], 0); + if (totalMessageBits > 0 && writtenOut.find(messageBitToWrite) == writtenOut.end()) { + printer->Print("result.$bit_field_name$ = to_$bit_field_name$;\n", + "bit_field_name", GetBitFieldName(messageBitToWrite)); + writtenOut.insert(messageBitToWrite); + } } for (auto oneof : oneofs_) { @@ -434,14 +496,7 @@ void MessageBuilderGenerator::GenerateCommonBuilderMethods( } printer->Outdent(); - - printer->Print(" onBuilt();\n"); - - printer->Print( - " return result;\n" - "}\n" - "\n", - "classname", name_resolver_->GetImmutableClassName(descriptor_)); + printer->Print("}\n"); // Override methods declared in GeneratedMessage to return the concrete // generated type so callsites won't depend on GeneratedMessage. This @@ -538,10 +593,20 @@ void MessageBuilderGenerator::GenerateCommonBuilderMethods( "classname", name_resolver_->GetImmutableClassName(descriptor_)); printer->Indent(); + int fields_in_function = 0; + int method_num = 1; + std::map variables; + variables["classname"] = name_resolver_->GetImmutableClassName(descriptor_); for (int i = 0; i < descriptor_->field_count(); i++) { if (!IsRealOneof(descriptor_->field(i))) { field_generators_.get(descriptor_->field(i)) .GenerateMergingCode(printer); + + MaybeSplitJavaMethod(printer, &fields_in_function, &method_num, + "return _mergeFrom_autosplit_$method_num$(other);\n", + "private Builder _mergeFrom_autosplit_$method_num$($classname$ other) {\n", + variables); + fields_in_function++; } } diff --git a/src/google/protobuf/compiler/java/message_serialization.h b/src/google/protobuf/compiler/java/message_serialization.h index 6145392f81f0..41453d86f2c5 100644 --- a/src/google/protobuf/compiler/java/message_serialization.h +++ b/src/google/protobuf/compiler/java/message_serialization.h @@ -32,6 +32,7 @@ #define GOOGLE_PROTOBUF_COMPILER_JAVA_MESSAGE_SERIALIZATION_H__ #include +#include #include #include @@ -54,10 +55,10 @@ void GenerateSerializeExtensionRange(io::Printer* printer, // // Templatized to support different field generator implementations. template -void GenerateSerializeFieldsAndExtensions( - io::Printer* printer, - const FieldGeneratorMap& field_generators, - const Descriptor* descriptor, const FieldDescriptor** sorted_fields) { + void + GenerateSerializeFieldsAndExtensions(io::Printer *printer, const FieldGeneratorMap &field_generators, + const Descriptor *descriptor, const FieldDescriptor **sorted_fields, + std::map variables) { std::vector sorted_extensions; sorted_extensions.reserve(descriptor->extension_range_count()); for (int i = 0; i < descriptor->extension_range_count(); ++i) { @@ -66,20 +67,50 @@ void GenerateSerializeFieldsAndExtensions( std::sort(sorted_extensions.begin(), sorted_extensions.end(), ExtensionRangeOrdering()); + int fields_in_function = 0; + int method_num = 1; + + std::size_t range_idx = 0; + // Merge the fields and the extension ranges, both sorted by field number. - for (int i = 0, j = 0; - i < descriptor->field_count() || j < sorted_extensions.size();) { - if (i == descriptor->field_count()) { - GenerateSerializeExtensionRange(printer, sorted_extensions[j++]); - } else if (j == sorted_extensions.size()) { - field_generators.get(sorted_fields[i++]) - .GenerateSerializationCode(printer); - } else if (sorted_fields[i]->number() < sorted_extensions[j]->start) { - field_generators.get(sorted_fields[i++]) - .GenerateSerializationCode(printer); + for (int i = 0; i < descriptor->field_count(); ++i) { + const FieldDescriptor* field = sorted_fields[i]; + + // Collapse all extension ranges up until the next field. This leads to + // shorter and more efficient codegen for messages containing a large + // number of extension ranges without fields in between them. + const Descriptor::ExtensionRange* range = nullptr; + while (range_idx < sorted_extensions.size() && + sorted_extensions[range_idx]->end <= field->number()) { + range = sorted_extensions[range_idx++]; + } + + if (range != nullptr) { + GenerateSerializeExtensionRange(printer, range); + } + field_generators.get(field).GenerateSerializationCode(printer); + } + + // After serializing all fields, serialize any remaining extensions via a + // single writeUntil call. + if (range_idx < sorted_extensions.size()) { + GenerateSerializeExtensionRange(printer, sorted_extensions.back()); + + if (descriptor->extension_range_count() > 0) { + MaybeSplitJavaMethod(printer, &fields_in_function, &method_num, + "_writeTo_autosplit_$method_num$(output, extensionWriter);\n", + "private void _writeTo_autosplit_$method_num$(com.google.protobuf.CodedOutputStream output," + "com.google.protobuf.GeneratedMessage$ver$.ExtendableMessage<$classname$>.ExtensionWriter extensionWriter) \n" + " throws java.io.IOException {\n", + variables); } else { - GenerateSerializeExtensionRange(printer, sorted_extensions[j++]); + MaybeSplitJavaMethod(printer, &fields_in_function, &method_num, + "_writeTo_autosplit_$method_num$(output);\n", + "private void _writeTo_autosplit_$method_num$(com.google.protobuf.CodedOutputStream output) \n" + " throws java.io.IOException {\n", + variables); } + fields_in_function++; } } diff --git a/src/google/protobuf/compiler/java/message_serialization_unittest.cc b/src/google/protobuf/compiler/java/message_serialization_unittest.cc new file mode 100644 index 000000000000..b2e1fcaed8bf --- /dev/null +++ b/src/google/protobuf/compiler/java/message_serialization_unittest.cc @@ -0,0 +1,124 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace google { +namespace protobuf { +namespace compiler { +namespace java { +namespace { + +using ::testing::ElementsAre; + +// Generates Java code for the specified Java proto, returning the compiler's +// exit status. +int CompileJavaProto(std::string proto_file_name) { + JavaGenerator java_generator; + + CommandLineInterface cli; + cli.RegisterGenerator("--java_out", &java_generator, /*help_text=*/""); + + std::string proto_path = StrCat( + "--proto_path=", + TestUtil::GetTestDataPath("third_party/protobuf/compiler/java")); + std::string java_out = StrCat("--java_out=", TestTempDir()); + + const char* argv[] = { + "protoc", + proto_path.c_str(), + java_out.c_str(), + proto_file_name.c_str(), + }; + + // Open-source codebase does not support ABSL_ARRAYSIZE. + return cli.Run(sizeof(argv) / sizeof(*argv), argv); +} + +TEST(MessageSerializationTest, CollapseAdjacentExtensionRanges) { + GOOGLE_CHECK_EQ(CompileJavaProto("message_serialization_unittest.proto"), 0); + + std::string java_source; + GOOGLE_CHECK_OK(File::GetContents( + // Open-source codebase does not support file::JoinPath, so we manually + // concatenate instead. + StrCat(TestTempDir(), + "/TestMessageWithManyExtensionRanges.java"), + &java_source, true)); + + // Open-source codebase does not support constexpr StringPiece. + static constexpr const char kWriteUntilCall[] = "extensionWriter.writeUntil("; + + std::vector range_ends; + + // Open-source codebase does not have Split overload taking a single + // char delimiter. + // + // NOLINTNEXTLINE(abseil-faster-strsplit-delimiter) + for (const auto& line : Split(java_source, "\n")) { + // Extract end position from writeUntil call. (Open-source codebase does not + // support RE2.) + std::size_t write_until_pos = line.find(kWriteUntilCall); + if (write_until_pos == std::string::npos) { + continue; + } + write_until_pos += (sizeof(kWriteUntilCall) - 1); + + std::size_t comma_pos = line.find(',', write_until_pos); + if (comma_pos == std::string::npos) { + continue; + } + + range_ends.push_back( + std::string(line.substr(write_until_pos, comma_pos - write_until_pos))); + } + + EXPECT_THAT(range_ends, ElementsAre("3", "13", "43")); +} + +} // namespace +} // namespace java +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/compiler/java/message_serialization_unittest.proto b/src/google/protobuf/compiler/java/message_serialization_unittest.proto new file mode 100644 index 000000000000..9cfdf42b3253 --- /dev/null +++ b/src/google/protobuf/compiler/java/message_serialization_unittest.proto @@ -0,0 +1,56 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +package protobuf_unittest; + +option java_multiple_files = true; +option java_package = ""; + +// Each batch of extension ranges not separated by a non-extension field should +// be serialized using a single ExtensionWriter#writeUntil call. +message TestMessageWithManyExtensionRanges { + // First extension range: ends at field number 3 (exclusive) + extensions 1 to 2; + + optional int32 foo = 3; + optional int32 bar = 5; + + // Second extension range: ends at field number 13 (exclusive) + extensions 6; + extensions 8; + extensions 10 to 12; + + optional int32 baz = 23; + + // Third extension range: ends at field number 43 (exclusive) + extensions 42; +} diff --git a/src/google/protobuf/compiler/main.cc b/src/google/protobuf/compiler/main.cc index 39609468ab62..15b65915e99e 100644 --- a/src/google/protobuf/compiler/main.cc +++ b/src/google/protobuf/compiler/main.cc @@ -66,6 +66,10 @@ int ProtobufMain(int argc, char* argv[]) { cli.RegisterGenerator("--java_out", "--java_opt", &java_generator, "Generate Java source file."); +#ifdef GOOGLE_PROTOBUF_RUNTIME_INCLUDE_BASE + java_generator.set_opensource_runtime(true); +#endif + // Proto2 Kotlin java::KotlinGenerator kt_generator; cli.RegisterGenerator("--kotlin_out", "--kotlin_opt", &kt_generator, @@ -76,6 +80,11 @@ int ProtobufMain(int argc, char* argv[]) { python::Generator py_generator; cli.RegisterGenerator("--python_out", "--python_opt", &py_generator, "Generate Python source file."); + +#ifdef GOOGLE_PROTOBUF_RUNTIME_INCLUDE_BASE + py_generator.set_opensource_runtime(true); +#endif + // Python pyi python::PyiGenerator pyi_generator; cli.RegisterGenerator("--pyi_out", &pyi_generator, diff --git a/src/google/protobuf/compiler/objectivec/objectivec_field.cc b/src/google/protobuf/compiler/objectivec/objectivec_field.cc index 004ea19fb832..a9e651742929 100644 --- a/src/google/protobuf/compiler/objectivec/objectivec_field.cc +++ b/src/google/protobuf/compiler/objectivec/objectivec_field.cc @@ -388,7 +388,7 @@ void RepeatedFieldGenerator::GeneratePropertyDeclaration( "$comments$" "$array_comment$" "@property(nonatomic, readwrite, strong, null_resettable) $array_property_type$ *$name$$storage_attribute$$deprecated_attribute$;\n" - "/** The number of items in @c $name$ without causing the array to be created. */\n" + "/** The number of items in @c $name$ without causing the container to be created. */\n" "@property(nonatomic, readonly) NSUInteger $name$_Count$deprecated_attribute$;\n"); if (IsInitName(variables_.find("name")->second)) { // If property name starts with init we need to annotate it to get past ARC. diff --git a/src/google/protobuf/compiler/php/php_generator.cc b/src/google/protobuf/compiler/php/php_generator.cc index 7c9a3e3db613..b2387039e888 100644 --- a/src/google/protobuf/compiler/php/php_generator.cc +++ b/src/google/protobuf/compiler/php/php_generator.cc @@ -48,29 +48,29 @@ const std::string kDescriptorMetadataFile = const std::string kDescriptorDirName = "Google/Protobuf/Internal"; const std::string kDescriptorPackageName = "Google\\Protobuf\\Internal"; const char* const kReservedNames[] = { - "abstract", "and", "array", "as", "break", - "callable", "case", "catch", "class", "clone", - "const", "continue", "declare", "default", "die", - "do", "echo", "else", "elseif", "empty", - "enddeclare", "endfor", "endforeach", "endif", "endswitch", - "endwhile", "eval", "exit", "extends", "final", - "finally", "fn", "for", "foreach", "function", - "global", "goto", "if", "implements", "include", - "include_once", "instanceof", "insteadof", "interface", "isset", - "list", "match", "namespace", "new", "or", - "parent", "print", "private", "protected", "public", - "require", "require_once", "return", "self", "static", - "switch", "throw", "trait", "try", "unset", - "use", "var", "while", "xor", "yield", - "int", "float", "bool", "string", "true", - "false", "null", "void", "iterable"}; + "abstract", "and", "array", "as", "break", + "callable", "case", "catch", "class", "clone", + "const", "continue", "declare", "default", "die", + "do", "echo", "else", "elseif", "empty", + "enddeclare", "endfor", "endforeach", "endif", "endswitch", + "endwhile", "eval", "exit", "extends", "final", + "finally", "fn", "for", "foreach", "function", + "global", "goto", "if", "implements", "include", + "include_once", "instanceof", "insteadof", "interface", "isset", + "list", "match", "namespace", "new", "or", + "parent", "print", "private", "protected", "public", + "readonly", "require", "require_once", "return", "self", + "static", "switch", "throw", "trait", "try", + "unset", "use", "var", "while", "xor", + "yield", "int", "float", "bool", "string", + "true", "false", "null", "void", "iterable"}; const char* const kValidConstantNames[] = { "int", "float", "bool", "string", "true", "false", "null", "void", "iterable", "parent", - "self" + "self", "readonly" }; -const int kReservedNamesSize = 79; -const int kValidConstantNamesSize = 11; +const int kReservedNamesSize = 80; +const int kValidConstantNamesSize = 12; const int kFieldSetter = 1; const int kFieldGetter = 2; const int kFieldProperty = 3; @@ -407,6 +407,34 @@ std::string GeneratedClassFileName(const DescriptorType* desc, return result + ".php"; } +template +std::string LegacyGeneratedClassFileName(const DescriptorType* desc, + const Options& options) { + std::string result = LegacyFullClassName(desc, options); + + for (int i = 0; i < result.size(); i++) { + if (result[i] == '\\') { + result[i] = '/'; + } + } + return result + ".php"; +} + +template +std::string LegacyReadOnlyGeneratedClassFileName(std::string php_namespace, + const DescriptorType* desc) { + if (!php_namespace.empty()) { + for (int i = 0; i < php_namespace.size(); i++) { + if (php_namespace[i] == '\\') { + php_namespace[i] = '/'; + } + } + return php_namespace + "/" + desc->name() + ".php"; + } + + return desc->name() + ".php"; +} + std::string GeneratedServiceFileName(const ServiceDescriptor* service, const Options& options) { std::string result = FullClassName(service, options) + "Interface"; @@ -1252,6 +1280,82 @@ void GenerateMetadataFile(const FileDescriptor* file, const Options& options, printer.Print("}\n\n"); } +template +void LegacyGenerateClassFile(const FileDescriptor* file, + const DescriptorType* desc, const Options& options, + GeneratorContext* generator_context) { + std::string filename = LegacyGeneratedClassFileName(desc, options); + std::unique_ptr output( + generator_context->Open(filename)); + io::Printer printer(output.get(), '^'); + + GenerateHead(file, &printer); + + std::string php_namespace = RootPhpNamespace(desc, options); + if (!php_namespace.empty()) { + printer.Print( + "namespace ^name^;\n\n", + "name", php_namespace); + } + std::string newname = FullClassName(desc, options); + printer.Print("if (false) {\n"); + Indent(&printer); + printer.Print("/**\n"); + printer.Print(" * This class is deprecated. Use ^new^ instead.\n", + "new", newname); + printer.Print(" * @deprecated\n"); + printer.Print(" */\n"); + printer.Print("class ^old^ {}\n", + "old", LegacyGeneratedClassName(desc)); + Outdent(&printer); + printer.Print("}\n"); + printer.Print("class_exists(^new^::class);\n", + "new", GeneratedClassNameImpl(desc)); + printer.Print("@trigger_error('^old^ is deprecated and will be removed in " + "the next major release. Use ^fullname^ instead', E_USER_DEPRECATED);\n\n", + "old", LegacyFullClassName(desc, options), + "fullname", newname); +} + +template +void LegacyReadOnlyGenerateClassFile(const FileDescriptor* file, + const DescriptorType* desc, const Options& options, + GeneratorContext* generator_context) { + std::string fullname = FullClassName(desc, options); + std::string php_namespace; + std::string classname; + int lastindex = fullname.find_last_of("\\"); + + if (lastindex != std::string::npos) { + php_namespace = fullname.substr(0, lastindex); + classname = fullname.substr(lastindex + 1); + } else { + php_namespace = ""; + classname = fullname; + } + + std::string filename = LegacyReadOnlyGeneratedClassFileName(php_namespace, desc); + std::unique_ptr output( + generator_context->Open(filename)); + io::Printer printer(output.get(), '^'); + + GenerateHead(file, &printer); + + if (!php_namespace.empty()) { + printer.Print( + "namespace ^name^;\n\n", + "name", php_namespace); + } + + printer.Print("class_exists(^new^::class); // autoload the new class, which " + "will also create an alias to the deprecated class\n", + "new", classname); + printer.Print("@trigger_error(__NAMESPACE__ . '\\^old^ is deprecated and will be removed in " + "the next major release. Use ^fullname^ instead', E_USER_DEPRECATED);\n\n", + "old", desc->name(), + "fullname", classname); +} + void GenerateEnumFile(const FileDescriptor* file, const EnumDescriptor* en, const Options& options, GeneratorContext* generator_context) { @@ -1372,6 +1476,19 @@ void GenerateEnumFile(const FileDescriptor* file, const EnumDescriptor* en, "new", fullname, "old", LegacyFullClassName(en, options)); } + + // Write legacy file for backwards compatibility with "readonly" keywword + std::string lower = en->name(); + std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); + if (lower == "readonly") { + printer.Print( + "// Adding a class alias for backwards compatibility with the \"readonly\" keyword.\n"); + printer.Print( + "class_alias(^new^::class, __NAMESPACE__ . '\\^old^');\n\n", + "new", fullname, + "old", en->name()); + LegacyReadOnlyGenerateClassFile(file, en, options, generator_context); + } } void GenerateMessageFile(const FileDescriptor* file, const Descriptor* message, @@ -1487,6 +1604,19 @@ void GenerateMessageFile(const FileDescriptor* file, const Descriptor* message, "old", LegacyFullClassName(message, options)); } + // Write legacy file for backwards compatibility with "readonly" keywword + std::string lower = message->name(); + std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); + if (lower == "readonly") { + printer.Print( + "// Adding a class alias for backwards compatibility with the \"readonly\" keyword.\n"); + printer.Print( + "class_alias(^new^::class, __NAMESPACE__ . '\\^old^');\n\n", + "new", fullname, + "old", message->name()); + LegacyReadOnlyGenerateClassFile(file, message, options, generator_context); + } + // Nested messages and enums. for (int i = 0; i < message->nested_type_count(); i++) { GenerateMessageFile(file, message->nested_type(i), options, diff --git a/src/google/protobuf/compiler/plugin.pb.h b/src/google/protobuf/compiler/plugin.pb.h index b73c9de2df83..51152e652551 100644 --- a/src/google/protobuf/compiler/plugin.pb.h +++ b/src/google/protobuf/compiler/plugin.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/compiler/plugin.proto b/src/google/protobuf/compiler/plugin.proto index 9242aacc5bd9..0f46a4548590 100644 --- a/src/google/protobuf/compiler/plugin.proto +++ b/src/google/protobuf/compiler/plugin.proto @@ -30,9 +30,6 @@ // Author: kenton@google.com (Kenton Varda) // -// WARNING: The plugin interface is currently EXPERIMENTAL and is subject to -// change. -// // protoc (aka the Protocol Compiler) can be extended via plugins. A plugin is // just a program that reads a CodeGeneratorRequest from stdin and writes a // CodeGeneratorResponse to stdout. diff --git a/src/google/protobuf/compiler/python/generator.cc b/src/google/protobuf/compiler/python/generator.cc index d8d6d74923e7..70b08042bb57 100644 --- a/src/google/protobuf/compiler/python/generator.cc +++ b/src/google/protobuf/compiler/python/generator.cc @@ -217,15 +217,12 @@ bool Generator::Generate(const FileDescriptor* file, GeneratorContext* context, std::string* error) const { // ----------------------------------------------------------------- // parse generator options - bool cpp_generated_lib_linked = false; std::vector > options; ParseGeneratorParameter(parameter, &options); for (int i = 0; i < options.size(); i++) { - if (options[i].first == "cpp_generated_lib_linked") { - cpp_generated_lib_linked = true; - } else if (options[i].first == "pyi_out") { + if (options[i].first == "pyi_out") { python::PyiGenerator pyi_generator; if (!pyi_generator.Generate(file, "", context, error)) { return false; @@ -247,10 +244,6 @@ bool Generator::Generate(const FileDescriptor* file, file_ = file; std::string filename = GetFileName(file, ".py"); - pure_python_workable_ = !cpp_generated_lib_linked; - if (HasPrefixString(file->name(), "google/protobuf/")) { - pure_python_workable_ = true; - } FileDescriptorProto fdp; file_->CopyTo(&fdp); @@ -263,55 +256,49 @@ bool Generator::Generate(const FileDescriptor* file, printer_ = &printer; PrintTopBoilerplate(printer_, file_, GeneratingDescriptorProto()); - if (pure_python_workable_) { - PrintImports(); - } + PrintImports(); PrintFileDescriptor(); - if (pure_python_workable_) { - if (GeneratingDescriptorProto()) { - printer_->Print("if _descriptor._USE_C_DESCRIPTORS == False:\n"); - printer_->Indent(); - // Create enums before message descriptors - PrintAllNestedEnumsInFile(); - PrintMessageDescriptors(); - FixForeignFieldsInDescriptors(); - printer_->Outdent(); - printer_->Print("else:\n"); - printer_->Indent(); - } - // Find the message descriptors first and then use the message - // descriptor to find enums. - printer_->Print( - "_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())\n"); - if (GeneratingDescriptorProto()) { - printer_->Outdent(); - } + if (GeneratingDescriptorProto()) { + printer_->Print("if _descriptor._USE_C_DESCRIPTORS == False:\n"); + printer_->Indent(); + // Create enums before message descriptors + PrintAllNestedEnumsInFile(); + PrintMessageDescriptors(); + FixForeignFieldsInDescriptors(); + printer_->Outdent(); + printer_->Print("else:\n"); + printer_->Indent(); + } + // Find the message descriptors first and then use the message + // descriptor to find enums. + printer_->Print( + "_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())\n"); + if (GeneratingDescriptorProto()) { + printer_->Outdent(); } std::string module_name = ModuleName(file->name()); printer_->Print( "_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, '$module_name$', " "globals())\n", "module_name", module_name); - if (pure_python_workable_) { - printer.Print("if _descriptor._USE_C_DESCRIPTORS == False:\n"); - printer_->Indent(); + printer.Print("if _descriptor._USE_C_DESCRIPTORS == False:\n"); + printer_->Indent(); - // We have to fix up the extensions after the message classes themselves, - // since they need to call static RegisterExtension() methods on these - // classes. - FixForeignFieldsInExtensions(); - // Descriptor options may have custom extensions. These custom options - // can only be successfully parsed after we register corresponding - // extensions. Therefore we parse all options again here to recognize - // custom options that may be unknown when we define the descriptors. - // This does not apply to services because they are not used by extensions. - FixAllDescriptorOptions(); + // We have to fix up the extensions after the message classes themselves, + // since they need to call static RegisterExtension() methods on these + // classes. + FixForeignFieldsInExtensions(); + // Descriptor options may have custom extensions. These custom options + // can only be successfully parsed after we register corresponding + // extensions. Therefore we parse all options again here to recognize + // custom options that may be unknown when we define the descriptors. + // This does not apply to services because they are not used by extensions. + FixAllDescriptorOptions(); - // Set serialized_start and serialized_end. - SetSerializedPbInterval(); + // Set serialized_start and serialized_end. + SetSerializedPbInterval(); - printer_->Outdent(); - } + printer_->Outdent(); if (HasGenericServices(file)) { printer_->Print( "_builder.BuildServices(DESCRIPTOR, '$module_name$', globals())\n", @@ -389,30 +376,26 @@ void Generator::PrintFileDescriptor() const { " create_key=_descriptor._internal_create_key,\n"; printer_->Print(m, file_descriptor_template); printer_->Indent(); - if (pure_python_workable_) { - printer_->Print("serialized_pb=b'$value$'\n", "value", - strings::CHexEscape(file_descriptor_serialized_)); - if (file_->dependency_count() != 0) { - printer_->Print(",\ndependencies=["); - for (int i = 0; i < file_->dependency_count(); ++i) { - std::string module_alias = ModuleAlias(file_->dependency(i)->name()); - printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias", - module_alias); - } - printer_->Print("]"); + printer_->Print("serialized_pb=b'$value$'\n", "value", + strings::CHexEscape(file_descriptor_serialized_)); + if (file_->dependency_count() != 0) { + printer_->Print(",\ndependencies=["); + for (int i = 0; i < file_->dependency_count(); ++i) { + std::string module_alias = ModuleAlias(file_->dependency(i)->name()); + printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias", + module_alias); } - if (file_->public_dependency_count() > 0) { - printer_->Print(",\npublic_dependencies=["); - for (int i = 0; i < file_->public_dependency_count(); ++i) { - std::string module_alias = - ModuleAlias(file_->public_dependency(i)->name()); - printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias", - module_alias); - } - printer_->Print("]"); + printer_->Print("]"); + } + if (file_->public_dependency_count() > 0) { + printer_->Print(",\npublic_dependencies=["); + for (int i = 0; i < file_->public_dependency_count(); ++i) { + std::string module_alias = + ModuleAlias(file_->public_dependency(i)->name()); + printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias", + module_alias); } - } else { - printer_->Print("serialized_pb=''\n"); + printer_->Print("]"); } // TODO(falk): Also print options and fix the message_type, enum_type, @@ -467,11 +450,9 @@ void Generator::PrintEnum(const EnumDescriptor& enum_descriptor) const { printer_->Indent(); printer_->Indent(); - if (pure_python_workable_) { - for (int i = 0; i < enum_descriptor.value_count(); ++i) { - PrintEnumValueDescriptor(*enum_descriptor.value(i)); - printer_->Print(",\n"); - } + for (int i = 0; i < enum_descriptor.value_count(); ++i) { + PrintEnumValueDescriptor(*enum_descriptor.value(i)); + printer_->Print(",\n"); } printer_->Outdent(); @@ -482,10 +463,8 @@ void Generator::PrintEnum(const EnumDescriptor& enum_descriptor) const { EnumDescriptorProto edp; printer_->Outdent(); printer_->Print(")\n"); - if (pure_python_workable_) { - printer_->Print("_sym_db.RegisterEnumDescriptor($name$)\n", "name", - module_level_descriptor_name); - } + printer_->Print("_sym_db.RegisterEnumDescriptor($name$)\n", "name", + module_level_descriptor_name); printer_->Print("\n"); } @@ -535,10 +514,6 @@ void Generator::PrintServiceDescriptor( void Generator::PrintDescriptorKeyAndModuleName( const ServiceDescriptor& descriptor) const { std::string name = ModuleLevelServiceDescriptorName(descriptor); - if (!pure_python_workable_) { - name = "_descriptor.ServiceDescriptor(full_name='" + - descriptor.full_name() + "')"; - } printer_->Print("$descriptor_key$ = $descriptor_name$,\n", "descriptor_key", kDescriptorKey, "descriptor_name", name); std::string module_name = ModuleName(file_->name()); @@ -728,12 +703,7 @@ void Generator::PrintMessage(const Descriptor& message_descriptor, PrintNestedMessages(message_descriptor, qualified_name, to_register); std::map m; m["descriptor_key"] = kDescriptorKey; - if (pure_python_workable_) { - m["descriptor_name"] = ModuleLevelDescriptorName(message_descriptor); - } else { - m["descriptor_name"] = "_descriptor.Descriptor(full_name='" + - message_descriptor.full_name() + "')"; - } + m["descriptor_name"] = ModuleLevelDescriptorName(message_descriptor); printer_->Print(m, "'$descriptor_key$' : $descriptor_name$,\n"); std::string module_name = ModuleName(file_->name()); printer_->Print("'__module__' : '$module_name$'\n", "module_name", diff --git a/src/google/protobuf/compiler/python/generator.h b/src/google/protobuf/compiler/python/generator.h index f1fecbc7358f..46b0ad183698 100644 --- a/src/google/protobuf/compiler/python/generator.h +++ b/src/google/protobuf/compiler/python/generator.h @@ -76,6 +76,10 @@ class PROTOC_EXPORT Generator : public CodeGenerator { uint64_t GetSupportedFeatures() const override; + void set_opensource_runtime(bool opensource) { + opensource_runtime_ = opensource; + } + private: void PrintImports() const; void PrintFileDescriptor() const; @@ -170,7 +174,8 @@ class PROTOC_EXPORT Generator : public CodeGenerator { mutable const FileDescriptor* file_; // Set in Generate(). Under mutex_. mutable std::string file_descriptor_serialized_; mutable io::Printer* printer_; // Set in Generate(). Under mutex_. - mutable bool pure_python_workable_; + + bool opensource_runtime_ = true; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Generator); }; diff --git a/src/google/protobuf/compiler/python/pyi_generator.cc b/src/google/protobuf/compiler/python/pyi_generator.cc index 1ccc9a219153..e954fd317fb2 100644 --- a/src/google/protobuf/compiler/python/pyi_generator.cc +++ b/src/google/protobuf/compiler/python/pyi_generator.cc @@ -44,39 +44,22 @@ namespace protobuf { namespace compiler { namespace python { -template -struct SortByName { - bool operator()(const DescriptorT* l, const DescriptorT* r) const { - return l->name() < r->name(); - } -}; - PyiGenerator::PyiGenerator() : file_(nullptr) {} PyiGenerator::~PyiGenerator() {} -void PyiGenerator::PrintItemMap( - const std::map& item_map) const { - for (const auto& entry : item_map) { - printer_->Print("$key$: $value$\n", "key", entry.first, "value", - entry.second); - } -} - template -std::string PyiGenerator::ModuleLevelName( - const DescriptorT& descriptor, - const std::map& import_map) const { +std::string PyiGenerator::ModuleLevelName(const DescriptorT& descriptor) const { std::string name = NamePrefixedWithNestedTypes(descriptor, "."); if (descriptor.file() != file_) { std::string module_alias; std::string filename = descriptor.file()->name(); - if (import_map.find(filename) == import_map.end()) { + if (import_map_.find(filename) == import_map_.end()) { std::string module_name = ModuleName(descriptor.file()->name()); std::vector tokens = Split(module_name, "."); module_alias = "_" + tokens.back(); } else { - module_alias = import_map.at(filename); + module_alias = import_map_.at(filename); } name = module_alias + "." + name; } @@ -156,7 +139,6 @@ void CheckImportModules(const Descriptor* descriptor, void PyiGenerator::PrintImportForDescriptor( const FileDescriptor& desc, - std::map* import_map, std::set* seen_aliases) const { const std::string& filename = desc.name(); std::string module_name = StrippedModuleName(filename); @@ -176,21 +158,19 @@ void PyiGenerator::PrintImportForDescriptor( } printer_->Print("$statement$ as $alias$\n", "statement", import_statement, "alias", alias); - (*import_map)[filename] = alias; + import_map_[filename] = alias; seen_aliases->insert(alias); } -void PyiGenerator::PrintImports( - std::map* item_map, - std::map* import_map) const { +void PyiGenerator::PrintImports() const { // Prints imported dependent _pb2 files. std::set seen_aliases; for (int i = 0; i < file_->dependency_count(); ++i) { const FileDescriptor* dep = file_->dependency(i); - PrintImportForDescriptor(*dep, import_map, &seen_aliases); + PrintImportForDescriptor(*dep, &seen_aliases); for (int j = 0; j < dep->public_dependency_count(); ++j) { PrintImportForDescriptor( - *dep->public_dependency(j), import_map, &seen_aliases); + *dep->public_dependency(j), &seen_aliases); } } @@ -254,7 +234,7 @@ void PyiGenerator::PrintImports( if (import_modules.has_union) { printer_->Print(", Union as _Union"); } - printer_->Print("\n\n"); + printer_->Print("\n"); // Public imports for (int i = 0; i < file_->public_dependency_count(); ++i) { @@ -272,17 +252,8 @@ void PyiGenerator::PrintImports( module_name, "enum_class", public_dep->enum_type(i)->name()); } - // Enum values for public imports - for (int i = 0; i < public_dep->enum_type_count(); ++i) { - const EnumDescriptor* enum_descriptor = public_dep->enum_type(i); - for (int j = 0; j < enum_descriptor->value_count(); ++j) { - (*item_map)[enum_descriptor->value(j)->name()] = - ModuleLevelName(*enum_descriptor, *import_map); - } - } - // Top level extensions for public imports - AddExtensions(*public_dep, item_map); } +printer_->Print("\n"); } void PyiGenerator::PrintEnum(const EnumDescriptor& enum_descriptor) const { @@ -293,20 +264,18 @@ void PyiGenerator::PrintEnum(const EnumDescriptor& enum_descriptor) const { "enum_name", enum_name); } -// Adds enum value to item map which will be ordered and printed later. -void PyiGenerator::AddEnumValue( - const EnumDescriptor& enum_descriptor, - std::map* item_map, - const std::map& import_map) const { +void PyiGenerator::PrintEnumValues( + const EnumDescriptor& enum_descriptor) const { // enum values - std::string module_enum_name = ModuleLevelName(enum_descriptor, import_map); + std::string module_enum_name = ModuleLevelName(enum_descriptor); for (int j = 0; j < enum_descriptor.value_count(); ++j) { const EnumValueDescriptor* value_descriptor = enum_descriptor.value(j); - (*item_map)[value_descriptor->name()] = module_enum_name; + printer_->Print("$name$: $module_enum_name$\n", + "name", value_descriptor->name(), + "module_enum_name", module_enum_name); } } -// Prints top level enums void PyiGenerator::PrintTopLevelEnums() const { for (int i = 0; i < file_->enum_type_count(); ++i) { printer_->Print("\n"); @@ -314,25 +283,22 @@ void PyiGenerator::PrintTopLevelEnums() const { } } -// Add top level extensions to item_map which will be ordered and -// printed later. template -void PyiGenerator::AddExtensions( - const DescriptorT& descriptor, - std::map* item_map) const { +void PyiGenerator::PrintExtensions(const DescriptorT& descriptor) const { for (int i = 0; i < descriptor.extension_count(); ++i) { const FieldDescriptor* extension_field = descriptor.extension(i); std::string constant_name = extension_field->name() + "_FIELD_NUMBER"; ToUpper(&constant_name); - (*item_map)[constant_name] = "_ClassVar[int]"; - (*item_map)[extension_field->name()] = "_descriptor.FieldDescriptor"; + printer_->Print("$constant_name$: _ClassVar[int]\n", + "constant_name", constant_name); + printer_->Print("$name$: _descriptor.FieldDescriptor\n", + "name", extension_field->name()); } } // Returns the string format of a field's cpp_type std::string PyiGenerator::GetFieldType( - const FieldDescriptor& field_des, const Descriptor& containing_des, - const std::map& import_map) const { + const FieldDescriptor& field_des, const Descriptor& containing_des) const { switch (field_des.cpp_type()) { case FieldDescriptor::CPPTYPE_INT32: case FieldDescriptor::CPPTYPE_UINT32: @@ -345,7 +311,7 @@ std::string PyiGenerator::GetFieldType( case FieldDescriptor::CPPTYPE_BOOL: return "bool"; case FieldDescriptor::CPPTYPE_ENUM: - return ModuleLevelName(*field_des.enum_type(), import_map); + return ModuleLevelName(*field_des.enum_type()); case FieldDescriptor::CPPTYPE_STRING: if (field_des.type() == FieldDescriptor::TYPE_STRING) { return "str"; @@ -356,7 +322,7 @@ std::string PyiGenerator::GetFieldType( // If the field is inside a nested message and the nested message has the // same name as a top-level message, then we need to prefix the field type // with the module name for disambiguation. - std::string name = ModuleLevelName(*field_des.message_type(), import_map); + std::string name = ModuleLevelName(*field_des.message_type()); if ((containing_des.containing_type() != nullptr && name == containing_des.name())) { std::string module = ModuleName(field_des.file()->name()); @@ -371,8 +337,7 @@ std::string PyiGenerator::GetFieldType( } void PyiGenerator::PrintMessage( - const Descriptor& message_descriptor, bool is_nested, - const std::map& import_map) const { + const Descriptor& message_descriptor, bool is_nested) const { if (!is_nested) { printer_->Print("\n"); } @@ -390,17 +355,11 @@ void PyiGenerator::PrintMessage( printer_->Indent(); printer_->Indent(); - std::vector fields; - fields.reserve(message_descriptor.field_count()); - for (int i = 0; i < message_descriptor.field_count(); ++i) { - fields.push_back(message_descriptor.field(i)); - } - std::sort(fields.begin(), fields.end(), SortByName()); - // Prints slots printer_->Print("__slots__ = [", "class_name", class_name); bool first_item = true; - for (const auto& field_des : fields) { + for (int i = 0; i < message_descriptor.field_count(); ++i) { + const FieldDescriptor* field_des = message_descriptor.field(i); if (IsPythonKeyword(field_des->name())) { continue; } @@ -413,48 +372,34 @@ void PyiGenerator::PrintMessage( } printer_->Print("]\n"); - std::map item_map; // Prints Extensions for extendable messages if (message_descriptor.extension_range_count() > 0) { - item_map["Extensions"] = "_python_message._ExtensionDict"; + printer_->Print("Extensions: _python_message._ExtensionDict\n"); } // Prints nested enums - std::vector nested_enums; - nested_enums.reserve(message_descriptor.enum_type_count()); for (int i = 0; i < message_descriptor.enum_type_count(); ++i) { - nested_enums.push_back(message_descriptor.enum_type(i)); - } - std::sort(nested_enums.begin(), nested_enums.end(), - SortByName()); - - for (const auto& entry : nested_enums) { - PrintEnum(*entry); - // Adds enum value to item_map which will be ordered and printed later - AddEnumValue(*entry, &item_map, import_map); + PrintEnum(*message_descriptor.enum_type(i)); + PrintEnumValues(*message_descriptor.enum_type(i)); } // Prints nested messages - std::vector nested_messages; - nested_messages.reserve(message_descriptor.nested_type_count()); for (int i = 0; i < message_descriptor.nested_type_count(); ++i) { - nested_messages.push_back(message_descriptor.nested_type(i)); - } - std::sort(nested_messages.begin(), nested_messages.end(), - SortByName()); - - for (const auto& entry : nested_messages) { - PrintMessage(*entry, true, import_map); + PrintMessage(*message_descriptor.nested_type(i), true); } - // Adds extensions to item_map which will be ordered and printed later - AddExtensions(message_descriptor, &item_map); + PrintExtensions(message_descriptor); - // Adds field number and field descriptor to item_map + // Prints field number + for (int i = 0; i < message_descriptor.field_count(); ++i) { + const FieldDescriptor& field_des = *message_descriptor.field(i); + printer_->Print( + "$field_number_name$: _ClassVar[int]\n", "field_number_name", + ToUpper(field_des.name()) + "_FIELD_NUMBER"); + } + // Prints field name and type for (int i = 0; i < message_descriptor.field_count(); ++i) { const FieldDescriptor& field_des = *message_descriptor.field(i); - item_map[ToUpper(field_des.name()) + "_FIELD_NUMBER"] = - "_ClassVar[int]"; if (IsPythonKeyword(field_des.name())) { continue; } @@ -465,27 +410,25 @@ void PyiGenerator::PrintMessage( field_type = (value_des->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE ? "_containers.MessageMap[" : "_containers.ScalarMap["); - field_type += GetFieldType(*key_des, message_descriptor, import_map); + field_type += GetFieldType(*key_des, message_descriptor); field_type += ", "; - field_type += GetFieldType(*value_des, message_descriptor, import_map); + field_type += GetFieldType(*value_des, message_descriptor); } else { if (field_des.is_repeated()) { field_type = (field_des.cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE ? "_containers.RepeatedCompositeFieldContainer[" : "_containers.RepeatedScalarFieldContainer["); } - field_type += GetFieldType(field_des, message_descriptor, import_map); + field_type += GetFieldType(field_des, message_descriptor); } if (field_des.is_repeated()) { field_type += "]"; } - item_map[field_des.name()] = field_type; + printer_->Print("$name$: $type$\n", + "name", field_des.name(), "type", field_type); } - // Prints all items in item_map - PrintItemMap(item_map); - // Prints __init__ printer_->Print("def __init__(self"); bool has_key_words = false; @@ -513,9 +456,9 @@ void PyiGenerator::PrintMessage( const Descriptor* map_entry = field_des->message_type(); printer_->Print( "_Mapping[$key_type$, $value_type$]", "key_type", - GetFieldType(*map_entry->field(0), message_descriptor, import_map), + GetFieldType(*map_entry->field(0), message_descriptor), "value_type", - GetFieldType(*map_entry->field(1), message_descriptor, import_map)); + GetFieldType(*map_entry->field(1), message_descriptor)); } else { if (field_des->is_repeated()) { printer_->Print("_Iterable["); @@ -523,15 +466,15 @@ void PyiGenerator::PrintMessage( if (field_des->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { printer_->Print( "_Union[$type_name$, _Mapping]", "type_name", - GetFieldType(*field_des, message_descriptor, import_map)); + GetFieldType(*field_des, message_descriptor)); } else { if (field_des->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) { printer_->Print("_Union[$type_name$, str]", "type_name", - ModuleLevelName(*field_des->enum_type(), import_map)); + ModuleLevelName(*field_des->enum_type())); } else { printer_->Print( "$type_name$", "type_name", - GetFieldType(*field_des, message_descriptor, import_map)); + GetFieldType(*field_des, message_descriptor)); } } if (field_des->is_repeated()) { @@ -553,36 +496,21 @@ void PyiGenerator::PrintMessage( printer_->Outdent(); } -void PyiGenerator::PrintMessages( - const std::map& import_map) const { +void PyiGenerator::PrintMessages() const { // Deterministically order the descriptors. - std::vector messages; - messages.reserve(file_->message_type_count()); for (int i = 0; i < file_->message_type_count(); ++i) { - messages.push_back(file_->message_type(i)); - } - std::sort(messages.begin(), messages.end(), SortByName()); - - for (const auto& entry : messages) { - PrintMessage(*entry, false, import_map); + PrintMessage(*file_->message_type(i), false); } } void PyiGenerator::PrintServices() const { - std::vector services; - services.reserve(file_->service_count()); - for (int i = 0; i < file_->service_count(); ++i) { - services.push_back(file_->service(i)); - } - std::sort(services.begin(), services.end(), SortByName()); - // Prints $Service$ and $Service$_Stub classes - for (const auto& entry : services) { + for (int i = 0; i < file_->service_count(); ++i) { printer_->Print("\n"); printer_->Print( "class $service_name$(_service.service): ...\n\n" "class $service_name$_Stub($service_name$): ...\n", - "service_name", entry->name()); + "service_name", file_->service(i)->name()); } } @@ -591,6 +519,7 @@ bool PyiGenerator::Generate(const FileDescriptor* file, GeneratorContext* context, std::string* error) const { MutexLock lock(&mutex_); + import_map_.clear(); // Calculate file name. file_ = file; std::string filename = @@ -601,29 +530,28 @@ bool PyiGenerator::Generate(const FileDescriptor* file, io::Printer printer(output.get(), '$'); printer_ = &printer; - // item map will store "DESCRIPTOR", top level extensions, top level enum - // values. The items will be sorted and printed later. - std::map item_map; - - // Adds "DESCRIPTOR" into item_map. - item_map["DESCRIPTOR"] = "_descriptor.FileDescriptor"; + PrintImports(); + printer_->Print("DESCRIPTOR: _descriptor.FileDescriptor\n"); - // import_map will be a mapping from filename to module alias, e.g. - // "google3/foo/bar.py" -> "_bar" - std::map import_map; + // Prints extensions and enums from imports. + for (int i = 0; i < file_->public_dependency_count(); ++i) { + const FileDescriptor* public_dep = file_->public_dependency(i); + PrintExtensions(*public_dep); + for (int i = 0; i < public_dep->enum_type_count(); ++i) { + const EnumDescriptor* enum_descriptor = public_dep->enum_type(i); + PrintEnumValues(*enum_descriptor); + } + } - PrintImports(&item_map, &import_map); - // Adds top level enum values to item_map. + PrintTopLevelEnums(); + // Prints top level enum values for (int i = 0; i < file_->enum_type_count(); ++i) { - AddEnumValue(*file_->enum_type(i), &item_map, import_map); + PrintEnumValues(*file_->enum_type(i)); } - // Adds top level extensions to item_map. - AddExtensions(*file_, &item_map); - // Prints item map - PrintItemMap(item_map); + // Prints top level Extensions + PrintExtensions(*file_); + PrintMessages(); - PrintMessages(import_map); - PrintTopLevelEnums(); if (HasGenericServices(file)) { PrintServices(); } diff --git a/src/google/protobuf/compiler/python/pyi_generator.h b/src/google/protobuf/compiler/python/pyi_generator.h index 9611ed43d135..40741e1e78f0 100644 --- a/src/google/protobuf/compiler/python/pyi_generator.h +++ b/src/google/protobuf/compiler/python/pyi_generator.h @@ -76,37 +76,29 @@ class PROTOC_EXPORT PyiGenerator : public google::protobuf::compiler::CodeGenera private: void PrintImportForDescriptor(const FileDescriptor& desc, - std::map* import_map, std::set* seen_aliases) const; - void PrintImports(std::map* item_map, - std::map* import_map) const; - void PrintEnum(const EnumDescriptor& enum_descriptor) const; - void AddEnumValue(const EnumDescriptor& enum_descriptor, - std::map* item_map, - const std::map& import_map) const; + void PrintImports() const; void PrintTopLevelEnums() const; + void PrintEnum(const EnumDescriptor& enum_descriptor) const; + void PrintEnumValues(const EnumDescriptor& enum_descriptor) const; template - void AddExtensions(const DescriptorT& descriptor, - std::map* item_map) const; - void PrintMessages( - const std::map& import_map) const; - void PrintMessage(const Descriptor& message_descriptor, bool is_nested, - const std::map& import_map) const; + void PrintExtensions(const DescriptorT& descriptor) const; + void PrintMessages() const; + void PrintMessage(const Descriptor& message_descriptor, bool is_nested) const; void PrintServices() const; - void PrintItemMap(const std::map& item_map) const; std::string GetFieldType( - const FieldDescriptor& field_des, const Descriptor& containing_des, - const std::map& import_map) const; + const FieldDescriptor& field_des, const Descriptor& containing_des) const; template - std::string ModuleLevelName( - const DescriptorT& descriptor, - const std::map& import_map) const; + std::string ModuleLevelName(const DescriptorT& descriptor) const; // Very coarse-grained lock to ensure that Generate() is reentrant. - // Guards file_ and printer_. + // Guards file_, printer_, and import_map_. mutable Mutex mutex_; mutable const FileDescriptor* file_; // Set in Generate(). Under mutex_. mutable io::Printer* printer_; // Set in Generate(). Under mutex_. + // import_map will be a mapping from filename to module alias, e.g. + // "google3/foo/bar.py" -> "_bar" + mutable std::map import_map_; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(PyiGenerator); }; diff --git a/src/google/protobuf/compiler/subprocess.cc b/src/google/protobuf/compiler/subprocess.cc index 6f547db79550..6faab0513764 100644 --- a/src/google/protobuf/compiler/subprocess.cc +++ b/src/google/protobuf/compiler/subprocess.cc @@ -46,8 +46,8 @@ #include #include #include -#include #include +#include namespace google { namespace protobuf { diff --git a/src/google/protobuf/descriptor.h b/src/google/protobuf/descriptor.h index 685e9235d804..beb3e641c655 100644 --- a/src/google/protobuf/descriptor.h +++ b/src/google/protobuf/descriptor.h @@ -1577,7 +1577,11 @@ class PROTOBUF_EXPORT FileDescriptor : private internal::SymbolBase { const FileOptions& options() const; // Syntax of this file. - enum Syntax { + enum Syntax +#ifndef SWIG + : int +#endif // !SWIG + { SYNTAX_UNKNOWN = 0, SYNTAX_PROTO2 = 2, SYNTAX_PROTO3 = 3, diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index f6f5da532503..439db9f0002e 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/duration.pb.h b/src/google/protobuf/duration.pb.h index 87729c4893b9..1e4a3e18f803 100644 --- a/src/google/protobuf/duration.pb.h +++ b/src/google/protobuf/duration.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/empty.pb.h b/src/google/protobuf/empty.pb.h index fdc398c3d900..c5f528b9e830 100644 --- a/src/google/protobuf/empty.pb.h +++ b/src/google/protobuf/empty.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/field_mask.pb.h b/src/google/protobuf/field_mask.pb.h index fd75b88acec5..01ecfacb1a7e 100644 --- a/src/google/protobuf/field_mask.pb.h +++ b/src/google/protobuf/field_mask.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index 0353b74fffb1..70014b2abb9c 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -2514,6 +2514,7 @@ const Type& Reflection::GetRawNonOneof(const Message& message, } void Reflection::PrepareSplitMessageForWrite(Message* message) const { + GOOGLE_DCHECK_NE(message, schema_.default_instance_); void** split = MutableSplitField(message); const void* default_split = GetSplitField(schema_.default_instance_); if (*split == default_split) { diff --git a/src/google/protobuf/generated_message_tctable_impl.h b/src/google/protobuf/generated_message_tctable_impl.h index 67349b47ff27..3fad18c7f7a0 100644 --- a/src/google/protobuf/generated_message_tctable_impl.h +++ b/src/google/protobuf/generated_message_tctable_impl.h @@ -38,7 +38,6 @@ #include #include #include -#include #include #include #include @@ -257,16 +256,6 @@ enum FieldType : uint16_t { // clang-format on } // namespace field_layout -// PROTOBUF_TC_PARAM_DECL are the parameters for tailcall functions, it is -// defined in port_def.inc. -// -// Note that this is performance sensitive: changing the parameters will change -// the registers used by the ABI calling convention, which subsequently affects -// register selection logic inside the function. - -// PROTOBUF_TC_PARAM_PASS passes values to match PROTOBUF_TC_PARAM_DECL. -#define PROTOBUF_TC_PARAM_PASS msg, ptr, ctx, table, hasbits, data - #ifndef NDEBUG template void AlignFail(uintptr_t address) { @@ -349,6 +338,28 @@ class PROTOBUF_EXPORT TcParser final { static const char* FastZ64P1(PROTOBUF_TC_PARAM_DECL); static const char* FastZ64P2(PROTOBUF_TC_PARAM_DECL); + // Manually unrolled and specialized Varint parsing. + template + static const char* SpecializedUnrolledVImpl1(PROTOBUF_TC_PARAM_DECL); + + template + static constexpr TailCallParseFunc SingularVarintNoZag1() { + if (data_offset < 100) { + return &SpecializedUnrolledVImpl1; + } else if (sizeof(FieldType) == 1) { + return &FastV8S1; + } else if (sizeof(FieldType) == 4) { + return &FastV32S1; + } else if (sizeof(FieldType) == 8) { + return &FastV64S1; + } else { + static_assert(sizeof(FieldType) == 1 || sizeof(FieldType) == 4 || + sizeof(FieldType) == 8, + ""); + return nullptr; + } + } + // Functions referenced by generated fast tables (closed enum): // E: closed enum (N.B.: open enums use V32, above) // r: enum range v: enum validator (_IsValid function) @@ -600,6 +611,135 @@ class PROTOBUF_EXPORT TcParser final { static const char* MpMap(PROTOBUF_TC_PARAM_DECL); }; +template +const char* TcParser::SpecializedUnrolledVImpl1(PROTOBUF_TC_PARAM_DECL) { + using TagType = uint8_t; + // super-early success test... + if (PROTOBUF_PREDICT_TRUE(((data.data) & 0x80FF) == 0)) { + ptr += sizeof(TagType); // Consume tag + if (hasbit_idx < 32) { + hasbits |= (uint64_t{1} << hasbit_idx); + } + uint8_t value = data.data >> 8; + RefAt(msg, data_offset) = value; + ptr += 1; + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + } + if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); + } + ptr += sizeof(TagType); // Consume tag + if (hasbit_idx < 32) { + hasbits |= (uint64_t{1} << hasbit_idx); + } + + // Few registers + auto* out = &RefAt(msg, data_offset); + uint64_t res = 0xFF & (data.data >> 8); + /* if (PROTOBUF_PREDICT_FALSE(res & 0x80)) */ { + res = RotRight7AndReplaceLowByte(res, ptr[1]); + if (PROTOBUF_PREDICT_FALSE(res & 0x80)) { + res = RotRight7AndReplaceLowByte(res, ptr[2]); + if (PROTOBUF_PREDICT_FALSE(res & 0x80)) { + res = RotRight7AndReplaceLowByte(res, ptr[3]); + if (PROTOBUF_PREDICT_FALSE(res & 0x80)) { + res = RotRight7AndReplaceLowByte(res, ptr[4]); + if (PROTOBUF_PREDICT_FALSE(res & 0x80)) { + res = RotRight7AndReplaceLowByte(res, ptr[5]); + if (PROTOBUF_PREDICT_FALSE(res & 0x80)) { + res = RotRight7AndReplaceLowByte(res, ptr[6]); + if (PROTOBUF_PREDICT_FALSE(res & 0x80)) { + res = RotRight7AndReplaceLowByte(res, ptr[7]); + if (PROTOBUF_PREDICT_FALSE(res & 0x80)) { + res = RotRight7AndReplaceLowByte(res, ptr[8]); + if (PROTOBUF_PREDICT_FALSE(res & 0x80)) { + if (ptr[9] & 0xFE) return nullptr; + res = RotateLeft(res, -7) & ~1; + res += ptr[9] & 1; + *out = RotateLeft(res, 63); + ptr += 10; + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + } + *out = RotateLeft(res, 56); + ptr += 9; + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + } + *out = RotateLeft(res, 49); + ptr += 8; + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + } + *out = RotateLeft(res, 42); + ptr += 7; + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + } + *out = RotateLeft(res, 35); + ptr += 6; + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + } + *out = RotateLeft(res, 28); + ptr += 5; + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + } + *out = RotateLeft(res, 21); + ptr += 4; + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + } + *out = RotateLeft(res, 14); + ptr += 3; + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + } + *out = RotateLeft(res, 7); + ptr += 2; + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + } + *out = res; + ptr += 1; + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); +} + +// Dispatch to the designated parse function +inline PROTOBUF_ALWAYS_INLINE const char* TcParser::TagDispatch( + PROTOBUF_TC_PARAM_DECL) { + const auto coded_tag = UnalignedLoad(ptr); + const size_t idx = coded_tag & table->fast_idx_mask; + PROTOBUF_ASSUME((idx & 7) == 0); + auto* fast_entry = table->fast_entry(idx >> 3); + data = fast_entry->bits; + data.data ^= coded_tag; + PROTOBUF_MUSTTAIL return fast_entry->target(PROTOBUF_TC_PARAM_PASS); +} + +// We can only safely call from field to next field if the call is optimized +// to a proper tail call. Otherwise we blow through stack. Clang and gcc +// reliably do this optimization in opt mode, but do not perform this in debug +// mode. Luckily the structure of the algorithm is such that it's always +// possible to just return and use the enclosing parse loop as a trampoline. +inline PROTOBUF_ALWAYS_INLINE const char* TcParser::ToTagDispatch( + PROTOBUF_TC_PARAM_DECL) { + constexpr bool always_return = !PROTOBUF_TAILCALL; + if (always_return || !ctx->DataAvailable(ptr)) { + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_PASS); + } + PROTOBUF_MUSTTAIL return TagDispatch(PROTOBUF_TC_PARAM_PASS); +} + +inline PROTOBUF_ALWAYS_INLINE const char* TcParser::ToParseLoop( + PROTOBUF_TC_PARAM_DECL) { + (void)data; + (void)ctx; + SyncHasbits(msg, hasbits, table); + return ptr; +} + +inline PROTOBUF_ALWAYS_INLINE const char* TcParser::Error( + PROTOBUF_TC_PARAM_DECL) { + (void)data; + (void)ctx; + (void)ptr; + SyncHasbits(msg, hasbits, table); + return nullptr; +} + } // namespace internal } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/generated_message_tctable_lite.cc b/src/google/protobuf/generated_message_tctable_lite.cc index aef780f54118..f9fb6cc4b915 100644 --- a/src/google/protobuf/generated_message_tctable_lite.cc +++ b/src/google/protobuf/generated_message_tctable_lite.cc @@ -30,6 +30,7 @@ #include #include +#include #include #include @@ -84,56 +85,13 @@ PROTOBUF_NOINLINE const char* TcParser::ParseLoop( // TODO(b/64614992): remove this asm asm("" : "+r"(table)); #endif - ptr = TagDispatch(msg, ptr, ctx, table - 1, 0, {}); + ptr = TagDispatch(msg, ptr, ctx, {}, table - 1, 0); if (ptr == nullptr) break; if (ctx->LastTag() != 1) break; // Ended on terminating tag } return ptr; } - // Dispatch to the designated parse function -inline PROTOBUF_ALWAYS_INLINE const char* TcParser::TagDispatch( - PROTOBUF_TC_PARAM_DECL) { - const auto coded_tag = UnalignedLoad(ptr); - const size_t idx = coded_tag & table->fast_idx_mask; - PROTOBUF_ASSUME((idx & 7) == 0); - auto* fast_entry = table->fast_entry(idx >> 3); - data = fast_entry->bits; - data.data ^= coded_tag; - PROTOBUF_MUSTTAIL return fast_entry->target(PROTOBUF_TC_PARAM_PASS); -} - -// We can only safely call from field to next field if the call is optimized -// to a proper tail call. Otherwise we blow through stack. Clang and gcc -// reliably do this optimization in opt mode, but do not perform this in debug -// mode. Luckily the structure of the algorithm is such that it's always -// possible to just return and use the enclosing parse loop as a trampoline. -inline PROTOBUF_ALWAYS_INLINE const char* TcParser::ToTagDispatch( - PROTOBUF_TC_PARAM_DECL) { - constexpr bool always_return = !PROTOBUF_TAILCALL; - if (always_return || !ctx->DataAvailable(ptr)) { - PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_PASS); - } - PROTOBUF_MUSTTAIL return TagDispatch(PROTOBUF_TC_PARAM_PASS); -} - -inline PROTOBUF_ALWAYS_INLINE const char* TcParser::ToParseLoop( - PROTOBUF_TC_PARAM_DECL) { - (void)data; - (void)ctx; - SyncHasbits(msg, hasbits, table); - return ptr; -} - -inline PROTOBUF_ALWAYS_INLINE const char* TcParser::Error( - PROTOBUF_TC_PARAM_DECL) { - (void)data; - (void)ctx; - (void)ptr; - SyncHasbits(msg, hasbits, table); - return nullptr; -} - // On the fast path, a (matching) 1-byte tag already has the decoded value. static uint32_t FastDecodeTag(uint8_t coded_tag) { return coded_tag; @@ -875,8 +833,31 @@ PROTOBUF_NOINLINE const char* TcParser::SingularVarBigint( } const char* TcParser::FastV8S1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularVarint( - PROTOBUF_TC_PARAM_PASS); + // Special case for a varint bool field with a tag of 1 byte: + // The coded_tag() field will actually contain the value too and we can check + // both at the same time. + auto coded_tag = data.coded_tag(); + if (PROTOBUF_PREDICT_TRUE(coded_tag == 0x0000 || coded_tag == 0x0100)) { + auto& field = RefAt(msg, data.offset()); + // Note: we use `data.data` because Clang generates suboptimal code when + // using coded_tag. + // In x86_64 this uses the CH register to read the second byte out of + // `data`. + uint8_t value = data.data >> 8; + // The assume allows using a mov instead of test+setne. + PROTOBUF_ASSUME(value <= 1); + field = static_cast(value); + + ptr += 2; // Consume the tag and the value. + hasbits |= (uint64_t{1} << data.hasbit_idx()); + + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); + } + + // If it didn't match above either the tag is wrong, or the value is encoded + // non-canonically. + // Jump to MiniParse as wrong tag is the most probable reason. + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); } const char* TcParser::FastV8S2(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_MUSTTAIL return SingularVarint( diff --git a/src/google/protobuf/io/coded_stream.h b/src/google/protobuf/io/coded_stream.h index a8b5eb57be73..e428f55b4b08 100644 --- a/src/google/protobuf/io/coded_stream.h +++ b/src/google/protobuf/io/coded_stream.h @@ -680,7 +680,7 @@ class PROTOBUF_EXPORT EpsCopyOutputStream { if (PROTOBUF_PREDICT_FALSE(end_ - ptr < size)) { return WriteRawFallback(data, size, ptr); } - std::memcpy(ptr, data, size); + std::memcpy(ptr, data, static_cast(size)); return ptr + size; } // Writes the buffer specified by data, size to the stream. Possibly by @@ -1776,7 +1776,7 @@ inline void CodedOutputStream::WriteRawMaybeAliased(const void* data, inline uint8_t* CodedOutputStream::WriteRawToArray(const void* data, int size, uint8_t* target) { - memcpy(target, data, size); + memcpy(target, data, static_cast(size)); return target + size; } diff --git a/src/google/protobuf/io/printer.h b/src/google/protobuf/io/printer.h index e30bfa6908b7..aa507cb014e7 100644 --- a/src/google/protobuf/io/printer.h +++ b/src/google/protobuf/io/printer.h @@ -251,7 +251,8 @@ class PROTOBUF_EXPORT Printer { template void Print(const char* text, const Args&... args) { std::map vars; - PrintInternal(&vars, text, args...); + FillMap(&vars, args...); + Print(vars, text); } // Indent text by two spaces. After calling Indent(), two spaces will be @@ -299,18 +300,13 @@ class PROTOBUF_EXPORT Printer { void Annotate(const char* begin_varname, const char* end_varname, const std::string& file_path, const std::vector& path); - // Base case - void PrintInternal(std::map* vars, - const char* text) { - Print(*vars, text); - } + void FillMap(std::map* vars) {} template - void PrintInternal(std::map* vars, const char* text, - const char* key, const std::string& value, - const Args&... args) { + void FillMap(std::map* vars, const std::string& key, + const std::string& value, const Args&... args) { (*vars)[key] = value; - PrintInternal(vars, text, args...); + FillMap(vars, args...); } // Copy size worth of bytes from data to buffer_. diff --git a/src/google/protobuf/io/tokenizer.cc b/src/google/protobuf/io/tokenizer.cc index f9e07763e736..9614d015ce6f 100644 --- a/src/google/protobuf/io/tokenizer.cc +++ b/src/google/protobuf/io/tokenizer.cc @@ -406,7 +406,7 @@ void Tokenizer::ConsumeString(char delimiter) { case '\n': { if (!allow_multiline_strings_) { - AddError("String literals cannot cross line boundaries."); + AddError("Multiline strings are not allowed. Did you miss a \"?."); return; } NextChar(); diff --git a/src/google/protobuf/io/tokenizer_unittest.cc b/src/google/protobuf/io/tokenizer_unittest.cc index 16ba940483ed..6233d6ae8bfc 100644 --- a/src/google/protobuf/io/tokenizer_unittest.cc +++ b/src/google/protobuf/io/tokenizer_unittest.cc @@ -1067,7 +1067,8 @@ ErrorCase kErrorCases[] = { {"'\\X' foo", true, "0:2: Invalid escape sequence in string literal.\n"}, {"'\\x' foo", true, "0:3: Expected hex digits for escape sequence.\n"}, {"'foo", false, "0:4: Unexpected end of string.\n"}, - {"'bar\nfoo", true, "0:4: String literals cannot cross line boundaries.\n"}, + {"'bar\nfoo", true, + "0:4: Multiline strings are not allowed. Did you miss a \"?.\n"}, {"'\\u01' foo", true, "0:5: Expected four hex digits for \\u escape sequence.\n"}, {"'\\u01' foo", true, diff --git a/src/google/protobuf/map.h b/src/google/protobuf/map.h index a0e1b610a833..a3f79f154569 100644 --- a/src/google/protobuf/map.h +++ b/src/google/protobuf/map.h @@ -333,6 +333,11 @@ inline size_t SpaceUsedInValues(const void*) { return 0; } } // namespace internal +#ifdef PROTOBUF_FUTURE_MAP_PAIR_UPGRADE +// This is the class for Map's internal value_type. +template +using MapPair = std::pair; +#else // This is the class for Map's internal value_type. Instead of using // std::pair as value_type, we use this class which provides us more control of // its process of construction and destruction. @@ -363,6 +368,7 @@ struct PROTOBUF_ATTRIBUTE_STANDALONE_DEBUG MapPair { friend class Arena; friend class Map; }; +#endif // Map is an associative container type used to store protobuf map // fields. Each Map instance may or may not use a different hash function, a @@ -379,6 +385,7 @@ class Map { public: using key_type = Key; using mapped_type = T; + using init_type = std::pair; using value_type = MapPair; using pointer = value_type*; @@ -421,6 +428,22 @@ class Map { ~Map() {} private: + template + struct SameAsElementReference + : std::is_same::type>::type, + typename std::remove_cv< + typename std::remove_reference

::type>::type> {}; + + template + using RequiresInsertable = + typename std::enable_if::value || + SameAsElementReference

::value, + int>::type; + template + using RequiresNotInit = + typename std::enable_if::value, int>::type; + using Allocator = internal::MapAllocator; // InnerMap is a generic hash-based map. It doesn't contain any @@ -1270,7 +1293,6 @@ class Map { const_iterator cbegin() const { return begin(); } const_iterator cend() const { return end(); } - // Capacity size_type size() const { return elements_.size(); } bool empty() const { return size() == 0; } @@ -1351,15 +1373,17 @@ class Map { elements_.try_emplace(std::forward(k), std::forward(args)...); return std::pair(iterator(p.first), p.second); } - std::pair insert(const value_type& value) { - return try_emplace(value.first, value.second); + std::pair insert(init_type&& value) { + return try_emplace(std::move(value.first), std::move(value.second)); } - std::pair insert(value_type&& value) { - return try_emplace(value.first, std::move(value.second)); + template = 0> + std::pair insert(P&& value) { + return try_emplace(std::forward

(value).first, + std::forward

(value).second); } template std::pair emplace(Args&&... args) { - return insert(value_type(std::forward(args)...)); + return EmplaceInternal(Rank0{}, std::forward(args)...); } template void insert(InputIt first, InputIt last) { @@ -1368,7 +1392,12 @@ class Map { try_emplace(pair.first, pair.second); } } - void insert(std::initializer_list values) { + void insert(std::initializer_list values) { + insert(values.begin(), values.end()); + } + template = 0, + RequiresInsertable = 0> + void insert(std::initializer_list

values) { insert(values.begin(), values.end()); } @@ -1429,6 +1458,23 @@ class Map { } private: + struct Rank1 {}; + struct Rank0 : Rank1 {}; + + // We try to construct `init_type` from `Args` with a fall back to + // `value_type`. The latter is less desired as it unconditionally makes a copy + // of `value_type::first`. + template + auto EmplaceInternal(Rank0, Args&&... args) -> + typename std::enable_if::value, + std::pair>::type { + return insert(init_type(std::forward(args)...)); + } + template + std::pair EmplaceInternal(Rank1, Args&&... args) { + return insert(value_type(std::forward(args)...)); + } + Arena* arena() const { return elements_.arena(); } InnerMap elements_; diff --git a/src/google/protobuf/map_test.cc b/src/google/protobuf/map_test.cc index f7c024cd16f7..fd895246b507 100644 --- a/src/google/protobuf/map_test.cc +++ b/src/google/protobuf/map_test.cc @@ -84,3 +84,5 @@ TEST(MapTest, Aligned8OnArena) { MapTest_Aligned(); } } // namespace internal } // namespace protobuf } // namespace google + +#include diff --git a/src/google/protobuf/map_test.inc b/src/google/protobuf/map_test.inc index 550d986af69c..1d55dd57e601 100644 --- a/src/google/protobuf/map_test.inc +++ b/src/google/protobuf/map_test.inc @@ -737,6 +737,47 @@ TEST_F(MapImplTest, InsertSingleRValue) { EXPECT_FALSE(result2.second); } +TEST_F(MapImplTest, InsertSingleBraceInitList) { + int32_t key = 0; + int32_t value1 = 100; + int32_t value2 = 101; + + // Insert a non-existing key. + auto result1 = map_.insert({key, value1}); + ExpectSingleElement(key, value1); + + auto it1 = result1.first; + EXPECT_EQ(key, it1->first); + EXPECT_EQ(value1, it1->second); + EXPECT_TRUE(result1.second); + + // Insert an existing key. + auto result2 = map_.insert({key, value2}); + ExpectSingleElement(key, value1); + + auto it2 = result2.first; + EXPECT_TRUE(it1 == it2); + EXPECT_FALSE(result2.second); +} + +TEST_F(MapImplTest, InsertSingleBraceInitListTypeMismatch) { + int32_t key = 0; + int32_t value1 = 100; + int32_t value2 = 101; + Map m; + + // Insert a non-existing key. + auto result1 = m.insert({key, value1}); + EXPECT_TRUE(result1.second); + + // Insert an existing key. + auto result2 = m.insert({key, value2}); + EXPECT_FALSE(result2.second); + + EXPECT_TRUE(result1.first == result2.first); +} + + TEST_F(MapImplTest, TryEmplace) { using ::testing::Pair; using ::testing::UnorderedElementsAre; @@ -769,6 +810,59 @@ TEST_F(MapImplTest, Emplace) { m, UnorderedElementsAre(Pair(1, "one"), Pair(2, "two"), Pair(42, "aaa"))); } +#ifndef PROTOBUF_FUTURE_MAP_PAIR_UPGRADE + +TEST_F(MapImplTest, EmplaceKeyOnly) { + using ::testing::Pair; + using ::testing::UnorderedElementsAre; + + Map m; + + m.emplace(1); + EXPECT_EQ(m.size(), 1); + + const int32_t key = 42; + m.emplace(key); + EXPECT_THAT(m, UnorderedElementsAre(Pair(1, ""), Pair(42, ""))); +} + +#else + +TEST_F(MapImplTest, ValueTypeNoImplicitConversion) { + using vt = typename Map::value_type; + + EXPECT_FALSE((std::is_convertible< + vt, std::pair>>::value)); +} + +enum class ConstructorType { + kDefault, + kCopy, + kMove, +}; + +struct ConstructorTag { + ConstructorTag() : invoked_constructor(ConstructorType::kDefault) {} + ConstructorTag(const ConstructorTag&) + : invoked_constructor(ConstructorType::kCopy) {} + ConstructorTag(ConstructorTag&&) + : invoked_constructor(ConstructorType::kMove) {} + + ConstructorType invoked_constructor; +}; + +TEST_F(MapImplTest, ValueTypeHasMoveConstructor) { + using vt = typename Map::value_type; + ConstructorTag l, r; + + vt pair(l, std::move(r)); + + EXPECT_EQ(pair.first.invoked_constructor, ConstructorType::kCopy); + EXPECT_EQ(pair.second.invoked_constructor, ConstructorType::kMove); +} + +#endif // !PROTOBUF_FUTURE_MAP_PAIR_UPGRADE + struct CountedInstance { CountedInstance() { ++num_created; } CountedInstance(const CountedInstance&) : CountedInstance() {} diff --git a/src/google/protobuf/message.cc b/src/google/protobuf/message.cc index 724a6220f0fd..5052b1c82772 100644 --- a/src/google/protobuf/message.cc +++ b/src/google/protobuf/message.cc @@ -214,8 +214,10 @@ uint64_t Message::GetInvariantPerBuild(uint64_t salt) { } namespace internal { -void* CreateSplitMessageGeneric(Arena* arena, void* default_split, - size_t size) { +void* CreateSplitMessageGeneric(Arena* arena, const void* default_split, + size_t size, const void* message, + const void* default_message) { + GOOGLE_DCHECK_NE(message, default_message); void* split = (arena == nullptr) ? ::operator new(size) : arena->AllocateAligned(size); memcpy(split, default_split, size); diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index 7042a13342a1..0d69a6bc38cc 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -411,7 +411,9 @@ class PROTOBUF_EXPORT Message : public MessageLite { namespace internal { // Creates and returns an allocation for a split message. -void* CreateSplitMessageGeneric(Arena* arena, void* default_split, size_t size); +void* CreateSplitMessageGeneric(Arena* arena, const void* default_split, + size_t size, const void* message, + const void* default_message); // Forward-declare interfaces used to implement RepeatedFieldRef. // These are protobuf internals that users shouldn't care about. @@ -483,6 +485,11 @@ class PROTOBUF_EXPORT Reflection final { return internal::ToIntSize(SpaceUsedLong(message)); } + // Returns true if the given message is a default message instance. + bool IsDefaultInstance(const Message& message) const { + return schema_.IsDefaultInstance(message); + } + // Check if the given non-repeated field is set. bool HasField(const Message& message, const FieldDescriptor* field) const; diff --git a/src/google/protobuf/message_lite.cc b/src/google/protobuf/message_lite.cc index 27a8b38bab56..2674b2698e1f 100644 --- a/src/google/protobuf/message_lite.cc +++ b/src/google/protobuf/message_lite.cc @@ -521,18 +521,14 @@ void GenericTypeHandler::Merge(const std::string& from, *to = from; } -// Non-inline implementations of InternalMetadata routines -#if defined(NDEBUG) || defined(_MSC_VER) -// for opt and MSVC builds, the destructor is defined in the header. -#else +// Non-inline implementations of InternalMetadata destructor // This is moved out of the header because the GOOGLE_DCHECK produces a lot of code. -InternalMetadata::~InternalMetadata() { +void InternalMetadata::CheckedDestruct() { if (HasMessageOwnedArenaTag()) { GOOGLE_DCHECK(!HasUnknownFieldsTag()); delete reinterpret_cast(ptr_ - kMessageOwnedArenaTagMask); } } -#endif // Non-inline variants of std::string specializations for // various InternalMetadata routines. diff --git a/src/google/protobuf/message_unittest.inc b/src/google/protobuf/message_unittest.inc index 44e3b3151c14..c7c29e6cd9f0 100644 --- a/src/google/protobuf/message_unittest.inc +++ b/src/google/protobuf/message_unittest.inc @@ -1174,11 +1174,12 @@ TEST(MESSAGE_TEST_NAME, PreservesFloatingPointNegative0) { std::signbit(out_message.optional_double())); } -std::string EncodeEnumValue(int number, int value, int non_canonical_bytes) { - uint8_t buf[100]; - uint8_t* p = buf; - - p = internal::WireFormatLite::WriteEnumToArray(number, value, p); +// Adds `non_canonical_bytes` bytes to the varint representation at the tail of +// the buffer. +// `buf` points to the start of the buffer, `p` points to one-past-the-end. +// Returns the new one-past-the-end pointer. +uint8_t* AddNonCanonicalBytes(const uint8_t* buf, uint8_t* p, + int non_canonical_bytes) { // varint can have a max of 10 bytes. while (non_canonical_bytes-- > 0 && p - buf < 10) { // Add a dummy byte at the end. @@ -1186,7 +1187,15 @@ std::string EncodeEnumValue(int number, int value, int non_canonical_bytes) { p[0] = 0; ++p; } + return p; +} +std::string EncodeEnumValue(int number, int value, int non_canonical_bytes) { + uint8_t buf[100]; + uint8_t* p = buf; + + p = internal::WireFormatLite::WriteEnumToArray(number, value, p); + p = AddNonCanonicalBytes(buf, p, non_canonical_bytes); return std::string(buf, p); } @@ -1201,6 +1210,16 @@ TEST(MESSAGE_TEST_NAME, TestEnumParsers) { const auto other_field = EncodeOtherField(); + // Encode a boolean field for many different cases and verify that it can be + // parsed as expected. + // There are: + // - optional/repeated/packed fields + // - field tags that encode in 1/2/3 bytes + // - canonical and non-canonical encodings of the varint + // - last vs not last field + // - label combinations to trigger different parsers: sequential, small + // sequential, non-validated. + constexpr int kInvalidValue = 0x900913; auto* ref = obj.GetReflection(); auto* descriptor = obj.descriptor(); @@ -1226,6 +1245,8 @@ TEST(MESSAGE_TEST_NAME, TestEnumParsers) { auto encoded = EncodeEnumValue(field->number(), value_desc->number(), non_canonical_bytes); if (use_tail_field) { + // Make sure that fields after this one can be parsed too. ie test + // that the "next" jump is correct too. encoded += other_field; } @@ -1263,5 +1284,72 @@ TEST(MESSAGE_TEST_NAME, TestEnumParsers) { } } +std::string EncodeBoolValue(int number, bool value, int non_canonical_bytes) { + uint8_t buf[100]; + uint8_t* p = buf; + + p = internal::WireFormatLite::WriteBoolToArray(number, value, p); + p = AddNonCanonicalBytes(buf, p, non_canonical_bytes); + return std::string(buf, p); +} + +TEST(MESSAGE_TEST_NAME, TestBoolParsers) { + UNITTEST::BoolParseTester obj; + + const auto other_field = EncodeOtherField(); + + // Encode a boolean field for many different cases and verify that it can be + // parsed as expected. + // There are: + // - optional/repeated/packed fields + // - field tags that encode in 1/2/3 bytes + // - canonical and non-canonical encodings of the varint + // - last vs not last field + + auto* ref = obj.GetReflection(); + auto* descriptor = obj.descriptor(); + for (bool use_tail_field : {false, true}) { + SCOPED_TRACE(use_tail_field); + for (int non_canonical_bytes = 0; non_canonical_bytes < 10; + ++non_canonical_bytes) { + SCOPED_TRACE(non_canonical_bytes); + for (int i = 0; i < descriptor->field_count(); ++i) { + const auto* field = descriptor->field(i); + if (field->name() == "other_field") continue; + SCOPED_TRACE(field->full_name()); + for (bool value : {false, true}) { + SCOPED_TRACE(value); + auto encoded = + EncodeBoolValue(field->number(), value, non_canonical_bytes); + if (use_tail_field) { + // Make sure that fields after this one can be parsed too. ie test + // that the "next" jump is correct too. + encoded += other_field; + } + + EXPECT_TRUE(obj.ParseFromString(encoded)); + if (field->is_repeated()) { + ASSERT_EQ(ref->FieldSize(obj, field), 1); + EXPECT_EQ(ref->GetRepeatedBool(obj, field, 0), value); + } else { + EXPECT_TRUE(ref->HasField(obj, field)); + EXPECT_EQ(ref->GetBool(obj, field), value); + } + auto& unknown = ref->GetUnknownFields(obj); + ASSERT_EQ(unknown.field_count(), 0); + } + } + } + } +} + +TEST(MESSAGE_TEST_NAME, IsDefaultInstance) { + UNITTEST::TestAllTypes msg; + const auto& default_msg = UNITTEST::TestAllTypes::default_instance(); + const auto* r = msg.GetReflection(); + EXPECT_TRUE(r->IsDefaultInstance(default_msg)); + EXPECT_FALSE(r->IsDefaultInstance(msg)); +} + } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/metadata_lite.h b/src/google/protobuf/metadata_lite.h index af840e5065c9..d015dc2f9da4 100644 --- a/src/google/protobuf/metadata_lite.h +++ b/src/google/protobuf/metadata_lite.h @@ -77,15 +77,19 @@ class PROTOBUF_EXPORT InternalMetadata { GOOGLE_DCHECK(!is_message_owned || arena != nullptr); } -#if defined(NDEBUG) || defined(_MSC_VER) + // To keep the ABI identical between debug and non-debug builds, + // the destructor is always defined here even though it may delegate + // to a non-inline private method. + // (see https://github.com/protocolbuffers/protobuf/issues/9947) ~InternalMetadata() { +#if defined(NDEBUG) || defined(_MSC_VER) if (HasMessageOwnedArenaTag()) { delete reinterpret_cast(ptr_ - kMessageOwnedArenaTagMask); } - } #else - ~InternalMetadata(); + CheckedDestruct(); #endif + } template void Delete() { @@ -264,6 +268,9 @@ class PROTOBUF_EXPORT InternalMetadata { PROTOBUF_NOINLINE void DoSwap(T* other) { mutable_unknown_fields()->Swap(other); } + + // Private helper with debug checks for ~InternalMetadata() + void CheckedDestruct(); }; // String Template specializations. diff --git a/src/google/protobuf/parse_context.h b/src/google/protobuf/parse_context.h index a7baf4d3336a..c72e19fe0d0e 100644 --- a/src/google/protobuf/parse_context.h +++ b/src/google/protobuf/parse_context.h @@ -617,6 +617,7 @@ PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE constexpr T RotateLeft( PROTOBUF_NODISCARD inline PROTOBUF_ALWAYS_INLINE uint64_t RotRight7AndReplaceLowByte(uint64_t res, const char& byte) { + // TODO(b/239808098): remove the inline assembly #if defined(__x86_64__) && defined(__GNUC__) // This will only use one register for `res`. // `byte` comes as a reference to allow the compiler to generate code like: diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index f0aa3deb201a..7d067fd2e8c7 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -154,6 +154,13 @@ # define PROTOBUF_GNUC_MIN(x, y) 0 #endif +#if defined(__clang__) && defined(__clang_major__) && defined(__clang_minor__) +#define PROTOBUF_CLANG_MIN(x, y) \ + (__clang_major__ > (x) || __clang_major__ == (x) && __clang_minor__ >= (y)) +#else +#define PROTOBUF_CLANG_MIN(x, y) 0 +#endif + // Portable check for MSVC minimum version: // https://docs.microsoft.com/en-us/cpp/preprocessor/predefined-macros #if defined(_MSC_VER) @@ -174,11 +181,15 @@ // Future versions of protobuf will include breaking changes to some APIs. // This macro can be set to enable these API changes ahead of time, so that // user code can be updated before upgrading versions of protobuf. -// PROTOBUF_FUTURE_FINAL is used on classes that are historically not marked as -// final, but that may be marked final in future (breaking) releases. #ifdef PROTOBUF_FUTURE_BREAKING_CHANGES -// Used on classes that are historically not marked as final. + +// Used to upgrade google::protobuf::MapPair to std::pair. +// Owner: mordberg@ +#define PROTOBUF_FUTURE_MAP_PAIR_UPGRADE 1 + +// Used on classes that are historically not marked as final, but that may be +// marked final in future (breaking) releases. // Owner: kfm@ #define PROTOBUF_FUTURE_FINAL final @@ -191,6 +202,7 @@ #define PROTOBUF_FUTURE_REMOVE_CLEARED_API 1 // Used to escape C++20 keywords. +// TODO(mkruskal): ping b/238664698 when this lands in opensource. // Owner: mkruskal@ #define PROTOBUF_FUTURE_CPP20_KEYWORDS 1 #else @@ -200,7 +212,7 @@ #ifdef PROTOBUF_VERSION #error PROTOBUF_VERSION was previously defined #endif -#define PROTOBUF_VERSION 3021002 +#define PROTOBUF_VERSION 3021004 #ifdef PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC #error PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC was previously defined @@ -559,6 +571,10 @@ #error PROTOBUF_FORCE_COPY_DEFAULT_STRING was previously defined #endif +#ifdef PROTOBUF_FORCE_ALLOCATION_ON_CONSTRUCTION +#error PROTOBUF_FORCE_ALLOCATION_ON_CONSTRUCTION was previously defined +#endif + #ifdef PROTOBUF_FALLTHROUGH_INTENDED #error PROTOBUF_FALLTHROUGH_INTENDED was previously defined #endif @@ -661,30 +677,39 @@ // This experiment is purely for the purpose of gathering data. All code guarded // by this flag is supposed to be removed after this experiment. #define PROTOBUF_MESSAGE_OWNED_ARENA_EXPERIMENT + #ifdef PROTOBUF_CONSTINIT #error PROTOBUF_CONSTINIT was previously defined #endif -#if defined(__cpp_constinit) && !defined(_MSC_VER) -#define PROTOBUF_CONSTINIT constinit -#define PROTOBUF_CONSTEXPR constexpr + +// Lexan sets both MSV_VER and clang, so handle it with the clang path. +#if defined(_MSC_VER) && !defined(__clang__) +// MSVC 17 currently seems to raise an error about constant-initialized pointers. +# if PROTOBUF_MSC_VER_MIN(1930) +# define PROTOBUF_CONSTINIT +# define PROTOBUF_CONSTEXPR constexpr +# endif +#else +# if defined(__cpp_constinit) && !defined(__CYGWIN__) +# define PROTOBUF_CONSTINIT constinit +# define PROTOBUF_CONSTEXPR constexpr // Some older Clang versions incorrectly raise an error about // constant-initializing weak default instance pointers. Versions 12.0 and // higher seem to work, except that XCode 12.5.1 shows the error even though it // uses Clang 12.0.5. -// Clang-cl on Windows raises error also. -#elif !defined(_MSC_VER) && __has_cpp_attribute(clang::require_constant_initialization) && \ - ((defined(__APPLE__) && __clang_major__ >= 13) || \ - (!defined(__APPLE__) && __clang_major__ >= 12)) -#define PROTOBUF_CONSTINIT [[clang::require_constant_initialization]] -#define PROTOBUF_CONSTEXPR constexpr -#elif PROTOBUF_GNUC_MIN(12, 2) -#define PROTOBUF_CONSTINIT __constinit -#define PROTOBUF_CONSTEXPR constexpr -// MSVC 17 currently seems to raise an error about constant-initialized pointers. -#elif defined(_MSC_VER) && _MSC_VER >= 1930 -#define PROTOBUF_CONSTINIT -#define PROTOBUF_CONSTEXPR constexpr -#else +# elif !defined(__CYGWIN__) && \ + __has_cpp_attribute(clang::require_constant_initialization) && \ + ((defined(__APPLE__) && PROTOBUF_CLANG_MIN(13, 0)) || \ + (!defined(__APPLE__) && PROTOBUF_CLANG_MIN(12, 0))) +# define PROTOBUF_CONSTINIT [[clang::require_constant_initialization]] +# define PROTOBUF_CONSTEXPR constexpr +# elif PROTOBUF_GNUC_MIN(12, 2) +# define PROTOBUF_CONSTINIT __constinit +# define PROTOBUF_CONSTEXPR constexpr +# endif +#endif + +#ifndef PROTOBUF_CONSTINIT #define PROTOBUF_CONSTINIT #define PROTOBUF_CONSTEXPR #endif @@ -807,11 +832,22 @@ #define PROTOBUF_TAIL_CALL_TABLE_PARSER_ENABLED 1 #endif -#define PROTOBUF_TC_PARAM_DECL \ - ::google::protobuf::MessageLite *msg, const char *ptr, \ - ::google::protobuf::internal::ParseContext *ctx, \ - const ::google::protobuf::internal::TcParseTableBase *table, \ - uint64_t hasbits, ::google::protobuf::internal::TcFieldData data +// Note that this is performance sensitive: changing the parameters will change +// the registers used by the ABI calling convention, which subsequently affects +// register selection logic inside the function. +// Arguments `msg`, `ptr` and `ctx` are the 1st/2nd/3rd argument to match the +// signature of ParseLoop. +// +// Note for x86_64: `data` must be the third or fourth argument for performance +// reasons. In order to efficiently read the second byte of `data` we need it to +// be passed in RDX or RCX. +#define PROTOBUF_TC_PARAM_DECL \ + ::google::protobuf::MessageLite *msg, const char *ptr, \ + ::google::protobuf::internal::ParseContext *ctx, \ + ::google::protobuf::internal::TcFieldData data, \ + const ::google::protobuf::internal::TcParseTableBase *table, uint64_t hasbits +// PROTOBUF_TC_PARAM_PASS passes values to match PROTOBUF_TC_PARAM_DECL. +#define PROTOBUF_TC_PARAM_PASS msg, ptr, ctx, data, table, hasbits #ifdef PROTOBUF_UNUSED #error PROTOBUF_UNUSED was previously defined @@ -933,6 +969,21 @@ #pragma warning(disable: 4125) #endif +#if PROTOBUF_ENABLE_DEBUG_LOGGING_MAY_LEAK_PII +#define PROTOBUF_DEBUG true +#else +#define PROTOBUF_DEBUG false +#endif + +// This `for` allows us to condition the `GOOGLE_LOG` on the define above, so that +// code can write `PROTOBUF_DLOG(INFO) << ...;` and have it turned off when +// debug logging is off. +// +// This is a `for`, not and `if`, to avoid it accidentally chaining with an +// `else` below it. +#define PROTOBUF_DLOG(x) \ + for (bool b = PROTOBUF_DEBUG; b; b = false) GOOGLE_LOG(x) + // We don't want code outside port_def doing complex testing, so // remove our portable condition test macros to nudge folks away from // using it themselves. diff --git a/src/google/protobuf/port_undef.inc b/src/google/protobuf/port_undef.inc index 5732b197c4cc..23eb789d1e76 100644 --- a/src/google/protobuf/port_undef.inc +++ b/src/google/protobuf/port_undef.inc @@ -40,6 +40,7 @@ #undef PROTOBUF_BUILTIN_BSWAP64 #undef PROTOBUF_BUILTIN_ATOMIC #undef PROTOBUF_GNUC_MIN +#undef PROTOBUF_CLANG_MIN #undef PROTOBUF_MSC_VER_MIN #undef PROTOBUF_CPLUSPLUS_MIN #undef PROTOBUF_NAMESPACE @@ -75,6 +76,7 @@ #undef PROTOBUF_FORCE_RESET_IN_CLEAR #undef PROTOBUF_FUZZ_MESSAGE_SPACE_USED_LONG #undef PROTOBUF_FORCE_COPY_DEFAULT_STRING +#undef PROTOBUF_FORCE_ALLOCATION_ON_CONSTRUCTION #undef PROTOBUF_NAMESPACE_OPEN #undef PROTOBUF_NAMESPACE_CLOSE #undef PROTOBUF_UNUSED @@ -106,9 +108,12 @@ #undef PROTOBUF_LOCKS_EXCLUDED #undef PROTOBUF_NO_THREAD_SAFETY_ANALYSIS #undef PROTOBUF_GUARDED_BY +#undef PROTOBUF_DEBUG +#undef PROTOBUF_DLOG #ifdef PROTOBUF_FUTURE_BREAKING_CHANGES #undef PROTOBUF_FUTURE_BREAKING_CHANGES +#undef PROTOBUF_FUTURE_MAP_PAIR_UPGRADE #undef PROTOBUF_FUTURE_REMOVE_DEFAULT_FIELD_COMPARATOR #undef PROTOBUF_FUTURE_REMOVE_CLEARED_API #endif diff --git a/src/google/protobuf/repeated_field.h b/src/google/protobuf/repeated_field.h index 3fbe4995cd0a..a7c1275b60d2 100644 --- a/src/google/protobuf/repeated_field.h +++ b/src/google/protobuf/repeated_field.h @@ -94,43 +94,30 @@ constexpr int RepeatedFieldLowerClampLimit() { constexpr int kRepeatedFieldUpperClampLimit = (std::numeric_limits::max() / 2) + 1; -// Swaps two blocks of memory of size sizeof(T). -template -inline void SwapBlock(char* p, char* q) { - T tmp; - memcpy(&tmp, p, sizeof(T)); - memcpy(p, q, sizeof(T)); - memcpy(q, &tmp, sizeof(T)); -} - // Swaps two blocks of memory of size kSize: -// template void memswap(char* p, char* q); -template -inline typename std::enable_if<(kSize == 0), void>::type memswap(char*, char*) { -} - -#define PROTO_MEMSWAP_DEF_SIZE(reg_type, max_size) \ - template \ - typename std::enable_if<(kSize >= sizeof(reg_type) && kSize < (max_size)), \ - void>::type \ - memswap(char* p, char* q) { \ - SwapBlock(p, q); \ - memswap(p + sizeof(reg_type), \ - q + sizeof(reg_type)); \ - } - -PROTO_MEMSWAP_DEF_SIZE(uint8_t, 2) -PROTO_MEMSWAP_DEF_SIZE(uint16_t, 4) -PROTO_MEMSWAP_DEF_SIZE(uint32_t, 8) - -#ifdef __SIZEOF_INT128__ -PROTO_MEMSWAP_DEF_SIZE(uint64_t, 16) -PROTO_MEMSWAP_DEF_SIZE(__uint128_t, (1u << 31)) +template +void memswap(char* a, char* b) { +#if __SIZEOF_INT128__ + using Buffer = __uint128_t; #else -PROTO_MEMSWAP_DEF_SIZE(uint64_t, (1u << 31)) + using Buffer = uint64_t; #endif -#undef PROTO_MEMSWAP_DEF_SIZE + constexpr size_t kBlockSize = sizeof(Buffer); + Buffer buf; + for (size_t i = 0; i < kSize / kBlockSize; ++i) { + memcpy(&buf, a, kBlockSize); + memcpy(a, b, kBlockSize); + memcpy(b, &buf, kBlockSize); + a += kBlockSize; + b += kBlockSize; + } + + // Swap the leftover bytes, could be zero. + memcpy(&buf, a, kSize % kBlockSize); + memcpy(a, b, kSize % kBlockSize); + memcpy(b, &buf, kSize % kBlockSize); +} template class RepeatedIterator; @@ -233,12 +220,6 @@ class RepeatedField final { // copies data between each other. void Swap(RepeatedField* other); - // Swaps entire contents with "other". Should be called only if the caller can - // guarantee that both repeated fields are on the same arena or are on the - // heap. Swapping between different arenas is disallowed and caught by a - // GOOGLE_DCHECK (see API docs for details). - void UnsafeArenaSwap(RepeatedField* other); - // Swaps two elements. void SwapElements(int index1, int index2); @@ -311,6 +292,7 @@ class RepeatedField final { // This is public due to it being called by generated code. inline void InternalSwap(RepeatedField* other); + private: template friend class Arena::InternalHelper; @@ -320,6 +302,12 @@ class RepeatedField final { : rep()->arena; } + // Swaps entire contents with "other". Should be called only if the caller can + // guarantee that both repeated fields are on the same arena or are on the + // heap. Swapping between different arenas is disallowed and caught by a + // GOOGLE_DCHECK (see API docs for details). + void UnsafeArenaSwap(RepeatedField* other); + static constexpr int kInitialSize = 0; // A note on the representation here (see also comment below for // RepeatedPtrFieldBase's struct Rep): @@ -338,7 +326,7 @@ class RepeatedField final { // current_size_. This function is intended to be the only place where // current_size_ is modified. inline int ExchangeCurrentSize(int new_size) { - int prev_size = current_size_; + const int prev_size = current_size_; current_size_ = new_size; return prev_size; } @@ -355,6 +343,7 @@ class RepeatedField final { } }; + // If total_size_ == 0 this points to an Arena otherwise it points to the // elements member of a Rep struct. Using this invariant allows the storage of // the arena pointer without an extra allocation in the constructor. @@ -471,7 +460,7 @@ class RepeatedField final { void Add(Element val) { if (index_ == capacity_) { - repeated_field_->ExchangeCurrentSize(index_); + repeated_field_->current_size_ = index_; repeated_field_->Reserve(index_ + 1); capacity_ = repeated_field_->total_size_; buffer_ = repeated_field_->unsafe_elements(); diff --git a/src/google/protobuf/repeated_field_unittest.cc b/src/google/protobuf/repeated_field_unittest.cc index c5211470bc0b..2cb6bab8c78b 100644 --- a/src/google/protobuf/repeated_field_unittest.cc +++ b/src/google/protobuf/repeated_field_unittest.cc @@ -305,6 +305,60 @@ TEST(RepeatedField, SwapLargeLarge) { } } +template +void TestMemswap() { + SCOPED_TRACE(kSize); + + const auto a_char = [](int i) -> char { return (i % ('z' - 'a')) + 'a'; }; + const auto b_char = [](int i) -> char { return (i % ('Z' - 'A')) + 'A'; }; + std::string a, b; + for (int i = 0; i < kSize; ++i) { + a += a_char(i); + b += b_char(i); + } + // We will not swap these. + a += "+"; + b += "-"; + + std::string expected_a = b, expected_b = a; + expected_a.back() = '+'; + expected_b.back() = '-'; + + internal::memswap(&a[0], &b[0]); + + // ODR use the functions in a way that forces the linker to keep them. That + // way we can see their generated code. + volatile auto odr_use_for_asm_dump = &internal::memswap; + (void)odr_use_for_asm_dump; + + EXPECT_EQ(expected_a, a); + EXPECT_EQ(expected_b, b); +} + +TEST(Memswap, VerifyWithSmallAndLargeSizes) { + // Arbitrary sizes + TestMemswap<0>(); + TestMemswap<1>(); + TestMemswap<10>(); + TestMemswap<100>(); + TestMemswap<1000>(); + TestMemswap<10000>(); + TestMemswap<100000>(); + TestMemswap<1000000>(); + + // Pointer aligned sizes + TestMemswap(); + TestMemswap(); + TestMemswap(); + TestMemswap(); + + // Test also just the block size and no leftover. + TestMemswap<64 * 1>(); + TestMemswap<64 * 2>(); + TestMemswap<64 * 3>(); + TestMemswap<64 * 4>(); +} + // Determines how much space was reserved by the given field by adding elements // to it until it re-allocates its space. static int ReservedSpace(RepeatedField* field) { diff --git a/src/google/protobuf/repeated_ptr_field.h b/src/google/protobuf/repeated_ptr_field.h index acdbee3afc0b..8a9c9e6c7312 100644 --- a/src/google/protobuf/repeated_ptr_field.h +++ b/src/google/protobuf/repeated_ptr_field.h @@ -41,8 +41,6 @@ // // This header covers RepeatedPtrField. -// IWYU pragma: private, include "net/proto2/public/repeated_field.h" - #ifndef GOOGLE_PROTOBUF_REPEATED_PTR_FIELD_H__ #define GOOGLE_PROTOBUF_REPEATED_PTR_FIELD_H__ diff --git a/src/google/protobuf/source_context.pb.h b/src/google/protobuf/source_context.pb.h index fb2e29f2eafd..899cfb012290 100644 --- a/src/google/protobuf/source_context.pb.h +++ b/src/google/protobuf/source_context.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/struct.pb.h b/src/google/protobuf/struct.pb.h index 7a0805d5b559..154ea45ec935 100644 --- a/src/google/protobuf/struct.pb.h +++ b/src/google/protobuf/struct.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/stubs/common.h b/src/google/protobuf/stubs/common.h index 9d90305126cb..c4d26369a392 100644 --- a/src/google/protobuf/stubs/common.h +++ b/src/google/protobuf/stubs/common.h @@ -82,7 +82,7 @@ namespace internal { // The current version, represented as a single integer to make comparison // easier: major * 10^6 + minor * 10^3 + micro -#define GOOGLE_PROTOBUF_VERSION 3021002 +#define GOOGLE_PROTOBUF_VERSION 3021004 // A suffix string for alpha, beta or rc releases. Empty for stable releases. #define GOOGLE_PROTOBUF_VERSION_SUFFIX "" diff --git a/src/google/protobuf/timestamp.pb.h b/src/google/protobuf/timestamp.pb.h index 3b5a469b2e22..771216e61b43 100644 --- a/src/google/protobuf/timestamp.pb.h +++ b/src/google/protobuf/timestamp.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/type.pb.h b/src/google/protobuf/type.pb.h index bcf1a444e0a7..2555602e930a 100644 --- a/src/google/protobuf/type.pb.h +++ b/src/google/protobuf/type.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/unittest.proto b/src/google/protobuf/unittest.proto index 4d08b444321d..de78f9a69d16 100644 --- a/src/google/protobuf/unittest.proto +++ b/src/google/protobuf/unittest.proto @@ -221,6 +221,12 @@ message TestReservedFields { reserved "bar", "baz"; } +enum TestReservedEnumFields { + UNKNOWN = 0; + reserved 2, 15, 9 to 11; + reserved "bar", "baz"; +} + message TestAllExtensions { extensions 1 to max; } @@ -340,6 +346,16 @@ extend TestAllExtensions { optional bytes oneof_bytes_extension = 114; } +message TestMixedFieldsAndExtensions { + optional int32 a = 1; + repeated fixed32 b = 3; + extensions 2, 4; + extend TestMixedFieldsAndExtensions { + optional int32 c = 2; + repeated fixed32 d = 4; + } +} + message TestGroup { optional group OptionalGroup = 16 { optional int32 a = 17; @@ -1521,3 +1537,20 @@ message EnumParseTester { optional int32 other_field = 99; }; +// This message contains different kind of bool fields to exercise the different +// parsers in table-drived. +message BoolParseTester { + optional bool optional_bool_lowfield = 1; + optional bool optional_bool_midfield = 1001; + optional bool optional_bool_hifield = 1000001; + repeated bool repeated_bool_lowfield = 2; + repeated bool repeated_bool_midfield = 1002; + repeated bool repeated_bool_hifield = 1000002; + repeated bool packed_bool_lowfield = 3 [packed = true]; + repeated bool packed_bool_midfield = 1003 [packed = true]; + repeated bool packed_bool_hifield = 1000003 [packed = true]; + + // An arbitrary field we can append to to break the runs of repeated fields. + optional int32 other_field = 99; +}; + diff --git a/src/google/protobuf/util/json_format_proto3.proto b/src/google/protobuf/util/json_format_proto3.proto index f9c5199e3a40..4df5eb9d3e17 100644 --- a/src/google/protobuf/util/json_format_proto3.proto +++ b/src/google/protobuf/util/json_format_proto3.proto @@ -189,6 +189,14 @@ message TestCustomJsonName { int32 value = 1 [json_name = "@value"]; } +message TestEvilJson { + int32 regular_value = 1 [json_name = "regular_name"]; + int32 script = 2 [json_name = ""]; + int32 quotes = 3 [json_name = "unbalanced\"quotes"]; + int32 script_and_quotes = 4 + [json_name = "\""]; +} + message TestExtensions { .protobuf_unittest.TestAllExtensions extensions = 1; } diff --git a/src/google/protobuf/util/json_util.cc b/src/google/protobuf/util/json_util.cc index 9c59d396894c..04cab9cf06e2 100644 --- a/src/google/protobuf/util/json_util.cc +++ b/src/google/protobuf/util/json_util.cc @@ -48,10 +48,12 @@ #include #include + // clang-format off #include // clang-format on + namespace google { namespace protobuf { namespace util { @@ -62,6 +64,7 @@ util::Status BinaryToJsonStream(TypeResolver* resolver, io::ZeroCopyInputStream* binary_input, io::ZeroCopyOutputStream* json_output, const JsonPrintOptions& options) { + io::CodedInputStream in_stream(binary_input); google::protobuf::Type type; RETURN_IF_ERROR(resolver->ResolveMessageType(type_url, &type)); @@ -152,6 +155,7 @@ util::Status JsonToBinaryStream(TypeResolver* resolver, io::ZeroCopyInputStream* json_input, io::ZeroCopyOutputStream* binary_output, const JsonParseOptions& options) { + google::protobuf::Type type; RETURN_IF_ERROR(resolver->ResolveMessageType(type_url, &type)); ZeroCopyStreamByteSink sink(binary_output); @@ -219,6 +223,7 @@ TypeResolver* GetGeneratedTypeResolver() { util::Status MessageToJsonString(const Message& message, std::string* output, const JsonOptions& options) { + const DescriptorPool* pool = message.GetDescriptor()->file()->pool(); TypeResolver* resolver = pool == DescriptorPool::generated_pool() @@ -235,6 +240,7 @@ util::Status MessageToJsonString(const Message& message, std::string* output, util::Status JsonStringToMessage(StringPiece input, Message* message, const JsonParseOptions& options) { + const DescriptorPool* pool = message->GetDescriptor()->file()->pool(); TypeResolver* resolver = pool == DescriptorPool::generated_pool() diff --git a/src/google/protobuf/util/json_util.h b/src/google/protobuf/util/json_util.h index 4f4594e2f9c7..6a5a4482b83f 100644 --- a/src/google/protobuf/util/json_util.h +++ b/src/google/protobuf/util/json_util.h @@ -33,6 +33,7 @@ #ifndef GOOGLE_PROTOBUF_UTIL_JSON_UTIL_H__ #define GOOGLE_PROTOBUF_UTIL_JSON_UTIL_H__ + #include #include #include @@ -93,6 +94,9 @@ typedef JsonPrintOptions JsonOptions; // Converts from protobuf message to JSON and appends it to |output|. This is a // simple wrapper of BinaryToJsonString(). It will use the DescriptorPool of the // passed-in message to resolve Any types. +// +// Please note that non-OK statuses are not a stable output of this API and +// subject to change without notice. PROTOBUF_EXPORT util::Status MessageToJsonString(const Message& message, std::string* output, const JsonOptions& options); @@ -105,6 +109,9 @@ inline util::Status MessageToJsonString(const Message& message, // Converts from JSON to protobuf message. This is a simple wrapper of // JsonStringToBinary(). It will use the DescriptorPool of the passed-in // message to resolve Any types. +// +// Please note that non-OK statuses are not a stable output of this API and +// subject to change without notice. PROTOBUF_EXPORT util::Status JsonStringToMessage( StringPiece input, Message* message, const JsonParseOptions& options); @@ -119,6 +126,9 @@ inline util::Status JsonStringToMessage(StringPiece input, // 2. input is not valid protobuf wire format, or conflicts with the type // information returned by TypeResolver. // Note that unknown fields will be discarded silently. +// +// Please note that non-OK statuses are not a stable output of this API and +// subject to change without notice. PROTOBUF_EXPORT util::Status BinaryToJsonStream( TypeResolver* resolver, const std::string& type_url, io::ZeroCopyInputStream* binary_input, @@ -150,6 +160,9 @@ inline util::Status BinaryToJsonString(TypeResolver* resolver, // 1. TypeResolver fails to resolve a type. // 2. input is not valid JSON format, or conflicts with the type // information returned by TypeResolver. +// +// Please note that non-OK statuses are not a stable output of this API and +// subject to change without notice. PROTOBUF_EXPORT util::Status JsonToBinaryStream( TypeResolver* resolver, const std::string& type_url, io::ZeroCopyInputStream* json_input, diff --git a/src/google/protobuf/util/json_util_test.cc b/src/google/protobuf/util/json_util_test.cc index 1ff638b66ece..99272d71ac34 100644 --- a/src/google/protobuf/util/json_util_test.cc +++ b/src/google/protobuf/util/json_util_test.cc @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -63,6 +64,11 @@ // Must be included last. #include +bool IsJson2() { + // Pay no attention to the person behind the curtain. + return false; +} + namespace google { namespace protobuf { namespace util { @@ -74,6 +80,9 @@ using ::proto3::TestMessage; using ::proto3::TestOneof; using ::proto3::TestWrapper; using ::proto_util_converter::testing::MapIn; +using ::testing::ElementsAre; +using ::testing::Not; +using ::testing::SizeIs; // TODO(b/234474291): Use the gtest versions once that's available in OSS. MATCHER_P(IsOkAndHolds, inner, @@ -153,7 +162,7 @@ class JsonTest : public testing::TestWithParam { util::StatusOr ToProto(StringPiece json, JsonParseOptions options = {}) { Proto proto; - RETURN_IF_ERROR(JsonStringToMessage(json, &proto, options)); + RETURN_IF_ERROR(ToProto(proto, json, options)); return proto; } @@ -268,13 +277,43 @@ TEST_P(JsonTest, TestDefaultValues) { R"("defaultString":"hello","defaultBytes":"d29ybGQ=","defaultNestedEnum":"BAR",)" R"("defaultForeignEnum":"FOREIGN_BAR","defaultImportEnum":"IMPORT_BAR",)" R"("defaultStringPiece":"abc","defaultCord":"123"})")); -} -TEST_P(JsonTest, TestPreserveProtoFieldNames) { - if (GetParam() == Codec::kResolver) { - GTEST_SKIP(); + // The ESF parser actually gets this wrong, and serializes floats whose + // default value is non-finite as 0. We make sure to reproduce this bug. + if (IsJson2()) { + EXPECT_THAT( + ToJson(protobuf_unittest::TestExtremeDefaultValues(), options), + IsOkAndHolds( + R"({"escapedBytes":"XDAwMFwwMDFcMDA3XDAxMFwwMTRcblxyXHRcMDEzXFxcJ1wiXDM3Ng==")" + R"(,"largeUint32":4294967295,"largeUint64":"18446744073709551615",)" + R"("smallInt32":-2147483647,"smallInt64":"-9223372036854775807",)" + R"("utf8String":"ሴ","zeroFloat":0,"oneFloat":1,"smallFloat":1.5,)" + R"("negativeOneFloat":-1,"negativeFloat":-1.5,"largeFloat":2e+08,)" + R"("smallNegativeFloat":-8e-28,"infDouble":0,"negInfDouble":0,)" + R"("nanDouble":0,"infFloat":0,"negInfFloat":0,"nanFloat":0,)" + R"("cppTrigraph":"? ? ?? ?? ??? ??/ ??-","reallySmallInt32":-2147483648)" + R"(,"reallySmallInt64":"-9223372036854775808","stringWithZero":"hel\u0000lo")" + R"(,"bytesWithZero":"d29yXDAwMGxk","stringPieceWithZero":"ab\u0000c")" + R"(,"cordWithZero":"12\u00003","replacementString":"${unknown}"})")); + } else { + EXPECT_THAT( + ToJson(protobuf_unittest::TestExtremeDefaultValues(), options), + IsOkAndHolds( + R"({"escapedBytes":"XDAwMFwwMDFcMDA3XDAxMFwwMTRcblxyXHRcMDEzXFxcJ1wiXDM3Ng==")" + R"(,"largeUint32":4294967295,"largeUint64":"18446744073709551615",)" + R"("smallInt32":-2147483647,"smallInt64":"-9223372036854775807")" + R"(,"reallySmallInt32":-2147483648,"reallySmallInt64":"-9223372036854775808",)" + R"("utf8String":"ሴ","zeroFloat":0,"oneFloat":1,"smallFloat":1.5,)" + R"("negativeOneFloat":-1,"negativeFloat":-1.5,"largeFloat":2e+08,)" + R"("smallNegativeFloat":-8e-28,"infDouble":0,"negInfDouble":0)" + R"(,"nanDouble":0,"infFloat":0,"negInfFloat":0,"nanFloat":0)" + R"(,"cppTrigraph":"? ? ?? ?? ??? ??/ ??-","stringWithZero":"hel\u0000lo")" + R"(,"bytesWithZero":"d29yXDAwMGxk","stringPieceWithZero":"ab\u0000c")" + R"(,"cordWithZero":"12\u00003","replacementString":"${unknown}"})")); } +} +TEST_P(JsonTest, TestPreserveProtoFieldNames) { TestMessage m; m.mutable_message_value(); @@ -284,6 +323,22 @@ TEST_P(JsonTest, TestPreserveProtoFieldNames) { } +TEST_P(JsonTest, Camels) { + protobuf_unittest::TestCamelCaseFieldNames m; + m.set_stringfield("sTRINGfIELD"); + + EXPECT_THAT(ToJson(m), IsOkAndHolds(R"({"StringField":"sTRINGfIELD"})")); +} + +TEST_P(JsonTest, EvilString) { + auto m = ToProto(R"json( + {"string_value": ")json" + "\n\r\b\f\1\2\3" + "\"}"); + ASSERT_OK(m); + EXPECT_EQ(m->string_value(), "\n\r\b\f\1\2\3"); +} + TEST_P(JsonTest, TestAlwaysPrintEnumsAsInts) { TestMessage orig; orig.set_enum_value(proto3::BAR); @@ -301,9 +356,8 @@ TEST_P(JsonTest, TestAlwaysPrintEnumsAsInts) { ASSERT_OK(parsed); EXPECT_EQ(parsed->enum_value(), proto3::BAR); - EXPECT_EQ(parsed->repeated_enum_value_size(), 2); - EXPECT_EQ(parsed->repeated_enum_value(0), proto3::FOO); - EXPECT_EQ(parsed->repeated_enum_value(1), proto3::BAR); + EXPECT_THAT(parsed->repeated_enum_value(), + ElementsAre(proto3::FOO, proto3::BAR)); } TEST_P(JsonTest, TestPrintEnumsAsIntsWithDefaultValue) { @@ -377,6 +431,7 @@ TEST_P(JsonTest, ParseMessage) { "repeatedEnumValue": [1, "FOO"], "repeatedMessageValue": [ {"value": 40}, + {}, {"value": 96} ] } @@ -398,33 +453,17 @@ TEST_P(JsonTest, ParseMessage) { EXPECT_EQ(m->enum_value(), proto3::EnumType::BAR); EXPECT_EQ(m->message_value().value(), 2048); - ASSERT_EQ(m->repeated_bool_value_size(), 1); - EXPECT_TRUE(m->repeated_bool_value(0)); - - ASSERT_EQ(m->repeated_int32_value_size(), 2); - EXPECT_EQ(m->repeated_int32_value(0), 0); - EXPECT_EQ(m->repeated_int32_value(1), -42); + EXPECT_THAT(m->repeated_bool_value(), ElementsAre(true)); + EXPECT_THAT(m->repeated_int32_value(), ElementsAre(0, -42)); + EXPECT_THAT(m->repeated_uint64_value(), ElementsAre(1, 2)); + EXPECT_THAT(m->repeated_double_value(), ElementsAre(1.5, -2)); + EXPECT_THAT(m->repeated_string_value(), ElementsAre("foo", "bar ", "")); + EXPECT_THAT(m->repeated_enum_value(), ElementsAre(proto3::BAR, proto3::FOO)); - ASSERT_EQ(m->repeated_uint64_value_size(), 2); - EXPECT_EQ(m->repeated_uint64_value(0), 1); - EXPECT_EQ(m->repeated_uint64_value(1), 2); - - ASSERT_EQ(m->repeated_double_value_size(), 2); - EXPECT_EQ(m->repeated_double_value(0), 1.5); - EXPECT_EQ(m->repeated_double_value(1), -2); - - ASSERT_EQ(m->repeated_string_value_size(), 3); - EXPECT_EQ(m->repeated_string_value(0), "foo"); - EXPECT_EQ(m->repeated_string_value(1), "bar "); - EXPECT_EQ(m->repeated_string_value(2), ""); - - ASSERT_EQ(m->repeated_enum_value_size(), 2); - EXPECT_EQ(m->repeated_enum_value(0), proto3::EnumType::BAR); - EXPECT_EQ(m->repeated_enum_value(1), proto3::EnumType::FOO); - - ASSERT_EQ(m->repeated_message_value_size(), 2); + ASSERT_THAT(m->repeated_message_value(), SizeIs(3)); EXPECT_EQ(m->repeated_message_value(0).value(), 40); - EXPECT_EQ(m->repeated_message_value(1).value(), 96); + EXPECT_EQ(m->repeated_message_value(1).value(), 0); + EXPECT_EQ(m->repeated_message_value(2).value(), 96); EXPECT_THAT( ToJson(*m), @@ -435,7 +474,7 @@ TEST_P(JsonTest, ParseMessage) { R"("messageValue":{"value":2048},"repeatedBoolValue":[true],"repeatedInt32Value":[0,-42])" R"(,"repeatedUint64Value":["1","2"],"repeatedDoubleValue":[1.5,-2],)" R"("repeatedStringValue":["foo","bar ",""],"repeatedEnumValue":["BAR","FOO"],)" - R"("repeatedMessageValue":[{"value":40},{"value":96}]})")); + R"("repeatedMessageValue":[{"value":40},{},{"value":96}]})")); } TEST_P(JsonTest, CurseOfAtob) { @@ -445,19 +484,18 @@ TEST_P(JsonTest, CurseOfAtob) { } )json"); ASSERT_OK(m); - EXPECT_EQ(m->repeated_bool_value_size(), 10); + EXPECT_THAT(m->repeated_bool_value(), + ElementsAre(false, true, false, true, false, true, false, true, + false, true)); +} - EXPECT_FALSE(m->repeated_bool_value(0)); - EXPECT_FALSE(m->repeated_bool_value(2)); - EXPECT_FALSE(m->repeated_bool_value(4)); - EXPECT_FALSE(m->repeated_bool_value(6)); - EXPECT_FALSE(m->repeated_bool_value(8)); +TEST_P(JsonTest, FloatPrecision) { + google::protobuf::Value v; + v.mutable_list_value()->add_values()->set_number_value(0.9900000095367432); + v.mutable_list_value()->add_values()->set_number_value(0.8799999952316284); - EXPECT_TRUE(m->repeated_bool_value(1)); - EXPECT_TRUE(m->repeated_bool_value(3)); - EXPECT_TRUE(m->repeated_bool_value(5)); - EXPECT_TRUE(m->repeated_bool_value(7)); - EXPECT_TRUE(m->repeated_bool_value(9)); + EXPECT_THAT(ToJson(v), + IsOkAndHolds("[0.99000000953674316,0.87999999523162842]")); } TEST_P(JsonTest, ParseLegacySingleRepeatedField) { @@ -469,16 +507,11 @@ TEST_P(JsonTest, ParseLegacySingleRepeatedField) { })json"); ASSERT_OK(m); - ASSERT_EQ(m->repeated_int32_value_size(), 1); - EXPECT_EQ(m->repeated_int32_value(0), 1997); - - ASSERT_EQ(m->repeated_string_value_size(), 1); - EXPECT_EQ(m->repeated_string_value(0), "oh no"); + EXPECT_THAT(m->repeated_int32_value(), ElementsAre(1997)); + EXPECT_THAT(m->repeated_string_value(), ElementsAre("oh no")); + EXPECT_THAT(m->repeated_enum_value(), ElementsAre(proto3::EnumType::BAR)); - ASSERT_EQ(m->repeated_enum_value_size(), 1); - EXPECT_EQ(m->repeated_enum_value(0), proto3::EnumType::BAR); - - ASSERT_EQ(m->repeated_message_value_size(), 1); + ASSERT_THAT(m->repeated_message_value(), SizeIs(1)); EXPECT_EQ(m->repeated_message_value(0).value(), -1); EXPECT_THAT(ToJson(*m), @@ -499,6 +532,15 @@ TEST_P(JsonTest, ParseMap) { EXPECT_EQ(other->DebugString(), message.DebugString()); } +TEST_P(JsonTest, RepeatedMapKey) { + EXPECT_THAT(ToProto(R"json({ + "string_map": { + "twiceKey": 0, + "twiceKey": 1 + } + })json"), StatusIs(util::StatusCode::kInvalidArgument)); +} + TEST_P(JsonTest, ParsePrimitiveMapIn) { MapIn message; JsonPrintOptions print_options; @@ -535,10 +577,43 @@ TEST_P(JsonTest, ParseOverOneof) { EXPECT_EQ(m.oneof_int32_value(), 5); } +TEST_P(JsonTest, RepeatedSingularKeys) { + auto m = ToProto(R"json({ + "int32Value": 1, + "int32Value": 2 + })json"); + EXPECT_OK(m); + EXPECT_EQ(m->int32_value(), 2); +} + +TEST_P(JsonTest, RepeatedRepeatedKeys) { + auto m = ToProto(R"json({ + "repeatedInt32Value": [1], + "repeatedInt32Value": [2, 3] + })json"); + EXPECT_OK(m); + EXPECT_THAT(m->repeated_int32_value(), ElementsAre(1, 2, 3)); +} + +TEST_P(JsonTest, RepeatedOneofKeys) { + EXPECT_THAT(ToProto(R"json({ + "oneofInt32Value": 1, + "oneofStringValue": "foo" + })json"), + StatusIs(util::StatusCode::kInvalidArgument)); +} + TEST_P(JsonTest, TestParseIgnoreUnknownFields) { JsonParseOptions options; options.ignore_unknown_fields = true; EXPECT_OK(ToProto(R"({"unknownName":0})", options)); + + TestMessage m; + m.GetReflection()->MutableUnknownFields(&m)->AddFixed32(9001, 9001); + m.GetReflection()->MutableUnknownFields(&m)->AddFixed64(9001, 9001); + m.GetReflection()->MutableUnknownFields(&m)->AddVarint(9001, 9001); + m.GetReflection()->MutableUnknownFields(&m)->AddLengthDelimited(9001, "9001"); + EXPECT_THAT(ToJson(m), IsOkAndHolds("{}")); } TEST_P(JsonTest, TestParseErrors) { @@ -578,9 +653,8 @@ TEST_P(JsonTest, TestDynamicMessage) { EXPECT_TRUE(generated.ParseFromString(message->SerializeAsString())); EXPECT_EQ(generated.int32_value(), 1024); - ASSERT_EQ(generated.repeated_int32_value_size(), 2); - EXPECT_EQ(generated.repeated_int32_value(0), 1); - EXPECT_EQ(generated.repeated_int32_value(1), 2); + EXPECT_THAT(generated.repeated_int32_value(), ElementsAre(1, 2)); + EXPECT_EQ(generated.message_value().value(), 2048); ASSERT_EQ(generated.repeated_message_value_size(), 2); EXPECT_EQ(generated.repeated_message_value(0).value(), 40); @@ -692,6 +766,69 @@ TEST_P(JsonTest, TestParsingNestedAnys) { R"("int32Value":5,"stringValue":"expected_value","messageValue":{"value":1}}}})")); } +TEST_P(JsonTest, TestParsingBrokenAny) { + auto m = ToProto(R"json( + { + "value": {} + } + )json"); + ASSERT_OK(m); + EXPECT_EQ(m->value().type_url(), ""); + EXPECT_EQ(m->value().value(), ""); + + EXPECT_THAT(ToProto(R"json( + { + "value": { + "type_url": "garbage", + "value": "bW9yZSBnYXJiYWdl" + } + } + )json"), + StatusIs(util::StatusCode::kInvalidArgument)); + + TestAny m2; + m2.mutable_value(); + EXPECT_THAT(ToJson(m2), IsOkAndHolds(R"({"value":{}})")); + m2.mutable_value()->set_value("garbage"); + // The ESF parser does not return InvalidArgument for this error. + EXPECT_THAT(ToJson(m2), Not(StatusIs(util::StatusCode::kOk))); + + m2.Clear(); + m2.mutable_value()->set_type_url("type.googleapis.com/proto3.TestMessage"); + EXPECT_THAT( + ToJson(m2), + IsOkAndHolds( + R"({"value":{"@type":"type.googleapis.com/proto3.TestMessage"}})")); +} + +TEST_P(JsonTest, TestFlatList) { + auto m = ToProto(R"json( + { + "repeatedInt32Value": [[[5]], [6]] + } + )json"); + ASSERT_OK(m); + EXPECT_THAT(m->repeated_int32_value(), ElementsAre(5, 6)); + + // The above flatteing behavior is suppressed for google::protobuf::ListValue. + auto m2 = ToProto(R"json( + { + "repeatedInt32Value": [[[5]], [6]] + } + )json"); + ASSERT_OK(m2); + auto fields = m2->struct_value().fields(); + auto list = fields["repeatedInt32Value"].list_value(); + EXPECT_EQ(list.values(0) + .list_value() + .values(0) + .list_value() + .values(0) + .number_value(), + 5); + EXPECT_EQ(list.values(1).list_value().values(0).number_value(), 6); +} + TEST_P(JsonTest, ParseWrappers) { auto m = ToProto(R"json( { @@ -749,6 +886,14 @@ TEST_P(JsonTest, TestParsingUnknownAnyFields) { EXPECT_EQ(t.string_value(), "expected_value"); } +TEST_P(JsonTest, TestHugeBareString) { + auto m = ToProto(R"json({ + "int64Value": 6009652459062546621 + })json"); + ASSERT_OK(m); + EXPECT_EQ(m->int64_value(), 6009652459062546621); +} + TEST_P(JsonTest, TestParsingUnknownEnumsProto2) { StringPiece input = R"json({"ayuLmao": "UNKNOWN_VALUE"})json"; @@ -825,11 +970,11 @@ TEST_P(JsonTest, TestParsingEnumCaseSensitive) { EXPECT_EQ(m.enum_value(), proto3::FOO); } - TEST_P(JsonTest, TestParsingEnumLowercase) { JsonParseOptions options; options.case_insensitive_enum_parsing = true; - auto m = ToProto(R"json({"enum_value": "TLSv1_2"})json", options); + auto m = + ToProto(R"json({"enum_value": "TLSv1_2"})json", options); ASSERT_OK(m); EXPECT_THAT(m->enum_value(), proto3::TLSv1_2); } @@ -844,6 +989,37 @@ TEST_P(JsonTest, TestParsingEnumIgnoreCase) { EXPECT_EQ(m.enum_value(), proto3::BAR); } +// This functionality is not correctly implemented by the ESF parser, so +// the test is only turned on when testing json2. +TEST_P(JsonTest, Extensions) { + if (GetParam() == Codec::kResolver || !IsJson2()) { + GTEST_SKIP(); + } + + auto m = ToProto(R"json({ + "[protobuf_unittest.TestMixedFieldsAndExtensions.c]": 42, + "a": 5, + "b": [1, 2, 3], + "[protobuf_unittest.TestMixedFieldsAndExtensions.d]": [1, 1, 2, 3, 5, 8, 13] + })json"); + ASSERT_OK(m); + EXPECT_EQ(m->a(), 5); + EXPECT_THAT(m->b(), ElementsAre(1, 2, 3)); + EXPECT_EQ(m->GetExtension(protobuf_unittest::TestMixedFieldsAndExtensions::c), + 42); + EXPECT_THAT( + m->GetRepeatedExtension(protobuf_unittest::TestMixedFieldsAndExtensions::d), + ElementsAre(1, 1, 2, 3, 5, 8, 13)); + + EXPECT_THAT( + ToJson(*m), + IsOkAndHolds( + R"({"a":5,)" + R"("[protobuf_unittest.TestMixedFieldsAndExtensions.c]":42,)" + R"("b":[1,2,3],)" + R"("[protobuf_unittest.TestMixedFieldsAndExtensions.d]":[1,1,2,3,5,8,13]})")); +} + // Parsing does NOT work like MergeFrom: existing repeated field values are // clobbered, not appended to. TEST_P(JsonTest, TestOverwriteRepeated) { @@ -851,10 +1027,7 @@ TEST_P(JsonTest, TestOverwriteRepeated) { m.add_repeated_int32_value(5); ASSERT_OK(ToProto(m, R"json({"repeated_int32_value": [1, 2, 3]})json")); - EXPECT_EQ(m.repeated_int32_value_size(), 3); - EXPECT_EQ(m.repeated_int32_value(0), 1); - EXPECT_EQ(m.repeated_int32_value(1), 2); - EXPECT_EQ(m.repeated_int32_value(2), 3); + EXPECT_THAT(m.repeated_int32_value(), ElementsAre(1, 2, 3)); } @@ -869,7 +1042,8 @@ TEST_P(JsonTest, TestDuration) { EXPECT_EQ(m->value().seconds(), 123456); EXPECT_EQ(m->value().nanos(), 789000000); - EXPECT_EQ(m->repeated_value().size(), 2); + + EXPECT_THAT(m->repeated_value(), SizeIs(2)); EXPECT_EQ(m->repeated_value(0).seconds(), 0); EXPECT_EQ(m->repeated_value(0).nanos(), 100000000); EXPECT_EQ(m->repeated_value(1).seconds(), 999); @@ -889,6 +1063,39 @@ TEST_P(JsonTest, TestDuration) { EXPECT_EQ(m2->value().seconds(), 4); EXPECT_EQ(m2->value().nanos(), 5); + + // Negative duration with zero seconds. + auto m3 = ToProto(R"json( + { + "value": {"nanos": -5}, + } + )json"); + ASSERT_OK(m3); + EXPECT_EQ(m3->value().seconds(), 0); + EXPECT_EQ(m3->value().nanos(), -5); + EXPECT_THAT(ToJson(m3->value()), IsOkAndHolds("\"-0.000000005s\"")); + + // Negative duration with zero nanos. + auto m4 = ToProto(R"json( + { + "value": {"seconds": -5}, + } + )json"); + ASSERT_OK(m4); + EXPECT_EQ(m4->value().seconds(), -5); + EXPECT_EQ(m4->value().nanos(), 0); + EXPECT_THAT(ToJson(m4->value()), IsOkAndHolds("\"-5s\"")); + + // Parse "0.5s" as a JSON string. + auto m5 = ToProto(R"json( + { + "value": "0.5s", + } + )json"); + ASSERT_OK(m5); + EXPECT_EQ(m5->value().seconds(), 0); + EXPECT_EQ(m5->value().nanos(), 500000000); + EXPECT_THAT(ToJson(m5->value()), IsOkAndHolds("\"0.500s\"")); } // These tests are not exhaustive; tests in //third_party/protobuf/conformance @@ -904,7 +1111,7 @@ TEST_P(JsonTest, TestTimestamp) { EXPECT_EQ(m->value().seconds(), 825422400); EXPECT_EQ(m->value().nanos(), 0); - EXPECT_EQ(m->repeated_value().size(), 1); + EXPECT_THAT(m->repeated_value(), SizeIs(1)); EXPECT_EQ(m->repeated_value(0).seconds(), 253402300799); EXPECT_EQ(m->repeated_value(0).nanos(), 0); @@ -954,10 +1161,7 @@ TEST_P(JsonTest, TestFieldMask) { )json"); ASSERT_OK(m); - EXPECT_EQ(m->value().paths_size(), 2); - EXPECT_EQ(m->value().paths(0), "foo"); - EXPECT_EQ(m->value().paths(1), "bar.baz_baz"); - + EXPECT_THAT(m->value().paths(), ElementsAre("foo", "bar.baz_baz")); EXPECT_THAT(ToJson(*m), IsOkAndHolds(R"({"value":"foo,bar.bazBaz"})")); auto m2 = ToProto(R"json( @@ -969,8 +1173,7 @@ TEST_P(JsonTest, TestFieldMask) { )json"); ASSERT_OK(m2); - EXPECT_EQ(m2->value().paths_size(), 1); - EXPECT_EQ(m2->value().paths(0), "yep.really"); + EXPECT_THAT(m2->value().paths(), ElementsAre("yep.really")); } TEST_P(JsonTest, TestLegalNullsInArray) { @@ -979,23 +1182,78 @@ TEST_P(JsonTest, TestLegalNullsInArray) { })json"); ASSERT_OK(m); - EXPECT_EQ(m->repeated_null_value_size(), 1); - EXPECT_EQ(m->repeated_null_value(0), google::protobuf::NULL_VALUE); + EXPECT_THAT(m->repeated_null_value(), + ElementsAre(google::protobuf::NULL_VALUE)); auto m2 = ToProto(R"json({ "repeatedValue": [null] })json"); ASSERT_OK(m2); - EXPECT_EQ(m2->repeated_value_size(), 1); + ASSERT_THAT(m2->repeated_value(), SizeIs(1)); EXPECT_TRUE(m2->repeated_value(0).has_null_value()); + + m2->Clear(); + m2->mutable_repeated_value(); // Materialize an empty singular Value. + m2->add_repeated_value(); + m2->add_repeated_value()->set_string_value("solitude"); + m2->add_repeated_value(); + EXPECT_THAT(ToJson(*m2), IsOkAndHolds(R"({"repeatedValue":["solitude"]})")); } -TEST_P(JsonTest, DISABLED_HtmlEscape) { +TEST_P(JsonTest, ListList) { + auto m = ToProto(R"json({ + "repeated_value": [["ayy", "lmao"]] + })json"); + ASSERT_OK(m); + + EXPECT_EQ(m->repeated_value(0).values(0).string_value(), "ayy"); + EXPECT_EQ(m->repeated_value(0).values(1).string_value(), "lmao"); + + m = ToProto(R"json({ + "repeated_value": [{ + "values": ["ayy", "lmao"] + }] + })json"); + ASSERT_OK(m); + + EXPECT_EQ(m->repeated_value(0).values(0).string_value(), "ayy"); + EXPECT_EQ(m->repeated_value(0).values(1).string_value(), "lmao"); +} + +TEST_P(JsonTest, HtmlEscape) { TestMessage m; m.set_string_value(""); EXPECT_THAT(ToJson(m), - IsOkAndHolds("{\"stringValue\":\"\\u003c/script\\u003e\"}")); + IsOkAndHolds(R"({"stringValue":"\u003c/script\u003e"})")); + + proto3::TestEvilJson m2; + JsonPrintOptions opts; + opts.always_print_primitive_fields = true; + EXPECT_THAT( + ToJson(m2, opts), + IsOkAndHolds( + R"({"regular_name":0,"\u003c/script\u003e":0,)" + R"("unbalanced\"quotes":0,)" + R"("\"\u003cscript\u003ealert('hello!);\u003c/script\u003e":0})")); +} + +TEST_P(JsonTest, FieldOrder) { + // $ protoscope -s <<< "3: 3 22: 2 1: 1 22: 2" + std::string out; + util::Status s = BinaryToJsonString( + resolver_.get(), "type.googleapis.com/proto3.TestMessage", + "\x18\x03\xb0\x01\x02\x08\x01\xb0\x01\x02", &out); + ASSERT_OK(s); + if (IsJson2()) { + EXPECT_EQ( + out, + R"({"boolValue":true,"int64Value":"3","repeatedInt32Value":[2,2]})"); + } else { + EXPECT_EQ( + out, + R"({"int64Value":"3","repeatedInt32Value":[2],"boolValue":true,"repeatedInt32Value":[2]})"); + } } } // namespace diff --git a/src/google/protobuf/util/message_differencer.h b/src/google/protobuf/util/message_differencer.h index 4df31524ba9f..c8291c119fa7 100644 --- a/src/google/protobuf/util/message_differencer.h +++ b/src/google/protobuf/util/message_differencer.h @@ -242,17 +242,17 @@ class PROTOBUF_EXPORT MessageDifferencer { // Reports that a field has been added into Message2. virtual void ReportAdded(const Message& message1, const Message& message2, - const std::vector& field_path) = 0; + const std::vector& field_path) {} // Reports that a field has been deleted from Message1. virtual void ReportDeleted( const Message& message1, const Message& message2, - const std::vector& field_path) = 0; + const std::vector& field_path) {} // Reports that the value of a field has been modified. virtual void ReportModified( const Message& message1, const Message& message2, - const std::vector& field_path) = 0; + const std::vector& field_path) {} // Reports that a repeated field has been moved to another location. This // only applies when using TreatAsSet or TreatAsMap() -- see below. Also diff --git a/src/google/protobuf/wire_format_lite.h b/src/google/protobuf/wire_format_lite.h index a7e64bf1e4fb..8f38f7c8d0bc 100644 --- a/src/google/protobuf/wire_format_lite.h +++ b/src/google/protobuf/wire_format_lite.h @@ -53,7 +53,6 @@ #include #include -// Do UTF-8 validation on string type in Debug build only #ifndef NDEBUG #define GOOGLE_PROTOBUF_UTF8_VALIDATION_ENABLED #endif diff --git a/src/google/protobuf/wrappers.pb.h b/src/google/protobuf/wrappers.pb.h index 72304a56d199..f629e5e8e2fe 100644 --- a/src/google/protobuf/wrappers.pb.h +++ b/src/google/protobuf/wrappers.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3021002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/tests.sh b/tests.sh index f2b481cc785b..65086740310f 100755 --- a/tests.sh +++ b/tests.sh @@ -45,44 +45,6 @@ build_cpp_tcmalloc() { PPROF_PATH=/usr/bin/google-pprof HEAPCHECK=strict ./protobuf-test } -build_cpp_distcheck() { - grep -q -- "-Og" src/Makefile.am && - echo "The -Og flag is incompatible with Clang versions older than 4.0." && - exit 1 - - # Initialize any submodules. - git submodule update --init --recursive - ./autogen.sh - ./configure - make dist - - # List all files that should be included in the distribution package. - git ls-files | grep "^\(java\|python\|objectivec\|csharp\|ruby\|php\|cmake\|examples\|src/google/protobuf/.*\.proto\)" |\ - grep -v ".gitignore" | grep -v "java/lite/proguard.pgcfg" |\ - grep -v "python/compatibility_tests" | grep -v "python/docs" | grep -v "python/.repo-metadata.json" |\ - grep -v "python/protobuf_distutils" | grep -v "csharp/compatibility_tests" > dist.lst - # Unzip the dist tar file. - DIST=`ls *.tar.gz` - tar -xf $DIST - cd ${DIST//.tar.gz} - # Check if every file exists in the dist tar file. - FILES_MISSING="" - for FILE in $(<../dist.lst); do - [ -f "$FILE" ] || { - echo "$FILE is not found!" - FILES_MISSING="$FILE $FILES_MISSING" - } - done - cd .. - if [ ! -z "$FILES_MISSING" ]; then - echo "Missing files in EXTRA_DIST: $FILES_MISSING" - exit 1 - fi - - # Do the regular dist-check for C++. - make distcheck -j$(nproc) -} - build_dist_install() { # Create a symlink pointing to python2 and put it at the beginning of $PATH. # This is necessary because the googletest build system involves a Python @@ -490,47 +452,8 @@ build_php_c() { test_php_c } -build_php7.0_mac() { - internal_build_cpp - # Install PHP - curl -s https://php-osx.liip.ch/install.sh | bash -s 7.0 - PHP_FOLDER=`find /usr/local -type d -name "php5-7.0*"` # The folder name may change upon time - test ! -z "$PHP_FOLDER" - export PATH="$PHP_FOLDER/bin:$PATH" - - # Install Composer - wget https://getcomposer.org/download/2.0.13/composer.phar --progress=dot:mega -O /usr/local/bin/composer - chmod a+x /usr/local/bin/composer - - # Install valgrind - echo "#! /bin/bash" > valgrind - chmod ug+x valgrind - sudo mv valgrind /usr/local/bin/valgrind - - # Test - test_php_c -} - -build_php7.3_mac() { +build_php_mac() { internal_build_cpp - # Install PHP - # We can't test PHP 7.4 with these binaries yet: - # https://github.com/liip/php-osx/issues/276 - curl -s https://php-osx.liip.ch/install.sh | bash -s 7.3 - PHP_FOLDER=`find /usr/local -type d -name "php5-7.3*"` # The folder name may change upon time - test ! -z "$PHP_FOLDER" - export PATH="$PHP_FOLDER/bin:$PATH" - - # Install Composer - wget https://getcomposer.org/download/2.0.13/composer.phar --progress=dot:mega -O /usr/local/bin/composer - chmod a+x /usr/local/bin/composer - - # Install valgrind - echo "#! /bin/bash" > valgrind - chmod ug+x valgrind - sudo mv valgrind /usr/local/bin/valgrind - - # Test test_php_c } @@ -578,7 +501,6 @@ build_benchmark() { if [ "$#" -ne 1 ]; then echo " Usage: $0 { cpp | - cpp_distcheck | csharp | java_jdk7 | java_oracle7 | diff --git a/version.json b/version.json index e50d63326187..42441278106b 100644 --- a/version.json +++ b/version.json @@ -1,17 +1,17 @@ { "main": { - "protoc_version": "21-dev", + "protoc_version": "22-dev", "lts": false, - "date": "2022-04-22", + "date": "2022-07-21", "languages": { - "cpp": "3.21-dev", - "csharp": "3.21-dev", - "java": "3.21-dev", - "javascript": "3.21-dev", - "objectivec": "3.21-dev", - "php": "3.21-dev", - "python": "4.21-dev", - "ruby": "3.21-dev" + "cpp": "3.22-dev", + "csharp": "3.22-dev", + "java": "3.22-dev", + "javascript": "3.22-dev", + "objectivec": "3.22-dev", + "php": "3.22-dev", + "python": "4.22-dev", + "ruby": "3.22-dev" } } }