Release prototype of Protobuf Editions.

This represents the future direction of protobuf, replacing proto2/proto3 syntax with editions. These will enable more incremental evolution of protobuf APIs through features, which are individual behaviors (such as whether field presence is explicit or implicit). For more details see https://protobuf.dev/editions/overview/. This PR contains a working implementation of editions for the protoc frontend and C++ code generation, along with various infrastructure improvements to support it. It gives early access for anyone who wants to a preview of editions, but has no effect on proto2/proto3 syntax. It is flag-guarded behind the `--experimental_editions` flag, and is an experimental feature with no guarantees. PiperOrigin-RevId: 544805690
protocolbuffers · Jul 1, 2023 · 4f9e417 · 4f9e417
1 parent bd589d9
commit 4f9e417
Show file tree

Hide file tree

Showing 82 changed files with 15,562 additions and 1,658 deletions.
diff --git a/BUILD.bazel b/BUILD.bazel
@@ -103,6 +103,12 @@ alias(
 # Built-in runtime protos: these are part of protobuf's internal
 # implementation, but are not Well-Known Types.
 
+alias(
+    name = "cpp_features_proto",
+    actual = "//src/google/protobuf:cpp_features_proto",  # proto_library
+    visibility = ["//visibility:public"],
+)
+
 alias(
     name = "descriptor_proto",
     actual = "//src/google/protobuf:descriptor_proto",  # proto_library

diff --git a/cmake/install.cmake b/cmake/install.cmake
@@ -65,6 +65,7 @@ set(protobuf_HEADERS
   ${libprotobuf_hdrs}
   ${libprotoc_hdrs}
   ${wkt_protos_files}
+  ${cpp_features_proto_proto_srcs}
   ${descriptor_proto_proto_srcs}
   ${plugin_proto_proto_srcs}
 )

diff --git a/cmake/tests.cmake b/cmake/tests.cmake
@@ -172,6 +172,7 @@ file(GLOB_RECURSE _local_hdrs
 
 # Exclude the bootstrapping that are directly used by tests.
 set(_exclude_hdrs
+  "${protobuf_SOURCE_DIR}/src/google/protobuf/cpp_features.pb.h"
   "${protobuf_SOURCE_DIR}/src/google/protobuf/descriptor.pb.h"
   "${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/plugin.pb.h")
 

diff --git a/generate_descriptor_proto.sh b/generate_descriptor_proto.sh
@@ -23,6 +23,7 @@ cd src
 declare -a RUNTIME_PROTO_FILES=(\
   google/protobuf/any.proto \
   google/protobuf/api.proto \
+  google/protobuf/cpp_features.proto \
   google/protobuf/descriptor.proto \
   google/protobuf/duration.proto \
   google/protobuf/empty.proto \

diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c
diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h
diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel
@@ -110,6 +110,7 @@ gen_file_lists(
         ":protoc": "libprotoc",
         # Protos:
         "//src/google/protobuf:well_known_type_protos": "wkt_protos",
+        "//src/google/protobuf:cpp_features_proto": "cpp_features_proto",
         "//src/google/protobuf:descriptor_proto": "descriptor_proto",
         "//src/google/protobuf/compiler:plugin_proto": "plugin_proto",
         # Test libraries:

diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c
diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h
diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel
@@ -109,6 +109,7 @@ WELL_KNOWN_TYPES = [
 
 proto_library(
     name = "wkt_proto",
+    strip_import_prefix = "/src",
     visibility = ["//visibility:private"],
     deps = [wkt + "_proto" for wkt in WELL_KNOWN_TYPES],
 )
@@ -177,6 +178,14 @@ proto_library(
     ],
 )
 
+proto_library(
+    name = "cpp_features_proto",
+    srcs = ["cpp_features.proto"],
+    strip_import_prefix = "/src",
+    visibility = ["//:__subpackages__"],
+    deps = [":descriptor_proto"],
+)
+
 ################################################################################
 # C++ Runtime Library
 ################################################################################
@@ -419,11 +428,13 @@ cc_library(
     name = "protobuf_nowkt",
     srcs = [
         "any.cc",
+        "cpp_features.pb.cc",
         "descriptor.cc",
         "descriptor.pb.cc",
         "descriptor_database.cc",
         "dynamic_message.cc",
         "extension_set_heavy.cc",
+        "feature_resolver.cc",
         "generated_message_bases.cc",
         "generated_message_reflection.cc",
         "generated_message_tctable_full.cc",
@@ -439,12 +450,14 @@ cc_library(
         "wire_format.cc",
     ],
     hdrs = [
+        "cpp_features.pb.h",
         "descriptor.h",
         "descriptor.pb.h",
         "descriptor_database.h",
         "descriptor_legacy.h",
         "descriptor_visitor.h",
         "dynamic_message.h",
+        "feature_resolver.h",
         "field_access_listener.h",
         "generated_enum_reflection.h",
         "generated_message_bases.h",
@@ -574,6 +587,12 @@ filegroup(
     visibility = ["//:__subpackages__"],
 )
 
+filegroup(
+    name = "cpp_features_proto_srcs",
+    srcs = ["cpp_features.proto"],
+    visibility = ["//src/google/protobuf/compiler/cpp:__pkg__"],
+)
+
 filegroup(
     name = "testdata",
     srcs = glob(["testdata/**/*"]) + [
@@ -630,8 +649,10 @@ filegroup(
         "unittest_embed_optimize_for.proto",
         "unittest_empty.proto",
         "unittest_enormous_descriptor.proto",
+        "unittest_features.proto",
         "unittest_import.proto",
         "unittest_import_public.proto",
+        "unittest_invalid_features.proto",
         "unittest_lazy_dependencies.proto",
         "unittest_lazy_dependencies_custom_option.proto",
         "unittest_lazy_dependencies_enum.proto",
@@ -793,6 +814,16 @@ filegroup(
     visibility = ["//src/google/protobuf/compiler/cpp:__pkg__"],
 )
 
+filegroup(
+    name = "cpp_features_cc_srcs",
+    testonly = 1,
+    data = [
+        "cpp_features.pb.cc",
+        "cpp_features.pb.h",
+    ],
+    visibility = ["//src/google/protobuf/compiler/cpp:__pkg__"],
+)
+
 cc_library(
     name = "lite_test_util",
     testonly = 1,
@@ -981,7 +1012,10 @@ cc_library(
     testonly = True,
     hdrs = ["test_textproto.h"],
     strip_include_prefix = "/src",
-    visibility = ["//pkg:__pkg__"],
+    visibility = [
+        "//pkg:__pkg__",
+        "//src/google/protobuf:__subpackages__",
+    ],
     deps = [
         ":protobuf",
         "@com_google_absl//absl/log:absl_check",
@@ -1044,6 +1078,23 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "feature_resolver_test",
+    srcs = ["feature_resolver_test.cc"],
+    copts = COPTS,
+    deps = [
+        ":cc_test_protos",
+        ":protobuf",
+        ":test_textproto",
+        ":test_util",
+        "//src/google/protobuf/compiler:importer",
+        "//src/google/protobuf/stubs",
+        "//src/google/protobuf/testing",
+        "@com_google_googletest//:gtest",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
 cc_test(
     name = "generated_message_reflection_unittest",
     srcs = ["generated_message_reflection_unittest.cc"],

diff --git a/src/google/protobuf/bridge/BUILD.bazel b/src/google/protobuf/bridge/BUILD.bazel
@@ -8,4 +8,4 @@ proto_library(
     name = "message_set_proto",
     srcs = ["message_set.proto"],
     strip_import_prefix = "/src",
-)
+)
diff --git a/src/google/protobuf/compiler/BUILD.bazel b/src/google/protobuf/compiler/BUILD.bazel
@@ -294,8 +294,8 @@ cc_library(
     copts = COPTS,
     strip_include_prefix = "/src",
     visibility = [
-      "//pkg:__pkg__",
-      "//src/google/protobuf/compiler:__subpackages__",
+        "//pkg:__pkg__",
+        "//src/google/protobuf/compiler:__subpackages__",
     ],
     deps = [
         ":code_generator",

diff --git a/src/google/protobuf/compiler/allowlists/editions.cc b/src/google/protobuf/compiler/allowlists/editions.cc
@@ -39,9 +39,12 @@ namespace compiler {
 // NOTE: These files have early default access to go/editions.  The protoc flag
 // `--experimental_editions` can also be used to enable editions.
 
-static constexpr auto kEarlyEditionsFile = internal::MakeAllowlist({
+static constexpr auto kEarlyEditionsFile = internal::MakeAllowlist(
+    {
 // Intentionally left blank.
-});
+        "google/protobuf/editions/",
+    },
+    internal::AllowlistFlags::kMatchPrefix);
 
 bool IsEarlyEditionsFile(absl::string_view file) {
   return kEarlyEditionsFile.Allows(file);

diff --git a/src/google/protobuf/compiler/code_generator.h b/src/google/protobuf/compiler/code_generator.h
@@ -126,6 +126,24 @@ class PROTOC_EXPORT CodeGenerator {
   // method can be removed.
   virtual bool HasGenerateAll() const { return true; }
 
+#ifdef PROTOBUF_FUTURE_EDITIONS
+
+ protected:
+  // Retrieves the resolved source features for a given descriptor.  These
+  // should be used to make any feature-based decisions during code generation.
+  template <typename DescriptorT>
+  static const FeatureSet& GetSourceFeatures(const DescriptorT& desc) {
+    return ::google::protobuf::internal::InternalFeatureHelper::GetFeatures(desc);
+  }
+
+  // Retrieves the raw source features for a given descriptor.  These should be
+  // used to validate the original .proto file and make any decisions about it
+  // during code generation.
+  template <typename DescriptorT>
+  static const FeatureSet& GetSourceRawFeatures(const DescriptorT& desc) {
+    return ::google::protobuf::internal::InternalFeatureHelper::GetRawFeatures(desc);
+  }
+#endif  // PROTOBUF_FUTURE_EDITIONS
 };
 
 // CodeGenerators generate one or more files in a given directory.  This

diff --git a/src/google/protobuf/compiler/command_line_interface.cc b/src/google/protobuf/compiler/command_line_interface.cc
@@ -1551,6 +1551,21 @@ bool CommandLineInterface::ParseInputFiles(
     }
     parsed_files->push_back(parsed_file);
 
+#ifdef PROTOBUF_FUTURE_EDITIONS
+    if (!experimental_editions_ && !IsEarlyEditionsFile(parsed_file->name())) {
+      if (FileDescriptorLegacy(parsed_file).syntax() ==
+          FileDescriptorLegacy::Syntax::SYNTAX_EDITIONS) {
+        std::cerr
+            << parsed_file->name()
+            << ": This file uses editions, but --experimental_editions has not "
+               "been enabled. This syntax is experimental and should be "
+               "avoided."
+            << std::endl;
+        result = false;
+        break;
+      }
+    }
+#endif  // PROTOBUF_FUTURE_EDITIONS
 
     // Enforce --disallow_services.
     if (disallow_services_ && parsed_file->service_count() > 0) {
@@ -1601,6 +1616,9 @@ void CommandLineInterface::Clear() {
   descriptor_set_out_name_.clear();
   dependency_out_name_.clear();
 
+#ifdef PROTOBUF_FUTURE_EDITIONS
+  experimental_editions_ = false;
+#endif  // PROTOBUF_FUTURE_EDITIONS
 
   mode_ = MODE_COMPILE;
   print_mode_ = PRINT_NONE;
@@ -1919,6 +1937,9 @@ bool CommandLineInterface::ParseArgument(const char* arg, std::string* name,
       *name == "--include_imports" || *name == "--include_source_info" ||
       *name == "--retain_options" || *name == "--version" ||
       *name == "--decode_raw" ||
+#ifdef PROTOBUF_FUTURE_EDITIONS
+      *name == "--experimental_editions" ||
+#endif  // PROTOBUF_FUTURE_EDITIONS
       *name == "--print_free_field_numbers" ||
       *name == "--experimental_allow_proto3_optional" ||
       *name == "--deterministic_output" || *name == "--fatal_warnings") {
@@ -2245,6 +2266,14 @@ CommandLineInterface::InterpretArgument(const std::string& name,
 #else
     ::setenv(io::Printer::kProtocCodegenTrace.data(), "yes", 0);
 #endif
+#ifdef PROTOBUF_FUTURE_EDITIONS
+  } else if (name == "--experimental_editions") {
+    // If you're reading this, you're probably wondering what
+    // --experimental_editions is for and thinking of turning it on. This is an
+    // experimental, undocumented, unsupported flag. Enable it at your own risk
+    // (or, just don't!).
+    experimental_editions_ = true;
+#endif  // PROTOBUF_FUTURE_EDITIONS
   } else {
     // Some other flag.  Look it up in the generators list.
     const GeneratorInfo* generator_info = FindGeneratorByFlag(name);

diff --git a/src/google/protobuf/compiler/command_line_interface.h b/src/google/protobuf/compiler/command_line_interface.h
@@ -428,6 +428,9 @@ class PROTOC_EXPORT CommandLineInterface {
   // dependency file will be written. Otherwise, empty.
   std::string dependency_out_name_;
 
+#ifdef PROTOBUF_FUTURE_EDITIONS
+  bool experimental_editions_ = false;
+#endif  // PROTOBUF_FUTURE_EDITIONS
 
   // True if --include_imports was given, meaning that we should
   // write all transitive dependencies to the DescriptorSet.  Otherwise, only