Merge remote-tracking branch 'upstream/main' into gro

.bazelrc

@@ -509,6 +509,7 @@ build:rbe-engflow --grpc_keepalive_time=30s
 build:rbe-engflow --remote_timeout=3600s
 build:rbe-engflow --bes_timeout=3600s
 build:rbe-engflow --bes_upload_mode=fully_async
+build:rbe-engflow --nolegacy_important_outputs
 
 build:cache-envoy-engflow --google_default_credentials=false
 build:cache-envoy-engflow --remote_cache=grpcs://morganite.cluster.engflow.com

.gitattributes

@@ -3,8 +3,8 @@
 /generated_api_shadow/envoy/** linguist-generated=true
 /generated_api_shadow/bazel/** linguist-generated=true
 *.svg binary
-/test/extensions/transport_sockets/tls/test_data/aes_128_key binary
-/test/extensions/transport_sockets/tls/test_data/ticket_key_* binary
+/test/common/tls/test_data/aes_128_key binary
+/test/common/tls/test_data/ticket_key_* binary
 /test/**/*_corpus/* linguist-generated=true
 requirements.txt binary
 package.lock binary

api/envoy/config/trace/v3/zipkin.proto

@@ -82,5 +82,10 @@ message ZipkinConfig {
   //   If this is set to true, then the
   //   :ref:`start_child_span of router <envoy_v3_api_field_extensions.filters.http.router.v3.Router.start_child_span>`
   //   SHOULD be set to true also to ensure the correctness of trace chain.
-  bool split_spans_for_request = 7;
+  //
+  //   Both this field and ``start_child_span`` are deprecated by the
+  //   :ref:`spawn_upstream_span <envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.Tracing.spawn_upstream_span>`.
+  //   Please use that ``spawn_upstream_span`` field to control the span creation.
+  bool split_spans_for_request = 7
+      [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"];
 }

api/envoy/extensions/filters/http/jwt_authn/v3/config.proto

@@ -311,6 +311,7 @@ message JwtProvider {
   // Add JWT claim to HTTP Header
   // Specify the claim name you want to copy in which HTTP header. For examples, following config:
   // The claim must be of type; string, int, double, bool. Array type claims are not supported
+  //
   // .. code-block:: yaml
   //
   //   claim_to_headers:

changelogs/current.yaml

@@ -51,6 +51,10 @@ minor_behavior_changes:
     Port migration is default turned off. QUIC client connections will no longer attempt to migrate to a new port when connections
     is degrading. Can be manually turned on via
     :ref:`port_migration <envoy_v3_api_field_config.core.v3.QuicProtocolOptions.num_timeouts_to_trigger_port_migration>`.
+- area: QUIC
+  change: |
+    Make each upstream connection to read as many as 32 packets in each event loop. This feature can be disabled by setting
+    ``envoy.reloadable_features.quic_upstream_reads_fixed_number_packets`` to false.
 - area: aws
   change: |
     AWS region string is now retrieved from environment and profile consistently within aws_request_signer and
@@ -73,6 +77,10 @@ minor_behavior_changes:
 
 bug_fixes:
 # *Changes expected to improve the state of the world and are unlikely to have negative effects*
+- area: http3_upstream
+  change: |
+    Fixing a bug with HTTP/3 upstream using a non-threadsafe cache cross-thread. Bumping HTTP/3 support down
+    to alpha as the severity of this bug indicates it is both not in use and not GA quality code.
 - area: tracers
   change: |
     use unary RPC calls for OpenTelemetry trace exports, rather than client-side streaming connections.
@@ -202,6 +210,11 @@ new_features:
     Added QUIC protocol option :ref:`send_disable_active_migration
     <envoy_v3_api_field_config.listener.v3.QuicProtocolOptions.send_disable_active_migration>` to make the server send clients a transport
     parameter to discourage client endpoints from active migration.
+- area: quic
+  change: |
+    Added support for QUIC listeners to :ref:`choose certificates based on SNI <arch_overview_ssl_cert_select>` and load certificates
+    from formats other than PEM, such as :ref:`pkcs12 <envoy_v3_api_field_extensions.transport_sockets.tls.v3.TlsCertificate.pkcs12>`.
+    This behavior can be disabled with runtime flag ``envoy.restart_features.quic_handle_certs_with_shared_tls_code``.
 - area: ext_proc
   change: |
     implemented
@@ -269,9 +282,17 @@ new_features:
 - area: tracing
   change: |
     Added support to configure a Dynatrace sampler for the OpenTelemetry tracer.
+- area: tracing
+  change: |
+    Added User-Agent header to OTLP trace exporters according to the OpenTelemetry specification.
 
 deprecated:
 - area: listener
   change: |
     deprecated runtime key ``overload.global_downstream_max_connections`` in favor of :ref:`downstream connections monitor
     <envoy_v3_api_msg_extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig>`.
+- area: tracing
+  change: |
+    deprecated :ref:`split_spans_for_request <envoy_v3_api_field_config.trace.v3.ZipkinConfig.split_spans_for_request>`
+    in favor of :ref:`spawn_upstream_span
+    <envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.Tracing.spawn_upstream_span>`.

contrib/cryptomb/private_key_providers/test/BUILD

@@ -52,7 +52,7 @@ envoy_cc_test(
     deps = [
         ":test_fake_factory",
         "//source/common/common:random_generator_lib",
-        "//source/extensions/transport_sockets/tls/private_key:private_key_manager_lib",
+        "//source/common/tls/private_key:private_key_manager_lib",
         "//test/mocks/runtime:runtime_mocks",
         "//test/mocks/server:server_mocks",
         "//test/mocks/ssl:ssl_mocks",
@@ -75,7 +75,7 @@ envoy_cc_test(
     ],
     deps = [
         ":test_fake_factory",
-        "//source/extensions/transport_sockets/tls/private_key:private_key_manager_lib",
+        "//source/common/tls/private_key:private_key_manager_lib",
         "//test/mocks/stats:stats_mocks",
         "//test/test_common:environment_lib",
         "//test/test_common:simulated_time_system_lib",

contrib/cryptomb/private_key_providers/test/config_test.cc

@@ -1,7 +1,7 @@
 #include <string>
 
 #include "source/common/common/random_generator.h"
-#include "source/extensions/transport_sockets/tls/private_key/private_key_manager_impl.h"
+#include "source/common/tls/private_key/private_key_manager_impl.h"
 
 #include "test/common/stats/stat_test_utility.h"
 #include "test/mocks/common.h"

contrib/cryptomb/private_key_providers/test/ops_test.cc

@@ -3,7 +3,7 @@
 #include <string>
 #include <vector>
 
-#include "source/extensions/transport_sockets/tls/private_key/private_key_manager_impl.h"
+#include "source/common/tls/private_key/private_key_manager_impl.h"
 
 #include "test/common/stats/stat_test_utility.h"
 #include "test/test_common/environment.h"

contrib/golang/filters/http/test/test_data/buffer/go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	github.com/envoyproxy/envoy v1.24.0
-	google.golang.org/protobuf v1.32.0
+	google.golang.org/protobuf v1.33.0
 )
 
 replace github.com/envoyproxy/envoy => ../../../../../../../

contrib/golang/filters/http/test/test_data/echo/go.mod

@@ -15,7 +15,7 @@ require (
 require (
 	github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	google.golang.org/protobuf v1.32.0
+	google.golang.org/protobuf v1.33.0
 )
 
 replace github.com/envoyproxy/envoy => ../../../../../../../

contrib/golang/filters/http/test/test_data/property/go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	github.com/envoyproxy/envoy v1.24.0
-	google.golang.org/protobuf v1.32.0
+	google.golang.org/protobuf v1.33.0
 )
 
 replace github.com/envoyproxy/envoy => ../../../../../../../

contrib/golang/filters/http/test/test_data/routeconfig/go.mod

@@ -15,7 +15,7 @@ require (
 require (
 	github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	google.golang.org/protobuf v1.32.0
+	google.golang.org/protobuf v1.33.0
 )
 
 replace github.com/envoyproxy/envoy => ../../../../../../../

contrib/golang/router/cluster_specifier/test/test_data/simple/go.mod

@@ -15,7 +15,7 @@ require (
 require (
 	github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	google.golang.org/protobuf v1.32.0
+	google.golang.org/protobuf v1.33.0
 )
 
 replace github.com/envoyproxy/envoy => ../../../../../../../

contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc

@@ -1,7 +1,7 @@
 #include "source/common/network/connection_impl.h"
+#include "source/common/tls/context_config_impl.h"
+#include "source/common/tls/ssl_socket.h"
 #include "source/extensions/filters/network/common/factory_base.h"
-#include "source/extensions/transport_sockets/tls/context_config_impl.h"
-#include "source/extensions/transport_sockets/tls/ssl_socket.h"
 
 #include "test/integration/fake_upstream.h"
 #include "test/integration/integration.h"

contrib/qat/private_key_providers/test/BUILD

@@ -51,7 +51,7 @@ envoy_cc_test(
     deps = [
         ":test_fake_factory",
         "//source/common/common:random_generator_lib",
-        "//source/extensions/transport_sockets/tls/private_key:private_key_manager_lib",
+        "//source/common/tls/private_key:private_key_manager_lib",
         "//test/mocks/runtime:runtime_mocks",
         "//test/mocks/server:server_mocks",
         "//test/mocks/ssl:ssl_mocks",
@@ -72,7 +72,7 @@ envoy_cc_test(
     ],
     deps = [
         ":test_fake_factory",
-        "//source/extensions/transport_sockets/tls/private_key:private_key_manager_lib",
+        "//source/common/tls/private_key:private_key_manager_lib",
         "//test/mocks/server:server_mocks",
         "//test/test_common:environment_lib",
         "//test/test_common:simulated_time_system_lib",

contrib/qat/private_key_providers/test/config_test.cc

@@ -1,7 +1,7 @@
 #include <string>
 
 #include "source/common/common/random_generator.h"
-#include "source/extensions/transport_sockets/tls/private_key/private_key_manager_impl.h"
+#include "source/common/tls/private_key/private_key_manager_impl.h"
 
 #include "test/common/stats/stat_test_utility.h"
 #include "test/mocks/common.h"

contrib/qat/private_key_providers/test/ops_test.cc

@@ -3,7 +3,7 @@
 #include <string>
 #include <vector>
 
-#include "source/extensions/transport_sockets/tls/private_key/private_key_manager_impl.h"
+#include "source/common/tls/private_key/private_key_manager_impl.h"
 
 #include "test/common/stats/stat_test_utility.h"
 #include "test/mocks/server/transport_socket_factory_context.h"

docs/root/configuration/http/http_conn_man/route_matching.rst

@@ -7,8 +7,14 @@ When Envoy matches a route, it uses the following procedure:
 
 #. The HTTP request's *host* or *:authority* header is matched to a :ref:`virtual host
    <envoy_v3_api_msg_config.route.v3.VirtualHost>`.
-#. Each :ref:`route entry <envoy_v3_api_msg_config.route.v3.Route>` in the virtual host is checked,
-   *in order*. If there is a match, the route is used and no further route checks are made.
+#. One of:
+
+   - Each :ref:`route entry <envoy_v3_api_msg_config.route.v3.Route>` in the virtual host is
+     checked, *in order*. If there is a match, the route is used and no further route checks are made.
+   - The :ref:`matcher entry <envoy_v3_api_msg_.xds.type.matcher.v3.Matcher.MatcherTree>` in the
+     virtual host is used to match a route. If there are many routes this will typically be more
+     efficient than the linear search of ``route``.
+
 #. Independently, each :ref:`virtual cluster <envoy_v3_api_msg_config.route.v3.VirtualCluster>` in the
    virtual host is checked, *in order*. If there is a match, the virtual cluster is used and no
    further virtual cluster checks are made.

docs/root/intro/arch_overview/http/http3.rst

@@ -8,8 +8,7 @@ HTTP/3 overview
   While HTTP/3 **downstream support is deemed ready for production use**, improvements are ongoing,
   tracked in the `area-quic <https://github.com/envoyproxy/envoy/labels/area%2Fquic>`_ tag.
 
-  HTTP/3 **upstream support is fine for locally controlled networks**, but is alpha for
-  general internet use - key features are implemented but have not been tested at scale.
+  HTTP/3 upstream support is alpha - key features are implemented but have not been tested at scale.
 
 .. _arch_overview_http3_downstream:
 

docs/root/intro/arch_overview/security/ssl.rst

@@ -97,8 +97,6 @@ See the reference for :ref:`UpstreamTlsContexts <envoy_v3_api_msg_extensions.tra
   subject name, hash, etc. Other validation context configuration is typically required depending
   on the deployment.
 
-.. _arch_overview_ssl_cert_select:
-
 Custom Certificate Validator
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -110,6 +108,8 @@ For example, Envoy can be configured to verify peer certificates following the `
 with multiple trust bundles in a single listener or cluster.
 For more detail, please refer to :ref:`the documentation of custom_validator_config field<envoy_v3_api_field_extensions.transport_sockets.tls.v3.CertificateValidationContext.custom_validator_config>`.
 
+.. _arch_overview_ssl_cert_select:
+
 Certificate selection
 ---------------------
 

envoy/network/BUILD

@@ -58,6 +58,12 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "parent_drained_callback_registrar_interface",
+    hdrs = ["parent_drained_callback_registrar.h"],
+    deps = [":address_interface"],
+)
+
 envoy_cc_library(
     name = "udp_packet_writer_handler_interface",
     hdrs = ["udp_packet_writer_handler.h"],

envoy/network/parent_drained_callback_registrar.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "envoy/network/address.h"
+
+#include "absl/functional/any_invocable.h"
+
+namespace Envoy {
+namespace Network {
+
+/**
+ * An interface through which a UDP listen socket, especially a QUIC socket, can
+ * postpone reading during hot restart until the parent instance is drained.
+ */
+class ParentDrainedCallbackRegistrar {
+public:
+  /**
+   * @param address is the address of the listener.
+   * @param callback the function to call when the listener matching address is
+   *                 drained on the parent instance.
+   */
+  virtual void registerParentDrainedCallback(const Address::InstanceConstSharedPtr& address,
+                                             absl::AnyInvocable<void()> callback) PURE;
+
+protected:
+  virtual ~ParentDrainedCallbackRegistrar() = default;
+};
+
+} // namespace Network
+} // namespace Envoy

envoy/network/socket.h

@@ -542,6 +542,13 @@ class Socket {
    * @return the socket options stored earlier with addOption() and addOptions() calls, if any.
    */
   virtual const OptionsSharedPtr& options() const PURE;
+
+  /**
+   * @return a ParentDrainedCallbackRegistrar for UDP listen sockets during hot restart.
+   */
+  virtual OptRef<class ParentDrainedCallbackRegistrar> parentDrainedCallbackRegistrar() const {
+    return absl::nullopt;
+  }
 };
 
 using SocketPtr = std::unique_ptr<Socket>;

envoy/server/hot_restart.h

@@ -62,6 +62,17 @@ class HotRestart {
   virtual void
   registerUdpForwardingListener(Network::Address::InstanceConstSharedPtr address,
                                 std::shared_ptr<Network::UdpListenerConfig> listener_config) PURE;
+
+  /**
+   * @return An interface on which registerParentDrainedCallback can be called during
+   *         creation of a listener, or nullopt if there is no parent instance.
+   *
+   *         If this is set, any UDP listener should start paused and only begin listening
+   *         when the parent instance is drained; this allows draining QUIC listeners to
+   *         catch their own packets and forward unrecognized packets to the child instance.
+   */
+  virtual OptRef<Network::ParentDrainedCallbackRegistrar> parentDrainedCallbackRegistrar() PURE;
+
   /**
    * Initialize the parent logic of our restarter. Meant to be called after initialization of a
    * new child has begun. The hot restart implementation needs to be created early to deal with

envoy/ssl/certificate_validation_context_config.h

@@ -19,7 +19,7 @@ namespace Ssl {
 //
 // When adding or changing this interface, it is likely that a change is needed to
 // `DefaultCertValidator::updateDigestForSessionId` in
-// `source/extensions/transport_sockets/tls/cert_validator/default_validator.cc`.
+// `source/common/tls/cert_validator/default_validator.cc`.
 class CertificateValidationContextConfig {
 public:
   virtual ~CertificateValidationContextConfig() = default;

envoy/ssl/context_manager.h

@@ -12,6 +12,12 @@
 namespace Envoy {
 namespace Ssl {
 
+// Opaque type defined and used by the ``ServerContext``.
+struct TlsContext;
+
+using ContextAdditionalInitFunc =
+    std::function<void(Ssl::TlsContext& context, const Ssl::TlsCertificateConfig& cert)>;
+
 /**
  * Manages all of the SSL contexts in the process
  */
@@ -30,7 +36,8 @@ class ContextManager {
    */
   virtual ServerContextSharedPtr
   createSslServerContext(Stats::Scope& scope, const ServerContextConfig& config,
-                         const std::vector<std::string>& server_names) PURE;
+                         const std::vector<std::string>& server_names,
+                         ContextAdditionalInitFunc additional_init) PURE;
 
   /**
    * @return the number of days until the next certificate being managed will expire, the value is

examples/ext_authz/auth/grpc-service/go.mod

@@ -6,5 +6,5 @@ require (
 	github.com/envoyproxy/go-control-plane v0.12.0
 	github.com/golang/protobuf v1.5.3
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80
-	google.golang.org/grpc v1.62.0
+	google.golang.org/grpc v1.62.1
 )

examples/ext_authz/auth/grpc-service/go.sum

@@ -2277,8 +2277,8 @@ google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSs
 google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
 google.golang.org/grpc v1.60.0/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM=
 google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM=
-google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk=
-google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE=
+google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk=
+google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

examples/golang-http/simple/go.mod

@@ -7,7 +7,7 @@ go 1.20
 require (
 	github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa
 	github.com/envoyproxy/envoy v1.24.0
-	google.golang.org/protobuf v1.32.0
+	google.golang.org/protobuf v1.33.0
 )
 
 require (