quiche: implement ActiveQuicListener (envoyproxy#7896)

Signed-off-by: Dan Zhang <danzh@google.com>

api/envoy/api/v2/listener/BUILD

@@ -28,3 +28,9 @@ api_proto_library_internal(
         "//envoy/api/v2/core:base",
     ],
 )
+
+api_proto_library_internal(
+    name = "quic_config",
+    srcs = ["quic_config.proto"],
+    visibility = ["//envoy/api/v2:friends"],
+)

api/envoy/api/v2/listener/quic_config.proto

@@ -0,0 +1,28 @@
+syntax = "proto3";
+
+package envoy.api.v2.listener;
+
+option java_outer_classname = "ListenerProto";
+option java_multiple_files = true;
+option java_package = "io.envoyproxy.envoy.api.v2.listener";
+option csharp_namespace = "Envoy.Api.V2.ListenerNS";
+option ruby_package = "Envoy::Api::V2::ListenerNS";
+
+import "google/protobuf/duration.proto";
+import "google/protobuf/wrappers.proto";
+
+// Configuration specific to the QUIC protocol.
+// Next id: 4
+message QuicProtocolOptions {
+  // Maximum number of streams that the client can negotiate per connection. 100
+  // if not specified.
+  google.protobuf.UInt32Value max_concurrent_streams = 1;
+
+  // Maximum number of milliseconds that connection will be alive when there is
+  // no network activity. 300000ms if not specified.
+  google.protobuf.Duration idle_timeout = 2;
+
+  // Connection timeout in milliseconds before the crypto handshake is finished.
+  // 20000ms if not specified.
+  google.protobuf.Duration crypto_handshake_timeout = 3;
+}

docs/root/api-v2/listeners/listeners.rst

@@ -8,3 +8,4 @@ Listeners
   ../api/v2/lds.proto
   ../api/v2/listener/listener.proto
   ../api/v2/listener/udp_listener_config.proto
+  ../api/v2/listener/quic_config.proto

include/envoy/event/dispatcher.h

@@ -145,8 +145,8 @@ class Dispatcher {
    * @param cb supplies the udp listener callbacks to invoke for listener events.
    * @return Network::ListenerPtr a new listener that is owned by the caller.
    */
-  virtual Network::ListenerPtr createUdpListener(Network::Socket& socket,
-                                                 Network::UdpListenerCallbacks& cb) PURE;
+  virtual Network::UdpListenerPtr createUdpListener(Network::Socket& socket,
+                                                    Network::UdpListenerCallbacks& cb) PURE;
   /**
    * Allocates a timer. @see Timer for docs on how to use the timer.
    * @param cb supplies the callback to invoke when the timer fires.

include/envoy/network/connection_handler.h

@@ -26,6 +26,19 @@ class ConnectionHandler {
    */
   virtual uint64_t numConnections() PURE;
 
+  /**
+   * Increment the return value of numConnections() by one.
+   * TODO(mattklein123): re-visit the connection accounting interface. Make TCP
+   * listener to do accounting through these interfaces instead of directly
+   * access the counter.
+   */
+  virtual void incNumConnections() PURE;
+
+  /**
+   * Decrement the return value of numConnections() by one.
+   */
+  virtual void decNumConnections() PURE;
+
   /**
    * Adds a listener to the handler.
    * @param config listener configuration options.

include/envoy/network/listener.h

@@ -242,6 +242,8 @@ class UdpListener : public virtual Listener {
   virtual Api::IoCallUint64Result send(const UdpSendData& data) PURE;
 };
 
+using UdpListenerPtr = std::unique_ptr<UdpListener>;
+
 /**
  * Thrown when there is a runtime error creating/binding a listener.
  */

include/envoy/server/active_udp_listener_config.h

@@ -2,6 +2,8 @@
 
 #include "envoy/network/connection_handler.h"
 
+#include "common/protobuf/protobuf.h"
+
 namespace Envoy {
 namespace Server {
 
@@ -13,6 +15,8 @@ class ActiveUdpListenerConfigFactory {
 public:
   virtual ~ActiveUdpListenerConfigFactory() = default;
 
+  virtual ProtobufTypes::MessagePtr createEmptyConfigProto() PURE;
+
   /**
    * Create an ActiveUdpListenerFactory object according to given message.
    */

source/common/event/dispatcher_impl.cc

@@ -139,10 +139,10 @@ DispatcherImpl::createListener(Network::Socket& socket, Network::ListenerCallbac
                                                         hand_off_restored_destination_connections)};
 }
 
-Network::ListenerPtr DispatcherImpl::createUdpListener(Network::Socket& socket,
-                                                       Network::UdpListenerCallbacks& cb) {
+Network::UdpListenerPtr DispatcherImpl::createUdpListener(Network::Socket& socket,
+                                                          Network::UdpListenerCallbacks& cb) {
   ASSERT(isThreadSafe());
-  return Network::ListenerPtr{new Network::UdpListenerImpl(*this, socket, cb, timeSource())};
+  return std::make_unique<Network::UdpListenerImpl>(*this, socket, cb, timeSource());
 }
 
 TimerPtr DispatcherImpl::createTimer(TimerCb cb) { return createTimerInternal(cb); }

source/common/event/dispatcher_impl.h

@@ -60,8 +60,8 @@ class DispatcherImpl : Logger::Loggable<Logger::Id::main>,
   Network::ListenerPtr createListener(Network::Socket& socket, Network::ListenerCallbacks& cb,
                                       bool bind_to_port,
                                       bool hand_off_restored_destination_connections) override;
-  Network::ListenerPtr createUdpListener(Network::Socket& socket,
-                                         Network::UdpListenerCallbacks& cb) override;
+  Network::UdpListenerPtr createUdpListener(Network::Socket& socket,
+                                            Network::UdpListenerCallbacks& cb) override;
   TimerPtr createTimer(TimerCb cb) override;
   void deferredDelete(DeferredDeletablePtr&& to_delete) override;
   void exit() override;

source/extensions/quic_listeners/quiche/BUILD

@@ -192,6 +192,37 @@ envoy_cc_library(
     hdrs = ["envoy_quic_simulated_watermark_buffer.h"],
 )
 
+envoy_cc_library(
+    name = "active_quic_listener_lib",
+    srcs = ["active_quic_listener.cc"],
+    hdrs = ["active_quic_listener.h"],
+    tags = ["nofips"],
+    deps = [
+        ":envoy_quic_alarm_factory_lib",
+        ":envoy_quic_connection_helper_lib",
+        ":envoy_quic_dispatcher_lib",
+        ":envoy_quic_packet_writer_lib",
+        ":envoy_quic_proof_source_lib",
+        ":envoy_quic_utils_lib",
+        "//include/envoy/network:listener_interface",
+        "//source/common/network:listener_lib",
+        "//source/common/protobuf:utility_lib",
+        "//source/server:connection_handler_lib",
+        "@envoy_api//envoy/api/v2/listener:quic_config_cc",
+    ],
+)
+
+envoy_cc_library(
+    name = "active_quic_listener_config_lib",
+    srcs = ["active_quic_listener_config.cc"],
+    hdrs = ["active_quic_listener_config.h"],
+    tags = ["nofips"],
+    deps = [
+        ":active_quic_listener_lib",
+        "//include/envoy/registry",
+    ],
+)
+
 envoy_cc_library(
     name = "envoy_quic_utils_lib",
     srcs = ["envoy_quic_utils.cc"],

source/extensions/quic_listeners/quiche/active_quic_listener.cc

@@ -0,0 +1,84 @@
+#include "extensions/quic_listeners/quiche/active_quic_listener.h"
+
+#include "extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h"
+#include "extensions/quic_listeners/quiche/envoy_quic_connection_helper.h"
+#include "extensions/quic_listeners/quiche/envoy_quic_dispatcher.h"
+#include "extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h"
+#include "extensions/quic_listeners/quiche/envoy_quic_packet_writer.h"
+#include "extensions/quic_listeners/quiche/envoy_quic_utils.h"
+
+namespace Envoy {
+namespace Quic {
+
+ActiveQuicListener::ActiveQuicListener(Event::Dispatcher& dispatcher,
+                                       Network::ConnectionHandler& parent, spdlog::logger& logger,
+                                       Network::ListenerConfig& listener_config,
+                                       const quic::QuicConfig& quic_config)
+    : ActiveQuicListener(dispatcher, parent,
+                         dispatcher.createUdpListener(listener_config.socket(), *this), logger,
+                         listener_config, quic_config) {}
+
+ActiveQuicListener::ActiveQuicListener(Event::Dispatcher& dispatcher,
+                                       Network::ConnectionHandler& parent,
+                                       Network::UdpListenerPtr&& listener, spdlog::logger& logger,
+                                       Network::ListenerConfig& listener_config,
+                                       const quic::QuicConfig& quic_config)
+    : ActiveQuicListener(dispatcher, parent, std::make_unique<EnvoyQuicPacketWriter>(*listener),
+                         std::move(listener), logger, listener_config, quic_config) {}
+
+ActiveQuicListener::ActiveQuicListener(Event::Dispatcher& dispatcher,
+                                       Network::ConnectionHandler& parent,
+                                       std::unique_ptr<quic::QuicPacketWriter> writer,
+                                       Network::UdpListenerPtr&& listener, spdlog::logger& logger,
+                                       Network::ListenerConfig& listener_config,
+                                       const quic::QuicConfig& quic_config)
+    : Server::ConnectionHandlerImpl::ActiveListenerImplBase(std::move(listener), listener_config),
+      logger_(logger), dispatcher_(dispatcher), version_manager_(quic::CurrentSupportedVersions()) {
+  quic::QuicRandom* const random = quic::QuicRandom::GetInstance();
+  random->RandBytes(random_seed_, sizeof(random_seed_));
+  crypto_config_ = std::make_unique<quic::QuicCryptoServerConfig>(
+      quic::QuicStringPiece(reinterpret_cast<char*>(random_seed_), sizeof(random_seed_)),
+      quic::QuicRandom::GetInstance(), std::make_unique<EnvoyQuicFakeProofSource>(),
+      quic::KeyExchangeSource::Default());
+  auto connection_helper = std::make_unique<EnvoyQuicConnectionHelper>(dispatcher_);
+  auto alarm_factory =
+      std::make_unique<EnvoyQuicAlarmFactory>(dispatcher_, *connection_helper->GetClock());
+  quic_dispatcher_ = std::make_unique<EnvoyQuicDispatcher>(
+      crypto_config_.get(), quic_config, &version_manager_, std::move(connection_helper),
+      std::move(alarm_factory), quic::kQuicDefaultConnectionIdLength, parent, config_, stats_,
+      dispatcher);
+  quic_dispatcher_->InitializeWithWriter(writer.release());
+}
+
+void ActiveQuicListener::onListenerShutdown() {
+  ENVOY_LOG_TO_LOGGER(logger_, info, "Quic listener {} shutdown.", config_.name());
+  quic_dispatcher_->Shutdown();
+}
+
+void ActiveQuicListener::onData(Network::UdpRecvData& data) {
+  quic::QuicSocketAddress peer_address(envoyAddressInstanceToQuicSocketAddress(data.peer_address_));
+  quic::QuicSocketAddress self_address(
+      envoyAddressInstanceToQuicSocketAddress(data.local_address_));
+  quic::QuicTime timestamp =
+      quic::QuicTime::Zero() +
+      quic::QuicTime::Delta::FromMilliseconds(std::chrono::duration_cast<std::chrono::milliseconds>(
+                                                  data.receive_time_.time_since_epoch())
+                                                  .count());
+  uint64_t num_slice = data.buffer_->getRawSlices(nullptr, 0);
+  ASSERT(num_slice == 1);
+  Buffer::RawSlice slice;
+  data.buffer_->getRawSlices(&slice, 1);
+  // TODO(danzh): pass in TTL and UDP header.
+  quic::QuicReceivedPacket packet(reinterpret_cast<char*>(slice.mem_), slice.len_, timestamp,
+                                  /*owns_buffer=*/false, /*ttl=*/0, /*ttl_valid=*/true,
+                                  /*packet_headers=*/nullptr, /*headers_length=*/0,
+                                  /*owns_header_buffer*/ false);
+  quic_dispatcher_->ProcessPacket(self_address, peer_address, packet);
+}
+
+void ActiveQuicListener::onWriteReady(const Network::Socket& /*socket*/) {
+  quic_dispatcher_->OnCanWrite();
+}
+
+} // namespace Quic
+} // namespace Envoy

source/extensions/quic_listeners/quiche/active_quic_listener.h

@@ -0,0 +1,107 @@
+#pragma once
+
+#include "envoy/api/v2/listener/quic_config.pb.h"
+#include "envoy/network/connection_handler.h"
+#include "envoy/network/listener.h"
+
+#include "common/protobuf/utility.h"
+
+#include "server/connection_handler_impl.h"
+
+#include "extensions/quic_listeners/quiche/envoy_quic_dispatcher.h"
+
+namespace Envoy {
+namespace Quic {
+
+// QUIC specific UdpListenerCallbacks implemention which delegates incoming
+// packets, write signals and listener errors to QuicDispatcher.
+class ActiveQuicListener : public Network::UdpListenerCallbacks,
+                           public Server::ConnectionHandlerImpl::ActiveListenerImplBase,
+                           // Inherits below two interfaces just to have common
+                           // interfaces. Not expected to support listener
+                           // filter.
+                           // TODO(danzh): clean up meaningless inheritance.
+                           public Network::UdpListenerFilterManager,
+                           public Network::UdpReadFilterCallbacks {
+public:
+  ActiveQuicListener(Event::Dispatcher& dispatcher, Network::ConnectionHandler& parent,
+                     spdlog::logger& logger, Network::ListenerConfig& listener_config,
+                     const quic::QuicConfig& quic_config);
+
+  ActiveQuicListener(Event::Dispatcher& dispatcher, Network::ConnectionHandler& parent,
+                     Network::UdpListenerPtr&& listener, spdlog::logger& logger,
+                     Network::ListenerConfig& listener_config, const quic::QuicConfig& quic_config);
+
+  // TODO(#7465): Make this a callback.
+  void onListenerShutdown();
+
+  // Network::UdpListenerCallbacks
+  void onData(Network::UdpRecvData& data) override;
+  void onWriteReady(const Network::Socket& socket) override;
+  void onReceiveError(const Network::UdpListenerCallbacks::ErrorCode& /*error_code*/,
+                      Api::IoError::IoErrorCode /*err*/) override {
+    // No-op. Quic can't do anything upon listener error.
+  }
+
+  // Network::UdpListenerFilterManager
+  void addReadFilter(Network::UdpListenerReadFilterPtr&& /*filter*/) override {
+    // QUIC doesn't support listener filter.
+    NOT_REACHED_GCOVR_EXCL_LINE;
+  }
+
+  // Network::UdpReadFilterCallbacks
+  Network::UdpListener& udpListener() override { NOT_REACHED_GCOVR_EXCL_LINE; }
+
+private:
+  friend class ActiveQuicListenerPeer;
+
+  ActiveQuicListener(Event::Dispatcher& dispatcher, Network::ConnectionHandler& parent,
+                     std::unique_ptr<quic::QuicPacketWriter> writer,
+                     Network::UdpListenerPtr&& listener, spdlog::logger& logger,
+                     Network::ListenerConfig& listener_config, const quic::QuicConfig& quic_config);
+
+  uint8_t random_seed_[16];
+  std::unique_ptr<quic::QuicCryptoServerConfig> crypto_config_;
+  spdlog::logger& logger_;
+  Event::Dispatcher& dispatcher_;
+  quic::QuicVersionManager version_manager_;
+  std::unique_ptr<EnvoyQuicDispatcher> quic_dispatcher_;
+};
+
+using ActiveQuicListenerPtr = std::unique_ptr<ActiveQuicListener>;
+
+// A factory to create ActiveQuicListener based on given config.
+class ActiveQuicListenerFactory : public Network::ActiveUdpListenerFactory {
+public:
+  ActiveQuicListenerFactory(const envoy::api::v2::listener::QuicProtocolOptions& config) {
+    uint64_t idle_network_timeout_ms =
+        config.has_idle_timeout() ? DurationUtil::durationToMilliseconds(config.idle_timeout())
+                                  : 300000;
+    quic_config_.SetIdleNetworkTimeout(
+        quic::QuicTime::Delta::FromMilliseconds(idle_network_timeout_ms),
+        quic::QuicTime::Delta::FromMilliseconds(idle_network_timeout_ms));
+    int32_t max_time_before_crypto_handshake_ms =
+        config.has_crypto_handshake_timeout()
+            ? DurationUtil::durationToMilliseconds(config.crypto_handshake_timeout())
+            : 20000;
+    quic_config_.set_max_time_before_crypto_handshake(
+        quic::QuicTime::Delta::FromMilliseconds(max_time_before_crypto_handshake_ms));
+    int32_t max_streams = PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_concurrent_streams, 100);
+    quic_config_.SetMaxIncomingBidirectionalStreamsToSend(max_streams);
+    quic_config_.SetMaxIncomingUnidirectionalStreamsToSend(max_streams);
+  }
+
+  Network::ConnectionHandler::ActiveListenerPtr
+  createActiveUdpListener(Network::ConnectionHandler& parent, Event::Dispatcher& disptacher,
+                          spdlog::logger& logger, Network::ListenerConfig& config) const override {
+    return std::make_unique<ActiveQuicListener>(disptacher, parent, logger, config, quic_config_);
+  }
+
+private:
+  friend class ActiveQuicListenerFactoryPeer;
+
+  quic::QuicConfig quic_config_;
+};
+
+} // namespace Quic
+} // namespace Envoy

source/extensions/quic_listeners/quiche/active_quic_listener_config.cc

@@ -0,0 +1,25 @@
+#include "extensions/quic_listeners/quiche/active_quic_listener_config.h"
+
+#include "envoy/api/v2/listener/quic_config.pb.h"
+
+#include "extensions/quic_listeners/quiche/active_quic_listener.h"
+
+namespace Envoy {
+namespace Quic {
+
+ProtobufTypes::MessagePtr ActiveQuicListenerConfigFactory::createEmptyConfigProto() {
+  return std::make_unique<envoy::api::v2::listener::QuicProtocolOptions>();
+}
+
+Network::ActiveUdpListenerFactoryPtr
+ActiveQuicListenerConfigFactory::createActiveUdpListenerFactory(const Protobuf::Message& message) {
+  auto& config = dynamic_cast<const envoy::api::v2::listener::QuicProtocolOptions&>(message);
+  return std::make_unique<ActiveQuicListenerFactory>(config);
+}
+
+std::string ActiveQuicListenerConfigFactory::name() { return QuicListenerName; }
+
+REGISTER_FACTORY(ActiveQuicListenerConfigFactory, Server::ActiveUdpListenerConfigFactory);
+
+} // namespace Quic
+} // namespace Envoy