api: add enable_reuse_port

Currently, the socket option `SO_REUSEPORT` is always set on the
Listener sockets and is no longer configurable. Previously
(before #315), this was configurable  via Envoy Listeners option
`enable_reuse_port`.

Therefore, this commit introduces the config property `enable_reuse_port`
on the `BPFMetadata` to reintroduce the possibility to configure this
behaviour.

Signed-off-by: Marco Hofstetter <marco.hofstetter@isovalent.com>

cilium/api/bpf_metadata.proto

@@ -44,4 +44,11 @@ message BpfMetadata {
   // proxy_id is passed to access log messages and allows relating access log messages to
   // listeners.
   uint32 proxy_id = 8;
+
+  // When this flag is set to true, listeners set the “SO_REUSEPORT“ socket option and
+  // create one socket for each worker thread. This makes inbound connections
+  // distribute among worker threads roughly evenly in cases where there are a high number
+  // of connections. When this flag is set to false, all worker threads share one socket. This field
+  // defaults to true.
+  bool enable_reuse_port = 9;
 }

cilium/bpf_metadata.cc

@@ -112,7 +112,8 @@ Config::Config(const ::cilium::BpfMetadata& config,
           Network::Utility::parseInternetAddressNoThrow(config.ipv4_source_address())),
       ipv6_source_address_(
           Network::Utility::parseInternetAddressNoThrow(config.ipv6_source_address())),
-      enforce_policy_on_l7lb_(config.enforce_policy_on_l7lb()) {
+      enforce_policy_on_l7lb_(config.enforce_policy_on_l7lb()),
+      enable_reuse_port_(config.enable_reuse_port()) {
   if (is_l7lb_ && is_ingress_) {
     throw EnvoyException("cilium.bpf_metadata: is_l7lb may not be set with is_ingress");
   }
@@ -422,7 +423,7 @@ bool Config::getMetadata(Network::ConnectionSocket& socket) {
   socket.addOption(std::make_shared<Cilium::SocketOption>(
       policy, mark, ingress_source_identity, source_identity, is_ingress_, is_l7lb_, dip->port(),
       std::move(pod_ip), std::move(src_address), std::move(ipv4_source_address),
-      std::move(ipv6_source_address), shared_from_this(), proxy_id_));
+      std::move(ipv6_source_address), shared_from_this(), proxy_id_, enable_reuse_port_));
   return true;
 }
 

cilium/bpf_metadata.h

@@ -43,6 +43,7 @@ class Config : public Cilium::PolicyResolver,
   Network::Address::InstanceConstSharedPtr ipv4_source_address_;
   Network::Address::InstanceConstSharedPtr ipv6_source_address_;
   bool enforce_policy_on_l7lb_;
+  bool enable_reuse_port_;
 
   std::shared_ptr<const Cilium::NetworkPolicyMap> npmap_{};
   Cilium::CtMapSharedPtr ct_maps_{};

cilium/socket_option.h

@@ -29,11 +29,13 @@ class SocketMarkOption : public Network::Socket::Option,
   SocketMarkOption(uint32_t mark, uint32_t identity,
                    Network::Address::InstanceConstSharedPtr original_source_address = nullptr,
                    Network::Address::InstanceConstSharedPtr ipv4_source_address = nullptr,
-                   Network::Address::InstanceConstSharedPtr ipv6_source_address = nullptr)
+                   Network::Address::InstanceConstSharedPtr ipv6_source_address = nullptr,
+                   bool enable_reuse_port = true)
       : identity_(identity), mark_(mark),
         original_source_address_(std::move(original_source_address)),
         ipv4_source_address_(std::move(ipv4_source_address)),
-        ipv6_source_address_(std::move(ipv6_source_address)) {}
+        ipv6_source_address_(std::move(ipv6_source_address)),
+        enable_reuse_port_(enable_reuse_port) {}
 
   absl::optional<Network::Socket::Option::Details>
   getOptionDetails(const Network::Socket&,
@@ -135,11 +137,14 @@ class SocketMarkOption : public Network::Socket::Option,
                   Envoy::errorDetails(status.errno_));
         return false;
       }
-      status = socket.setSocketOption(SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one));
-      if (status.return_value_ < 0) {
-        ENVOY_LOG(critical, "Failed to set socket option SO_REUSEPORT: {}",
-                  Envoy::errorDetails(status.errno_));
-        return false;
+
+      if (enable_reuse_port_) {
+        status = socket.setSocketOption(SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one));
+        if (status.return_value_ < 0) {
+          ENVOY_LOG(critical, "Failed to set socket option SO_REUSEPORT: {}",
+                    Envoy::errorDetails(status.errno_));
+          return false;
+        }
       }
     }
 
@@ -200,6 +205,8 @@ class SocketMarkOption : public Network::Socket::Option,
   // connecting to IPv6 or IPv4 address, respectively.
   Network::Address::InstanceConstSharedPtr ipv4_source_address_;
   Network::Address::InstanceConstSharedPtr ipv6_source_address_;
+
+  bool enable_reuse_port_;
 };
 
 class SocketOption : public SocketMarkOption {
@@ -210,21 +217,22 @@ class SocketOption : public SocketMarkOption {
                Network::Address::InstanceConstSharedPtr original_source_address,
                Network::Address::InstanceConstSharedPtr ipv4_source_address,
                Network::Address::InstanceConstSharedPtr ipv6_source_address,
-               const std::shared_ptr<PolicyResolver>& policy_id_resolver, uint32_t proxy_id)
+               const std::shared_ptr<PolicyResolver>& policy_id_resolver, uint32_t proxy_id,
+               bool enable_reuse_port)
       : SocketMarkOption(mark, source_identity, original_source_address, ipv4_source_address,
-                         ipv6_source_address),
+                         ipv6_source_address, enable_reuse_port),
         ingress_source_identity_(ingress_source_identity), initial_policy_(policy),
         ingress_(ingress), is_l7lb_(l7lb), port_(port), pod_ip_(std::move(pod_ip)),
         proxy_id_(proxy_id), policy_id_resolver_(policy_id_resolver) {
     ENVOY_LOG(debug,
               "Cilium SocketOption(): source_identity: {}, "
               "ingress: {}, port: {}, pod_ip: {}, source_addresses: {}/{}/{}, mark: {:x} (magic "
-              "mark: {:x}, cluster: {}, ID: {}), proxy_id: {}",
+              "mark: {:x}, cluster: {}, ID: {}), proxy_id: {}, enable_reuse_port: {}",
               identity_, ingress_, port_, pod_ip_,
               original_source_address_ ? original_source_address_->asString() : "",
               ipv4_source_address_ ? ipv4_source_address_->asString() : "",
               ipv6_source_address_ ? ipv6_source_address_->asString() : "", mark_, mark & 0xff00,
-              mark & 0xff, mark >> 16, proxy_id_);
+              mark & 0xff, mark >> 16, proxy_id_, enable_reuse_port_);
     ASSERT(initial_policy_ != nullptr);
   }
 

tests/bpf_metadata.cc

@@ -174,7 +174,7 @@ bool TestConfig::getMetadata(Network::ConnectionSocket& socket) {
 
   socket.addOption(std::make_shared<Cilium::SocketOption>(
       policy, 0, 0, source_identity, is_ingress_, false, port, std::move(pod_ip), nullptr, nullptr,
-      nullptr, shared_from_this(), 0));
+      nullptr, shared_from_this(), 0, true));
 
   return true;
 }