diff --git a/cilium/api/bpf_metadata.proto b/cilium/api/bpf_metadata.proto index e3dd02567..c5dbbc89d 100644 --- a/cilium/api/bpf_metadata.proto +++ b/cilium/api/bpf_metadata.proto @@ -44,4 +44,11 @@ message BpfMetadata { // proxy_id is passed to access log messages and allows relating access log messages to // listeners. uint32 proxy_id = 8; + + // When this flag is set to true, listeners set the “SO_REUSEPORT“ socket option and + // create one socket for each worker thread. This makes inbound connections + // distribute among worker threads roughly evenly in cases where there are a high number + // of connections. When this flag is set to false, all worker threads share one socket. This field + // defaults to true. + bool enable_reuse_port = 9; } diff --git a/cilium/bpf_metadata.cc b/cilium/bpf_metadata.cc index 79d513389..018db776b 100644 --- a/cilium/bpf_metadata.cc +++ b/cilium/bpf_metadata.cc @@ -43,7 +43,7 @@ class BpfMetadataConfigFactory : public NamedListenerFilterConfigFactory { std::shared_ptr options = std::make_shared(); uint32_t mark = (config->is_ingress_) ? 0x0A00 : 0x0B00; - options->push_back(std::make_shared(mark, 0)); + options->push_back(std::make_shared(mark, 0, config->enable_reuse_port_)); context.addListenSocketOptions(options); return [listener_filter_matcher, @@ -112,7 +112,8 @@ Config::Config(const ::cilium::BpfMetadata& config, Network::Utility::parseInternetAddressNoThrow(config.ipv4_source_address())), ipv6_source_address_( Network::Utility::parseInternetAddressNoThrow(config.ipv6_source_address())), - enforce_policy_on_l7lb_(config.enforce_policy_on_l7lb()) { + enforce_policy_on_l7lb_(config.enforce_policy_on_l7lb()), + enable_reuse_port_(config.enable_reuse_port()) { if (is_l7lb_ && is_ingress_) { throw EnvoyException("cilium.bpf_metadata: is_l7lb may not be set with is_ingress"); } @@ -422,7 +423,7 @@ bool Config::getMetadata(Network::ConnectionSocket& socket) { socket.addOption(std::make_shared( policy, mark, ingress_source_identity, source_identity, is_ingress_, is_l7lb_, dip->port(), std::move(pod_ip), std::move(src_address), std::move(ipv4_source_address), - std::move(ipv6_source_address), shared_from_this(), proxy_id_)); + std::move(ipv6_source_address), shared_from_this(), proxy_id_, enable_reuse_port_)); return true; } diff --git a/cilium/bpf_metadata.h b/cilium/bpf_metadata.h index 869f0942c..23a9099da 100644 --- a/cilium/bpf_metadata.h +++ b/cilium/bpf_metadata.h @@ -43,6 +43,7 @@ class Config : public Cilium::PolicyResolver, Network::Address::InstanceConstSharedPtr ipv4_source_address_; Network::Address::InstanceConstSharedPtr ipv6_source_address_; bool enforce_policy_on_l7lb_; + bool enable_reuse_port_; std::shared_ptr npmap_{}; Cilium::CtMapSharedPtr ct_maps_{}; diff --git a/cilium/socket_option.h b/cilium/socket_option.h index aea3cd565..0d1e11bd4 100644 --- a/cilium/socket_option.h +++ b/cilium/socket_option.h @@ -26,14 +26,15 @@ class PolicyResolver { class SocketMarkOption : public Network::Socket::Option, public Logger::Loggable { public: - SocketMarkOption(uint32_t mark, uint32_t identity, + SocketMarkOption(uint32_t mark, uint32_t identity, bool enable_reuse_port, Network::Address::InstanceConstSharedPtr original_source_address = nullptr, Network::Address::InstanceConstSharedPtr ipv4_source_address = nullptr, Network::Address::InstanceConstSharedPtr ipv6_source_address = nullptr) : identity_(identity), mark_(mark), original_source_address_(std::move(original_source_address)), ipv4_source_address_(std::move(ipv4_source_address)), - ipv6_source_address_(std::move(ipv6_source_address)) {} + ipv6_source_address_(std::move(ipv6_source_address)), + enable_reuse_port_(enable_reuse_port) {} absl::optional getOptionDetails(const Network::Socket&, @@ -135,11 +136,14 @@ class SocketMarkOption : public Network::Socket::Option, Envoy::errorDetails(status.errno_)); return false; } - status = socket.setSocketOption(SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one)); - if (status.return_value_ < 0) { - ENVOY_LOG(critical, "Failed to set socket option SO_REUSEPORT: {}", - Envoy::errorDetails(status.errno_)); - return false; + + if (enable_reuse_port_) { + status = socket.setSocketOption(SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one)); + if (status.return_value_ < 0) { + ENVOY_LOG(critical, "Failed to set socket option SO_REUSEPORT: {}", + Envoy::errorDetails(status.errno_)); + return false; + } } } @@ -200,6 +204,8 @@ class SocketMarkOption : public Network::Socket::Option, // connecting to IPv6 or IPv4 address, respectively. Network::Address::InstanceConstSharedPtr ipv4_source_address_; Network::Address::InstanceConstSharedPtr ipv6_source_address_; + + bool enable_reuse_port_; }; class SocketOption : public SocketMarkOption { @@ -210,8 +216,10 @@ class SocketOption : public SocketMarkOption { Network::Address::InstanceConstSharedPtr original_source_address, Network::Address::InstanceConstSharedPtr ipv4_source_address, Network::Address::InstanceConstSharedPtr ipv6_source_address, - const std::shared_ptr& policy_id_resolver, uint32_t proxy_id) - : SocketMarkOption(mark, source_identity, original_source_address, ipv4_source_address, + const std::shared_ptr& policy_id_resolver, uint32_t proxy_id, + bool enable_reuse_port) + : SocketMarkOption(mark, source_identity, enable_reuse_port, + original_source_address, ipv4_source_address, ipv6_source_address), ingress_source_identity_(ingress_source_identity), initial_policy_(policy), ingress_(ingress), is_l7lb_(l7lb), port_(port), pod_ip_(std::move(pod_ip)), @@ -219,12 +227,12 @@ class SocketOption : public SocketMarkOption { ENVOY_LOG(debug, "Cilium SocketOption(): source_identity: {}, " "ingress: {}, port: {}, pod_ip: {}, source_addresses: {}/{}/{}, mark: {:x} (magic " - "mark: {:x}, cluster: {}, ID: {}), proxy_id: {}", + "mark: {:x}, cluster: {}, ID: {}), proxy_id: {}, enable_reuse_port: {}", identity_, ingress_, port_, pod_ip_, original_source_address_ ? original_source_address_->asString() : "", ipv4_source_address_ ? ipv4_source_address_->asString() : "", ipv6_source_address_ ? ipv6_source_address_->asString() : "", mark_, mark & 0xff00, - mark & 0xff, mark >> 16, proxy_id_); + mark & 0xff, mark >> 16, proxy_id_, enable_reuse_port_); ASSERT(initial_policy_ != nullptr); } diff --git a/go/cilium/api/bpf_metadata.pb.go b/go/cilium/api/bpf_metadata.pb.go index a8a62eec9..34d331cd3 100644 --- a/go/cilium/api/bpf_metadata.pb.go +++ b/go/cilium/api/bpf_metadata.pb.go @@ -63,6 +63,12 @@ type BpfMetadata struct { // proxy_id is passed to access log messages and allows relating access log messages to // listeners. ProxyId uint32 `protobuf:"varint,8,opt,name=proxy_id,json=proxyId,proto3" json:"proxy_id,omitempty"` + // When this flag is set to true, listeners set the “SO_REUSEPORT“ socket option and + // create one socket for each worker thread. This makes inbound connections + // distribute among worker threads roughly evenly in cases where there are a high number + // of connections. When this flag is set to false, all worker threads share one socket. This field + // defaults to true. + EnableReusePort bool `protobuf:"varint,9,opt,name=enable_reuse_port,json=enableReusePort,proto3" json:"enable_reuse_port,omitempty"` } func (x *BpfMetadata) Reset() { @@ -153,12 +159,19 @@ func (x *BpfMetadata) GetProxyId() uint32 { return 0 } +func (x *BpfMetadata) GetEnableReusePort() bool { + if x != nil { + return x.EnableReusePort + } + return false +} + var File_cilium_api_bpf_metadata_proto protoreflect.FileDescriptor var file_cilium_api_bpf_metadata_proto_rawDesc = []byte{ 0x0a, 0x1d, 0x63, 0x69, 0x6c, 0x69, 0x75, 0x6d, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x62, 0x70, 0x66, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, - 0x06, 0x63, 0x69, 0x6c, 0x69, 0x75, 0x6d, 0x22, 0xcf, 0x02, 0x0a, 0x0b, 0x42, 0x70, 0x66, 0x4d, + 0x06, 0x63, 0x69, 0x6c, 0x69, 0x75, 0x6d, 0x22, 0xfb, 0x02, 0x0a, 0x0b, 0x42, 0x70, 0x66, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x19, 0x0a, 0x08, 0x62, 0x70, 0x66, 0x5f, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x70, 0x66, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, @@ -179,11 +192,13 @@ var file_cilium_api_bpf_metadata_proto_rawDesc = []byte{ 0x37, 0x6c, 0x62, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x65, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4f, 0x6e, 0x4c, 0x37, 0x6c, 0x62, 0x12, 0x19, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x07, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x64, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x69, 0x6c, 0x69, 0x75, 0x6d, 0x2f, 0x70, - 0x72, 0x6f, 0x78, 0x79, 0x2f, 0x67, 0x6f, 0x2f, 0x63, 0x69, 0x6c, 0x69, 0x75, 0x6d, 0x2f, 0x61, - 0x70, 0x69, 0x3b, 0x63, 0x69, 0x6c, 0x69, 0x75, 0x6d, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x52, 0x07, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x64, 0x12, 0x2a, 0x0a, 0x11, 0x65, 0x6e, 0x61, + 0x62, 0x6c, 0x65, 0x5f, 0x72, 0x65, 0x75, 0x73, 0x65, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x09, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x75, 0x73, + 0x65, 0x50, 0x6f, 0x72, 0x74, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x69, 0x6c, 0x69, 0x75, 0x6d, 0x2f, 0x70, 0x72, 0x6f, 0x78, 0x79, + 0x2f, 0x67, 0x6f, 0x2f, 0x63, 0x69, 0x6c, 0x69, 0x75, 0x6d, 0x2f, 0x61, 0x70, 0x69, 0x3b, 0x63, + 0x69, 0x6c, 0x69, 0x75, 0x6d, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/go/cilium/api/bpf_metadata.pb.validate.go b/go/cilium/api/bpf_metadata.pb.validate.go index 2c3fff7c3..5b022af76 100644 --- a/go/cilium/api/bpf_metadata.pb.validate.go +++ b/go/cilium/api/bpf_metadata.pb.validate.go @@ -73,6 +73,8 @@ func (m *BpfMetadata) validate(all bool) error { // no validation rules for ProxyId + // no validation rules for EnableReusePort + if len(errors) > 0 { return BpfMetadataMultiError(errors) } diff --git a/tests/bpf_metadata.cc b/tests/bpf_metadata.cc index 04349121e..8de1314d6 100644 --- a/tests/bpf_metadata.cc +++ b/tests/bpf_metadata.cc @@ -174,7 +174,7 @@ bool TestConfig::getMetadata(Network::ConnectionSocket& socket) { socket.addOption(std::make_shared( policy, 0, 0, source_identity, is_ingress_, false, port, std::move(pod_ip), nullptr, nullptr, - nullptr, shared_from_this(), 0)); + nullptr, shared_from_this(), 0, true)); return true; }