http: allowing an optional proxy proto header when terminating CONNEC…

…T requests (envoyproxy#10975)

Signed-off-by: Alyssa Wilk <alyssar@chromium.org>

api/envoy/config/route/v3/route_components.proto

@@ -3,6 +3,7 @@ syntax = "proto3";
 package envoy.config.route.v3;
 
 import "envoy/config/core/v3/base.proto";
+import "envoy/config/core/v3/proxy_protocol.proto";
 import "envoy/type/matcher/v3/regex.proto";
 import "envoy/type/matcher/v3/string.proto";
 import "envoy/type/tracing/v3/custom_tag.proto";
@@ -724,7 +725,8 @@ message RouteAction {
     // Configuration for sending data upstream as a raw data payload. This is used for
     // CONNECT requests, when forwarding CONNECT payload as raw TCP.
     message ConnectConfig {
-      // TODO(alyssawilk) add proxy proto configuration here.
+      // If present, the proxy protocol header will be prepended to the CONNECT payload sent upstream.
+      core.v3.ProxyProtocolConfig proxy_protocol_config = 1;
     }
 
     // The case-insensitive name of this upgrade, e.g. "websocket".

api/envoy/config/route/v4alpha/route_components.proto

@@ -3,6 +3,7 @@ syntax = "proto3";
 package envoy.config.route.v4alpha;
 
 import "envoy/config/core/v4alpha/base.proto";
+import "envoy/config/core/v4alpha/proxy_protocol.proto";
 import "envoy/type/matcher/v3/regex.proto";
 import "envoy/type/matcher/v3/string.proto";
 import "envoy/type/tracing/v3/custom_tag.proto";
@@ -727,10 +728,11 @@ message RouteAction {
     // Configuration for sending data upstream as a raw data payload. This is used for
     // CONNECT requests, when forwarding CONNECT payload as raw TCP.
     message ConnectConfig {
-      // TODO(alyssawilk) add proxy proto configuration here.
-
       option (udpa.annotations.versioning).previous_message_type =
           "envoy.config.route.v3.RouteAction.UpgradeConfig.ConnectConfig";
+
+      // If present, the proxy protocol header will be prepended to the CONNECT payload sent upstream.
+      core.v4alpha.ProxyProtocolConfig proxy_protocol_config = 1;
     }
 
     // The case-insensitive name of this upgrade, e.g. "websocket".

source/common/router/BUILD

@@ -309,6 +309,7 @@ envoy_cc_library(
         "//source/common/stream_info:uint32_accessor_lib",
         "//source/common/tracing:http_tracer_lib",
         "//source/common/upstream:load_balancer_lib",
+        "//source/extensions/common/proxy_protocol:proxy_protocol_header_lib",
         "@envoy_api//envoy/extensions/filters/http/router/v3:pkg_cc_proto",
     ],
 )

source/common/router/upstream_request.cc

@@ -35,6 +35,7 @@
 #include "common/stream_info/uint32_accessor_impl.h"
 #include "common/tracing/http_tracer_impl.h"
 
+#include "extensions/common/proxy_protocol/proxy_protocol_header.h"
 #include "extensions/filters/http/well_known_names.h"
 
 namespace Envoy {
@@ -538,9 +539,19 @@ void TcpUpstream::encodeData(Buffer::Instance& data, bool end_stream) {
 }
 
 void TcpUpstream::encodeHeaders(const Http::RequestHeaderMap&, bool end_stream) {
-  if (end_stream) {
-    Buffer::OwnedImpl data;
-    upstream_conn_data_->connection().write(data, true);
+  // Headers should only happen once, so use this opportunity to add the proxy
+  // proto header, if configured.
+  ASSERT(upstream_request_->parent().routeEntry()->connectConfig().has_value());
+  Buffer::OwnedImpl data;
+  auto& connect_config = upstream_request_->parent().routeEntry()->connectConfig().value();
+  if (connect_config.has_proxy_protocol_config()) {
+    const Network::Connection& connection = *upstream_request_->parent().callbacks()->connection();
+    Extensions::Common::ProxyProtocol::generateProxyProtoHeader(
+        connect_config.proxy_protocol_config(), connection, data);
+  }
+
+  if (data.length() != 0 || end_stream) {
+    upstream_conn_data_->connection().write(data, end_stream);
   }
 }
 

source/common/router/upstream_request.h

@@ -138,6 +138,7 @@ class UpstreamRequest : public Logger::Loggable<Logger::Id::router>,
   bool createPerTryTimeoutOnRequestComplete() {
     return create_per_try_timeout_on_request_complete_;
   }
+  RouterFilterInterface& parent() { return parent_; }
 
 private:
   RouterFilterInterface& parent_;

source/extensions/common/proxy_protocol/BUILD

@@ -15,6 +15,8 @@ envoy_cc_library(
     deps = [
         "//include/envoy/buffer:buffer_interface",
         "//include/envoy/network:address_interface",
+        "//include/envoy/network:connection_interface",
         "//source/common/network:address_lib",
+        "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
     ],
 )

source/extensions/common/proxy_protocol/proxy_protocol_header.cc

@@ -35,6 +35,12 @@ void generateV1Header(const std::string& src_addr, const std::string& dst_addr,
   out.add(stream.str());
 }
 
+void generateV1Header(const Network::Address::Ip& source_address,
+                      const Network::Address::Ip& dest_address, Buffer::Instance& out) {
+  generateV1Header(source_address.addressAsString(), dest_address.addressAsString(),
+                   source_address.port(), dest_address.port(), source_address.version(), out);
+}
+
 void generateV2Header(const std::string& src_addr, const std::string& dst_addr, uint32_t src_port,
                       uint32_t dst_port, Network::Address::IpVersion ip_version,
                       Buffer::Instance& out) {
@@ -95,6 +101,23 @@ void generateV2Header(const std::string& src_addr, const std::string& dst_addr,
   out.add(ports, 4);
 }
 
+void generateV2Header(const Network::Address::Ip& source_address,
+                      const Network::Address::Ip& dest_address, Buffer::Instance& out) {
+  generateV2Header(source_address.addressAsString(), dest_address.addressAsString(),
+                   source_address.port(), dest_address.port(), source_address.version(), out);
+}
+
+void generateProxyProtoHeader(const envoy::config::core::v3::ProxyProtocolConfig& config,
+                              const Network::Connection& connection, Buffer::Instance& out) {
+  const Network::Address::Ip& dest_address = *connection.localAddress()->ip();
+  const Network::Address::Ip& source_address = *connection.remoteAddress()->ip();
+  if (config.version() == envoy::config::core::v3::ProxyProtocolConfig::V1) {
+    generateV1Header(source_address, dest_address, out);
+  } else if (config.version() == envoy::config::core::v3::ProxyProtocolConfig::V2) {
+    generateV2Header(source_address, dest_address, out);
+  }
+}
+
 void generateV2LocalHeader(Buffer::Instance& out) {
   out.add(PROXY_PROTO_V2_SIGNATURE, PROXY_PROTO_V2_SIGNATURE_LEN);
   const uint8_t addr_fam_protocol_and_length[4]{PROXY_PROTO_V2_VERSION << 4, 0, 0, 0};
@@ -104,4 +127,4 @@ void generateV2LocalHeader(Buffer::Instance& out) {
 } // namespace ProxyProtocol
 } // namespace Common
 } // namespace Extensions
-} // namespace Envoy
+} // namespace Envoy

source/extensions/common/proxy_protocol/proxy_protocol_header.h

@@ -1,7 +1,9 @@
 #pragma once
 
 #include "envoy/buffer/buffer.h"
+#include "envoy/config/core/v3/proxy_protocol.pb.h"
 #include "envoy/network/address.h"
+#include "envoy/network/connection.h"
 
 namespace Envoy {
 namespace Extensions {
@@ -41,15 +43,25 @@ constexpr uint32_t PROXY_PROTO_V2_ADDR_LEN_UNIX = 216;
 void generateV1Header(const std::string& src_addr, const std::string& dst_addr, uint32_t src_port,
                       uint32_t dst_port, Network::Address::IpVersion ip_version,
                       Buffer::Instance& out);
+void generateV1Header(const Network::Address::Ip& source_address,
+                      const Network::Address::Ip& dest_address, Buffer::Instance& out);
+
 // Generates the v2 PROXY protocol header and adds it to the specified buffer
 // TCP is assumed as the transport protocol
 void generateV2Header(const std::string& src_addr, const std::string& dst_addr, uint32_t src_port,
                       uint32_t dst_port, Network::Address::IpVersion ip_version,
                       Buffer::Instance& out);
+void generateV2Header(const Network::Address::Ip& source_address,
+                      const Network::Address::Ip& dest_address, Buffer::Instance& out);
+
+// Generates the appropriate proxy proto header and appends it to the supplied buffer.
+void generateProxyProtoHeader(const envoy::config::core::v3::ProxyProtocolConfig& config,
+                              const Network::Connection& connection, Buffer::Instance& out);
+
 // Generates the v2 PROXY protocol local command header and adds it to the specified buffer
 void generateV2LocalHeader(Buffer::Instance& out);
 
 } // namespace ProxyProtocol
 } // namespace Common
 } // namespace Extensions
-} // namespace Envoy
+} // namespace Envoy

test/common/router/upstream_request_test.cc

@@ -3,6 +3,8 @@
 #include "common/router/router.h"
 #include "common/router/upstream_request.h"
 
+#include "extensions/common/proxy_protocol/proxy_protocol_header.h"
+
 #include "test/common/http/common.h"
 #include "test/mocks/http/mocks.h"
 #include "test/mocks/router/mocks.h"
@@ -55,6 +57,9 @@ class MockRouterFilterInterface : public RouterFilterInterface {
     ON_CALL(*this, cluster()).WillByDefault(Return(cluster_info_));
     ON_CALL(*this, upstreamRequests()).WillByDefault(ReturnRef(requests_));
     EXPECT_CALL(callbacks_.dispatcher_, setTrackedObject(_)).Times(AnyNumber());
+    ON_CALL(*this, routeEntry()).WillByDefault(Return(&route_entry_));
+    ON_CALL(callbacks_, connection()).WillByDefault(Return(&client_connection_));
+    route_entry_.connect_config_.emplace(RouteEntry::ConnectConfig());
   }
 
   MOCK_METHOD(void, onUpstream100ContinueHeaders,
@@ -89,6 +94,8 @@ class MockRouterFilterInterface : public RouterFilterInterface {
   MOCK_METHOD(TimeSource&, timeSource, ());
 
   NiceMock<Http::MockStreamDecoderFilterCallbacks> callbacks_;
+  NiceMock<MockRouteEntry> route_entry_;
+  NiceMock<Network::MockConnection> client_connection_;
 
   envoy::extensions::filters::http::router::v3::Router router_proto;
   NiceMock<Server::Configuration::MockFactoryContext> context_;
@@ -173,6 +180,7 @@ class TcpUpstreamTest : public ::testing::Test {
 
 TEST_F(TcpUpstreamTest, Basic) {
   // Swallow the headers.
+  EXPECT_CALL(connection_, write(_, false)).Times(0);
   tcp_upstream_->encodeHeaders(request_, false);
 
   // Proxy the data.
@@ -197,6 +205,52 @@ TEST_F(TcpUpstreamTest, Basic) {
   tcp_upstream_->onUpstreamData(response2, false);
 }
 
+TEST_F(TcpUpstreamTest, V1Header) {
+  envoy::config::core::v3::ProxyProtocolConfig* proxy_config =
+      mock_router_filter_.route_entry_.connect_config_->mutable_proxy_protocol_config();
+  proxy_config->set_version(envoy::config::core::v3::ProxyProtocolConfig::V1);
+  mock_router_filter_.client_connection_.remote_address_ =
+      std::make_shared<Network::Address::Ipv4Instance>("1.2.3.4", 5);
+  mock_router_filter_.client_connection_.local_address_ =
+      std::make_shared<Network::Address::Ipv4Instance>("4.5.6.7", 8);
+
+  Buffer::OwnedImpl expected_data;
+  Extensions::Common::ProxyProtocol::generateProxyProtoHeader(
+      *proxy_config, mock_router_filter_.client_connection_, expected_data);
+
+  // encodeHeaders now results in the proxy proto header being sent.
+  EXPECT_CALL(connection_, write(BufferEqual(&expected_data), false));
+  tcp_upstream_->encodeHeaders(request_, false);
+
+  // Data is proxied as usual.
+  EXPECT_CALL(connection_, write(BufferStringEqual("foo"), false));
+  Buffer::OwnedImpl buffer("foo");
+  tcp_upstream_->encodeData(buffer, false);
+}
+
+TEST_F(TcpUpstreamTest, V2Header) {
+  envoy::config::core::v3::ProxyProtocolConfig* proxy_config =
+      mock_router_filter_.route_entry_.connect_config_->mutable_proxy_protocol_config();
+  proxy_config->set_version(envoy::config::core::v3::ProxyProtocolConfig::V2);
+  mock_router_filter_.client_connection_.remote_address_ =
+      std::make_shared<Network::Address::Ipv4Instance>("1.2.3.4", 5);
+  mock_router_filter_.client_connection_.local_address_ =
+      std::make_shared<Network::Address::Ipv4Instance>("4.5.6.7", 8);
+
+  Buffer::OwnedImpl expected_data;
+  Extensions::Common::ProxyProtocol::generateProxyProtoHeader(
+      *proxy_config, mock_router_filter_.client_connection_, expected_data);
+
+  // encodeHeaders now results in the proxy proto header being sent.
+  EXPECT_CALL(connection_, write(BufferEqual(&expected_data), false));
+  tcp_upstream_->encodeHeaders(request_, false);
+
+  // Data is proxied as usual.
+  EXPECT_CALL(connection_, write(BufferStringEqual("foo"), false));
+  Buffer::OwnedImpl buffer("foo");
+  tcp_upstream_->encodeData(buffer, false);
+}
+
 TEST_F(TcpUpstreamTest, TrailersEndStream) {
   // Swallow the headers.
   tcp_upstream_->encodeHeaders(request_, false);

test/extensions/common/proxy_protocol/BUILD

@@ -14,6 +14,7 @@ envoy_cc_test(
     deps = [
         "//source/common/buffer:buffer_lib",
         "//source/extensions/common/proxy_protocol:proxy_protocol_header_lib",
+        "//test/mocks/network:connection_mocks",
         "//test/test_common:utility_lib",
     ],
 )

test/extensions/common/proxy_protocol/proxy_protocol_header_test.cc

@@ -4,6 +4,7 @@
 
 #include "extensions/common/proxy_protocol/proxy_protocol_header.h"
 
+#include "test/mocks/network/connection.h"
 #include "test/test_common/utility.h"
 
 #include "gmock/gmock.h"
@@ -28,6 +29,16 @@ TEST(ProxyProtocolHeaderTest, GeneratesV1IPv4Header) {
   generateV1Header(src_addr, dst_addr, src_port, dst_port, version, buff);
 
   EXPECT_TRUE(TestUtility::buffersEqual(expectedBuff, buff));
+
+  // Make sure the wrapper utility generates the same output.
+  testing::NiceMock<Network::MockClientConnection> connection;
+  connection.remote_address_ = Network::Utility::resolveUrl("tcp://174.2.2.222:50000");
+  connection.local_address_ = Network::Utility::resolveUrl("tcp://172.0.0.1:80");
+  Buffer::OwnedImpl util_buf;
+  envoy::config::core::v3::ProxyProtocolConfig config;
+  config.set_version(envoy::config::core::v3::ProxyProtocolConfig::V1);
+  generateProxyProtoHeader(config, connection, util_buf);
+  EXPECT_TRUE(TestUtility::buffersEqual(expectedBuff, util_buf));
 }
 
 TEST(ProxyProtocolHeaderTest, GeneratesV1IPv6Header) {
@@ -79,6 +90,16 @@ TEST(ProxyProtocolHeaderTest, GeneratesV2IPv6Header) {
   generateV2Header(src_addr, dst_addr, src_port, dst_port, version, buff);
 
   EXPECT_TRUE(TestUtility::buffersEqual(expectedBuff, buff));
+
+  // Make sure the wrapper utility generates the same output.
+  testing::NiceMock<Network::MockConnection> connection;
+  connection.remote_address_ = Network::Utility::resolveUrl("tcp://[1:2:3::4]:8");
+  connection.local_address_ = Network::Utility::resolveUrl("tcp://[1:100:200:3::]:2");
+  Buffer::OwnedImpl util_buf;
+  envoy::config::core::v3::ProxyProtocolConfig config;
+  config.set_version(envoy::config::core::v3::ProxyProtocolConfig::V2);
+  generateProxyProtoHeader(config, connection, util_buf);
+  EXPECT_TRUE(TestUtility::buffersEqual(expectedBuff, util_buf));
 }
 
 TEST(ProxyProtocolHeaderTest, GeneratesV2LocalHeader) {
@@ -96,4 +117,4 @@ TEST(ProxyProtocolHeaderTest, GeneratesV2LocalHeader) {
 } // namespace ProxyProtocol
 } // namespace Common
 } // namespace Extensions
-} // namespace Envoy
+} // namespace Envoy