[network] add so_mark as a socket option (#5352)

IP Source transparency involves non-local IP addresses being routed as though they were local. This requires some magic in the stack to ensure that those flows are sent back to envoy from the upstream host, rather than back to the original source IP address. We plan on using SO_MARK to do this. So, add it into the socket option factory. My intention is for it to be used by a follow-up PR to #5337.

This was cherry-picked from PR #5035 where it was already reviewed. I plan on closing that PR.

Risk Level: Low. No code invoked in production yet.
Testing: Ran newly added UT and other UT in network.
Docs Changes: None until we expose this through config.
Release Notes: None

Signed-off-by: Kyle Larose <kyle@agilicus.com>

source/common/network/socket_option_factory.cc

@@ -8,7 +8,7 @@ namespace Network {
 
 std::unique_ptr<Socket::Options>
 SocketOptionFactory::buildTcpKeepaliveOptions(Network::TcpKeepaliveConfig keepalive_config) {
-  std::unique_ptr<Socket::Options> options = absl::make_unique<Socket::Options>();
+  std::unique_ptr<Socket::Options> options = std::make_unique<Socket::Options>();
   options->push_back(std::make_shared<Network::SocketOptionImpl>(
       envoy::api::v2::core::SocketOption::STATE_PREBIND, ENVOY_SOCKET_SO_KEEPALIVE, 1));
 
@@ -31,15 +31,15 @@ SocketOptionFactory::buildTcpKeepaliveOptions(Network::TcpKeepaliveConfig keepal
 }
 
 std::unique_ptr<Socket::Options> SocketOptionFactory::buildIpFreebindOptions() {
-  std::unique_ptr<Socket::Options> options = absl::make_unique<Socket::Options>();
+  std::unique_ptr<Socket::Options> options = std::make_unique<Socket::Options>();
   options->push_back(std::make_shared<Network::AddrFamilyAwareSocketOptionImpl>(
       envoy::api::v2::core::SocketOption::STATE_PREBIND, ENVOY_SOCKET_IP_FREEBIND,
       ENVOY_SOCKET_IPV6_FREEBIND, 1));
   return options;
 }
 
 std::unique_ptr<Socket::Options> SocketOptionFactory::buildIpTransparentOptions() {
-  std::unique_ptr<Socket::Options> options = absl::make_unique<Socket::Options>();
+  std::unique_ptr<Socket::Options> options = std::make_unique<Socket::Options>();
   options->push_back(std::make_shared<Network::AddrFamilyAwareSocketOptionImpl>(
       envoy::api::v2::core::SocketOption::STATE_PREBIND, ENVOY_SOCKET_IP_TRANSPARENT,
       ENVOY_SOCKET_IPV6_TRANSPARENT, 1));
@@ -49,9 +49,18 @@ std::unique_ptr<Socket::Options> SocketOptionFactory::buildIpTransparentOptions(
   return options;
 }
 
+std::unique_ptr<Socket::Options> SocketOptionFactory::buildSocketMarkOptions(uint32_t mark) {
+  std::unique_ptr<Socket::Options> options = std::make_unique<Socket::Options>();
+  // we need this to happen prior to binding or prior to connecting. In both cases, PREBIND will
+  // fire.
+  options->push_back(std::make_shared<Network::SocketOptionImpl>(
+      envoy::api::v2::core::SocketOption::STATE_PREBIND, ENVOY_SOCKET_SO_MARK, mark));
+  return options;
+}
+
 std::unique_ptr<Socket::Options> SocketOptionFactory::buildLiteralOptions(
     const Protobuf::RepeatedPtrField<envoy::api::v2::core::SocketOption>& socket_options) {
-  auto options = absl::make_unique<Socket::Options>();
+  auto options = std::make_unique<Socket::Options>();
   for (const auto& socket_option : socket_options) {
     std::string buf;
     int int_value;
@@ -78,7 +87,7 @@ std::unique_ptr<Socket::Options> SocketOptionFactory::buildLiteralOptions(
 
 std::unique_ptr<Socket::Options>
 SocketOptionFactory::buildTcpFastOpenOptions(uint32_t queue_length) {
-  std::unique_ptr<Socket::Options> options = absl::make_unique<Socket::Options>();
+  std::unique_ptr<Socket::Options> options = std::make_unique<Socket::Options>();
   options->push_back(std::make_shared<Network::SocketOptionImpl>(
       envoy::api::v2::core::SocketOption::STATE_LISTENING, ENVOY_SOCKET_TCP_FASTOPEN,
       queue_length));

source/common/network/socket_option_factory.h

@@ -29,6 +29,7 @@ class SocketOptionFactory : Logger::Loggable<Logger::Id::connection> {
   buildTcpKeepaliveOptions(Network::TcpKeepaliveConfig keepalive_config);
   static std::unique_ptr<Socket::Options> buildIpFreebindOptions();
   static std::unique_ptr<Socket::Options> buildIpTransparentOptions();
+  static std::unique_ptr<Socket::Options> buildSocketMarkOptions(uint32_t mark);
   static std::unique_ptr<Socket::Options> buildTcpFastOpenOptions(uint32_t queue_length);
   static std::unique_ptr<Socket::Options> buildLiteralOptions(
       const Protobuf::RepeatedPtrField<envoy::api::v2::core::SocketOption>& socket_options);

source/common/network/socket_option_impl.h

@@ -47,6 +47,12 @@ namespace Network {
 #define ENVOY_SOCKET_SO_KEEPALIVE Network::SocketOptionName()
 #endif
 
+#ifdef SO_MARK
+#define ENVOY_SOCKET_SO_MARK Network::SocketOptionName(std::make_pair(SOL_SOCKET, SO_MARK))
+#else
+#define ENVOY_SOCKET_SO_MARK Network::SocketOptionName()
+#endif
+
 #ifdef TCP_KEEPCNT
 #define ENVOY_SOCKET_TCP_KEEPCNT Network::SocketOptionName(std::make_pair(IPPROTO_TCP, TCP_KEEPCNT))
 #else

test/common/network/BUILD

@@ -183,6 +183,20 @@ envoy_cc_test(
     ],
 )
 
+envoy_cc_test(
+    name = "socket_option_factory_test",
+    srcs = ["socket_option_factory_test.cc"],
+    deps = [
+        "//source/common/network:address_lib",
+        "//source/common/network:socket_option_factory_lib",
+        "//source/common/network:socket_option_lib",
+        "//test/mocks/api:api_mocks",
+        "//test/mocks/network:network_mocks",
+        "//test/test_common:environment_lib",
+        "//test/test_common:threadsafe_singleton_injector_lib",
+    ],
+)
+
 envoy_cc_test(
     name = "addr_family_aware_socket_option_impl_test",
     srcs = ["addr_family_aware_socket_option_impl_test.cc"],

test/common/network/socket_option_factory_test.cc

@@ -0,0 +1,126 @@
+#include "common/network/address_impl.h"
+#include "common/network/socket_option_factory.h"
+#include "common/network/socket_option_impl.h"
+
+#include "test/mocks/api/mocks.h"
+#include "test/mocks/network/mocks.h"
+#include "test/test_common/threadsafe_singleton_injector.h"
+
+#include "gtest/gtest.h"
+
+using testing::_;
+
+namespace Envoy {
+namespace Network {
+
+class SocketOptionFactoryTest : public testing::Test {
+public:
+  SocketOptionFactoryTest() = default;
+
+  TestThreadsafeSingletonInjector<Api::OsSysCallsImpl> os_calls_{[this]() {
+    // Before injecting OsSysCallsImpl, make sure validateIpv{4,6}Supported is called so the static
+    // bool is initialized without requiring to mock ::socket and ::close. :( :(
+    std::make_unique<Address::Ipv4Instance>("1.2.3.4", 5678);
+    std::make_unique<Address::Ipv6Instance>("::1:2:3:4", 5678);
+    return &os_sys_calls_mock_;
+  }()};
+
+protected:
+  testing::NiceMock<MockListenSocket> socket_mock_;
+  Api::MockOsSysCalls os_sys_calls_mock_;
+
+  void SetUp() override { socket_mock_.local_address_.reset(); }
+  void makeSocketV4() {
+    socket_mock_.local_address_ = std::make_unique<Address::Ipv4Instance>("1.2.3.4", 5678);
+  }
+  void makeSocketV6() {
+    socket_mock_.local_address_ = std::make_unique<Address::Ipv6Instance>("::1:2:3:4", 5678);
+  }
+};
+
+#define CHECK_OPTION_SUPPORTED(option)                                                             \
+  if (!option.has_value()) {                                                                       \
+    return;                                                                                        \
+  }
+
+// TODO(klarose): Simplify these tests once https://github.com/envoyproxy/envoy/pull/5351 is merged.
+
+TEST_F(SocketOptionFactoryTest, TestBuildSocketMarkOptions) {
+
+  // use a shared_ptr due to applyOptions requiring one
+  std::shared_ptr<Socket::Options> options = SocketOptionFactory::buildSocketMarkOptions(100);
+
+  const auto expected_option = ENVOY_SOCKET_SO_MARK;
+  CHECK_OPTION_SUPPORTED(expected_option);
+
+  const int type = expected_option.value().first;
+  const int option = expected_option.value().second;
+  EXPECT_CALL(os_sys_calls_mock_, setsockopt_(_, _, _, _, sizeof(int)))
+      .WillOnce(Invoke([type, option](int, int input_type, int input_option, const void* optval,
+                                      socklen_t) -> int {
+        EXPECT_EQ(100, *static_cast<const int*>(optval));
+        EXPECT_EQ(type, input_type);
+        EXPECT_EQ(option, input_option);
+        return 0;
+      }));
+
+  EXPECT_TRUE(Network::Socket::applyOptions(options, socket_mock_,
+                                            envoy::api::v2::core::SocketOption::STATE_PREBIND));
+}
+
+TEST_F(SocketOptionFactoryTest, TestBuildIpv4TransparentOptions) {
+  makeSocketV4();
+
+  // use a shared_ptr due to applyOptions requiring one
+  std::shared_ptr<Socket::Options> options = SocketOptionFactory::buildIpTransparentOptions();
+
+  const auto expected_option = ENVOY_SOCKET_IP_TRANSPARENT;
+  CHECK_OPTION_SUPPORTED(expected_option);
+
+  const int type = expected_option.value().first;
+  const int option = expected_option.value().second;
+  EXPECT_CALL(os_sys_calls_mock_, setsockopt_(_, _, _, _, sizeof(int)))
+      .Times(2)
+      .WillRepeatedly(Invoke([type, option](int, int input_type, int input_option,
+                                            const void* optval, socklen_t) -> int {
+        EXPECT_EQ(type, input_type);
+        EXPECT_EQ(option, input_option);
+        EXPECT_EQ(1, *static_cast<const int*>(optval));
+        return 0;
+      }));
+
+  EXPECT_TRUE(Network::Socket::applyOptions(options, socket_mock_,
+                                            envoy::api::v2::core::SocketOption::STATE_PREBIND));
+  EXPECT_TRUE(Network::Socket::applyOptions(options, socket_mock_,
+                                            envoy::api::v2::core::SocketOption::STATE_BOUND));
+}
+
+TEST_F(SocketOptionFactoryTest, TestBuildIpv6TransparentOptions) {
+  makeSocketV6();
+
+  // use a shared_ptr due to applyOptions requiring one
+  std::shared_ptr<Socket::Options> options = SocketOptionFactory::buildIpTransparentOptions();
+
+  const auto expected_option = ENVOY_SOCKET_IPV6_TRANSPARENT;
+  CHECK_OPTION_SUPPORTED(expected_option);
+
+  const int type = expected_option.value().first;
+  const int option = expected_option.value().second;
+  EXPECT_CALL(os_sys_calls_mock_, setsockopt_(_, _, _, _, sizeof(int)))
+      .Times(2)
+      .WillRepeatedly(Invoke([type, option](int, int input_type, int input_option,
+                                            const void* optval, socklen_t) -> int {
+        EXPECT_EQ(type, input_type);
+        EXPECT_EQ(option, input_option);
+        EXPECT_EQ(1, *static_cast<const int*>(optval));
+        return 0;
+      }));
+
+  EXPECT_TRUE(Network::Socket::applyOptions(options, socket_mock_,
+                                            envoy::api::v2::core::SocketOption::STATE_PREBIND));
+  EXPECT_TRUE(Network::Socket::applyOptions(options, socket_mock_,
+                                            envoy::api::v2::core::SocketOption::STATE_BOUND));
+}
+
+} // namespace Network
+} // namespace Envoy