Skip to content

Commit

Permalink
network: support interface binding on Android (#1897)
Browse files Browse the repository at this point in the history
Description: `SO_BINDTODEVICE` will generally only be useable by an Android app running as root, so the present interface-binding approach on Android is mostly a non-starter. Instead, this change binds the interface indirectly by finding the local IP of the interface and setting that as the source IP of the socket.

To do so, it passes a "synthetic" socket option through to the internal connection API. When applied, instead of setting an actual socket option on the socket, this object sets the source IP property on the connection before bind() is called.
Risk Level: Moderate
Testing: Local & On Device

Signed-off-by: Mike Schore <mike.schore@gmail.com>
  • Loading branch information
goaway authored Nov 1, 2021
1 parent b1a8c1b commit 41eba57
Show file tree
Hide file tree
Showing 8 changed files with 194 additions and 53 deletions.
34 changes: 17 additions & 17 deletions library/common/engine.cc
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,10 @@ envoy_status_t Engine::main(const std::string config, const std::string log_leve

network_configurator_ =
Network::ConfiguratorFactory{server_->serverFactoryContext()}.get();
logInterfaces();
auto v4_interfaces = network_configurator_->enumerateV4Interfaces();
auto v6_interfaces = network_configurator_->enumerateV4Interfaces();
logInterfaces("netconf_get_v4_interfaces", v4_interfaces);
logInterfaces("netconf_get_v6_interfaces", v6_interfaces);
client_scope_ = server_->serverFactoryContext().scope().createScope("pulse.");
// StatNameSet is lock-free, the benefit of using it is being able to create StatsName
// on-the-fly without risking contention on system with lots of threads.
Expand Down Expand Up @@ -300,22 +303,19 @@ void Engine::drainConnections() {
server_->clusterManager().drainConnections();
}

void Engine::logInterfaces() {
auto v4_vec = network_configurator_->enumerateV4Interfaces();
auto v4_vec_unique_end = std::unique(v4_vec.begin(), v4_vec.end());
std::string v4_names = std::accumulate(v4_vec.begin(), v4_vec_unique_end, std::string{},
[](std::string acc, std::string next) {
return acc.empty() ? next : std::move(acc) + "," + next;
});

auto v6_vec = network_configurator_->enumerateV6Interfaces();
auto v6_vec_unique_end = std::unique(v6_vec.begin(), v6_vec.end());
std::string v6_names = std::accumulate(v6_vec.begin(), v6_vec_unique_end, std::string{},
[](std::string acc, std::string next) {
return acc.empty() ? next : std::move(acc) + "," + next;
});
ENVOY_LOG_EVENT(debug, "netconf_get_v4_interfaces", v4_names);
ENVOY_LOG_EVENT(debug, "netconf_get_v6_interfaces", v6_names);
void Engine::logInterfaces(absl::string_view event,
std::vector<Network::InterfacePair>& interfaces) {
std::vector<std::string> names;
names.resize(interfaces.size());
std::transform(interfaces.begin(), interfaces.end(), names.begin(),
[](Network::InterfacePair& pair) { return std::get<0>(pair); });

auto unique_end = std::unique(names.begin(), names.end());
std::string all_names = std::accumulate(names.begin(), unique_end, std::string{},
[](std::string acc, std::string next) {
return acc.empty() ? next : std::move(acc) + "," + next;
});
ENVOY_LOG_EVENT(debug, event, all_names);
}

} // namespace Envoy
3 changes: 2 additions & 1 deletion library/common/engine.h
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,8 @@ class Engine : public Logger::Loggable<Logger::Id::main> {

private:
envoy_status_t main(std::string config, std::string log_level);
void logInterfaces();
static void logInterfaces(absl::string_view event,
std::vector<Network::InterfacePair>& interfaces);

Event::Dispatcher* event_dispatcher_{};
Stats::ScopePtr client_scope_;
Expand Down
13 changes: 13 additions & 0 deletions library/common/network/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ envoy_cc_library(
}),
repository = "@envoy",
deps = [
"//library/common/network:src_addr_socket_option_lib",
"//library/common/types:c_types_lib",
"@envoy//envoy/network:socket_interface",
"@envoy//envoy/singleton:manager_interface",
Expand All @@ -35,6 +36,18 @@ envoy_cc_library(
],
)

envoy_cc_library(
name = "src_addr_socket_option_lib",
srcs = ["src_addr_socket_option_impl.cc"],
hdrs = ["src_addr_socket_option_impl.h"],
repository = "@envoy",
deps = [
"@envoy//envoy/network:address_interface",
"@envoy//source/common/network:socket_option_lib",
"@envoy_api//envoy/config/core/v3:pkg_cc_proto",
],
)

envoy_cc_library(
name = "synthetic_address_lib",
hdrs = ["synthetic_address_impl.h"],
Expand Down
78 changes: 50 additions & 28 deletions library/common/network/configurator.cc
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,12 @@
#include "source/common/common/scalar_to_byte_vector.h"
#include "source/common/common/utility.h"
#include "source/common/network/addr_family_aware_socket_option_impl.h"
#include "source/common/network/address_impl.h"
#include "source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h"

// Used on Linux/Android
#include "library/common/network/src_addr_socket_option_impl.h"

// Used on Linux (requires root/CAP_NET_RAW)
#ifdef SO_BINDTODEVICE
#define ENVOY_SOCKET_SO_BINDTODEVICE ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_BINDTODEVICE)
#else
Expand Down Expand Up @@ -152,10 +155,11 @@ void Configurator::reportNetworkUsage(envoy_netconf_t configuration_key, bool ne
if (network_state_.socket_mode_ == DefaultPreferredNetworkMode) {
ENVOY_LOG_EVENT(debug, "netconf_mode_switch", "DefaultPreferredNetworkMode");
} else if (network_state_.socket_mode_ == AlternateBoundInterfaceMode) {
auto& v4_interface = getActiveAlternateInterface(network_state_.network_, AF_INET);
auto& v6_interface = getActiveAlternateInterface(network_state_.network_, AF_INET6);
auto v4_pair = getActiveAlternateInterface(network_state_.network_, AF_INET);
auto v6_pair = getActiveAlternateInterface(network_state_.network_, AF_INET6);
ENVOY_LOG_EVENT(debug, "netconf_mode_switch", "AlternateBoundInterfaceMode [{}|{}]",
v4_interface, v6_interface);
std::get<const std::string>(v4_pair),
std::get<const std::string>(v6_pair));
}
}
}
Expand Down Expand Up @@ -192,11 +196,11 @@ void Configurator::refreshDns(envoy_netconf_t configuration_key) {
}
}

std::vector<std::string> Configurator::enumerateV4Interfaces() {
std::vector<InterfacePair> Configurator::enumerateV4Interfaces() {
return enumerateInterfaces(AF_INET, 0, 0);
}

std::vector<std::string> Configurator::enumerateV6Interfaces() {
std::vector<InterfacePair> Configurator::enumerateV6Interfaces() {
return enumerateInterfaces(AF_INET6, 0, 0);
}

Expand All @@ -220,26 +224,36 @@ Socket::OptionsSharedPtr Configurator::getUpstreamSocketOptions(envoy_network_t
}

Socket::OptionsSharedPtr Configurator::getAlternateInterfaceSocketOptions(envoy_network_t network) {
auto& v4_interface = getActiveAlternateInterface(network, AF_INET);
auto& v6_interface = getActiveAlternateInterface(network, AF_INET6);
auto v4_pair = getActiveAlternateInterface(network, AF_INET);
auto v6_pair = getActiveAlternateInterface(network, AF_INET6);
ENVOY_LOG(debug, "found active alternate interface (ipv4): {} {}", std::get<0>(v4_pair),
std::get<1>(v4_pair));
ENVOY_LOG(debug, "found active alternate interface (ipv6): {} {}", std::get<0>(v6_pair),
std::get<1>(v6_pair));

auto options = std::make_shared<Socket::Options>();

// Android
#ifdef SO_BINDTODEVICE
options->push_back(std::make_shared<AddrFamilyAwareSocketOptionImpl>(
envoy::config::core::v3::SocketOption::STATE_PREBIND, ENVOY_SOCKET_SO_BINDTODEVICE,
v4_interface, ENVOY_SOCKET_SO_BINDTODEVICE, v6_interface));
#endif // SO_BINDTODEVICE

// iOS
#ifdef IP_BOUND_IF
int v4_idx = if_nametoindex(v4_interface.c_str());
int v6_idx = if_nametoindex(v6_interface.c_str());
// iOS
// On platforms where it exists, IP_BOUND_IF/IPV6_BOUND_IF provide a straightforward way to bind
// a socket explicitly to specific interface. (The Linux alternative is SO_BINDTODEVICE, but has
// other restriction; see below.)
int v4_idx = if_nametoindex(std::get<const std::string>(v4_pair).c_str());
int v6_idx = if_nametoindex(std::get<const std::string>(v6_pair).c_str());
options->push_back(std::make_shared<AddrFamilyAwareSocketOptionImpl>(
envoy::config::core::v3::SocketOption::STATE_PREBIND, ENVOY_SOCKET_IP_BOUND_IF, v4_idx,
ENVOY_SOCKET_IPV6_BOUND_IF, v6_idx));
#endif // IP_BOUND_IF
#else
// Android
// SO_BINDTODEVICE is defined on Android, but applying it requires root privileges (or more
// specifically, CAP_NET_RAW). As a workaround, this binds the socket to the interface by
// attaching "synthetic" socket option, which sets the socket's source address to the local
// address of the interface. This is not quite as precise, since it's possible that multiple
// interfaces share the same local address, but this is all best-effort anyways.
options->push_back(std::make_shared<AddrFamilyAwareSocketOptionImpl>(
std::make_unique<SrcAddrSocketOptionImpl>(std::get<1>(v4_pair)),
std::make_unique<SrcAddrSocketOptionImpl>(std::get<1>(v6_pair))));
#endif

return options;
}
Expand All @@ -261,30 +275,30 @@ envoy_netconf_t Configurator::addUpstreamSocketOptions(Socket::OptionsSharedPtr
return configuration_key;
}

const std::string Configurator::getActiveAlternateInterface(envoy_network_t network,
unsigned short family) {
InterfacePair Configurator::getActiveAlternateInterface(envoy_network_t network,
unsigned short family) {
// Attempt to derive an active interface that differs from the passed network parameter.
if (network == ENVOY_NET_WWAN) {
// Network is cellular, so look for a WiFi interface.
// WiFi should always support multicast, and will not be point-to-point.
auto interfaces =
enumerateInterfaces(family, IFF_UP | IFF_MULTICAST, IFF_LOOPBACK | IFF_POINTOPOINT);
return interfaces.size() > 0 ? interfaces[0] : "";
return interfaces.size() > 0 ? interfaces[0] : std::make_pair("", nullptr);
} else if (network == ENVOY_NET_WLAN) {
// Network is WiFi, so look for a cellular interface.
// Cellular networks should be point-to-point.
auto interfaces = enumerateInterfaces(family, IFF_UP | IFF_POINTOPOINT, IFF_LOOPBACK);
return interfaces.size() > 0 ? interfaces[0] : "";
return interfaces.size() > 0 ? interfaces[0] : std::make_pair("", nullptr);
} else {
return "";
return std::make_pair("", nullptr);
}
}

std::vector<std::string>
std::vector<InterfacePair>
Configurator::enumerateInterfaces([[maybe_unused]] unsigned short family,
[[maybe_unused]] unsigned int select_flags,
[[maybe_unused]] unsigned int reject_flags) {
std::vector<std::string> names{};
std::vector<InterfacePair> pairs{};

#ifdef SUPPORTS_GETIFADDRS
struct ifaddrs* interfaces = nullptr;
Expand All @@ -300,13 +314,21 @@ Configurator::enumerateInterfaces([[maybe_unused]] unsigned short family,
if ((ifa->ifa_flags & (select_flags ^ reject_flags)) != select_flags) {
continue;
}
names.push_back(std::string{ifa->ifa_name});

const sockaddr_storage* ss = reinterpret_cast<sockaddr_storage*>(ifa->ifa_addr);
size_t ss_len = family == AF_INET ? sizeof(sockaddr_in) : sizeof(sockaddr_in6);
StatusOr<Address::InstanceConstSharedPtr> address =
Address::addressFromSockAddr(*ss, ss_len, family == AF_INET6);
if (!address.ok()) {
continue;
}
pairs.push_back(std::make_pair(std::string{ifa->ifa_name}, *address));
}

freeifaddrs(interfaces);
#endif // SUPPORTS_GETIFADDRS

return names;
return pairs;
}

ConfiguratorSharedPtr ConfiguratorFactory::get() {
Expand Down
11 changes: 6 additions & 5 deletions library/common/network/configurator.h
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ namespace Envoy {
namespace Network {

using DnsCacheManagerSharedPtr = Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr;
using InterfacePair = std::pair<const std::string, Address::InstanceConstSharedPtr>;

/**
* Object responsible for tracking network state, especially with respect to multiple interfaces,
Expand All @@ -68,21 +69,21 @@ class Configurator : public Logger::Loggable<Logger::Id::upstream>, public Singl
/**
* @returns a list of local network interfaces supporting IPv4.
*/
std::vector<std::string> enumerateV4Interfaces();
std::vector<InterfacePair> enumerateV4Interfaces();

/**
* @returns a list of local network interfaces supporting IPv6.
*/
std::vector<std::string> enumerateV6Interfaces();
std::vector<InterfacePair> enumerateV6Interfaces();

/**
* @param family, network family of the interface.
* @param select_flags, flags which MUST be set for each returned interface.
* @param reject_flags, flags which MUST NOT be set for any returned interface.
* @returns a list of local network interfaces filtered by the providered flags.
*/
std::vector<std::string> enumerateInterfaces(unsigned short family, unsigned int select_flags,
unsigned int reject_flags);
std::vector<InterfacePair> enumerateInterfaces(unsigned short family, unsigned int select_flags,
unsigned int reject_flags);

/**
* @returns the current OS default/preferred network class.
Expand Down Expand Up @@ -149,7 +150,7 @@ class Configurator : public Logger::Loggable<Logger::Id::upstream>, public Singl
Thread::MutexBasicLockable mutex_;
};
Socket::OptionsSharedPtr getAlternateInterfaceSocketOptions(envoy_network_t network);
const std::string getActiveAlternateInterface(envoy_network_t network, unsigned short family);
InterfacePair getActiveAlternateInterface(envoy_network_t network, unsigned short family);

bool enable_interface_binding_;
DnsCacheManagerSharedPtr dns_cache_manager_;
Expand Down
58 changes: 58 additions & 0 deletions library/common/network/src_addr_socket_option_impl.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#include "library/common/network/src_addr_socket_option_impl.h"

#include "envoy/config/core/v3/base.pb.h"

#include "source/common/common/assert.h"

namespace Envoy {
namespace Network {

SrcAddrSocketOptionImpl::SrcAddrSocketOptionImpl(
Network::Address::InstanceConstSharedPtr source_address)
: source_address_(std::move(source_address)) {
ASSERT(source_address_->type() == Network::Address::Type::Ip);
}

bool SrcAddrSocketOptionImpl::setOption(
Network::Socket& socket, envoy::config::core::v3::SocketOption::SocketState state) const {

if (state == envoy::config::core::v3::SocketOption::STATE_PREBIND) {
socket.connectionInfoProvider().setLocalAddress(source_address_);
}

return true;
}

/**
* Inserts an address, already in network order, to a byte array.
*/
template <typename T> void addressIntoVector(std::vector<uint8_t>& vec, const T& address) {
const uint8_t* byte_array = reinterpret_cast<const uint8_t*>(&address);
vec.insert(vec.end(), byte_array, byte_array + sizeof(T));
}

void SrcAddrSocketOptionImpl::hashKey(std::vector<uint8_t>& key) const {

// Note: we're assuming that there cannot be a conflict between IPv6 addresses here. If an IPv4
// address is mapped into an IPv6 address using an IPv4-Mapped IPv6 Address (RFC4921), then it's
// possible the hashes will be different despite the IP address used by the connection being
// the same.
if (source_address_->ip()->version() == Network::Address::IpVersion::v4) {
// note raw_address is already in network order
uint32_t raw_address = source_address_->ip()->ipv4()->address();
addressIntoVector(key, raw_address);
} else if (source_address_->ip()->version() == Network::Address::IpVersion::v6) {
// note raw_address is already in network order
absl::uint128 raw_address = source_address_->ip()->ipv6()->address();
addressIntoVector(key, raw_address);
}
}

absl::optional<Network::Socket::Option::Details> SrcAddrSocketOptionImpl::getOptionDetails(
const Network::Socket&, envoy::config::core::v3::SocketOption::SocketState) const {
// no details for this option.
return absl::nullopt;
}

} // namespace Network
} // namespace Envoy
46 changes: 46 additions & 0 deletions library/common/network/src_addr_socket_option_impl.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#pragma once

#include "envoy/config/core/v3/base.pb.h"
#include "envoy/network/address.h"
#include "envoy/network/listen_socket.h"

namespace Envoy {
namespace Network {

/**
* This is a "synthetic" socket option implementation, which sets the source IP/port of a socket
* using a provided IP address (and maybe port) during bind.
*
* Based on the OriginalSrcSocketOption extension.
*/
class SrcAddrSocketOptionImpl : public Network::Socket::Option {
public:
/**
* Constructs a socket option which will set the socket to use source @c source_address
*/
SrcAddrSocketOptionImpl(Network::Address::InstanceConstSharedPtr source_address);
~SrcAddrSocketOptionImpl() override = default;

/**
* Updates the source address of the socket to match `source_address_`.
* Adds socket options to the socket to allow this to work.
*/
bool setOption(Network::Socket& socket,
envoy::config::core::v3::SocketOption::SocketState state) const override;

/**
* Appends a key which uniquely identifies the address being tracked.
*/
void hashKey(std::vector<uint8_t>& key) const override;

absl::optional<Details>
getOptionDetails(const Network::Socket& socket,
envoy::config::core::v3::SocketOption::SocketState state) const override;
bool isSupported() const override { return true; }

private:
Network::Address::InstanceConstSharedPtr source_address_;
};

} // namespace Network
} // namespace Envoy
Loading

0 comments on commit 41eba57

Please sign in to comment.