network: implement initial heuristic for binding alternate interface (#…

…1858)

Description: Introduces an initial heuristic for attempting a connection using a socket with a bound interface. The heuristic is entirely contained within reportNetworkUsage(...), and as such is straightforward to iterate and experiment upon. In support of the heuristic, current network/configuration state is identified with a configuration_key that prevents stale information from influencing internal tracking. ~Current network state is stored and updated automatically.~ This allows static accessors to begin to report OS updates during Envoy's startup period, and ensures OS updates have a timely effect on new requests.

The initial heuristic is as follows:

For both WiFi and cellular network types, there is a default socket configuration that does not use a bound socket, and an alternative "override" configuration that will attempt to use an interface associated with the *opposite* network type (a cellular-bound socket when the preferred network type is WiFi, and a WiFi-bound socket when network type is cellular).

A "network fault" is defined as a request that terminates in error while having received no upstream bytes.
If a connection has never handled a request without a fault, it is allowed only one fault before the alternative will be tried.
If a connection has ever handled a request without a fault, it is allowed up to three consecutive faults before the alternative will be tried.

The above behavior is disabled by default, and may be enabled by settings enableInterfaceBinding(true) on the public EngineBuilders.

Risk Level: Moderate
Testing: Manual and new/updated coverage

Signed-off-by: Mike Schore <mike.schore@gmail.com>

examples/kotlin/hello_world/MainActivity.kt

@@ -47,6 +47,7 @@ class MainActivity : Activity() {
 
     engine = AndroidEngineBuilder(application)
       .addLogLevel(LogLevel.DEBUG)
+      .enableInterfaceBinding(true)
       .addPlatformFilter(::DemoFilter)
       .addPlatformFilter(::BufferDemoFilter)
       .addPlatformFilter(::AsyncDemoFilter)

library/common/extensions/filters/http/network_configuration/BUILD

@@ -23,6 +23,7 @@ envoy_cc_extension(
         ":filter_cc_proto",
         "//library/common/http:internal_headers_lib",
         "//library/common/network:configurator_lib",
+        "//library/common/stream_info:extra_stream_info_lib",
         "//library/common/types:c_types_lib",
         "@envoy//envoy/http:codes_interface",
         "@envoy//envoy/http:filter_interface",

library/common/extensions/filters/http/network_configuration/filter.cc

@@ -7,24 +7,55 @@ namespace Extensions {
 namespace HttpFilters {
 namespace NetworkConfiguration {
 
-Http::FilterHeadersStatus NetworkConfigurationFilter::decodeHeaders(Http::RequestHeaderMap&, bool) {
-  ENVOY_LOG(debug, "NetworkConfigurationFilter::decodeHeaders");
-
-  envoy_network_t network = network_configurator_->getPreferredNetwork();
-  ENVOY_LOG(debug, "current preferred network: {}", network);
-
-  if (enable_interface_binding_) {
-    override_interface_ = network_configurator_->overrideInterface(network);
-  }
-  ENVOY_LOG(debug, "will override interface: {}", override_interface_);
+void NetworkConfigurationFilter::setDecoderFilterCallbacks(
+    Http::StreamDecoderFilterCallbacks& callbacks) {
+  ENVOY_LOG(debug, "NetworkConfigurationFilter::setDecoderFilterCallbacks");
+
+  auto new_extra_stream_info = std::make_unique<StreamInfo::ExtraStreamInfo>();
+  extra_stream_info_ = new_extra_stream_info.get();
+
+  decoder_callbacks_ = &callbacks;
+  decoder_callbacks_->streamInfo().filterState()->setData(
+      StreamInfo::ExtraStreamInfo::key(), std::move(new_extra_stream_info),
+      StreamInfo::FilterState::StateType::Mutable, StreamInfo::FilterState::LifeSpan::Request);
+
+  auto options = std::make_shared<Network::Socket::Options>();
+  network_configurator_->setInterfaceBindingEnabled(enable_interface_binding_);
+  extra_stream_info_->configuration_key_ = network_configurator_->addUpstreamSocketOptions(options);
+  decoder_callbacks_->addUpstreamSocketOptions(options);
+}
 
-  auto connection_options =
-      network_configurator_->getUpstreamSocketOptions(network, override_interface_);
-  decoder_callbacks_->addUpstreamSocketOptions(connection_options);
+Http::FilterHeadersStatus NetworkConfigurationFilter::encodeHeaders(Http::ResponseHeaderMap&,
+                                                                    bool) {
+  ENVOY_LOG(debug, "NetworkConfigurationFilter::encodeHeaders");
+  // Report request status to network configurator, so that socket configuration may be adapted
+  // to current network conditions. Receiving headers from upstream always means some level of
+  // network transmission was successful, so we unconditionally set network_fault to false.
+  network_configurator_->reportNetworkUsage(extra_stream_info_->configuration_key_.value(),
+                                            false /* network_fault */);
 
   return Http::FilterHeadersStatus::Continue;
 }
 
+Http::LocalErrorStatus NetworkConfigurationFilter::onLocalReply(const LocalReplyData& reply) {
+  ENVOY_LOG(debug, "NetworkConfigurationFilter::onLocalReply");
+
+  bool success_status = static_cast<int>(reply.code_) < 400;
+  // Envoy uses local replies to report various local errors, including networking failures (which
+  // Envoy Mobile later surfaces as errors). As a proxy for the many different types of network
+  // errors, this code interprets any local error where a stream received no bytes from the upstream
+  // as a network fault. This status is passed to the configurator below when we report network
+  // usage, where it may be factored into future socket configuration.
+  bool network_fault = !success_status &&
+                       !decoder_callbacks_->streamInfo().firstUpstreamRxByteReceived().has_value();
+  // Report request status to network configurator, so that socket configuration may be adapted
+  // to current network conditions.
+  network_configurator_->reportNetworkUsage(extra_stream_info_->configuration_key_.value(),
+                                            network_fault);
+
+  return Http::LocalErrorStatus::ContinueAndResetStream;
+}
+
 } // namespace NetworkConfiguration
 } // namespace HttpFilters
 } // namespace Extensions

library/common/extensions/filters/http/network_configuration/filter.h

@@ -7,6 +7,7 @@
 
 #include "library/common/extensions/filters/http/network_configuration/filter.pb.h"
 #include "library/common/network/configurator.h"
+#include "library/common/stream_info/extra_stream_info.h"
 #include "library/common/types/c_types.h"
 
 namespace Envoy {
@@ -23,16 +24,20 @@ class NetworkConfigurationFilter final : public Http::PassThroughFilter,
   NetworkConfigurationFilter(Network::ConfiguratorSharedPtr network_configurator,
                              bool enable_interface_binding)
       : network_configurator_(network_configurator),
-        enable_interface_binding_(enable_interface_binding), override_interface_(false) {}
+        extra_stream_info_(nullptr), // always set in setDecoderFilterCallbacks
+        enable_interface_binding_(enable_interface_binding) {}
 
   // Http::StreamDecoderFilter
-  Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap& headers,
-                                          bool end_stream) override;
+  void setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) override;
+  // Http::StreamEncoderFilter
+  Http::FilterHeadersStatus encodeHeaders(Http::ResponseHeaderMap&, bool) override;
+  // Http::StreamFilterBase
+  Http::LocalErrorStatus onLocalReply(const LocalReplyData&) override;
 
 private:
   Network::ConfiguratorSharedPtr network_configurator_;
+  StreamInfo::ExtraStreamInfo* extra_stream_info_;
   bool enable_interface_binding_;
-  bool override_interface_;
 };
 
 } // namespace NetworkConfiguration

library/common/extensions/retry/options/network_configuration/BUILD

@@ -22,6 +22,7 @@ envoy_cc_extension(
     deps = [
         ":predicate_cc_proto",
         "//library/common/network:configurator_lib",
+        "//library/common/stream_info:extra_stream_info_lib",
         "@envoy//envoy/upstream:retry_interface",
     ],
 )

library/common/extensions/retry/options/network_configuration/predicate.cc

@@ -1,5 +1,7 @@
 #include "library/common/extensions/retry/options/network_configuration/predicate.h"
 
+#include "library/common/stream_info/extra_stream_info.h"
+
 namespace Envoy {
 namespace Extensions {
 namespace Retry {
@@ -15,8 +17,53 @@ NetworkConfigurationRetryOptionsPredicate::NetworkConfigurationRetryOptionsPredi
 
 Upstream::RetryOptionsPredicate::UpdateOptionsReturn
 NetworkConfigurationRetryOptionsPredicate::updateOptions(
-    const Upstream::RetryOptionsPredicate::UpdateOptionsParameters&) const {
-  return {};
+    const Upstream::RetryOptionsPredicate::UpdateOptionsParameters& parameters) const {
+
+  auto options = std::make_shared<Network::Socket::Options>();
+  auto& stream_info = parameters.retriable_request_stream_info_;
+  auto filter_state = stream_info.filterState();
+
+  // ExtraStreamInfo is added by the NetworkConfigurationFilter and should normally always be
+  // present - this check is mostly defensive.
+  if (!filter_state->hasData<StreamInfo::ExtraStreamInfo>(StreamInfo::ExtraStreamInfo::key())) {
+    ENVOY_LOG(warn, "extra stream info is missing");
+
+    // Returning nullopt results in existing socket options being preserved.
+    return Upstream::RetryOptionsPredicate::UpdateOptionsReturn{absl::nullopt};
+  }
+
+  auto& extra_stream_info =
+      filter_state->getDataMutable<StreamInfo::ExtraStreamInfo>(StreamInfo::ExtraStreamInfo::key());
+
+  // This check is also defensive. The NetworkConfigurationFilter should always set this when
+  // ExtraStreaminfo is created.
+  if (!extra_stream_info.configuration_key_.has_value()) {
+    ENVOY_LOG(warn, "network configuration key is missing");
+
+    // Returning nullopt results in existing socket options being preserved.
+    return Upstream::RetryOptionsPredicate::UpdateOptionsReturn{absl::nullopt};
+  }
+
+  // As a proxy for the many different types of network errors, this code interprets any failure
+  // where a stream received no bytes from the upstream as a network fault. This status is passed to
+  // the configurator below when we report network usage, where it may be factored into future
+  // socket configuration.
+  bool network_fault = !stream_info.firstUpstreamRxByteReceived().has_value();
+
+  // Report request status to network configurator, so that socket configuration may be adapted
+  // to current network conditions.
+  network_configurator_->reportNetworkUsage(extra_stream_info.configuration_key_.value(),
+                                            network_fault);
+
+  // Update socket configuration for next retry attempt.
+  extra_stream_info.configuration_key_ = network_configurator_->addUpstreamSocketOptions(options);
+
+  // The options returned here replace any existing socket options used for a prior attempt. At
+  // present, all socket options set in Envoy Mobile are provided by the NetworkConfigurator, so
+  // it's safe to simply replace them.
+  // TODO(goaway): If additional socket options are ever provided by a source other than the
+  // NetworkConfigurator, we need to account for the potential presence of those options here.
+  return Upstream::RetryOptionsPredicate::UpdateOptionsReturn{options};
 }
 
 } // namespace Options

library/common/extensions/retry/options/network_configuration/predicate.h

@@ -2,6 +2,8 @@
 
 #include "envoy/upstream/retry.h"
 
+#include "source/common/common/logger.h"
+
 #include "library/common/extensions/retry/options/network_configuration/predicate.pb.h"
 #include "library/common/extensions/retry/options/network_configuration/predicate.pb.validate.h"
 #include "library/common/network/configurator.h"
@@ -11,7 +13,8 @@ namespace Extensions {
 namespace Retry {
 namespace Options {
 
-class NetworkConfigurationRetryOptionsPredicate : public Upstream::RetryOptionsPredicate {
+class NetworkConfigurationRetryOptionsPredicate : public Upstream::RetryOptionsPredicate,
+                                                  public Logger::Loggable<Logger::Id::upstream> {
 public:
   NetworkConfigurationRetryOptionsPredicate(
       const envoymobile::extensions::retry::options::network_configuration::

library/common/http/BUILD

@@ -20,6 +20,7 @@ envoy_cc_library(
         "//library/common/http:header_utility_lib",
         "//library/common/network:configurator_lib",
         "//library/common/network:synthetic_address_lib",
+        "//library/common/stream_info:extra_stream_info_lib",
         "//library/common/types:c_types_lib",
         "@envoy//envoy/buffer:buffer_interface",
         "@envoy//envoy/common:scope_tracker_interface",

library/common/http/client.cc

@@ -14,6 +14,7 @@
 #include "library/common/http/header_utility.h"
 #include "library/common/http/headers.h"
 #include "library/common/network/configurator.h"
+#include "library/common/stream_info/extra_stream_info.h"
 
 namespace Envoy {
 namespace Http {

library/common/main_interface.cc

@@ -88,14 +88,12 @@ envoy_status_t reset_stream(envoy_stream_t stream) {
 }
 
 envoy_status_t set_preferred_network(envoy_network_t network) {
-  envoy_network_t previous = Envoy::Network::Configurator::setPreferredNetwork(network);
-  if (previous != network && previous != ENVOY_NET_GENERIC) {
-    if (auto e = engine()) {
-      e->dispatcher().post([network]() -> void {
-        if (auto e = engine())
-          e->networkConfigurator().refreshDns(network);
-      });
-    }
+  envoy_netconf_t configuration_key = Envoy::Network::Configurator::setPreferredNetwork(network);
+  if (auto e = engine()) {
+    e->dispatcher().post([configuration_key]() -> void {
+      if (auto e = engine())
+        e->networkConfigurator().refreshDns(configuration_key);
+    });
   }
   return ENVOY_SUCCESS;
 }