From 8997f0e7a5e3cc1305cdb87b54e1f359f5672d02 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 28 Oct 2020 20:43:57 -0400 Subject: [PATCH 1/6] http: fixing a bug with IPv6 hosts (#13798) Fixing a bug where HTTP parser offsets for IPv6 hosts did not include [] and Envoy assumed it did. This results in mis-parsing addresses for IPv6 CONNECT requests and IPv6 hosts in fully URLs over HTTP/1.1 Signed-off-by: Alyssa Wilk Signed-off-by: Shikugawa --- docs/root/version_history/current.rst | 21 +++++++++ source/common/http/utility.cc | 47 +++++++++++++++---- test/common/http/utility_test.cc | 32 +++++++++++-- .../proxy_filter_integration_test.cc | 3 +- test/integration/integration_test.cc | 36 ++++++++++++++ 5 files changed, 125 insertions(+), 14 deletions(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 84ab09df6d44..d5edb4e9a7e1 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -5,5 +5,26 @@ Changes ------- * listener: fix crash when disabling or re-enabling listeners due to overload while processing LDS updates. * proxy_proto: fixed a bug where network filters would not have the correct downstreamRemoteAddress() when accessed from the StreamInfo. This could result in incorrect enforcement of RBAC rules in the RBAC network filter (but not in the RBAC HTTP filter), or incorrect access log addresses from tcp_proxy. +Incompatible Behavior Changes +----------------------------- +*Changes that are expected to cause an incompatibility if applicable; deployment changes are likely required* + +Minor Behavior Changes +---------------------- +*Changes that may cause incompatibilities for some users, but should not for most* + +* build: the Alpine based debug images are no longer built in CI, use Ubuntu based images instead. +* cluster manager: the cluster which can't extract secret entity by SDS to be warming and never activate. This feature is disabled by default and is controlled by runtime guard `envoy.reloadable_features.cluster_keep_warming_no_secret_entity`. +* ext_authz filter: disable `envoy.reloadable_features.ext_authz_measure_timeout_on_check_created` by default. +* ext_authz filter: the deprecated field :ref:`use_alpha ` is no longer supported and cannot be set anymore. +* grpc_web filter: if a `grpc-accept-encoding` header is present it's passed as-is to the upstream and if it isn't `grpc-accept-encoding:identity` is sent instead. The header was always overwriten with `grpc-accept-encoding:identity,deflate,gzip` before. +* watchdog: the watchdog action :ref:`abort_action ` is now the default action to terminate the process if watchdog kill / multikill is enabled. + +Bug Fixes +--------- +*Changes expected to improve the state of the world and are unlikely to have negative effects* + +* http: fixed URL parsing for HTTP/1.1 fully qualified URLs and connect requests containing IPv6 addresses. +* http: sending CONNECT_ERROR for HTTP/2 where appropriate during CONNECT requests. * tls: fix read resumption after triggering buffer high-watermark and all remaining request/response bytes are stored in the SSL connection's internal buffers. * udp: fixed issue in which receiving truncated UDP datagrams would cause Envoy to crash. diff --git a/source/common/http/utility.cc b/source/common/http/utility.cc index 204743ff52f6..3c0176536805 100644 --- a/source/common/http/utility.cc +++ b/source/common/http/utility.cc @@ -207,6 +207,30 @@ namespace Http { static const char kDefaultPath[] = "/"; +// If http_parser encounters an IP address [address] as the host it will set the offset and +// length to point to 'address' rather than '[address]'. Fix this by adjusting the offset +// and length to include the brackets. +// @param absolute_url the absolute URL. This is usually of the form // http://host/path +// but may be host:port for CONNECT requests +// @param offset the offset for the first character of the host. For IPv6 hosts +// this will point to the first character inside the brackets and will be +// adjusted to point at the brackets +// @param len the length of the host-and-port field. For IPv6 hosts this will +// not include the brackets and will be adjusted to do so. +bool maybeAdjustForIpv6(absl::string_view absolute_url, uint64_t& offset, uint64_t& len) { + // According to https://tools.ietf.org/html/rfc3986#section-3.2.2 the only way a hostname + // may begin with '[' is if it's an ipv6 address. + if (offset == 0 || *(absolute_url.data() + offset - 1) != '[') { + return false; + } + // Start one character sooner and end one character later. + offset--; + len += 2; + // HTTP parser ensures that any [ has a closing ] + ASSERT(absolute_url.length() >= offset + len); + return true; +} + bool Utility::Url::initialize(absl::string_view absolute_url, bool is_connect) { struct http_parser_url u; http_parser_url_init(&u); @@ -223,20 +247,27 @@ bool Utility::Url::initialize(absl::string_view absolute_url, bool is_connect) { scheme_ = absl::string_view(absolute_url.data() + u.field_data[UF_SCHEMA].off, u.field_data[UF_SCHEMA].len); - uint16_t authority_len = u.field_data[UF_HOST].len; + uint64_t authority_len = u.field_data[UF_HOST].len; if ((u.field_set & (1 << UF_PORT)) == (1 << UF_PORT)) { authority_len = authority_len + u.field_data[UF_PORT].len + 1; } - host_and_port_ = - absl::string_view(absolute_url.data() + u.field_data[UF_HOST].off, authority_len); + + uint64_t authority_beginning = u.field_data[UF_HOST].off; + const bool is_ipv6 = maybeAdjustForIpv6(absolute_url, authority_beginning, authority_len); + host_and_port_ = absl::string_view(absolute_url.data() + authority_beginning, authority_len); + if (is_ipv6 && !parseAuthority(host_and_port_).is_ip_address_) { + return false; + } // RFC allows the absolute-uri to not end in /, but the absolute path form - // must start with - uint64_t path_len = absolute_url.length() - (u.field_data[UF_HOST].off + hostAndPort().length()); - if (path_len > 0) { - uint64_t path_beginning = u.field_data[UF_HOST].off + hostAndPort().length(); - path_and_query_params_ = absl::string_view(absolute_url.data() + path_beginning, path_len); + // must start with. Determine if there's a non-zero path, and if so determine + // the length of the path, query params etc. + uint64_t path_etc_len = absolute_url.length() - (authority_beginning + hostAndPort().length()); + if (path_etc_len > 0) { + uint64_t path_beginning = authority_beginning + hostAndPort().length(); + path_and_query_params_ = absl::string_view(absolute_url.data() + path_beginning, path_etc_len); } else if (!is_connect) { + ASSERT((u.field_set & (1 << UF_PATH)) == 0); path_and_query_params_ = absl::string_view(kDefaultPath, 1); } return true; diff --git a/test/common/http/utility_test.cc b/test/common/http/utility_test.cc index 8751acddb024..472381c7d9c2 100644 --- a/test/common/http/utility_test.cc +++ b/test/common/http/utility_test.cc @@ -1090,6 +1090,9 @@ TEST(Url, ParsingFails) { EXPECT_FALSE(url.initialize("random_scheme://host.com/path", false)); EXPECT_FALSE(url.initialize("http://www.foo.com", true)); EXPECT_FALSE(url.initialize("foo.com", true)); + EXPECT_FALSE(url.initialize("http://[notaddress]:80/?query=param", false)); + EXPECT_FALSE(url.initialize("http://[1::z::2]:80/?query=param", false)); + EXPECT_FALSE(url.initialize("http://1.2.3.4:65536/?query=param", false)); } void validateUrl(absl::string_view raw_url, absl::string_view expected_scheme, @@ -1101,12 +1104,17 @@ void validateUrl(absl::string_view raw_url, absl::string_view expected_scheme, EXPECT_EQ(url.pathAndQueryParams(), expected_path); } -void validateConnectUrl(absl::string_view raw_url, absl::string_view expected_host_port) { +void validateConnectUrl(absl::string_view raw_url) { Utility::Url url; ASSERT_TRUE(url.initialize(raw_url, true)) << "Failed to initialize " << raw_url; EXPECT_TRUE(url.scheme().empty()); EXPECT_TRUE(url.pathAndQueryParams().empty()); - EXPECT_EQ(url.hostAndPort(), expected_host_port); + EXPECT_EQ(url.hostAndPort(), raw_url); +} + +void invalidConnectUrl(absl::string_view raw_url) { + Utility::Url url; + ASSERT_FALSE(url.initialize(raw_url, true)) << "Unexpectedly initialized " << raw_url; } TEST(Url, ParsingTest) { @@ -1141,6 +1149,14 @@ TEST(Url, ParsingTest) { validateUrl("http://www.host.com:80/?query=param", "http", "www.host.com:80", "/?query=param"); validateUrl("http://www.host.com/?query=param", "http", "www.host.com", "/?query=param"); + // Test with an ipv4 host address. + validateUrl("http://1.2.3.4/?query=param", "http", "1.2.3.4", "/?query=param"); + validateUrl("http://1.2.3.4:80/?query=param", "http", "1.2.3.4:80", "/?query=param"); + + // Test with an ipv6 address + validateUrl("http://[1::2:3]/?query=param", "http", "[1::2:3]", "/?query=param"); + validateUrl("http://[1::2:3]:80/?query=param", "http", "[1::2:3]:80", "/?query=param"); + // Test url with query parameter but without slash validateUrl("http://www.host.com:80?query=param", "http", "www.host.com:80", "?query=param"); validateUrl("http://www.host.com?query=param", "http", "www.host.com", "?query=param"); @@ -1163,8 +1179,16 @@ TEST(Url, ParsingTest) { } TEST(Url, ParsingForConnectTest) { - validateConnectUrl("host.com:443", "host.com:443"); - validateConnectUrl("host.com:80", "host.com:80"); + validateConnectUrl("host.com:443"); + validateConnectUrl("host.com:80"); + validateConnectUrl("1.2.3.4:80"); + validateConnectUrl("[1:2::3:4]:80"); + + invalidConnectUrl("[::12345678]:80"); + invalidConnectUrl("[1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1]:80"); + invalidConnectUrl("[1:1]:80"); + invalidConnectUrl("[:::]:80"); + invalidConnectUrl("[::1::]:80"); } void validatePercentEncodingEncodeDecode(absl::string_view source, diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc index 49c7c08f50ac..f988a9994b42 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -355,8 +355,7 @@ TEST_P(ProxyFilterIntegrationTest, UpstreamTlsWithIpHost) { {":method", "POST"}, {":path", "/test/long/url"}, {":scheme", "http"}, - {":authority", fmt::format("{}:{}", Network::Test::getLoopbackAddressUrlString(GetParam()), - fake_upstreams_[0]->localAddress()->ip()->port())}}; + {":authority", fake_upstreams_[0]->localAddress()->asString()}}; auto response = codec_client_->makeHeaderOnlyRequest(request_headers); waitForNextUpstreamRequest(); diff --git a/test/integration/integration_test.cc b/test/integration/integration_test.cc index 3d90d5be067c..02d784476fb7 100644 --- a/test/integration/integration_test.cc +++ b/test/integration/integration_test.cc @@ -777,6 +777,41 @@ TEST_P(IntegrationTest, AbsolutePath) { EXPECT_FALSE(response.find("HTTP/1.1 404 Not Found\r\n") == 0); } +// Make that both IPv4 and IPv6 hosts match when using relative and absolute URLs. +TEST_P(IntegrationTest, TestHostWithAddress) { + useAccessLog("%REQ(Host)%\n"); + std::string address_string; + if (GetParam() == Network::Address::IpVersion::v4) { + address_string = TestUtility::getIpv4Loopback(); + } else { + address_string = "[::1]"; + } + + auto host = config_helper_.createVirtualHost(address_string.c_str(), "/"); + host.set_require_tls(envoy::config::route::v3::VirtualHost::ALL); + config_helper_.addVirtualHost(host); + + initialize(); + std::string response; + + // Test absolute URL with ipv6. + sendRawHttpAndWaitForResponse( + lookupPort("http"), absl::StrCat("GET http://", address_string, " HTTP/1.1\r\n\r\n").c_str(), + &response, true); + EXPECT_FALSE(response.find("HTTP/1.1 404 Not Found\r\n") == 0); + EXPECT_TRUE(response.find("301") != std::string::npos); + EXPECT_THAT(waitForAccessLog(access_log_name_), HasSubstr(address_string)); + + // Test normal IPv6 request as well. + response.clear(); + sendRawHttpAndWaitForResponse( + lookupPort("http"), + absl::StrCat("GET / HTTP/1.1\r\nHost: ", address_string, "\r\n\r\n").c_str(), &response, + true); + EXPECT_FALSE(response.find("HTTP/1.1 404 Not Found\r\n") == 0); + EXPECT_TRUE(response.find("301") != std::string::npos); +} + TEST_P(IntegrationTest, AbsolutePathWithPort) { // Configure www.namewithport.com:1234 to send a redirect, and ensure the redirect is // encountered via absolute URL with a port. @@ -789,6 +824,7 @@ TEST_P(IntegrationTest, AbsolutePathWithPort) { lookupPort("http"), "GET http://www.namewithport.com:1234 HTTP/1.1\r\nHost: host\r\n\r\n", &response, true); EXPECT_FALSE(response.find("HTTP/1.1 404 Not Found\r\n") == 0); + EXPECT_TRUE(response.find("301") != std::string::npos); } TEST_P(IntegrationTest, AbsolutePathWithoutPort) { From f7da8855b841868ebd0ff96cbaaebc8ab57e79d5 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Sat, 5 Dec 2020 01:52:50 +0900 Subject: [PATCH 2/6] docs Signed-off-by: Shikugawa --- docs/root/version_history/current.rst | 22 +------------------ .../proxy_filter_integration_test.cc | 3 ++- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index d5edb4e9a7e1..e5c802df9e9d 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -3,28 +3,8 @@ Changes ------- +* http: fixed URL parsing for HTTP/1.1 fully qualified URLs and connect requests containing IPv6 addresses. * listener: fix crash when disabling or re-enabling listeners due to overload while processing LDS updates. * proxy_proto: fixed a bug where network filters would not have the correct downstreamRemoteAddress() when accessed from the StreamInfo. This could result in incorrect enforcement of RBAC rules in the RBAC network filter (but not in the RBAC HTTP filter), or incorrect access log addresses from tcp_proxy. -Incompatible Behavior Changes ------------------------------ -*Changes that are expected to cause an incompatibility if applicable; deployment changes are likely required* - -Minor Behavior Changes ----------------------- -*Changes that may cause incompatibilities for some users, but should not for most* - -* build: the Alpine based debug images are no longer built in CI, use Ubuntu based images instead. -* cluster manager: the cluster which can't extract secret entity by SDS to be warming and never activate. This feature is disabled by default and is controlled by runtime guard `envoy.reloadable_features.cluster_keep_warming_no_secret_entity`. -* ext_authz filter: disable `envoy.reloadable_features.ext_authz_measure_timeout_on_check_created` by default. -* ext_authz filter: the deprecated field :ref:`use_alpha ` is no longer supported and cannot be set anymore. -* grpc_web filter: if a `grpc-accept-encoding` header is present it's passed as-is to the upstream and if it isn't `grpc-accept-encoding:identity` is sent instead. The header was always overwriten with `grpc-accept-encoding:identity,deflate,gzip` before. -* watchdog: the watchdog action :ref:`abort_action ` is now the default action to terminate the process if watchdog kill / multikill is enabled. - -Bug Fixes ---------- -*Changes expected to improve the state of the world and are unlikely to have negative effects* - -* http: fixed URL parsing for HTTP/1.1 fully qualified URLs and connect requests containing IPv6 addresses. -* http: sending CONNECT_ERROR for HTTP/2 where appropriate during CONNECT requests. * tls: fix read resumption after triggering buffer high-watermark and all remaining request/response bytes are stored in the SSL connection's internal buffers. * udp: fixed issue in which receiving truncated UDP datagrams would cause Envoy to crash. diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc index f988a9994b42..49c7c08f50ac 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -355,7 +355,8 @@ TEST_P(ProxyFilterIntegrationTest, UpstreamTlsWithIpHost) { {":method", "POST"}, {":path", "/test/long/url"}, {":scheme", "http"}, - {":authority", fake_upstreams_[0]->localAddress()->asString()}}; + {":authority", fmt::format("{}:{}", Network::Test::getLoopbackAddressUrlString(GetParam()), + fake_upstreams_[0]->localAddress()->ip()->port())}}; auto response = codec_client_->makeHeaderOnlyRequest(request_headers); waitForNextUpstreamRequest(); From 1a8444766e60f4f73083743fbe39222d111de01f Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Sat, 5 Dec 2020 10:17:19 +0900 Subject: [PATCH 3/6] Kick CI Signed-off-by: Shikugawa From 4a59fc7f321af3f72e1af266a459cdd7f4c20993 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 9 Dec 2020 11:34:09 +0900 Subject: [PATCH 4/6] release 1.15.4 Signed-off-by: Shikugawa --- docs/root/version_history/current.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 60a43af25066..770f02f3ba62 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -3,8 +3,5 @@ Changes ------- + * http: fixed URL parsing for HTTP/1.1 fully qualified URLs and connect requests containing IPv6 addresses. -* listener: fix crash when disabling or re-enabling listeners due to overload while processing LDS updates. -* proxy_proto: fixed a bug where network filters would not have the correct downstreamRemoteAddress() when accessed from the StreamInfo. This could result in incorrect enforcement of RBAC rules in the RBAC network filter (but not in the RBAC HTTP filter), or incorrect access log addresses from tcp_proxy. -* tls: fix read resumption after triggering buffer high-watermark and all remaining request/response bytes are stored in the SSL connection's internal buffers. -* udp: fixed issue in which receiving truncated UDP datagrams would cause Envoy to crash. From e6583ebfdc79933cd0c682277fed49421e52d825 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 9 Dec 2020 19:43:41 +0900 Subject: [PATCH 5/6] Kick CI Signed-off-by: Shikugawa From 117d08d95d5a58eaf92c3b1235a1e998a6bde84d Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Fri, 18 Dec 2020 10:46:07 +0900 Subject: [PATCH 6/6] Kick CI Signed-off-by: Shikugawa