From 73cfa65eceb4bc9ca8001c38026d112dcce61ad3 Mon Sep 17 00:00:00 2001 From: John W Higgins Date: Thu, 9 Feb 2023 17:06:06 -0800 Subject: [PATCH 1/5] Add scaffolding for http proxy --- src/http/client.cr | 69 +++++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/src/http/client.cr b/src/http/client.cr index c9b9e1d699da..f5cbe45ba858 100644 --- a/src/http/client.cr +++ b/src/http/client.cr @@ -109,7 +109,7 @@ class HTTP::Client getter! tls : Nil alias TLSContext = Bool | Nil {% else %} - getter! tls : OpenSSL::SSL::Context::Client + getter! tls : OpenSSL::SSL::Context::Client | Bool alias TLSContext = OpenSSL::SSL::Context::Client | Bool | Nil {% end %} @@ -128,25 +128,9 @@ class HTTP::Client # be used depending on the *tls* arguments: 80 for if *tls* is `false`, # 443 if *tls* is truthy. If *tls* is `true` a new `OpenSSL::SSL::Context::Client` will # be used, else the given one. In any case the active context can be accessed through `tls`. - def initialize(@host : String, port = nil, tls : TLSContext = nil) + def initialize(@host : String, port = nil, @tls : TLSContext = nil) check_host_only(@host) - {% if flag?(:without_openssl) %} - if tls - raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time" - end - @tls = nil - {% else %} - @tls = case tls - when true - OpenSSL::SSL::Context::Client.new - when OpenSSL::SSL::Context::Client - tls - when false, nil - nil - end - {% end %} - @port = (port || (@tls ? 443 : 80)).to_i end @@ -791,26 +775,49 @@ class HTTP::Client raise "This HTTP::Client cannot be reconnected" end - hostname = @host.starts_with?('[') && @host.ends_with?(']') ? @host[1..-2] : @host - io = TCPSocket.new hostname, @port, @dns_timeout, @connect_timeout + @io = create_proxy_io || create_io(host, port, @tls) + end + + private def create_io(host, port, tls) + hostname = host.starts_with?('[') && host.ends_with?(']') ? host[1..-2] : host + io = TCPSocket.new hostname, port, @dns_timeout, @connect_timeout io.read_timeout = @read_timeout if @read_timeout io.write_timeout = @write_timeout if @write_timeout io.sync = false + tls ? create_tls(io, host, tls) : io + end - {% if !flag?(:without_openssl) %} - if tls = @tls - tcp_socket = io - begin - io = OpenSSL::SSL::Socket::Client.new(tcp_socket, context: tls, sync_close: true, hostname: @host.rchop('.')) - rescue exc - # don't leak the TCP socket when the SSL connection failed - tcp_socket.close - raise exc - end + private def create_proxy_io + return unless proxy = find_proxy + + proxy_uri = URI.parse(proxy) + tls = HTTP::Client.tls_flag(proxy_uri, nil) + proxy_port = proxy_uri.port || (tls ? 443 : 80) + host = HTTP::Client.validate_host(proxy_uri) + + create_io(host, proxy_port, tls) + end + + private def create_tls(io, host, tls) + {% if flag?(:without_openssl) %} + raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time" + {% else %} + tcp_socket = io + begin + tls = tls.is_a?(OpenSSL::SSL::Context::Client) ? tls : OpenSSL::SSL::Context::Client.new + OpenSSL::SSL::Socket::Client.new(tcp_socket, context: tls, sync_close: true, hostname: host.rchop('.')) + rescue exc + # don't leak the TCP socket when the SSL connection failed + tcp_socket.close + raise exc end {% end %} + end + + private def find_proxy + return unless ENV.has_key?("http_proxy") - @io = io + ENV["http_proxy"] end private def host_header From b4287796a190cc6a910e1a10fb9f7a7664499cc6 Mon Sep 17 00:00:00 2001 From: John W Higgins Date: Fri, 10 Feb 2023 11:38:37 -0800 Subject: [PATCH 2/5] Add https proxy via connect --- src/http/client.cr | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/http/client.cr b/src/http/client.cr index f5cbe45ba858..2f86657caf0f 100644 --- a/src/http/client.cr +++ b/src/http/client.cr @@ -795,7 +795,12 @@ class HTTP::Client proxy_port = proxy_uri.port || (tls ? 443 : 80) host = HTTP::Client.validate_host(proxy_uri) - create_io(host, proxy_port, tls) + io = create_io(host, proxy_port, tls) + + return io unless @tls + + send_connect_request(io) + create_tls(io, @host, @tls) end private def create_tls(io, host, tls) @@ -815,9 +820,24 @@ class HTTP::Client end private def find_proxy - return unless ENV.has_key?("http_proxy") + type = @tls ? "https_proxy" : "http_proxy" + return unless ENV.has_key?(type) + + ENV[type] + end - ENV["http_proxy"] + private def send_connect_request(io) + io << "CONNECT #{@host}:#{@port} HTTP/1.1\r\n" + io << "Host: #{@host}:#{@port}\r\n" + io << "\r\n" + io.flush + + resp = HTTP::Client::Response.from_io(io, ignore_body: true) + + unless resp.success? + io.close + raise IO::Error.new("Unable to connect to https proxy") + end end private def host_header From 9463e09f9baf03335786b729f2d3ff9a467a44b7 Mon Sep 17 00:00:00 2001 From: John W Higgins Date: Sat, 11 Feb 2023 10:05:45 -0800 Subject: [PATCH 3/5] Add proxy property to HTTP::Client class and instance --- src/http/client.cr | 45 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/src/http/client.cr b/src/http/client.cr index 2f86657caf0f..c17ac0e05a6f 100644 --- a/src/http/client.cr +++ b/src/http/client.cr @@ -74,6 +74,17 @@ class HTTP::Client # The set of possible valid body types. alias BodyType = String | Bytes | IO | Nil + alias ProxyURL = String | URI | Nil + + # HTTP Proxy URL. Empty string equates to no proxy. + # + # Order of usage is instance -> class -> proxy environment variables + class_property proxy : ProxyURL + + # HTTP Proxy URL. Emoty string equates to no proxy. + # + # Order of usage is instance -> class -> proxy environment variables + property proxy : ProxyURL # Returns the target host. # @@ -790,10 +801,9 @@ class HTTP::Client private def create_proxy_io return unless proxy = find_proxy - proxy_uri = URI.parse(proxy) - tls = HTTP::Client.tls_flag(proxy_uri, nil) - proxy_port = proxy_uri.port || (tls ? 443 : 80) - host = HTTP::Client.validate_host(proxy_uri) + tls = HTTP::Client.tls_flag(proxy, nil) + proxy_port = proxy.port || (tls ? 443 : 80) + host = HTTP::Client.validate_host(proxy) io = create_io(host, proxy_port, tls) @@ -819,11 +829,32 @@ class HTTP::Client {% end %} end - private def find_proxy + private def find_proxy : URI | Nil + # Order of lookup is class -> instance -> environment + # Both class and instance properties will stop on empty strings. + + case proxy + when "" + return nil + when String + return URI.parse(proxy.as(String)) + when URI + return proxy.as(URI) + end + + case HTTP::Client.proxy + when "" + return nil + when String + return URI.parse(HTTP::Client.proxy.as(String)) + when URI + return HTTP::Client.proxy.as(URI) + end + type = @tls ? "https_proxy" : "http_proxy" - return unless ENV.has_key?(type) + return nil unless ENV.has_key?(type) - ENV[type] + URI.parse(ENV[type]) end private def send_connect_request(io) From 58bea9b322766cadb3170d443bbce0660f2fddcb Mon Sep 17 00:00:00 2001 From: John W Higgins Date: Sat, 11 Feb 2023 13:13:00 -0800 Subject: [PATCH 4/5] Modify HTTP::Client.proxy to allow for subclasses --- src/http/client.cr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/http/client.cr b/src/http/client.cr index c17ac0e05a6f..609d35bfd368 100644 --- a/src/http/client.cr +++ b/src/http/client.cr @@ -842,13 +842,13 @@ class HTTP::Client return proxy.as(URI) end - case HTTP::Client.proxy + case self.class.proxy when "" return nil when String - return URI.parse(HTTP::Client.proxy.as(String)) + return URI.parse(self.class.proxy.as(String)) when URI - return HTTP::Client.proxy.as(URI) + return self.class.proxy.as(URI) end type = @tls ? "https_proxy" : "http_proxy" From 08754bc8ac17c1c2ff29484944756caef2999dcc Mon Sep 17 00:00:00 2001 From: John W Higgins Date: Sun, 12 Feb 2023 09:01:12 -0800 Subject: [PATCH 5/5] Allow all_proxy and upper case (where applicable) environment variables for proxies --- src/http/client.cr | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/http/client.cr b/src/http/client.cr index 609d35bfd368..244a849e83e9 100644 --- a/src/http/client.cr +++ b/src/http/client.cr @@ -851,10 +851,18 @@ class HTTP::Client return self.class.proxy.as(URI) end + # Find an appropriate environment variable type = @tls ? "https_proxy" : "http_proxy" - return nil unless ENV.has_key?(type) - - URI.parse(ENV[type]) + key = if ENV.has_key?(type) + type + elsif @tls && ENV.has_key?("HTTPS_PROXY") + "HTTPS_PROXY" + elsif ENV.has_key?("all_proxy") + "all_proxy" + elsif ENV.has_key?("ALL_PROXY") + "ALL_PROXY" + end + return URI.parse(ENV[key]) if key end private def send_connect_request(io)