SOCKS4/4a/5 proxy support in SocketsHttpHandler (#48883)

* Separate method for proxy scheme validation.

* Pass socks connection kind.

* Unauthorized socks5 connection.

* Username and password auth.

* Fix response address.

* Fix proxyUri value and assertion.

* Use HttpConnectionKind for SOCKS.

* Handle more connection kind assertions.

* SOCKS4/4a support.

* Move version selection into SocksHelper.

* Call sync version of read write.

* Cancellation by disposing stream.

* Dispose cancellation registration.

* IP addressing for SOCKS5.

* IP addressing for SOCKS4.

* Wrap write method.

* Cancellation and optimization.

* Optimize.

* Apply suggestions from code review

Co-authored-by: Miha Zupan <mihazupan.zupan1@gmail.com>

* Clarify logic.

* Remove ssl assertion.

* SocksException.

* Make SocksException derive from IOException.

* Use binary primitives to write port in BE.

* Socks loopback test.

* Expand test matrix.

* Try to solve certificate issue.

* Pass handler to httpclient.

* Update ConnectToTcpHostAsync.

* Remove custom self-signed cert use from Socks test

* Fix LoopbackSocksServer's parsing of Socks4a domain name

* Only set RequestVersionExact for H2C

Setting it in general breaks H2 => H1.1 downgrade on platforms without ALPN

* Add auth test.

* Add IP in test matrix.

* Only override host when required.

* Don't attempt NT Auth for Socks proxies

* Skip HTTP2 ssl test on platforms without ALPN support

* Use NetworkCredential directly

* Pass AddressFamily to sync Dns resolution too

* Consistently check encoded string lengths

* Fix Socks5 user/pass auth

* Add IPv6 test for socks5

* Exception nits

* Add exceptional tests.

* Fix exceptional test.

* Fix NRT compilation

Co-authored-by: Miha Zupan <mihazupan.zupan1@gmail.com>

* Server shouldn't wait for request in exceptional test.

* Add exception message to test.

* Update auth failure handling.

* SOCKS4 and 5 uses different auth model, requires different error message.

* Revert accidental indent change.

* Expand test matrix to include Sync HTTP1

* Read received bytes before returning error response in Socks4 loopback

* Use named bool arguments

* Improve exception messages

* !IsEmpty => Length != 0

* Improve exception messages 2

* Avoid enforing Socks4 VN value

Co-authored-by: Miha Zupan <mihazupan.zupan1@gmail.com>

src/libraries/System.Net.Http/src/Resources/Strings.resx

@@ -406,7 +406,7 @@
     <value>Client certificate was not found in the personal (\"MY\") certificate store. In UWP, client certificates are only supported if they have been added to that certificate store.</value>
   </data>
   <data name="net_http_invalid_proxy_scheme" xml:space="preserve">
-    <value>Only the 'http' scheme is allowed for proxies.</value>
+    <value>Only the 'http', 'socks4', 'socks4a' and 'socks5' schemes are allowed for proxies.</value>
   </data>
   <data name="net_http_request_invalid_char_encoding" xml:space="preserve">
     <value>Request headers must contain only ASCII characters.</value>
@@ -606,6 +606,33 @@
   <data name="net_http_synchronous_reads_not_supported" xml:space="preserve">
     <value>Synchronous reads are not supported, use ReadAsync instead.</value>
   </data>
+  <data name="net_socks_auth_failed" xml:space="preserve">
+    <value>Failed to authenticate with the SOCKS server.</value>
+  </data>
+  <data name="net_socks_bad_address_type" xml:space="preserve">
+    <value>SOCKS server returned an unknown address type.</value>
+  </data>
+  <data name="net_socks_connection_failed" xml:space="preserve">
+    <value>SOCKS server failed to connect to the destination.</value>
+  </data>
+  <data name="net_socks_ipv6_notsupported" xml:space="preserve">
+    <value>SOCKS4 does not support IPv6 addresses.</value>
+  </data>
+  <data name="net_socks_no_auth_method" xml:space="preserve">
+    <value>SOCKS server did not return a suitable authentication method.</value>
+  </data>
+  <data name="net_socks_no_ipv4_address" xml:space="preserve">
+    <value>Failed to resolve the destination host to an IPv4 address.</value>
+  </data>
+  <data name="net_socks_unexpected_version" xml:space="preserve">
+    <value>Unexpected SOCKS protocol version. Required {0}, got {1}.</value>
+  </data>
+  <data name="net_socks_string_too_long" xml:space="preserve">
+    <value>Encoding the {0} took more than the maximum of 255 bytes.</value>
+  </data>
+  <data name="net_socks_auth_required" xml:space="preserve">
+    <value>SOCKS server requested username &amp; password authentication.</value>
+  </data>
   <data name="net_http_proxy_tunnel_returned_failure_status_code" xml:space="preserve">
     <value>The proxy tunnel request to proxy '{0}' failed with status code '{1}'."</value>
   </data>

src/libraries/System.Net.Http/src/System.Net.Http.csproj

@@ -179,6 +179,8 @@
     <Compile Include="System\Net\Http\HttpTelemetry.AnyOS.cs" />
     <Compile Include="System\Net\Http\HttpUtilities.AnyOS.cs" />
     <Compile Include="System\Net\Http\SocketsHttpHandler\SystemProxyInfo.cs" />
+    <Compile Include="System\Net\Http\SocketsHttpHandler\SocksHelper.cs" />
+    <Compile Include="System\Net\Http\SocketsHttpHandler\SocksException.cs" />
     <Compile Include="$(CommonPath)System\Net\NTAuthentication.Common.cs"
              Link="Common\System\Net\NTAuthentication.Common.cs" />
     <Compile Include="$(CommonPath)System\Net\ContextFlagsPal.cs"

src/libraries/System.Net.Http/src/System/Net/Http/HttpUtilities.cs

@@ -34,6 +34,14 @@ internal static bool IsNonSecureWebSocketScheme(string scheme) =>
         internal static bool IsSecureWebSocketScheme(string scheme) =>
             string.Equals(scheme, "wss", StringComparison.OrdinalIgnoreCase);
 
+        internal static bool IsSupportedProxyScheme(string scheme) =>
+            string.Equals(scheme, "http", StringComparison.OrdinalIgnoreCase) || IsSocksScheme(scheme);
+
+        internal static bool IsSocksScheme(string scheme) =>
+            string.Equals(scheme, "socks5", StringComparison.OrdinalIgnoreCase) ||
+            string.Equals(scheme, "socks4a", StringComparison.OrdinalIgnoreCase) ||
+            string.Equals(scheme, "socks4", StringComparison.OrdinalIgnoreCase);
+
         // Always specify TaskScheduler.Default to prevent us from using a user defined TaskScheduler.Current.
         //
         // Since we're not doing any CPU and/or I/O intensive operations, continue on the same thread.

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionKind.cs

@@ -10,6 +10,8 @@ internal enum HttpConnectionKind : byte
         Proxy,              // HTTP proxy usage for non-secure (HTTP) requests.
         ProxyTunnel,        // Non-secure websocket (WS) connection using CONNECT tunneling through proxy.
         SslProxyTunnel,     // HTTP proxy usage for secure (HTTPS/WSS) requests using SSL and proxy CONNECT.
-        ProxyConnect        // Connection used for proxy CONNECT. Tunnel will be established on top of this.
+        ProxyConnect,       // Connection used for proxy CONNECT. Tunnel will be established on top of this.
+        SocksTunnel,        // SOCKS proxy usage for HTTP requests.
+        SslSocksTunnel      // SOCKS proxy usage for HTTPS requests.
     }
 }

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs

@@ -190,8 +190,17 @@ public HttpConnectionPool(HttpConnectionPoolManager poolManager, HttpConnectionK
                     _http3Enabled = false;
                     break;
 
+                case HttpConnectionKind.SocksTunnel:
+                case HttpConnectionKind.SslSocksTunnel:
+                    Debug.Assert(host != null);
+                    Debug.Assert(port != 0);
+                    Debug.Assert(proxyUri != null);
+
+                    _http3Enabled = false; // TODO: SOCKS supports UDP and may be used for HTTP3
+                    break;
+
                 default:
-                    Debug.Fail("Unkown HttpConnectionKind in HttpConnectionPool.ctor");
+                    Debug.Fail("Unknown HttpConnectionKind in HttpConnectionPool.ctor");
                     break;
             }
 
@@ -317,7 +326,7 @@ private static SslClientAuthenticationOptions ConstructSslOptions(HttpConnection
         public HttpAuthority? OriginAuthority => _originAuthority;
         public HttpConnectionSettings Settings => _poolManager.Settings;
         public HttpConnectionKind Kind => _kind;
-        public bool IsSecure => _kind == HttpConnectionKind.Https || _kind == HttpConnectionKind.SslProxyTunnel;
+        public bool IsSecure => _kind == HttpConnectionKind.Https || _kind == HttpConnectionKind.SslProxyTunnel || _kind == HttpConnectionKind.SslSocksTunnel;
         public Uri? ProxyUri => _proxyUri;
         public ICredentials? ProxyCredentials => _poolManager.ProxyCredentials;
         public byte[]? HostHeaderValueBytes => _hostHeaderValueBytes;
@@ -339,10 +348,10 @@ public byte[] Http2AltSvcOriginUri
 
                     Debug.Assert(_originAuthority != null);
                     sb
-                        .Append(_kind == HttpConnectionKind.Https ? "https://" : "http://")
+                        .Append(IsSecure ? "https://" : "http://")
                         .Append(_originAuthority.IdnHost);
 
-                    if (_originAuthority.Port != (_kind == HttpConnectionKind.Https ? DefaultHttpsPort : DefaultHttpPort))
+                    if (_originAuthority.Port != (IsSecure ? DefaultHttpsPort : DefaultHttpPort))
                     {
                         sb
                             .Append(':')
@@ -547,7 +556,7 @@ private async ValueTask<HttpConnectionBase> GetHttp11ConnectionAsync(HttpRequest
 
         private async ValueTask<HttpConnectionBase> GetHttp2ConnectionAsync(HttpRequestMessage request, bool async, CancellationToken cancellationToken)
         {
-            Debug.Assert(_kind == HttpConnectionKind.Https || _kind == HttpConnectionKind.SslProxyTunnel || _kind == HttpConnectionKind.Http);
+            Debug.Assert(_kind == HttpConnectionKind.Https || _kind == HttpConnectionKind.SslProxyTunnel || _kind == HttpConnectionKind.Http || _kind == HttpConnectionKind.SocksTunnel || _kind == HttpConnectionKind.SslSocksTunnel);
 
             // See if we have an HTTP2 connection
             Http2Connection? http2Connection = GetExistingHttp2Connection();
@@ -603,7 +612,7 @@ private async ValueTask<HttpConnectionBase> GetHttp2ConnectionAsync(HttpRequestM
 
                     sslStream = stream as SslStream;
 
-                    if (_kind == HttpConnectionKind.Http)
+                    if (!IsSecure)
                     {
                         http2Connection = await ConstructHttp2ConnectionAsync(stream, request, cancellationToken).ConfigureAwait(false);
 
@@ -1148,7 +1157,7 @@ internal void BlocklistAuthority(HttpAuthority badAuthority)
             Debug.Assert(_altSvcBlocklistTimerCancellation != null);
             if (added)
             {
-               _ = Task.Delay(AltSvcBlocklistTimeoutInMilliseconds)
+                _ = Task.Delay(AltSvcBlocklistTimeoutInMilliseconds)
                     .ContinueWith(t =>
                     {
                         lock (altSvcBlocklist)
@@ -1264,6 +1273,14 @@ public ValueTask<HttpResponseMessage> SendAsync(HttpRequestMessage request, bool
                     case HttpConnectionKind.SslProxyTunnel:
                         stream = await EstablishProxyTunnelAsync(async, request.HasHeaders ? request.Headers : null, cancellationToken).ConfigureAwait(false);
                         break;
+
+                    case HttpConnectionKind.SocksTunnel:
+                    case HttpConnectionKind.SslSocksTunnel:
+                        Debug.Assert(_originAuthority != null);
+                        Debug.Assert(_proxyUri != null);
+                        (socket, stream) = await ConnectToTcpHostAsync(_proxyUri.IdnHost, _proxyUri.Port, request, async, cancellationToken).ConfigureAwait(false);
+                        await SocksHelper.EstablishSocksTunnelAsync(stream, _originAuthority.IdnHost, _originAuthority.Port, _proxyUri, ProxyCredentials, async, cancellationToken).ConfigureAwait(false);
+                        break;
                 }
 
                 Debug.Assert(stream != null);

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPoolManager.cs

@@ -282,8 +282,20 @@ private HttpConnectionKey GetConnectionKey(HttpRequestMessage request, Uri? prox
 
             if (proxyUri != null)
             {
-                Debug.Assert(HttpUtilities.IsSupportedNonSecureScheme(proxyUri.Scheme));
-                if (sslHostName == null)
+                Debug.Assert(HttpUtilities.IsSupportedProxyScheme(proxyUri.Scheme));
+                if (HttpUtilities.IsSocksScheme(proxyUri.Scheme))
+                {
+                    // Socks proxy
+                    if (sslHostName != null)
+                    {
+                        return new HttpConnectionKey(HttpConnectionKind.SslSocksTunnel, uri.IdnHost, uri.Port, sslHostName, proxyUri, identity);
+                    }
+                    else
+                    {
+                        return new HttpConnectionKey(HttpConnectionKind.SocksTunnel, uri.IdnHost, uri.Port, null, proxyUri, identity);
+                    }
+                }
+                else if (sslHostName == null)
                 {
                     if (HttpUtilities.IsNonSecureWebSocketScheme(uri.Scheme))
                     {
@@ -394,7 +406,7 @@ public ValueTask<HttpResponseMessage> SendAsync(HttpRequestMessage request, bool
                 if (NetEventSource.Log.IsEnabled()) NetEventSource.Error(this, $"Exception from {_proxy.GetType().Name}.GetProxy({request.RequestUri}): {ex}");
             }
 
-            if (proxyUri != null && proxyUri.Scheme != UriScheme.Http)
+            if (proxyUri != null && !HttpUtilities.IsSupportedProxyScheme(proxyUri.Scheme))
             {
                 throw new NotSupportedException(SR.net_http_invalid_proxy_scheme);
             }

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/SocksException.cs

@@ -0,0 +1,14 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.IO;
+
+namespace System.Net.Http
+{
+    internal class SocksException : IOException
+    {
+        public SocksException(string message) : base(message) { }
+
+        public SocksException(string message, Exception innerException) : base(message, innerException) { }
+    }
+}