Resiliency

Core/Core/Api/GraphQL/Client.cs

@@ -362,7 +362,7 @@ private static GraphQLHttpClient CreateGraphQLClient(Account account, HttpClient
 
   private static HttpClient CreateHttpClient(Account account)
   {
-    var httpClient = Http.GetHttpProxyClient(null, TimeSpan.FromSeconds(30));
+    var httpClient = Http.GetHttpProxyClient(new SpeckleHttpClientHandler(Http.HttpAsyncPolicy(timeoutSeconds: 30)));
     Http.AddAuthHeader(httpClient, account.token);
 
     httpClient.DefaultRequestHeaders.Add("apollographql-client-name", Setup.HostApplication);

Core/Core/Helpers/Http.cs

@@ -10,7 +10,7 @@
 using Polly;
 using Polly.Contrib.WaitAndRetry;
 using Polly.Extensions.Http;
-using Polly.Retry;
+using Polly.Timeout;
 using Serilog.Context;
 using Speckle.Core.Credentials;
 using Speckle.Core.Logging;
@@ -19,15 +19,21 @@ namespace Speckle.Core.Helpers;
 
 public static class Http
 {
+  public const int DEFAULT_TIMEOUT_SECONDS = 60;
+
   public static IEnumerable<TimeSpan> DefaultDelay()
   {
-    return Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromMilliseconds(100), 5);
+    return Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromMilliseconds(200), 5);
   }
 
-  public static AsyncRetryPolicy<HttpResponseMessage> HttpAsyncPolicy(IEnumerable<TimeSpan>? delay = null)
+  public static IAsyncPolicy<HttpResponseMessage> HttpAsyncPolicy(
+    IEnumerable<TimeSpan>? delay = null,
+    int timeoutSeconds = DEFAULT_TIMEOUT_SECONDS
+  )
   {
-    return HttpPolicyExtensions
+    var retryPolicy = HttpPolicyExtensions
       .HandleTransientHttpError()
+      .Or<TimeoutRejectedException>()
       .WaitAndRetryAsync(
         delay ?? DefaultDelay(),
         (ex, timeSpan, retryAttempt, context) =>
@@ -36,6 +42,10 @@ public static AsyncRetryPolicy<HttpResponseMessage> HttpAsyncPolicy(IEnumerable<
           context.Add("retryCount", retryAttempt);
         }
       );
+
+    var timeoutPolicy = Policy.TimeoutAsync<HttpResponseMessage>(timeoutSeconds);
+
+    return Policy.WrapAsync(retryPolicy, timeoutPolicy);
   }
 
   /// <summary>
@@ -145,13 +155,17 @@ public static async Task<HttpResponseMessage> HttpPing(Uri uri)
     }
   }
 
-  public static HttpClient GetHttpProxyClient(SpeckleHttpClientHandler? handler = null, TimeSpan? timeout = null)
+  public static HttpClient GetHttpProxyClient(SpeckleHttpClientHandler? speckleHttpClientHandler = null)
   {
     IWebProxy proxy = WebRequest.GetSystemWebProxy();
     proxy.Credentials = CredentialCache.DefaultCredentials;
 
-    handler ??= new SpeckleHttpClientHandler();
-    var client = new HttpClient(handler) { Timeout = timeout ?? TimeSpan.FromSeconds(100) };
+    speckleHttpClientHandler ??= new SpeckleHttpClientHandler(HttpAsyncPolicy());
+
+    var client = new HttpClient(speckleHttpClientHandler)
+    {
+      Timeout = Timeout.InfiniteTimeSpan //timeout is configured on the SpeckleHttpClientHandler through policy
+    };
     return client;
   }
 
@@ -178,11 +192,11 @@ public static void AddAuthHeader(HttpClient client, string? authToken)
 
 public sealed class SpeckleHttpClientHandler : HttpClientHandler
 {
-  private readonly IEnumerable<TimeSpan> _delay;
+  private readonly IAsyncPolicy<HttpResponseMessage> _resiliencePolicy;
 
-  public SpeckleHttpClientHandler(IEnumerable<TimeSpan>? delay = null)
+  public SpeckleHttpClientHandler(IAsyncPolicy<HttpResponseMessage> resiliencePolicy)
   {
-    _delay = delay ?? Http.DefaultDelay();
+    _resiliencePolicy = resiliencePolicy;
   }
 
   /// <exception cref="OperationCanceledException"><paramref name="cancellationToken"/> requested cancel</exception>
@@ -206,7 +220,7 @@ CancellationToken cancellationToken
 
       request.Headers.Add("x-request-id", context.CorrelationId.ToString());
 
-      var policyResult = await Http.HttpAsyncPolicy(_delay)
+      var policyResult = await _resiliencePolicy
         .ExecuteAndCaptureAsync(
           ctx =>
           {

Core/Core/Transports/Server.cs

@@ -141,7 +141,7 @@ private void Initialize(string baseUri, string streamId, string authorizationTok
     StreamId = streamId;
 
     Client = Http.GetHttpProxyClient(
-      new SpeckleHttpClientHandler { AutomaticDecompression = DecompressionMethods.GZip }
+      new SpeckleHttpClientHandler(Http.HttpAsyncPolicy()) { AutomaticDecompression = DecompressionMethods.GZip }
     );
 
     Client.BaseAddress = new Uri(baseUri);

Core/Core/Transports/ServerUtils/ServerAPI.cs

@@ -32,23 +32,24 @@ public sealed class ServerApi : IDisposable, IServerApi
 
   private readonly HttpClient _client;
 
-  public ServerApi(string baseUri, string? authorizationToken, string blobStorageFolder, int timeoutSeconds = 60)
+  public ServerApi(string baseUri, string? authorizationToken, string blobStorageFolder, int timeoutSeconds = 120)
   {
     CancellationToken = CancellationToken.None;
 
     BlobStorageFolder = blobStorageFolder;
 
     _client = Http.GetHttpProxyClient(
-      new SpeckleHttpClientHandler { AutomaticDecompression = DecompressionMethods.GZip }
+      new SpeckleHttpClientHandler(Http.HttpAsyncPolicy(timeoutSeconds: timeoutSeconds))
+      {
+        AutomaticDecompression = DecompressionMethods.GZip
+      }
     );
 
     _client.BaseAddress = new Uri(baseUri);
-    _client.Timeout = TimeSpan.FromSeconds(timeoutSeconds);
 
     Http.AddAuthHeader(_client, authorizationToken);
   }
 
-  private int RetriedCount { get; set; }
   public CancellationToken CancellationToken { get; set; }
   public bool CompressPayloads { get; set; } = true;
 

Core/Core/Transports/ServerV2.cs

@@ -73,7 +73,7 @@ public void SaveBlob(Blob obj)
     {
       if (IsInErrorState)
       {
-        return;
+        throw new TransportException("Server transport is in an errored state", _exception);
       }
 
       _sendBuffer.Add(($"blob:{hash}", obj.filePath));
@@ -223,7 +223,7 @@ public void SaveObject(string id, string serializedObject)
     {
       if (IsInErrorState)
       {
-        return;
+        throw new TransportException("Server transport is in an errored state", _exception);
       }
 
       _sendBuffer.Add((id, serializedObject));