[release/8.0] [tests] Use ResiliencePipeline in IntegrationTests (#3623)

* [tests] Use ResiliencePipeline in IntegrationTests .. with a policy to retry with 1 sec delays, and a total timeout of 90 secs on the service side. The test(client) side has an overall http request timeout for 120 secs which would allow the real errors from the service to get surfaced to the client. * address review feedback from @ eerhardt * Use the pipeline for MySql also * IntegrationServiceFixture.DumpComponentLogsAsync: error out if more than one resource is given * Rename TestUtils -> ResilienceUtils - feedback from @ eerhardt * [tests] Ignore `CA1305` for tests https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/quality-rules/ca1305 ``` dotnet_diagnostic.CA1305.severity = none ``` Addresses review feedback from @ eerhardt . * Remove Polly reference * Make code consistent. * add comment addressing review feedback from @ eerhardt --------- Co-authored-by: Ankit Jain <radical@gmail.com> Co-authored-by: Eric Erhardt <eric.erhardt@microsoft.com>
dotnet · Apr 12, 2024 · 2f7c0c2 · 2f7c0c2
1 parent b1dd348
commit 2f7c0c2
Show file tree

Hide file tree

Showing 10 changed files with 82 additions and 28 deletions.
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -108,7 +108,6 @@
     <PackageVersion Include="Npgsql.DependencyInjection" Version="8.0.2" />
     <PackageVersion Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.2" />
     <PackageVersion Include="Oracle.EntityFrameworkCore" Version="8.21.121" />
-    <PackageVersion Include="Polly" Version="8.3.0" />
     <PackageVersion Include="Polly.Core" Version="8.3.0" />
     <PackageVersion Include="Polly.Extensions" Version="8.3.0" />
     <PackageVersion Include="Pomelo.EntityFrameworkCore.MySql" Version="8.0.1" />

diff --git a/src/Aspire.Hosting/Aspire.Hosting.csproj b/src/Aspire.Hosting/Aspire.Hosting.csproj
@@ -34,7 +34,7 @@
     <PackageReference Include="Grpc.AspNetCore" />
     <PackageReference Include="KubernetesClient" />
     <PackageReference Include="Microsoft.Extensions.Hosting" />
-    <PackageReference Include="Polly" />
+    <PackageReference Include="Polly.Core" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/tests/.editorconfig b/tests/.editorconfig
@@ -2,3 +2,6 @@
 
 # CA2007: Consider calling ConfigureAwait on the awaited task
 dotnet_diagnostic.CA2007.severity = silent
+
+# CA1305: Specify IFormatProvider
+dotnet_diagnostic.CA1305.severity = none
diff --git a/tests/Aspire.EndToEnd.Tests/IntegrationServicesFixture.cs b/tests/Aspire.EndToEnd.Tests/IntegrationServicesFixture.cs
@@ -277,6 +277,15 @@ public async Task DumpDockerInfoAsync(ITestOutputHelper? testOutputArg = null)
 
     public async Task DumpComponentLogsAsync(TestResourceNames resource, ITestOutputHelper? testOutputArg = null)
     {
+        if (resource == TestResourceNames.None)
+        {
+            return;
+        }
+        if (resource == TestResourceNames.All || !Enum.IsDefined<TestResourceNames>(resource))
+        {
+            throw new ArgumentException($"Only one resource is supported at a time. resource: {resource}");
+        }
+
         string component = resource switch
         {
             TestResourceNames.cosmos => "cosmos",

diff --git a/tests/testproject/TestProject.IntegrationServiceA/Cosmos/CosmosExtensions.cs b/tests/testproject/TestProject.IntegrationServiceA/Cosmos/CosmosExtensions.cs
@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text;
+using Aspire.TestProject;
 using Microsoft.Azure.Cosmos;
 using Polly;
 
@@ -13,15 +15,17 @@ public static void MapCosmosApi(this WebApplication app)
 
     private static async Task<IResult> VerifyCosmosAsync(CosmosClient cosmosClient)
     {
+        StringBuilder errorMessageBuilder = new();
         try
         {
-            var policy = Policy
-                .Handle<HttpRequestException>()
-                // retry 60 times with a 1 second delay between retries
-                .WaitAndRetryAsync(60, retryAttempt => TimeSpan.FromSeconds(1));
+            ResiliencePipeline pipeline = ResilienceUtils.GetDefaultResiliencePipelineBuilder<HttpRequestException>(args =>
+            {
+                errorMessageBuilder.AppendLine($"{Environment.NewLine}Service retry #{args.AttemptNumber} due to {args.Outcome.Exception}");
+                return ValueTask.CompletedTask;
+            }).Build();
 
-            var db = await policy.ExecuteAsync(
-                async () => (await cosmosClient.CreateDatabaseIfNotExistsAsync("db")).Database);
+            var db = await pipeline.ExecuteAsync(
+                async token => (await cosmosClient.CreateDatabaseIfNotExistsAsync("db", cancellationToken: token)).Database);
 
             var container = (await db.CreateContainerIfNotExistsAsync("todos", "/id")).Container;
 
@@ -38,7 +42,7 @@ private static async Task<IResult> VerifyCosmosAsync(CosmosClient cosmosClient)
         }
         catch (Exception e)
         {
-            return Results.Problem(e.ToString());
+            return Results.Problem($"Error: {e}{Environment.NewLine}** Previous retries: {errorMessageBuilder}");
         }
     }
 }
diff --git a/tests/testproject/TestProject.IntegrationServiceA/MySql/MySqlExtensions.cs b/tests/testproject/TestProject.IntegrationServiceA/MySql/MySqlExtensions.cs
@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text;
+using Aspire.TestProject;
 using MySqlConnector;
 using Polly;
 
@@ -13,14 +15,16 @@ public static void MapMySqlApi(this WebApplication app)
 
     private static async Task<IResult> VerifyMySqlAsync(MySqlConnection connection)
     {
+        StringBuilder errorMessageBuilder = new();
         try
         {
-            var policy = Policy
-                .Handle<MySqlException>()
-                // retry 60 times with a 1 second delay between retries
-                .WaitAndRetryAsync(60, retryAttempt => TimeSpan.FromSeconds(1));
+            ResiliencePipeline pipeline = ResilienceUtils.GetDefaultResiliencePipelineBuilder<MySqlException>(args =>
+            {
+                errorMessageBuilder.AppendLine($"{Environment.NewLine}Service retry #{args.AttemptNumber} due to {args.Outcome.Exception}");
+                return ValueTask.CompletedTask;
+            }).Build();
 
-            await policy.ExecuteAsync(connection.OpenAsync);
+            await pipeline.ExecuteAsync(async token => await connection.OpenAsync(token));
 
             var command = connection.CreateCommand();
             command.CommandText = $"SELECT 1";
@@ -30,7 +34,7 @@ private static async Task<IResult> VerifyMySqlAsync(MySqlConnection connection)
         }
         catch (Exception e)
         {
-            return Results.Problem(e.ToString());
+            return Results.Problem($"Error: {e}{Environment.NewLine}** Previous retries: {errorMessageBuilder}");
         }
     }
 }
diff --git a/tests/testproject/TestProject.IntegrationServiceA/Oracle/OracleDatabaseExtensions.cs b/tests/testproject/TestProject.IntegrationServiceA/Oracle/OracleDatabaseExtensions.cs
@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text;
+using Aspire.TestProject;
 using Microsoft.EntityFrameworkCore;
 using Oracle.ManagedDataAccess.Client;
 using Polly;
@@ -14,22 +16,24 @@ public static void MapOracleDatabaseApi(this WebApplication app)
 
     private static IResult VerifyOracleDatabase(MyDbContext context)
     {
+        StringBuilder errorMessageBuilder = new();
         try
         {
-            var policy = Policy
-                .Handle<OracleException>()
-                // retry 60 times with a 1 second delay between retries
-                .WaitAndRetry(60, retryAttempt => TimeSpan.FromSeconds(1));
+            ResiliencePipeline pipeline = ResilienceUtils.GetDefaultResiliencePipelineBuilder<OracleException>(args =>
+            {
+                errorMessageBuilder.AppendLine($"{Environment.NewLine}Service retry #{args.AttemptNumber} due to {args.Outcome.Exception}");
+                return ValueTask.CompletedTask;
+            }).Build();
 
-            return policy.Execute(() =>
+            return pipeline.Execute(() =>
             {
                 var results = context.Database.SqlQueryRaw<int>("SELECT 1 FROM DUAL");
                 return results.Any() ? Results.Ok("Success!") : Results.Problem("Failed");
             });
         }
         catch (Exception e)
         {
-            return Results.Problem(e.ToString());
+            return Results.Problem($"Error: {e}{Environment.NewLine}** Previous retries: {errorMessageBuilder}");
         }
     }
 }
diff --git a/tests/testproject/TestProject.IntegrationServiceA/ResilienceUtils.cs b/tests/testproject/TestProject.IntegrationServiceA/ResilienceUtils.cs
@@ -0,0 +1,27 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Polly;
+using Polly.Retry;
+
+namespace Aspire.TestProject;
+
+public static class ResilienceUtils
+{
+    public static ResiliencePipelineBuilder GetDefaultResiliencePipelineBuilder<TException>(Func<OnRetryArguments<object>, ValueTask> onRetry, int overallTimeoutSecs = 90) where TException : Exception
+    {
+        // Retry for upto 20 times with delay of 1 sec between
+        // attempts, and also stop before an overall timeout of
+        // @overallTimeoutSecs
+        var optionsOnRetry = new RetryStrategyOptions
+        {
+            MaxRetryAttempts = 20,
+            ShouldHandle = new PredicateBuilder().Handle<TException>(),
+            Delay = TimeSpan.FromSeconds(1),
+            OnRetry = onRetry
+        };
+        return new ResiliencePipelineBuilder()
+                    .AddTimeout(TimeSpan.FromSeconds(overallTimeoutSecs))
+                    .AddRetry(optionsOnRetry);
+    }
+}
diff --git a/tests/testproject/TestProject.IntegrationServiceA/SqlServer/SqlServerExtensions.cs b/tests/testproject/TestProject.IntegrationServiceA/SqlServer/SqlServerExtensions.cs
@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text;
+using Aspire.TestProject;
 using Microsoft.Data.SqlClient;
 using Polly;
 
@@ -13,14 +15,16 @@ public static void MapSqlServerApi(this WebApplication app)
 
     private static async Task<IResult> VerifySqlServerAsync(SqlConnection connection)
     {
+        StringBuilder errorMessageBuilder = new();
         try
         {
-            var policy = Policy
-                .Handle<SqlException>()
-                // retry 60 times with a 1 second delay between retries
-                .WaitAndRetryAsync(60, retryAttempt => TimeSpan.FromSeconds(1));
+            ResiliencePipeline pipeline = ResilienceUtils.GetDefaultResiliencePipelineBuilder<SqlException>(args =>
+            {
+                errorMessageBuilder.AppendLine($"{Environment.NewLine}Service retry #{args.AttemptNumber} due to {args.Outcome.Exception}");
+                return ValueTask.CompletedTask;
+            }).Build();
 
-            await policy.ExecuteAsync(connection.OpenAsync);
+            await pipeline.ExecuteAsync(async token => await connection.OpenAsync(token));
 
             var command = connection.CreateCommand();
             command.CommandText = $"SELECT 1";
@@ -30,7 +34,7 @@ private static async Task<IResult> VerifySqlServerAsync(SqlConnection connection
         }
         catch (Exception e)
         {
-            return Results.Problem(e.ToString());
+            return Results.Problem($"Error: {e}{Environment.NewLine}** Previous retries: {errorMessageBuilder}");
         }
     }
 }
diff --git a/tests/testproject/TestProject.IntegrationServiceA/TestProject.IntegrationServiceA.csproj b/tests/testproject/TestProject.IntegrationServiceA/TestProject.IntegrationServiceA.csproj
@@ -31,7 +31,7 @@
   <ItemGroup>
     <ProjectReference Condition="'$(TestsRunningOutsideOfRepo)' != 'true'" Include="@(ComponentReferenceForTests -> '$(RepoRoot)src\Components\%(Identity)\%(Identity).csproj')" />
     <PackageReference Condition="'$(TestsRunningOutsideOfRepo)' == 'true'" Include="@(ComponentReferenceForTests)" />
-    <PackageReference Include="Polly" />
+    <PackageReference Include="Polly.Core" />
   </ItemGroup>
 
 </Project>