[Internal] Pipelines: Fixes FaultInjection Release Pipelines (#4913)

# Pull Request Template ## Description Fixes FaultInjection Release Pipelines: - Removes build-fault-injection-test.yml, this file is unneeded and including it just runs the same tests twice increasing cogs - Updates FaultInjection Tests, these tests were originally made using a session consistency account, the live account we use for integration testing in running with strong consistency. Changes to the tests needed to be made in order for them to run on strong consistency. ![image](https://github.com/user-attachments/assets/e12b056a-a558-47e3-bda3-2c9a234743a5) ## Type of change Please delete options that are not relevant. - [] Bug fix (non-breaking change which fixes an issue) ## Closing issues To automatically close an issue: closes #IssueNumber
Azure · Nov 28, 2024 · 7b4128a · 7b4128a
1 parent d9f4b0c
commit 7b4128a
Show file tree

Hide file tree

Showing 5 changed files with 80 additions and 82 deletions.
diff --git a/Microsoft.Azure.Cosmos/FaultInjection/tests/FaultInjectionDirectModeTests.cs b/Microsoft.Azure.Cosmos/FaultInjection/tests/FaultInjectionDirectModeTests.cs
@@ -17,6 +17,7 @@ namespace Microsoft.Azure.Cosmos.FaultInjection.Tests
     using Microsoft.Azure.Cosmos.Routing;
     using Microsoft.Azure.Documents;
     using static Microsoft.Azure.Cosmos.FaultInjection.Tests.Utils.TestCommon;
+    using ConsistencyLevel = ConsistencyLevel;
     using CosmosSystemTextJsonSerializer = Utils.TestCommon.CosmosSystemTextJsonSerializer;
     using Database = Database;
     using PartitionKey = PartitionKey;
@@ -69,7 +70,7 @@ public async Task Initialize()
 
             CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
             {
-                ConnectionMode = ConnectionMode.Gateway,
+                ConsistencyLevel = ConsistencyLevel.Session,
                 Serializer = this.serializer,
             };
 
@@ -157,6 +158,7 @@ public async Task FaultInjectionServerErrorRule_OperationTypeTest(int operation)
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
                 };
@@ -330,6 +332,7 @@ public async Task FaultInjectionServerErrorRule_OperationTypeAddressTest(int ope
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ApplicationPreferredRegions = preferredRegions,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
@@ -454,6 +457,7 @@ public async Task FaultInjectionServerErrorRule_RegionTest()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
                 };
@@ -555,6 +559,7 @@ public async Task FaultInjectionServerErrorRule_PartitionTest()
 
             CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
             {
+                ConsistencyLevel = ConsistencyLevel.Session,
                 FaultInjector = faultInjector,
                 Serializer = this.serializer,
                 MaxRetryAttemptsOnRateLimitedRequests = 0,
@@ -635,6 +640,7 @@ public async Task FaultInjectionServerErrorRule_ServerSendDelay()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer,
                     EnableContentResponseOnWrite = true,
@@ -727,6 +733,7 @@ public async Task FaultInjectionServerErrorRule_ServerResponseDelay()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer,
                     EnableContentResponseOnWrite = true,
@@ -821,6 +828,7 @@ public async Task FaultInjectionServerErrorRule_ServerTimeout()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
                 };
@@ -904,6 +912,7 @@ public async Task FaultInjectionServerErrorRule_ConnecitonTimeout()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer,
                     EnableContentResponseOnWrite = true,
@@ -984,6 +993,7 @@ public async Task FaultInjectionServerErrorRule_ConnecitonDelay()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer,
                     EnableContentResponseOnWrite = true,
@@ -1091,6 +1101,7 @@ public async Task FaultInjectionServerErrorRule_ServerErrorResponseTest(
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer,
                 };
@@ -1171,6 +1182,7 @@ public async Task FaultInjectionServerErrorRule_HitCountTest()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
                 };
@@ -1266,6 +1278,7 @@ public async Task FaultInjectionServerErrorRule_IncludePrimaryTest()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
                 };
@@ -1339,6 +1352,7 @@ public async Task FaultInjectionServerErrorRule_InjectionRateTest()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
                 };
@@ -1424,6 +1438,7 @@ public async Task FaultInjectionConnectionErrorRule_Test()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
                 };
@@ -1437,21 +1452,29 @@ public async Task FaultInjectionConnectionErrorRule_Test()
                 ChaosInterceptor interceptor = faultInjector.GetChaosInterceptor() as ChaosInterceptor;
                 Assert.IsNotNull(interceptor);
 
-                await this.PerformDocumentOperationAndCheckApplication(
-                    this.fiContainer,
-                    OperationType.Create,
-                    createdItem,
-                    connectionErrorRule,
-                    0,
-                    0);
+                try
+                {
+                    await this.fiContainer.CreateItemAsync<FaultInjectionTestObject>(createdItem);
+                }
+                catch
+                {
+                    //ignore
+                }
 
                 FaultInjectionDynamicChannelStore channelStore = interceptor.GetChannelStore();
                 Assert.IsTrue(channelStore.GetAllChannels().Count > 0);
                 List<Guid> channelGuids = channelStore.GetAllChannelIds();
 
-                await Task.Delay(TimeSpan.FromSeconds(2));
+                try
+                {
+                    await Task.Delay(TimeSpan.FromSeconds(2));
 
-                await this.fiContainer.CreateItemAsync<FaultInjectionTestObject>(createdItem2);
+                    await this.fiContainer.CreateItemAsync<FaultInjectionTestObject>(createdItem2);
+                }
+                catch
+                {
+                    //ignore
+                }
 
                 Assert.IsTrue(connectionErrorRule.GetHitCount() >= 1);
 
@@ -1547,6 +1570,7 @@ public async Task FaultInjectionServerErrorRule_ReadFeedTest()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
                 };
@@ -1832,6 +1856,7 @@ private void ValidateFaultInjectionRuleApplication(
             FaultInjectionRule rule)
         {
             string diagnosticsString = diagnostics.ToString();
+            Console.WriteLine(diagnostics.ToString());
             Assert.IsTrue(1 <= rule.GetHitCount());
             Assert.IsTrue(1 <= diagnostics.GetFailedRequestCount());
             Assert.IsTrue(diagnosticsString.Contains(statusCode.ToString()));

diff --git a/Microsoft.Azure.Cosmos/FaultInjection/tests/FaultInjectionGatewayModeTests.cs b/Microsoft.Azure.Cosmos/FaultInjection/tests/FaultInjectionGatewayModeTests.cs
@@ -17,6 +17,7 @@ namespace Microsoft.Azure.Cosmos.FaultInjection.Tests
     using Microsoft.Azure.Cosmos.Routing;
     using Microsoft.Azure.Documents;
     using static Microsoft.Azure.Cosmos.FaultInjection.Tests.Utils.TestCommon;
+    using ConsistencyLevel = ConsistencyLevel;
     using CosmosSystemTextJsonSerializer = Utils.TestCommon.CosmosSystemTextJsonSerializer;
     using Database = Database;
     using PartitionKey = PartitionKey;
@@ -60,6 +61,7 @@ public async Task Initialize()
 
             CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
             {
+                ConsistencyLevel = ConsistencyLevel.Session,
                 ConnectionMode = ConnectionMode.Gateway,
                 Serializer = this.serializer,
             };
@@ -79,8 +81,8 @@ public async Task Cleanup()
             {
                 await this.highThroughputContainer.DeleteContainerAsync();
             }
-            this.client.Dispose();
-            this.fiClient.Dispose();
+            this.client?.Dispose();
+            this.fiClient?.Dispose();
         }
 
         //<summary>
@@ -150,6 +152,7 @@ public async Task FIGatewayRegion()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ConnectionMode = ConnectionMode.Gateway,
                     Serializer = this.serializer
                 };
@@ -182,6 +185,15 @@ public async Task FIGatewayRegion()
                         (int)HttpStatusCode.Gone,
                         localRegionRule);
                 }
+                catch (CosmosException ex)
+                {
+                    this.ValidateHitCount(localRegionRule, 1);
+                    this.ValidateHitCount(remoteRegionRule, 0);
+                    this.ValidateFaultInjectionRuleApplication(
+                        ex,
+                        (int)HttpStatusCode.Gone,
+                        localRegionRule);
+                }
             }
             finally
             {
@@ -256,6 +268,7 @@ public async Task FIGatewayPartitionTest()
 
             CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
             {
+                ConsistencyLevel = ConsistencyLevel.Session,
                 ConnectionMode = ConnectionMode.Gateway,
                 Serializer = this.serializer,
                 MaxRetryAttemptsOnRateLimitedRequests = 0,
@@ -392,6 +405,7 @@ public async Task FIGatewayResponseDelay()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ConnectionMode = ConnectionMode.Gateway,
                     Serializer = this.serializer,
                     EnableContentResponseOnWrite = true,
@@ -486,6 +500,7 @@ public async Task FIGatewaySendDelay()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ConnectionMode = ConnectionMode.Gateway,
                     Serializer = this.serializer,
                     EnableContentResponseOnWrite = true,
@@ -557,13 +572,13 @@ public async Task FIGatewaySendDelay()
         [DataRow(FaultInjectionOperationType.ReadItem, FaultInjectionServerErrorType.Timeout, (int)StatusCodes.RequestTimeout, (int)SubStatusCodes.Unknown, DisplayName = "Timeout")]
         [DataRow(FaultInjectionOperationType.ReadItem, FaultInjectionServerErrorType.PartitionIsMigrating, (int)StatusCodes.Gone, (int)SubStatusCodes.CompletingPartitionMigration, DisplayName = "PartitionIsMigrating")]
         [DataRow(FaultInjectionOperationType.ReadItem, FaultInjectionServerErrorType.PartitionIsSplitting, (int)StatusCodes.Gone, (int)SubStatusCodes.CompletingSplit, DisplayName = "PartitionIsSplitting")]
-        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.Gone, (int)StatusCodes.Gone, (int)SubStatusCodes.ServerGenerated410, DisplayName = "Gone")]
-        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.InternalServerError, (int)StatusCodes.InternalServerError, (int)SubStatusCodes.Unknown, DisplayName = "InternalServerError")]
-        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.TooManyRequests, (int)StatusCodes.TooManyRequests, (int)SubStatusCodes.RUBudgetExceeded, DisplayName = "TooManyRequests")]
-        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.ReadSessionNotAvailable, (int)SubStatusCodes.ReadSessionNotAvailable, DisplayName = "ReadSessionNotAvailable")]
-        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.Timeout, (int)StatusCodes.RequestTimeout, (int)SubStatusCodes.Unknown, DisplayName = "Timeout")]
-        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.PartitionIsMigrating, (int)StatusCodes.Gone, (int)SubStatusCodes.CompletingPartitionMigration, DisplayName = "PartitionIsMigrating")]
-        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.PartitionIsSplitting, (int)StatusCodes.Gone, (int)SubStatusCodes.CompletingSplit, DisplayName = "PartitionIsSplitting")]
+        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.Gone, (int)StatusCodes.Gone, (int)SubStatusCodes.ServerGenerated410, DisplayName = "Gone - write")]
+        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.InternalServerError, (int)StatusCodes.InternalServerError, (int)SubStatusCodes.Unknown, DisplayName = "InternalServerError - write")]
+        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.TooManyRequests, (int)StatusCodes.TooManyRequests, (int)SubStatusCodes.RUBudgetExceeded, DisplayName = "TooManyRequests - write")]
+        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.ReadSessionNotAvailable, (int)StatusCodes.NotFound, (int)SubStatusCodes.ReadSessionNotAvailable, DisplayName = "ReadSessionNotAvailable - write")]
+        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.Timeout, (int)StatusCodes.RequestTimeout, (int)SubStatusCodes.Unknown, DisplayName = "Timeout - write")]
+        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.PartitionIsMigrating, (int)StatusCodes.Gone, (int)SubStatusCodes.CompletingPartitionMigration, DisplayName = "PartitionIsMigrating - write")]
+        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.PartitionIsSplitting, (int)StatusCodes.Gone, (int)SubStatusCodes.CompletingSplit, DisplayName = "PartitionIsSplitting - write")]
         public async Task FIGatewayServerResponse(
             FaultInjectionOperationType faultInjectionOperationType, 
             FaultInjectionServerErrorType faultInjectionServerErrorType,
@@ -596,6 +611,7 @@ public async Task FIGatewayServerResponse(
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ConnectionMode = ConnectionMode.Gateway,
                     Serializer = this.serializer,
                     MaxRetryAttemptsOnRateLimitedRequests = 0,
@@ -722,6 +738,7 @@ public async Task FIGatewayHitLimit()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ConnectionMode = ConnectionMode.Gateway,
                     Serializer = this.serializer
                 };
@@ -756,6 +773,11 @@ public async Task FIGatewayHitLimit()
                         this.ValidateFaultInjectionRuleApplication(ex, (int)HttpStatusCode.Gone, hitCountRule);
                         this.ValidateHitCount(hitCountRule, i + 1);
                     }
+                    catch (CosmosException ex)
+                    {
+                        this.ValidateFaultInjectionRuleApplication(ex, (int)HttpStatusCode.Gone, hitCountRule);
+                        this.ValidateHitCount(hitCountRule, i + 1);
+                    }
                 }
             }
             finally
@@ -798,6 +820,7 @@ public async Task FIGatewayInjectionRate()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ConnectionMode = ConnectionMode.Gateway,
                     Serializer = this.serializer
                 };
@@ -867,6 +890,7 @@ public async Task FIOnlyGateway()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ConnectionMode = ConnectionMode.Direct,
                     Serializer = this.serializer
                 };
@@ -893,6 +917,7 @@ public async Task FIOnlyGateway()
                 //Test on gateway mode client
                 cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ConnectionMode = ConnectionMode.Gateway,
                     Serializer = this.serializer
                 };

diff --git a/azure-pipelines-faultinjection.yml b/azure-pipelines-faultinjection.yml
@@ -17,12 +17,12 @@ stages:
         BuildConfiguration: '${{ variables.BuildConfiguration }}'
         VmImage: '${{ variables.VmImage }}'
 
-    - template:  templates/build-fault-injection-test.yml
-      parameters:
-        BuildConfiguration: '${{ variables.BuildConfiguration }}'
-        Arguments: '${{ variables.ReleaseArguments }}'
-        VmImage: '${{ variables.VmImage }}'
-        MultiRegionConnectionString: $(COSMOSDB_MULTI_REGION)
+    # - template:  templates/build-fault-injection-test.yml
+    #   parameters:
+    #     BuildConfiguration: '${{ variables.BuildConfiguration }}'
+    #     Arguments: '${{ variables.ReleaseArguments }}'
+    #     VmImage: '${{ variables.VmImage }}'
+    #     MultiRegionConnectionString: $(COSMOSDB_MULTI_REGION)
 
     - job:
       displayName: FaultInjection Integration Tests ${{ variables.BuildConfiguration }}
@@ -58,7 +58,7 @@ stages:
           publishTestResults: true
           testRunTitle: FaultInjectionTests
         env:
-          COSMOSDB_ACCOUNT_CONNECTION_STRING: $(COSMOSDB_ACCOUNT_CONNECTION_STRING) # Real Account Connection String used by Integration Tests while running as part of release pipeline
+          COSMOSDB_MULTI_REGION: $(COSMOSDB_MULTI_REGION) # Real Account Connection String used by Integration Tests while running as part of release pipeline
 
 - stage:
   displayName: Publish