[Internal] Pipelines: Fixes FaultInjection Release Pipelines

Microsoft.Azure.Cosmos/FaultInjection/tests/FaultInjectionDirectModeTests.cs

@@ -17,6 +17,7 @@ namespace Microsoft.Azure.Cosmos.FaultInjection.Tests
     using Microsoft.Azure.Cosmos.Routing;
     using Microsoft.Azure.Documents;
     using static Microsoft.Azure.Cosmos.FaultInjection.Tests.Utils.TestCommon;
+    using ConsistencyLevel = ConsistencyLevel;
     using CosmosSystemTextJsonSerializer = Utils.TestCommon.CosmosSystemTextJsonSerializer;
     using Database = Database;
     using PartitionKey = PartitionKey;
@@ -69,7 +70,7 @@ public async Task Initialize()
 
             CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
             {
-                ConnectionMode = ConnectionMode.Gateway,
+                ConsistencyLevel = ConsistencyLevel.Session,
                 Serializer = this.serializer,
             };
 
@@ -157,6 +158,7 @@ public async Task FaultInjectionServerErrorRule_OperationTypeTest(int operation)
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
                 };
@@ -330,6 +332,7 @@ public async Task FaultInjectionServerErrorRule_OperationTypeAddressTest(int ope
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ApplicationPreferredRegions = preferredRegions,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
@@ -454,6 +457,7 @@ public async Task FaultInjectionServerErrorRule_RegionTest()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
                 };
@@ -555,6 +559,7 @@ public async Task FaultInjectionServerErrorRule_PartitionTest()
 
             CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
             {
+                ConsistencyLevel = ConsistencyLevel.Session,
                 FaultInjector = faultInjector,
                 Serializer = this.serializer,
                 MaxRetryAttemptsOnRateLimitedRequests = 0,
@@ -635,6 +640,7 @@ public async Task FaultInjectionServerErrorRule_ServerSendDelay()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer,
                     EnableContentResponseOnWrite = true,
@@ -727,6 +733,7 @@ public async Task FaultInjectionServerErrorRule_ServerResponseDelay()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer,
                     EnableContentResponseOnWrite = true,
@@ -821,6 +828,7 @@ public async Task FaultInjectionServerErrorRule_ServerTimeout()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
                 };
@@ -904,6 +912,7 @@ public async Task FaultInjectionServerErrorRule_ConnecitonTimeout()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer,
                     EnableContentResponseOnWrite = true,
@@ -984,6 +993,7 @@ public async Task FaultInjectionServerErrorRule_ConnecitonDelay()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer,
                     EnableContentResponseOnWrite = true,
@@ -1091,6 +1101,7 @@ public async Task FaultInjectionServerErrorRule_ServerErrorResponseTest(
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer,
                 };
@@ -1171,6 +1182,7 @@ public async Task FaultInjectionServerErrorRule_HitCountTest()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
                 };
@@ -1266,6 +1278,7 @@ public async Task FaultInjectionServerErrorRule_IncludePrimaryTest()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
                 };
@@ -1339,6 +1352,7 @@ public async Task FaultInjectionServerErrorRule_InjectionRateTest()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
                 };
@@ -1424,6 +1438,7 @@ public async Task FaultInjectionConnectionErrorRule_Test()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
                 };
@@ -1437,21 +1452,29 @@ public async Task FaultInjectionConnectionErrorRule_Test()
                 ChaosInterceptor interceptor = faultInjector.GetChaosInterceptor() as ChaosInterceptor;
                 Assert.IsNotNull(interceptor);
 
-                await this.PerformDocumentOperationAndCheckApplication(
-                    this.fiContainer,
-                    OperationType.Create,
-                    createdItem,
-                    connectionErrorRule,
-                    0,
-                    0);
+                try
+                {
+                    await this.fiContainer.CreateItemAsync<FaultInjectionTestObject>(createdItem);
+                }
+                catch
+                {
+                    //ignore
+                }
 
                 FaultInjectionDynamicChannelStore channelStore = interceptor.GetChannelStore();
                 Assert.IsTrue(channelStore.GetAllChannels().Count > 0);
                 List<Guid> channelGuids = channelStore.GetAllChannelIds();
 
-                await Task.Delay(TimeSpan.FromSeconds(2));
+                try
+                {
+                    await Task.Delay(TimeSpan.FromSeconds(2));
 
-                await this.fiContainer.CreateItemAsync<FaultInjectionTestObject>(createdItem2);
+                    await this.fiContainer.CreateItemAsync<FaultInjectionTestObject>(createdItem2);
+                }
+                catch
+                {
+                    //ignore
+                }
 
                 Assert.IsTrue(connectionErrorRule.GetHitCount() >= 1);
 
@@ -1547,6 +1570,7 @@ public async Task FaultInjectionServerErrorRule_ReadFeedTest()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     FaultInjector = faultInjector,
                     Serializer = this.serializer
                 };
@@ -1832,6 +1856,7 @@ private void ValidateFaultInjectionRuleApplication(
             FaultInjectionRule rule)
         {
             string diagnosticsString = diagnostics.ToString();
+            Console.WriteLine(diagnostics.ToString());
             Assert.IsTrue(1 <= rule.GetHitCount());
             Assert.IsTrue(1 <= diagnostics.GetFailedRequestCount());
             Assert.IsTrue(diagnosticsString.Contains(statusCode.ToString()));

Microsoft.Azure.Cosmos/FaultInjection/tests/FaultInjectionGatewayModeTests.cs

@@ -17,6 +17,7 @@ namespace Microsoft.Azure.Cosmos.FaultInjection.Tests
     using Microsoft.Azure.Cosmos.Routing;
     using Microsoft.Azure.Documents;
     using static Microsoft.Azure.Cosmos.FaultInjection.Tests.Utils.TestCommon;
+    using ConsistencyLevel = ConsistencyLevel;
     using CosmosSystemTextJsonSerializer = Utils.TestCommon.CosmosSystemTextJsonSerializer;
     using Database = Database;
     using PartitionKey = PartitionKey;
@@ -60,6 +61,7 @@ public async Task Initialize()
 
             CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
             {
+                ConsistencyLevel = ConsistencyLevel.Session,
                 ConnectionMode = ConnectionMode.Gateway,
                 Serializer = this.serializer,
             };
@@ -79,8 +81,8 @@ public async Task Cleanup()
             {
                 await this.highThroughputContainer.DeleteContainerAsync();
             }
-            this.client.Dispose();
-            this.fiClient.Dispose();
+            this.client?.Dispose();
+            this.fiClient?.Dispose();
         }
 
         //<summary>
@@ -150,6 +152,7 @@ public async Task FIGatewayRegion()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ConnectionMode = ConnectionMode.Gateway,
                     Serializer = this.serializer
                 };
@@ -182,6 +185,15 @@ public async Task FIGatewayRegion()
                         (int)HttpStatusCode.Gone,
                         localRegionRule);
                 }
+                catch (CosmosException ex)
+                {
+                    this.ValidateHitCount(localRegionRule, 1);
+                    this.ValidateHitCount(remoteRegionRule, 0);
+                    this.ValidateFaultInjectionRuleApplication(
+                        ex,
+                        (int)HttpStatusCode.Gone,
+                        localRegionRule);
+                }
             }
             finally
             {
@@ -256,6 +268,7 @@ public async Task FIGatewayPartitionTest()
 
             CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
             {
+                ConsistencyLevel = ConsistencyLevel.Session,
                 ConnectionMode = ConnectionMode.Gateway,
                 Serializer = this.serializer,
                 MaxRetryAttemptsOnRateLimitedRequests = 0,
@@ -392,6 +405,7 @@ public async Task FIGatewayResponseDelay()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ConnectionMode = ConnectionMode.Gateway,
                     Serializer = this.serializer,
                     EnableContentResponseOnWrite = true,
@@ -486,6 +500,7 @@ public async Task FIGatewaySendDelay()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ConnectionMode = ConnectionMode.Gateway,
                     Serializer = this.serializer,
                     EnableContentResponseOnWrite = true,
@@ -557,13 +572,13 @@ public async Task FIGatewaySendDelay()
         [DataRow(FaultInjectionOperationType.ReadItem, FaultInjectionServerErrorType.Timeout, (int)StatusCodes.RequestTimeout, (int)SubStatusCodes.Unknown, DisplayName = "Timeout")]
         [DataRow(FaultInjectionOperationType.ReadItem, FaultInjectionServerErrorType.PartitionIsMigrating, (int)StatusCodes.Gone, (int)SubStatusCodes.CompletingPartitionMigration, DisplayName = "PartitionIsMigrating")]
         [DataRow(FaultInjectionOperationType.ReadItem, FaultInjectionServerErrorType.PartitionIsSplitting, (int)StatusCodes.Gone, (int)SubStatusCodes.CompletingSplit, DisplayName = "PartitionIsSplitting")]
-        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.Gone, (int)StatusCodes.Gone, (int)SubStatusCodes.ServerGenerated410, DisplayName = "Gone")]
-        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.InternalServerError, (int)StatusCodes.InternalServerError, (int)SubStatusCodes.Unknown, DisplayName = "InternalServerError")]
-        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.TooManyRequests, (int)StatusCodes.TooManyRequests, (int)SubStatusCodes.RUBudgetExceeded, DisplayName = "TooManyRequests")]
-        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.ReadSessionNotAvailable, (int)SubStatusCodes.ReadSessionNotAvailable, DisplayName = "ReadSessionNotAvailable")]
-        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.Timeout, (int)StatusCodes.RequestTimeout, (int)SubStatusCodes.Unknown, DisplayName = "Timeout")]
-        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.PartitionIsMigrating, (int)StatusCodes.Gone, (int)SubStatusCodes.CompletingPartitionMigration, DisplayName = "PartitionIsMigrating")]
-        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.PartitionIsSplitting, (int)StatusCodes.Gone, (int)SubStatusCodes.CompletingSplit, DisplayName = "PartitionIsSplitting")]
+        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.Gone, (int)StatusCodes.Gone, (int)SubStatusCodes.ServerGenerated410, DisplayName = "Gone - write")]
+        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.InternalServerError, (int)StatusCodes.InternalServerError, (int)SubStatusCodes.Unknown, DisplayName = "InternalServerError - write")]
+        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.TooManyRequests, (int)StatusCodes.TooManyRequests, (int)SubStatusCodes.RUBudgetExceeded, DisplayName = "TooManyRequests - write")]
+        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.ReadSessionNotAvailable, (int)StatusCodes.NotFound, (int)SubStatusCodes.ReadSessionNotAvailable, DisplayName = "ReadSessionNotAvailable - write")]
+        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.Timeout, (int)StatusCodes.RequestTimeout, (int)SubStatusCodes.Unknown, DisplayName = "Timeout - write")]
+        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.PartitionIsMigrating, (int)StatusCodes.Gone, (int)SubStatusCodes.CompletingPartitionMigration, DisplayName = "PartitionIsMigrating - write")]
+        [DataRow(FaultInjectionOperationType.CreateItem, FaultInjectionServerErrorType.PartitionIsSplitting, (int)StatusCodes.Gone, (int)SubStatusCodes.CompletingSplit, DisplayName = "PartitionIsSplitting - write")]
         public async Task FIGatewayServerResponse(
             FaultInjectionOperationType faultInjectionOperationType, 
             FaultInjectionServerErrorType faultInjectionServerErrorType,
@@ -596,6 +611,7 @@ public async Task FIGatewayServerResponse(
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ConnectionMode = ConnectionMode.Gateway,
                     Serializer = this.serializer,
                     MaxRetryAttemptsOnRateLimitedRequests = 0,
@@ -722,6 +738,7 @@ public async Task FIGatewayHitLimit()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ConnectionMode = ConnectionMode.Gateway,
                     Serializer = this.serializer
                 };
@@ -756,6 +773,11 @@ public async Task FIGatewayHitLimit()
                         this.ValidateFaultInjectionRuleApplication(ex, (int)HttpStatusCode.Gone, hitCountRule);
                         this.ValidateHitCount(hitCountRule, i + 1);
                     }
+                    catch (CosmosException ex)
+                    {
+                        this.ValidateFaultInjectionRuleApplication(ex, (int)HttpStatusCode.Gone, hitCountRule);
+                        this.ValidateHitCount(hitCountRule, i + 1);
+                    }
                 }
             }
             finally
@@ -798,6 +820,7 @@ public async Task FIGatewayInjectionRate()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ConnectionMode = ConnectionMode.Gateway,
                     Serializer = this.serializer
                 };
@@ -867,6 +890,7 @@ public async Task FIOnlyGateway()
 
                 CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ConnectionMode = ConnectionMode.Direct,
                     Serializer = this.serializer
                 };
@@ -893,6 +917,7 @@ public async Task FIOnlyGateway()
                 //Test on gateway mode client
                 cosmosClientOptions = new CosmosClientOptions()
                 {
+                    ConsistencyLevel = ConsistencyLevel.Session,
                     ConnectionMode = ConnectionMode.Gateway,
                     Serializer = this.serializer
                 };

azure-pipelines-faultinjection.yml

@@ -17,12 +17,12 @@ stages:
         BuildConfiguration: '${{ variables.BuildConfiguration }}'
         VmImage: '${{ variables.VmImage }}'
 
-    - template:  templates/build-fault-injection-test.yml
-      parameters:
-        BuildConfiguration: '${{ variables.BuildConfiguration }}'
-        Arguments: '${{ variables.ReleaseArguments }}'
-        VmImage: '${{ variables.VmImage }}'
-        MultiRegionConnectionString: $(COSMOSDB_MULTI_REGION)
+    # - template:  templates/build-fault-injection-test.yml
+    #   parameters:
+    #     BuildConfiguration: '${{ variables.BuildConfiguration }}'
+    #     Arguments: '${{ variables.ReleaseArguments }}'
+    #     VmImage: '${{ variables.VmImage }}'
+    #     MultiRegionConnectionString: $(COSMOSDB_MULTI_REGION)
 
     - job:
       displayName: FaultInjection Integration Tests ${{ variables.BuildConfiguration }}
@@ -58,7 +58,7 @@ stages:
           publishTestResults: true
           testRunTitle: FaultInjectionTests
         env:
-          COSMOSDB_ACCOUNT_CONNECTION_STRING: $(COSMOSDB_ACCOUNT_CONNECTION_STRING) # Real Account Connection String used by Integration Tests while running as part of release pipeline
+          COSMOSDB_MULTI_REGION: $(COSMOSDB_MULTI_REGION) # Real Account Connection String used by Integration Tests while running as part of release pipeline
 
 - stage:
   displayName: Publish