Fix for timing out SQL integration tests

test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureDisabledTests.cs

@@ -40,7 +40,7 @@ public async Task GivenADatabaseSupportsResourceChangeCapture_WhenResourceChange
             FhirStorageTestsFixture fhirStorageTestsFixture = null;
             try
             {
-                string databaseName = $"FHIRRESOURCECHANGEDISABLEDTEST_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}_{BigInteger.Abs(new BigInteger(Guid.NewGuid().ToByteArray()))}";
+                string databaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName("ChangeCaptureDisabled");
 
                 // this will either create the database or upgrade the schema.
                 var coreFeatureConfigOptions = Options.Create(new CoreFeatureConfiguration() { SupportsResourceChangeCapture = false });

test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureFixture.cs

@@ -31,7 +31,7 @@ public class SqlServerFhirResourceChangeCaptureFixture : IAsyncLifetime
 
         public SqlServerFhirResourceChangeCaptureFixture()
         {
-            _databaseName = $"FHIRRESOURCECHANGEINTTEST_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}_{BigInteger.Abs(new BigInteger(Guid.NewGuid().ToByteArray()))}";
+            _databaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName($"ChangeCapture");
             _coreFeatureConfigOptions = Options.Create(new CoreFeatureConfiguration() { SupportsResourceChangeCapture = true });
             _sqlFixture = new SqlServerFhirStorageTestsFixture(SchemaVersionConstants.Max, _databaseName, _coreFeatureConfigOptions);
             _storageFixture = new FhirStorageTestsFixture(_sqlFixture);

test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlCustomQueryTests.cs

@@ -75,7 +75,7 @@ public async Task GivenASqlQuery_IfAStoredProcExistsWithMatchingHash_ThenStoredP
             // Query after adding an sproc to the database
             var sw = Stopwatch.StartNew();
             var sprocWasUsed = false;
-            while (sw.Elapsed.TotalSeconds < 10) // previous single try after 1.1 sec delay was not reliable.
+            while (sw.Elapsed.TotalSeconds < 100) // previous single try after 1.1 sec delay was not reliable.
             {
                 await Task.Delay(300);
                 await _fixture.SearchService.SearchAsync(KnownResourceTypes.Patient, query, CancellationToken.None);

test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestHelper.cs

@@ -40,7 +40,9 @@ public class SqlServerFhirStorageTestHelper : IFhirStorageTestHelper, ISqlServer
         private readonly ISqlConnectionBuilder _sqlConnectionBuilder;
         private readonly AsyncRetryPolicy _dbSetupRetryPolicy;
         private readonly TestQueueClient _queueClient;
-        private static readonly SemaphoreSlim DbSetupSemaphore = new(4);
+        private static readonly SemaphoreSlim DbSetupSemaphore = new(14); // max number of concurrent requests to the master database is 64 and we run 4 FHIR versions in parallel
+        private static readonly object _locker = new object();
+        private static bool? _isAzure = null;
 
         public SqlServerFhirStorageTestHelper(
             string initialConnectionString,
@@ -61,12 +63,10 @@ public SqlServerFhirStorageTestHelper(
             _dbSetupRetryPolicy = Policy
                 .Handle<Exception>()
                 .WaitAndRetryAsync(
-                    retryCount: 20,
+                    retryCount: 5,
                     sleepDurationProvider: retryAttempt => TimeSpan.FromSeconds(3));
         }
 
-        internal bool DropDatabase => true;
-
         public async Task CreateAndInitializeDatabase(string databaseName, int maximumSupportedSchemaVersion, bool forceIncrementalSchemaUpgrade, SchemaInitializer schemaInitializer = null, CancellationToken cancellationToken = default)
         {
             string testConnectionString = new SqlConnectionStringBuilder(_initialConnectionString) { InitialCatalog = databaseName }.ToString();
@@ -79,7 +79,7 @@ await _dbSetupRetryPolicy.ExecuteAsync(
                     await connection.OpenAsync(cancellationToken);
 
                     await using SqlCommand command = connection.CreateCommand();
-                    command.CommandTimeout = 600;
+                    command.CommandTimeout = 300;
                     command.CommandText = @$"
                         IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = '{databaseName}')
                         BEGIN
@@ -211,7 +211,7 @@ public async Task ExecuteSqlCmd(string sql)
 
         public async Task DeleteDatabase(string databaseName, CancellationToken cancellationToken = default)
         {
-            if (!DropDatabase)
+            if (IsAzure())
             {
                 return;
             }
@@ -226,9 +226,8 @@ public async Task DeleteDatabase(string databaseName, CancellationToken cancella
                     await using SqlConnection connection = await _sqlConnectionBuilder.GetSqlConnectionAsync(_masterDatabaseName, null, cancellationToken);
                     await connection.OpenAsync(cancellationToken);
                     await using SqlCommand command = connection.CreateCommand();
-                    command.CommandTimeout = 600;
+                    command.CommandTimeout = 15;
                     command.CommandText = $"DROP DATABASE IF EXISTS {databaseName}";
-
                     await command.ExecuteNonQueryAsync(cancellationToken);
                     await connection.CloseAsync();
                 }
@@ -384,5 +383,25 @@ protected SqlConnection GetSqlConnection(string connectionString)
             var result = new SqlConnection(connectionBuilder.ToString());
             return result;
         }
+
+        private bool IsAzure()
+        {
+            if (!_isAzure.HasValue)
+            {
+                lock (_locker)
+                {
+                    if (!_isAzure.HasValue)
+                    {
+                        using var conn = _sqlConnectionBuilder.GetSqlConnection(_masterDatabaseName);  // cannot await in the lock
+                        using var cmd = new SqlCommand("SELECT patindex('%Azure%',@@version)", conn);
+                        conn.Open();
+                        var value = cmd.ExecuteScalar();
+                        _isAzure = (int)value > 0;
+                    }
+                }
+            }
+
+            return _isAzure.Value;
+        }
     }
 }

test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs

@@ -76,7 +76,7 @@ public class SqlServerFhirStorageTestsFixture : IServiceProvider, IAsyncLifetime
         private SqlQueueClient _sqlQueueClient;
 
         public SqlServerFhirStorageTestsFixture()
-            : this(SchemaVersionConstants.Max, $"FHIRINTEGRATIONTEST_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}_{BigInteger.Abs(new BigInteger(Guid.NewGuid().ToByteArray()))}")
+            : this(SchemaVersionConstants.Max, GetDatabaseName())
         {
         }
 
@@ -125,6 +125,11 @@ internal SqlServerFhirStorageTestsFixture(int maximumSupportedSchemaVersion, str
 
         internal ISqlQueryHashCalculator SqlQueryHashCalculator { get; private set; }
 
+        internal static string GetDatabaseName(string test = null)
+        {
+            return $"{ModelInfoProvider.Version}{(test == null ? string.Empty : $"_{test}")}_{DateTimeOffset.UtcNow.ToString("s").Replace("-", string.Empty).Replace(":", string.Empty)}_{Guid.NewGuid().ToString().Replace("-", string.Empty)}";
+        }
+
         public async Task InitializeAsync()
         {
             var scriptProvider = new ScriptProvider<SchemaVersion>();

test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSchemaUpgradeTests.cs

@@ -54,8 +54,8 @@ public SqlServerSchemaUpgradeTests()
         [Fact]
         public async Task GivenTwoSchemaInitializationMethods_WhenCreatingTwoDatabases_BothSchemasShouldBeEquivalent()
         {
-            var snapshotDatabaseName = $"SNAPSHOT_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}_{BigInteger.Abs(new BigInteger(Guid.NewGuid().ToByteArray()))}";
-            var diffDatabaseName = $"DIFF_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}_{BigInteger.Abs(new BigInteger(Guid.NewGuid().ToByteArray()))}";
+            var snapshotDatabaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName($"Upgrade_Snapshot");
+            var diffDatabaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName($"Upgrade_Diff");
 
             SqlServerFhirStorageTestHelper testHelper1 = null;
             SqlServerFhirStorageTestHelper testHelper2 = null;
@@ -92,7 +92,7 @@ public void GivenASchemaVersion_WhenApplyingDiffTwice_ShouldSucceed()
                 // The schema upgrade scripts starting from v7 were made idempotent.
                 if (version >= 7 && version >= SchemaVersionConstants.MinForUpgrade) // no sense in checking not supported versions
                 {
-                    var snapshotDatabaseName = $"SNAPSHOT_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}_{BigInteger.Abs(new BigInteger(Guid.NewGuid().ToByteArray()))}";
+                    var snapshotDatabaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName($"Upgrade_Snapshot_Diff");
 
                     SqlServerFhirStorageTestHelper testHelper = null;
                     SchemaUpgradeRunner upgradeRunner;

test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSqlCompatibilityTests.cs

@@ -34,7 +34,7 @@ public class SqlServerSqlCompatibilityTests
         [Fact]
         public async Task GivenADatabaseWithAnEarlierSupportedSchemaAndUpgraded_WhenUpsertingAfter_OperationSucceeds()
         {
-            string databaseName = $"FHIRCOMPATIBILITYTEST_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}";
+            string databaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName("Compatibility");
             var insertedElements = new List<string>();
 
             FhirStorageTestsFixture fhirStorageTestsFixture = null;
@@ -93,7 +93,7 @@ public async Task GivenADatabaseWithAnEarlierSupportedSchema_WhenUpserting_Opera
             {
                 if (version >= Math.Max(SchemaVersionConstants.Min, SchemaVersionConstants.Max - 5) && version <= SchemaVersionConstants.Max)
                 {
-                    string databaseName = $"FHIRCOMPATIBILITYTEST_V{version}_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}";
+                    string databaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName($"Compatibility_V{version}");
                     FhirStorageTestsFixture fhirStorageTestsFixture = null;
                     try
                     {
@@ -135,7 +135,7 @@ public async Task GivenADatabaseWithAnEarlierSupportedSchema_WhenSearchingWithSo
         {
             Skip.If(SchemaVersionConstants.AddMinMaxForDateAndStringSearchParamVersion < SchemaVersionConstants.Min, "Schema version required for this test is not supported");
 
-            string databaseName = $"FHIRCOMPATIBILITYTEST_SORT_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}";
+            string databaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName($"Compatibility_Sort");
             int schemaVersion = SchemaVersionConstants.AddMinMaxForDateAndStringSearchParamVersion - 1;
             var fhirStorageTestsFixture = new FhirStorageTestsFixture(new SqlServerFhirStorageTestsFixture(
                 schemaVersion,