Fix for timing out SQL integration tests

test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureDisabledTests.cs

@@ -40,7 +40,7 @@ public async Task GivenADatabaseSupportsResourceChangeCapture_WhenResourceChange
             FhirStorageTestsFixture fhirStorageTestsFixture = null;
             try
             {
-                string databaseName = $"FHIRRESOURCECHANGEDISABLEDTEST_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}_{BigInteger.Abs(new BigInteger(Guid.NewGuid().ToByteArray()))}";
+                string databaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName($"ChangeCaptureDisabled");
 
                 // this will either create the database or upgrade the schema.
                 var coreFeatureConfigOptions = Options.Create(new CoreFeatureConfiguration() { SupportsResourceChangeCapture = false });

test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureFixture.cs

@@ -31,7 +31,7 @@ public class SqlServerFhirResourceChangeCaptureFixture : IAsyncLifetime
 
         public SqlServerFhirResourceChangeCaptureFixture()
         {
-            _databaseName = $"FHIRRESOURCECHANGEINTTEST_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}_{BigInteger.Abs(new BigInteger(Guid.NewGuid().ToByteArray()))}";
+            _databaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName($"ChangeCapture");
             _coreFeatureConfigOptions = Options.Create(new CoreFeatureConfiguration() { SupportsResourceChangeCapture = true });
             _sqlFixture = new SqlServerFhirStorageTestsFixture(SchemaVersionConstants.Max, _databaseName, _coreFeatureConfigOptions);
             _storageFixture = new FhirStorageTestsFixture(_sqlFixture);

test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlCustomQueryTests.cs

@@ -75,7 +75,7 @@ public async Task GivenASqlQuery_IfAStoredProcExistsWithMatchingHash_ThenStoredP
             // Query after adding an sproc to the database
             var sw = Stopwatch.StartNew();
             var sprocWasUsed = false;
-            while (sw.Elapsed.TotalSeconds < 10) // previous single try after 1.1 sec delay was not reliable.
+            while (sw.Elapsed.TotalSeconds < 100) // previous single try after 1.1 sec delay was not reliable.
             {
                 await Task.Delay(300);
                 await _fixture.SearchService.SearchAsync(KnownResourceTypes.Patient, query, CancellationToken.None);

test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestHelper.cs

@@ -40,7 +40,8 @@
         private readonly ISqlConnectionBuilder _sqlConnectionBuilder;
         private readonly AsyncRetryPolicy _dbSetupRetryPolicy;
         private readonly TestQueueClient _queueClient;
-        private static readonly SemaphoreSlim DbSetupSemaphore = new(4);
+        private static readonly object _locker = new object();
+        private static bool? _isAzure = null;
 
         public SqlServerFhirStorageTestHelper(
             string initialConnectionString,
@@ -61,12 +62,10 @@
             _dbSetupRetryPolicy = Policy
                 .Handle<Exception>()
                 .WaitAndRetryAsync(
-                    retryCount: 20,
+                    retryCount: 5,
                     sleepDurationProvider: retryAttempt => TimeSpan.FromSeconds(3));
         }
 
-        internal bool DropDatabase => true;
-
         public async Task CreateAndInitializeDatabase(string databaseName, int maximumSupportedSchemaVersion, bool forceIncrementalSchemaUpgrade, SchemaInitializer schemaInitializer = null, CancellationToken cancellationToken = default)
         {
             string testConnectionString = new SqlConnectionStringBuilder(_initialConnectionString) { InitialCatalog = databaseName }.ToString();
@@ -79,23 +78,13 @@
                     await connection.OpenAsync(cancellationToken);
 
                     await using SqlCommand command = connection.CreateCommand();
-                    command.CommandTimeout = 600;
+                    command.CommandTimeout = 300;
                     command.CommandText = @$"
                         IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = '{databaseName}')
                         BEGIN
                           CREATE DATABASE {databaseName};
                         END";
-
-                    await DbSetupSemaphore.WaitAsync(cancellationToken);
-                    try
-                    {
-                        await command.ExecuteNonQueryAsync(cancellationToken);
-                    }
-                    finally
-                    {
-                        DbSetupSemaphore.Release();
-                    }
-
+                    await command.ExecuteNonQueryAsync(cancellationToken);
                     await connection.CloseAsync();
                 });
 
@@ -211,31 +200,34 @@
 
         public async Task DeleteDatabase(string databaseName, CancellationToken cancellationToken = default)
         {
-            if (!DropDatabase)
+            if (!_isAzure.HasValue)
+            {
+                lock (_locker)
+                {
+                    if (!_isAzure.HasValue)
+                    {
+                        _isAzure = IsAzure(); // cannot await in the lock
+                    }
+                }
+            }
+
+            if (_isAzure.Value)
             {
                 return;
             }
 
             try
             {
-                await DbSetupSemaphore.WaitAsync(cancellationToken);
-                try
-                {
-                    SqlConnection.ClearAllPools();
+                SqlConnection.ClearAllPools();
 
-                    await using SqlConnection connection = await _sqlConnectionBuilder.GetSqlConnectionAsync(_masterDatabaseName, null, cancellationToken);
-                    await connection.OpenAsync(cancellationToken);
-                    await using SqlCommand command = connection.CreateCommand();
-                    command.CommandTimeout = 600;
-                    command.CommandText = $"DROP DATABASE IF EXISTS {databaseName}";
+                await using SqlConnection connection = await _sqlConnectionBuilder.GetSqlConnectionAsync(_masterDatabaseName, null, cancellationToken);
+                await connection.OpenAsync(cancellationToken);
+                await using SqlCommand command = connection.CreateCommand();
+                command.CommandTimeout = 15;
+                command.CommandText = $"DROP DATABASE IF EXISTS {databaseName}";
 
-                    await command.ExecuteNonQueryAsync(cancellationToken);
-                    await connection.CloseAsync();
-                }
-                finally
-                {
-                    DbSetupSemaphore.Release();
-                }
+                await command.ExecuteNonQueryAsync(cancellationToken);
+                await connection.CloseAsync();
             }
             catch (Exception ex)
             {
@@ -384,5 +376,14 @@
             var result = new SqlConnection(connectionBuilder.ToString());
             return result;
         }
+
+        private bool IsAzure()
+        {
+            using var conn = _sqlConnectionBuilder.GetSqlConnection(_masterDatabaseName);
+            using var cmd = new SqlCommand("SELECT patindex('%Azure SQL%',@@version)", conn);
+            conn.Open();
+            var value = cmd.ExecuteScalar();
+            return (int)value > 0;
+        }
     }
 }

test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs

@@ -76,7 +76,7 @@ public class SqlServerFhirStorageTestsFixture : IServiceProvider, IAsyncLifetime
         private SqlQueueClient _sqlQueueClient;
 
         public SqlServerFhirStorageTestsFixture()
-            : this(SchemaVersionConstants.Max, $"FHIRINTEGRATIONTEST_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}_{BigInteger.Abs(new BigInteger(Guid.NewGuid().ToByteArray()))}")
+            : this(SchemaVersionConstants.Max, GetDatabaseName())
         {
         }
 
@@ -125,6 +125,11 @@ internal SqlServerFhirStorageTestsFixture(int maximumSupportedSchemaVersion, str
 
         internal ISqlQueryHashCalculator SqlQueryHashCalculator { get; private set; }
 
+        internal static string GetDatabaseName(string test = null)
+        {
+            return $"{ModelInfoProvider.Version}{(test == null ? string.Empty : $"_{test}")}_{DateTimeOffset.UtcNow.ToString("s").Replace("-", string.Empty).Replace(":", string.Empty)}_{Guid.NewGuid().ToString().Replace("-", string.Empty)}";
+        }
+
         public async Task InitializeAsync()
         {
             var scriptProvider = new ScriptProvider<SchemaVersion>();

test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSchemaUpgradeTests.cs

@@ -54,8 +54,8 @@ public SqlServerSchemaUpgradeTests()
         [Fact]
         public async Task GivenTwoSchemaInitializationMethods_WhenCreatingTwoDatabases_BothSchemasShouldBeEquivalent()
         {
-            var snapshotDatabaseName = $"SNAPSHOT_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}_{BigInteger.Abs(new BigInteger(Guid.NewGuid().ToByteArray()))}";
-            var diffDatabaseName = $"DIFF_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}_{BigInteger.Abs(new BigInteger(Guid.NewGuid().ToByteArray()))}";
+            var snapshotDatabaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName($"Upgrade_Snapshot");
+            var diffDatabaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName($"Upgrade_Diff");
 
             SqlServerFhirStorageTestHelper testHelper1 = null;
             SqlServerFhirStorageTestHelper testHelper2 = null;
@@ -92,7 +92,7 @@ public void GivenASchemaVersion_WhenApplyingDiffTwice_ShouldSucceed()
                 // The schema upgrade scripts starting from v7 were made idempotent.
                 if (version >= 7 && version >= SchemaVersionConstants.MinForUpgrade) // no sense in checking not supported versions
                 {
-                    var snapshotDatabaseName = $"SNAPSHOT_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}_{BigInteger.Abs(new BigInteger(Guid.NewGuid().ToByteArray()))}";
+                    var snapshotDatabaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName($"Upgrade_Snapshot_Diff");
 
                     SqlServerFhirStorageTestHelper testHelper = null;
                     SchemaUpgradeRunner upgradeRunner;

test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSqlCompatibilityTests.cs

@@ -34,7 +34,7 @@ public class SqlServerSqlCompatibilityTests
         [Fact]
         public async Task GivenADatabaseWithAnEarlierSupportedSchemaAndUpgraded_WhenUpsertingAfter_OperationSucceeds()
         {
-            string databaseName = $"FHIRCOMPATIBILITYTEST_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}";
+            string databaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName("Compatibility");
             var insertedElements = new List<string>();
 
             FhirStorageTestsFixture fhirStorageTestsFixture = null;
@@ -93,7 +93,7 @@ public async Task GivenADatabaseWithAnEarlierSupportedSchema_WhenUpserting_Opera
             {
                 if (version >= Math.Max(SchemaVersionConstants.Min, SchemaVersionConstants.Max - 5) && version <= SchemaVersionConstants.Max)
                 {
-                    string databaseName = $"FHIRCOMPATIBILITYTEST_V{version}_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}";
+                    string databaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName($"Compatibility_V{version}");
                     FhirStorageTestsFixture fhirStorageTestsFixture = null;
                     try
                     {
@@ -135,7 +135,7 @@ public async Task GivenADatabaseWithAnEarlierSupportedSchema_WhenSearchingWithSo
         {
             Skip.If(SchemaVersionConstants.AddMinMaxForDateAndStringSearchParamVersion < SchemaVersionConstants.Min, "Schema version required for this test is not supported");
 
-            string databaseName = $"FHIRCOMPATIBILITYTEST_SORT_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}";
+            string databaseName = SqlServerFhirStorageTestsFixture.GetDatabaseName($"Compatibility_Sort");
             int schemaVersion = SchemaVersionConstants.AddMinMaxForDateAndStringSearchParamVersion - 1;
             var fhirStorageTestsFixture = new FhirStorageTestsFixture(new SqlServerFhirStorageTestsFixture(
                 schemaVersion,