From c25fb311302024f67b1bdaaaf20c4f158acd18ec Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Fri, 8 Mar 2024 21:09:50 +0530 Subject: [PATCH 01/28] a)Changes done on existing Microsoft.Health.Fhir.CosmosDb Project 1.Icollectionupdater copied to Istoreprocedure in cosmos db old project 2.older Istoreprocedure logic moved to new project with filename IStoredProcedureInstaller 3.StoredProcedureInstaller.cs removed from old db(Microsoft.Health.Fhir.CosmosDb) and moved this logic to new project with filename DataPlaneStoredProcedureInstaller.cs 4.Configs folder contains classes like CosmosDataStoreConfiguration,CosmosCollectionConfiguration moved to Microsoft.Health.Fhir.CosmosDb.Core project 5.Following classes moved to Microsoft.Health.Fhir.CosmosDb.Core project -SystemData -KnownDocumentProperties -ICollectionUpdater -CollectionVersion -CollectionUpgradeManager -IUpgradeManager -CollectionInitializer : we have removed depenedancy for RetryExceptionPolicyFactory in constructor -ICollectionInitializer -CosmosClientExtensions -ICosmosDbDistributedLock -ICosmosDbDistributedLockFactory -ICosmosClientTestProvider 6. Following existing Classes/interfaces modified -ICosmosClientInitializer -FhirCosmosClientInitializer -CosmosDbFhirStorageTestsFixture // TODO: TODO: need modify this class to call new CosmosDB .Initialization functions // now some existing call are commented -CosmosContainerProvider // TODO: TODO: need modify this class to call new CosmosDB .Initialization functions // now some existing call are commented b) New Microsoft.Health.Fhir.CosmosDb.Initialization project added -Initialization of cosmos DB functionality moved here - Moved Storeprocedure installation logic here Added classes like DataPlaneStoredProcedureInstaller,StoredProcedureMetadataBase - Also we have DataPlaneCollectionSetup class which will implement following methods -CreateDatabaseAsync - CreateCollection -UpdateFhirCollectionSettings c)Unit Test Project Added Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests project d)Also Added Microsoft.Health.Fhir.CosmosDb.Core project - We have moved common functionalities from Microsoft.Health.Fhir.CosmosDb project #User Story 112343: [Cosmos][OSS] Move Cosmos DB initialization logic to a new assembly --- Microsoft.Health.Fhir.sln | 23 ++- .../Configs/CosmosCollectionConfiguration.cs | 2 +- .../Configs/CosmosDataStoreConfiguration.cs | 2 +- .../CosmosDataStoreParallelQueryOptions.cs | 2 +- .../Configs/CosmosDataStoreRetryOptions.cs | 2 +- .../Constants.cs | 13 ++ .../Storage/CosmosClientExtensions.cs | 2 +- .../Storage/ICosmosClientTestProvider.cs | 4 +- .../Storage/ICosmosDbDistributedLock.cs | 2 +- .../ICosmosDbDistributedLockFactory.cs | 2 +- .../Storage/KnownDocumentProperties.cs | 2 +- .../Features/Storage/SystemData.cs | 2 +- .../Versioning/CollectionUpgradeManager.cs | 6 +- .../Storage/Versioning/CollectionVersion.cs | 2 +- .../Storage/Versioning/ICollectionUpdater.cs | 2 +- .../Storage/Versioning/IUpgradeManager.cs | 2 +- ...Microsoft.Health.Fhir.CosmosDb.Core.csproj | 10 ++ .../CosmosDbInitializationTests.cs | 28 ++++ ...r.CosmosDb.Initialization.UnitTests.csproj | 12 ++ .../Features/Storage/CollectionInitializer.cs | 20 ++- .../Storage/DataPlaneCollectionSetup.cs | 149 ++++++++++++++++++ .../Storage/ICollectionInitializer.cs | 5 +- .../Features/Storage/ICollectionSetup.cs | 26 +++ .../Features/Storage/NullcollectionSetup.cs | 35 ++++ .../AcquireExportJobsMetadata.cs | 13 ++ .../AcquireExportJobs/acquireExportJobs.js | 0 .../AcquireReindexJobsMetadata.cs | 13 ++ .../AcquireReindexJobs/acquireReindexJobs.js | 0 .../DataPlaneStoredProcedureInstaller.cs} | 13 +- .../HardDelete/HardDeleteMetadata.cs | 13 ++ .../StoredProcedures/HardDelete/hardDelete.js | 0 .../IStoredProcedureInstaller.cs | 16 ++ .../IStoredProcedureMetadata.cs | 21 +++ .../Replace/ReplaceSingleResourceMetadata.cs | 12 ++ .../Replace/replaceSingleResource.js | 0 .../StoredProcedureMetadataBase.cs | 76 +++++++++ ...dateUnsupportedSearchParametersMetadata.cs | 13 ++ .../updateUnsupportedSearchParameters.js | 0 ...Health.Fhir.CosmosDb.Initialization.csproj | 19 +++ .../Features/Health/CosmosHealthCheckTests.cs | 3 +- .../Features/Health/TestCosmosHealthCheck.cs | 3 +- .../Storage/CosmosFhirDataStoreTests.cs | 2 +- .../FhirCosmosClientInitializerTests.cs | 4 +- ...DbSearchParameterStatusInitializerTests.cs | 2 +- .../CollectionUpgradeManagerTests.cs | 4 +- .../Features/Health/CosmosHealthCheck.cs | 3 +- .../Search/CosmosDbSortingValidator.cs | 4 +- .../Search/FhirCosmosSearchService.cs | 10 +- .../Search/Queries/ExpressionQueryBuilder.cs | 2 +- .../Features/Search/Queries/QueryBuilder.cs | 1 + .../CosmosClientReadWriteTestProvider.cs | 3 +- .../Storage/CosmosContainerProvider.cs | 12 +- ...CosmosDbCollectionPhysicalPartitionInfo.cs | 2 +- .../Storage/CosmosDbDistributedLock.cs | 1 + .../Storage/CosmosDbDistributedLockFactory.cs | 1 + .../Features/Storage/CosmosFhirDataStore.cs | 18 +-- .../Storage/CosmosResponseProcessor.cs | 6 +- .../Storage/FhirCosmosClientInitializer.cs | 9 +- .../Storage/FhirCosmosResourceWrapper.cs | 1 + .../Storage/FhirCosmosResponseHandler.cs | 2 +- .../Storage/ICosmosClientInitializer.cs | 6 +- .../CosmosFhirOperationDataStore.cs | 11 +- .../Operations/CosmosJobRecordWrapper.cs | 1 + .../Export/CosmosExportJobRecordWrapper.cs | 1 + .../Reindex/CosmosReindexJobRecordWrapper.cs | 1 + .../Storage/Queues/CosmosQueueClient.cs | 1 + .../Storage/Queues/JobGroupWrapper.cs | 1 + .../CosmosDbSearchParameterStatusDataStore.cs | 2 +- ...osmosDbSearchParameterStatusInitializer.cs | 4 +- .../Registry/SearchParameterStatusWrapper.cs | 1 + .../Storage/RetryExceptionPolicyFactory.cs | 2 +- .../AcquireExportJobs/AcquireExportJobs.cs | 6 + .../AcquireReindexJobs/AcquireReindexJobs.cs | 5 + .../StoredProcedures/HardDelete/HardDelete.cs | 5 + .../StoredProcedures/IStoredProcedure.cs | 7 +- .../Replace/ReplaceSingleResource.cs | 5 + .../StoredProcedures/StoredProcedureBase.cs | 40 +++-- .../UpdateUnsupportedSearchParameters.cs | 5 + .../FhirCollectionSettingsUpdater.cs | 7 +- .../Microsoft.Health.Fhir.CosmosDb.csproj | 11 +- ...erBuilderCosmosDbRegistrationExtensions.cs | 14 +- .../CosmosDbFhirStorageTestsFixture.cs | 25 ++- 82 files changed, 692 insertions(+), 126 deletions(-) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Core}/Configs/CosmosCollectionConfiguration.cs (90%) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Core}/Configs/CosmosDataStoreConfiguration.cs (98%) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Core}/Configs/CosmosDataStoreParallelQueryOptions.cs (94%) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Core}/Configs/CosmosDataStoreRetryOptions.cs (93%) create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Core/Constants.cs rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Core}/Features/Storage/CosmosClientExtensions.cs (93%) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Core}/Features/Storage/ICosmosClientTestProvider.cs (86%) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Core}/Features/Storage/ICosmosDbDistributedLock.cs (95%) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Core}/Features/Storage/ICosmosDbDistributedLockFactory.cs (90%) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Core}/Features/Storage/KnownDocumentProperties.cs (93%) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Core}/Features/Storage/SystemData.cs (95%) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Core}/Features/Storage/Versioning/CollectionUpgradeManager.cs (95%) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Core}/Features/Storage/Versioning/CollectionVersion.cs (91%) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Core}/Features/Storage/Versioning/ICollectionUpdater.cs (88%) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Core}/Features/Storage/Versioning/IUpgradeManager.cs (87%) create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Core/Microsoft.Health.Fhir.CosmosDb.Core.csproj create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Initialization}/Features/Storage/CollectionInitializer.cs (85%) create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Initialization}/Features/Storage/ICollectionInitializer.cs (77%) create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionSetup.cs create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/NullcollectionSetup.cs create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobsMetadata.cs rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Initialization}/Features/Storage/StoredProcedures/AcquireExportJobs/acquireExportJobs.js (100%) create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobsMetadata.cs rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Initialization}/Features/Storage/StoredProcedures/AcquireReindexJobs/acquireReindexJobs.js (100%) rename src/{Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/StoredProcedureInstaller.cs => Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/DataPlaneStoredProcedureInstaller.cs} (68%) create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/HardDelete/HardDeleteMetadata.cs rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Initialization}/Features/Storage/StoredProcedures/HardDelete/hardDelete.js (100%) create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/IStoredProcedureInstaller.cs create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/IStoredProcedureMetadata.cs create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/Replace/ReplaceSingleResourceMetadata.cs rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Initialization}/Features/Storage/StoredProcedures/Replace/replaceSingleResource.js (100%) create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/StoredProcedureMetadataBase.cs create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParametersMetadata.cs rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Initialization}/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/updateUnsupportedSearchParameters.js (100%) create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj diff --git a/Microsoft.Health.Fhir.sln b/Microsoft.Health.Fhir.sln index 5b198b5284..788977da40 100644 --- a/Microsoft.Health.Fhir.sln +++ b/Microsoft.Health.Fhir.sln @@ -199,6 +199,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PerfTester", "tools\PerfTes EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SqlScriptRunner", "tools\SqlScriptRunner\SqlScriptRunner.csproj", "{76C29222-8D35-43A2-89C5-43114D113C10}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.CosmosDb.Initialization", "src\Microsoft.Health.Fhir.CosmosDb.Initialization\Microsoft.Health.Fhir.CosmosDb.Initialization.csproj", "{10661BC9-01B0-4E35-9751-3B5CE97E25C0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests", "src\Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests\Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj", "{B9AAA11D-8C8C-44C3-AADE-801376EF82F0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.CosmosDb.Core", "src\Microsoft.Health.Fhir.CosmosDb.Core\Microsoft.Health.Fhir.CosmosDb.Core.csproj", "{1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -465,6 +471,18 @@ Global {76C29222-8D35-43A2-89C5-43114D113C10}.Debug|Any CPU.Build.0 = Debug|Any CPU {76C29222-8D35-43A2-89C5-43114D113C10}.Release|Any CPU.ActiveCfg = Release|Any CPU {76C29222-8D35-43A2-89C5-43114D113C10}.Release|Any CPU.Build.0 = Release|Any CPU + {10661BC9-01B0-4E35-9751-3B5CE97E25C0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {10661BC9-01B0-4E35-9751-3B5CE97E25C0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {10661BC9-01B0-4E35-9751-3B5CE97E25C0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {10661BC9-01B0-4E35-9751-3B5CE97E25C0}.Release|Any CPU.Build.0 = Release|Any CPU + {B9AAA11D-8C8C-44C3-AADE-801376EF82F0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B9AAA11D-8C8C-44C3-AADE-801376EF82F0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B9AAA11D-8C8C-44C3-AADE-801376EF82F0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B9AAA11D-8C8C-44C3-AADE-801376EF82F0}.Release|Any CPU.Build.0 = Release|Any CPU + {1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -557,10 +575,13 @@ Global {71A1A274-23A9-441B-BB47-1FC561FF0F35} = {B70945F4-01A6-4351-955B-C4A2943B5E3B} {5E0B69FE-40DE-42E8-9F67-4BB7410AF67C} = {B70945F4-01A6-4351-955B-C4A2943B5E3B} {76C29222-8D35-43A2-89C5-43114D113C10} = {B70945F4-01A6-4351-955B-C4A2943B5E3B} + {10661BC9-01B0-4E35-9751-3B5CE97E25C0} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} + {B9AAA11D-8C8C-44C3-AADE-801376EF82F0} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} + {1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {E370FB31-CF95-47D1-B1E1-863A77973FF8} RESX_SortFileContentOnSave = True + SolutionGuid = {E370FB31-CF95-47D1-B1E1-863A77973FF8} EndGlobalSection GlobalSection(SharedMSBuildProjectFiles) = preSolution test\Microsoft.Health.Fhir.Shared.Tests.E2E.Common\Microsoft.Health.Fhir.Shared.Tests.E2E.Common.projitems*{0478b687-7105-40f6-a2dc-81057890e944}*SharedItemsImports = 13 diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Configs/CosmosCollectionConfiguration.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Configs/CosmosCollectionConfiguration.cs similarity index 90% rename from src/Microsoft.Health.Fhir.CosmosDb/Configs/CosmosCollectionConfiguration.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Configs/CosmosCollectionConfiguration.cs index 2130fe6ffe..7447ee7eea 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Configs/CosmosCollectionConfiguration.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Configs/CosmosCollectionConfiguration.cs @@ -3,7 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -namespace Microsoft.Health.Fhir.CosmosDb.Configs +namespace Microsoft.Health.Fhir.CosmosDb.Core.Configs { public class CosmosCollectionConfiguration { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Configs/CosmosDataStoreConfiguration.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Configs/CosmosDataStoreConfiguration.cs similarity index 98% rename from src/Microsoft.Health.Fhir.CosmosDb/Configs/CosmosDataStoreConfiguration.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Configs/CosmosDataStoreConfiguration.cs index f6d85f5852..eb21be6264 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Configs/CosmosDataStoreConfiguration.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Configs/CosmosDataStoreConfiguration.cs @@ -6,7 +6,7 @@ using System.Collections.Generic; using Microsoft.Azure.Cosmos; -namespace Microsoft.Health.Fhir.CosmosDb.Configs +namespace Microsoft.Health.Fhir.CosmosDb.Core.Configs { public class CosmosDataStoreConfiguration { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Configs/CosmosDataStoreParallelQueryOptions.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Configs/CosmosDataStoreParallelQueryOptions.cs similarity index 94% rename from src/Microsoft.Health.Fhir.CosmosDb/Configs/CosmosDataStoreParallelQueryOptions.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Configs/CosmosDataStoreParallelQueryOptions.cs index a161523b83..78c28292e0 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Configs/CosmosDataStoreParallelQueryOptions.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Configs/CosmosDataStoreParallelQueryOptions.cs @@ -5,7 +5,7 @@ using System; -namespace Microsoft.Health.Fhir.CosmosDb.Configs +namespace Microsoft.Health.Fhir.CosmosDb.Core.Configs { public class CosmosDataStoreParallelQueryOptions { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Configs/CosmosDataStoreRetryOptions.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Configs/CosmosDataStoreRetryOptions.cs similarity index 93% rename from src/Microsoft.Health.Fhir.CosmosDb/Configs/CosmosDataStoreRetryOptions.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Configs/CosmosDataStoreRetryOptions.cs index 9e789d508a..e99ad62f2a 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Configs/CosmosDataStoreRetryOptions.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Configs/CosmosDataStoreRetryOptions.cs @@ -3,7 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -namespace Microsoft.Health.Fhir.CosmosDb.Configs +namespace Microsoft.Health.Fhir.CosmosDb.Core.Configs { public class CosmosDataStoreRetryOptions { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Constants.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Constants.cs new file mode 100644 index 0000000000..e5c4b4d54c --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Constants.cs @@ -0,0 +1,13 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.CosmosDb.Core +{ + public static class Constants + { + public const string CollectionConfigurationName = "fhirCosmosDb"; + public const string RawResource = "rawResource"; + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosClientExtensions.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/CosmosClientExtensions.cs similarity index 93% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosClientExtensions.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/CosmosClientExtensions.cs index 6c37bf27c5..d43cfb79ee 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosClientExtensions.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/CosmosClientExtensions.cs @@ -7,7 +7,7 @@ using System.Threading.Tasks; using Microsoft.Azure.Cosmos; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage { public static class CosmosClientExtensions { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICosmosClientTestProvider.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICosmosClientTestProvider.cs similarity index 86% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICosmosClientTestProvider.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICosmosClientTestProvider.cs index 5b2f98d058..8f376ddd28 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICosmosClientTestProvider.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICosmosClientTestProvider.cs @@ -6,9 +6,9 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage { public interface ICosmosClientTestProvider { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICosmosDbDistributedLock.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICosmosDbDistributedLock.cs similarity index 95% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICosmosDbDistributedLock.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICosmosDbDistributedLock.cs index 438fb873d0..a230b97dc6 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICosmosDbDistributedLock.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICosmosDbDistributedLock.cs @@ -7,7 +7,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage { public interface ICosmosDbDistributedLock : IAsyncDisposable, IDisposable { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICosmosDbDistributedLockFactory.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICosmosDbDistributedLockFactory.cs similarity index 90% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICosmosDbDistributedLockFactory.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICosmosDbDistributedLockFactory.cs index c315b4ef67..095997af51 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICosmosDbDistributedLockFactory.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICosmosDbDistributedLockFactory.cs @@ -5,7 +5,7 @@ using Microsoft.Azure.Cosmos; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage { public interface ICosmosDbDistributedLockFactory { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/KnownDocumentProperties.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/KnownDocumentProperties.cs similarity index 93% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/KnownDocumentProperties.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/KnownDocumentProperties.cs index 44ec2d0451..a3ac60a4e0 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/KnownDocumentProperties.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/KnownDocumentProperties.cs @@ -3,7 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage { public static class KnownDocumentProperties { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/SystemData.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/SystemData.cs similarity index 95% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/SystemData.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/SystemData.cs index 040d80d6a5..c7b1c06a86 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/SystemData.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/SystemData.cs @@ -5,7 +5,7 @@ using Newtonsoft.Json; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage { /// /// Defines a base object for serializing System/Meta information to a CosmosDb collection diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/CollectionUpgradeManager.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs similarity index 95% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/CollectionUpgradeManager.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs index 0ee8d49834..d229ab346a 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/CollectionUpgradeManager.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs @@ -11,11 +11,11 @@ using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning { - internal class CollectionUpgradeManager : IUpgradeManager + public class CollectionUpgradeManager : IUpgradeManager { private readonly IEnumerable _collectionUpdater; private readonly CosmosDataStoreConfiguration _configuration; diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/CollectionVersion.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionVersion.cs similarity index 91% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/CollectionVersion.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionVersion.cs index c298ea0bb5..caa00fce17 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/CollectionVersion.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionVersion.cs @@ -5,7 +5,7 @@ using Newtonsoft.Json; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning { public class CollectionVersion : SystemData { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/ICollectionUpdater.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/ICollectionUpdater.cs similarity index 88% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/ICollectionUpdater.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/ICollectionUpdater.cs index c2842808a0..08a3c0b980 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/ICollectionUpdater.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/ICollectionUpdater.cs @@ -7,7 +7,7 @@ using System.Threading.Tasks; using Microsoft.Azure.Cosmos; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning { public interface ICollectionUpdater { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/IUpgradeManager.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/IUpgradeManager.cs similarity index 87% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/IUpgradeManager.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/IUpgradeManager.cs index 2d169b97ed..9282cb7b5e 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/IUpgradeManager.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/IUpgradeManager.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; using Microsoft.Azure.Cosmos; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning { public interface IUpgradeManager { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Microsoft.Health.Fhir.CosmosDb.Core.csproj b/src/Microsoft.Health.Fhir.CosmosDb.Core/Microsoft.Health.Fhir.CosmosDb.Core.csproj new file mode 100644 index 0000000000..04072db7f7 --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Microsoft.Health.Fhir.CosmosDb.Core.csproj @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs new file mode 100644 index 0000000000..56b4277032 --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Linq; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; +using Xunit; + +namespace CosmosDb.Initialization.UnitTests +{ + public class CosmosDbInitializationTests + { + [Fact] + public void Test1() + { + var storeProcs = new string[] { "AcquireExportJobsMetadata", "AcquireReindexJobsMetadata", "HardDeleteMetadata", "ReplaceSingleResourceMetadata", "UpdateUnsupportedSearchParametersMetadata" }; + var fhirStoredProcsClasses = typeof(StoredProcedureMetadataBase).Assembly + .GetTypes().Where(x => !x.IsAbstract && typeof(StoredProcedureMetadataBase).IsAssignableFrom(x)) + .ToArray(); + foreach (var storeproc in fhirStoredProcsClasses) + { + Assert.Contains(storeproc.Name, storeProcs); + } + } + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj new file mode 100644 index 0000000000..a31b125781 --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CollectionInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs similarity index 85% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CollectionInitializer.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs index 3791292609..622c91f6a4 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CollectionInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs @@ -10,18 +10,19 @@ using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging; using Microsoft.Health.Abstractions.Exceptions; -using Microsoft.Health.Fhir.CosmosDb.Configs; -using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; using Polly; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage { public class CollectionInitializer : ICollectionInitializer { private readonly CosmosCollectionConfiguration _cosmosCollectionConfiguration; private readonly CosmosDataStoreConfiguration _cosmosDataStoreConfiguration; private readonly IUpgradeManager _upgradeManager; - private readonly RetryExceptionPolicyFactory _retryExceptionPolicyFactory; + private readonly ICosmosClientTestProvider _clientTestProvider; private readonly ILogger _logger; @@ -29,35 +30,32 @@ public CollectionInitializer( CosmosCollectionConfiguration cosmosCollectionConfiguration, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, IUpgradeManager upgradeManager, - RetryExceptionPolicyFactory retryExceptionPolicyFactory, ICosmosClientTestProvider clientTestProvider, ILogger logger) { EnsureArg.IsNotNull(cosmosCollectionConfiguration, nameof(cosmosCollectionConfiguration)); EnsureArg.IsNotNull(cosmosCollectionConfiguration.CollectionId, nameof(CosmosCollectionConfiguration.CollectionId)); + EnsureArg.IsNotNull(clientTestProvider, nameof(clientTestProvider)); EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration)); EnsureArg.IsNotNull(upgradeManager, nameof(upgradeManager)); - EnsureArg.IsNotNull(retryExceptionPolicyFactory, nameof(retryExceptionPolicyFactory)); EnsureArg.IsNotNull(logger, nameof(logger)); _cosmosCollectionConfiguration = cosmosCollectionConfiguration; _cosmosDataStoreConfiguration = cosmosDataStoreConfiguration; _upgradeManager = upgradeManager; - _retryExceptionPolicyFactory = retryExceptionPolicyFactory; + _clientTestProvider = clientTestProvider; _logger = logger; } - public async Task InitializeCollectionAsync(CosmosClient client, CancellationToken cancellationToken = default) + public async Task InitializeCollectionAsync(CosmosClient client, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) { Database database = client.GetDatabase(_cosmosDataStoreConfiguration.DatabaseId); Container containerClient = database.GetContainer(_cosmosCollectionConfiguration.CollectionId); _logger.LogInformation("Finding Container: {CollectionId}", _cosmosCollectionConfiguration.CollectionId); - AsyncPolicy retryPolicy = _retryExceptionPolicyFactory.RetryPolicy; - var existingContainer = await retryPolicy.ExecuteAsync(async () => await database.TryGetContainerAsync(_cosmosCollectionConfiguration.CollectionId)); _logger.LogInformation("Creating Cosmos Container if not exits: {CollectionId}", _cosmosCollectionConfiguration.CollectionId); @@ -84,7 +82,7 @@ await retryPolicy.ExecuteAsync(async () => { try { - await _clientTestProvider.PerformTestAsync(existingContainer, _cosmosDataStoreConfiguration, _cosmosCollectionConfiguration, cancellationToken); + await _clientTestProvider.PerformTestAsync(existingContainer, _cosmosDataStoreConfiguration, _cosmosCollectionConfiguration, cancellationToken); } catch (CosmosException e) when (e.StatusCode == HttpStatusCode.TooManyRequests) { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs new file mode 100644 index 0000000000..6e07248201 --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs @@ -0,0 +1,149 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Azure.Cosmos; +using Microsoft.Extensions.Logging; +using Microsoft.Health.Abstractions.Exceptions; +using Microsoft.Health.Core.Features.Context; +using Microsoft.Health.Fhir.CosmosDb.Core; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; +using Polly; +using Polly.Retry; + +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage +{ + public class DataPlaneCollectionSetup : ICollectionSetup + { + private readonly ILogger _logger; + private const int CollectionSettingsVersion = 3; + + public DataPlaneCollectionSetup(ILogger logger) + { + EnsureArg.IsNotNull(logger, nameof(logger)); + _logger = logger; + } + + public async Task CreateDatabaseAsync(CosmosClient client, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) + { + EnsureArg.IsNotNull(client, nameof(client)); + EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration)); + + try + { + _logger.LogInformation("Creating Cosmos DB Database {DatabaseId}", cosmosDataStoreConfiguration.DatabaseId); + + if (cosmosDataStoreConfiguration.AllowDatabaseCreation) + { + _logger.LogInformation("CreateDatabaseIfNotExists {DatabaseId}", cosmosDataStoreConfiguration.DatabaseId); + + await retryPolicy.ExecuteAsync( + async () => + await client.CreateDatabaseIfNotExistsAsync( + cosmosDataStoreConfiguration.DatabaseId, + cosmosDataStoreConfiguration.InitialDatabaseThroughput.HasValue ? ThroughputProperties.CreateManualThroughput(cosmosDataStoreConfiguration.InitialDatabaseThroughput.Value) : null, + cancellationToken: cancellationToken)); + } + + _logger.LogInformation("Cosmos DB Database {DatabaseId} successfully initialized", cosmosDataStoreConfiguration.DatabaseId); + } + catch (Exception ex) + { + LogLevel logLevel = ex is RequestRateExceededException ? LogLevel.Warning : LogLevel.Critical; + _logger.Log(logLevel, ex, "Cosmos DB Database {DatabaseId} initialization failed", cosmosDataStoreConfiguration.DatabaseId); + throw; + } + } + + public async Task CreateCollection(CosmosClient client, IEnumerable collectionInitializers, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) + { + EnsureArg.IsNotNull(client, nameof(client)); + EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration)); + EnsureArg.IsNotNull(collectionInitializers, nameof(collectionInitializers)); + + try + { + _logger.LogInformation("Initializing Cosmos DB Database {DatabaseId} and collections", cosmosDataStoreConfiguration.DatabaseId); + + foreach (var collectionInitializer in collectionInitializers) + { + await collectionInitializer.InitializeCollectionAsync(client, retryPolicy, cancellationToken); + } + + _logger.LogInformation("Collections successfully initialized"); + } + catch (Exception ex) + { + LogLevel logLevel = ex is RequestRateExceededException ? LogLevel.Warning : LogLevel.Critical; + _logger.Log(logLevel, ex, "Collections initialization failed"); + throw; + } + } + + public async Task UpdateFhirCollectionSettings(Container container, CancellationToken cancellationToken) + { + EnsureArg.IsNotNull(container, nameof(container)); + + var thisVersion = await GetLatestCollectionVersion(container, cancellationToken); + + if (thisVersion.Version < 2) + { + var containerResponse = await container.ReadContainerAsync(cancellationToken: cancellationToken); + + // For more information on setting indexing policies refer to: + // https://docs.microsoft.com/en-us/azure/cosmos-db/how-to-manage-indexing-policy + // It is no longer necessary to explicitly set the kind (range/hash) + containerResponse.Resource.IndexingPolicy.IncludedPaths.Clear(); + containerResponse.Resource.IndexingPolicy.IncludedPaths.Add(new IncludedPath + { + Path = "/*", + }); + + containerResponse.Resource.IndexingPolicy.ExcludedPaths.Clear(); + containerResponse.Resource.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath { Path = $"/{Constants.RawResource}/*", }); + containerResponse.Resource.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath { Path = $"/\"_etag\"/?", }); + + // Setting the DefaultTTL to -1 means that by default all documents in the collection will live forever + // but the Cosmos DB service should monitor this collection for documents that have overridden this default. + // See: https://docs.microsoft.com/en-us/azure/cosmos-db/time-to-live + containerResponse.Resource.DefaultTimeToLive = -1; + + await container.ReplaceContainerAsync(containerResponse, cancellationToken: cancellationToken); + + thisVersion.Version = CollectionSettingsVersion; + await container.UpsertItemAsync(thisVersion, new PartitionKey(thisVersion.PartitionKey), cancellationToken: cancellationToken); + } + + // TODO:Need to Validate this logic + /* else if (thisVersion.Version < CollectionSettingsVersion) + { + await _updateSP.Execute(container.Scripts, cancellationToken); + thisVersion.Version = CollectionSettingsVersion; + await container.UpsertItemAsync(thisVersion, new PartitionKey(thisVersion.PartitionKey), cancellationToken: cancellationToken); + } */ + } + + private static async Task GetLatestCollectionVersion(Container container, CancellationToken cancellationToken) + { + FeedIterator query = container.GetItemQueryIterator( + new QueryDefinition("SELECT * FROM root r"), + requestOptions: new QueryRequestOptions + { + PartitionKey = new PartitionKey(CollectionVersion.CollectionVersionPartition), + }); + + FeedResponse result = await query.ReadNextAsync(cancellationToken); + + return result.FirstOrDefault() ?? new CollectionVersion(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICollectionInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionInitializer.cs similarity index 77% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICollectionInitializer.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionInitializer.cs index 3fb4ab2e87..e145290c86 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICollectionInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionInitializer.cs @@ -6,11 +6,12 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos; +using Polly; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage { public interface ICollectionInitializer { - Task InitializeCollectionAsync(CosmosClient client, CancellationToken cancellationToken = default); + Task InitializeCollectionAsync(CosmosClient client, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default); } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionSetup.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionSetup.cs new file mode 100644 index 0000000000..1a7863ee42 --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionSetup.cs @@ -0,0 +1,26 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Cosmos; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Polly; + +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage +{ + public interface ICollectionSetup + { + public Task CreateDatabaseAsync(CosmosClient client, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken); + + public Task CreateCollection(CosmosClient client, IEnumerable collectionInitializers, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default); + + public Task UpdateFhirCollectionSettings(Container container, CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/NullcollectionSetup.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/NullcollectionSetup.cs new file mode 100644 index 0000000000..cec6fcb96a --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/NullcollectionSetup.cs @@ -0,0 +1,35 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Cosmos; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Polly; + +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage +{ + public class NullcollectionSetup : ICollectionSetup + { + public Task CreateCollection(CosmosClient client, IEnumerable collectionInitializers, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) + { + return Task.CompletedTask; + } + + public Task CreateDatabaseAsync(CosmosClient client, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) + { + return Task.CompletedTask; + } + + public Task UpdateFhirCollectionSettings(Container container, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobsMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobsMetadata.cs new file mode 100644 index 0000000000..7bab9c289a --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobsMetadata.cs @@ -0,0 +1,13 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; + +namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.AcquireExportJobs +{ + public sealed class AcquireExportJobsMetadata : StoredProcedureMetadataBase + { + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/acquireExportJobs.js b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireExportJobs/acquireExportJobs.js similarity index 100% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/acquireExportJobs.js rename to src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireExportJobs/acquireExportJobs.js diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobsMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobsMetadata.cs new file mode 100644 index 0000000000..fb697e7773 --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobsMetadata.cs @@ -0,0 +1,13 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; + +namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.AcquireReindexJobs +{ + public sealed class AcquireReindexJobsMetadata : StoredProcedureMetadataBase + { + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/acquireReindexJobs.js b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireReindexJobs/acquireReindexJobs.js similarity index 100% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/acquireReindexJobs.js rename to src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireReindexJobs/acquireReindexJobs.js diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/StoredProcedureInstaller.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/DataPlaneStoredProcedureInstaller.cs similarity index 68% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/StoredProcedureInstaller.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/DataPlaneStoredProcedureInstaller.cs index 1f05340a12..40530d83ff 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/StoredProcedureInstaller.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/DataPlaneStoredProcedureInstaller.cs @@ -9,24 +9,25 @@ using System.Threading.Tasks; using EnsureThat; using Microsoft.Azure.Cosmos; -using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures { - public class StoredProcedureInstaller : ICollectionUpdater + public class DataPlaneStoredProcedureInstaller : IStoredProcedureInstaller { - private readonly IEnumerable _storedProcedures; + private readonly IEnumerable _storedProcedures; - public StoredProcedureInstaller(IEnumerable storedProcedures) + // TODO: refactor constructor to have dependency on container + public DataPlaneStoredProcedureInstaller(IEnumerable storedProcedures) { EnsureArg.IsNotNull(storedProcedures, nameof(storedProcedures)); _storedProcedures = storedProcedures; } + // TODO: refactor method to have dependency on IReadOnlyList public async Task ExecuteAsync(Container container, CancellationToken cancellationToken) { - foreach (IStoredProcedure storedProc in _storedProcedures) + foreach (IStoredProcedureMetadata storedProc in _storedProcedures) { try { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/HardDelete/HardDeleteMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/HardDelete/HardDeleteMetadata.cs new file mode 100644 index 0000000000..a6182f8755 --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/HardDelete/HardDeleteMetadata.cs @@ -0,0 +1,13 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; + +namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.HardDelete +{ + public sealed class HardDeleteMetadata : StoredProcedureMetadataBase + { + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/hardDelete.js b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/HardDelete/hardDelete.js similarity index 100% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/hardDelete.js rename to src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/HardDelete/hardDelete.js diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/IStoredProcedureInstaller.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/IStoredProcedureInstaller.cs new file mode 100644 index 0000000000..bf93cd59d3 --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/IStoredProcedureInstaller.cs @@ -0,0 +1,16 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Cosmos; + +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures +{ + public interface IStoredProcedureInstaller + { + Task ExecuteAsync(Container container, CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/IStoredProcedureMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/IStoredProcedureMetadata.cs new file mode 100644 index 0000000000..b015f24e5a --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/IStoredProcedureMetadata.cs @@ -0,0 +1,21 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Azure.Cosmos.Scripts; + +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures +{ + public interface IStoredProcedureMetadata + { + string FullName { get; } + + StoredProcedureProperties ToStoredProcedureProperties(); + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/Replace/ReplaceSingleResourceMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/Replace/ReplaceSingleResourceMetadata.cs new file mode 100644 index 0000000000..a1f6e08d59 --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/Replace/ReplaceSingleResourceMetadata.cs @@ -0,0 +1,12 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; + +namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.Replace +{ + public sealed class ReplaceSingleResourceMetadata : StoredProcedureMetadataBase + { + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/replaceSingleResource.js b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/Replace/replaceSingleResource.js similarity index 100% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/replaceSingleResource.js rename to src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/Replace/replaceSingleResource.js diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/StoredProcedureMetadataBase.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/StoredProcedureMetadataBase.cs new file mode 100644 index 0000000000..9b3f1cd6df --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/StoredProcedureMetadataBase.cs @@ -0,0 +1,76 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Azure.Cosmos; +using Microsoft.Azure.Cosmos.Scripts; +using Microsoft.Health.Core.Extensions; + +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures +{ + public class StoredProcedureMetadataBase : IStoredProcedureMetadata + { + private readonly Lazy _body; + private readonly Lazy _versionedName; + + public StoredProcedureMetadataBase() + { + _body = new Lazy(GetBody); + _versionedName = new Lazy(() => $"{Name}_{_body.Value.ComputeHash()}"); + } + + public virtual string Name => CamelCase(RemoveSuffix(GetType().Name, "Metadata")); + + public string FullName => _versionedName.Value; + + public StoredProcedureProperties ToStoredProcedureProperties() + { + return new StoredProcedureProperties + { + Id = FullName, + Body = _body.Value, + }; + } + + public Uri GetUri(Uri collection) + { + return new Uri($"{collection}/sprocs/{Uri.EscapeDataString(FullName)}", UriKind.Relative); + } + + private string GetBody() + { + // Assumed convention is the stored proc is in the same directory as the cs file + var resourceName = $"{GetType().Namespace}.{Name}.js"; + + using (Stream resourceStream = GetType().Assembly.GetManifestResourceStream(resourceName)) + using (var reader = new StreamReader(resourceStream)) + { + return reader.ReadToEnd(); + } + } + + private static string CamelCase(string str) + { + EnsureArg.IsNotEmpty(str, nameof(str)); + EnsureArg.IsTrue(str.Length > 1, nameof(str)); + + return string.Concat(char.ToLowerInvariant(str[0]), str.Substring(1)); + } + + private static string RemoveSuffix(string str, string suffix) + { + if (str.EndsWith(suffix, StringComparison.Ordinal)) + { + return str.Substring(0, str.Length - suffix.Length); + } + + return str; + } + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParametersMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParametersMetadata.cs new file mode 100644 index 0000000000..4efe888c43 --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParametersMetadata.cs @@ -0,0 +1,13 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; + +namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.UpdateUnsupportedSearchParametersToUnsupported +{ + public sealed class UpdateUnsupportedSearchParametersMetadata : StoredProcedureMetadataBase + { + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/updateUnsupportedSearchParameters.js b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/updateUnsupportedSearchParameters.js similarity index 100% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/updateUnsupportedSearchParameters.js rename to src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/updateUnsupportedSearchParameters.js diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj new file mode 100644 index 0000000000..ad9de11322 --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Health/CosmosHealthCheckTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Health/CosmosHealthCheckTests.cs index ddfab51622..fbb8101214 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Health/CosmosHealthCheckTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Health/CosmosHealthCheckTests.cs @@ -17,7 +17,8 @@ using Microsoft.Health.Abstractions.Exceptions; using Microsoft.Health.Core.Features.Health; using Microsoft.Health.Fhir.Core.Features.Health; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Features.Storage; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Health/TestCosmosHealthCheck.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Health/TestCosmosHealthCheck.cs index 3fd980d170..5622335466 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Health/TestCosmosHealthCheck.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Health/TestCosmosHealthCheck.cs @@ -7,7 +7,8 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Health.Extensions.DependencyInjection; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Features.Health; using Microsoft.Health.Fhir.CosmosDb.Features.Storage; diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosFhirDataStoreTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosFhirDataStoreTests.cs index 4da108a6e6..ecb9da05b4 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosFhirDataStoreTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosFhirDataStoreTests.cs @@ -30,7 +30,7 @@ using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Core.UnitTests.Extensions; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; using Microsoft.Health.Fhir.CosmosDb.Features.Queries; using Microsoft.Health.Fhir.CosmosDb.Features.Storage; using Microsoft.Health.Fhir.Tests.Common; diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/FhirCosmosClientInitializerTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/FhirCosmosClientInitializerTests.cs index d1f7fbc74d..8fc4f329e3 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/FhirCosmosClientInitializerTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/FhirCosmosClientInitializerTests.cs @@ -8,8 +8,10 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Health.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Context; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using NSubstitute; diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializerTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializerTests.cs index df694110f7..68fa42451a 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializerTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializerTests.cs @@ -11,7 +11,7 @@ using Microsoft.Health.Core; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Search.Registry; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; using Microsoft.Health.Fhir.CosmosDb.Features.Queries; using Microsoft.Health.Fhir.CosmosDb.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Registry; diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs index 72036938a0..aef620acf2 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs @@ -7,7 +7,9 @@ using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; using Microsoft.Health.Fhir.CosmosDb.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning; using Microsoft.Health.Fhir.Tests.Common; diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Health/CosmosHealthCheck.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Health/CosmosHealthCheck.cs index 41a3f20b54..7beeaccd71 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Health/CosmosHealthCheck.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Health/CosmosHealthCheck.cs @@ -16,7 +16,8 @@ using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Health; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Features.Storage; namespace Microsoft.Health.Fhir.CosmosDb.Features.Health diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/CosmosDbSortingValidator.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/CosmosDbSortingValidator.cs index 642be83c8b..036e968157 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/CosmosDbSortingValidator.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/CosmosDbSortingValidator.cs @@ -46,10 +46,10 @@ public bool ValidateSorting(IReadOnlyList<(SearchParameterInfo searchParameter, return true; } - errorMessages = new[] { string.Format(CultureInfo.InvariantCulture, Core.Resources.SearchSortParameterNotSupported, parameter.searchParameter.Code) }; + errorMessages = new[] { string.Format(CultureInfo.InvariantCulture, Microsoft.Health.Fhir.Core.Resources.SearchSortParameterNotSupported, parameter.searchParameter.Code) }; return false; default: - errorMessages = new[] { Core.Resources.MultiSortParameterNotSupported }; + errorMessages = new[] { Microsoft.Health.Fhir.Core.Resources.MultiSortParameterNotSupported }; return false; } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/FhirCosmosSearchService.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/FhirCosmosSearchService.cs index 992050d761..df80f9032a 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/FhirCosmosSearchService.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/FhirCosmosSearchService.cs @@ -23,7 +23,7 @@ using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Expressions; using Microsoft.Health.Fhir.Core.Models; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; using Microsoft.Health.Fhir.CosmosDb.Features.Search.Queries; using Microsoft.Health.Fhir.CosmosDb.Features.Storage; using Microsoft.Health.Fhir.ValueSets; @@ -299,7 +299,7 @@ private async Task RecurseChainedExpression(ChainedExpression expres } catch (OperationCanceledException) { - throw new RequestTooCostlyException(Core.Resources.ConditionalRequestTooCostly); + throw new RequestTooCostlyException(Microsoft.Health.Fhir.Core.Resources.ConditionalRequestTooCostly); } if (!chainedResults.Any()) @@ -551,9 +551,9 @@ private async Task ExecuteCountSearchAsync( new OperationOutcomeIssue( OperationOutcomeConstants.IssueSeverity.Error, OperationOutcomeConstants.IssueType.NotSupported, - string.Format(Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue))); + string.Format(Microsoft.Health.Fhir.Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue))); - throw new InvalidSearchOperationException(string.Format(Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue)); + throw new InvalidSearchOperationException(string.Format(Microsoft.Health.Fhir.Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue)); } return (int)count; @@ -567,7 +567,7 @@ private SearchResult CreateSearchResult(SearchOptions searchOptions, IEnumerable new OperationOutcomeIssue( OperationOutcomeConstants.IssueSeverity.Warning, OperationOutcomeConstants.IssueType.Incomplete, - Core.Resources.TruncatedIncludeMessage)); + Microsoft.Health.Fhir.Core.Resources.TruncatedIncludeMessage)); } return new SearchResult( diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/ExpressionQueryBuilder.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/ExpressionQueryBuilder.cs index beb43255fd..aa5935309d 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/ExpressionQueryBuilder.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/ExpressionQueryBuilder.cs @@ -200,7 +200,7 @@ public object VisitChained(ChainedExpression expression, Context context) public object VisitSortParameter(SortExpression expression, Context context) { - throw new SearchOperationNotSupportedException(Core.Resources.SortNotSupported); + throw new SearchOperationNotSupportedException(Microsoft.Health.Fhir.Core.Resources.SortNotSupported); } public object VisitMissingField(MissingFieldExpression expression, Context context) diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/QueryBuilder.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/QueryBuilder.cs index 08be51683a..07d7b97359 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/QueryBuilder.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/QueryBuilder.cs @@ -14,6 +14,7 @@ using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Expressions; using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Features.Queries; using Microsoft.Health.Fhir.CosmosDb.Features.Storage; diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosClientReadWriteTestProvider.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosClientReadWriteTestProvider.cs index 057501a5b9..6008d6e206 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosClientReadWriteTestProvider.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosClientReadWriteTestProvider.cs @@ -6,7 +6,8 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Newtonsoft.Json; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs index 5241a149a6..1bb98b06dd 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs @@ -10,6 +10,7 @@ using EnsureThat; using MediatR; using Microsoft.Azure.Cosmos; +using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -17,7 +18,8 @@ using Microsoft.Health.Core; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Messages.Storage; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage { @@ -25,6 +27,8 @@ namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage /// Provides an instance that is opened and whose collection has been properly initialized for use. /// Initialization starts asynchronously during application startup and is guaranteed to complete before any web request is handled by a controller. /// + /// TODO: need modify this class to call new CosmosDB .Initialization functions + /// // now some existing call are commented public class CosmosContainerProvider : IHostedService, IRequireInitializationOnFirstRequest, IDisposable { private readonly ILogger _logger; @@ -52,8 +56,10 @@ public CosmosContainerProvider( string collectionId = collectionConfiguration.Get(Constants.CollectionConfigurationName).CollectionId; _client = cosmosClientInitializer.CreateCosmosClient(cosmosDataStoreConfiguration); - _initializationOperation = new RetryableInitializationOperation( - () => cosmosClientInitializer.InitializeDataStoreAsync(_client, cosmosDataStoreConfiguration, collectionInitializers)); + _initializationOperation = new RetryableInitializationOperation(() => { return Task.CompletedTask; }); + + // _initializationOperation = new RetryableInitializationOperation( + // () => cosmosClientInitializer.InitializeDataStoreAsync(_client, cosmosDataStoreConfiguration, collectionInitializers)); _container = new Lazy(() => cosmosClientInitializer.CreateFhirContainer( _client, diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosDbCollectionPhysicalPartitionInfo.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosDbCollectionPhysicalPartitionInfo.cs index 2c7d315501..9f2876d547 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosDbCollectionPhysicalPartitionInfo.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosDbCollectionPhysicalPartitionInfo.cs @@ -18,7 +18,7 @@ using Microsoft.Health.Abstractions.Exceptions; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Extensions; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosDbDistributedLock.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosDbDistributedLock.cs index 99433011ab..ca1327b7f4 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosDbDistributedLock.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosDbDistributedLock.cs @@ -13,6 +13,7 @@ using Microsoft.Health.Abstractions.Exceptions; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Newtonsoft.Json; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosDbDistributedLockFactory.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosDbDistributedLockFactory.cs index 88de9ffefb..1a81230679 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosDbDistributedLockFactory.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosDbDistributedLockFactory.cs @@ -8,6 +8,7 @@ using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging; using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs index 08396b73b4..4733662382 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs @@ -31,7 +31,7 @@ using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Persistence.Orchestration; using Microsoft.Health.Fhir.Core.Models; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; using Microsoft.Health.Fhir.CosmosDb.Features.Queries; using Microsoft.Health.Fhir.CosmosDb.Features.Search; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.HardDelete; @@ -268,7 +268,7 @@ private async Task InternalUpsertAsync( if (cosmosWrapper.SearchIndices == null || cosmosWrapper.SearchIndices.Count == 0) { - throw new MissingSearchIndicesException(string.Format(Core.Resources.MissingSearchIndices, resource.ResourceTypeName)); + throw new MissingSearchIndicesException(string.Format(Microsoft.Health.Fhir.Core.Resources.MissingSearchIndices, resource.ResourceTypeName)); } var partitionKey = new PartitionKey(cosmosWrapper.PartitionKey); @@ -308,11 +308,11 @@ await retryPolicy.ExecuteAsync( if (_modelInfoProvider.Version == FhirSpecification.Stu3) { _logger.LogInformation("PreconditionFailed: IfMatchHeaderRequiredForResource"); - throw new PreconditionFailedException(string.Format(Core.Resources.IfMatchHeaderRequiredForResource, resource.ResourceTypeName)); + throw new PreconditionFailedException(string.Format(Microsoft.Health.Fhir.Core.Resources.IfMatchHeaderRequiredForResource, resource.ResourceTypeName)); } _logger.LogInformation("BadRequest: IfMatchHeaderRequiredForResource"); - throw new BadRequestException(string.Format(Core.Resources.IfMatchHeaderRequiredForResource, resource.ResourceTypeName)); + throw new BadRequestException(string.Format(Microsoft.Health.Fhir.Core.Resources.IfMatchHeaderRequiredForResource, resource.ResourceTypeName)); } while (true) @@ -337,13 +337,13 @@ await retryPolicy.ExecuteAsync( if (weakETag != null) { _logger.LogInformation("ResourceNotFound: ResourceNotFoundByIdAndVersion"); - throw new ResourceNotFoundException(string.Format(Core.Resources.ResourceNotFoundByIdAndVersion, resource.ResourceTypeName, resource.ResourceId, weakETag.VersionId)); + throw new ResourceNotFoundException(string.Format(Microsoft.Health.Fhir.Core.Resources.ResourceNotFoundByIdAndVersion, resource.ResourceTypeName, resource.ResourceId, weakETag.VersionId)); } if (!allowCreate) { _logger.LogInformation("MethodNotAllowed: ResourceCreationNotAllowed"); - throw new MethodNotAllowedException(Core.Resources.ResourceCreationNotAllowed); + throw new MethodNotAllowedException(Microsoft.Health.Fhir.Core.Resources.ResourceCreationNotAllowed); } throw; @@ -359,7 +359,7 @@ await retryPolicy.ExecuteAsync( } _logger.LogInformation("PreconditionFailed: ResourceVersionConflict"); - throw new PreconditionFailedException(string.Format(Core.Resources.ResourceVersionConflict, weakETag.VersionId)); + throw new PreconditionFailedException(string.Format(Microsoft.Health.Fhir.Core.Resources.ResourceVersionConflict, weakETag.VersionId)); } if (existingItemResource.IsDeleted && cosmosWrapper.IsDeleted) @@ -557,8 +557,8 @@ public async Task UpdateSearchParameterIndicesAsync(ResourceWra switch (exception.GetSubStatusCode()) { case HttpStatusCode.PreconditionFailed: - _logger.LogError(string.Format(Core.Resources.ResourceVersionConflict, WeakETag.FromVersionId(resourceWrapper.Version))); - throw new PreconditionFailedException(string.Format(Core.Resources.ResourceVersionConflict, WeakETag.FromVersionId(resourceWrapper.Version))); + _logger.LogError(string.Format(Microsoft.Health.Fhir.Core.Resources.ResourceVersionConflict, WeakETag.FromVersionId(resourceWrapper.Version))); + throw new PreconditionFailedException(string.Format(Microsoft.Health.Fhir.Core.Resources.ResourceVersionConflict, WeakETag.FromVersionId(resourceWrapper.Version))); case HttpStatusCode.ServiceUnavailable: _logger.LogError("Failed to reindex resource because the Cosmos service was unavailable."); diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosResponseProcessor.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosResponseProcessor.cs index 89561c4233..88f09e43e4 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosResponseProcessor.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosResponseProcessor.cs @@ -70,7 +70,7 @@ public async Task ProcessErrorResponseAsync(HttpStatusCode statusCode, Headers h } else if (errorMessage.Contains("Invalid Continuation Token", StringComparison.OrdinalIgnoreCase) || errorMessage.Contains("Malformed Continuation Token", StringComparison.OrdinalIgnoreCase)) { - exception = new Core.Exceptions.RequestNotValidException(Core.Resources.InvalidContinuationToken); + exception = new Microsoft.Health.Fhir.Core.Exceptions.RequestNotValidException(Microsoft.Health.Fhir.Core.Resources.InvalidContinuationToken); } else if (statusCode == HttpStatusCode.RequestEntityTooLarge || (statusCode == HttpStatusCode.BadRequest && errorMessage.Contains("Request size is too large", StringComparison.OrdinalIgnoreCase))) @@ -78,14 +78,14 @@ public async Task ProcessErrorResponseAsync(HttpStatusCode statusCode, Headers h // There are multiple known failures relating to RequestEntityTooLarge. // 1. When the document size is ~2mb (just under or at the limit) it can make it into the stored proc and fail on create // 2. Larger documents are rejected by CosmosDb with HttpStatusCode.RequestEntityTooLarge - exception = new Core.Exceptions.RequestEntityTooLargeException(); + exception = new Microsoft.Health.Fhir.Core.Exceptions.RequestEntityTooLargeException(); } else if (statusCode == HttpStatusCode.Forbidden) { int? subStatusValue = headers.GetSubStatusValue(); if (subStatusValue.HasValue && Enum.IsDefined(typeof(KnownCosmosDbCmkSubStatusValue), subStatusValue)) { - exception = new Core.Exceptions.CustomerManagedKeyException(GetCustomerManagedKeyErrorMessage(subStatusValue.Value)); + exception = new Microsoft.Health.Fhir.Core.Exceptions.CustomerManagedKeyException(GetCustomerManagedKeyErrorMessage(subStatusValue.Value)); } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs index 778eff6bcb..3da793c038 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs @@ -14,7 +14,9 @@ using Microsoft.Azure.Cosmos.Fluent; using Microsoft.Extensions.Logging; using Microsoft.Health.Abstractions.Exceptions; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage; using Microsoft.IO; using Newtonsoft.Json; using Newtonsoft.Json.Converters; @@ -111,7 +113,8 @@ await _retryExceptionPolicyFactory.RetryPolicy.ExecuteAsync(async () => } } - /// + // TODO: seperate control pane and data plane operations + /* public async Task InitializeDataStoreAsync(CosmosClient client, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, IEnumerable collectionInitializers, CancellationToken cancellationToken = default) { EnsureArg.IsNotNull(client, nameof(client)); @@ -147,7 +150,7 @@ await client.CreateDatabaseIfNotExistsAsync( _logger.Log(logLevel, ex, "Cosmos DB Database {DatabaseId} and collections initialization failed", cosmosDataStoreConfiguration.DatabaseId); throw; } - } + } */ private class FhirCosmosSerializer : CosmosSerializer { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosResourceWrapper.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosResourceWrapper.cs index 8b40d8300a..008c0628ce 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosResourceWrapper.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosResourceWrapper.cs @@ -9,6 +9,7 @@ using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Search; using Newtonsoft.Json; diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosResponseHandler.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosResponseHandler.cs index 6dad67bbc4..8af92bc601 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosResponseHandler.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosResponseHandler.cs @@ -13,7 +13,7 @@ using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Persistence; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; using Microsoft.Health.Fhir.CosmosDb.Features.Queries; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICosmosClientInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICosmosClientInitializer.cs index c7ae2075d1..1a5773ddc0 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICosmosClientInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICosmosClientInitializer.cs @@ -7,7 +7,8 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage { @@ -32,7 +33,7 @@ public interface ICosmosClientInitializer /// The collection configuration for the query to use Task OpenCosmosClient(CosmosClient client, CosmosDataStoreConfiguration configuration, CosmosCollectionConfiguration cosmosCollectionConfiguration); - /// + /* /// /// Ensures that the necessary database and collection exist with the proper indexing policy and stored procedures /// /// The instance to use for initialization. @@ -41,6 +42,7 @@ public interface ICosmosClientInitializer /// Cancellation token. /// A task Task InitializeDataStoreAsync(CosmosClient client, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, IEnumerable collectionInitializers, CancellationToken cancellationToken = default); + */ /// /// Creates a new Container instance for access the Cosmos API diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Operations/CosmosFhirOperationDataStore.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Operations/CosmosFhirOperationDataStore.cs index 59de7f62b5..d047eb66e0 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Operations/CosmosFhirOperationDataStore.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Operations/CosmosFhirOperationDataStore.cs @@ -22,7 +22,8 @@ using Microsoft.Health.Fhir.Core.Features.Operations.Export.Models; using Microsoft.Health.Fhir.Core.Features.Operations.Reindex.Models; using Microsoft.Health.Fhir.Core.Features.Persistence; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Operations.Export; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Operations.Reindex; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.AcquireExportJobs; @@ -228,7 +229,7 @@ async Task ILegacyExportOperationDataStore.GetLegacyExportJobB if (dce.StatusCode == HttpStatusCode.NotFound) { - throw new JobNotFoundException(string.Format(Core.Resources.JobNotFound, id)); + throw new JobNotFoundException(string.Format(Microsoft.Health.Fhir.Core.Resources.JobNotFound, id)); } _logger.LogError(dce, "Failed to get an export job by id."); @@ -274,7 +275,7 @@ async Task ILegacyExportOperationDataStore.UpdateLegacyExportJ } else if (dce.StatusCode == HttpStatusCode.NotFound) { - throw new JobNotFoundException(string.Format(Core.Resources.JobNotFound, jobRecord.Id)); + throw new JobNotFoundException(string.Format(Microsoft.Health.Fhir.Core.Resources.JobNotFound, jobRecord.Id)); } _logger.LogError(dce, "Failed to update an export job."); @@ -391,7 +392,7 @@ public override async Task GetReindexJobByIdAsync(string jobI } else if (dce.StatusCode == HttpStatusCode.NotFound) { - throw new JobNotFoundException(string.Format(Core.Resources.JobNotFound, jobId)); + throw new JobNotFoundException(string.Format(Microsoft.Health.Fhir.Core.Resources.JobNotFound, jobId)); } _logger.LogError(dce, "Failed to get reindex job by id: {JobId}.", jobId); @@ -437,7 +438,7 @@ public override async Task UpdateReindexJobAsync(ReindexJobRe } else if (dce.StatusCode == HttpStatusCode.NotFound) { - throw new JobNotFoundException(string.Format(Core.Resources.JobNotFound, jobRecord.Id)); + throw new JobNotFoundException(string.Format(Microsoft.Health.Fhir.Core.Resources.JobNotFound, jobRecord.Id)); } _logger.LogError(dce, "Failed to update a reindex job."); diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Operations/CosmosJobRecordWrapper.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Operations/CosmosJobRecordWrapper.cs index 074642e35c..6dfb99560f 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Operations/CosmosJobRecordWrapper.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Operations/CosmosJobRecordWrapper.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Newtonsoft.Json; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.Operations diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Operations/Export/CosmosExportJobRecordWrapper.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Operations/Export/CosmosExportJobRecordWrapper.cs index 7af53492b5..bb6faecc75 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Operations/Export/CosmosExportJobRecordWrapper.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Operations/Export/CosmosExportJobRecordWrapper.cs @@ -6,6 +6,7 @@ using EnsureThat; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Operations.Export.Models; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Newtonsoft.Json; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.Operations.Export diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Operations/Reindex/CosmosReindexJobRecordWrapper.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Operations/Reindex/CosmosReindexJobRecordWrapper.cs index 15ec768fe8..9974d468ab 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Operations/Reindex/CosmosReindexJobRecordWrapper.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Operations/Reindex/CosmosReindexJobRecordWrapper.cs @@ -6,6 +6,7 @@ using EnsureThat; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Operations.Reindex.Models; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Newtonsoft.Json; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.Operations.Reindex diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Queues/CosmosQueueClient.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Queues/CosmosQueueClient.cs index 35342f1b96..863aefb5a9 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Queues/CosmosQueueClient.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Queues/CosmosQueueClient.cs @@ -18,6 +18,7 @@ using Microsoft.Health.Core.Extensions; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Features.Queries; using Microsoft.Health.JobManagement; using Polly; diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Queues/JobGroupWrapper.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Queues/JobGroupWrapper.cs index d4527b8565..c68b63172a 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Queues/JobGroupWrapper.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Queues/JobGroupWrapper.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Newtonsoft.Json; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.Queues; diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusDataStore.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusDataStore.cs index 0f6c2eeb5a..302ccf07b3 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusDataStore.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusDataStore.cs @@ -14,7 +14,7 @@ using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Search.Registry; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.Registry { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs index c49ddcc3e7..21add8245b 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs @@ -11,7 +11,9 @@ using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Models; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.Registry diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/SearchParameterStatusWrapper.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/SearchParameterStatusWrapper.cs index 25af50dc70..615601d25f 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/SearchParameterStatusWrapper.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/SearchParameterStatusWrapper.cs @@ -7,6 +7,7 @@ using Microsoft.Health.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Newtonsoft.Json; using Newtonsoft.Json.Converters; diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/RetryExceptionPolicyFactory.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/RetryExceptionPolicyFactory.cs index ebdbfd9044..2850384550 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/RetryExceptionPolicyFactory.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/RetryExceptionPolicyFactory.cs @@ -12,7 +12,7 @@ using Microsoft.Health.Core.Features.Context; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Context; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; using Polly; using Polly.Retry; diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs index d1fea46599..d807065867 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs @@ -9,11 +9,17 @@ using EnsureThat; using Microsoft.Azure.Cosmos.Scripts; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Operations.Export; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.AcquireExportJobs { internal class AcquireExportJobs : StoredProcedureBase { + public AcquireExportJobs() + : base(new AcquireExportJobsMetadata()) + { + } + public async Task>> ExecuteAsync( Scripts client, ushort numberOfJobsToAcquire, diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs index 879250aae0..03a5fe96e6 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs @@ -14,6 +14,11 @@ namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.Acqui { internal class AcquireReindexJobs : StoredProcedureBase { + public AcquireReindexJobs() + : base(new AcquireReindexJobsMetadata()) + { + } + public async Task>> ExecuteAsync( Scripts client, ushort maximumNumberOfConcurrentJobsAllowed, diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs index 2e6a2bbe10..e24e2e1357 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs @@ -14,6 +14,11 @@ namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.HardD { internal class HardDelete : StoredProcedureBase { + public HardDelete() + : base(new HardDeleteMetadata()) + { + } + public async Task>> Execute(Scripts client, ResourceKey key, bool keepCurrentVersion, CancellationToken cancellationToken) { EnsureArg.IsNotNull(client, nameof(client)); diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/IStoredProcedure.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/IStoredProcedure.cs index 84fb9ffc41..904773a67f 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/IStoredProcedure.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/IStoredProcedure.cs @@ -2,15 +2,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- - +using System.Threading; +using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Scripts; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures { public interface IStoredProcedure { - string FullName { get; } - - StoredProcedureProperties ToStoredProcedureProperties(); + Task> ExecuteStoredProc(Scripts client, string partitionId, CancellationToken cancellationToken, params object[] parameters); } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/ReplaceSingleResource.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/ReplaceSingleResource.cs index 92ebac081b..7f9657c09d 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/ReplaceSingleResource.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/ReplaceSingleResource.cs @@ -12,6 +12,11 @@ namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.Repla { internal class ReplaceSingleResource : StoredProcedureBase { + public ReplaceSingleResource() + : base(new ReplaceSingleResourceMetadata()) + { + } + public async Task Execute(Scripts client, FhirCosmosResourceWrapper resource, string matchVersionId, CancellationToken cancellationToken) { EnsureArg.IsNotNull(client, nameof(client)); diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/StoredProcedureBase.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/StoredProcedureBase.cs index edb7ce70e3..da20f2b24c 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/StoredProcedureBase.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/StoredProcedureBase.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -11,34 +12,41 @@ using Microsoft.Azure.Cosmos; using Microsoft.Azure.Cosmos.Scripts; using Microsoft.Health.Core.Extensions; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures { public abstract class StoredProcedureBase : IStoredProcedure { - private readonly Lazy _body; - private readonly Lazy _versionedName; + // private readonly Lazy _body; + // private readonly Lazy _versionedName; + private readonly IStoredProcedureMetadata _storedProcedureMetadata; - protected StoredProcedureBase() + /* protected StoredProcedureBase() { _body = new Lazy(GetBody); _versionedName = new Lazy(() => $"{Name}_{_body.Value.ComputeHash()}"); + }*/ + + protected StoredProcedureBase(IStoredProcedureMetadata storedProcedure) + { + _storedProcedureMetadata = storedProcedure; } - protected virtual string Name => CamelCase(GetType().Name); + // protected virtual string Name => CamelCase(GetType().Name); - public string FullName => _versionedName.Value; + public string FullName => _storedProcedureMetadata.FullName; - public StoredProcedureProperties ToStoredProcedureProperties() - { - return new StoredProcedureProperties - { - Id = FullName, - Body = _body.Value, - }; - } + // public StoredProcedureProperties ToStoredProcedureProperties() + // { + // return new StoredProcedureProperties + // { + // Id = FullName, + // Body = _body.Value, + // }; + // } - protected async Task> ExecuteStoredProc(Scripts client, string partitionId, CancellationToken cancellationToken, params object[] parameters) + public async Task> ExecuteStoredProc(Scripts client, string partitionId, CancellationToken cancellationToken, params object[] parameters) { EnsureArg.IsNotNull(client, nameof(client)); EnsureArg.IsNotNull(partitionId, nameof(partitionId)); @@ -52,7 +60,7 @@ protected async Task> ExecuteStoredProc(Scr return results; } - public Uri GetUri(Uri collection) + /* public Uri GetUri(Uri collection) { return new Uri($"{collection}/sprocs/{Uri.EscapeDataString(FullName)}", UriKind.Relative); } @@ -75,6 +83,6 @@ private static string CamelCase(string str) EnsureArg.IsTrue(str.Length > 1, nameof(str)); return string.Concat(char.ToLowerInvariant(str[0]), str.Substring(1)); - } + }*/ } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs index e3a735d29c..c7c9829c0f 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs @@ -13,6 +13,11 @@ namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.Updat { internal class UpdateUnsupportedSearchParameters : StoredProcedureBase { + public UpdateUnsupportedSearchParameters() + : base(new UpdateUnsupportedSearchParametersMetadata()) + { + } + public async Task> Execute(Scripts client, CancellationToken cancellationToken) { EnsureArg.IsNotNull(client, nameof(client)); diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/FhirCollectionSettingsUpdater.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/FhirCollectionSettingsUpdater.cs index 64542c8dd2..4499c8441f 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/FhirCollectionSettingsUpdater.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/FhirCollectionSettingsUpdater.cs @@ -11,7 +11,8 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Health.Fhir.Core.Features.Persistence; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.UpdateUnsupportedSearchParametersToUnsupported; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning @@ -19,6 +20,8 @@ namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning /// /// Updates a document collection to the latest index /// + /// + // TODO: rename to dataplanecollectionupdate and needs to move to initialization project public sealed class FhirCollectionSettingsUpdater : ICollectionUpdater { private readonly ILogger _logger; @@ -75,6 +78,8 @@ public async Task ExecuteAsync(Container container, CancellationToken cancellati thisVersion.Version = CollectionSettingsVersion; await container.UpsertItemAsync(thisVersion, new PartitionKey(thisVersion.PartitionKey), cancellationToken: cancellationToken); } + + // TODO:Validate this logic else if (thisVersion.Version < CollectionSettingsVersion) { await _updateSP.Execute(container.Scripts, cancellationToken); diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.csproj b/src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.csproj index 9a2cf11ffa..0abfc3618b 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.csproj +++ b/src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.csproj @@ -1,14 +1,4 @@  - - - - - - - - - - @@ -28,6 +18,7 @@ + diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs index 548063e869..07ddaccf90 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs @@ -21,7 +21,9 @@ using Microsoft.Health.Fhir.Core.Messages.Storage; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Core.Registration; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; using Microsoft.Health.Fhir.CosmosDb.Features.Health; using Microsoft.Health.Fhir.CosmosDb.Features.Operations; using Microsoft.Health.Fhir.CosmosDb.Features.Operations.Export; @@ -35,6 +37,8 @@ using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Registry; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; using Microsoft.Health.JobManagement; using Constants = Microsoft.Health.Fhir.CosmosDb.Constants; @@ -49,6 +53,7 @@ public static class FhirServerBuilderCosmosDbRegistrationExtensions /// The FHIR server builder. /// Configure action. Defaulted to null. /// The builder. + /// TODO: FhirServerBuilderCosmosDbRegistrationExtensions commented retryExceptionPolicyFactory in constructor as we have modified CollectionInitializer code public static IFhirServerBuilder AddCosmosDb(this IFhirServerBuilder fhirServerBuilder, Action configureAction = null) { EnsureArg.IsNotNull(fhirServerBuilder, nameof(fhirServerBuilder)); @@ -159,15 +164,16 @@ private static IFhirServerBuilder AddCosmosDbPersistence(this IFhirServerBuilder cosmosCollectionConfiguration, config, upgradeManager, - retryExceptionPolicyFactory, cosmosClientTestProvider, loggerFactory.CreateLogger()); + + // retryExceptionPolicyFactory, }) .Singleton() .AsService(); - services.Add() + services.Add() .Transient() - .AsService(); + .AsService(); services.Add() .Transient() diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs index 127a5c42a8..72b32ff57b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs @@ -30,7 +30,9 @@ using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Core.UnitTests.Extensions; -using Microsoft.Health.Fhir.CosmosDb.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; using Microsoft.Health.Fhir.CosmosDb.Features.Queries; using Microsoft.Health.Fhir.CosmosDb.Features.Search; using Microsoft.Health.Fhir.CosmosDb.Features.Search.Queries; @@ -40,6 +42,8 @@ using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Registry; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; using Microsoft.Health.JobManagement; using Microsoft.Health.JobManagement.UnitTests; using NSubstitute; @@ -47,6 +51,8 @@ namespace Microsoft.Health.Fhir.Tests.Integration.Persistence { + // TODO: need modify this class to call new CosmosDB .Initialization project + // now some existing call are commented public class CosmosDbFhirStorageTestsFixture : IServiceProvider, IAsyncLifetime { private static readonly SemaphoreSlim CollectionInitializationSemaphore = new SemaphoreSlim(1, 1); @@ -93,11 +99,11 @@ public CosmosDbFhirStorageTestsFixture() public virtual async Task InitializeAsync() { - var fhirStoredProcs = typeof(IStoredProcedure).Assembly + var fhirStoredProcs = typeof(IStoredProcedureMetadata).Assembly .GetTypes() - .Where(x => !x.IsAbstract && typeof(IStoredProcedure).IsAssignableFrom(x)) + .Where(x => !x.IsAbstract && typeof(IStoredProcedureMetadata).IsAssignableFrom(x)) .ToArray() - .Select(type => (IStoredProcedure)Activator.CreateInstance(type)); + .Select(type => (IStoredProcedureMetadata)Activator.CreateInstance(type)); var optionsMonitor = Substitute.For>(); @@ -118,7 +124,9 @@ public virtual async Task InitializeAsync() var updaters = new ICollectionUpdater[] { new FhirCollectionSettingsUpdater(_cosmosDataStoreConfiguration, optionsMonitor, NullLogger.Instance), - new StoredProcedureInstaller(fhirStoredProcs), + + // new DataPlaneStoredProcedureInstaller(fhirStoredProcs), + new CosmosDbSearchParameterStatusInitializer( () => _filebasedSearchParameterStatusDataStore, new CosmosQueryFactory( @@ -139,7 +147,8 @@ public virtual async Task InitializeAsync() var retryExceptionPolicyFactory = new RetryExceptionPolicyFactory(_cosmosDataStoreConfiguration, _fhirRequestContextAccessor); var documentClientInitializer = new FhirCosmosClientInitializer(testProvider, () => new[] { handler }, retryExceptionPolicyFactory, NullLogger.Instance); _cosmosClient = documentClientInitializer.CreateCosmosClient(_cosmosDataStoreConfiguration); - var fhirCollectionInitializer = new CollectionInitializer(_cosmosCollectionConfiguration, _cosmosDataStoreConfiguration, upgradeManager, retryExceptionPolicyFactory, testProvider, NullLogger.Instance); + + // var fhirCollectionInitializer = new CollectionInitializer(_cosmosCollectionConfiguration, _cosmosDataStoreConfiguration, upgradeManager, retryExceptionPolicyFactory, testProvider, NullLogger.Instance); // Cosmos DB emulators throws errors when multiple collections are initialized concurrently. // Use the semaphore to only allow one initialization at a time. @@ -147,8 +156,8 @@ public virtual async Task InitializeAsync() try { - await documentClientInitializer.InitializeDataStoreAsync(_cosmosClient, _cosmosDataStoreConfiguration, new List { fhirCollectionInitializer }, CancellationToken.None); - _container = documentClientInitializer.CreateFhirContainer(_cosmosClient, _cosmosDataStoreConfiguration.DatabaseId, _cosmosCollectionConfiguration.CollectionId); + // await documentClientInitializer.InitializeDataStoreAsync(_cosmosClient, _cosmosDataStoreConfiguration, new List { fhirCollectionInitializer }, CancellationToken.None); + _container = documentClientInitializer.CreateFhirContainer(_cosmosClient, _cosmosDataStoreConfiguration.DatabaseId, _cosmosCollectionConfiguration.CollectionId); } finally { From 223dc0eb859858f12217b5a2f8f3b7e5163c2f45 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Mon, 11 Mar 2024 13:25:23 +0530 Subject: [PATCH 02/28] versions updated for some packages --- Directory.Packages.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index ff9fe22975..9725a3cddd 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -59,7 +59,7 @@ - + @@ -100,7 +100,7 @@ - + From c33ac2e5dde3641a7007f192c07bfde9883eabd5 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Fri, 15 Mar 2024 13:20:15 +0530 Subject: [PATCH 03/28] Moved to CosmosDB.Core -StoredProcedureBase -IStoredProcedure -IStoredProcedureMetadata -StoredProcedureMetadata Moved to CosmosDB.Initialization: -UpdateUnsupportedSearchParameters -FhirCollectionSettingsUpdater --- .../Storage/StoreProcedures/IStoredProcedure.cs | 15 +++++++++++++++ .../IStoredProcedureMetadata.cs | 2 +- .../StoreProcedures}/StoredProcedureBase.cs | 3 +-- .../StoredProcedureMetadataBase.cs | 2 +- .../CosmosDbInitializationTests.cs | 2 +- .../AcquireExportJobsMetadata.cs | 1 + .../AcquireReindexJobsMetadata.cs | 1 + .../DataPlaneStoredProcedureInstaller.cs | 1 + .../HardDelete/HardDeleteMetadata.cs | 1 + .../Replace/ReplaceSingleResourceMetadata.cs | 1 + .../UpdateUnsupportedSearchParameters.cs | 10 ++++++---- ...UpdateUnsupportedSearchParametersMetadata.cs | 3 ++- .../Versioning/FhirCollectionSettingsUpdater.cs | 10 +++++----- .../Versioning/CollectionUpgradeManagerTests.cs | 2 +- .../Features/Storage/CosmosContainerProvider.cs | 17 +++++++++++------ .../CosmosDbSearchParameterStatusInitializer.cs | 7 ++++++- .../AcquireExportJobs/AcquireExportJobs.cs | 1 + .../AcquireReindexJobs/AcquireReindexJobs.cs | 1 + .../StoredProcedures/HardDelete/HardDelete.cs | 1 + .../Replace/ReplaceSingleResource.cs | 1 + ...rverBuilderCosmosDbRegistrationExtensions.cs | 7 ++++++- .../CosmosDbFhirStorageTestsFixture.cs | 3 ++- 22 files changed, 67 insertions(+), 25 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedure.cs rename src/{Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures => Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures}/IStoredProcedureMetadata.cs (88%) rename src/{Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures => Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures}/StoredProcedureBase.cs (95%) rename src/{Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures => Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures}/StoredProcedureMetadataBase.cs (96%) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Initialization}/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs (62%) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Initialization}/Features/Storage/Versioning/FhirCollectionSettingsUpdater.cs (92%) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedure.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedure.cs new file mode 100644 index 0000000000..2dd5a0eb60 --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedure.cs @@ -0,0 +1,15 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Cosmos.Scripts; + +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures +{ + public interface IStoredProcedure + { + Task> ExecuteStoredProc(Scripts client, string partitionId, CancellationToken cancellationToken, params object[] parameters); + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/IStoredProcedureMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedureMetadata.cs similarity index 88% rename from src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/IStoredProcedureMetadata.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedureMetadata.cs index b015f24e5a..1182444973 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/IStoredProcedureMetadata.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedureMetadata.cs @@ -10,7 +10,7 @@ using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Scripts; -namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures { public interface IStoredProcedureMetadata { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/StoredProcedureBase.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs similarity index 95% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/StoredProcedureBase.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs index da20f2b24c..521ad607e8 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/StoredProcedureBase.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs @@ -12,9 +12,8 @@ using Microsoft.Azure.Cosmos; using Microsoft.Azure.Cosmos.Scripts; using Microsoft.Health.Core.Extensions; -using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures { public abstract class StoredProcedureBase : IStoredProcedure { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/StoredProcedureMetadataBase.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureMetadataBase.cs similarity index 96% rename from src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/StoredProcedureMetadataBase.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureMetadataBase.cs index 9b3f1cd6df..5e9dd3c850 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/StoredProcedureMetadataBase.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureMetadataBase.cs @@ -12,7 +12,7 @@ using Microsoft.Azure.Cosmos.Scripts; using Microsoft.Health.Core.Extensions; -namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures { public class StoredProcedureMetadataBase : IStoredProcedureMetadata { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs index 56b4277032..d0d08c089f 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs @@ -5,7 +5,7 @@ using System; using System.Linq; -using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Xunit; namespace CosmosDb.Initialization.UnitTests diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobsMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobsMetadata.cs index 7bab9c289a..c58dd8cc5b 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobsMetadata.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobsMetadata.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.AcquireExportJobs diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobsMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobsMetadata.cs index fb697e7773..3d787328d5 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobsMetadata.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobsMetadata.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.AcquireReindexJobs diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/DataPlaneStoredProcedureInstaller.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/DataPlaneStoredProcedureInstaller.cs index 40530d83ff..d699b3216b 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/DataPlaneStoredProcedureInstaller.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/DataPlaneStoredProcedureInstaller.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using EnsureThat; using Microsoft.Azure.Cosmos; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/HardDelete/HardDeleteMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/HardDelete/HardDeleteMetadata.cs index a6182f8755..c1cd550c12 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/HardDelete/HardDeleteMetadata.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/HardDelete/HardDeleteMetadata.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.HardDelete diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/Replace/ReplaceSingleResourceMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/Replace/ReplaceSingleResourceMetadata.cs index a1f6e08d59..4bd1f63d01 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/Replace/ReplaceSingleResourceMetadata.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/Replace/ReplaceSingleResourceMetadata.cs @@ -2,6 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.Replace diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs similarity index 62% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs index c7c9829c0f..0cc639ab7d 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs @@ -7,12 +7,14 @@ using System.Threading.Tasks; using EnsureThat; using Microsoft.Azure.Cosmos.Scripts; -using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Operations.Reindex; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.UpdateUnsupportedSearchParametersToUnsupported +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.UpdateUnsupportedSearchParametersToUnsupported { - internal class UpdateUnsupportedSearchParameters : StoredProcedureBase + public class UpdateUnsupportedSearchParameters : StoredProcedureBase { + private const string _searchParameterStatusPartitionKey = "__searchparameterstatus__"; + public UpdateUnsupportedSearchParameters() : base(new UpdateUnsupportedSearchParametersMetadata()) { @@ -22,7 +24,7 @@ public async Task> Execute(Scripts client { EnsureArg.IsNotNull(client, nameof(client)); - return await ExecuteStoredProc(client, CosmosDbReindexConstants.SearchParameterStatusPartitionKey, cancellationToken); + return await ExecuteStoredProc(client, _searchParameterStatusPartitionKey, cancellationToken); } } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParametersMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParametersMetadata.cs index 4efe888c43..e918199802 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParametersMetadata.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParametersMetadata.cs @@ -3,9 +3,10 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.UpdateUnsupportedSearchParametersToUnsupported +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.UpdateUnsupportedSearchParametersToUnsupported { public sealed class UpdateUnsupportedSearchParametersMetadata : StoredProcedureMetadataBase { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/FhirCollectionSettingsUpdater.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/Versioning/FhirCollectionSettingsUpdater.cs similarity index 92% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/FhirCollectionSettingsUpdater.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/Versioning/FhirCollectionSettingsUpdater.cs index 4499c8441f..4be9b43651 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Versioning/FhirCollectionSettingsUpdater.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/Versioning/FhirCollectionSettingsUpdater.cs @@ -10,12 +10,12 @@ using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.CosmosDb.Core; using Microsoft.Health.Fhir.CosmosDb.Core.Configs; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; -using Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.UpdateUnsupportedSearchParametersToUnsupported; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.UpdateUnsupportedSearchParametersToUnsupported; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.Versioning { /// /// Updates a document collection to the latest index @@ -65,7 +65,7 @@ public async Task ExecuteAsync(Container container, CancellationToken cancellati }); containerResponse.Resource.IndexingPolicy.ExcludedPaths.Clear(); - containerResponse.Resource.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath { Path = $"/{KnownResourceWrapperProperties.RawResource}/*", }); + containerResponse.Resource.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath { Path = $"/{Constants.RawResource}/*", }); containerResponse.Resource.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath { Path = $"/\"_etag\"/?", }); // Setting the DefaultTTL to -1 means that by default all documents in the collection will live forever @@ -79,7 +79,7 @@ public async Task ExecuteAsync(Container container, CancellationToken cancellati await container.UpsertItemAsync(thisVersion, new PartitionKey(thisVersion.PartitionKey), cancellationToken: cancellationToken); } - // TODO:Validate this logic + // TODO: This logic needs to move cosmosdbsearchpaarmetersinitializer else if (thisVersion.Version < CollectionSettingsVersion) { await _updateSP.Execute(container.Scripts, cancellationToken); diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs index aef620acf2..c491520ba3 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs @@ -11,7 +11,7 @@ using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; using Microsoft.Health.Fhir.CosmosDb.Features.Storage; -using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.Versioning; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using NSubstitute; diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs index 1bb98b06dd..20b49862ea 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs @@ -41,6 +41,8 @@ public CosmosContainerProvider( CosmosDataStoreConfiguration cosmosDataStoreConfiguration, IOptionsMonitor collectionConfiguration, ICosmosClientInitializer cosmosClientInitializer, + ICollectionSetup collectionSetup, + RetryExceptionPolicyFactory retryPolicyFactory, ILogger logger, IMediator mediator, IEnumerable collectionInitializers) @@ -55,16 +57,19 @@ public CosmosContainerProvider( string collectionId = collectionConfiguration.Get(Constants.CollectionConfigurationName).CollectionId; _client = cosmosClientInitializer.CreateCosmosClient(cosmosDataStoreConfiguration); - - _initializationOperation = new RetryableInitializationOperation(() => { return Task.CompletedTask; }); - - // _initializationOperation = new RetryableInitializationOperation( - // () => cosmosClientInitializer.InitializeDataStoreAsync(_client, cosmosDataStoreConfiguration, collectionInitializers)); - _container = new Lazy(() => cosmosClientInitializer.CreateFhirContainer( _client, cosmosDataStoreConfiguration.DatabaseId, collectionId)); + _initializationOperation = new RetryableInitializationOperation(async () => + { + await collectionSetup.CreateDatabaseAsync(_client, cosmosDataStoreConfiguration, retryPolicyFactory.RetryPolicy, CancellationToken.None); // We need valid cancellation token + await collectionSetup.CreateCollection(_client, collectionInitializers, cosmosDataStoreConfiguration, retryPolicyFactory.RetryPolicy, CancellationToken.None); + await collectionSetup.UpdateFhirCollectionSettings(_container.Value, CancellationToken.None); + }); + + // _initializationOperation = new RetryableInitializationOperation( + // () => cosmosClientInitializer.InitializeDataStoreAsync(_client, cosmosDataStoreConfiguration, collectionInitializers)); } public Container Container diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs index 21add8245b..6b8cc9e09a 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs @@ -14,7 +14,7 @@ using Microsoft.Health.Fhir.CosmosDb.Core.Configs; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; -using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning; +using Microsoft.Health.Fhir.CosmosDb.Features.Storage; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.Registry { @@ -72,6 +72,11 @@ public async Task ExecuteAsync(Container container, CancellationToken cancellati await transaction.ExecuteAsync(cancellationToken); } } + else + { + // TODO: new logic for update searchparameter + // check if the storeproc tobeexecuted more than once + } } } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs index d807065867..31b9630838 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using EnsureThat; using Microsoft.Azure.Cosmos.Scripts; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Operations.Export; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs index 03a5fe96e6..f0e4a8faea 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using EnsureThat; using Microsoft.Azure.Cosmos.Scripts; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Operations.Reindex; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.AcquireReindexJobs diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs index e24e2e1357..d92181e6a2 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs @@ -9,6 +9,7 @@ using EnsureThat; using Microsoft.Azure.Cosmos.Scripts; using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.HardDelete { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/ReplaceSingleResource.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/ReplaceSingleResource.cs index 7f9657c09d..0959f2113d 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/ReplaceSingleResource.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/ReplaceSingleResource.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using EnsureThat; using Microsoft.Azure.Cosmos.Scripts; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.Replace { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs index 07ddaccf90..0ee73cacdb 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs @@ -36,9 +36,9 @@ using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Queues; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Registry; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures; -using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.Versioning; using Microsoft.Health.JobManagement; using Constants = Microsoft.Health.Fhir.CosmosDb.Constants; @@ -171,6 +171,7 @@ private static IFhirServerBuilder AddCosmosDbPersistence(this IFhirServerBuilder }) .Singleton() .AsService(); + services.Add() .Transient() .AsService(); @@ -200,6 +201,10 @@ private static IFhirServerBuilder AddCosmosDbPersistence(this IFhirServerBuilder .Singleton() .AsService(); + services.Add() + .Singleton() + .AsService(); + services.Add() .Singleton() .AsSelf() diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs index 72b32ff57b..2462f3a165 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs @@ -32,6 +32,7 @@ using Microsoft.Health.Fhir.Core.UnitTests.Extensions; using Microsoft.Health.Fhir.CosmosDb.Core.Configs; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; using Microsoft.Health.Fhir.CosmosDb.Features.Queries; using Microsoft.Health.Fhir.CosmosDb.Features.Search; @@ -41,9 +42,9 @@ using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Queues; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Registry; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures; -using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Versioning; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.Versioning; using Microsoft.Health.JobManagement; using Microsoft.Health.JobManagement.UnitTests; using NSubstitute; From c12e3e9f8babbaed83736fc82cf4a826805b4158 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Wed, 20 Mar 2024 18:49:12 +0530 Subject: [PATCH 04/28] 1. Added ICollectionDataUpdater interface in cosmosdb project 2.Replaced ICollectionUpdater with ICollectionDataUpdater in CosmosDbSearchParameterStatusInitializer 3.moved else part logic from FhirCollectionSettingsUpdater to CosmosDbSearchParameterStatusInitializer 4.FhirCollectionSettingsUpdater is removed from solution --- .../FhirCollectionSettingsUpdater.cs | 105 ------------------ ...Health.Fhir.CosmosDb.Initialization.csproj | 4 + .../CollectionUpgradeManagerTests.cs | 3 +- .../Storage/CosmosContainerProvider.cs | 2 + .../Storage/ICollectionDataUpdater.cs | 16 +++ ...osmosDbSearchParameterStatusInitializer.cs | 31 +++++- ...erBuilderCosmosDbRegistrationExtensions.cs | 5 - .../CosmosDbFhirStorageTestsFixture.cs | 15 ++- 8 files changed, 58 insertions(+), 123 deletions(-) delete mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/Versioning/FhirCollectionSettingsUpdater.cs create mode 100644 src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICollectionDataUpdater.cs diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/Versioning/FhirCollectionSettingsUpdater.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/Versioning/FhirCollectionSettingsUpdater.cs deleted file mode 100644 index 4be9b43651..0000000000 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/Versioning/FhirCollectionSettingsUpdater.cs +++ /dev/null @@ -1,105 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using EnsureThat; -using Microsoft.Azure.Cosmos; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.Health.Fhir.CosmosDb.Core; -using Microsoft.Health.Fhir.CosmosDb.Core.Configs; -using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; -using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.UpdateUnsupportedSearchParametersToUnsupported; - -namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.Versioning -{ - /// - /// Updates a document collection to the latest index - /// - /// - // TODO: rename to dataplanecollectionupdate and needs to move to initialization project - public sealed class FhirCollectionSettingsUpdater : ICollectionUpdater - { - private readonly ILogger _logger; - private readonly CosmosDataStoreConfiguration _configuration; - private readonly CosmosCollectionConfiguration _collectionConfiguration; - private readonly UpdateUnsupportedSearchParameters _updateSP = new UpdateUnsupportedSearchParameters(); - - private const int CollectionSettingsVersion = 3; - - public FhirCollectionSettingsUpdater( - CosmosDataStoreConfiguration configuration, - IOptionsMonitor namedCosmosCollectionConfigurationAccessor, - ILogger logger) - { - EnsureArg.IsNotNull(configuration, nameof(configuration)); - EnsureArg.IsNotNull(namedCosmosCollectionConfigurationAccessor, nameof(namedCosmosCollectionConfigurationAccessor)); - EnsureArg.IsNotNull(logger, nameof(logger)); - - _configuration = configuration; - _collectionConfiguration = namedCosmosCollectionConfigurationAccessor.Get(Constants.CollectionConfigurationName); - _logger = logger; - } - - public async Task ExecuteAsync(Container container, CancellationToken cancellationToken) - { - EnsureArg.IsNotNull(container, nameof(container)); - - var thisVersion = await GetLatestCollectionVersion(container, cancellationToken); - - if (thisVersion.Version < 2) - { - var containerResponse = await container.ReadContainerAsync(cancellationToken: cancellationToken); - - // For more information on setting indexing policies refer to: - // https://docs.microsoft.com/en-us/azure/cosmos-db/how-to-manage-indexing-policy - // It is no longer necessary to explicitly set the kind (range/hash) - containerResponse.Resource.IndexingPolicy.IncludedPaths.Clear(); - containerResponse.Resource.IndexingPolicy.IncludedPaths.Add(new IncludedPath - { - Path = "/*", - }); - - containerResponse.Resource.IndexingPolicy.ExcludedPaths.Clear(); - containerResponse.Resource.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath { Path = $"/{Constants.RawResource}/*", }); - containerResponse.Resource.IndexingPolicy.ExcludedPaths.Add(new ExcludedPath { Path = $"/\"_etag\"/?", }); - - // Setting the DefaultTTL to -1 means that by default all documents in the collection will live forever - // but the Cosmos DB service should monitor this collection for documents that have overridden this default. - // See: https://docs.microsoft.com/en-us/azure/cosmos-db/time-to-live - containerResponse.Resource.DefaultTimeToLive = -1; - - await container.ReplaceContainerAsync(containerResponse, cancellationToken: cancellationToken); - - thisVersion.Version = CollectionSettingsVersion; - await container.UpsertItemAsync(thisVersion, new PartitionKey(thisVersion.PartitionKey), cancellationToken: cancellationToken); - } - - // TODO: This logic needs to move cosmosdbsearchpaarmetersinitializer - else if (thisVersion.Version < CollectionSettingsVersion) - { - await _updateSP.Execute(container.Scripts, cancellationToken); - thisVersion.Version = CollectionSettingsVersion; - await container.UpsertItemAsync(thisVersion, new PartitionKey(thisVersion.PartitionKey), cancellationToken: cancellationToken); - } - } - - private static async Task GetLatestCollectionVersion(Container container, CancellationToken cancellationToken) - { - FeedIterator query = container.GetItemQueryIterator( - new QueryDefinition("SELECT * FROM root r"), - requestOptions: new QueryRequestOptions - { - PartitionKey = new PartitionKey(CollectionVersion.CollectionVersionPartition), - }); - - FeedResponse result = await query.ReadNextAsync(cancellationToken); - - return result.FirstOrDefault() ?? new CollectionVersion(); - } - } -} diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj index ad9de11322..ee127b9ccf 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj @@ -16,4 +16,8 @@ + + + + diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs index c491520ba3..d32d7f5b49 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs @@ -11,7 +11,6 @@ using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; using Microsoft.Health.Fhir.CosmosDb.Features.Storage; -using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.Versioning; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using NSubstitute; @@ -64,7 +63,7 @@ public CollectionUpgradeManagerTests() collectionVersionWrappers.ReadNextAsync(Arg.Any()) .Returns(Substitute.ForPartsOf>()); - var updaters = new ICollectionUpdater[] { new FhirCollectionSettingsUpdater(_cosmosDataStoreConfiguration, optionsMonitor, NullLogger.Instance), }; + var updaters = new ICollectionUpdater[] { /*new FhirCollectionSettingsUpdater(_cosmosDataStoreConfiguration, optionsMonitor, NullLogger.Instance),*/ }; _manager = new CollectionUpgradeManager(updaters, _cosmosDataStoreConfiguration, optionsMonitor, factory, NullLogger.Instance); _containerResponse = Substitute.ForPartsOf(); diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs index 20b49862ea..f85c0201c5 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs @@ -42,6 +42,7 @@ public CosmosContainerProvider( IOptionsMonitor collectionConfiguration, ICosmosClientInitializer cosmosClientInitializer, ICollectionSetup collectionSetup, + ICollectionDataUpdater collectionDataUpdater, RetryExceptionPolicyFactory retryPolicyFactory, ILogger logger, IMediator mediator, @@ -66,6 +67,7 @@ public CosmosContainerProvider( await collectionSetup.CreateDatabaseAsync(_client, cosmosDataStoreConfiguration, retryPolicyFactory.RetryPolicy, CancellationToken.None); // We need valid cancellation token await collectionSetup.CreateCollection(_client, collectionInitializers, cosmosDataStoreConfiguration, retryPolicyFactory.RetryPolicy, CancellationToken.None); await collectionSetup.UpdateFhirCollectionSettings(_container.Value, CancellationToken.None); + await collectionDataUpdater.ExecuteAsync(_container.Value, CancellationToken.None); }); // _initializationOperation = new RetryableInitializationOperation( diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICollectionDataUpdater.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICollectionDataUpdater.cs new file mode 100644 index 0000000000..31fc3e70c6 --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICollectionDataUpdater.cs @@ -0,0 +1,16 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Cosmos; + +namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage +{ + public interface ICollectionDataUpdater + { + Task ExecuteAsync(Container container, CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs index 6b8cc9e09a..3a7d3ca85c 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs @@ -15,14 +15,19 @@ using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; using Microsoft.Health.Fhir.CosmosDb.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.UpdateUnsupportedSearchParametersToUnsupported; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.Registry { - public class CosmosDbSearchParameterStatusInitializer : ICollectionUpdater + // Note:We should keep this class as is and we need new Interface like ICollectionDataUpdater instead of ICollectionUpdater + // ICollectionDataUpdater should only be in cosmosdb project + public class CosmosDbSearchParameterStatusInitializer : ICollectionDataUpdater { private readonly ISearchParameterStatusDataStore _filebasedSearchParameterStatusDataStore; private readonly ICosmosQueryFactory _queryFactory; private readonly CosmosDataStoreConfiguration _configuration; + private readonly UpdateUnsupportedSearchParameters _updateSP = new UpdateUnsupportedSearchParameters(); + private const int CollectionSettingsVersion = 3; public CosmosDbSearchParameterStatusInitializer( FilebasedSearchParameterStatusDataStore.Resolver filebasedSearchParameterStatusDataStoreResolver, @@ -41,6 +46,8 @@ public async Task ExecuteAsync(Container container, CancellationToken cancellati { EnsureArg.IsNotNull(container, nameof(container)); + var thisVersion = await GetLatestCollectionVersion(container, cancellationToken); + // Detect if registry has been initialized var partitionKey = new PartitionKey(SearchParameterStatusWrapper.SearchParameterStatusPartitionKey); var query = _queryFactory.Create( @@ -74,9 +81,27 @@ public async Task ExecuteAsync(Container container, CancellationToken cancellati } else { - // TODO: new logic for update searchparameter - // check if the storeproc tobeexecuted more than once + // TODO: new logic for update searchparameter + await _updateSP.Execute(container.Scripts, cancellationToken); + thisVersion.Version = CollectionSettingsVersion; + await container.UpsertItemAsync(thisVersion, new PartitionKey(thisVersion.PartitionKey), cancellationToken: cancellationToken); + + // check if the storeproc tobeexecuted more than once } } + + private static async Task GetLatestCollectionVersion(Container container, CancellationToken cancellationToken) + { + FeedIterator query = container.GetItemQueryIterator( + new QueryDefinition("SELECT * FROM root r"), + requestOptions: new QueryRequestOptions + { + PartitionKey = new PartitionKey(CollectionVersion.CollectionVersionPartition), + }); + + FeedResponse result = await query.ReadNextAsync(cancellationToken); + + return result.FirstOrDefault() ?? new CollectionVersion(); + } } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs index 0ee73cacdb..bd8a5024b0 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs @@ -38,7 +38,6 @@ using Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; -using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.Versioning; using Microsoft.Health.JobManagement; using Constants = Microsoft.Health.Fhir.CosmosDb.Constants; @@ -176,10 +175,6 @@ private static IFhirServerBuilder AddCosmosDbPersistence(this IFhirServerBuilder .Transient() .AsService(); - services.Add() - .Transient() - .AsService(); - services.Add() .Transient() .AsService(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs index 2462f3a165..a5934a6bf7 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs @@ -44,7 +44,6 @@ using Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; -using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.Versioning; using Microsoft.Health.JobManagement; using Microsoft.Health.JobManagement.UnitTests; using NSubstitute; @@ -124,16 +123,16 @@ public virtual async Task InitializeAsync() var updaters = new ICollectionUpdater[] { - new FhirCollectionSettingsUpdater(_cosmosDataStoreConfiguration, optionsMonitor, NullLogger.Instance), + // new FhirCollectionSettingsUpdater(_cosmosDataStoreConfiguration, optionsMonitor, NullLogger.Instance), // new DataPlaneStoredProcedureInstaller(fhirStoredProcs), - new CosmosDbSearchParameterStatusInitializer( - () => _filebasedSearchParameterStatusDataStore, - new CosmosQueryFactory( - new CosmosResponseProcessor(_fhirRequestContextAccessor, mediator, Substitute.For(), NullLogger.Instance), - NullFhirCosmosQueryLogger.Instance), - _cosmosDataStoreConfiguration), + // new CosmosDbSearchParameterStatusInitializer( + // () => _filebasedSearchParameterStatusDataStore, + // new CosmosQueryFactory( + // new CosmosResponseProcessor(_fhirRequestContextAccessor, mediator, Substitute.For(), NullLogger.Instance), + // NullFhirCosmosQueryLogger.Instance), + // _cosmosDataStoreConfiguration), }; var dbLock = new CosmosDbDistributedLockFactory(Substitute.For>>(), NullLogger.Instance); From b296c6bd674699c977f5d37fa626189c1092bad7 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Fri, 22 Mar 2024 16:24:12 +0530 Subject: [PATCH 05/28] worked on code review comments --- .../Storage/ICollectionDataUpdater.cs | 2 +- .../StoreProcedures/IStoredProcedure.cs | 2 +- .../IStoredProcedureInstaller.cs | 2 +- .../StoreProcedures/StoredProcedureBase.cs | 46 +------------------ .../StoredProcedureMetadataBase.cs | 4 +- .../Versioning/CollectionUpgradeManager.cs | 23 +++++----- .../CosmosDbInitializationTests.cs | 11 +++-- .../Features/Storage/CollectionInitializer.cs | 4 +- .../Storage/DataPlaneCollectionSetup.cs | 17 ++----- ...tionSetup.cs => DefaultCollectionSetup.cs} | 10 ++-- .../Features/Storage/ICollectionSetup.cs | 4 +- .../DataPlaneStoredProcedureInstaller.cs | 8 ++-- .../UpdateUnsupportedSearchParameters.cs | 2 +- .../CollectionUpgradeManagerTests.cs | 14 +++++- .../Storage/CosmosContainerProvider.cs | 31 +++++++++++-- .../AcquireExportJobs/AcquireExportJobs.cs | 2 +- .../AcquireReindexJobs/AcquireReindexJobs.cs | 2 +- .../StoredProcedures/HardDelete/HardDelete.cs | 2 +- .../StoredProcedures/IStoredProcedure.cs | 15 ------ .../Replace/ReplaceSingleResource.cs | 2 +- ...erBuilderCosmosDbRegistrationExtensions.cs | 1 + .../CosmosDbFhirStorageTestsFixture.cs | 31 ++++++------- 22 files changed, 102 insertions(+), 133 deletions(-) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Core}/Features/Storage/ICollectionDataUpdater.cs (90%) rename src/{Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures => Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures}/IStoredProcedureInstaller.cs (86%) rename src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/{NullcollectionSetup.cs => DefaultCollectionSetup.cs} (64%) delete mode 100644 src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/IStoredProcedure.cs diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICollectionDataUpdater.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionDataUpdater.cs similarity index 90% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICollectionDataUpdater.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionDataUpdater.cs index 31fc3e70c6..93cc4bc43d 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICollectionDataUpdater.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionDataUpdater.cs @@ -7,7 +7,7 @@ using System.Threading.Tasks; using Microsoft.Azure.Cosmos; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage { public interface ICollectionDataUpdater { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedure.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedure.cs index 2dd5a0eb60..8c731baf36 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedure.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedure.cs @@ -10,6 +10,6 @@ namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures { public interface IStoredProcedure { - Task> ExecuteStoredProc(Scripts client, string partitionId, CancellationToken cancellationToken, params object[] parameters); + Task> ExecuteStoredProcAsync(Scripts client, string partitionId, CancellationToken cancellationToken, params object[] parameters); } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/IStoredProcedureInstaller.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedureInstaller.cs similarity index 86% rename from src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/IStoredProcedureInstaller.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedureInstaller.cs index bf93cd59d3..b44e5fb46f 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/IStoredProcedureInstaller.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedureInstaller.cs @@ -7,7 +7,7 @@ using System.Threading.Tasks; using Microsoft.Azure.Cosmos; -namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures { public interface IStoredProcedureInstaller { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs index 521ad607e8..492532fa96 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs @@ -17,35 +17,16 @@ namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures { public abstract class StoredProcedureBase : IStoredProcedure { - // private readonly Lazy _body; - // private readonly Lazy _versionedName; private readonly IStoredProcedureMetadata _storedProcedureMetadata; - /* protected StoredProcedureBase() - { - _body = new Lazy(GetBody); - _versionedName = new Lazy(() => $"{Name}_{_body.Value.ComputeHash()}"); - }*/ - protected StoredProcedureBase(IStoredProcedureMetadata storedProcedure) { _storedProcedureMetadata = storedProcedure; } - // protected virtual string Name => CamelCase(GetType().Name); - public string FullName => _storedProcedureMetadata.FullName; - // public StoredProcedureProperties ToStoredProcedureProperties() - // { - // return new StoredProcedureProperties - // { - // Id = FullName, - // Body = _body.Value, - // }; - // } - - public async Task> ExecuteStoredProc(Scripts client, string partitionId, CancellationToken cancellationToken, params object[] parameters) + public async Task> ExecuteStoredProcAsync(Scripts client, string partitionId, CancellationToken cancellationToken, params object[] parameters) { EnsureArg.IsNotNull(client, nameof(client)); EnsureArg.IsNotNull(partitionId, nameof(partitionId)); @@ -58,30 +39,5 @@ public async Task> ExecuteStoredProc(Script return results; } - - /* public Uri GetUri(Uri collection) - { - return new Uri($"{collection}/sprocs/{Uri.EscapeDataString(FullName)}", UriKind.Relative); - } - - private string GetBody() - { - // Assumed convention is the stored proc is in the same directory as the cs file - var resourceName = $"{GetType().Namespace}.{Name}.js"; - - using (Stream resourceStream = GetType().Assembly.GetManifestResourceStream(resourceName)) - using (var reader = new StreamReader(resourceStream)) - { - return reader.ReadToEnd(); - } - } - - private static string CamelCase(string str) - { - EnsureArg.IsNotEmpty(str, nameof(str)); - EnsureArg.IsTrue(str.Length > 1, nameof(str)); - - return string.Concat(char.ToLowerInvariant(str[0]), str.Substring(1)); - }*/ } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureMetadataBase.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureMetadataBase.cs index 5e9dd3c850..e6cbc94eee 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureMetadataBase.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureMetadataBase.cs @@ -14,12 +14,12 @@ namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures { - public class StoredProcedureMetadataBase : IStoredProcedureMetadata + public abstract class StoredProcedureMetadataBase : IStoredProcedureMetadata { private readonly Lazy _body; private readonly Lazy _versionedName; - public StoredProcedureMetadataBase() + protected StoredProcedureMetadataBase() { _body = new Lazy(GetBody); _versionedName = new Lazy(() => $"{Name}_{_body.Value.ComputeHash()}"); diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs index d229ab346a..0a7d07596a 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs @@ -12,31 +12,36 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning { public class CollectionUpgradeManager : IUpgradeManager { - private readonly IEnumerable _collectionUpdater; + private readonly ICollectionDataUpdater _collectionDataUpdater; + private readonly IStoredProcedureInstaller _storedProcedureInstaller; private readonly CosmosDataStoreConfiguration _configuration; private readonly CosmosCollectionConfiguration _collectionConfiguration; private readonly ICosmosDbDistributedLockFactory _lockFactory; private readonly ILogger _logger; public CollectionUpgradeManager( - IEnumerable collectionUpdater, + ICollectionDataUpdater collectionDataUpdater, + IStoredProcedureInstaller storedProcedureInstaller, CosmosDataStoreConfiguration configuration, IOptionsMonitor namedCosmosCollectionConfigurationAccessor, ICosmosDbDistributedLockFactory lockFactory, ILogger logger) { - EnsureArg.IsNotNull(collectionUpdater, nameof(collectionUpdater)); + EnsureArg.IsNotNull(collectionDataUpdater, nameof(collectionDataUpdater)); + EnsureArg.IsNotNull(storedProcedureInstaller, nameof(storedProcedureInstaller)); EnsureArg.IsNotNull(configuration, nameof(configuration)); EnsureArg.IsNotNull(namedCosmosCollectionConfigurationAccessor, nameof(namedCosmosCollectionConfigurationAccessor)); EnsureArg.IsNotNull(lockFactory, nameof(lockFactory)); EnsureArg.IsNotNull(logger, nameof(logger)); - _collectionUpdater = collectionUpdater; + _collectionDataUpdater = collectionDataUpdater; + _storedProcedureInstaller = storedProcedureInstaller; _configuration = configuration; _collectionConfiguration = GetCosmosCollectionConfiguration(namedCosmosCollectionConfigurationAccessor, Constants.CollectionConfigurationName); _lockFactory = lockFactory; @@ -59,14 +64,8 @@ public async Task SetupContainerAsync(Container container) _logger.LogDebug("Attempting to acquire upgrade lock"); await distributedLock.AcquireLock(cancellationTokenSource.Token); - - foreach (var updater in _collectionUpdater) - { - _logger.LogDebug("Running {CollectionUpdater} on {CollectionId}", updater.GetType().Name, _collectionConfiguration.CollectionId); - - await updater.ExecuteAsync(container, CancellationToken.None); - } - + await _storedProcedureInstaller.ExecuteAsync(container, CancellationToken.None); + await _collectionDataUpdater.ExecuteAsync(container, CancellationToken.None); await distributedLock.ReleaseLock(); } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs index d0d08c089f..5d5b92c04b 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs @@ -8,20 +8,21 @@ using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Xunit; -namespace CosmosDb.Initialization.UnitTests +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests { public class CosmosDbInitializationTests { [Fact] - public void Test1() + public void GivenList_ThenStoreProcedresshouldreturnValidList() { var storeProcs = new string[] { "AcquireExportJobsMetadata", "AcquireReindexJobsMetadata", "HardDeleteMetadata", "ReplaceSingleResourceMetadata", "UpdateUnsupportedSearchParametersMetadata" }; var fhirStoredProcsClasses = typeof(StoredProcedureMetadataBase).Assembly - .GetTypes().Where(x => !x.IsAbstract && typeof(StoredProcedureMetadataBase).IsAssignableFrom(x)) - .ToArray(); + .GetTypes() + .Where(x => !x.IsAbstract && typeof(StoredProcedureMetadataBase).IsAssignableFrom(x)) + .ToArray(); foreach (var storeproc in fhirStoredProcsClasses) { - Assert.Contains(storeproc.Name, storeProcs); + Assert.Contains(storeproc.Name, storeProcs); } } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs index 622c91f6a4..ba6661063c 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs @@ -61,7 +61,7 @@ public async Task InitializeCollectionAsync(CosmosClient client, Asyn _logger.LogInformation("Creating Cosmos Container if not exits: {CollectionId}", _cosmosCollectionConfiguration.CollectionId); ContainerResponse containerResponse = await retryPolicy.ExecuteAsync(async () => - await database.CreateContainerIfNotExistsAsync( + await database.CreateContainerIfNotExistsAsync( _cosmosCollectionConfiguration.CollectionId, $"/{KnownDocumentProperties.PartitionKey}", _cosmosCollectionConfiguration.InitialCollectionThroughput)); @@ -82,7 +82,7 @@ await retryPolicy.ExecuteAsync(async () => { try { - await _clientTestProvider.PerformTestAsync(existingContainer, _cosmosDataStoreConfiguration, _cosmosCollectionConfiguration, cancellationToken); + await _clientTestProvider.PerformTestAsync(existingContainer, _cosmosDataStoreConfiguration, _cosmosCollectionConfiguration, cancellationToken); } catch (CosmosException e) when (e.StatusCode == HttpStatusCode.TooManyRequests) { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs index 6e07248201..b9a6fe0728 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs @@ -37,6 +37,7 @@ public async Task CreateDatabaseAsync(CosmosClient client, CosmosDataStoreConfig { EnsureArg.IsNotNull(client, nameof(client)); EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration)); + EnsureArg.IsNotNull(retryPolicy, nameof(retryPolicy)); try { @@ -64,7 +65,7 @@ await client.CreateDatabaseIfNotExistsAsync( } } - public async Task CreateCollection(CosmosClient client, IEnumerable collectionInitializers, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) + public async Task CreateCollectionAsync(CosmosClient client, IEnumerable collectionInitializers, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) { EnsureArg.IsNotNull(client, nameof(client)); EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration)); @@ -89,11 +90,11 @@ public async Task CreateCollection(CosmosClient client, IEnumerable GetLatestCollectionVersion(Container container, CancellationToken cancellationToken) + private static async Task GetLatestCollectionVersionAsync(Container container, CancellationToken cancellationToken) { FeedIterator query = container.GetItemQueryIterator( new QueryDefinition("SELECT * FROM root r"), diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/NullcollectionSetup.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DefaultCollectionSetup.cs similarity index 64% rename from src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/NullcollectionSetup.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DefaultCollectionSetup.cs index cec6fcb96a..de60b602b9 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/NullcollectionSetup.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DefaultCollectionSetup.cs @@ -15,9 +15,13 @@ namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage { - public class NullcollectionSetup : ICollectionSetup + /// + /// This class does not execute any real initialization. + /// It's used as a placeholder to external initialization + /// + public class DefaultCollectionSetup : ICollectionSetup { - public Task CreateCollection(CosmosClient client, IEnumerable collectionInitializers, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) + public Task CreateCollectionAsync(CosmosClient client, IEnumerable collectionInitializers, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) { return Task.CompletedTask; } @@ -27,7 +31,7 @@ public Task CreateDatabaseAsync(CosmosClient client, CosmosDataStoreConfiguratio return Task.CompletedTask; } - public Task UpdateFhirCollectionSettings(Container container, CancellationToken cancellationToken) + public Task UpdateFhirCollectionSettingsAsync(Container container, CancellationToken cancellationToken) { return Task.CompletedTask; } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionSetup.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionSetup.cs index 1a7863ee42..11d48c5d18 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionSetup.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionSetup.cs @@ -19,8 +19,8 @@ public interface ICollectionSetup { public Task CreateDatabaseAsync(CosmosClient client, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken); - public Task CreateCollection(CosmosClient client, IEnumerable collectionInitializers, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default); + public Task CreateCollectionAsync(CosmosClient client, IEnumerable collectionInitializers, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default); - public Task UpdateFhirCollectionSettings(Container container, CancellationToken cancellationToken); + public Task UpdateFhirCollectionSettingsAsync(Container container, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/DataPlaneStoredProcedureInstaller.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/DataPlaneStoredProcedureInstaller.cs index d699b3216b..636ba4fab5 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/DataPlaneStoredProcedureInstaller.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/DataPlaneStoredProcedureInstaller.cs @@ -13,22 +13,22 @@ namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures { - public class DataPlaneStoredProcedureInstaller : IStoredProcedureInstaller + public sealed class DataPlaneStoredProcedureInstaller : IStoredProcedureInstaller { - private readonly IEnumerable _storedProcedures; + private readonly IEnumerable _storeProceduresMetadata; // TODO: refactor constructor to have dependency on container public DataPlaneStoredProcedureInstaller(IEnumerable storedProcedures) { EnsureArg.IsNotNull(storedProcedures, nameof(storedProcedures)); - _storedProcedures = storedProcedures; + _storeProceduresMetadata = storedProcedures; } // TODO: refactor method to have dependency on IReadOnlyList public async Task ExecuteAsync(Container container, CancellationToken cancellationToken) { - foreach (IStoredProcedureMetadata storedProc in _storedProcedures) + foreach (IStoredProcedureMetadata storedProc in _storeProceduresMetadata) { try { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs index 0cc639ab7d..4d8f6b15fb 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs @@ -24,7 +24,7 @@ public async Task> Execute(Scripts client { EnsureArg.IsNotNull(client, nameof(client)); - return await ExecuteStoredProc(client, _searchParameterStatusPartitionKey, cancellationToken); + return await ExecuteStoredProcAsync(client, _searchParameterStatusPartitionKey, cancellationToken); } } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs index d32d7f5b49..6707397c85 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs @@ -9,8 +9,10 @@ using Microsoft.Extensions.Options; using Microsoft.Health.Fhir.CosmosDb.Core.Configs; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; using Microsoft.Health.Fhir.CosmosDb.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using NSubstitute; @@ -47,7 +49,8 @@ public CollectionUpgradeManagerTests() var factory = Substitute.For(); var cosmosDbDistributedLock = Substitute.For(); var optionsMonitor = Substitute.For>(); - + var collectionDataupdater = Substitute.For(); + var storeProcedureInstaller = Substitute.For(); optionsMonitor.Get(Constants.CollectionConfigurationName).Returns(_cosmosCollectionConfiguration); factory.Create(Arg.Any(), Arg.Any()).Returns(cosmosDbDistributedLock); @@ -63,8 +66,15 @@ public CollectionUpgradeManagerTests() collectionVersionWrappers.ReadNextAsync(Arg.Any()) .Returns(Substitute.ForPartsOf>()); + collectionDataupdater.ExecuteAsync(Arg.Any(), Arg.Any()) + .Returns(Task.CompletedTask); + + storeProcedureInstaller.ExecuteAsync(Arg.Any(), Arg.Any()) + .Returns(Task.CompletedTask); + + // TODO: need to changes as per CosmosDbFhirStorageTestsFixture for CollectionUpgradeManager class var updaters = new ICollectionUpdater[] { /*new FhirCollectionSettingsUpdater(_cosmosDataStoreConfiguration, optionsMonitor, NullLogger.Instance),*/ }; - _manager = new CollectionUpgradeManager(updaters, _cosmosDataStoreConfiguration, optionsMonitor, factory, NullLogger.Instance); + _manager = new CollectionUpgradeManager(collectionDataupdater, storeProcedureInstaller, _cosmosDataStoreConfiguration, optionsMonitor, factory, NullLogger.Instance); _containerResponse = Substitute.ForPartsOf(); diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs index f85c0201c5..b920657ead 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs @@ -19,6 +19,7 @@ using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Messages.Storage; using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage @@ -27,7 +28,6 @@ namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage /// Provides an instance that is opened and whose collection has been properly initialized for use. /// Initialization starts asynchronously during application startup and is guaranteed to complete before any web request is handled by a controller. /// - /// TODO: need modify this class to call new CosmosDB .Initialization functions /// // now some existing call are commented public class CosmosContainerProvider : IHostedService, IRequireInitializationOnFirstRequest, IDisposable { @@ -64,10 +64,7 @@ public CosmosContainerProvider( collectionId)); _initializationOperation = new RetryableInitializationOperation(async () => { - await collectionSetup.CreateDatabaseAsync(_client, cosmosDataStoreConfiguration, retryPolicyFactory.RetryPolicy, CancellationToken.None); // We need valid cancellation token - await collectionSetup.CreateCollection(_client, collectionInitializers, cosmosDataStoreConfiguration, retryPolicyFactory.RetryPolicy, CancellationToken.None); - await collectionSetup.UpdateFhirCollectionSettings(_container.Value, CancellationToken.None); - await collectionDataUpdater.ExecuteAsync(_container.Value, CancellationToken.None); + await InitializeDataStoreAsync(collectionSetup, collectionDataUpdater, cosmosDataStoreConfiguration, retryPolicyFactory, collectionInitializers); }); // _initializationOperation = new RetryableInitializationOperation( @@ -89,6 +86,30 @@ public Container Container } } + private async Task InitializeDataStoreAsync( + ICollectionSetup collectionSetup, + ICollectionDataUpdater collectionDataUpdater, + CosmosDataStoreConfiguration cosmosDataStoreConfiguration, + RetryExceptionPolicyFactory retryPolicyFactory, + IEnumerable collectionInitializers) + { + try + { + _logger.LogInformation("Initializing Cosmos DB Database {DatabaseId} and collections", cosmosDataStoreConfiguration.DatabaseId); + + await collectionSetup.CreateDatabaseAsync(_client, cosmosDataStoreConfiguration, retryPolicyFactory.RetryPolicy, CancellationToken.None); // We need valid cancellation token + await collectionSetup.CreateCollectionAsync(_client, collectionInitializers, cosmosDataStoreConfiguration, retryPolicyFactory.RetryPolicy, CancellationToken.None); + await collectionSetup.UpdateFhirCollectionSettingsAsync(_container.Value, CancellationToken.None); + await collectionDataUpdater.ExecuteAsync(_container.Value, CancellationToken.None); + } + catch (Exception ex) + { + LogLevel logLevel = LogLevel.Critical; + _logger.Log(logLevel, ex, "Cosmos DB Database {DatabaseId}", cosmosDataStoreConfiguration.DatabaseId); + throw; + } + } + /// /// Starts the initialization of the document client and cosmos data store. /// diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs index 31b9630838..a02bb8b78d 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs @@ -29,7 +29,7 @@ public async Task>( + return await ExecuteStoredProcAsync>( client, CosmosDbExportConstants.ExportJobPartitionKey, cancellationToken, diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs index f0e4a8faea..23485fb86c 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs @@ -28,7 +28,7 @@ public async Task>( + return await ExecuteStoredProcAsync>( client, CosmosDbReindexConstants.ReindexJobPartitionKey, cancellationToken, diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs index d92181e6a2..7c7b3c6891 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs @@ -25,7 +25,7 @@ public async Task>> Execute(Scripts EnsureArg.IsNotNull(client, nameof(client)); EnsureArg.IsNotNull(key, nameof(key)); - return await ExecuteStoredProc>(client, key.ToPartitionKey(), cancellationToken, key.ResourceType, key.Id, keepCurrentVersion); + return await ExecuteStoredProcAsync>(client, key.ToPartitionKey(), cancellationToken, key.ResourceType, key.Id, keepCurrentVersion); } } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/IStoredProcedure.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/IStoredProcedure.cs deleted file mode 100644 index 904773a67f..0000000000 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/IStoredProcedure.cs +++ /dev/null @@ -1,15 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Azure.Cosmos.Scripts; - -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures -{ - public interface IStoredProcedure - { - Task> ExecuteStoredProc(Scripts client, string partitionId, CancellationToken cancellationToken, params object[] parameters); - } -} diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/ReplaceSingleResource.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/ReplaceSingleResource.cs index 0959f2113d..529a1a0f1b 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/ReplaceSingleResource.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/ReplaceSingleResource.cs @@ -24,7 +24,7 @@ public async Task Execute(Scripts client, FhirCosmosR EnsureArg.IsNotNull(resource, nameof(resource)); StoredProcedureExecuteResponse result = - await ExecuteStoredProc( + await ExecuteStoredProcAsync( client, resource.PartitionKey, cancellationToken, diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs index bd8a5024b0..b7e57cf62d 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs @@ -23,6 +23,7 @@ using Microsoft.Health.Fhir.Core.Registration; using Microsoft.Health.Fhir.CosmosDb.Core.Configs; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; using Microsoft.Health.Fhir.CosmosDb.Features.Health; using Microsoft.Health.Fhir.CosmosDb.Features.Operations; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs index a5934a6bf7..dba9d3d60f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs @@ -12,6 +12,7 @@ using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; +using Microsoft.Health.Core.Extensions; using Microsoft.Health.Core.Features.Context; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Configs; @@ -121,23 +122,18 @@ public virtual async Task InitializeAsync() IMediator mediator = Substitute.For(); - var updaters = new ICollectionUpdater[] - { - // new FhirCollectionSettingsUpdater(_cosmosDataStoreConfiguration, optionsMonitor, NullLogger.Instance), - - // new DataPlaneStoredProcedureInstaller(fhirStoredProcs), + IStoredProcedureInstaller storedProcedureInstallter = new DataPlaneStoredProcedureInstaller(fhirStoredProcs); - // new CosmosDbSearchParameterStatusInitializer( - // () => _filebasedSearchParameterStatusDataStore, - // new CosmosQueryFactory( - // new CosmosResponseProcessor(_fhirRequestContextAccessor, mediator, Substitute.For(), NullLogger.Instance), - // NullFhirCosmosQueryLogger.Instance), - // _cosmosDataStoreConfiguration), - }; + ICollectionDataUpdater dataCollectionUpdater = new CosmosDbSearchParameterStatusInitializer( + () => _filebasedSearchParameterStatusDataStore, + new CosmosQueryFactory( + new CosmosResponseProcessor(_fhirRequestContextAccessor, mediator, Substitute.For(), NullLogger.Instance), + NullFhirCosmosQueryLogger.Instance), + _cosmosDataStoreConfiguration); var dbLock = new CosmosDbDistributedLockFactory(Substitute.For>>(), NullLogger.Instance); - var upgradeManager = new CollectionUpgradeManager(updaters, _cosmosDataStoreConfiguration, optionsMonitor, dbLock, NullLogger.Instance); + var upgradeManager = new CollectionUpgradeManager(dataCollectionUpdater, storedProcedureInstallter, _cosmosDataStoreConfiguration, optionsMonitor, dbLock, NullLogger.Instance); ICosmosClientTestProvider testProvider = new CosmosClientReadWriteTestProvider(); var cosmosResponseProcessor = Substitute.For(); @@ -148,16 +144,19 @@ public virtual async Task InitializeAsync() var documentClientInitializer = new FhirCosmosClientInitializer(testProvider, () => new[] { handler }, retryExceptionPolicyFactory, NullLogger.Instance); _cosmosClient = documentClientInitializer.CreateCosmosClient(_cosmosDataStoreConfiguration); - // var fhirCollectionInitializer = new CollectionInitializer(_cosmosCollectionConfiguration, _cosmosDataStoreConfiguration, upgradeManager, retryExceptionPolicyFactory, testProvider, NullLogger.Instance); + var fhirCollectionInitializer = new CollectionInitializer(_cosmosCollectionConfiguration, _cosmosDataStoreConfiguration, upgradeManager, testProvider, NullLogger.Instance); // Cosmos DB emulators throws errors when multiple collections are initialized concurrently. // Use the semaphore to only allow one initialization at a time. await CollectionInitializationSemaphore.WaitAsync(); - + var datacollectionSetup = new DataPlaneCollectionSetup(NullLogger.Instance); try { + // TODO: we need to replce this logic with DataPlanecollectionsetup createDatabase method // await documentClientInitializer.InitializeDataStoreAsync(_cosmosClient, _cosmosDataStoreConfiguration, new List { fhirCollectionInitializer }, CancellationToken.None); - _container = documentClientInitializer.CreateFhirContainer(_cosmosClient, _cosmosDataStoreConfiguration.DatabaseId, _cosmosCollectionConfiguration.CollectionId); + await datacollectionSetup.CreateDatabaseAsync(_cosmosClient, _cosmosDataStoreConfiguration, retryExceptionPolicyFactory.RetryPolicy, CancellationToken.None); + await datacollectionSetup.CreateCollectionAsync(_cosmosClient, new List { fhirCollectionInitializer }, _cosmosDataStoreConfiguration, retryExceptionPolicyFactory.RetryPolicy, CancellationToken.None); + _container = documentClientInitializer.CreateFhirContainer(_cosmosClient, _cosmosDataStoreConfiguration.DatabaseId, _cosmosCollectionConfiguration.CollectionId); } finally { From d4b9649295d122782ceadfdc09a641410cd4c631 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Wed, 27 Mar 2024 14:24:43 +0530 Subject: [PATCH 06/28] worked on review comments and cleaned up code --- .../Features/Storage/DataPlaneCollectionSetup.cs | 1 + .../Versioning/CollectionUpgradeManagerTests.cs | 2 -- .../Features/Storage/CosmosContainerProvider.cs | 6 +----- .../CosmosDbSearchParameterStatusInitializer.cs | 3 --- .../Persistence/CosmosDbFhirStorageTestsFixture.cs | 10 +++------- 5 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs index b9a6fe0728..8f4868ea28 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs @@ -70,6 +70,7 @@ public async Task CreateCollectionAsync(CosmosClient client, IEnumerable(), Arg.Any()) .Returns(Task.CompletedTask); - // TODO: need to changes as per CosmosDbFhirStorageTestsFixture for CollectionUpgradeManager class - var updaters = new ICollectionUpdater[] { /*new FhirCollectionSettingsUpdater(_cosmosDataStoreConfiguration, optionsMonitor, NullLogger.Instance),*/ }; _manager = new CollectionUpgradeManager(collectionDataupdater, storeProcedureInstaller, _cosmosDataStoreConfiguration, optionsMonitor, factory, NullLogger.Instance); _containerResponse = Substitute.ForPartsOf(); diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs index b920657ead..f009829fef 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs @@ -28,7 +28,6 @@ namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage /// Provides an instance that is opened and whose collection has been properly initialized for use. /// Initialization starts asynchronously during application startup and is guaranteed to complete before any web request is handled by a controller. /// - /// // now some existing call are commented public class CosmosContainerProvider : IHostedService, IRequireInitializationOnFirstRequest, IDisposable { private readonly ILogger _logger; @@ -66,9 +65,6 @@ public CosmosContainerProvider( { await InitializeDataStoreAsync(collectionSetup, collectionDataUpdater, cosmosDataStoreConfiguration, retryPolicyFactory, collectionInitializers); }); - - // _initializationOperation = new RetryableInitializationOperation( - // () => cosmosClientInitializer.InitializeDataStoreAsync(_client, cosmosDataStoreConfiguration, collectionInitializers)); } public Container Container @@ -105,7 +101,7 @@ private async Task InitializeDataStoreAsync( catch (Exception ex) { LogLevel logLevel = LogLevel.Critical; - _logger.Log(logLevel, ex, "Cosmos DB Database {DatabaseId}", cosmosDataStoreConfiguration.DatabaseId); + _logger.Log(logLevel, ex, "Cosmos DB Database {DatabaseId} Initialization has failed.", cosmosDataStoreConfiguration.DatabaseId); throw; } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs index 3a7d3ca85c..e29910a809 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs @@ -81,12 +81,9 @@ public async Task ExecuteAsync(Container container, CancellationToken cancellati } else { - // TODO: new logic for update searchparameter await _updateSP.Execute(container.Scripts, cancellationToken); thisVersion.Version = CollectionSettingsVersion; await container.UpsertItemAsync(thisVersion, new PartitionKey(thisVersion.PartitionKey), cancellationToken: cancellationToken); - - // check if the storeproc tobeexecuted more than once } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs index dba9d3d60f..ae61e8ffce 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs @@ -52,8 +52,6 @@ namespace Microsoft.Health.Fhir.Tests.Integration.Persistence { - // TODO: need modify this class to call new CosmosDB .Initialization project - // now some existing call are commented public class CosmosDbFhirStorageTestsFixture : IServiceProvider, IAsyncLifetime { private static readonly SemaphoreSlim CollectionInitializationSemaphore = new SemaphoreSlim(1, 1); @@ -152,11 +150,9 @@ public virtual async Task InitializeAsync() var datacollectionSetup = new DataPlaneCollectionSetup(NullLogger.Instance); try { - // TODO: we need to replce this logic with DataPlanecollectionsetup createDatabase method - // await documentClientInitializer.InitializeDataStoreAsync(_cosmosClient, _cosmosDataStoreConfiguration, new List { fhirCollectionInitializer }, CancellationToken.None); - await datacollectionSetup.CreateDatabaseAsync(_cosmosClient, _cosmosDataStoreConfiguration, retryExceptionPolicyFactory.RetryPolicy, CancellationToken.None); - await datacollectionSetup.CreateCollectionAsync(_cosmosClient, new List { fhirCollectionInitializer }, _cosmosDataStoreConfiguration, retryExceptionPolicyFactory.RetryPolicy, CancellationToken.None); - _container = documentClientInitializer.CreateFhirContainer(_cosmosClient, _cosmosDataStoreConfiguration.DatabaseId, _cosmosCollectionConfiguration.CollectionId); + await datacollectionSetup.CreateDatabaseAsync(_cosmosClient, _cosmosDataStoreConfiguration, retryExceptionPolicyFactory.RetryPolicy, CancellationToken.None); + await datacollectionSetup.CreateCollectionAsync(_cosmosClient, new List { fhirCollectionInitializer }, _cosmosDataStoreConfiguration, retryExceptionPolicyFactory.RetryPolicy, CancellationToken.None); + _container = documentClientInitializer.CreateFhirContainer(_cosmosClient, _cosmosDataStoreConfiguration.DatabaseId, _cosmosCollectionConfiguration.CollectionId); } finally { From f875929dd7c3d707958fd0e83a25ac20bfa96219 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Thu, 4 Apr 2024 13:23:58 +0530 Subject: [PATCH 07/28] done with end to end flow testing with CosmosDbFhirStorageTestsFixture class --- .../AcquireExportJobs/AcquireExportJobsMetadata.cs | 2 +- .../AcquireReindexJobs/AcquireReindexJobsMetadata.cs | 2 +- .../Storage/StoredProcedures/HardDelete/HardDeleteMetadata.cs | 2 +- .../StoredProcedures/Replace/ReplaceSingleResourceMetadata.cs | 2 +- .../StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs | 1 + .../StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs | 1 + .../Features/Storage/StoredProcedures/HardDelete/HardDelete.cs | 1 + .../Storage/StoredProcedures/Replace/ReplaceSingleResource.cs | 1 + .../Persistence/CosmosDbFhirStorageTestsFixture.cs | 2 +- 9 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobsMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobsMetadata.cs index c58dd8cc5b..43663b9a7e 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobsMetadata.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobsMetadata.cs @@ -6,7 +6,7 @@ using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.AcquireExportJobs +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.AcquireExportJobs { public sealed class AcquireExportJobsMetadata : StoredProcedureMetadataBase { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobsMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobsMetadata.cs index 3d787328d5..3367ce8544 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobsMetadata.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobsMetadata.cs @@ -6,7 +6,7 @@ using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.AcquireReindexJobs +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.AcquireReindexJobs { public sealed class AcquireReindexJobsMetadata : StoredProcedureMetadataBase { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/HardDelete/HardDeleteMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/HardDelete/HardDeleteMetadata.cs index c1cd550c12..b886170034 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/HardDelete/HardDeleteMetadata.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/HardDelete/HardDeleteMetadata.cs @@ -6,7 +6,7 @@ using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.HardDelete +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.HardDelete { public sealed class HardDeleteMetadata : StoredProcedureMetadataBase { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/Replace/ReplaceSingleResourceMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/Replace/ReplaceSingleResourceMetadata.cs index 4bd1f63d01..a5616a657c 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/Replace/ReplaceSingleResourceMetadata.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/Replace/ReplaceSingleResourceMetadata.cs @@ -5,7 +5,7 @@ using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.Replace +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.Replace { public sealed class ReplaceSingleResourceMetadata : StoredProcedureMetadataBase { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs index a02bb8b78d..8e590bf48b 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireExportJobs/AcquireExportJobs.cs @@ -11,6 +11,7 @@ using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Operations.Export; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.AcquireExportJobs; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.AcquireExportJobs { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs index 23485fb86c..c6745ffba3 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/AcquireReindexJobs/AcquireReindexJobs.cs @@ -10,6 +10,7 @@ using Microsoft.Azure.Cosmos.Scripts; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Operations.Reindex; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.AcquireReindexJobs; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.AcquireReindexJobs { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs index 7c7b3c6891..1f2261c4fb 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs @@ -10,6 +10,7 @@ using Microsoft.Azure.Cosmos.Scripts; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.HardDelete; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.HardDelete { diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/ReplaceSingleResource.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/ReplaceSingleResource.cs index 529a1a0f1b..db80fd45f9 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/ReplaceSingleResource.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/Replace/ReplaceSingleResource.cs @@ -8,6 +8,7 @@ using EnsureThat; using Microsoft.Azure.Cosmos.Scripts; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.Replace; namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures.Replace { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs index ae61e8ffce..f1fbed97cc 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs @@ -98,7 +98,7 @@ public CosmosDbFhirStorageTestsFixture() public virtual async Task InitializeAsync() { - var fhirStoredProcs = typeof(IStoredProcedureMetadata).Assembly + var fhirStoredProcs = typeof(DataPlaneCollectionSetup).Assembly .GetTypes() .Where(x => !x.IsAbstract && typeof(IStoredProcedureMetadata).IsAssignableFrom(x)) .ToArray() From 012e288d60356d54525af4075ce99a12fe77d745 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Fri, 5 Apr 2024 12:24:39 +0530 Subject: [PATCH 08/28] resolved installed storeprocedure issue --- .../Storage/Versioning/ICollectionUpdater.cs | 16 ------- ...Health.Fhir.CosmosDb.Initialization.csproj | 5 +++ ...DataPlaneCosmosDbRegistrationExtensions.cs | 42 +++++++++++++++++++ ...osmosDbSearchParameterStatusInitializer.cs | 2 - ...erBuilderCosmosDbRegistrationExtensions.cs | 7 ++-- 5 files changed, 51 insertions(+), 21 deletions(-) delete mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/ICollectionUpdater.cs create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.Initialization/Registration/DataPlaneCosmosDbRegistrationExtensions.cs diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/ICollectionUpdater.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/ICollectionUpdater.cs deleted file mode 100644 index 08a3c0b980..0000000000 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/ICollectionUpdater.cs +++ /dev/null @@ -1,16 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Azure.Cosmos; - -namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning -{ - public interface ICollectionUpdater - { - Task ExecuteAsync(Container container, CancellationToken cancellationToken); - } -} diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj index ee127b9ccf..88c7942789 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj @@ -1,5 +1,10 @@  + + + + + diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Registration/DataPlaneCosmosDbRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Registration/DataPlaneCosmosDbRegistrationExtensions.cs new file mode 100644 index 0000000000..fbe0926402 --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Registration/DataPlaneCosmosDbRegistrationExtensions.cs @@ -0,0 +1,42 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Extensions.DependencyInjection; + +using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.AcquireExportJobs; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.HardDelete; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.Replace; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.UpdateUnsupportedSearchParametersToUnsupported; + +namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Registration +{ + public static class DataPlaneCosmosDbRegistrationExtensions + { + public static void AddDataPlaneDepencies(this IServiceCollection services) + { + // services.TypesInSameAssemblyAs() + // .AssignableTo() + // .Singleton() + // .AsSelf() + // .AsImplementedInterfaces() + // .AsService(); + + services.Add() + .Transient() + .AsImplementedInterfaces(); + services.Add() + .Transient() + .AsImplementedInterfaces(); + services.Add() + .Transient() + .AsImplementedInterfaces(); + services.Add() + .Transient() + .AsImplementedInterfaces(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs index e29910a809..54a36a0d0b 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializer.cs @@ -19,8 +19,6 @@ namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage.Registry { - // Note:We should keep this class as is and we need new Interface like ICollectionDataUpdater instead of ICollectionUpdater - // ICollectionDataUpdater should only be in cosmosdb project public class CosmosDbSearchParameterStatusInitializer : ICollectionDataUpdater { private readonly ISearchParameterStatusDataStore _filebasedSearchParameterStatusDataStore; diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs index b7e57cf62d..30ebd1313c 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs @@ -39,6 +39,7 @@ using Microsoft.Health.Fhir.CosmosDb.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Registration; using Microsoft.Health.JobManagement; using Constants = Microsoft.Health.Fhir.CosmosDb.Constants; @@ -166,8 +167,6 @@ private static IFhirServerBuilder AddCosmosDbPersistence(this IFhirServerBuilder upgradeManager, cosmosClientTestProvider, loggerFactory.CreateLogger()); - - // retryExceptionPolicyFactory, }) .Singleton() .AsService(); @@ -178,7 +177,7 @@ private static IFhirServerBuilder AddCosmosDbPersistence(this IFhirServerBuilder services.Add() .Transient() - .AsService(); + .AsService(); services.TypesInSameAssemblyAs() .AssignableTo() @@ -186,6 +185,8 @@ private static IFhirServerBuilder AddCosmosDbPersistence(this IFhirServerBuilder .AsSelf() .AsService(); + services.AddDataPlaneDepencies(); + services.Add() .Scoped() .AsSelf() From b6fd34fc021b1ad0b5febc721c2a300b2270ed77 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Fri, 12 Apr 2024 13:16:38 +0530 Subject: [PATCH 09/28] Unit test cases updated . DataPlaneCosmosDbRegistarionExtensions issue is resolved --- .../CosmosDbInitializationTests.cs | 50 +++++++++++++++++-- ...DataPlaneCosmosDbRegistrationExtensions.cs | 38 ++++++-------- .../Storage/FhirCosmosClientInitializer.cs | 39 --------------- 3 files changed, 60 insertions(+), 67 deletions(-) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs index 5d5b92c04b..69a58ace76 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs @@ -5,7 +5,10 @@ using System; using System.Linq; +using Microsoft.Azure.Cosmos.Scripts; +using Microsoft.Health.Core.Extensions; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; using Xunit; namespace Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests @@ -13,16 +16,53 @@ namespace Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests public class CosmosDbInitializationTests { [Fact] - public void GivenList_ThenStoreProcedresshouldreturnValidList() + public void GivenAListOfExpectedStoredProcedures_ThenAllStoredProceduresInTheAssemblyShouldMatch() { - var storeProcs = new string[] { "AcquireExportJobsMetadata", "AcquireReindexJobsMetadata", "HardDeleteMetadata", "ReplaceSingleResourceMetadata", "UpdateUnsupportedSearchParametersMetadata" }; - var fhirStoredProcsClasses = typeof(StoredProcedureMetadataBase).Assembly + string[] storeProcs = new string[] + { + "AcquireExportJobsMetadata", + "AcquireReindexJobsMetadata", + "HardDeleteMetadata", + "ReplaceSingleResourceMetadata", + "UpdateUnsupportedSearchParametersMetadata", + }; + + Type[] fhirStoredProcsClasses = typeof(DataPlaneStoredProcedureInstaller).Assembly + .GetTypes() + .Where(x => !x.IsAbstract && typeof(StoredProcedureMetadataBase).IsAssignableFrom(x)) + .ToArray(); + + Assert.NotEmpty(fhirStoredProcsClasses); + + foreach (Type storeproc in fhirStoredProcsClasses) + { + Assert.Contains(storeproc.Name, storeProcs); + } + } + + [Fact] + public void GivenAllStoredProceduresInTheAssembly_ThenAllStructureShouldBeAsExpected() + { + Type[] fhirStoredProcsClasses = typeof(DataPlaneStoredProcedureInstaller).Assembly .GetTypes() .Where(x => !x.IsAbstract && typeof(StoredProcedureMetadataBase).IsAssignableFrom(x)) .ToArray(); - foreach (var storeproc in fhirStoredProcsClasses) + + Assert.NotEmpty(fhirStoredProcsClasses); + + foreach (Type storeproc in fhirStoredProcsClasses) { - Assert.Contains(storeproc.Name, storeProcs); + StoredProcedureMetadataBase storedProcedureMetadata = storeproc.GetConstructors().First().Invoke(null) as StoredProcedureMetadataBase; + + Assert.NotNull(storedProcedureMetadata); + Assert.NotNull(storedProcedureMetadata.FullName); + + StoredProcedureProperties metadataProperties = storedProcedureMetadata.ToStoredProcedureProperties(); + Assert.NotNull(metadataProperties.Id); + Assert.False(string.IsNullOrEmpty(metadataProperties.Body)); + + string storedProcedureHash = metadataProperties.Body.ComputeHash(); + Assert.Equal($"{storedProcedureMetadata.Name}_{storedProcedureHash}", storedProcedureMetadata.FullName); } } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Registration/DataPlaneCosmosDbRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Registration/DataPlaneCosmosDbRegistrationExtensions.cs index fbe0926402..7990536eca 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Registration/DataPlaneCosmosDbRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Registration/DataPlaneCosmosDbRegistrationExtensions.cs @@ -2,15 +2,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- - +using System; +using System.Linq; +using Microsoft.Azure.Cosmos; using Microsoft.Extensions.DependencyInjection; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; -using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.AcquireExportJobs; -using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.HardDelete; -using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.Replace; -using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures.UpdateUnsupportedSearchParametersToUnsupported; namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Registration { @@ -18,25 +16,19 @@ public static class DataPlaneCosmosDbRegistrationExtensions { public static void AddDataPlaneDepencies(this IServiceCollection services) { - // services.TypesInSameAssemblyAs() - // .AssignableTo() - // .Singleton() - // .AsSelf() - // .AsImplementedInterfaces() - // .AsService(); + Type[] storedProcedureMetadataTypes = typeof(DataPlaneCosmosDbRegistrationExtensions).Assembly.GetTypes() + .Where(x => !x.IsAbstract && typeof(StoredProcedureMetadataBase).IsAssignableFrom(x)) + .Where(x => x.IsClass && !x.IsAbstract && !x.ContainsGenericParameters) + .ToArray(); - services.Add() - .Transient() - .AsImplementedInterfaces(); - services.Add() - .Transient() - .AsImplementedInterfaces(); - services.Add() - .Transient() - .AsImplementedInterfaces(); - services.Add() - .Transient() - .AsImplementedInterfaces(); + foreach (Type type in storedProcedureMetadataTypes) + { + services + .Add(type) + .Singleton() + .AsSelf() + .AsService(); + } } } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs index 3da793c038..0383d80fa5 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs @@ -113,45 +113,6 @@ await _retryExceptionPolicyFactory.RetryPolicy.ExecuteAsync(async () => } } - // TODO: seperate control pane and data plane operations - /* - public async Task InitializeDataStoreAsync(CosmosClient client, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, IEnumerable collectionInitializers, CancellationToken cancellationToken = default) - { - EnsureArg.IsNotNull(client, nameof(client)); - EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration)); - EnsureArg.IsNotNull(collectionInitializers, nameof(collectionInitializers)); - - try - { - _logger.LogInformation("Initializing Cosmos DB Database {DatabaseId} and collections", cosmosDataStoreConfiguration.DatabaseId); - - if (cosmosDataStoreConfiguration.AllowDatabaseCreation) - { - _logger.LogInformation("CreateDatabaseIfNotExists {DatabaseId}", cosmosDataStoreConfiguration.DatabaseId); - - await _retryExceptionPolicyFactory.RetryPolicy.ExecuteAsync( - async () => - await client.CreateDatabaseIfNotExistsAsync( - cosmosDataStoreConfiguration.DatabaseId, - cosmosDataStoreConfiguration.InitialDatabaseThroughput.HasValue ? ThroughputProperties.CreateManualThroughput(cosmosDataStoreConfiguration.InitialDatabaseThroughput.Value) : null, - cancellationToken: cancellationToken)); - } - - foreach (var collectionInitializer in collectionInitializers) - { - await collectionInitializer.InitializeCollectionAsync(client, cancellationToken); - } - - _logger.LogInformation("Cosmos DB Database {DatabaseId} and collections successfully initialized", cosmosDataStoreConfiguration.DatabaseId); - } - catch (Exception ex) - { - LogLevel logLevel = ex is RequestRateExceededException ? LogLevel.Warning : LogLevel.Critical; - _logger.Log(logLevel, ex, "Cosmos DB Database {DatabaseId} and collections initialization failed", cosmosDataStoreConfiguration.DatabaseId); - throw; - } - } */ - private class FhirCosmosSerializer : CosmosSerializer { private const int BlobSizeThresholdWarningInBytes = 1000000; // 1MB threshold. From 25cd8c7bb546f5af3e7a1b7f914afeb693e3c07d Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Fri, 12 Apr 2024 16:58:27 +0530 Subject: [PATCH 10/28] resolved build issue for hardDelete --- .../Features/Storage/StoredProcedures/HardDelete/HardDelete.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs index 2a5761857d..e8c7a1a5a7 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/StoredProcedures/HardDelete/HardDelete.cs @@ -26,7 +26,7 @@ public async Task> Execute(Scripts client, R EnsureArg.IsNotNull(client, nameof(client)); EnsureArg.IsNotNull(key, nameof(key)); - return await ExecuteStoredProc(client, key.ToPartitionKey(), cancellationToken, key.ResourceType, key.Id, keepCurrentVersion, allowPartialSuccess); + return await ExecuteStoredProcAsync(client, key.ToPartitionKey(), cancellationToken, key.ResourceType, key.Id, keepCurrentVersion, allowPartialSuccess); } } } From ca6ddfaa1feb0d7232c169a45024d680ddd36d6d Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Wed, 17 Apr 2024 12:08:32 +0530 Subject: [PATCH 11/28] identity package updated to latest version to solve vulnerability issue --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 704c84ce75..69f4986cdf 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -32,7 +32,7 @@ - + From 780183a7e736810f557356f4e30a73e827b7e730 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Fri, 26 Apr 2024 12:00:52 +0530 Subject: [PATCH 12/28] Solve the test failures in PR Disconnect ICollectionSetup from CosmosClient and Container. --- .../Storage/ICollectionInitializer.cs | 2 +- .../Features/Storage/ICollectionSetup.cs | 8 +-- .../Storage/ICosmosClientInitializer.cs | 3 +- .../Versioning/CollectionUpgradeManager.cs | 14 ++-- .../Storage/Versioning/IUpgradeManager.cs | 3 +- .../Features/Storage/CollectionInitializer.cs | 2 +- .../Storage/DataPlaneCollectionSetup.cs | 62 +++++++++++------- .../Storage/DefaultCollectionSetup.cs | 8 +-- .../CollectionUpgradeManagerTests.cs | 65 +++++++++++++------ .../Storage/CosmosContainerProvider.cs | 6 +- .../CosmosDbFhirStorageTestsFixture.cs | 10 +-- 11 files changed, 117 insertions(+), 66 deletions(-) rename src/{Microsoft.Health.Fhir.CosmosDb.Initialization => Microsoft.Health.Fhir.CosmosDb.Core}/Features/Storage/ICollectionInitializer.cs (90%) rename src/{Microsoft.Health.Fhir.CosmosDb.Initialization => Microsoft.Health.Fhir.CosmosDb.Core}/Features/Storage/ICollectionSetup.cs (51%) rename src/{Microsoft.Health.Fhir.CosmosDb => Microsoft.Health.Fhir.CosmosDb.Core}/Features/Storage/ICosmosClientInitializer.cs (95%) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionInitializer.cs similarity index 90% rename from src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionInitializer.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionInitializer.cs index e145290c86..db39a41123 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionInitializer.cs @@ -8,7 +8,7 @@ using Microsoft.Azure.Cosmos; using Polly; -namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage { public interface ICollectionInitializer { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionSetup.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionSetup.cs similarity index 51% rename from src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionSetup.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionSetup.cs index 11d48c5d18..aa722dad5c 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ICollectionSetup.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionSetup.cs @@ -13,14 +13,14 @@ using Microsoft.Health.Fhir.CosmosDb.Core.Configs; using Polly; -namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage { public interface ICollectionSetup { - public Task CreateDatabaseAsync(CosmosClient client, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken); + public Task CreateDatabaseAsync(AsyncPolicy retryPolicy, CancellationToken cancellationToken); - public Task CreateCollectionAsync(CosmosClient client, IEnumerable collectionInitializers, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default); + public Task CreateCollectionAsync(IEnumerable collectionInitializers, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default); - public Task UpdateFhirCollectionSettingsAsync(Container container, CancellationToken cancellationToken); + public Task UpdateFhirCollectionSettingsAsync(CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICosmosClientInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICosmosClientInitializer.cs similarity index 95% rename from src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICosmosClientInitializer.cs rename to src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICosmosClientInitializer.cs index 1a5773ddc0..2924e6bc44 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/ICosmosClientInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICosmosClientInitializer.cs @@ -8,9 +8,8 @@ using System.Threading.Tasks; using Microsoft.Azure.Cosmos; using Microsoft.Health.Fhir.CosmosDb.Core.Configs; -using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage; -namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage +namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage { /// /// Provides methods for creating a CosmosClient instance and initializing a collection. diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs index 0a7d07596a..650f57c508 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -20,6 +21,7 @@ public class CollectionUpgradeManager : IUpgradeManager { private readonly ICollectionDataUpdater _collectionDataUpdater; private readonly IStoredProcedureInstaller _storedProcedureInstaller; + private readonly ICollectionSetup _collectionSetup; private readonly CosmosDataStoreConfiguration _configuration; private readonly CosmosCollectionConfiguration _collectionConfiguration; private readonly ICosmosDbDistributedLockFactory _lockFactory; @@ -28,6 +30,7 @@ public class CollectionUpgradeManager : IUpgradeManager public CollectionUpgradeManager( ICollectionDataUpdater collectionDataUpdater, IStoredProcedureInstaller storedProcedureInstaller, + ICollectionSetup collectionSetup, CosmosDataStoreConfiguration configuration, IOptionsMonitor namedCosmosCollectionConfigurationAccessor, ICosmosDbDistributedLockFactory lockFactory, @@ -35,11 +38,12 @@ public CollectionUpgradeManager( { EnsureArg.IsNotNull(collectionDataUpdater, nameof(collectionDataUpdater)); EnsureArg.IsNotNull(storedProcedureInstaller, nameof(storedProcedureInstaller)); + EnsureArg.IsNotNull(collectionSetup, nameof(collectionSetup)); EnsureArg.IsNotNull(configuration, nameof(configuration)); EnsureArg.IsNotNull(namedCosmosCollectionConfigurationAccessor, nameof(namedCosmosCollectionConfigurationAccessor)); EnsureArg.IsNotNull(lockFactory, nameof(lockFactory)); EnsureArg.IsNotNull(logger, nameof(logger)); - + _collectionSetup = collectionSetup; _collectionDataUpdater = collectionDataUpdater; _storedProcedureInstaller = storedProcedureInstaller; _configuration = configuration; @@ -53,7 +57,8 @@ public CollectionUpgradeManager( /// public int CollectionSettingsVersion { get; } = 3; - public async Task SetupContainerAsync(Container container) + // TODO: Add a cancellation token parameter, and replace CancellationToken.None with the cancellation token. + public async Task SetupContainerAsync(Container container, CancellationToken cancellationToken) { EnsureArg.IsNotNull(container, nameof(container)); @@ -64,8 +69,9 @@ public async Task SetupContainerAsync(Container container) _logger.LogDebug("Attempting to acquire upgrade lock"); await distributedLock.AcquireLock(cancellationTokenSource.Token); - await _storedProcedureInstaller.ExecuteAsync(container, CancellationToken.None); - await _collectionDataUpdater.ExecuteAsync(container, CancellationToken.None); + await _collectionSetup.UpdateFhirCollectionSettingsAsync(cancellationToken); + await _storedProcedureInstaller.ExecuteAsync(container, cancellationToken); + await _collectionDataUpdater.ExecuteAsync(container, cancellationToken); await distributedLock.ReleaseLock(); } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/IUpgradeManager.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/IUpgradeManager.cs index 9282cb7b5e..f7cae70eb4 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/IUpgradeManager.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/IUpgradeManager.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos; @@ -10,6 +11,6 @@ namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning { public interface IUpgradeManager { - Task SetupContainerAsync(Container container); + Task SetupContainerAsync(Container container, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs index ba6661063c..e69241f097 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs @@ -92,7 +92,7 @@ await retryPolicy.ExecuteAsync(async () => } }); - await _upgradeManager.SetupContainerAsync(containerClient); + await _upgradeManager.SetupContainerAsync(containerClient, cancellationToken); return existingContainer; } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs index 8f4868ea28..7533f3e035 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs @@ -12,10 +12,12 @@ using EnsureThat; using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.Health.Abstractions.Exceptions; using Microsoft.Health.Core.Features.Context; using Microsoft.Health.Fhir.CosmosDb.Core; using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; using Polly; using Polly.Retry; @@ -25,60 +27,74 @@ namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage public class DataPlaneCollectionSetup : ICollectionSetup { private readonly ILogger _logger; + private readonly CosmosClient _client; + private Lazy _container; private const int CollectionSettingsVersion = 3; + private readonly CosmosDataStoreConfiguration _cosmosDataStoreConfiguration; - public DataPlaneCollectionSetup(ILogger logger) + // TODO: move loggerto be the last parameter. + public DataPlaneCollectionSetup(CosmosDataStoreConfiguration cosmosDataStoreConfiguration, IOptionsMonitor collectionConfiguration, ICosmosClientInitializer cosmosClientInitializer, ILogger logger) { + EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration)); + EnsureArg.IsNotNull(cosmosClientInitializer, nameof(cosmosClientInitializer)); + EnsureArg.IsNotNull(collectionConfiguration, nameof(collectionConfiguration)); EnsureArg.IsNotNull(logger, nameof(logger)); + + string collectionId = collectionConfiguration.Get(Constants.CollectionConfigurationName).CollectionId; + _cosmosDataStoreConfiguration = cosmosDataStoreConfiguration; + _client = cosmosClientInitializer.CreateCosmosClient(cosmosDataStoreConfiguration); + _container = new Lazy(() => cosmosClientInitializer.CreateFhirContainer( + _client, + cosmosDataStoreConfiguration.DatabaseId, + collectionId)); + _logger = logger; } - public async Task CreateDatabaseAsync(CosmosClient client, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) + public async Task CreateDatabaseAsync(AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) { - EnsureArg.IsNotNull(client, nameof(client)); - EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration)); EnsureArg.IsNotNull(retryPolicy, nameof(retryPolicy)); try { - _logger.LogInformation("Creating Cosmos DB Database {DatabaseId}", cosmosDataStoreConfiguration.DatabaseId); + _logger.LogInformation("Creating Cosmos DB Database {DatabaseId}", _cosmosDataStoreConfiguration.DatabaseId); - if (cosmosDataStoreConfiguration.AllowDatabaseCreation) + if (_cosmosDataStoreConfiguration.AllowDatabaseCreation) { - _logger.LogInformation("CreateDatabaseIfNotExists {DatabaseId}", cosmosDataStoreConfiguration.DatabaseId); + _logger.LogInformation("CreateDatabaseIfNotExists {DatabaseId}", _cosmosDataStoreConfiguration.DatabaseId); await retryPolicy.ExecuteAsync( async () => - await client.CreateDatabaseIfNotExistsAsync( - cosmosDataStoreConfiguration.DatabaseId, - cosmosDataStoreConfiguration.InitialDatabaseThroughput.HasValue ? ThroughputProperties.CreateManualThroughput(cosmosDataStoreConfiguration.InitialDatabaseThroughput.Value) : null, + await _client.CreateDatabaseIfNotExistsAsync( + _cosmosDataStoreConfiguration.DatabaseId, + _cosmosDataStoreConfiguration.InitialDatabaseThroughput.HasValue ? ThroughputProperties.CreateManualThroughput(_cosmosDataStoreConfiguration.InitialDatabaseThroughput.Value) : null, cancellationToken: cancellationToken)); } - _logger.LogInformation("Cosmos DB Database {DatabaseId} successfully initialized", cosmosDataStoreConfiguration.DatabaseId); + _logger.LogInformation("Cosmos DB Database {DatabaseId} successfully initialized", _cosmosDataStoreConfiguration.DatabaseId); } catch (Exception ex) { LogLevel logLevel = ex is RequestRateExceededException ? LogLevel.Warning : LogLevel.Critical; - _logger.Log(logLevel, ex, "Cosmos DB Database {DatabaseId} initialization failed", cosmosDataStoreConfiguration.DatabaseId); + _logger.Log(logLevel, ex, "Cosmos DB Database {DatabaseId} initialization failed", _cosmosDataStoreConfiguration.DatabaseId); throw; } } - public async Task CreateCollectionAsync(CosmosClient client, IEnumerable collectionInitializers, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) + public async Task CreateCollectionAsync(IEnumerable collectionInitializers, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) { - EnsureArg.IsNotNull(client, nameof(client)); - EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration)); + EnsureArg.IsNotNull(_client, nameof(_client)); + EnsureArg.IsNotNull(_cosmosDataStoreConfiguration, nameof(_cosmosDataStoreConfiguration)); EnsureArg.IsNotNull(collectionInitializers, nameof(collectionInitializers)); EnsureArg.IsNotNull(retryPolicy, nameof(retryPolicy)); try { - _logger.LogInformation("Initializing Cosmos DB Database {DatabaseId} and collections", cosmosDataStoreConfiguration.DatabaseId); + _logger.LogInformation("Initializing Cosmos DB Database {DatabaseId} and collections", _cosmosDataStoreConfiguration.DatabaseId); foreach (var collectionInitializer in collectionInitializers) { - await collectionInitializer.InitializeCollectionAsync(client, retryPolicy, cancellationToken); + await collectionInitializer.InitializeCollectionAsync(_client, retryPolicy, cancellationToken); } _logger.LogInformation("Collections successfully initialized"); @@ -91,15 +107,15 @@ public async Task CreateCollectionAsync(CosmosClient client, IEnumerable /// This class does not execute any real initialization. @@ -21,17 +21,17 @@ namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage /// public class DefaultCollectionSetup : ICollectionSetup { - public Task CreateCollectionAsync(CosmosClient client, IEnumerable collectionInitializers, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) + public Task CreateCollectionAsync(IEnumerable collectionInitializers, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) { return Task.CompletedTask; } - public Task CreateDatabaseAsync(CosmosClient client, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) + public Task CreateDatabaseAsync(AsyncPolicy retryPolicy, CancellationToken cancellationToken = default) { return Task.CompletedTask; } - public Task UpdateFhirCollectionSettingsAsync(Container container, CancellationToken cancellationToken) + public Task UpdateFhirCollectionSettingsAsync(CancellationToken cancellationToken) { return Task.CompletedTask; } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs index 84e2920d43..f88db6eec6 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs @@ -3,7 +3,9 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; using System.Threading; +using DotLiquid.Util; using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; @@ -16,6 +18,7 @@ using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using NSubstitute; +using NSubstitute.ClearExtensions; using Xunit; using Task = System.Threading.Tasks.Task; @@ -41,38 +44,56 @@ public class CollectionUpgradeManagerTests }; private readonly CollectionUpgradeManager _manager; - private readonly Container _client; + private readonly Container _container; private readonly ContainerResponse _containerResponse; public CollectionUpgradeManagerTests() { - var factory = Substitute.For(); + _container = Substitute.For(); + var cosmosDbDistributedLock = Substitute.For(); + cosmosDbDistributedLock.TryAcquireLock().Returns(true); + + var factory = Substitute.For(); + factory.Create(Arg.Any(), Arg.Any()).Returns(cosmosDbDistributedLock); + var optionsMonitor = Substitute.For>(); - var collectionDataupdater = Substitute.For(); - var storeProcedureInstaller = Substitute.For(); optionsMonitor.Get(Constants.CollectionConfigurationName).Returns(_cosmosCollectionConfiguration); - factory.Create(Arg.Any(), Arg.Any()).Returns(cosmosDbDistributedLock); - cosmosDbDistributedLock.TryAcquireLock().Returns(true); + var collectionDataUpdater = Substitute.For(); + collectionDataUpdater.ExecuteAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(true)); - _client = Substitute.For(); + var storeProcedureInstaller = Substitute.For(); + storeProcedureInstaller.ExecuteAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(true)); + + var collectionInitializer = Substitute.For(); + collectionInitializer.CreateFhirContainer(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_container); + + var dataPlaneCollectionSetup = new DataPlaneCollectionSetup( + _cosmosDataStoreConfiguration, + optionsMonitor, + collectionInitializer, + NullLogger.Instance); var collectionVersionWrappers = Substitute.ForPartsOf>(); - _client.GetItemQueryIterator(Arg.Any(), Arg.Any(), Arg.Any()) + _container.GetItemQueryIterator(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(collectionVersionWrappers); collectionVersionWrappers.ReadNextAsync(Arg.Any()) .Returns(Substitute.ForPartsOf>()); - collectionDataupdater.ExecuteAsync(Arg.Any(), Arg.Any()) - .Returns(Task.CompletedTask); - - storeProcedureInstaller.ExecuteAsync(Arg.Any(), Arg.Any()) - .Returns(Task.CompletedTask); - - _manager = new CollectionUpgradeManager(collectionDataupdater, storeProcedureInstaller, _cosmosDataStoreConfiguration, optionsMonitor, factory, NullLogger.Instance); + _manager = new CollectionUpgradeManager( + collectionDataUpdater, + storeProcedureInstaller, + dataPlaneCollectionSetup, + _cosmosDataStoreConfiguration, + optionsMonitor, + factory, + NullLogger.Instance); _containerResponse = Substitute.ForPartsOf(); @@ -84,7 +105,7 @@ public CollectionUpgradeManagerTests() }; _containerResponse.Resource.Returns(containerProperties); - _client.ReadContainerAsync(Arg.Any(), Arg.Any()) + _container.ReadContainerAsync(Arg.Any(), Arg.Any()) .Returns(_containerResponse); } @@ -93,7 +114,7 @@ public async Task GivenACollection_WhenSettingUpCollection_ThenTheCollectionInde { await UpdateCollectionAsync(); - await _client.Received(1).ReplaceContainerAsync(Arg.Any(), null, Arg.Any()); + await _container.Received(1).ReplaceContainerAsync(Arg.Any(), null, Arg.Any()); } [Fact] @@ -101,7 +122,7 @@ public async Task GivenACollection_WhenSettingUpCollection_ThenTheCollectionVers { await UpdateCollectionAsync(); - await _client.Received(1) + await _container.Received(1) .UpsertItemAsync(Arg.Is(x => x.Version == _manager.CollectionSettingsVersion), Arg.Any(), null, Arg.Any()); } @@ -115,7 +136,13 @@ public async Task GivenACollection_WhenSettingUpCollection_ThenTheCollectionTTLI private async Task UpdateCollectionAsync() { - await _manager.SetupContainerAsync(_client); + await _manager.SetupContainerAsync(_container, cancellationToken: GetCancellationToken()); + } + + private static CancellationToken GetCancellationToken() + { + CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + return tokenSource.Token; } } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs index f009829fef..46713f2ff5 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs @@ -93,9 +93,9 @@ private async Task InitializeDataStoreAsync( { _logger.LogInformation("Initializing Cosmos DB Database {DatabaseId} and collections", cosmosDataStoreConfiguration.DatabaseId); - await collectionSetup.CreateDatabaseAsync(_client, cosmosDataStoreConfiguration, retryPolicyFactory.RetryPolicy, CancellationToken.None); // We need valid cancellation token - await collectionSetup.CreateCollectionAsync(_client, collectionInitializers, cosmosDataStoreConfiguration, retryPolicyFactory.RetryPolicy, CancellationToken.None); - await collectionSetup.UpdateFhirCollectionSettingsAsync(_container.Value, CancellationToken.None); + await collectionSetup.CreateDatabaseAsync(retryPolicyFactory.RetryPolicy, CancellationToken.None); // We need valid cancellation token + await collectionSetup.CreateCollectionAsync(collectionInitializers, retryPolicyFactory.RetryPolicy, CancellationToken.None); + await collectionSetup.UpdateFhirCollectionSettingsAsync(CancellationToken.None); await collectionDataUpdater.ExecuteAsync(_container.Value, CancellationToken.None); } catch (Exception ex) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs index f1fbed97cc..11a7c3e047 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs @@ -104,6 +104,8 @@ public virtual async Task InitializeAsync() .ToArray() .Select(type => (IStoredProcedureMetadata)Activator.CreateInstance(type)); + var dataPlaneCollectionSetup = Substitute.For(); + var optionsMonitor = Substitute.For>(); optionsMonitor.Get(CosmosDb.Constants.CollectionConfigurationName).Returns(_cosmosCollectionConfiguration); @@ -131,7 +133,7 @@ public virtual async Task InitializeAsync() var dbLock = new CosmosDbDistributedLockFactory(Substitute.For>>(), NullLogger.Instance); - var upgradeManager = new CollectionUpgradeManager(dataCollectionUpdater, storedProcedureInstallter, _cosmosDataStoreConfiguration, optionsMonitor, dbLock, NullLogger.Instance); + var upgradeManager = new CollectionUpgradeManager(dataCollectionUpdater, storedProcedureInstallter, dataPlaneCollectionSetup, _cosmosDataStoreConfiguration, optionsMonitor, dbLock, NullLogger.Instance); ICosmosClientTestProvider testProvider = new CosmosClientReadWriteTestProvider(); var cosmosResponseProcessor = Substitute.For(); @@ -147,11 +149,11 @@ public virtual async Task InitializeAsync() // Cosmos DB emulators throws errors when multiple collections are initialized concurrently. // Use the semaphore to only allow one initialization at a time. await CollectionInitializationSemaphore.WaitAsync(); - var datacollectionSetup = new DataPlaneCollectionSetup(NullLogger.Instance); + var datacollectionSetup = new DataPlaneCollectionSetup(_cosmosDataStoreConfiguration, optionsMonitor, documentClientInitializer, NullLogger.Instance); try { - await datacollectionSetup.CreateDatabaseAsync(_cosmosClient, _cosmosDataStoreConfiguration, retryExceptionPolicyFactory.RetryPolicy, CancellationToken.None); - await datacollectionSetup.CreateCollectionAsync(_cosmosClient, new List { fhirCollectionInitializer }, _cosmosDataStoreConfiguration, retryExceptionPolicyFactory.RetryPolicy, CancellationToken.None); + await datacollectionSetup.CreateDatabaseAsync(retryExceptionPolicyFactory.RetryPolicy, CancellationToken.None); + await datacollectionSetup.CreateCollectionAsync(new List { fhirCollectionInitializer }, retryExceptionPolicyFactory.RetryPolicy, CancellationToken.None); _container = documentClientInitializer.CreateFhirContainer(_cosmosClient, _cosmosDataStoreConfiguration.DatabaseId, _cosmosCollectionConfiguration.CollectionId); } finally From 15a047d9de88a9d6aeb8fa0cbe084f41acdb1c15 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Mon, 29 Apr 2024 19:33:43 +0530 Subject: [PATCH 13/28] fixed review comments --- .../Features/Storage/ICollectionSetup.cs | 4 ---- .../Features/Storage/ICosmosClientInitializer.cs | 11 ----------- .../Storage/StoreProcedures/StoredProcedureBase.cs | 2 -- .../Features/Storage/CollectionInitializer.cs | 11 ++++------- .../Features/Storage/DataPlaneCollectionSetup.cs | 2 +- .../Features/Storage/DefaultCollectionSetup.cs | 4 ---- .../UpdateUnsupportedSearchParameters.cs | 2 +- .../DataPlaneCosmosDbRegistrationExtensions.cs | 2 +- .../Versioning/CollectionUpgradeManagerTests.cs | 4 ++-- .../Features/Storage/CosmosContainerProvider.cs | 2 ++ ...FhirServerBuilderCosmosDbRegistrationExtensions.cs | 11 +++++------ 11 files changed, 16 insertions(+), 39 deletions(-) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionSetup.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionSetup.cs index aa722dad5c..9c45a2244e 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionSetup.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionSetup.cs @@ -5,12 +5,8 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; -using Microsoft.Azure.Cosmos; -using Microsoft.Health.Fhir.CosmosDb.Core.Configs; using Polly; namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICosmosClientInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICosmosClientInitializer.cs index 2924e6bc44..f5e2d179c5 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICosmosClientInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICosmosClientInitializer.cs @@ -32,17 +32,6 @@ public interface ICosmosClientInitializer /// The collection configuration for the query to use Task OpenCosmosClient(CosmosClient client, CosmosDataStoreConfiguration configuration, CosmosCollectionConfiguration cosmosCollectionConfiguration); - /* /// - /// Ensures that the necessary database and collection exist with the proper indexing policy and stored procedures - /// - /// The instance to use for initialization. - /// The data store configuration. - /// The collection of collection initializers. - /// Cancellation token. - /// A task - Task InitializeDataStoreAsync(CosmosClient client, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, IEnumerable collectionInitializers, CancellationToken cancellationToken = default); - */ - /// /// Creates a new Container instance for access the Cosmos API /// diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs index 492532fa96..f71090f43e 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs @@ -4,8 +4,6 @@ // ------------------------------------------------------------------------------------------------- using System; -using System.Collections.Generic; -using System.IO; using System.Threading; using System.Threading.Tasks; using EnsureThat; diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs index e69241f097..36ed37f64d 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs @@ -22,7 +22,6 @@ public class CollectionInitializer : ICollectionInitializer private readonly CosmosCollectionConfiguration _cosmosCollectionConfiguration; private readonly CosmosDataStoreConfiguration _cosmosDataStoreConfiguration; private readonly IUpgradeManager _upgradeManager; - private readonly ICosmosClientTestProvider _clientTestProvider; private readonly ILogger _logger; @@ -35,7 +34,6 @@ public CollectionInitializer( { EnsureArg.IsNotNull(cosmosCollectionConfiguration, nameof(cosmosCollectionConfiguration)); EnsureArg.IsNotNull(cosmosCollectionConfiguration.CollectionId, nameof(CosmosCollectionConfiguration.CollectionId)); - EnsureArg.IsNotNull(clientTestProvider, nameof(clientTestProvider)); EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration)); EnsureArg.IsNotNull(upgradeManager, nameof(upgradeManager)); @@ -44,7 +42,6 @@ public CollectionInitializer( _cosmosCollectionConfiguration = cosmosCollectionConfiguration; _cosmosDataStoreConfiguration = cosmosDataStoreConfiguration; _upgradeManager = upgradeManager; - _clientTestProvider = clientTestProvider; _logger = logger; } @@ -61,10 +58,10 @@ public async Task InitializeCollectionAsync(CosmosClient client, Asyn _logger.LogInformation("Creating Cosmos Container if not exits: {CollectionId}", _cosmosCollectionConfiguration.CollectionId); ContainerResponse containerResponse = await retryPolicy.ExecuteAsync(async () => - await database.CreateContainerIfNotExistsAsync( - _cosmosCollectionConfiguration.CollectionId, - $"/{KnownDocumentProperties.PartitionKey}", - _cosmosCollectionConfiguration.InitialCollectionThroughput)); + await database.CreateContainerIfNotExistsAsync( + _cosmosCollectionConfiguration.CollectionId, + $"/{KnownDocumentProperties.PartitionKey}", + _cosmosCollectionConfiguration.InitialCollectionThroughput)); if (containerResponse.StatusCode == HttpStatusCode.Created || containerResponse.Resource.DefaultTimeToLive != -1) { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs index 7533f3e035..e0c8ee2832 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs @@ -28,7 +28,7 @@ public class DataPlaneCollectionSetup : ICollectionSetup { private readonly ILogger _logger; private readonly CosmosClient _client; - private Lazy _container; + private readonly Lazy _container; private const int CollectionSettingsVersion = 3; private readonly CosmosDataStoreConfiguration _cosmosDataStoreConfiguration; diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DefaultCollectionSetup.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DefaultCollectionSetup.cs index 71763cf12d..d74ae3edf9 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DefaultCollectionSetup.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DefaultCollectionSetup.cs @@ -5,12 +5,8 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; -using Microsoft.Azure.Cosmos; -using Microsoft.Health.Fhir.CosmosDb.Core.Configs; using Polly; namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs index 4d8f6b15fb..b0329acfb7 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/UpdateUnsupportedSearchParametersToUnsupported/UpdateUnsupportedSearchParameters.cs @@ -16,7 +16,7 @@ public class UpdateUnsupportedSearchParameters : StoredProcedureBase private const string _searchParameterStatusPartitionKey = "__searchparameterstatus__"; public UpdateUnsupportedSearchParameters() - : base(new UpdateUnsupportedSearchParametersMetadata()) + : base(new UpdateUnsupportedSearchParametersMetadata()) { } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Registration/DataPlaneCosmosDbRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Registration/DataPlaneCosmosDbRegistrationExtensions.cs index 7990536eca..1e2d9eea57 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Registration/DataPlaneCosmosDbRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Registration/DataPlaneCosmosDbRegistrationExtensions.cs @@ -14,7 +14,7 @@ namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Registration { public static class DataPlaneCosmosDbRegistrationExtensions { - public static void AddDataPlaneDepencies(this IServiceCollection services) + public static void AddCosmosDbInitializationDependencies(this IServiceCollection services) { Type[] storedProcedureMetadataTypes = typeof(DataPlaneCosmosDbRegistrationExtensions).Assembly.GetTypes() .Where(x => !x.IsAbstract && typeof(StoredProcedureMetadataBase).IsAssignableFrom(x)) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs index f88db6eec6..e16b56714e 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs @@ -141,8 +141,8 @@ private async Task UpdateCollectionAsync() private static CancellationToken GetCancellationToken() { - CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); - return tokenSource.Token; + using CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + return tokenSource.Token; } } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs index 46713f2ff5..fb8ec74687 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs @@ -52,6 +52,8 @@ public CosmosContainerProvider( EnsureArg.IsNotNull(cosmosClientInitializer, nameof(cosmosClientInitializer)); EnsureArg.IsNotNull(logger, nameof(logger)); EnsureArg.IsNotNull(collectionInitializers, nameof(collectionInitializers)); + EnsureArg.IsNotNull(collectionSetup, nameof(collectionSetup)); + EnsureArg.IsNotNull(collectionDataUpdater, nameof(collectionDataUpdater)); _logger = logger; _mediator = mediator; diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs index 30ebd1313c..bec843d067 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Registration/FhirServerBuilderCosmosDbRegistrationExtensions.cs @@ -54,7 +54,6 @@ public static class FhirServerBuilderCosmosDbRegistrationExtensions /// The FHIR server builder. /// Configure action. Defaulted to null. /// The builder. - /// TODO: FhirServerBuilderCosmosDbRegistrationExtensions commented retryExceptionPolicyFactory in constructor as we have modified CollectionInitializer code public static IFhirServerBuilder AddCosmosDb(this IFhirServerBuilder fhirServerBuilder, Action configureAction = null) { EnsureArg.IsNotNull(fhirServerBuilder, nameof(fhirServerBuilder)); @@ -179,13 +178,17 @@ private static IFhirServerBuilder AddCosmosDbPersistence(this IFhirServerBuilder .Transient() .AsService(); + services.Add() + .Singleton() + .AsService(); + services.TypesInSameAssemblyAs() .AssignableTo() .Singleton() .AsSelf() .AsService(); - services.AddDataPlaneDepencies(); + services.AddCosmosDbInitializationDependencies(); services.Add() .Scoped() @@ -198,10 +201,6 @@ private static IFhirServerBuilder AddCosmosDbPersistence(this IFhirServerBuilder .Singleton() .AsService(); - services.Add() - .Singleton() - .AsService(); - services.Add() .Singleton() .AsSelf() From 642310ee5604e33d9bae1f453400c439d9e26f70 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Tue, 30 Apr 2024 21:59:50 +0530 Subject: [PATCH 14/28] solved issues with unit test --- ...DbSearchParameterStatusInitializerTests.cs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializerTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializerTests.cs index 68fa42451a..73761adc29 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializerTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Registry/CosmosDbSearchParameterStatusInitializerTests.cs @@ -7,11 +7,13 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Microsoft.AspNetCore.Http.Features; using Microsoft.Azure.Cosmos; using Microsoft.Health.Core; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; using Microsoft.Health.Fhir.CosmosDb.Features.Queries; using Microsoft.Health.Fhir.CosmosDb.Features.Storage; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Registry; @@ -66,7 +68,19 @@ public async Task GivenARegistryInitializer_WhenDatabaseIsNew_SearchParametersSh .ExecuteNextAsync() .Returns(Substitute.ForPartsOf>()); + var feedResponse = Substitute.For>(); + + var feedIterator = Substitute.For>(); + feedIterator.ReadNextAsync(Arg.Any()) + .Returns(feedResponse); + Container container = Substitute.For(); + container + .GetItemQueryIterator( + queryDefinition: Arg.Any(), + continuationToken: Arg.Any(), + requestOptions: Arg.Any()) + .Returns(feedIterator); await _initializer.ExecuteAsync(container, CancellationToken.None); @@ -90,7 +104,19 @@ public async Task GivenARegistryInitializer_WhenDatabaseIsExisting_NothingNeedsT .ExecuteNextAsync() .Returns(info => response); + var feedResponse = Substitute.For>(); + + var feedIterator = Substitute.For>(); + feedIterator.ReadNextAsync(Arg.Any()) + .Returns(feedResponse); + Container container = Substitute.For(); + container + .GetItemQueryIterator( + queryDefinition: Arg.Any(), + continuationToken: Arg.Any(), + requestOptions: Arg.Any()) + .Returns(feedIterator); await _initializer.ExecuteAsync(container, CancellationToken.None); From 7698d560e6313dc13d334ca37e65af42c897b084 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Fri, 3 May 2024 11:22:44 +0530 Subject: [PATCH 15/28] Added new project refrences to docker file --- build/docker/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index b51fc0a65d..5fcf8e813d 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -43,6 +43,12 @@ COPY ./src/Microsoft.Health.Fhir.Api/Microsoft.Health.Fhir.Api.csproj \ COPY ./src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.csproj \ ./src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.csproj +COPY ./src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.Core.csproj \ + ./src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.Core.csproj + +COPY ./src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj \ + ./src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj + COPY ./src/Microsoft.Health.Fhir.${FHIR_VERSION}.Core/Microsoft.Health.Fhir.${FHIR_VERSION}.Core.csproj \ ./src/Microsoft.Health.Fhir.${FHIR_VERSION}.Core/Microsoft.Health.Fhir.${FHIR_VERSION}.Core.csproj From e6aa26177108b8aa112e2c80effb73983136a634 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Fri, 3 May 2024 11:37:55 +0530 Subject: [PATCH 16/28] updated docker file --- build/docker/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 5fcf8e813d..e81629e623 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -43,11 +43,11 @@ COPY ./src/Microsoft.Health.Fhir.Api/Microsoft.Health.Fhir.Api.csproj \ COPY ./src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.csproj \ ./src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.csproj -COPY ./src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.Core.csproj \ - ./src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.Core.csproj +COPY ./src/Microsoft.Health.Fhir.CosmosDb.Core/Microsoft.Health.Fhir.CosmosDb.Core.csproj \ + ./src/Microsoft.Health.Fhir.CosmosDb.Core/Microsoft.Health.Fhir.CosmosDb.Core.csproj -COPY ./src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj \ - ./src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj +COPY ./src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj \ + ./src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj COPY ./src/Microsoft.Health.Fhir.${FHIR_VERSION}.Core/Microsoft.Health.Fhir.${FHIR_VERSION}.Core.csproj \ ./src/Microsoft.Health.Fhir.${FHIR_VERSION}.Core/Microsoft.Health.Fhir.${FHIR_VERSION}.Core.csproj From 814c32b9c7ccbe0925e50ca79d982848c16e9aa8 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Fri, 10 May 2024 16:52:47 +0530 Subject: [PATCH 17/28] removed extra call to fhircollectionsetting updater --- .../Storage/Versioning/CollectionUpgradeManager.cs | 3 --- .../Features/Storage/CosmosContainerProvider.cs | 12 +++++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs index 650f57c508..617b92ddea 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs @@ -21,7 +21,6 @@ public class CollectionUpgradeManager : IUpgradeManager { private readonly ICollectionDataUpdater _collectionDataUpdater; private readonly IStoredProcedureInstaller _storedProcedureInstaller; - private readonly ICollectionSetup _collectionSetup; private readonly CosmosDataStoreConfiguration _configuration; private readonly CosmosCollectionConfiguration _collectionConfiguration; private readonly ICosmosDbDistributedLockFactory _lockFactory; @@ -43,7 +42,6 @@ public CollectionUpgradeManager( EnsureArg.IsNotNull(namedCosmosCollectionConfigurationAccessor, nameof(namedCosmosCollectionConfigurationAccessor)); EnsureArg.IsNotNull(lockFactory, nameof(lockFactory)); EnsureArg.IsNotNull(logger, nameof(logger)); - _collectionSetup = collectionSetup; _collectionDataUpdater = collectionDataUpdater; _storedProcedureInstaller = storedProcedureInstaller; _configuration = configuration; @@ -69,7 +67,6 @@ public async Task SetupContainerAsync(Container container, CancellationToken can _logger.LogDebug("Attempting to acquire upgrade lock"); await distributedLock.AcquireLock(cancellationTokenSource.Token); - await _collectionSetup.UpdateFhirCollectionSettingsAsync(cancellationToken); await _storedProcedureInstaller.ExecuteAsync(container, cancellationToken); await _collectionDataUpdater.ExecuteAsync(container, cancellationToken); await distributedLock.ReleaseLock(); diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs index fb8ec74687..def19b349e 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosContainerProvider.cs @@ -94,11 +94,13 @@ private async Task InitializeDataStoreAsync( try { _logger.LogInformation("Initializing Cosmos DB Database {DatabaseId} and collections", cosmosDataStoreConfiguration.DatabaseId); - - await collectionSetup.CreateDatabaseAsync(retryPolicyFactory.RetryPolicy, CancellationToken.None); // We need valid cancellation token - await collectionSetup.CreateCollectionAsync(collectionInitializers, retryPolicyFactory.RetryPolicy, CancellationToken.None); - await collectionSetup.UpdateFhirCollectionSettingsAsync(CancellationToken.None); - await collectionDataUpdater.ExecuteAsync(_container.Value, CancellationToken.None); + using (var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(5))) + { + await collectionSetup.CreateDatabaseAsync(retryPolicyFactory.RetryPolicy, cancellationTokenSource.Token); // We need valid cancellation token + await collectionSetup.CreateCollectionAsync(collectionInitializers, retryPolicyFactory.RetryPolicy, cancellationTokenSource.Token); + await collectionSetup.UpdateFhirCollectionSettingsAsync(cancellationTokenSource.Token); + await collectionDataUpdater.ExecuteAsync(_container.Value, cancellationTokenSource.Token); + } } catch (Exception ex) { From 1d53f73d54c7456f2843421a98c4528820874a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Henrique=20Inoc=C3=AAncio=20Borba=20Ferreira?= Date: Wed, 29 May 2024 08:09:03 -0700 Subject: [PATCH 18/28] Refactoring to avoid circle dependency (#3877) --- .../Versioning/CollectionUpgradeManager.cs | 7 -- .../Storage/DataPlaneCollectionSetup.cs | 24 ++++-- .../CollectionUpgradeManagerTests.cs | 3 +- .../Storage/FhirCosmosClientInitializer.cs | 84 +++++++++++++------ .../CosmosDbFhirStorageTestsFixture.cs | 13 +-- 5 files changed, 85 insertions(+), 46 deletions(-) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs index 617b92ddea..59785f3deb 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs @@ -20,7 +20,6 @@ namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning public class CollectionUpgradeManager : IUpgradeManager { private readonly ICollectionDataUpdater _collectionDataUpdater; - private readonly IStoredProcedureInstaller _storedProcedureInstaller; private readonly CosmosDataStoreConfiguration _configuration; private readonly CosmosCollectionConfiguration _collectionConfiguration; private readonly ICosmosDbDistributedLockFactory _lockFactory; @@ -28,22 +27,17 @@ public class CollectionUpgradeManager : IUpgradeManager public CollectionUpgradeManager( ICollectionDataUpdater collectionDataUpdater, - IStoredProcedureInstaller storedProcedureInstaller, - ICollectionSetup collectionSetup, CosmosDataStoreConfiguration configuration, IOptionsMonitor namedCosmosCollectionConfigurationAccessor, ICosmosDbDistributedLockFactory lockFactory, ILogger logger) { EnsureArg.IsNotNull(collectionDataUpdater, nameof(collectionDataUpdater)); - EnsureArg.IsNotNull(storedProcedureInstaller, nameof(storedProcedureInstaller)); - EnsureArg.IsNotNull(collectionSetup, nameof(collectionSetup)); EnsureArg.IsNotNull(configuration, nameof(configuration)); EnsureArg.IsNotNull(namedCosmosCollectionConfigurationAccessor, nameof(namedCosmosCollectionConfigurationAccessor)); EnsureArg.IsNotNull(lockFactory, nameof(lockFactory)); EnsureArg.IsNotNull(logger, nameof(logger)); _collectionDataUpdater = collectionDataUpdater; - _storedProcedureInstaller = storedProcedureInstaller; _configuration = configuration; _collectionConfiguration = GetCosmosCollectionConfiguration(namedCosmosCollectionConfigurationAccessor, Constants.CollectionConfigurationName); _lockFactory = lockFactory; @@ -67,7 +61,6 @@ public async Task SetupContainerAsync(Container container, CancellationToken can _logger.LogDebug("Attempting to acquire upgrade lock"); await distributedLock.AcquireLock(cancellationTokenSource.Token); - await _storedProcedureInstaller.ExecuteAsync(container, cancellationToken); await _collectionDataUpdater.ExecuteAsync(container, cancellationToken); await distributedLock.ReleaseLock(); } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs index e0c8ee2832..5ba155e3d0 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs @@ -18,6 +18,7 @@ using Microsoft.Health.Fhir.CosmosDb.Core; using Microsoft.Health.Fhir.CosmosDb.Core.Configs; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; using Polly; using Polly.Retry; @@ -26,18 +27,25 @@ namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage { public class DataPlaneCollectionSetup : ICollectionSetup { + private const int CollectionSettingsVersion = 3; + private readonly ILogger _logger; private readonly CosmosClient _client; private readonly Lazy _container; - private const int CollectionSettingsVersion = 3; private readonly CosmosDataStoreConfiguration _cosmosDataStoreConfiguration; - - // TODO: move loggerto be the last parameter. - public DataPlaneCollectionSetup(CosmosDataStoreConfiguration cosmosDataStoreConfiguration, IOptionsMonitor collectionConfiguration, ICosmosClientInitializer cosmosClientInitializer, ILogger logger) + private readonly IStoredProcedureInstaller _storedProcedureInstaller; + + public DataPlaneCollectionSetup( + CosmosDataStoreConfiguration cosmosDataStoreConfiguration, + IOptionsMonitor collectionConfiguration, + ICosmosClientInitializer cosmosClientInitializer, + IStoredProcedureInstaller storedProcedureInstaller, + ILogger logger) { EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration)); EnsureArg.IsNotNull(cosmosClientInitializer, nameof(cosmosClientInitializer)); EnsureArg.IsNotNull(collectionConfiguration, nameof(collectionConfiguration)); + EnsureArg.IsNotNull(storedProcedureInstaller, nameof(storedProcedureInstaller)); EnsureArg.IsNotNull(logger, nameof(logger)); string collectionId = collectionConfiguration.Get(Constants.CollectionConfigurationName).CollectionId; @@ -47,7 +55,7 @@ public DataPlaneCollectionSetup(CosmosDataStoreConfiguration cosmosDataStoreConf _client, cosmosDataStoreConfiguration.DatabaseId, collectionId)); - + _storedProcedureInstaller = storedProcedureInstaller; _logger = logger; } @@ -98,6 +106,12 @@ public async Task CreateCollectionAsync(IEnumerable coll } _logger.LogInformation("Collections successfully initialized"); + + _logger.LogInformation("Installing Stored Procedures"); + + await _storedProcedureInstaller.ExecuteAsync(_container.Value, cancellationToken); + + _logger.LogInformation("Stored Procedures are installed"); } catch (Exception ex) { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs index e16b56714e..7c72c68db3 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs @@ -76,6 +76,7 @@ public CollectionUpgradeManagerTests() _cosmosDataStoreConfiguration, optionsMonitor, collectionInitializer, + storeProcedureInstaller, NullLogger.Instance); var collectionVersionWrappers = Substitute.ForPartsOf>(); @@ -88,8 +89,6 @@ public CollectionUpgradeManagerTests() _manager = new CollectionUpgradeManager( collectionDataUpdater, - storeProcedureInstaller, - dataPlaneCollectionSetup, _cosmosDataStoreConfiguration, optionsMonitor, factory, diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs index 0383d80fa5..c91f4a7c53 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -29,6 +30,8 @@ public class FhirCosmosClientInitializer : ICosmosClientInitializer private readonly ILogger _logger; private readonly Func> _requestHandlerFactory; private readonly RetryExceptionPolicyFactory _retryExceptionPolicyFactory; + private readonly ConcurrentDictionary _cosmosClients; + private readonly object _lockObject; public FhirCosmosClientInitializer( ICosmosClientTestProvider testProvider, @@ -45,6 +48,8 @@ public FhirCosmosClientInitializer( _requestHandlerFactory = requestHandlerFactory; _retryExceptionPolicyFactory = retryExceptionPolicyFactory; _logger = logger; + _cosmosClients = new ConcurrentDictionary(); + _lockObject = new object(); } /// @@ -52,6 +57,58 @@ public CosmosClient CreateCosmosClient(CosmosDataStoreConfiguration configuratio { EnsureArg.IsNotNull(configuration, nameof(configuration)); + // Thread-safe logic to ensure that a single instance of CosmosClient is created per host. + if (_cosmosClients.TryGetValue(configuration.Host, out CosmosClient client1)) + { + return client1; + } + else + { + lock (_lockObject) + { + if (_cosmosClients.TryGetValue(configuration.Host, out CosmosClient client2)) + { + return client2; + } + else + { + var client = CreateCosmosClientInternal(configuration); + _cosmosClients.TryAdd(configuration.Host, client); + return client; + } + } + } + } + + public Container CreateFhirContainer(CosmosClient client, string databaseId, string collectionId) + { + return client.GetContainer(databaseId, collectionId); + } + + /// + public async Task OpenCosmosClient(CosmosClient client, CosmosDataStoreConfiguration configuration, CosmosCollectionConfiguration cosmosCollectionConfiguration) + { + EnsureArg.IsNotNull(client, nameof(client)); + EnsureArg.IsNotNull(configuration, nameof(configuration)); + + _logger.LogInformation("Opening CosmosClient connection to {CollectionId}", cosmosCollectionConfiguration.CollectionId); + try + { + await _retryExceptionPolicyFactory.RetryPolicy.ExecuteAsync(async () => + await _testProvider.PerformTestAsync(client.GetContainer(configuration.DatabaseId, cosmosCollectionConfiguration.CollectionId), configuration, cosmosCollectionConfiguration)); + + _logger.LogInformation("Established CosmosClient connection to {CollectionId}", cosmosCollectionConfiguration.CollectionId); + } + catch (Exception e) + { + LogLevel logLevel = e is RequestRateExceededException ? LogLevel.Warning : LogLevel.Critical; + _logger.Log(logLevel, e, "Failed to connect to CosmosClient collection {CollectionId}", cosmosCollectionConfiguration.CollectionId); + throw; + } + } + + private CosmosClient CreateCosmosClientInternal(CosmosDataStoreConfiguration configuration) + { var host = configuration.Host; var key = configuration.Key; @@ -86,33 +143,6 @@ public CosmosClient CreateCosmosClient(CosmosDataStoreConfiguration configuratio return builder.Build(); } - public Container CreateFhirContainer(CosmosClient client, string databaseId, string collectionId) - { - return client.GetContainer(databaseId, collectionId); - } - - /// - public async Task OpenCosmosClient(CosmosClient client, CosmosDataStoreConfiguration configuration, CosmosCollectionConfiguration cosmosCollectionConfiguration) - { - EnsureArg.IsNotNull(client, nameof(client)); - EnsureArg.IsNotNull(configuration, nameof(configuration)); - - _logger.LogInformation("Opening CosmosClient connection to {CollectionId}", cosmosCollectionConfiguration.CollectionId); - try - { - await _retryExceptionPolicyFactory.RetryPolicy.ExecuteAsync(async () => - await _testProvider.PerformTestAsync(client.GetContainer(configuration.DatabaseId, cosmosCollectionConfiguration.CollectionId), configuration, cosmosCollectionConfiguration)); - - _logger.LogInformation("Established CosmosClient connection to {CollectionId}", cosmosCollectionConfiguration.CollectionId); - } - catch (Exception e) - { - LogLevel logLevel = e is RequestRateExceededException ? LogLevel.Warning : LogLevel.Critical; - _logger.Log(logLevel, e, "Failed to connect to CosmosClient collection {CollectionId}", cosmosCollectionConfiguration.CollectionId); - throw; - } - } - private class FhirCosmosSerializer : CosmosSerializer { private const int BlobSizeThresholdWarningInBytes = 1000000; // 1MB threshold. diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs index 11a7c3e047..5c07ab68c2 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs @@ -104,7 +104,7 @@ public virtual async Task InitializeAsync() .ToArray() .Select(type => (IStoredProcedureMetadata)Activator.CreateInstance(type)); - var dataPlaneCollectionSetup = Substitute.For(); + IStoredProcedureInstaller storedProcedureInstallter = new DataPlaneStoredProcedureInstaller(fhirStoredProcs); var optionsMonitor = Substitute.For>(); @@ -122,8 +122,6 @@ public virtual async Task InitializeAsync() IMediator mediator = Substitute.For(); - IStoredProcedureInstaller storedProcedureInstallter = new DataPlaneStoredProcedureInstaller(fhirStoredProcs); - ICollectionDataUpdater dataCollectionUpdater = new CosmosDbSearchParameterStatusInitializer( () => _filebasedSearchParameterStatusDataStore, new CosmosQueryFactory( @@ -133,7 +131,7 @@ public virtual async Task InitializeAsync() var dbLock = new CosmosDbDistributedLockFactory(Substitute.For>>(), NullLogger.Instance); - var upgradeManager = new CollectionUpgradeManager(dataCollectionUpdater, storedProcedureInstallter, dataPlaneCollectionSetup, _cosmosDataStoreConfiguration, optionsMonitor, dbLock, NullLogger.Instance); + var upgradeManager = new CollectionUpgradeManager(dataCollectionUpdater, _cosmosDataStoreConfiguration, optionsMonitor, dbLock, NullLogger.Instance); ICosmosClientTestProvider testProvider = new CosmosClientReadWriteTestProvider(); var cosmosResponseProcessor = Substitute.For(); @@ -149,7 +147,12 @@ public virtual async Task InitializeAsync() // Cosmos DB emulators throws errors when multiple collections are initialized concurrently. // Use the semaphore to only allow one initialization at a time. await CollectionInitializationSemaphore.WaitAsync(); - var datacollectionSetup = new DataPlaneCollectionSetup(_cosmosDataStoreConfiguration, optionsMonitor, documentClientInitializer, NullLogger.Instance); + var datacollectionSetup = new DataPlaneCollectionSetup( + _cosmosDataStoreConfiguration, + optionsMonitor, + documentClientInitializer, + storedProcedureInstallter, + NullLogger.Instance); try { await datacollectionSetup.CreateDatabaseAsync(retryExceptionPolicyFactory.RetryPolicy, CancellationToken.None); From 12e3c3000d17e31b135e7d5e76181556ff9ac9c2 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Wed, 29 May 2024 20:52:36 +0530 Subject: [PATCH 19/28] merging refatoring changes with local branch --- .../StoreProcedures/IStoredProcedureMetadata.cs | 3 --- .../Storage/StoreProcedures/StoredProcedureBase.cs | 8 +++++--- .../Storage/Versioning/CollectionUpgradeManager.cs | 2 ++ .../Features/Storage/CollectionInitializer.cs | 12 ++++++------ .../Features/Storage/DataPlaneCollectionSetup.cs | 9 +-------- .../DataPlaneCosmosDbRegistrationExtensions.cs | 1 - 6 files changed, 14 insertions(+), 21 deletions(-) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedureMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedureMetadata.cs index 1182444973..ede1522be7 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedureMetadata.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedureMetadata.cs @@ -2,9 +2,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs index f71090f43e..a9cbee8abd 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs @@ -15,14 +15,16 @@ namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures { public abstract class StoredProcedureBase : IStoredProcedure { - private readonly IStoredProcedureMetadata _storedProcedureMetadata; + private readonly IStoredProcedureMetadata storedProcedureMetadata; protected StoredProcedureBase(IStoredProcedureMetadata storedProcedure) { - _storedProcedureMetadata = storedProcedure; + EnsureArg.IsNotNull(storedProcedureMetadata, nameof(storedProcedureMetadata)); + + storedProcedureMetadata = storedProcedure; } - public string FullName => _storedProcedureMetadata.FullName; + public string FullName => storedProcedureMetadata.FullName; public async Task> ExecuteStoredProcAsync(Scripts client, string partitionId, CancellationToken cancellationToken, params object[] parameters) { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs index 59785f3deb..bf1576d681 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs @@ -27,12 +27,14 @@ public class CollectionUpgradeManager : IUpgradeManager public CollectionUpgradeManager( ICollectionDataUpdater collectionDataUpdater, + IStoredProcedureInstaller storedProcedureInstaller, CosmosDataStoreConfiguration configuration, IOptionsMonitor namedCosmosCollectionConfigurationAccessor, ICosmosDbDistributedLockFactory lockFactory, ILogger logger) { EnsureArg.IsNotNull(collectionDataUpdater, nameof(collectionDataUpdater)); + EnsureArg.IsNotNull(storedProcedureInstaller, nameof(storedProcedureInstaller)); EnsureArg.IsNotNull(configuration, nameof(configuration)); EnsureArg.IsNotNull(namedCosmosCollectionConfigurationAccessor, nameof(namedCosmosCollectionConfigurationAccessor)); EnsureArg.IsNotNull(lockFactory, nameof(lockFactory)); diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs index 36ed37f64d..8689d160c3 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/CollectionInitializer.cs @@ -54,14 +54,14 @@ public async Task InitializeCollectionAsync(CosmosClient client, Asyn _logger.LogInformation("Finding Container: {CollectionId}", _cosmosCollectionConfiguration.CollectionId); var existingContainer = await retryPolicy.ExecuteAsync(async () => await database.TryGetContainerAsync(_cosmosCollectionConfiguration.CollectionId)); - _logger.LogInformation("Creating Cosmos Container if not exits: {CollectionId}", _cosmosCollectionConfiguration.CollectionId); - ContainerResponse containerResponse = await retryPolicy.ExecuteAsync(async () => - await database.CreateContainerIfNotExistsAsync( - _cosmosCollectionConfiguration.CollectionId, - $"/{KnownDocumentProperties.PartitionKey}", - _cosmosCollectionConfiguration.InitialCollectionThroughput)); + ContainerResponse containerResponse = await retryPolicy + .ExecuteAsync(async () => + await database.CreateContainerIfNotExistsAsync( + _cosmosCollectionConfiguration.CollectionId, + $"/{KnownDocumentProperties.PartitionKey}", + _cosmosCollectionConfiguration.InitialCollectionThroughput)); if (containerResponse.StatusCode == HttpStatusCode.Created || containerResponse.Resource.DefaultTimeToLive != -1) { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs index 5ba155e3d0..ced0cb6bd3 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs @@ -28,19 +28,13 @@ namespace Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage public class DataPlaneCollectionSetup : ICollectionSetup { private const int CollectionSettingsVersion = 3; - private readonly ILogger _logger; private readonly CosmosClient _client; private readonly Lazy _container; private readonly CosmosDataStoreConfiguration _cosmosDataStoreConfiguration; private readonly IStoredProcedureInstaller _storedProcedureInstaller; - public DataPlaneCollectionSetup( - CosmosDataStoreConfiguration cosmosDataStoreConfiguration, - IOptionsMonitor collectionConfiguration, - ICosmosClientInitializer cosmosClientInitializer, - IStoredProcedureInstaller storedProcedureInstaller, - ILogger logger) + public DataPlaneCollectionSetup(CosmosDataStoreConfiguration cosmosDataStoreConfiguration, IOptionsMonitor collectionConfiguration, ICosmosClientInitializer cosmosClientInitializer, ILogger logger) { EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration)); EnsureArg.IsNotNull(cosmosClientInitializer, nameof(cosmosClientInitializer)); @@ -55,7 +49,6 @@ public DataPlaneCollectionSetup( _client, cosmosDataStoreConfiguration.DatabaseId, collectionId)); - _storedProcedureInstaller = storedProcedureInstaller; _logger = logger; } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Registration/DataPlaneCosmosDbRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Registration/DataPlaneCosmosDbRegistrationExtensions.cs index 1e2d9eea57..5208b86158 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Registration/DataPlaneCosmosDbRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Registration/DataPlaneCosmosDbRegistrationExtensions.cs @@ -6,7 +6,6 @@ using System.Linq; using Microsoft.Azure.Cosmos; using Microsoft.Extensions.DependencyInjection; - using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; From 5f981cc42361d180c597f4b8d736e9de7aac475b Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Wed, 29 May 2024 21:40:17 +0530 Subject: [PATCH 20/28] solved merge issues --- .../Features/Storage/DataPlaneCollectionSetup.cs | 8 +++++++- .../Storage/Versioning/CollectionUpgradeManagerTests.cs | 1 + .../Persistence/CosmosDbFhirStorageTestsFixture.cs | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs index ced0cb6bd3..f07aeac0ea 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/DataPlaneCollectionSetup.cs @@ -34,7 +34,12 @@ public class DataPlaneCollectionSetup : ICollectionSetup private readonly CosmosDataStoreConfiguration _cosmosDataStoreConfiguration; private readonly IStoredProcedureInstaller _storedProcedureInstaller; - public DataPlaneCollectionSetup(CosmosDataStoreConfiguration cosmosDataStoreConfiguration, IOptionsMonitor collectionConfiguration, ICosmosClientInitializer cosmosClientInitializer, ILogger logger) + public DataPlaneCollectionSetup( + CosmosDataStoreConfiguration cosmosDataStoreConfiguration, + IOptionsMonitor collectionConfiguration, + ICosmosClientInitializer cosmosClientInitializer, + IStoredProcedureInstaller storedProcedureInstaller, + ILogger logger) { EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration)); EnsureArg.IsNotNull(cosmosClientInitializer, nameof(cosmosClientInitializer)); @@ -49,6 +54,7 @@ public DataPlaneCollectionSetup(CosmosDataStoreConfiguration cosmosDataStoreConf _client, cosmosDataStoreConfiguration.DatabaseId, collectionId)); + _storedProcedureInstaller = storedProcedureInstaller; _logger = logger; } diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs index 7c72c68db3..7b27a9ed83 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs @@ -89,6 +89,7 @@ public CollectionUpgradeManagerTests() _manager = new CollectionUpgradeManager( collectionDataUpdater, + storeProcedureInstaller, _cosmosDataStoreConfiguration, optionsMonitor, factory, diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs index 5c07ab68c2..68874ebd50 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs @@ -131,7 +131,7 @@ public virtual async Task InitializeAsync() var dbLock = new CosmosDbDistributedLockFactory(Substitute.For>>(), NullLogger.Instance); - var upgradeManager = new CollectionUpgradeManager(dataCollectionUpdater, _cosmosDataStoreConfiguration, optionsMonitor, dbLock, NullLogger.Instance); + var upgradeManager = new CollectionUpgradeManager(dataCollectionUpdater, storedProcedureInstallter, _cosmosDataStoreConfiguration, optionsMonitor, dbLock, NullLogger.Instance); ICosmosClientTestProvider testProvider = new CosmosClientReadWriteTestProvider(); var cosmosResponseProcessor = Substitute.For(); From 9269d5063b0ddeb2c8cd8508ddbfe3ca271568b2 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Wed, 29 May 2024 23:09:20 +0530 Subject: [PATCH 21/28] fixes namespace issue for Core.Resources --- .../Features/Search/FhirCosmosSearchService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/FhirCosmosSearchService.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/FhirCosmosSearchService.cs index 069cfdcf68..3717558f6a 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/FhirCosmosSearchService.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/FhirCosmosSearchService.cs @@ -302,7 +302,7 @@ private async Task RecurseChainedExpression(ChainedExpression expres catch (OperationCanceledException) { _logger.LogWarning("Request Too Costly (ConditionalRequestTooCostly)"); - throw new RequestTooCostlyException(Core.Resources.ConditionalRequestTooCostly); + throw new RequestTooCostlyException(Microsoft.Health.Fhir.Core.Resources.ConditionalRequestTooCostly); } if (!chainedResults.Any()) @@ -558,7 +558,7 @@ private async Task ExecuteCountSearchAsync( string.Format(Microsoft.Health.Fhir.Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue))); _logger.LogWarning("Invalid Search Operation (SearchCountResultsExceedLimit)"); - throw new InvalidSearchOperationException(string.Format(Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue)); + throw new InvalidSearchOperationException(string.Format(Microsoft.Health.Fhir.Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue)); } return (int)count; From fd6169e43b70fd3ae78914f1870666e4940c62a8 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Thu, 30 May 2024 21:38:44 +0530 Subject: [PATCH 22/28] resolves unit test issue for FHIRCosmosClientInitializer --- .../Storage/StoreProcedures/StoredProcedureBase.cs | 8 ++++---- .../Storage/Versioning/CollectionUpgradeManagerTests.cs | 7 ------- .../Features/Storage/FhirCosmosClientInitializer.cs | 8 +++++--- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs index a9cbee8abd..55f512634f 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/StoredProcedureBase.cs @@ -15,16 +15,16 @@ namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures { public abstract class StoredProcedureBase : IStoredProcedure { - private readonly IStoredProcedureMetadata storedProcedureMetadata; + private readonly IStoredProcedureMetadata _storedProcedureMetadata; - protected StoredProcedureBase(IStoredProcedureMetadata storedProcedure) + protected StoredProcedureBase(IStoredProcedureMetadata storedProcedureMetadata) { EnsureArg.IsNotNull(storedProcedureMetadata, nameof(storedProcedureMetadata)); - storedProcedureMetadata = storedProcedure; + _storedProcedureMetadata = storedProcedureMetadata; } - public string FullName => storedProcedureMetadata.FullName; + public string FullName => _storedProcedureMetadata.FullName; public async Task> ExecuteStoredProcAsync(Scripts client, string partitionId, CancellationToken cancellationToken, params object[] parameters) { diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs index 7b27a9ed83..22bb39a07d 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs @@ -72,13 +72,6 @@ public CollectionUpgradeManagerTests() collectionInitializer.CreateFhirContainer(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(_container); - var dataPlaneCollectionSetup = new DataPlaneCollectionSetup( - _cosmosDataStoreConfiguration, - optionsMonitor, - collectionInitializer, - storeProcedureInstaller, - NullLogger.Instance); - var collectionVersionWrappers = Substitute.ForPartsOf>(); _container.GetItemQueryIterator(Arg.Any(), Arg.Any(), Arg.Any()) diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs index c91f4a7c53..36ba1c1f11 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs @@ -57,8 +57,10 @@ public CosmosClient CreateCosmosClient(CosmosDataStoreConfiguration configuratio { EnsureArg.IsNotNull(configuration, nameof(configuration)); + string key = string.IsNullOrEmpty(configuration.Host) ? "local" : configuration.Host; + // Thread-safe logic to ensure that a single instance of CosmosClient is created per host. - if (_cosmosClients.TryGetValue(configuration.Host, out CosmosClient client1)) + if (_cosmosClients.TryGetValue(key, out CosmosClient client1)) { return client1; } @@ -66,14 +68,14 @@ public CosmosClient CreateCosmosClient(CosmosDataStoreConfiguration configuratio { lock (_lockObject) { - if (_cosmosClients.TryGetValue(configuration.Host, out CosmosClient client2)) + if (_cosmosClients.TryGetValue(key, out CosmosClient client2)) { return client2; } else { var client = CreateCosmosClientInternal(configuration); - _cosmosClients.TryAdd(configuration.Host, client); + _cosmosClients.TryAdd(key, client); return client; } } From 004109447ad67de1da262be9042c2547273b685e Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Wed, 5 Jun 2024 18:17:31 +0530 Subject: [PATCH 23/28] Resolves unite test issues --- .../Versioning/CollectionUpgradeManager.cs | 3 - .../CollectionUpgradeManagerTests.cs | 33 +----- .../DataPlaneCollectionSetupTests.cs | 108 ++++++++++++++++++ .../CosmosDbFhirStorageTestsFixture.cs | 2 +- 4 files changed, 115 insertions(+), 31 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/DataPlaneCollectionSetupTests.cs diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs index bf1576d681..61b3668588 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/Versioning/CollectionUpgradeManager.cs @@ -27,14 +27,12 @@ public class CollectionUpgradeManager : IUpgradeManager public CollectionUpgradeManager( ICollectionDataUpdater collectionDataUpdater, - IStoredProcedureInstaller storedProcedureInstaller, CosmosDataStoreConfiguration configuration, IOptionsMonitor namedCosmosCollectionConfigurationAccessor, ICosmosDbDistributedLockFactory lockFactory, ILogger logger) { EnsureArg.IsNotNull(collectionDataUpdater, nameof(collectionDataUpdater)); - EnsureArg.IsNotNull(storedProcedureInstaller, nameof(storedProcedureInstaller)); EnsureArg.IsNotNull(configuration, nameof(configuration)); EnsureArg.IsNotNull(namedCosmosCollectionConfigurationAccessor, nameof(namedCosmosCollectionConfigurationAccessor)); EnsureArg.IsNotNull(lockFactory, nameof(lockFactory)); @@ -51,7 +49,6 @@ public CollectionUpgradeManager( /// public int CollectionSettingsVersion { get; } = 3; - // TODO: Add a cancellation token parameter, and replace CancellationToken.None with the cancellation token. public async Task SetupContainerAsync(Container container, CancellationToken cancellationToken) { EnsureArg.IsNotNull(container, nameof(container)); diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs index 22bb39a07d..83e9392a95 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/CollectionUpgradeManagerTests.cs @@ -44,6 +44,7 @@ public class CollectionUpgradeManagerTests }; private readonly CollectionUpgradeManager _manager; + private readonly ICollectionDataUpdater _collectionDataUpdater; private readonly Container _container; private readonly ContainerResponse _containerResponse; @@ -60,12 +61,8 @@ public CollectionUpgradeManagerTests() var optionsMonitor = Substitute.For>(); optionsMonitor.Get(Constants.CollectionConfigurationName).Returns(_cosmosCollectionConfiguration); - var collectionDataUpdater = Substitute.For(); - collectionDataUpdater.ExecuteAsync(Arg.Any(), Arg.Any()) - .Returns(Task.FromResult(true)); - - var storeProcedureInstaller = Substitute.For(); - storeProcedureInstaller.ExecuteAsync(Arg.Any(), Arg.Any()) + _collectionDataUpdater = Substitute.For(); + _collectionDataUpdater.ExecuteAsync(Arg.Any(), Arg.Any()) .Returns(Task.FromResult(true)); var collectionInitializer = Substitute.For(); @@ -81,8 +78,7 @@ public CollectionUpgradeManagerTests() .Returns(Substitute.ForPartsOf>()); _manager = new CollectionUpgradeManager( - collectionDataUpdater, - storeProcedureInstaller, + _collectionDataUpdater, _cosmosDataStoreConfiguration, optionsMonitor, factory, @@ -103,28 +99,11 @@ public CollectionUpgradeManagerTests() } [Fact] - public async Task GivenACollection_WhenSettingUpCollection_ThenTheCollectionIndexIsUpdated() - { - await UpdateCollectionAsync(); - - await _container.Received(1).ReplaceContainerAsync(Arg.Any(), null, Arg.Any()); - } - - [Fact] - public async Task GivenACollection_WhenSettingUpCollection_ThenTheCollectionVersionWrapperIsSaved() - { - await UpdateCollectionAsync(); - - await _container.Received(1) - .UpsertItemAsync(Arg.Is(x => x.Version == _manager.CollectionSettingsVersion), Arg.Any(), null, Arg.Any()); - } - - [Fact] - public async Task GivenACollection_WhenSettingUpCollection_ThenTheCollectionTTLIsSetToNeg1() + public async Task GivenACollection_WhenSettingUpCollection_ThenSearchParameterIsRegistered() { await UpdateCollectionAsync(); - Assert.Equal(-1, _containerResponse.Resource.DefaultTimeToLive); + await _collectionDataUpdater.Received(1).ExecuteAsync(_container, Arg.Any()); } private async Task UpdateCollectionAsync() diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/DataPlaneCollectionSetupTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/DataPlaneCollectionSetupTests.cs new file mode 100644 index 0000000000..8692bb9d6e --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/DataPlaneCollectionSetupTests.cs @@ -0,0 +1,108 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Cosmos; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; +using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.CosmosDb.UnitTests.Features.Storage.Versioning +{ + public class DataPlaneCollectionSetupTests + { + private readonly Container _container; + private readonly DataPlaneCollectionSetup _setup; + private readonly ContainerResponse _containerResponse; + private readonly ILogger _logger = Substitute.For>(); + + private readonly CosmosDataStoreConfiguration _cosmosDataStoreConfiguration = new CosmosDataStoreConfiguration + { + AllowDatabaseCreation = false, + ConnectionMode = ConnectionMode.Direct, + DatabaseId = "testdatabaseid", + Host = "https://fakehost", + Key = "ZmFrZWtleQ==", // "fakekey" + PreferredLocations = null, + }; + + private readonly CosmosCollectionConfiguration _cosmosCollectionConfiguration = new CosmosCollectionConfiguration + { + CollectionId = "testcollectionid", + }; + + public DataPlaneCollectionSetupTests() + { + _container = Substitute.For(); + _containerResponse = Substitute.ForPartsOf(); + + var containerProperties = new ContainerProperties(); + containerProperties.IndexingPolicy = new IndexingPolicy + { + IncludedPaths = { }, + ExcludedPaths = { }, + }; + + _containerResponse.Resource.Returns(containerProperties); + _container.ReadContainerAsync(Arg.Any(), Arg.Any()) + .Returns(_containerResponse); + + var optionsMonitor = Substitute.For>(); + optionsMonitor.Get(Constants.CollectionConfigurationName).Returns(_cosmosCollectionConfiguration); + + var collectionInitializer = Substitute.For(); + collectionInitializer.CreateFhirContainer(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_container); + + var collectionVersionWrappers = Substitute.ForPartsOf>(); + + _container.GetItemQueryIterator(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(collectionVersionWrappers); + + collectionVersionWrappers.ReadNextAsync(Arg.Any()) + .Returns(Substitute.ForPartsOf>()); + + var storeProcedureInstaller = Substitute.For(); + storeProcedureInstaller.ExecuteAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(true)); + + _setup = new DataPlaneCollectionSetup(_cosmosDataStoreConfiguration, optionsMonitor, collectionInitializer, storeProcedureInstaller, _logger); + } + + [Fact] + public async Task GivenACollection_WhenSettingUpCollection_ThenTheCollectionIndexIsUpdated() + { + await _setup.UpdateFhirCollectionSettingsAsync(CancellationToken.None); + await _container.Received(1).ReplaceContainerAsync(Arg.Any(), null, Arg.Any()); + } + + [Fact] + public async Task GivenACollection_WhenSettingUpCollection_ThenTheCollectionVersionWrapperIsSaved() + { + await _setup.UpdateFhirCollectionSettingsAsync(CancellationToken.None); + + await _container.Received(1) + .UpsertItemAsync(Arg.Is(x => x.Version == 3), Arg.Any(), null, Arg.Any()); + } + + [Fact] + public async Task GivenACollection_WhenSettingUpCollection_ThenTheCollectionTTLIsSetToNeg1() + { + await _setup.UpdateFhirCollectionSettingsAsync(CancellationToken.None); + Assert.Equal(-1, _containerResponse.Resource.DefaultTimeToLive); + } + } +} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs index 68874ebd50..5c07ab68c2 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs @@ -131,7 +131,7 @@ public virtual async Task InitializeAsync() var dbLock = new CosmosDbDistributedLockFactory(Substitute.For>>(), NullLogger.Instance); - var upgradeManager = new CollectionUpgradeManager(dataCollectionUpdater, storedProcedureInstallter, _cosmosDataStoreConfiguration, optionsMonitor, dbLock, NullLogger.Instance); + var upgradeManager = new CollectionUpgradeManager(dataCollectionUpdater, _cosmosDataStoreConfiguration, optionsMonitor, dbLock, NullLogger.Instance); ICosmosClientTestProvider testProvider = new CosmosClientReadWriteTestProvider(); var cosmosResponseProcessor = Substitute.For(); From 6f8bf1cbae64a44854adc14036cc0b15bad47c8c Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Wed, 5 Jun 2024 19:53:42 +0530 Subject: [PATCH 24/28] added missing decorators on DataPlaneCollectionSetupTests --- .../Storage/Versioning/DataPlaneCollectionSetupTests.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/DataPlaneCollectionSetupTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/DataPlaneCollectionSetupTests.cs index 8692bb9d6e..bfaceae3b5 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/DataPlaneCollectionSetupTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Versioning/DataPlaneCollectionSetupTests.cs @@ -17,11 +17,15 @@ using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.Versioning; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; namespace Microsoft.Health.Fhir.CosmosDb.UnitTests.Features.Storage.Versioning { + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.DataSourceValidation)] public class DataPlaneCollectionSetupTests { private readonly Container _container; From 810cb16ad34cf33beb40d31f740feeb9d6109243 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Thu, 6 Jun 2024 15:00:29 +0530 Subject: [PATCH 25/28] work on review comments --- .../Features/Storage/ICollectionSetup.cs | 1 - .../Storage/StoreProcedures/IStoredProcedureMetadata.cs | 4 +--- .../StoredProcedures/DataPlaneStoredProcedureInstaller.cs | 2 -- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionSetup.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionSetup.cs index 9c45a2244e..03b07d0e66 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionSetup.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/ICollectionSetup.cs @@ -3,7 +3,6 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedureMetadata.cs b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedureMetadata.cs index ede1522be7..9131d31eff 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedureMetadata.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Core/Features/Storage/StoreProcedures/IStoredProcedureMetadata.cs @@ -2,9 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System.Linq; -using System.Text; -using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Scripts; namespace Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/DataPlaneStoredProcedureInstaller.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/DataPlaneStoredProcedureInstaller.cs index 636ba4fab5..3f162b1e16 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/DataPlaneStoredProcedureInstaller.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization/Features/Storage/StoredProcedures/DataPlaneStoredProcedureInstaller.cs @@ -17,7 +17,6 @@ public sealed class DataPlaneStoredProcedureInstaller : IStoredProcedureInstalle { private readonly IEnumerable _storeProceduresMetadata; - // TODO: refactor constructor to have dependency on container public DataPlaneStoredProcedureInstaller(IEnumerable storedProcedures) { EnsureArg.IsNotNull(storedProcedures, nameof(storedProcedures)); @@ -25,7 +24,6 @@ public DataPlaneStoredProcedureInstaller(IEnumerable s _storeProceduresMetadata = storedProcedures; } - // TODO: refactor method to have dependency on IReadOnlyList public async Task ExecuteAsync(Container container, CancellationToken cancellationToken) { foreach (IStoredProcedureMetadata storedProc in _storeProceduresMetadata) From 8a95303be3e40ad070e9c14f7263e185bcd36659 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Tue, 18 Jun 2024 00:00:00 +0530 Subject: [PATCH 26/28] Adds classification attibutes --- .../CosmosDbInitializationTests.cs | 4 ++++ ...osoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs index 69a58ace76..3063eb343d 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/CosmosDbInitializationTests.cs @@ -9,10 +9,14 @@ using Microsoft.Health.Core.Extensions; using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage.StoredProcedures; using Microsoft.Health.Fhir.CosmosDb.Initialization.Features.Storage.StoredProcedures; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; using Xunit; namespace Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests { + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.DataSourceValidation)] public class CosmosDbInitializationTests { [Fact] diff --git a/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj index a31b125781..57eb3e5ddb 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj +++ b/src/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests/Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj @@ -1,5 +1,6 @@  + @@ -7,6 +8,7 @@ + From b15ee623fe7dbf26640a4e92e50a9dcbd63c0b13 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Tue, 2 Jul 2024 23:41:33 +0530 Subject: [PATCH 27/28] -Adding newly added project references to R4.slnf and R5.slnf -Adding simple threadsafe logic for creating CosmosClient instance --- R4.slnf | 3 +++ R5.slnf | 3 +++ .../Storage/FhirCosmosClientInitializer.cs | 19 +++++-------------- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/R4.slnf b/R4.slnf index 4469743c86..0e555d8325 100644 --- a/R4.slnf +++ b/R4.slnf @@ -10,6 +10,9 @@ "src\\Microsoft.Health.Fhir.Azure\\Microsoft.Health.Fhir.Azure.csproj", "src\\Microsoft.Health.Fhir.Core.UnitTests\\Microsoft.Health.Fhir.Core.UnitTests.csproj", "src\\Microsoft.Health.Fhir.Core\\Microsoft.Health.Fhir.Core.csproj", + "src\Microsoft.Health.Fhir.CosmosDb.Core\Microsoft.Health.Fhir.CosmosDb.Core.csproj", + "src\Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests\Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj", + "src\Microsoft.Health.Fhir.CosmosDb.Initialization\Microsoft.Health.Fhir.CosmosDb.Initialization.csproj", "src\\Microsoft.Health.Fhir.CosmosDb.UnitTests\\Microsoft.Health.Fhir.CosmosDb.UnitTests.csproj", "src\\Microsoft.Health.Fhir.CosmosDb\\Microsoft.Health.Fhir.CosmosDb.csproj", "src\\Microsoft.Health.Fhir.R4.Api\\Microsoft.Health.Fhir.R4.Api.csproj", diff --git a/R5.slnf b/R5.slnf index b34a610a38..61bdc352fd 100644 --- a/R5.slnf +++ b/R5.slnf @@ -10,6 +10,9 @@ "src\\Microsoft.Health.Fhir.Azure\\Microsoft.Health.Fhir.Azure.csproj", "src\\Microsoft.Health.Fhir.Core.UnitTests\\Microsoft.Health.Fhir.Core.UnitTests.csproj", "src\\Microsoft.Health.Fhir.Core\\Microsoft.Health.Fhir.Core.csproj", + "src\Microsoft.Health.Fhir.CosmosDb.Core\Microsoft.Health.Fhir.CosmosDb.Core.csproj", + "src\Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests\Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj", + "src\Microsoft.Health.Fhir.CosmosDb.Initialization\Microsoft.Health.Fhir.CosmosDb.Initialization.csproj", "src\\Microsoft.Health.Fhir.CosmosDb.UnitTests\\Microsoft.Health.Fhir.CosmosDb.UnitTests.csproj", "src\\Microsoft.Health.Fhir.CosmosDb\\Microsoft.Health.Fhir.CosmosDb.csproj", "src\\Microsoft.Health.Fhir.R5.Api\\Microsoft.Health.Fhir.R5.Api.csproj", diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs index 36ba1c1f11..ae54926ece 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs @@ -30,7 +30,7 @@ public class FhirCosmosClientInitializer : ICosmosClientInitializer private readonly ILogger _logger; private readonly Func> _requestHandlerFactory; private readonly RetryExceptionPolicyFactory _retryExceptionPolicyFactory; - private readonly ConcurrentDictionary _cosmosClients; + private static CosmosClient _cosmosClient; private readonly object _lockObject; public FhirCosmosClientInitializer( @@ -48,7 +48,6 @@ public FhirCosmosClientInitializer( _requestHandlerFactory = requestHandlerFactory; _retryExceptionPolicyFactory = retryExceptionPolicyFactory; _logger = logger; - _cosmosClients = new ConcurrentDictionary(); _lockObject = new object(); } @@ -60,24 +59,16 @@ public CosmosClient CreateCosmosClient(CosmosDataStoreConfiguration configuratio string key = string.IsNullOrEmpty(configuration.Host) ? "local" : configuration.Host; // Thread-safe logic to ensure that a single instance of CosmosClient is created per host. - if (_cosmosClients.TryGetValue(key, out CosmosClient client1)) + if (_cosmosClient != null) { - return client1; + return _cosmosClient; } else { lock (_lockObject) { - if (_cosmosClients.TryGetValue(key, out CosmosClient client2)) - { - return client2; - } - else - { - var client = CreateCosmosClientInternal(configuration); - _cosmosClients.TryAdd(key, client); - return client; - } + _cosmosClient = CreateCosmosClientInternal(configuration); + return _cosmosClient; } } } From 4cff2cfcf3cd0e4347c6103bbaff6ad7ecfe0735 Mon Sep 17 00:00:00 2001 From: Anil Mahajan Date: Wed, 3 Jul 2024 00:49:24 +0530 Subject: [PATCH 28/28] refactoring CreateCosmosClient logic to have validation logic --- .../Features/Storage/FhirCosmosClientInitializer.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs index ae54926ece..42785f1d05 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs @@ -30,7 +30,7 @@ public class FhirCosmosClientInitializer : ICosmosClientInitializer private readonly ILogger _logger; private readonly Func> _requestHandlerFactory; private readonly RetryExceptionPolicyFactory _retryExceptionPolicyFactory; - private static CosmosClient _cosmosClient; + private CosmosClient _cosmosClient; private readonly object _lockObject; public FhirCosmosClientInitializer( @@ -56,9 +56,7 @@ public CosmosClient CreateCosmosClient(CosmosDataStoreConfiguration configuratio { EnsureArg.IsNotNull(configuration, nameof(configuration)); - string key = string.IsNullOrEmpty(configuration.Host) ? "local" : configuration.Host; - - // Thread-safe logic to ensure that a single instance of CosmosClient is created per host. + // Thread-safe logic to ensure that a single instance of CosmosClient is created. if (_cosmosClient != null) { return _cosmosClient; @@ -67,7 +65,11 @@ public CosmosClient CreateCosmosClient(CosmosDataStoreConfiguration configuratio { lock (_lockObject) { - _cosmosClient = CreateCosmosClientInternal(configuration); + if (_cosmosClient == null) + { + _cosmosClient = CreateCosmosClientInternal(configuration); + } + return _cosmosClient; } }