From 09887b1bff6ec7d77e69e0256edc76bac1ec6a82 Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Fri, 23 Sep 2022 09:20:35 -0700 Subject: [PATCH] Ability to switch temporary storage to use either memory or disk (#166) * Ability to switch temporary storage to use either memory or disk Signed-off-by: Victor Chang * Fix configuration name for temp data storage. Signed-off-by: Victor Chang * Validate storage configurations based on temp storage location Signed-off-by: Victor Chang * Log time for a payload to complete end-to-end within the service. Signed-off-by: Victor Chang Signed-off-by: Victor Chang --- src/Api/Storage/Payload.cs | 2 + src/Api/Storage/StorageObjectMetadata.cs | 25 ++++++- src/Configuration/ConfigurationValidator.cs | 38 +++++++++- src/Configuration/StorageConfiguration.cs | 15 ++++ .../TemporaryDataStorageLocation.cs | 24 ++++++ .../Test/ConfigurationValidatorTest.cs | 38 ++++++++-- src/Database/PayloadConfiguration.cs | 1 + .../Common/FileStorageMetadataExtensions.cs | 75 ++++++++++++++++--- .../Logging/Log.700.PayloadService.cs | 4 +- .../Connectors/DataRetrievalService.cs | 9 ++- .../PayloadNotificationActionHandler.cs | 2 +- .../Services/DicomWeb/IStreamsWriter.cs | 8 +- .../Services/Fhir/FhirJsonReader.cs | 11 ++- .../Services/Fhir/FhirService.cs | 7 +- .../Services/Fhir/FhirXmlReader.cs | 11 ++- .../Services/HealthLevel7/MllpService.cs | 5 +- .../Services/Scp/ApplicationEntityHandler.cs | 12 ++- .../DicomFileStorageMetadataExtensionsTest.cs | 40 +++++++++- .../Connectors/DataRetrievalServiceTest.cs | 4 + .../Services/DicomWeb/StreamsWriterTest.cs | 22 +++--- .../Test/Services/Fhir/FhirJsonReaderTest.cs | 17 +++-- .../Test/Services/Fhir/FhirServiceTest.cs | 4 + .../Test/Services/Fhir/FhirXmlReaderTest.cs | 19 +++-- .../Services/HealthLevel7/MllpServiceTest.cs | 5 +- .../Scp/ApplicationEntityHandlerTest.cs | 24 ++++-- .../Storage/ObjectUploadServiceTest.cs | 11 ++- 26 files changed, 353 insertions(+), 80 deletions(-) create mode 100644 src/Configuration/TemporaryDataStorageLocation.cs diff --git a/src/Api/Storage/Payload.cs b/src/Api/Storage/Payload.cs index d0fcb4e3e..af584bb20 100644 --- a/src/Api/Storage/Payload.cs +++ b/src/Api/Storage/Payload.cs @@ -66,6 +66,8 @@ public enum PayloadState public bool HasTimedOut { get => ElapsedTime().TotalSeconds >= Timeout; } + public TimeSpan Elapsed { get { return DateTime.UtcNow.Subtract(DateTimeCreated); } } + public string CallingAeTitle { get => Files.OfType().Select(p => p.CallingAeTitle).FirstOrDefault(); } public string CalledAeTitle { get => Files.OfType().Select(p => p.CalledAeTitle).FirstOrDefault(); } diff --git a/src/Api/Storage/StorageObjectMetadata.cs b/src/Api/Storage/StorageObjectMetadata.cs index 7e17289da..fae88b73a 100644 --- a/src/Api/Storage/StorageObjectMetadata.cs +++ b/src/Api/Storage/StorageObjectMetadata.cs @@ -16,6 +16,7 @@ using System; using System.IO; +using System.Runtime; using System.Text.Json.Serialization; using Ardalis.GuardClauses; @@ -123,9 +124,27 @@ public void SetUploaded(string bucketName) if (Data is not null && Data.CanSeek) { - Data.Close(); - Data.Dispose(); - Data = null; + if (Data is FileStream fileStream) + { + var filename = fileStream.Name; + Data.Close(); + Data.Dispose(); + Data = null; + System.IO.File.Delete(filename); + } + else // MemoryStream + { + Data.Close(); + Data.Dispose(); + Data = null; + + // When IG stores all received/downloaded data in-memory using MemoryStream, LOH grows tremendously and thus impacts the performance and + // memory usage. The following makes sure LOH is compacted after the data is uploaded. + GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; +#pragma warning disable S1215 // "GC.Collect" should not be called + GC.Collect(); +#pragma warning restore S1215 // "GC.Collect" should not be called + } } } diff --git a/src/Configuration/ConfigurationValidator.cs b/src/Configuration/ConfigurationValidator.cs index bd8c18db2..86d7cb51d 100644 --- a/src/Configuration/ConfigurationValidator.cs +++ b/src/Configuration/ConfigurationValidator.cs @@ -17,6 +17,8 @@ using System; using System.Collections.Generic; +using System.IO; +using System.IO.Abstractions; using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -29,6 +31,7 @@ namespace Monai.Deploy.InformaticsGateway.Configuration public class ConfigurationValidator : IValidateOptions { private readonly ILogger _logger; + private readonly IFileSystem _fileSystem; private readonly List _validationErrors; /// @@ -36,9 +39,10 @@ public class ConfigurationValidator : IValidateOptions /// InformaticsGatewayConfiguration to be validated /// Logger to be used by ConfigurationValidator - public ConfigurationValidator(ILogger logger) + public ConfigurationValidator(ILogger logger, IFileSystem fileSystem) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); _validationErrors = new List(); } @@ -82,6 +86,38 @@ private bool IsStorageValid(StorageConfiguration storage) valid &= IsValueInRange("InformaticsGateway>storage>reserveSpaceGB", 1, 999, storage.ReserveSpaceGB); valid &= IsValueInRange("InformaticsGateway>storage>payloadProcessThreads", 1, 128, storage.PayloadProcessThreads); valid &= IsValueInRange("InformaticsGateway>storage>concurrentUploads", 1, 128, storage.ConcurrentUploads); + valid &= IsValueInRange("InformaticsGateway>storage>memoryThreshold", 1, int.MaxValue, storage.MemoryThreshold); + + if (storage.TemporaryDataStorage == TemporaryDataStorageLocation.Disk) + { + valid &= IsNotNullOrWhiteSpace("InformaticsGateway>storage>bufferRootPath", storage.BufferStorageRootPath); + valid &= IsValidDirectory("InformaticsGateway>storage>bufferRootPath", storage.BufferStorageRootPath); + valid &= IsValueInRange("InformaticsGateway>storage>bufferSize", 1, int.MaxValue, storage.BufferSize); + } + + return valid; + } + + private bool IsValidDirectory(string source, string directory) + { + var valid = true; + try + { + if (!_fileSystem.Directory.Exists(directory)) + { + valid = false; + _validationErrors.Add($"Directory `{directory}` specified in `{source}` does not exist."); + } + else + { + using var _ = _fileSystem.File.Create(Path.Combine(directory, Path.GetRandomFileName()), 1, FileOptions.DeleteOnClose); + } + } + catch (Exception ex) + { + valid = false; + _validationErrors.Add($"Directory `{directory}` specified in `{source}` is not accessible: {ex.Message}."); + } return valid; } diff --git a/src/Configuration/StorageConfiguration.cs b/src/Configuration/StorageConfiguration.cs index 8995e52c8..871216522 100644 --- a/src/Configuration/StorageConfiguration.cs +++ b/src/Configuration/StorageConfiguration.cs @@ -22,6 +22,13 @@ namespace Monai.Deploy.InformaticsGateway.Configuration { public class StorageConfiguration : StorageServiceConfiguration { + /// + /// Gets or sets whether to store temporary data in Memory or on Disk. + /// Defaults to Memory. + /// + [ConfigurationKeyName("tempStorageLocation")] + public TemporaryDataStorageLocation TemporaryDataStorage { get; set; } = TemporaryDataStorageLocation.Memory; + /// /// Gets or sets the path used for buffering incoming data. /// Defaults to ./temp. @@ -29,6 +36,14 @@ public class StorageConfiguration : StorageServiceConfiguration [ConfigurationKeyName("bufferRootPath")] public string BufferStorageRootPath { get; set; } = "./temp"; + /// + /// Gets or sets the number of bytes buffered for reads and writes to the temporary file. + /// Defaults to 128000. + /// + [ConfigurationKeyName("bufferSize")] + public int BufferSize { get; set; } = 128000; + + /// /// Gets or set the maximum memory buffer size in bytes with default to 30MiB. /// diff --git a/src/Configuration/TemporaryDataStorageLocation.cs b/src/Configuration/TemporaryDataStorageLocation.cs new file mode 100644 index 000000000..01fe8305d --- /dev/null +++ b/src/Configuration/TemporaryDataStorageLocation.cs @@ -0,0 +1,24 @@ +/* + * Copyright 2022 MONAI Consortium + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Monai.Deploy.InformaticsGateway.Configuration +{ + public enum TemporaryDataStorageLocation + { + Memory, + Disk + } +} diff --git a/src/Configuration/Test/ConfigurationValidatorTest.cs b/src/Configuration/Test/ConfigurationValidatorTest.cs index 9c60a5c4b..fa282b697 100644 --- a/src/Configuration/Test/ConfigurationValidatorTest.cs +++ b/src/Configuration/Test/ConfigurationValidatorTest.cs @@ -15,6 +15,8 @@ */ using System; +using System.IO; +using System.IO.Abstractions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Monai.Deploy.InformaticsGateway.SharedTest; @@ -26,17 +28,19 @@ namespace Monai.Deploy.InformaticsGateway.Configuration.Test public class ConfigurationValidatorTest { private readonly Mock> _logger; + private readonly Mock _fileSystem; public ConfigurationValidatorTest() { _logger = new Mock>(); + _fileSystem = new Mock(); } [Fact(DisplayName = "ConfigurationValidator test with all valid settings")] public void AllValid() { var config = MockValidConfiguration(); - var valid = new ConfigurationValidator(_logger.Object).Validate("", config); + var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config); Assert.True(valid == ValidateOptionsResult.Success); } @@ -46,7 +50,7 @@ public void InvalidScpPort() var config = MockValidConfiguration(); config.Dicom.Scp.Port = Int32.MaxValue; - var valid = new ConfigurationValidator(_logger.Object).Validate("", config); + var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config); var validationMessage = $"Invalid port number '{Int32.MaxValue}' specified for InformaticsGateway>dicom>scp>port."; Assert.Equal(validationMessage, valid.FailureMessage); @@ -59,7 +63,7 @@ public void InvalidScpMaxAssociations() var config = MockValidConfiguration(); config.Dicom.Scp.MaximumNumberOfAssociations = 0; - var valid = new ConfigurationValidator(_logger.Object).Validate("", config); + var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config); var validationMessage = $"Value of InformaticsGateway>dicom>scp>max-associations must be between {1} and {1000}."; Assert.Equal(validationMessage, valid.FailureMessage); @@ -72,7 +76,7 @@ public void StorageWithInvalidWatermark() var config = MockValidConfiguration(); config.Storage.Watermark = 1000; - var valid = new ConfigurationValidator(_logger.Object).Validate("", config); + var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config); var validationMessage = "Value of InformaticsGateway>storage>watermark must be between 1 and 100."; Assert.Equal(validationMessage, valid.FailureMessage); @@ -85,7 +89,7 @@ public void StorageWithInvalidReservedSpace() var config = MockValidConfiguration(); config.Storage.ReserveSpaceGB = 9999; - var valid = new ConfigurationValidator(_logger.Object).Validate("", config); + var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config); var validationMessage = "Value of InformaticsGateway>storage>reserveSpaceGB must be between 1 and 999."; Assert.Equal(validationMessage, valid.FailureMessage); @@ -98,7 +102,7 @@ public void StorageWithInvalidTemporaryBucketName() var config = MockValidConfiguration(); config.Storage.TemporaryStorageBucket = " "; - var valid = new ConfigurationValidator(_logger.Object).Validate("", config); + var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config); var validationMessages = new[] { "Value for InformaticsGateway>storage>temporaryBucketName is required.", "Value for InformaticsGateway>storage>temporaryBucketName does not conform to Amazon S3 bucket naming requirements." }; Assert.Equal(string.Join(Environment.NewLine, validationMessages), valid.FailureMessage); @@ -114,7 +118,7 @@ public void StorageWithInvalidBucketName() var config = MockValidConfiguration(); config.Storage.StorageServiceBucketName = ""; - var valid = new ConfigurationValidator(_logger.Object).Validate("", config); + var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config); var validationMessages = new[] { "Value for InformaticsGateway>storage>bucketName is required.", "Value for InformaticsGateway>storage>bucketName does not conform to Amazon S3 bucket naming requirements." }; Assert.Equal(string.Join(Environment.NewLine, validationMessages), valid.FailureMessage); @@ -124,6 +128,26 @@ public void StorageWithInvalidBucketName() } } + [Fact(DisplayName = "ConfigurationValidator test with inaccessible directory")] + public void StorageWithInaccessbleDirectory() + { + _fileSystem.Setup(p => p.Directory.Exists(It.IsAny())).Returns(true); + _fileSystem.Setup(p => p.File.Create(It.IsAny(), It.IsAny(), It.IsAny())).Throws(new UnauthorizedAccessException("error")); + + var config = MockValidConfiguration(); + config.Storage.TemporaryDataStorage = TemporaryDataStorageLocation.Disk; + config.Storage.BufferStorageRootPath = "/blabla"; + + var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config); + + var validationMessages = new[] { $"Directory `/blabla` specified in `InformaticsGateway>storage>bufferRootPath` is not accessible: error." }; + Assert.Equal(string.Join(Environment.NewLine, validationMessages), valid.FailureMessage); + foreach (var message in validationMessages) + { + _logger.VerifyLogging(message, LogLevel.Error, Times.Once()); + } + } + private static InformaticsGatewayConfiguration MockValidConfiguration() { var config = new InformaticsGatewayConfiguration(); diff --git a/src/Database/PayloadConfiguration.cs b/src/Database/PayloadConfiguration.cs index 171b10ac1..b40c9e7c1 100644 --- a/src/Database/PayloadConfiguration.cs +++ b/src/Database/PayloadConfiguration.cs @@ -57,6 +57,7 @@ public void Configure(EntityTypeBuilder builder) builder.Ignore(j => j.CalledAeTitle); builder.Ignore(j => j.CallingAeTitle); builder.Ignore(j => j.HasTimedOut); + builder.Ignore(j => j.Elapsed); builder.Ignore(j => j.Count); } } diff --git a/src/InformaticsGateway/Common/FileStorageMetadataExtensions.cs b/src/InformaticsGateway/Common/FileStorageMetadataExtensions.cs index dabb80437..27988b2fa 100644 --- a/src/InformaticsGateway/Common/FileStorageMetadataExtensions.cs +++ b/src/InformaticsGateway/Common/FileStorageMetadataExtensions.cs @@ -14,41 +14,92 @@ * limitations under the License. */ -using System.IO; +using System.IO.Abstractions; using System.Text; using System.Threading.Tasks; using Ardalis.GuardClauses; using FellowOakDicom; using Monai.Deploy.InformaticsGateway.Api.Storage; +using Monai.Deploy.InformaticsGateway.Configuration; namespace Monai.Deploy.InformaticsGateway.Common { internal static class FileStorageMetadataExtensions { - public static async Task SetDataStreams(this DicomFileStorageMetadata dicomFileStorageMetadata, DicomFile dicomFile, string dicomJson) + public static async Task SetDataStreams( + this DicomFileStorageMetadata dicomFileStorageMetadata, + DicomFile dicomFile, + string dicomJson, + TemporaryDataStorageLocation storageLocation, + IFileSystem fileSystem = null, + string temporaryStoragePath = "") { Guard.Against.Null(dicomFile, nameof(dicomFile)); Guard.Against.Null(dicomJson, nameof(dicomJson)); // allow empty here - dicomFileStorageMetadata.File.Data = new MemoryStream(); + switch (storageLocation) + { + case TemporaryDataStorageLocation.Disk: + Guard.Against.Null(fileSystem, nameof(fileSystem)); + Guard.Against.NullOrWhiteSpace(temporaryStoragePath, nameof(temporaryStoragePath)); + + var tempFile = fileSystem.Path.Combine(temporaryStoragePath, $@"{System.DateTime.UtcNow.Ticks}.tmp"); + dicomFileStorageMetadata.File.Data = fileSystem.File.Create(tempFile); + break; + default: + dicomFileStorageMetadata.File.Data = new System.IO.MemoryStream(); + break; + } + await dicomFile.SaveAsync(dicomFileStorageMetadata.File.Data).ConfigureAwait(false); - dicomFileStorageMetadata.File.Data.Seek(0, SeekOrigin.Begin); + dicomFileStorageMetadata.File.Data.Seek(0, System.IO.SeekOrigin.Begin); - SetTextStream(dicomFileStorageMetadata.JsonFile, dicomJson); + await SetTextStream(dicomFileStorageMetadata.JsonFile, dicomJson, storageLocation, fileSystem, temporaryStoragePath); } - public static void SetDataStream(this FhirFileStorageMetadata fhirFileStorageMetadata, string json) - => SetTextStream(fhirFileStorageMetadata.File, json); + public static async Task SetDataStream( + this FhirFileStorageMetadata fhirFileStorageMetadata, + string json, + TemporaryDataStorageLocation storageLocation, + IFileSystem fileSystem = null, + string temporaryStoragePath = "") + => await SetTextStream(fhirFileStorageMetadata.File, json, storageLocation, fileSystem, temporaryStoragePath); - public static void SetDataStream(this Hl7FileStorageMetadata hl7FileStorageMetadata, string message) - => SetTextStream(hl7FileStorageMetadata.File, message); + public static async Task SetDataStream( + this Hl7FileStorageMetadata hl7FileStorageMetadata, + string message, + TemporaryDataStorageLocation storageLocation, + IFileSystem fileSystem = null, + string temporaryStoragePath = "") + => await SetTextStream(hl7FileStorageMetadata.File, message, storageLocation, fileSystem, temporaryStoragePath); - private static void SetTextStream(StorageObjectMetadata storageObjectMetadata, string message) + private static async Task SetTextStream( + StorageObjectMetadata storageObjectMetadata, + string message, + TemporaryDataStorageLocation storageLocation, + IFileSystem fileSystem = null, + string temporaryStoragePath = "") { Guard.Against.Null(message, nameof(message)); // allow empty here - storageObjectMetadata.Data = new MemoryStream(Encoding.UTF8.GetBytes(message)); - storageObjectMetadata.Data.Seek(0, SeekOrigin.Begin); + switch (storageLocation) + { + case TemporaryDataStorageLocation.Disk: + Guard.Against.Null(fileSystem, nameof(fileSystem)); + Guard.Against.NullOrWhiteSpace(temporaryStoragePath, nameof(temporaryStoragePath)); + + var tempFile = fileSystem.Path.Combine(temporaryStoragePath, $@"{System.DateTime.UtcNow.Ticks}.tmp"); + var stream = fileSystem.File.Create(tempFile); + var data = Encoding.UTF8.GetBytes(message); + await stream.WriteAsync(data, 0, data.Length); + storageObjectMetadata.Data = stream; + break; + default: + storageObjectMetadata.Data = new System.IO.MemoryStream(Encoding.UTF8.GetBytes(message)); + break; + } + + storageObjectMetadata.Data.Seek(0, System.IO.SeekOrigin.Begin); } } } diff --git a/src/InformaticsGateway/Logging/Log.700.PayloadService.cs b/src/InformaticsGateway/Logging/Log.700.PayloadService.cs index 588669efe..5cb9aaa3f 100644 --- a/src/InformaticsGateway/Logging/Log.700.PayloadService.cs +++ b/src/InformaticsGateway/Logging/Log.700.PayloadService.cs @@ -53,8 +53,8 @@ public static partial class Log [LoggerMessage(EventId = 711, Level = LogLevel.Information, Message = "Publishing workflow request message ID={messageId}...")] public static partial void PublishingWorkflowRequest(this ILogger logger, string messageId); - [LoggerMessage(EventId = 712, Level = LogLevel.Information, Message = "Workflow request published to {queue}, message ID={messageId}.")] - public static partial void WorkflowRequestPublished(this ILogger logger, string queue, string messageId); + [LoggerMessage(EventId = 712, Level = LogLevel.Information, Message = "Workflow request published to {queue}, message ID={messageId}. Payload took {payloadElapsedTime} to complete.")] + public static partial void WorkflowRequestPublished(this ILogger logger, string queue, string messageId, TimeSpan payloadElapsedTime); [LoggerMessage(EventId = 713, Level = LogLevel.Information, Message = "Restoring payloads from database.")] public static partial void StartupRestoreFromDatabase(this ILogger logger); diff --git a/src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs b/src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs index ea39f5d0b..ada4e956f 100644 --- a/src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs +++ b/src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; +using System.IO.Abstractions; using System.Linq; using System.Net.Http; using System.Net.Http.Headers; @@ -55,6 +56,7 @@ internal class DataRetrievalService : IHostedService, IMonaiService, IDisposable private readonly IObjectUploadQueue _uploadQueue; private readonly IPayloadAssembler _payloadAssembler; private readonly IDicomToolkit _dicomToolkit; + private readonly IFileSystem _fileSystem; private bool _disposedValue; public ServiceStatus Status { get; set; } @@ -77,6 +79,7 @@ public DataRetrievalService( _uploadQueue = _rootScope.ServiceProvider.GetService() ?? throw new ServiceNotFoundException(nameof(IObjectUploadQueue)); _payloadAssembler = _rootScope.ServiceProvider.GetService() ?? throw new ServiceNotFoundException(nameof(IPayloadAssembler)); _dicomToolkit = _rootScope.ServiceProvider.GetService() ?? throw new ServiceNotFoundException(nameof(IDicomToolkit)); + _fileSystem = _rootScope.ServiceProvider.GetService() ?? throw new ServiceNotFoundException(nameof(IFileSystem)); } public Task StartAsync(CancellationToken cancellationToken) @@ -330,7 +333,7 @@ private async Task RetrieveFhirResource(string transactionId, HttpClient h } var fhirFile = new FhirFileStorageMetadata(transactionId, resource.Type, resource.Id, fhirFormat); - fhirFile.SetDataStream(json); + await fhirFile.SetDataStream(json, _options.Value.Storage.TemporaryDataStorage, _fileSystem, _options.Value.Storage.BufferStorageRootPath); retrievedResources.Add(fhirFile.Id, fhirFile); return true; } @@ -509,7 +512,7 @@ private async Task RetrieveInstances(string transactionId, IDicomWebClient dicom } var dicomFileStorageMetadata = SaveFile(transactionId, file, uids); - await dicomFileStorageMetadata.SetDataStreams(file, file.ToJson(_options.Value.Dicom.WriteDicomJson, _options.Value.Dicom.ValidateDicomOnSerialization)).ConfigureAwait(false); + await dicomFileStorageMetadata.SetDataStreams(file, file.ToJson(_options.Value.Dicom.WriteDicomJson, _options.Value.Dicom.ValidateDicomOnSerialization), _options.Value.Storage.TemporaryDataStorage, _fileSystem, _options.Value.Storage.BufferStorageRootPath).ConfigureAwait(false); retrievedInstance.Add(uids.Identifier, dicomFileStorageMetadata); count++; } @@ -539,7 +542,7 @@ private async Task SaveFiles(string transactionId, IAsyncEnumerable f } var dicomFileStorageMetadata = SaveFile(transactionId, file, uids); - await dicomFileStorageMetadata.SetDataStreams(file, file.ToJson(_options.Value.Dicom.WriteDicomJson, _options.Value.Dicom.ValidateDicomOnSerialization)).ConfigureAwait(false); + await dicomFileStorageMetadata.SetDataStreams(file, file.ToJson(_options.Value.Dicom.WriteDicomJson, _options.Value.Dicom.ValidateDicomOnSerialization), _options.Value.Storage.TemporaryDataStorage, _fileSystem, _options.Value.Storage.BufferStorageRootPath).ConfigureAwait(false); retrievedInstance.Add(uids.Identifier, dicomFileStorageMetadata); } } diff --git a/src/InformaticsGateway/Services/Connectors/PayloadNotificationActionHandler.cs b/src/InformaticsGateway/Services/Connectors/PayloadNotificationActionHandler.cs index c45c1cbc4..4a50c3fd7 100644 --- a/src/InformaticsGateway/Services/Connectors/PayloadNotificationActionHandler.cs +++ b/src/InformaticsGateway/Services/Connectors/PayloadNotificationActionHandler.cs @@ -138,7 +138,7 @@ await _messageBrokerPublisherService.Publish( _options.Value.Messaging.Topics.WorkflowRequest, message.ToMessage()).ConfigureAwait(false); - _logger.WorkflowRequestPublished(_options.Value.Messaging.Topics.WorkflowRequest, message.MessageId); + _logger.WorkflowRequestPublished(_options.Value.Messaging.Topics.WorkflowRequest, message.MessageId, payload.Elapsed); } private async Task UpdatePayloadState(Payload payload) diff --git a/src/InformaticsGateway/Services/DicomWeb/IStreamsWriter.cs b/src/InformaticsGateway/Services/DicomWeb/IStreamsWriter.cs index 35cf29a15..c09943d4c 100644 --- a/src/InformaticsGateway/Services/DicomWeb/IStreamsWriter.cs +++ b/src/InformaticsGateway/Services/DicomWeb/IStreamsWriter.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; using Ardalis.GuardClauses; @@ -43,6 +44,7 @@ internal interface IStreamsWriter internal class StreamsWriter : IStreamsWriter { private readonly ILogger _logger; + private readonly IFileSystem _fileSystem; private readonly IObjectUploadQueue _uploadQueue; private readonly IDicomToolkit _dicomToolkit; private readonly IPayloadAssembler _payloadAssembler; @@ -56,13 +58,15 @@ public StreamsWriter( IDicomToolkit dicomToolkit, IPayloadAssembler payloadAssembler, IOptions configuration, - ILogger logger) + ILogger logger, + IFileSystem fileSystem) { _uploadQueue = fileStore ?? throw new ArgumentNullException(nameof(fileStore)); _dicomToolkit = dicomToolkit ?? throw new ArgumentNullException(nameof(dicomToolkit)); _payloadAssembler = payloadAssembler ?? throw new ArgumentNullException(nameof(payloadAssembler)); _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); _resultDicomDataset = new DicomDataset(); _failureCount = 0; _storedCount = 0; @@ -165,7 +169,7 @@ private async Task SaveInstance(Stream stream, string studyInstanceUid, string w dicomInfo.SetWorkflows(workflowName); } - await dicomInfo.SetDataStreams(dicomFile, dicomFile.ToJson(_configuration.Value.Dicom.WriteDicomJson, _configuration.Value.Dicom.ValidateDicomOnSerialization)).ConfigureAwait(false); + await dicomInfo.SetDataStreams(dicomFile, dicomFile.ToJson(_configuration.Value.Dicom.WriteDicomJson, _configuration.Value.Dicom.ValidateDicomOnSerialization), _configuration.Value.Storage.TemporaryDataStorage, _fileSystem, _configuration.Value.Storage.BufferStorageRootPath).ConfigureAwait(false); _uploadQueue.Queue(dicomInfo); // for DICOMweb, use correlation ID as the grouping key diff --git a/src/InformaticsGateway/Services/Fhir/FhirJsonReader.cs b/src/InformaticsGateway/Services/Fhir/FhirJsonReader.cs index 9b8ab8b99..2f56dffb5 100644 --- a/src/InformaticsGateway/Services/Fhir/FhirJsonReader.cs +++ b/src/InformaticsGateway/Services/Fhir/FhirJsonReader.cs @@ -16,6 +16,7 @@ using System; using System.IO; +using System.IO.Abstractions; using System.Text.Json; using System.Text.Json.Nodes; using System.Threading; @@ -23,9 +24,11 @@ using Ardalis.GuardClauses; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.Net.Http.Headers; using Monai.Deploy.InformaticsGateway.Api.Storage; using Monai.Deploy.InformaticsGateway.Common; +using Monai.Deploy.InformaticsGateway.Configuration; using Monai.Deploy.InformaticsGateway.Logging; namespace Monai.Deploy.InformaticsGateway.Services.Fhir @@ -33,10 +36,14 @@ namespace Monai.Deploy.InformaticsGateway.Services.Fhir internal class FhirJsonReader : IFHirRequestReader { private readonly ILogger _logger; + private readonly IOptions _options; + private readonly IFileSystem _fileSystem; - public FhirJsonReader(ILogger logger) + public FhirJsonReader(ILogger logger, IOptions options, IFileSystem fileSystem) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); } public async Task GetContentAsync(HttpRequest request, string correlationId, string resourceType, MediaTypeHeaderValue mediaTypeHeaderValue, CancellationToken cancellationToken) @@ -68,7 +75,7 @@ public async Task GetContentAsync(HttpRequest request, string c result.RawData = jsonDoc.ToJsonString(new JsonSerializerOptions { WriteIndented = true }); var fileMetadata = new FhirFileStorageMetadata(correlationId, result.InternalResourceType, resourceId, Api.Rest.FhirStorageFormat.Json); - fileMetadata.SetDataStream(result.RawData); + await fileMetadata.SetDataStream(result.RawData, _options.Value.Storage.TemporaryDataStorage, _fileSystem, _options.Value.Storage.BufferStorageRootPath); result.Metadata = fileMetadata; return result; diff --git a/src/InformaticsGateway/Services/Fhir/FhirService.cs b/src/InformaticsGateway/Services/Fhir/FhirService.cs index 59c3b8cf0..f66dbc830 100644 --- a/src/InformaticsGateway/Services/Fhir/FhirService.cs +++ b/src/InformaticsGateway/Services/Fhir/FhirService.cs @@ -15,6 +15,7 @@ */ using System; +using System.IO.Abstractions; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -41,6 +42,7 @@ internal class FhirService : IFhirService private readonly ILogger _logger; private readonly IPayloadAssembler _payloadAssembler; private readonly IObjectUploadQueue _uploadQueue; + private readonly IFileSystem _fileSystem; public FhirService(IServiceScopeFactory serviceScopeFactory, IOptions configuration) { @@ -51,6 +53,7 @@ public FhirService(IServiceScopeFactory serviceScopeFactory, IOptions>() ?? throw new ServiceNotFoundException(nameof(ILogger)); _payloadAssembler = scope.ServiceProvider.GetService() ?? throw new ServiceNotFoundException(nameof(IPayloadAssembler)); _uploadQueue = scope.ServiceProvider.GetService() ?? throw new ServiceNotFoundException(nameof(IObjectUploadQueue)); + _fileSystem = scope.ServiceProvider.GetService() ?? throw new ServiceNotFoundException(nameof(IFileSystem)); } public async Task StoreAsync(HttpRequest request, string correlationId, string resourceType, CancellationToken cancellationToken) @@ -100,13 +103,13 @@ private IFHirRequestReader GetRequestReader(MediaTypeHeaderValue mediaTypeHeader if (mediaTypeHeaderValue.MediaType.Equals(ContentTypes.ApplicationFhirJson, StringComparison.OrdinalIgnoreCase)) { var logger = scope.ServiceProvider.GetService>() ?? throw new ServiceNotFoundException(nameof(ILogger)); - return new FhirJsonReader(logger); + return new FhirJsonReader(logger, _configuration, _fileSystem); } if (mediaTypeHeaderValue.MediaType.Equals(ContentTypes.ApplicationFhirXml, StringComparison.OrdinalIgnoreCase)) { var logger = scope.ServiceProvider.GetService>() ?? throw new ServiceNotFoundException(nameof(ILogger)); - return new FhirXmlReader(logger); + return new FhirXmlReader(logger, _configuration, _fileSystem); } throw new UnsupportedContentTypeException($"Media type of '{mediaTypeHeaderValue.MediaType}' is not supported."); diff --git a/src/InformaticsGateway/Services/Fhir/FhirXmlReader.cs b/src/InformaticsGateway/Services/Fhir/FhirXmlReader.cs index 07610c6b2..2d975db8a 100644 --- a/src/InformaticsGateway/Services/Fhir/FhirXmlReader.cs +++ b/src/InformaticsGateway/Services/Fhir/FhirXmlReader.cs @@ -16,6 +16,7 @@ using System; using System.IO; +using System.IO.Abstractions; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -23,9 +24,11 @@ using Ardalis.GuardClauses; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.Net.Http.Headers; using Monai.Deploy.InformaticsGateway.Api.Storage; using Monai.Deploy.InformaticsGateway.Common; +using Monai.Deploy.InformaticsGateway.Configuration; using Monai.Deploy.InformaticsGateway.Logging; namespace Monai.Deploy.InformaticsGateway.Services.Fhir @@ -33,10 +36,14 @@ namespace Monai.Deploy.InformaticsGateway.Services.Fhir internal class FhirXmlReader : IFHirRequestReader { private readonly ILogger _logger; + private readonly IOptions _options; + private readonly IFileSystem _fileSystem; - public FhirXmlReader(ILogger logger) + public FhirXmlReader(ILogger logger, IOptions options, IFileSystem fileSystem) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); } public async Task GetContentAsync(HttpRequest request, string correlationId, string resourceType, MediaTypeHeaderValue mediaTypeHeaderValue, CancellationToken cancellationToken) @@ -80,7 +87,7 @@ public async Task GetContentAsync(HttpRequest request, string c result.InternalResourceType = rootNode.Name; var fileMetadata = new FhirFileStorageMetadata(correlationId, result.InternalResourceType, resourceId, Api.Rest.FhirStorageFormat.Xml); - fileMetadata.SetDataStream(result.RawData); + await fileMetadata.SetDataStream(result.RawData, _options.Value.Storage.TemporaryDataStorage, _fileSystem, _options.Value.Storage.BufferStorageRootPath); result.Metadata = fileMetadata; return result; diff --git a/src/InformaticsGateway/Services/HealthLevel7/MllpService.cs b/src/InformaticsGateway/Services/HealthLevel7/MllpService.cs index 3ba770ffd..ba657f8b2 100644 --- a/src/InformaticsGateway/Services/HealthLevel7/MllpService.cs +++ b/src/InformaticsGateway/Services/HealthLevel7/MllpService.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; using Ardalis.GuardClauses; @@ -43,6 +44,7 @@ internal sealed class MllpService : IHostedService, IDisposable, IMonaiService private readonly IMllpClientFactory _mllpClientFactory; private readonly IObjectUploadQueue _uploadQueue; private readonly IPayloadAssembler _payloadAssembler; + private readonly IFileSystem _fileSystem; private readonly IServiceScope _serviceScope; private readonly ILoggerFactory _logginFactory; private readonly ILogger _logger; @@ -79,6 +81,7 @@ public MllpService(IServiceScopeFactory serviceScopeFactory, _mllpClientFactory = _serviceScope.ServiceProvider.GetService() ?? throw new ServiceNotFoundException(nameof(IMllpClientFactory)); _uploadQueue = _serviceScope.ServiceProvider.GetService() ?? throw new ServiceNotFoundException(nameof(IObjectUploadQueue)); _payloadAssembler = _serviceScope.ServiceProvider.GetService() ?? throw new ServiceNotFoundException(nameof(IPayloadAssembler)); + _fileSystem = _serviceScope.ServiceProvider.GetService() ?? throw new ServiceNotFoundException(nameof(IFileSystem)); _activeTasks = new ConcurrentDictionary(); } @@ -158,7 +161,7 @@ private async Task OnDisconnect(IMllpClient client, MllpClientResult result) foreach (var message in result.Messages) { var hl7Fileetadata = new Hl7FileStorageMetadata(client.ClientId.ToString()); - hl7Fileetadata.SetDataStream(message.HL7Message); + await hl7Fileetadata.SetDataStream(message.HL7Message, _configuration.Value.Storage.TemporaryDataStorage, _fileSystem, _configuration.Value.Storage.BufferStorageRootPath); _uploadQueue.Queue(hl7Fileetadata); await _payloadAssembler.Queue(client.ClientId.ToString(), hl7Fileetadata).ConfigureAwait(false); } diff --git a/src/InformaticsGateway/Services/Scp/ApplicationEntityHandler.cs b/src/InformaticsGateway/Services/Scp/ApplicationEntityHandler.cs index 46bec278e..a856c09fd 100644 --- a/src/InformaticsGateway/Services/Scp/ApplicationEntityHandler.cs +++ b/src/InformaticsGateway/Services/Scp/ApplicationEntityHandler.cs @@ -15,12 +15,14 @@ */ using System; +using System.IO.Abstractions; using System.Linq; using System.Threading.Tasks; using Ardalis.GuardClauses; using FellowOakDicom.Network; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Monai.Deploy.InformaticsGateway.Api; using Monai.Deploy.InformaticsGateway.Api.Storage; using Monai.Deploy.InformaticsGateway.Common; @@ -34,11 +36,12 @@ namespace Monai.Deploy.InformaticsGateway.Services.Scp internal class ApplicationEntityHandler : IDisposable, IApplicationEntityHandler { private readonly ILogger _logger; + private readonly IOptions _options; private readonly IServiceScope _serviceScope; private readonly IPayloadAssembler _payloadAssembler; private readonly IObjectUploadQueue _uploadQueue; - + private readonly IFileSystem _fileSystem; private MonaiApplicationEntity _configuration; private DicomJsonOptions _dicomJsonOptions; private bool _validateDicomValueOnJsonSerialization; @@ -46,14 +49,17 @@ internal class ApplicationEntityHandler : IDisposable, IApplicationEntityHandler public ApplicationEntityHandler( IServiceScopeFactory serviceScopeFactory, - ILogger logger) + ILogger logger, + IOptions options) { Guard.Against.Null(serviceScopeFactory, nameof(serviceScopeFactory)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _options = options ?? throw new ArgumentNullException(nameof(options)); _serviceScope = serviceScopeFactory.CreateScope(); _payloadAssembler = _serviceScope.ServiceProvider.GetService() ?? throw new ServiceNotFoundException(nameof(IPayloadAssembler)); _uploadQueue = _serviceScope.ServiceProvider.GetService() ?? throw new ServiceNotFoundException(nameof(IObjectUploadQueue)); + _fileSystem = _serviceScope.ServiceProvider.GetService() ?? throw new ServiceNotFoundException(nameof(IFileSystem)); } public void Configure(MonaiApplicationEntity monaiApplicationEntity, DicomJsonOptions dicomJsonOptions, bool validateDicomValuesOnJsonSerialization) @@ -95,7 +101,7 @@ public async Task HandleInstanceAsync(DicomCStoreRequest request, string calledA dicomInfo.SetWorkflows(_configuration.Workflows.ToArray()); } - await dicomInfo.SetDataStreams(request.File, request.File.ToJson(_dicomJsonOptions, _validateDicomValueOnJsonSerialization)).ConfigureAwait(false); + await dicomInfo.SetDataStreams(request.File, request.File.ToJson(_dicomJsonOptions, _validateDicomValueOnJsonSerialization), _options.Value.Storage.TemporaryDataStorage, _fileSystem, _options.Value.Storage.BufferStorageRootPath).ConfigureAwait(false); _uploadQueue.Queue(dicomInfo); var dicomTag = FellowOakDicom.DicomTag.Parse(_configuration.Grouping); diff --git a/src/InformaticsGateway/Test/Common/DicomFileStorageMetadataExtensionsTest.cs b/src/InformaticsGateway/Test/Common/DicomFileStorageMetadataExtensionsTest.cs index 3f389a152..a572279f4 100644 --- a/src/InformaticsGateway/Test/Common/DicomFileStorageMetadataExtensionsTest.cs +++ b/src/InformaticsGateway/Test/Common/DicomFileStorageMetadataExtensionsTest.cs @@ -16,6 +16,7 @@ using System; using System.IO; +using System.IO.Abstractions.TestingHelpers; using System.Text; using System.Threading.Tasks; using FellowOakDicom; @@ -31,7 +32,7 @@ namespace Monai.Deploy.InformaticsGateway.Test.Common public class DicomFileStorageMetadataExtensionsTest { [Fact] - public async Task GivenADicomFileStorageMetadata_WhenSetDataStreamsIsCalled_ExpectDataStreamsAreSet() + public async Task GivenADicomFileStorageMetadata_WhenSetDataStreamsIsCalledWithInMemoryStore_ExpectDataStreamsAreSet() { var metadata = new DicomFileStorageMetadata( Guid.NewGuid().ToString(), @@ -42,7 +43,7 @@ public async Task GivenADicomFileStorageMetadata_WhenSetDataStreamsIsCalled_Expe var dicom = InstanceGenerator.GenerateDicomFile(); var json = dicom.ToJson(DicomJsonOptions.Complete, false); - await metadata.SetDataStreams(dicom, json).ConfigureAwait(false); + await metadata.SetDataStreams(dicom, json, TemporaryDataStorageLocation.Memory).ConfigureAwait(false); Assert.NotNull(metadata.File.Data); Assert.NotNull(metadata.JsonFile.Data); @@ -59,7 +60,38 @@ public async Task GivenADicomFileStorageMetadata_WhenSetDataStreamsIsCalled_Expe } [Fact] - public async Task GivenADicomFileStorageMetadataWithInvalidDSValue_WhenSetDataStreamsIsCalledWithValidation_ThrowsFormatException() + public async Task GivenADicomFileStorageMetadata_WhenSetDataStreamsIsCalledWithDiskStore_ExpectDataStreamsAreSet() + { + var fileSystem = new MockFileSystem(); + fileSystem.AddDirectory("/temp"); + + var metadata = new DicomFileStorageMetadata( + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString()); + + var dicom = InstanceGenerator.GenerateDicomFile(); + var json = dicom.ToJson(DicomJsonOptions.Complete, false); + await metadata.SetDataStreams(dicom, json, TemporaryDataStorageLocation.Disk, fileSystem, "/temp").ConfigureAwait(false); + + Assert.NotNull(metadata.File.Data); + Assert.NotNull(metadata.JsonFile.Data); + + var ms = new MemoryStream(); + await dicom.SaveAsync(ms).ConfigureAwait(false); + Assert.Equal(ms.ToArray(), (metadata.File.Data as MemoryStream).ToArray()); + + var jsonFromStream = Encoding.UTF8.GetString((metadata.JsonFile.Data as MemoryStream).ToArray()); + Assert.Equal(json.Trim(), jsonFromStream.Trim()); + + var dicomFileFromJson = DicomJson.ConvertJsonToDicom(json); + Assert.Equal(dicom.Dataset, dicomFileFromJson); + } + + [Fact] + public void GivenADicomFileStorageMetadataWithInvalidDSValue_WhenSetDataStreamsIsCalledWithValidation_ThrowsFormatException() { var metadata = new DicomFileStorageMetadata( Guid.NewGuid().ToString(), @@ -94,7 +126,7 @@ public async Task GivenADicomFileStorageMetadataWithInvalidDSValue_WhenSetDataSt dicom.Dataset.Add(DicomTag.PixelSpacing, "0.68300002813334234392234", "0.2354257587243524352345"); var json = dicom.ToJson(DicomJsonOptions.Complete, false); - await metadata.SetDataStreams(dicom, json).ConfigureAwait(false); + await metadata.SetDataStreams(dicom, json, TemporaryDataStorageLocation.Memory).ConfigureAwait(false); Assert.NotNull(metadata.File.Data); Assert.NotNull(metadata.JsonFile.Data); diff --git a/src/InformaticsGateway/Test/Services/Connectors/DataRetrievalServiceTest.cs b/src/InformaticsGateway/Test/Services/Connectors/DataRetrievalServiceTest.cs index bb2f0fc69..8b1751733 100644 --- a/src/InformaticsGateway/Test/Services/Connectors/DataRetrievalServiceTest.cs +++ b/src/InformaticsGateway/Test/Services/Connectors/DataRetrievalServiceTest.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Abstractions; using System.Net.Http; using System.Net.Http.Headers; using System.Text.Json; @@ -54,6 +55,7 @@ public class DataRetrievalServiceTest private readonly Mock _uploadQueue; private readonly Mock _payloadAssembler; private readonly Mock _dicomToolkit; + private readonly Mock _fileSystem; private readonly Mock _inferenceRequestStore; private readonly Mock> _loggerDicomWebClient; @@ -74,6 +76,7 @@ public DataRetrievalServiceTest() _serviceScopeFactory = new Mock(); _uploadQueue = new Mock(); _dicomToolkit = new Mock(); + _fileSystem = new Mock(); _options = Options.Create(new InformaticsGatewayConfiguration()); _serviceScope = new Mock(); @@ -90,6 +93,7 @@ public DataRetrievalServiceTest() services.AddScoped(p => _payloadAssembler.Object); services.AddScoped(p => _dicomToolkit.Object); services.AddScoped(p => _inferenceRequestStore.Object); + services.AddScoped(p => _fileSystem.Object); _serviceProvider = services.BuildServiceProvider(); _serviceScopeFactory.Setup(p => p.CreateScope()).Returns(_serviceScope.Object); diff --git a/src/InformaticsGateway/Test/Services/DicomWeb/StreamsWriterTest.cs b/src/InformaticsGateway/Test/Services/DicomWeb/StreamsWriterTest.cs index c6295fc2f..451d0633d 100644 --- a/src/InformaticsGateway/Test/Services/DicomWeb/StreamsWriterTest.cs +++ b/src/InformaticsGateway/Test/Services/DicomWeb/StreamsWriterTest.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Abstractions.TestingHelpers; using System.Threading.Tasks; using FellowOakDicom; using FellowOakDicom.Network; @@ -38,6 +39,7 @@ namespace Monai.Deploy.InformaticsGateway.Test.Services.DicomWeb public class StreamsWriterTest { private readonly Mock> _logger; + private readonly MockFileSystem _fileSystem; private readonly Mock _uploadQueue; private readonly Mock _dicomToolkit; private readonly Mock _payloadAssembler; @@ -50,17 +52,19 @@ public StreamsWriterTest() _payloadAssembler = new Mock(); _configuration = Options.Create(new InformaticsGatewayConfiguration()); _logger = new Mock>(); + _fileSystem = new MockFileSystem(); } [Fact] public void GivenAStreamsWriter_WhenInitialized_ExpectParametersToBeValidated() { - Assert.Throws(() => new StreamsWriter(null, null, null, null, null)); - Assert.Throws(() => new StreamsWriter(_uploadQueue.Object, null, null, null, null)); - Assert.Throws(() => new StreamsWriter(_uploadQueue.Object, _dicomToolkit.Object, null, null, null)); - Assert.Throws(() => new StreamsWriter(_uploadQueue.Object, _dicomToolkit.Object, _payloadAssembler.Object, null, null)); - Assert.Throws(() => new StreamsWriter(_uploadQueue.Object, _dicomToolkit.Object, _payloadAssembler.Object, _configuration, null)); - var exception = Record.Exception(() => new StreamsWriter(_uploadQueue.Object, _dicomToolkit.Object, _payloadAssembler.Object, _configuration, _logger.Object)); + Assert.Throws(() => new StreamsWriter(null, null, null, null, null, null)); + Assert.Throws(() => new StreamsWriter(_uploadQueue.Object, null, null, null, null, null)); + Assert.Throws(() => new StreamsWriter(_uploadQueue.Object, _dicomToolkit.Object, null, null, null, null)); + Assert.Throws(() => new StreamsWriter(_uploadQueue.Object, _dicomToolkit.Object, _payloadAssembler.Object, null, null, null)); + Assert.Throws(() => new StreamsWriter(_uploadQueue.Object, _dicomToolkit.Object, _payloadAssembler.Object, _configuration, null, null)); + Assert.Throws(() => new StreamsWriter(_uploadQueue.Object, _dicomToolkit.Object, _payloadAssembler.Object, _configuration, _logger.Object, null)); + var exception = Record.Exception(() => new StreamsWriter(_uploadQueue.Object, _dicomToolkit.Object, _payloadAssembler.Object, _configuration, _logger.Object, _fileSystem)); Assert.Null(exception); } @@ -72,7 +76,7 @@ public async Task GivenAHttpStream_WhenFailedToOpenAsDicomInstance_ExpectStatus4 .Throws(new Exception("error")); var studyInstanceUid = DicomUIDGenerator.GenerateDerivedFromUUID().UID; - var writer = new StreamsWriter(_uploadQueue.Object, _dicomToolkit.Object, _payloadAssembler.Object, _configuration, _logger.Object); + var writer = new StreamsWriter(_uploadQueue.Object, _dicomToolkit.Object, _payloadAssembler.Object, _configuration, _logger.Object, _fileSystem); var streams = GenerateDicomStreams(studyInstanceUid); var result = await writer.Save( @@ -94,7 +98,7 @@ public async Task GivingADicomInstanceWithoutAMatchingStudyInstanceUid_WhenSavei _payloadAssembler.Setup(p => p.Queue(It.IsAny(), It.IsAny(), It.IsAny())); var studyInstanceUid = DicomUIDGenerator.GenerateDerivedFromUUID().UID; - var writer = new StreamsWriter(_uploadQueue.Object, _dicomToolkit.Object, _payloadAssembler.Object, _configuration, _logger.Object); + var writer = new StreamsWriter(_uploadQueue.Object, _dicomToolkit.Object, _payloadAssembler.Object, _configuration, _logger.Object, _fileSystem); var correlationId = Guid.NewGuid().ToString(); var streams = GenerateDicomStreams(studyInstanceUid); @@ -132,7 +136,7 @@ public async Task GivingAValidDicomInstance_WhenSaveingInstance_ExpectInstanceTo _payloadAssembler.Setup(p => p.Queue(It.IsAny(), It.IsAny(), It.IsAny())); var studyInstanceUid = DicomUIDGenerator.GenerateDerivedFromUUID().UID; - var writer = new StreamsWriter(_uploadQueue.Object, _dicomToolkit.Object, _payloadAssembler.Object, _configuration, _logger.Object); + var writer = new StreamsWriter(_uploadQueue.Object, _dicomToolkit.Object, _payloadAssembler.Object, _configuration, _logger.Object, _fileSystem); var correlationId = Guid.NewGuid().ToString(); var streams = GenerateDicomStreams(studyInstanceUid); diff --git a/src/InformaticsGateway/Test/Services/Fhir/FhirJsonReaderTest.cs b/src/InformaticsGateway/Test/Services/Fhir/FhirJsonReaderTest.cs index d224f7a05..7444f4177 100644 --- a/src/InformaticsGateway/Test/Services/Fhir/FhirJsonReaderTest.cs +++ b/src/InformaticsGateway/Test/Services/Fhir/FhirJsonReaderTest.cs @@ -16,11 +16,14 @@ using System; using System.IO; +using System.IO.Abstractions; +using System.IO.Abstractions.TestingHelpers; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.Net.Http.Headers; using Monai.Deploy.InformaticsGateway.Configuration; using Monai.Deploy.InformaticsGateway.Services.Fhir; @@ -31,13 +34,15 @@ namespace Monai.Deploy.InformaticsGateway.Test.Services.Fhir { public class FhirJsonReaderTest { - private readonly InformaticsGatewayConfiguration _config; private readonly Mock> _logger; + private readonly IOptions _options; + private readonly IFileSystem _fileSystem; public FhirJsonReaderTest() { - _config = new InformaticsGatewayConfiguration(); _logger = new Mock>(); + _options = Options.Create(new InformaticsGatewayConfiguration()); + _fileSystem = new MockFileSystem(); } [Fact] @@ -46,7 +51,7 @@ public async Task GetContentAsync_WhenCalled_EnsuresArgumentsAreValid() var request = new Mock(); var correlationId = Guid.NewGuid().ToString(); var resourceType = "Patient"; - var reader = new FhirJsonReader(_logger.Object); + var reader = new FhirJsonReader(_logger.Object, _options, _fileSystem); await Assert.ThrowsAsync(async () => await reader.GetContentAsync(null, null, null, null, CancellationToken.None)); await Assert.ThrowsAsync(async () => await reader.GetContentAsync(request.Object, null, null, null, CancellationToken.None)); @@ -62,7 +67,7 @@ public async Task GetContentAsync_WhenCalledWithEmptyContent_ThrowsException() var correlationId = Guid.NewGuid().ToString(); var resourceType = "Patient"; var contentType = new MediaTypeHeaderValue(ContentTypes.ApplicationFhirJson); - var reader = new FhirJsonReader(_logger.Object); + var reader = new FhirJsonReader(_logger.Object, _options, _fileSystem); await Assert.ThrowsAsync(async () => { @@ -83,7 +88,7 @@ public async Task GetContentAsync_WhenCalledWithNonXmlContent_ThrowsException() var correlationId = Guid.NewGuid().ToString(); var resourceType = "Patient"; var contentType = new MediaTypeHeaderValue(ContentTypes.ApplicationFhirJson); - var reader = new FhirJsonReader(_logger.Object); + var reader = new FhirJsonReader(_logger.Object, _options, _fileSystem); await Assert.ThrowsAnyAsync(async () => { @@ -106,7 +111,7 @@ public async Task GetContentAsync_WhenCalledWithNoId_ReturnsOriginalWithId(strin var correlationId = Guid.NewGuid().ToString(); var resourceType = "Patient"; var contentType = new MediaTypeHeaderValue(ContentTypes.ApplicationFhirJson); - var reader = new FhirJsonReader(_logger.Object); + var reader = new FhirJsonReader(_logger.Object, _options, _fileSystem); var data = System.Text.Encoding.UTF8.GetBytes(xml); using var stream = new System.IO.MemoryStream(); diff --git a/src/InformaticsGateway/Test/Services/Fhir/FhirServiceTest.cs b/src/InformaticsGateway/Test/Services/Fhir/FhirServiceTest.cs index ccb22751e..422fe6bdf 100644 --- a/src/InformaticsGateway/Test/Services/Fhir/FhirServiceTest.cs +++ b/src/InformaticsGateway/Test/Services/Fhir/FhirServiceTest.cs @@ -16,6 +16,7 @@ using System; using System.IO; +using System.IO.Abstractions; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -43,6 +44,7 @@ public class FhirServiceTest private readonly Mock> _logger; private readonly Mock> _loggerJson; private readonly Mock> _loggerXml; + private readonly Mock _fileSystem; private readonly Mock _payloadAssembler; private readonly Mock _uploadQueue; @@ -62,6 +64,7 @@ public FhirServiceTest() _logger = new Mock>(); _loggerJson = new Mock>(); _loggerXml = new Mock>(); + _fileSystem = new Mock(); _httpRequest = new Mock(); @@ -73,6 +76,7 @@ public FhirServiceTest() services.AddScoped(p => _loggerXml.Object); services.AddScoped(p => _uploadQueue.Object); services.AddScoped(p => _payloadAssembler.Object); + services.AddScoped(p => _fileSystem.Object); _serviceProvider = services.BuildServiceProvider(); _serviceScopeFactory.Setup(p => p.CreateScope()).Returns(_serviceScope.Object); _serviceScope.Setup(p => p.ServiceProvider).Returns(_serviceProvider); diff --git a/src/InformaticsGateway/Test/Services/Fhir/FhirXmlReaderTest.cs b/src/InformaticsGateway/Test/Services/Fhir/FhirXmlReaderTest.cs index 1475e954a..606026b73 100644 --- a/src/InformaticsGateway/Test/Services/Fhir/FhirXmlReaderTest.cs +++ b/src/InformaticsGateway/Test/Services/Fhir/FhirXmlReaderTest.cs @@ -16,12 +16,16 @@ using System; using System.IO; +using System.IO.Abstractions; +using System.IO.Abstractions.TestingHelpers; using System.Threading; using System.Threading.Tasks; using System.Xml; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.Net.Http.Headers; +using Monai.Deploy.InformaticsGateway.Configuration; using Monai.Deploy.InformaticsGateway.Services.Fhir; using Moq; using Xunit; @@ -31,10 +35,15 @@ namespace Monai.Deploy.InformaticsGateway.Test.Services.Fhir public class FhirXmlReaderTest { private readonly Mock> _logger; + private readonly IOptions _options; + private readonly IFileSystem _fileSystem; public FhirXmlReaderTest() { _logger = new Mock>(); + _options = Options.Create(new InformaticsGatewayConfiguration()); + _fileSystem = new MockFileSystem(); + } [Fact] @@ -43,7 +52,7 @@ public async Task GetContentAsync_WhenCalled_EnsuresArgumentsAreValid() var request = new Mock(); var correlationId = Guid.NewGuid().ToString(); var resourceType = "Patient"; - var reader = new FhirXmlReader(_logger.Object); + var reader = new FhirXmlReader(_logger.Object, _options, _fileSystem); await Assert.ThrowsAsync(async () => await reader.GetContentAsync(null, null, null, null, CancellationToken.None)); await Assert.ThrowsAsync(async () => await reader.GetContentAsync(request.Object, null, null, null, CancellationToken.None)); @@ -59,7 +68,7 @@ public async Task GetContentAsync_WhenCalledWithEmptyContent_ThrowsException() var correlationId = Guid.NewGuid().ToString(); var resourceType = "Patient"; var contentType = new MediaTypeHeaderValue(ContentTypes.ApplicationFhirXml); - var reader = new FhirXmlReader(_logger.Object); + var reader = new FhirXmlReader(_logger.Object, _options, _fileSystem); await Assert.ThrowsAsync(async () => { @@ -80,7 +89,7 @@ public async Task GetContentAsync_WhenCalledWithNonXmlContent_ThrowsException() var correlationId = Guid.NewGuid().ToString(); var resourceType = "Patient"; var contentType = new MediaTypeHeaderValue(ContentTypes.ApplicationFhirXml); - var reader = new FhirXmlReader(_logger.Object); + var reader = new FhirXmlReader(_logger.Object, _options, _fileSystem); await Assert.ThrowsAsync(async () => { @@ -100,7 +109,7 @@ public async Task GetContentAsync_WhenCalledWithNonFhirContent_ThrowsException() var correlationId = Guid.NewGuid().ToString(); var resourceType = "Patient"; var contentType = new MediaTypeHeaderValue(ContentTypes.ApplicationFhirXml); - var reader = new FhirXmlReader(_logger.Object); + var reader = new FhirXmlReader(_logger.Object, _options, _fileSystem); await Assert.ThrowsAsync(async () => { @@ -123,7 +132,7 @@ public async Task GetContentAsync_WhenCalledWithNoId_ReturnsOriginalWithId(strin var correlationId = Guid.NewGuid().ToString(); var resourceType = "Patient"; var contentType = new MediaTypeHeaderValue(ContentTypes.ApplicationFhirXml); - var reader = new FhirXmlReader(_logger.Object); + var reader = new FhirXmlReader(_logger.Object, _options, _fileSystem); var data = System.Text.Encoding.UTF8.GetBytes(xml); using var stream = new System.IO.MemoryStream(); diff --git a/src/InformaticsGateway/Test/Services/HealthLevel7/MllpServiceTest.cs b/src/InformaticsGateway/Test/Services/HealthLevel7/MllpServiceTest.cs index d4f374425..61a2dc23b 100644 --- a/src/InformaticsGateway/Test/Services/HealthLevel7/MllpServiceTest.cs +++ b/src/InformaticsGateway/Test/Services/HealthLevel7/MllpServiceTest.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; +using System.IO.Abstractions; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -47,7 +48,7 @@ public class MllpServiceTest private readonly Mock _uploadQueue; private readonly Mock _payloadAssembler; private readonly Mock _tcpListener; - + private readonly Mock _fileSystem; private readonly CancellationTokenSource _cancellationTokenSource; private readonly Mock _serviceScope; private readonly Mock> _logger; @@ -64,6 +65,7 @@ public MllpServiceTest() _uploadQueue = new Mock(); _payloadAssembler = new Mock(); _tcpListener = new Mock(); + _fileSystem = new Mock(); _cancellationTokenSource = new CancellationTokenSource(); _serviceScope = new Mock(); @@ -77,6 +79,7 @@ public MllpServiceTest() services.AddScoped(p => _mllpClientFactory.Object); services.AddScoped(p => _uploadQueue.Object); services.AddScoped(p => _payloadAssembler.Object); + services.AddScoped(p => _fileSystem.Object); _serviceProvider = services.BuildServiceProvider(); _serviceScopeFactory.Setup(p => p.CreateScope()).Returns(_serviceScope.Object); _serviceScope.Setup(p => p.ServiceProvider).Returns(_serviceProvider); diff --git a/src/InformaticsGateway/Test/Services/Scp/ApplicationEntityHandlerTest.cs b/src/InformaticsGateway/Test/Services/Scp/ApplicationEntityHandlerTest.cs index 0d7cad791..d81eb7528 100644 --- a/src/InformaticsGateway/Test/Services/Scp/ApplicationEntityHandlerTest.cs +++ b/src/InformaticsGateway/Test/Services/Scp/ApplicationEntityHandlerTest.cs @@ -16,14 +16,17 @@ using System; using System.Collections.Generic; +using System.IO.Abstractions; using System.Threading.Tasks; using FellowOakDicom; using FellowOakDicom.Network; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Monai.Deploy.InformaticsGateway.Api; using Monai.Deploy.InformaticsGateway.Api.Storage; using Monai.Deploy.InformaticsGateway.Common; +using Monai.Deploy.InformaticsGateway.Configuration; using Monai.Deploy.InformaticsGateway.Services.Connectors; using Monai.Deploy.InformaticsGateway.Services.Scp; using Monai.Deploy.InformaticsGateway.Services.Storage; @@ -41,7 +44,8 @@ public class ApplicationEntityHandlerTest private readonly Mock _serviceScope; private readonly Mock _payloadAssembler; private readonly Mock _uploadQueue; - + private readonly IOptions _options; + private readonly Mock _fileSystem; private readonly IServiceProvider _serviceProvider; public ApplicationEntityHandlerTest() @@ -52,10 +56,13 @@ public ApplicationEntityHandlerTest() _payloadAssembler = new Mock(); _uploadQueue = new Mock(); + _options = Options.Create(new InformaticsGatewayConfiguration()); + _fileSystem = new Mock(); var services = new ServiceCollection(); services.AddScoped(p => _payloadAssembler.Object); services.AddScoped(p => _uploadQueue.Object); + services.AddScoped(p => _fileSystem.Object); _serviceProvider = services.BuildServiceProvider(); _serviceScopeFactory.Setup(p => p.CreateScope()).Returns(_serviceScope.Object); _serviceScope.Setup(p => p.ServiceProvider).Returns(_serviceProvider); @@ -66,10 +73,11 @@ public ApplicationEntityHandlerTest() [RetryFact(5, 250)] public void GivenAApplicationEntityHandler_WhenInitialized_ExpectParametersToBeValidated() { - Assert.Throws(() => new ApplicationEntityHandler(null, null)); - Assert.Throws(() => new ApplicationEntityHandler(_serviceScopeFactory.Object, null)); + Assert.Throws(() => new ApplicationEntityHandler(null, null, null)); + Assert.Throws(() => new ApplicationEntityHandler(_serviceScopeFactory.Object, null, null)); + Assert.Throws(() => new ApplicationEntityHandler(_serviceScopeFactory.Object, _logger.Object, null)); - _ = new ApplicationEntityHandler(_serviceScopeFactory.Object, _logger.Object); + _ = new ApplicationEntityHandler(_serviceScopeFactory.Object, _logger.Object, _options); } [RetryFact(5, 250)] @@ -83,7 +91,7 @@ public async Task GivenAApplicationEntityHandler_WhenHandleInstanceAsyncIsCalled IgnoredSopClasses = new List { DicomUID.SecondaryCaptureImageStorage.UID } }; - var handler = new ApplicationEntityHandler(_serviceScopeFactory.Object, _logger.Object); + var handler = new ApplicationEntityHandler(_serviceScopeFactory.Object, _logger.Object, _options); var request = GenerateRequest(); var dicomToolkit = new DicomToolkit(); @@ -103,7 +111,7 @@ public async Task GivenACStoreRequest_WhenTheSopClassIsInTheIgnoreList_ExpectIns IgnoredSopClasses = new List { DicomUID.SecondaryCaptureImageStorage.UID } }; - var handler = new ApplicationEntityHandler(_serviceScopeFactory.Object, _logger.Object); + var handler = new ApplicationEntityHandler(_serviceScopeFactory.Object, _logger.Object, _options); handler.Configure(aet, Configuration.DicomJsonOptions.Complete, true); var request = GenerateRequest(); @@ -127,7 +135,7 @@ public async Task GivenACStoreRequest_WhenTheSopClassIsNotInTheAllowedList_Expec AllowedSopClasses = new List { DicomUID.UltrasoundImageStorage.UID } }; - var handler = new ApplicationEntityHandler(_serviceScopeFactory.Object, _logger.Object); + var handler = new ApplicationEntityHandler(_serviceScopeFactory.Object, _logger.Object, _options); handler.Configure(aet, Configuration.DicomJsonOptions.Complete, true); var request = GenerateRequest(); @@ -150,7 +158,7 @@ public async Task GivenACStoreRequest_WhenHandleInstanceAsyncIsCalled_ExpectADic Workflows = new List() { "AppA", "AppB", Guid.NewGuid().ToString() } }; - var handler = new ApplicationEntityHandler(_serviceScopeFactory.Object, _logger.Object); + var handler = new ApplicationEntityHandler(_serviceScopeFactory.Object, _logger.Object, _options); handler.Configure(aet, Configuration.DicomJsonOptions.Complete, true); var request = GenerateRequest(); diff --git a/src/InformaticsGateway/Test/Services/Storage/ObjectUploadServiceTest.cs b/src/InformaticsGateway/Test/Services/Storage/ObjectUploadServiceTest.cs index 5a7e445c3..d844eaec8 100644 --- a/src/InformaticsGateway/Test/Services/Storage/ObjectUploadServiceTest.cs +++ b/src/InformaticsGateway/Test/Services/Storage/ObjectUploadServiceTest.cs @@ -45,7 +45,6 @@ public class ObjectUploadServiceTest private readonly IObjectUploadQueue _uploadQueue; private readonly Mock _storageService; private readonly Mock _storageMetadataWrapperRepository; - private readonly CancellationTokenSource _cancellationTokenSource; private readonly ServiceProvider _serviceProvider; private readonly Mock _serviceScope; @@ -124,7 +123,7 @@ public async Task GivenADicomFileStorageMetadata_WhenQueuedForUpload_ExpectTwoFi } [Fact] - public void GivenAFhirFileStorageMetadata_WhenQueuedForUpload_ExpectSingleFileToBeUploaded() + public async Task GivenAFhirFileStorageMetadata_WhenQueuedForUpload_ExpectSingleFileToBeUploaded() { var countdownEvent = new CountdownEvent(1); _storageService.Setup(p => p.PutObjectAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny())) @@ -137,7 +136,7 @@ public void GivenAFhirFileStorageMetadata_WhenQueuedForUpload_ExpectSingleFileTo Assert.Equal(ServiceStatus.Running, svc.Status); - var file = GenerateFhirFileStorageMetadata(); + var file = await GenerateFhirFileStorageMetadata(); _uploadQueue.Queue(file); Assert.True(countdownEvent.Wait(TimeSpan.FromSeconds(3))); @@ -145,11 +144,11 @@ public void GivenAFhirFileStorageMetadata_WhenQueuedForUpload_ExpectSingleFileTo _storageService.Verify(p => p.PutObjectAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny()), Times.Once()); } - private FhirFileStorageMetadata GenerateFhirFileStorageMetadata() + private async Task GenerateFhirFileStorageMetadata() { var file = new FhirFileStorageMetadata(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), FhirStorageFormat.Json); - file.SetDataStream("[]"); + await file.SetDataStream("[]", TemporaryDataStorageLocation.Memory); return file; } @@ -169,7 +168,7 @@ private async Task GenerateDicomFileStorageMetadata() { DicomTag.SOPClassUID, DicomUID.SecondaryCaptureImageStorage.UID } }; var dicomFile = new DicomFile(dataset); - await file.SetDataStreams(dicomFile, "[]"); + await file.SetDataStreams(dicomFile, "[]", TemporaryDataStorageLocation.Memory); return file; } }