Ability to switch temporary storage to use either memory or disk (#166)

* Ability to switch temporary storage to use either memory or disk

Signed-off-by: Victor Chang <vicchang@nvidia.com>

* Fix configuration name for temp data storage.

Signed-off-by: Victor Chang <vicchang@nvidia.com>

* Validate storage configurations based on temp storage location

Signed-off-by: Victor Chang <vicchang@nvidia.com>

* Log time for a payload to complete end-to-end within the service.

Signed-off-by: Victor Chang <vicchang@nvidia.com>

Signed-off-by: Victor Chang <vicchang@nvidia.com>

src/Api/Storage/Payload.cs

@@ -66,6 +66,8 @@ public enum PayloadState
 
         public bool HasTimedOut { get => ElapsedTime().TotalSeconds >= Timeout; }
 
+        public TimeSpan Elapsed { get { return DateTime.UtcNow.Subtract(DateTimeCreated); } }
+
         public string CallingAeTitle { get => Files.OfType<DicomFileStorageMetadata>().Select(p => p.CallingAeTitle).FirstOrDefault(); }
 
         public string CalledAeTitle { get => Files.OfType<DicomFileStorageMetadata>().Select(p => p.CalledAeTitle).FirstOrDefault(); }

src/Api/Storage/StorageObjectMetadata.cs

@@ -16,6 +16,7 @@
 
 using System;
 using System.IO;
+using System.Runtime;
 using System.Text.Json.Serialization;
 using Ardalis.GuardClauses;
 
@@ -123,9 +124,27 @@ public void SetUploaded(string bucketName)
 
             if (Data is not null && Data.CanSeek)
             {
-                Data.Close();
-                Data.Dispose();
-                Data = null;
+                if (Data is FileStream fileStream)
+                {
+                    var filename = fileStream.Name;
+                    Data.Close();
+                    Data.Dispose();
+                    Data = null;
+                    System.IO.File.Delete(filename);
+                }
+                else // MemoryStream
+                {
+                    Data.Close();
+                    Data.Dispose();
+                    Data = null;
+
+                    // When IG stores all received/downloaded data in-memory using MemoryStream, LOH grows tremendously and thus impacts the performance and
+                    //  memory usage. The following makes sure LOH is compacted after the data is uploaded.
+                    GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
+#pragma warning disable S1215 // "GC.Collect" should not be called
+                    GC.Collect();
+#pragma warning restore S1215 // "GC.Collect" should not be called
+                }
             }
         }
 

src/Configuration/ConfigurationValidator.cs

@@ -17,6 +17,8 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO;
+using System.IO.Abstractions;
 using System.Text.RegularExpressions;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
@@ -29,16 +31,18 @@ namespace Monai.Deploy.InformaticsGateway.Configuration
     public class ConfigurationValidator : IValidateOptions<InformaticsGatewayConfiguration>
     {
         private readonly ILogger<ConfigurationValidator> _logger;
+        private readonly IFileSystem _fileSystem;
         private readonly List<string> _validationErrors;
 
         /// <summary>
         /// Initializes an instance of the <see cref="ConfigurationValidator"/> class.
         /// </summary>
         /// <param name="configuration">InformaticsGatewayConfiguration to be validated</param>
         /// <param name="logger">Logger to be used by ConfigurationValidator</param>
-        public ConfigurationValidator(ILogger<ConfigurationValidator> logger)
+        public ConfigurationValidator(ILogger<ConfigurationValidator> logger, IFileSystem fileSystem)
         {
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+            _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
             _validationErrors = new List<string>();
         }
 
@@ -82,6 +86,38 @@ private bool IsStorageValid(StorageConfiguration storage)
             valid &= IsValueInRange("InformaticsGateway>storage>reserveSpaceGB", 1, 999, storage.ReserveSpaceGB);
             valid &= IsValueInRange("InformaticsGateway>storage>payloadProcessThreads", 1, 128, storage.PayloadProcessThreads);
             valid &= IsValueInRange("InformaticsGateway>storage>concurrentUploads", 1, 128, storage.ConcurrentUploads);
+            valid &= IsValueInRange("InformaticsGateway>storage>memoryThreshold", 1, int.MaxValue, storage.MemoryThreshold);
+
+            if (storage.TemporaryDataStorage == TemporaryDataStorageLocation.Disk)
+            {
+                valid &= IsNotNullOrWhiteSpace("InformaticsGateway>storage>bufferRootPath", storage.BufferStorageRootPath);
+                valid &= IsValidDirectory("InformaticsGateway>storage>bufferRootPath", storage.BufferStorageRootPath);
+                valid &= IsValueInRange("InformaticsGateway>storage>bufferSize", 1, int.MaxValue, storage.BufferSize);
+            }
+
+            return valid;
+        }
+
+        private bool IsValidDirectory(string source, string directory)
+        {
+            var valid = true;
+            try
+            {
+                if (!_fileSystem.Directory.Exists(directory))
+                {
+                    valid = false;
+                    _validationErrors.Add($"Directory `{directory}` specified in `{source}` does not exist.");
+                }
+                else
+                {
+                    using var _ = _fileSystem.File.Create(Path.Combine(directory, Path.GetRandomFileName()), 1, FileOptions.DeleteOnClose);
+                }
+            }
+            catch (Exception ex)
+            {
+                valid = false;
+                _validationErrors.Add($"Directory `{directory}` specified in `{source}` is not accessible: {ex.Message}.");
+            }
             return valid;
         }
 

src/Configuration/StorageConfiguration.cs

@@ -22,13 +22,28 @@ namespace Monai.Deploy.InformaticsGateway.Configuration
 {
     public class StorageConfiguration : StorageServiceConfiguration
     {
+        /// <summary>
+        /// Gets or sets whether to store temporary data in <c>Memory</c> or on <c>Disk</c>.
+        /// Defaults to <c>Memory</c>.
+        /// </summary>
+        [ConfigurationKeyName("tempStorageLocation")]
+        public TemporaryDataStorageLocation TemporaryDataStorage { get; set; } = TemporaryDataStorageLocation.Memory;
+
         /// <summary>
         /// Gets or sets the path used for buffering incoming data.
         /// Defaults to <c>./temp</c>.
         /// </summary>
         [ConfigurationKeyName("bufferRootPath")]
         public string BufferStorageRootPath { get; set; } = "./temp";
 
+        /// <summary>
+        /// Gets or sets the number of bytes buffered for reads and writes to the temporary file.
+        /// Defaults to <c>128000</c>.
+        /// </summary>
+        [ConfigurationKeyName("bufferSize")]
+        public int BufferSize { get; set; } = 128000;
+
+
         /// <summary>
         /// Gets or set the maximum memory buffer size in bytes with default to 30MiB.
         /// </summary>

src/Configuration/TemporaryDataStorageLocation.cs

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2022 MONAI Consortium
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Monai.Deploy.InformaticsGateway.Configuration
+{
+    public enum TemporaryDataStorageLocation
+    {
+        Memory,
+        Disk
+    }
+}

src/Configuration/Test/ConfigurationValidatorTest.cs

@@ -15,6 +15,8 @@
  */
 
 using System;
+using System.IO;
+using System.IO.Abstractions;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Monai.Deploy.InformaticsGateway.SharedTest;
@@ -26,17 +28,19 @@ namespace Monai.Deploy.InformaticsGateway.Configuration.Test
     public class ConfigurationValidatorTest
     {
         private readonly Mock<ILogger<ConfigurationValidator>> _logger;
+        private readonly Mock<IFileSystem> _fileSystem;
 
         public ConfigurationValidatorTest()
         {
             _logger = new Mock<ILogger<ConfigurationValidator>>();
+            _fileSystem = new Mock<IFileSystem>();
         }
 
         [Fact(DisplayName = "ConfigurationValidator test with all valid settings")]
         public void AllValid()
         {
             var config = MockValidConfiguration();
-            var valid = new ConfigurationValidator(_logger.Object).Validate("", config);
+            var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
             Assert.True(valid == ValidateOptionsResult.Success);
         }
 
@@ -46,7 +50,7 @@ public void InvalidScpPort()
             var config = MockValidConfiguration();
             config.Dicom.Scp.Port = Int32.MaxValue;
 
-            var valid = new ConfigurationValidator(_logger.Object).Validate("", config);
+            var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
 
             var validationMessage = $"Invalid port number '{Int32.MaxValue}' specified for InformaticsGateway>dicom>scp>port.";
             Assert.Equal(validationMessage, valid.FailureMessage);
@@ -59,7 +63,7 @@ public void InvalidScpMaxAssociations()
             var config = MockValidConfiguration();
             config.Dicom.Scp.MaximumNumberOfAssociations = 0;
 
-            var valid = new ConfigurationValidator(_logger.Object).Validate("", config);
+            var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
 
             var validationMessage = $"Value of InformaticsGateway>dicom>scp>max-associations must be between {1} and {1000}.";
             Assert.Equal(validationMessage, valid.FailureMessage);
@@ -72,7 +76,7 @@ public void StorageWithInvalidWatermark()
             var config = MockValidConfiguration();
             config.Storage.Watermark = 1000;
 
-            var valid = new ConfigurationValidator(_logger.Object).Validate("", config);
+            var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
 
             var validationMessage = "Value of InformaticsGateway>storage>watermark must be between 1 and 100.";
             Assert.Equal(validationMessage, valid.FailureMessage);
@@ -85,7 +89,7 @@ public void StorageWithInvalidReservedSpace()
             var config = MockValidConfiguration();
             config.Storage.ReserveSpaceGB = 9999;
 
-            var valid = new ConfigurationValidator(_logger.Object).Validate("", config);
+            var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
 
             var validationMessage = "Value of InformaticsGateway>storage>reserveSpaceGB must be between 1 and 999.";
             Assert.Equal(validationMessage, valid.FailureMessage);
@@ -98,7 +102,7 @@ public void StorageWithInvalidTemporaryBucketName()
             var config = MockValidConfiguration();
             config.Storage.TemporaryStorageBucket = " ";
 
-            var valid = new ConfigurationValidator(_logger.Object).Validate("", config);
+            var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
 
             var validationMessages = new[] { "Value for InformaticsGateway>storage>temporaryBucketName is required.", "Value for InformaticsGateway>storage>temporaryBucketName does not conform to Amazon S3 bucket naming requirements." };
             Assert.Equal(string.Join(Environment.NewLine, validationMessages), valid.FailureMessage);
@@ -114,7 +118,7 @@ public void StorageWithInvalidBucketName()
             var config = MockValidConfiguration();
             config.Storage.StorageServiceBucketName = "";
 
-            var valid = new ConfigurationValidator(_logger.Object).Validate("", config);
+            var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
 
             var validationMessages = new[] { "Value for InformaticsGateway>storage>bucketName is required.", "Value for InformaticsGateway>storage>bucketName does not conform to Amazon S3 bucket naming requirements." };
             Assert.Equal(string.Join(Environment.NewLine, validationMessages), valid.FailureMessage);
@@ -124,6 +128,26 @@ public void StorageWithInvalidBucketName()
             }
         }
 
+        [Fact(DisplayName = "ConfigurationValidator test with inaccessible directory")]
+        public void StorageWithInaccessbleDirectory()
+        {
+            _fileSystem.Setup(p => p.Directory.Exists(It.IsAny<string>())).Returns(true);
+            _fileSystem.Setup(p => p.File.Create(It.IsAny<string>(), It.IsAny<int>(), It.IsAny<FileOptions>())).Throws(new UnauthorizedAccessException("error"));
+
+            var config = MockValidConfiguration();
+            config.Storage.TemporaryDataStorage = TemporaryDataStorageLocation.Disk;
+            config.Storage.BufferStorageRootPath = "/blabla";
+
+            var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
+
+            var validationMessages = new[] { $"Directory `/blabla` specified in `InformaticsGateway>storage>bufferRootPath` is not accessible: error." };
+            Assert.Equal(string.Join(Environment.NewLine, validationMessages), valid.FailureMessage);
+            foreach (var message in validationMessages)
+            {
+                _logger.VerifyLogging(message, LogLevel.Error, Times.Once());
+            }
+        }
+
         private static InformaticsGatewayConfiguration MockValidConfiguration()
         {
             var config = new InformaticsGatewayConfiguration();

src/Database/PayloadConfiguration.cs

@@ -57,6 +57,7 @@ public void Configure(EntityTypeBuilder<Payload> builder)
             builder.Ignore(j => j.CalledAeTitle);
             builder.Ignore(j => j.CallingAeTitle);
             builder.Ignore(j => j.HasTimedOut);
+            builder.Ignore(j => j.Elapsed);
             builder.Ignore(j => j.Count);
         }
     }

src/InformaticsGateway/Common/FileStorageMetadataExtensions.cs

@@ -14,41 +14,92 @@
  * limitations under the License.
  */
 
-using System.IO;
+using System.IO.Abstractions;
 using System.Text;
 using System.Threading.Tasks;
 using Ardalis.GuardClauses;
 using FellowOakDicom;
 using Monai.Deploy.InformaticsGateway.Api.Storage;
+using Monai.Deploy.InformaticsGateway.Configuration;
 
 namespace Monai.Deploy.InformaticsGateway.Common
 {
     internal static class FileStorageMetadataExtensions
     {
-        public static async Task SetDataStreams(this DicomFileStorageMetadata dicomFileStorageMetadata, DicomFile dicomFile, string dicomJson)
+        public static async Task SetDataStreams(
+            this DicomFileStorageMetadata dicomFileStorageMetadata,
+            DicomFile dicomFile,
+            string dicomJson,
+            TemporaryDataStorageLocation storageLocation,
+            IFileSystem fileSystem = null,
+            string temporaryStoragePath = "")
         {
             Guard.Against.Null(dicomFile, nameof(dicomFile));
             Guard.Against.Null(dicomJson, nameof(dicomJson)); // allow empty here
 
-            dicomFileStorageMetadata.File.Data = new MemoryStream();
+            switch (storageLocation)
+            {
+                case TemporaryDataStorageLocation.Disk:
+                    Guard.Against.Null(fileSystem, nameof(fileSystem));
+                    Guard.Against.NullOrWhiteSpace(temporaryStoragePath, nameof(temporaryStoragePath));
+
+                    var tempFile = fileSystem.Path.Combine(temporaryStoragePath, $@"{System.DateTime.UtcNow.Ticks}.tmp");
+                    dicomFileStorageMetadata.File.Data = fileSystem.File.Create(tempFile);
+                    break;
+                default:
+                    dicomFileStorageMetadata.File.Data = new System.IO.MemoryStream();
+                    break;
+            }
+
             await dicomFile.SaveAsync(dicomFileStorageMetadata.File.Data).ConfigureAwait(false);
-            dicomFileStorageMetadata.File.Data.Seek(0, SeekOrigin.Begin);
+            dicomFileStorageMetadata.File.Data.Seek(0, System.IO.SeekOrigin.Begin);
 
-            SetTextStream(dicomFileStorageMetadata.JsonFile, dicomJson);
+            await SetTextStream(dicomFileStorageMetadata.JsonFile, dicomJson, storageLocation, fileSystem, temporaryStoragePath);
         }
 
-        public static void SetDataStream(this FhirFileStorageMetadata fhirFileStorageMetadata, string json)
-            => SetTextStream(fhirFileStorageMetadata.File, json);
+        public static async Task SetDataStream(
+            this FhirFileStorageMetadata fhirFileStorageMetadata,
+            string json,
+            TemporaryDataStorageLocation storageLocation,
+            IFileSystem fileSystem = null,
+            string temporaryStoragePath = "")
+            => await SetTextStream(fhirFileStorageMetadata.File, json, storageLocation, fileSystem, temporaryStoragePath);
 
-        public static void SetDataStream(this Hl7FileStorageMetadata hl7FileStorageMetadata, string message)
-            => SetTextStream(hl7FileStorageMetadata.File, message);
+        public static async Task SetDataStream(
+            this Hl7FileStorageMetadata hl7FileStorageMetadata,
+             string message,
+             TemporaryDataStorageLocation storageLocation,
+            IFileSystem fileSystem = null,
+             string temporaryStoragePath = "")
+            => await SetTextStream(hl7FileStorageMetadata.File, message, storageLocation, fileSystem, temporaryStoragePath);
 
-        private static void SetTextStream(StorageObjectMetadata storageObjectMetadata, string message)
+        private static async Task SetTextStream(
+            StorageObjectMetadata storageObjectMetadata,
+            string message,
+            TemporaryDataStorageLocation storageLocation,
+            IFileSystem fileSystem = null,
+            string temporaryStoragePath = "")
         {
             Guard.Against.Null(message, nameof(message)); // allow empty here
 
-            storageObjectMetadata.Data = new MemoryStream(Encoding.UTF8.GetBytes(message));
-            storageObjectMetadata.Data.Seek(0, SeekOrigin.Begin);
+            switch (storageLocation)
+            {
+                case TemporaryDataStorageLocation.Disk:
+                    Guard.Against.Null(fileSystem, nameof(fileSystem));
+                    Guard.Against.NullOrWhiteSpace(temporaryStoragePath, nameof(temporaryStoragePath));
+
+                    var tempFile = fileSystem.Path.Combine(temporaryStoragePath, $@"{System.DateTime.UtcNow.Ticks}.tmp");
+                    var stream = fileSystem.File.Create(tempFile);
+                    var data = Encoding.UTF8.GetBytes(message);
+                    await stream.WriteAsync(data, 0, data.Length);
+                    storageObjectMetadata.Data = stream;
+                    break;
+                default:
+                    storageObjectMetadata.Data = new System.IO.MemoryStream(Encoding.UTF8.GetBytes(message));
+                    break;
+            }
+
+            storageObjectMetadata.Data.Seek(0, System.IO.SeekOrigin.Begin);
         }
     }
 }