workign on release engine

Rdmp.Core/DataExport/DataRelease/AWSReleaseEngine.cs

@@ -0,0 +1,92 @@
+using Amazon.S3.Model;
+using Rdmp.Core.DataExport.Data;
+using Rdmp.Core.DataExport.DataRelease.Pipeline;
+using Rdmp.Core.DataExport.DataRelease.Potential;
+using Rdmp.Core.ReusableLibraryCode.AWS;
+using Rdmp.Core.ReusableLibraryCode.Progress;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Rdmp.Core.DataExport.DataRelease
+{
+    public class AWSReleaseEngine : ReleaseEngine
+    {
+
+        AWSS3 _s3Helper;
+        S3Bucket _bucket;
+
+        public AWSReleaseEngine(Project project, ReleaseEngineSettings settings, AWSS3 s3Helper, S3Bucket bucket, IDataLoadEventListener listener, ReleaseAudit releaseAudit) : base(project, settings, listener, releaseAudit)
+        {
+            _s3Helper = s3Helper;
+            _bucket = bucket;
+        }
+
+        public override void DoRelease(Dictionary<IExtractionConfiguration, List<ReleasePotential>> toRelease, Dictionary<IExtractionConfiguration, ReleaseEnvironmentPotential> environments, bool isPatch)
+        {
+
+            //base.DoRelease(toRelease, environments, isPatch);   
+            ConfigurationsToRelease = toRelease;
+            var auditFilePath = Path.Combine(Path.GetTempPath(), "contents.txt");
+            using (var sw = PrepareAuditFile(auditFilePath))
+            {
+                ReleaseGlobalFolder();
+                // Audit Global Folder if there are any
+                if (ReleaseAudit.SourceGlobalFolder != null)
+                {
+                    AuditDirectoryCreation(ReleaseAudit.SourceGlobalFolder.FullName, sw, 0);
+
+                    foreach (var fileInfo in ReleaseAudit.SourceGlobalFolder.GetFiles())
+                        AuditFileCreation(fileInfo.Name, sw, 1);
+                }
+
+                //ReleaseAllExtractionConfigurations(toRelease, sw, environments, isPatch);
+                sw.Flush();
+                sw.Close();
+                //_bucket = Task.Run(async () => await _s3Helper.GetBucket(BucketName)).Result;
+
+                Task.Run(async () => await _s3Helper.PutObject(_bucket.BucketName, "contents.txt", auditFilePath));
+                File.Delete(auditFilePath);
+
+            }
+            ReleaseSuccessful = true;
+        }
+
+        protected void ReleaseGlobalFolder(DirectoryInfo directory = null)
+        {
+            if (directory == null)
+                directory = ReleaseAudit.SourceGlobalFolder;
+
+            if (ReleaseAudit.SourceGlobalFolder != null)
+            {
+                foreach(var dir in directory.GetDirectories())
+                {
+                    ReleaseGlobalFolder(dir);// todo this won't put it in the currect subfolder
+                }
+                foreach(var file in directory.EnumerateFiles())
+                {
+                    Task.Run(async ()=> await _s3Helper.PutObject(_bucket.BucketName, file.Name, file.FullName));
+                }
+            }
+        }
+
+
+        protected StreamWriter PrepareAuditFile(string path)
+        {
+            var sw = new StreamWriter(path);
+
+            sw.WriteLine($"----------Details Of Release---------:{DateTime.Now}");
+            sw.WriteLine($"ProjectName:{Project.Name}");
+            sw.WriteLine($"ProjectNumber:{Project.ProjectNumber}");
+            sw.WriteLine($"Project.ID:{Project.ID}");
+            sw.WriteLine($"ThisFileWasCreated:{DateTime.Now}");
+
+            sw.WriteLine($"----------Contents Of Directory---------:{DateTime.Now}");
+
+            return sw;
+        }
+    }
+}

Rdmp.Core/DataExport/DataRelease/AWSS3BucketReleaseDestination.cs

@@ -0,0 +1,156 @@
+using Rdmp.Core.DataExport.Data;
+using Rdmp.Core.DataExport.DataRelease.Pipeline;
+using Rdmp.Core.DataFlowPipeline.Requirements;
+using Rdmp.Core.DataFlowPipeline;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Rdmp.Core.ReusableLibraryCode.Progress;
+using Rdmp.Core.ReusableLibraryCode.Checks;
+using Rdmp.Core.Curation.Data;
+using Rdmp.Core.CommandExecution;
+using Amazon.S3.Model;
+using Rdmp.Core.ReusableLibraryCode.AWS;
+using Amazon;
+using Org.BouncyCastle.Crypto.Fpe;
+using System.IO;
+using System.Globalization;
+
+namespace Rdmp.Core.DataExport.DataRelease
+{
+    public class AWSS3BucketReleaseDestination : IPluginDataFlowComponent<ReleaseAudit>, IDataFlowDestination<ReleaseAudit>,
+    IPipelineRequirement<Project>, IPipelineRequirement<ReleaseData>
+    {
+
+        [DemandsNestedInitialization] public ReleaseEngineSettings ReleaseSettings { get; set; }
+        [DemandsInitialization("The local AWS profile you wish to use for the extraction")]
+        public string AWS_Profile { get; set; }
+
+        [DemandsInitialization("The name of the bucket you wish to write to")]
+        public string BucketName { get; set; }
+
+        [DemandsInitialization("The AWS Region you wish to use")]
+        public string AWS_Region { get; set; }
+
+
+        [DemandsInitialization("The folder in the S3 bucket you wish to release to", defaultValue: "")]
+        public string BucketFolder { get; set; }
+
+        private ReleaseData _releaseData;
+        private Project _project;
+        private AWSReleaseEngine _engine;
+        private List<IExtractionConfiguration> _configurationReleased;
+        private DirectoryInfo _destinationFolder;
+
+        private IBasicActivateItems _activator;
+
+
+        public void SetActivator(IBasicActivateItems activator)
+        {
+            _activator = activator;
+        }
+
+
+        private AWSS3 _s3Helper;
+        private RegionEndpoint _region;
+        private S3Bucket _bucket;
+
+        public void Abort(IDataLoadEventListener listener)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void Check(ICheckNotifier notifier)
+        {
+            ((ICheckable)ReleaseSettings).Check(notifier);
+            //TODO make the aws stuff pop up if not configured
+            if (string.IsNullOrWhiteSpace(AWS_Region))
+            {
+                notifier.OnCheckPerformed(new CheckEventArgs("No AWS Region Specified.", CheckResult.Fail));
+                return;
+            }
+            _region = RegionEndpoint.GetBySystemName(AWS_Region);
+            if (string.IsNullOrWhiteSpace(AWS_Profile))
+            {
+                notifier.OnCheckPerformed(new CheckEventArgs("No AWS Profile Specified.", CheckResult.Fail));
+                return;
+            }
+
+            _s3Helper = new AWSS3(AWS_Profile, _region);
+            try
+            {
+                _bucket = Task.Run(async() => await _s3Helper.GetBucket(BucketName)).Result;
+            }
+            catch (Exception e)
+            {
+                notifier.OnCheckPerformed(new CheckEventArgs(e.Message, CheckResult.Fail));
+                return;
+            }
+            //todo check location on bucket doesn't already exists
+
+        }
+
+        public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureExceptionIfAny)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void PreInitialize(Project value, IDataLoadEventListener listener)
+        {
+            _project = value;
+        }
+
+        public void PreInitialize(ReleaseData value, IDataLoadEventListener listener)
+        {
+            _releaseData = value;
+        }
+
+        public ReleaseAudit ProcessPipelineData(ReleaseAudit releaseAudit, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken)
+        {
+            if (releaseAudit == null)
+                return null;
+
+            if (releaseAudit.ReleaseFolder == null)
+            {
+                if (string.IsNullOrWhiteSpace(BucketFolder))
+                {
+                    listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "No Release Folder or S3 Bucket path were specified. Files will be placed in the root of the S3 Bucket"));
+                }
+            }
+            if (_releaseData.ReleaseState == ReleaseState.DoingPatch)
+            {
+                //TODO this is untested, but a blind copy from the other release destination
+                listener.OnNotify(this,
+                new NotifyEventArgs(ProgressEventType.Information,
+                    "CumulativeExtractionResults for datasets not included in the Patch will now be erased."));
+
+                var recordsDeleted = 0;
+
+                foreach (var (configuration, potentials) in _releaseData.ConfigurationsForRelease)
+                    //foreach existing CumulativeExtractionResults if it is not included in the patch then it should be deleted
+                    foreach (var redundantResult in configuration.CumulativeExtractionResults.Where(r =>
+                                 potentials.All(rp => rp.DataSet.ID != r.ExtractableDataSet_ID)))
+                    {
+                        redundantResult.DeleteInDatabase();
+                        recordsDeleted++;
+                    }
+
+                listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
+                    $"Deleted {recordsDeleted} old CumulativeExtractionResults (That were not included in the final Patch you are preparing)"));
+
+            }
+            _region = RegionEndpoint.GetBySystemName(AWS_Region);
+            _s3Helper = new AWSS3(AWS_Profile, _region);
+            _engine = new AWSReleaseEngine(_project, ReleaseSettings, _s3Helper, _bucket, listener, releaseAudit);
+            _engine.DoRelease(_releaseData.ConfigurationsForRelease, _releaseData.EnvironmentPotentials,
+                _releaseData.ReleaseState == ReleaseState.DoingPatch);
+
+            _destinationFolder = _engine.ReleaseAudit.ReleaseFolder;
+            _configurationReleased = _engine.ConfigurationsReleased;
+
+            return null;
+        }
+    }
+}