-
Notifications
You must be signed in to change notification settings - Fork 16
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
2 changed files
with
248 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,92 @@ | ||
using Amazon.S3.Model; | ||
using Rdmp.Core.DataExport.Data; | ||
using Rdmp.Core.DataExport.DataRelease.Pipeline; | ||
using Rdmp.Core.DataExport.DataRelease.Potential; | ||
using Rdmp.Core.ReusableLibraryCode.AWS; | ||
using Rdmp.Core.ReusableLibraryCode.Progress; | ||
using System; | ||
using System.Collections.Generic; | ||
using System.IO; | ||
using System.Linq; | ||
using System.Text; | ||
using System.Threading.Tasks; | ||
|
||
namespace Rdmp.Core.DataExport.DataRelease | ||
{ | ||
public class AWSReleaseEngine : ReleaseEngine | ||
{ | ||
|
||
AWSS3 _s3Helper; | ||
S3Bucket _bucket; | ||
|
||
public AWSReleaseEngine(Project project, ReleaseEngineSettings settings, AWSS3 s3Helper, S3Bucket bucket, IDataLoadEventListener listener, ReleaseAudit releaseAudit) : base(project, settings, listener, releaseAudit) | ||
{ | ||
_s3Helper = s3Helper; | ||
_bucket = bucket; | ||
} | ||
|
||
public override void DoRelease(Dictionary<IExtractionConfiguration, List<ReleasePotential>> toRelease, Dictionary<IExtractionConfiguration, ReleaseEnvironmentPotential> environments, bool isPatch) | ||
{ | ||
|
||
//base.DoRelease(toRelease, environments, isPatch); | ||
ConfigurationsToRelease = toRelease; | ||
Check failure on line 32 in Rdmp.Core/DataExport/DataRelease/AWSReleaseEngine.cs
|
||
var auditFilePath = Path.Combine(Path.GetTempPath(), "contents.txt"); | ||
using (var sw = PrepareAuditFile(auditFilePath)) | ||
{ | ||
ReleaseGlobalFolder(); | ||
// Audit Global Folder if there are any | ||
if (ReleaseAudit.SourceGlobalFolder != null) | ||
{ | ||
AuditDirectoryCreation(ReleaseAudit.SourceGlobalFolder.FullName, sw, 0); | ||
|
||
foreach (var fileInfo in ReleaseAudit.SourceGlobalFolder.GetFiles()) | ||
AuditFileCreation(fileInfo.Name, sw, 1); | ||
} | ||
|
||
//ReleaseAllExtractionConfigurations(toRelease, sw, environments, isPatch); | ||
sw.Flush(); | ||
sw.Close(); | ||
//_bucket = Task.Run(async () => await _s3Helper.GetBucket(BucketName)).Result; | ||
|
||
Task.Run(async () => await _s3Helper.PutObject(_bucket.BucketName, "contents.txt", auditFilePath)); | ||
File.Delete(auditFilePath); | ||
|
||
} | ||
ReleaseSuccessful = true; | ||
} | ||
|
||
protected void ReleaseGlobalFolder(DirectoryInfo directory = null) | ||
{ | ||
if (directory == null) | ||
directory = ReleaseAudit.SourceGlobalFolder; | ||
|
||
if (ReleaseAudit.SourceGlobalFolder != null) | ||
{ | ||
foreach(var dir in directory.GetDirectories()) | ||
{ | ||
ReleaseGlobalFolder(dir);// todo this won't put it in the currect subfolder | ||
} | ||
foreach(var file in directory.EnumerateFiles()) | ||
{ | ||
Task.Run(async ()=> await _s3Helper.PutObject(_bucket.BucketName, file.Name, file.FullName)); | ||
} | ||
} | ||
} | ||
|
||
|
||
protected StreamWriter PrepareAuditFile(string path) | ||
{ | ||
var sw = new StreamWriter(path); | ||
|
||
sw.WriteLine($"----------Details Of Release---------:{DateTime.Now}"); | ||
sw.WriteLine($"ProjectName:{Project.Name}"); | ||
sw.WriteLine($"ProjectNumber:{Project.ProjectNumber}"); | ||
sw.WriteLine($"Project.ID:{Project.ID}"); | ||
sw.WriteLine($"ThisFileWasCreated:{DateTime.Now}"); | ||
|
||
sw.WriteLine($"----------Contents Of Directory---------:{DateTime.Now}"); | ||
|
||
return sw; | ||
} | ||
} | ||
} |
156 changes: 156 additions & 0 deletions
156
Rdmp.Core/DataExport/DataRelease/AWSS3BucketReleaseDestination.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,156 @@ | ||
using Rdmp.Core.DataExport.Data; | ||
using Rdmp.Core.DataExport.DataRelease.Pipeline; | ||
using Rdmp.Core.DataFlowPipeline.Requirements; | ||
using Rdmp.Core.DataFlowPipeline; | ||
using System; | ||
using System.Collections.Generic; | ||
using System.Linq; | ||
using System.Text; | ||
using System.Threading.Tasks; | ||
using Rdmp.Core.ReusableLibraryCode.Progress; | ||
using Rdmp.Core.ReusableLibraryCode.Checks; | ||
using Rdmp.Core.Curation.Data; | ||
using Rdmp.Core.CommandExecution; | ||
using Amazon.S3.Model; | ||
using Rdmp.Core.ReusableLibraryCode.AWS; | ||
using Amazon; | ||
using Org.BouncyCastle.Crypto.Fpe; | ||
using System.IO; | ||
using System.Globalization; | ||
|
||
namespace Rdmp.Core.DataExport.DataRelease | ||
{ | ||
public class AWSS3BucketReleaseDestination : IPluginDataFlowComponent<ReleaseAudit>, IDataFlowDestination<ReleaseAudit>, | ||
IPipelineRequirement<Project>, IPipelineRequirement<ReleaseData> | ||
{ | ||
|
||
[DemandsNestedInitialization] public ReleaseEngineSettings ReleaseSettings { get; set; } | ||
[DemandsInitialization("The local AWS profile you wish to use for the extraction")] | ||
public string AWS_Profile { get; set; } | ||
|
||
[DemandsInitialization("The name of the bucket you wish to write to")] | ||
public string BucketName { get; set; } | ||
|
||
[DemandsInitialization("The AWS Region you wish to use")] | ||
public string AWS_Region { get; set; } | ||
|
||
|
||
[DemandsInitialization("The folder in the S3 bucket you wish to release to", defaultValue: "")] | ||
public string BucketFolder { get; set; } | ||
|
||
private ReleaseData _releaseData; | ||
private Project _project; | ||
private AWSReleaseEngine _engine; | ||
private List<IExtractionConfiguration> _configurationReleased; | ||
private DirectoryInfo _destinationFolder; | ||
|
||
private IBasicActivateItems _activator; | ||
|
||
|
||
public void SetActivator(IBasicActivateItems activator) | ||
{ | ||
_activator = activator; | ||
} | ||
|
||
|
||
private AWSS3 _s3Helper; | ||
private RegionEndpoint _region; | ||
private S3Bucket _bucket; | ||
|
||
public void Abort(IDataLoadEventListener listener) | ||
{ | ||
throw new NotImplementedException(); | ||
} | ||
|
||
public void Check(ICheckNotifier notifier) | ||
{ | ||
((ICheckable)ReleaseSettings).Check(notifier); | ||
//TODO make the aws stuff pop up if not configured | ||
if (string.IsNullOrWhiteSpace(AWS_Region)) | ||
{ | ||
notifier.OnCheckPerformed(new CheckEventArgs("No AWS Region Specified.", CheckResult.Fail)); | ||
return; | ||
} | ||
_region = RegionEndpoint.GetBySystemName(AWS_Region); | ||
if (string.IsNullOrWhiteSpace(AWS_Profile)) | ||
{ | ||
notifier.OnCheckPerformed(new CheckEventArgs("No AWS Profile Specified.", CheckResult.Fail)); | ||
return; | ||
} | ||
|
||
_s3Helper = new AWSS3(AWS_Profile, _region); | ||
try | ||
{ | ||
_bucket = Task.Run(async() => await _s3Helper.GetBucket(BucketName)).Result; | ||
} | ||
catch (Exception e) | ||
{ | ||
notifier.OnCheckPerformed(new CheckEventArgs(e.Message, CheckResult.Fail)); | ||
return; | ||
} | ||
//todo check location on bucket doesn't already exists | ||
|
||
} | ||
|
||
public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureExceptionIfAny) | ||
{ | ||
throw new NotImplementedException(); | ||
} | ||
|
||
public void PreInitialize(Project value, IDataLoadEventListener listener) | ||
{ | ||
_project = value; | ||
} | ||
|
||
public void PreInitialize(ReleaseData value, IDataLoadEventListener listener) | ||
{ | ||
_releaseData = value; | ||
} | ||
|
||
public ReleaseAudit ProcessPipelineData(ReleaseAudit releaseAudit, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) | ||
{ | ||
if (releaseAudit == null) | ||
return null; | ||
|
||
if (releaseAudit.ReleaseFolder == null) | ||
{ | ||
if (string.IsNullOrWhiteSpace(BucketFolder)) | ||
{ | ||
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "No Release Folder or S3 Bucket path were specified. Files will be placed in the root of the S3 Bucket")); | ||
} | ||
} | ||
if (_releaseData.ReleaseState == ReleaseState.DoingPatch) | ||
{ | ||
//TODO this is untested, but a blind copy from the other release destination | ||
listener.OnNotify(this, | ||
new NotifyEventArgs(ProgressEventType.Information, | ||
"CumulativeExtractionResults for datasets not included in the Patch will now be erased.")); | ||
|
||
var recordsDeleted = 0; | ||
|
||
foreach (var (configuration, potentials) in _releaseData.ConfigurationsForRelease) | ||
//foreach existing CumulativeExtractionResults if it is not included in the patch then it should be deleted | ||
foreach (var redundantResult in configuration.CumulativeExtractionResults.Where(r => | ||
potentials.All(rp => rp.DataSet.ID != r.ExtractableDataSet_ID))) | ||
{ | ||
redundantResult.DeleteInDatabase(); | ||
recordsDeleted++; | ||
} | ||
|
||
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, | ||
$"Deleted {recordsDeleted} old CumulativeExtractionResults (That were not included in the final Patch you are preparing)")); | ||
|
||
} | ||
_region = RegionEndpoint.GetBySystemName(AWS_Region); | ||
_s3Helper = new AWSS3(AWS_Profile, _region); | ||
_engine = new AWSReleaseEngine(_project, ReleaseSettings, _s3Helper, _bucket, listener, releaseAudit); | ||
_engine.DoRelease(_releaseData.ConfigurationsForRelease, _releaseData.EnvironmentPotentials, | ||
_releaseData.ReleaseState == ReleaseState.DoingPatch); | ||
|
||
_destinationFolder = _engine.ReleaseAudit.ReleaseFolder; | ||
_configurationReleased = _engine.ConfigurationsReleased; | ||
|
||
return null; | ||
} | ||
} | ||
} |