Skip to content

Commit

Permalink
add exists check
Browse files Browse the repository at this point in the history
  • Loading branch information
@JFriel
JFriel committed Aug 8, 2024
1 parent 4f815b0 commit 928ebba
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 11 deletions.
5 changes: 3 additions & 2 deletions Rdmp.Core/DataExport/DataRelease/AWSReleaseEngine.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ public override void DoRelease(Dictionary<IExtractionConfiguration, List<Release
{
ReleaseGlobalFolder();
// Audit Global Folder if there are any
//TODO actually test globals
if (ReleaseAudit.SourceGlobalFolder != null)
{
AuditDirectoryCreation(ReleaseAudit.SourceGlobalFolder.FullName, sw, 0);
Expand All @@ -55,7 +56,7 @@ public override void DoRelease(Dictionary<IExtractionConfiguration, List<Release

Task.Run(async () =>
{
await _s3Helper.PutObject(_bucket.BucketName, "contents.txt", auditFilePath, GetLocation(ReleaseAudit.ReleaseFolder != null ? ReleaseAudit.ReleaseFolder.Name : null, null));
await _s3Helper.PutObject(_bucket.BucketName, "contents.txt", auditFilePath, GetLocation( null, null));
}
).Wait();
File.Delete(auditFilePath);
Expand Down Expand Up @@ -100,7 +101,7 @@ protected override void ReleaseAllExtractionConfigurations(Dictionary<IExtractio
foreach (var kvp in toRelease)
{
var extractionIdentifier = $"{kvp.Key.Name}_{kvp.Key.ID}";
var locationWithinBucket = GetLocation(ReleaseAudit.ReleaseFolder != null ? ReleaseAudit.ReleaseFolder.Name : null, extractionIdentifier);// $"{!string.IsNullOrWhiteSpace(_bucketFolder)?}{ReleaseAudit.ReleaseFolder.Name}/{extractionIdentifier}";
var locationWithinBucket = GetLocation( null, extractionIdentifier);
AuditExtractionConfigurationDetails(sw, locationWithinBucket, kvp, extractionIdentifier);
AuditDirectoryCreation(locationWithinBucket, sw, 0);
var customDataFolder = ReleaseCustomData(kvp, locationWithinBucket);
Expand Down
25 changes: 16 additions & 9 deletions Rdmp.Core/DataExport/DataRelease/AWSS3BucketReleaseDestination.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,14 +81,21 @@ public void Check(ICheckNotifier notifier)
_s3Helper = new AWSS3(AWS_Profile, _region);
try
{
_bucket = Task.Run(async() => await _s3Helper.GetBucket(BucketName)).Result;
_bucket = Task.Run(async () => await _s3Helper.GetBucket(BucketName)).Result;
}
catch (Exception e)
{
notifier.OnCheckPerformed(new CheckEventArgs(e.Message, CheckResult.Fail));
return;
}
//todo check location on bucket doesn't already exists
//check if the folder exists
if (_s3Helper.DoesObjectExists(!string.IsNullOrWhiteSpace(BucketFolder) ? $"{BucketFolder}/contents.txt" : "contents.txt", _bucket.BucketName).Result)
{
notifier.OnCheckPerformed(new CheckEventArgs("Bucket Folder Already exists", CheckResult.Fail));
return;
}


}

Expand All @@ -114,13 +121,13 @@ public ReleaseAudit ProcessPipelineData(ReleaseAudit releaseAudit, IDataLoadEven
_region = RegionEndpoint.GetBySystemName(AWS_Region);
_s3Helper = new AWSS3(AWS_Profile, _region);
_bucket = Task.Run(async () => await _s3Helper.GetBucket(BucketName)).Result;
if (releaseAudit.ReleaseFolder == null)
{
if (string.IsNullOrWhiteSpace(BucketFolder))
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "No Release Folder or S3 Bucket path were specified. Files will be placed in the root of the S3 Bucket"));
}
}
//if (releaseAudit.ReleaseFolder == null)
//{
// if (string.IsNullOrWhiteSpace(BucketFolder))
// {
// listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "No Release Folder or S3 Bucket path were specified. Files will be placed in the root of the S3 Bucket"));
// }
//}
if (_releaseData.ReleaseState == ReleaseState.DoingPatch)
{
//TODO this is untested, but a blind copy from the other release destination
Expand All @@ -145,7 +152,7 @@ public ReleaseAudit ProcessPipelineData(ReleaseAudit releaseAudit, IDataLoadEven
}
_region = RegionEndpoint.GetBySystemName(AWS_Region);
_s3Helper = new AWSS3(AWS_Profile, _region);
_engine = new AWSReleaseEngine(_project, ReleaseSettings, _s3Helper, _bucket, BucketFolder,listener, releaseAudit);
_engine = new AWSReleaseEngine(_project, ReleaseSettings, _s3Helper, _bucket, BucketFolder, listener, releaseAudit);
_engine.DoRelease(_releaseData.ConfigurationsForRelease, _releaseData.EnvironmentPotentials,
_releaseData.ReleaseState == ReleaseState.DoingPatch);

Expand Down
22 changes: 22 additions & 0 deletions Rdmp.Core/ReusableLibraryCode/AWS/AWSS3.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using Amazon.S3;
using Amazon.S3.Model;
using Azure;
using MathNet.Numerics.Statistics;
using Renci.SshNet;
using SharpCompress.Common;
using System;
Expand Down Expand Up @@ -60,6 +61,27 @@ public static string KeyGenerator(string path, string file)
return Path.Join(path, file).Replace("\\", "/");//todo there is probably a better way to do this
}

public async Task<bool> DoesObjectExists(string Key, string bucketName)
{
ListObjectsResponse response = null;
try
{

ListObjectsRequest request = new ListObjectsRequest
{
BucketName = bucketName,
Prefix = Key
};
response = await _client.ListObjectsAsync(request);

}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
return (response != null && response.S3Objects != null && response.S3Objects.Count > 0 && response.S3Objects.Any(o => o.Key == Key));
}


public bool ObjectExists(string fileKey, string bucketName)
{
Expand Down

0 comments on commit 928ebba

Please sign in to comment.