From 17ff49e13c227c434ff19b594e037de3fd4df1c1 Mon Sep 17 00:00:00 2001 From: Ersan Date: Tue, 26 Jul 2022 01:04:52 -0700 Subject: [PATCH 1/5] Adds 2 more bucket notification functional tests --- Minio.Functional.Tests/FunctionalTest.cs | 508 ++++++++++++++++------- Minio.Functional.Tests/Program.cs | 2 + 2 files changed, 363 insertions(+), 147 deletions(-) diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index a812b4ead..cf601500c 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -1352,153 +1352,6 @@ internal static async Task RemoveIncompleteUpload_Test(MinioClient minio) } } - - #region Bucket Notifications - - internal static async Task ListenBucketNotificationsAsync_Test1(MinioClient minio) - { - var startTime = DateTime.Now; - var bucketName = GetRandomName(15); - var objectName = GetRandomName(10); - var contentType = "application/octet-stream"; - IDisposable subscription = null; - var args = new Dictionary - { - { "bucketName", bucketName }, - { "objectName", objectName }, - { "contentType", contentType }, - { "size", "1KB" } - }; - try - { - await Setup_Test(minio, bucketName); - - var received = new List(); - - var eventsList = new List(); - eventsList.Add(EventType.ObjectCreatedAll); - - var listenArgs = new ListenBucketNotificationsArgs() - .WithBucket(bucketName) - .WithEvents(eventsList); - var events = minio.ListenBucketNotificationsAsync(listenArgs); - subscription = events.Subscribe( - ev => received.Add(ev), - ex => { }, - () => { } - ); - - await PutObject_Tester(minio, bucketName, objectName, null, contentType, - 0, null, rsg.GenerateStreamFromSeed(1 * KB)); - - // wait for notifications - var eventDetected = false; - for (var attempt = 0; attempt < 10; attempt++) - if (received.Count > 0) - { - // Check if there is any unexpected error returned - // and captured in the receivedJson list, like - // "NotImplemented" api error. If so, we throw an exception - // and skip running this test - if (received.Count > 1 && received[1].json.StartsWith("")) - { - // Although the attribute is called "json", - // returned data in list "received" is in xml - // format and it is an error.Here, we convert xml - // into json format. - var receivedJson = XmlStrToJsonStr(received[1].json); - - - // Cleanup the "Error" key encapsulating "receivedJson" - // data. This is required to match and convert json data - // "receivedJson" into class "ErrorResponse" - var len = "{'Error':".Length; - var trimmedFront = receivedJson.Substring(len); - var trimmedFull = trimmedFront.Substring(0, trimmedFront.Length - 1); - - var err = JsonConvert.DeserializeObject(trimmedFull); - - Exception ex = new UnexpectedMinioException(err.Message); - if (err.Code == "NotImplemented") - ex = new NotImplementedException(err.Message); - - throw ex; - } - - var notification = JsonConvert.DeserializeObject(received[0].json); - - if (notification.Records != null) - { - Assert.AreEqual(1, notification.Records.Length); - Assert.IsTrue(notification.Records[0].eventName.Contains("s3:ObjectCreated:Put")); - Assert.IsTrue( - objectName.Contains(HttpUtility.UrlDecode(notification.Records[0].s3.objectMeta.key))); - Assert.IsTrue(contentType.Contains(notification.Records[0].s3.objectMeta.contentType)); - eventDetected = true; - break; - } - } - - // subscription.Dispose(); - if (!eventDetected) - throw new UnexpectedMinioException("Failed to detect the expected bucket notification event."); - - new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for small object", - TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); - } - catch (NotImplementedException ex) - { - new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for small object", - TestStatus.NA, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - } - catch (Exception ex) - { - if (ex.Message == "Listening for bucket notification is specific" + - " only to `minio` server endpoints") - { - // This is expected when bucket notification - // is requested against AWS. - // Check if endPoint is AWS - bool isAWS(string endPoint) - { - var rgx = new Regex("^s3\\.?.*\\.amazonaws\\.com", RegexOptions.IgnoreCase); - var matches = rgx.Matches(endPoint); - return matches.Count > 0; - } - - if (Environment.GetEnvironmentVariable("AWS_ENDPOINT") != null || - isAWS(Environment.GetEnvironmentVariable("SERVER_ENDPOINT"))) - // This is a PASS - new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for small object", - TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); - } - else - { - new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for small object", - TestStatus.FAIL, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - throw; - } - } - finally - { - await TearDown(minio, bucketName); - if (subscription != null) - subscription.Dispose(); - } - } - - #endregion - #region Select Object Content internal static async Task SelectObjectContent_Test(MinioClient minio) @@ -2540,6 +2393,367 @@ internal static async Task ObjectRetentionAsync_Test1(MinioClient minio) #endregion + + #region Bucket Notifications + + internal static async Task ListenBucketNotificationsAsync_Test1(MinioClient minio) + { + var startTime = DateTime.Now; + var bucketName = GetRandomName(15); + var objectName = GetRandomName(10); + var contentType = "application/octet-stream"; + IDisposable subscription = null; + var args = new Dictionary + { + { "bucketName", bucketName }, + { "objectName", objectName }, + { "contentType", contentType }, + { "size", "1KB" } + }; + try + { + await Setup_Test(minio, bucketName); + + var received = new List(); + + var eventsList = new List(); + eventsList.Add(EventType.ObjectCreatedAll); + + var listenArgs = new ListenBucketNotificationsArgs() + .WithBucket(bucketName) + .WithEvents(eventsList); + var events = minio.ListenBucketNotificationsAsync(listenArgs); + subscription = events.Subscribe( + ev => received.Add(ev), + ex => { }, + () => { } + ); + + await PutObject_Tester(minio, bucketName, objectName, null, contentType, + 0, null, rsg.GenerateStreamFromSeed(1 * KB)); + + // wait for notifications + var eventDetected = false; + for (var attempt = 0; attempt < 10; attempt++) + if (received.Count > 0) + { + // Check if there is any unexpected error returned + // and captured in the receivedJson list, like + // "NotImplemented" api error. If so, we throw an exception + // and skip running this test + if (received.Count > 1 && received[1].json.StartsWith("")) + { + // Although the attribute is called "json", + // returned data in list "received" is in xml + // format and it is an error.Here, we convert xml + // into json format. + var receivedJson = XmlStrToJsonStr(received[1].json); + + + // Cleanup the "Error" key encapsulating "receivedJson" + // data. This is required to match and convert json data + // "receivedJson" into class "ErrorResponse" + var len = "{'Error':".Length; + var trimmedFront = receivedJson.Substring(len); + var trimmedFull = trimmedFront.Substring(0, trimmedFront.Length - 1); + + var err = JsonConvert.DeserializeObject(trimmedFull); + + Exception ex = new UnexpectedMinioException(err.Message); + if (err.Code == "NotImplemented") + ex = new NotImplementedException(err.Message); + + throw ex; + } + + var notification = JsonConvert.DeserializeObject(received[0].json); + + if (notification.Records != null) + { + Assert.AreEqual(1, notification.Records.Length); + Assert.IsTrue(notification.Records[0].eventName.Contains("s3:ObjectCreated:Put")); + Assert.IsTrue( + objectName.Contains(HttpUtility.UrlDecode(notification.Records[0].s3.objectMeta.key))); + Assert.IsTrue(contentType.Contains(notification.Records[0].s3.objectMeta.contentType)); + eventDetected = true; + break; + } + } + + // subscription.Dispose(); + if (!eventDetected) + throw new UnexpectedMinioException("Failed to detect the expected bucket notification event."); + + new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for small object", + TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + } + catch (NotImplementedException ex) + { + new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for small object", + TestStatus.NA, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + } + catch (Exception ex) + { + if (ex.Message == "Listening for bucket notification is specific" + + " only to `minio` server endpoints") + { + // This is expected when bucket notification + // is requested against AWS. + // Check if endPoint is AWS + bool isAWS(string endPoint) + { + var rgx = new Regex("^s3\\.?.*\\.amazonaws\\.com", RegexOptions.IgnoreCase); + var matches = rgx.Matches(endPoint); + return matches.Count > 0; + } + + if (Environment.GetEnvironmentVariable("AWS_ENDPOINT") != null || + isAWS(Environment.GetEnvironmentVariable("SERVER_ENDPOINT"))) + // This is a PASS + new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for small object", + TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + } + else + { + new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for small object", + TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + throw; + } + } + finally + { + await TearDown(minio, bucketName); + if (subscription != null) + subscription.Dispose(); + } + } + + + internal static async Task ListenBucketNotificationsAsync_Test2(MinioClient minio) + { + var startTime = DateTime.Now; + var events = new List(); + events.Add(EventType.ObjectCreatedAll); + var rxEvents = new List(); + IDisposable subscription = null; + var bucketName = GetRandomName(15); + var contentType = "application/json"; + var args = new Dictionary + { + { "bucketName", bucketName }, + { "contentType", contentType }, + { "size", "16B" } + }; + + try + { + static Stream ToStream(string input) + { + var stream = new MemoryStream(); + var writer = new StreamWriter(stream); + writer.Write(input); + writer.Flush(); + stream.Position = 0; + + return stream; + } + + ; + + var bucketExistsArgs = new BucketExistsArgs() + .WithBucket(bucketName); + var found = await minio.BucketExistsAsync(bucketExistsArgs).ConfigureAwait(false); + if (!found) + { + var makeBucketArgs = new MakeBucketArgs() + .WithBucket(bucketName); + await minio.MakeBucketAsync(makeBucketArgs).ConfigureAwait(false); + } + + void Notify(MinioNotificationRaw data) + { + var notification = JsonConvert.DeserializeObject(data.json); + if (notification is not { Records: { } }) return; + + foreach (var @event in notification.Records) rxEvents.Add(@event); + } + + var listenArgs = new ListenBucketNotificationsArgs() + .WithBucket(bucketName) + .WithEvents(events); + var observable = minio.ListenBucketNotificationsAsync(listenArgs); + + subscription = observable.Subscribe( + ev => Notify(ev), + ex => throw new Exception($"OnError: {ex.Message}"), + () => throw new Exception("STOPPED LISTENING FOR BUCKET NOTIFICATIONS\n")); + + var modelJson = "{\"test\": \"test\"}"; + await using var stream = ToStream(modelJson); + var putObjectArgs = new PutObjectArgs() + .WithObject("test.json") + .WithBucket(bucketName) + .WithContentType(contentType) + .WithStreamData(stream) + .WithObjectSize(stream.Length); + + await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); + + // Waits until the Put event is detected + // Times out if the event is not caught in 3 seconds + var timeoutDuration = 3; // seconds + var stTime = DateTime.UtcNow; + var timeout = TimeSpan.FromSeconds(timeoutDuration); + while (rxEvents.Count < 1) + { + await Task.Delay(25); + if (DateTime.UtcNow - stTime >= timeout) + throw new Exception("Timeout: while waiting for events"); + } + + foreach (var ev in rxEvents) Assert.AreEqual("s3:ObjectCreated:Put", ev.eventName); + + new MintLogger(nameof(ListenBucketNotificationsAsync_Test2), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for longer event processing", + TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + } + catch (Exception ex) + { + new MintLogger(nameof(ListenBucketNotificationsAsync_Test2), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for longer event processing", + TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + throw; + } + + finally + { + await TearDown(minio, bucketName); + if (subscription != null) + subscription.Dispose(); + } + } + + internal static async Task ListenBucketNotificationsAsync_Test3(MinioClient minio) + { + var startTime = DateTime.Now; + var events = new List(); + events.Add(EventType.ObjectCreatedAll); + var rxEventsData = new MinioNotificationRaw(""); + IDisposable disposable = null; + var bucketName = GetRandomName(15); + var suffix = ".json"; + var contentType = "application/json"; + var args = new Dictionary + { + { "bucketName", bucketName }, + { "contentType", contentType }, + { "suffix", suffix }, + { "size", "16B" } + }; + + try + { + var bucketExistsArgs = new BucketExistsArgs() + .WithBucket(bucketName); + if (!minio.BucketExistsAsync(bucketExistsArgs).Result) + { + var makeBucketArgs = new MakeBucketArgs() + .WithBucket(bucketName); + // await minio.MakeBucketAsync(makeBucketArgs); + minio.MakeBucketAsync(makeBucketArgs).Wait(); + } + + // Console.WriteLine($"Bucket created"); + + var notificationsArgs = new ListenBucketNotificationsArgs() + .WithBucket(bucketName) + .WithSuffix(suffix) + .WithEvents(events); + + var notifications = minio.ListenBucketNotificationsAsync(notificationsArgs); + + var testState = "fail"; + Exception exception = null; + disposable = notifications.Subscribe( + x => + { + rxEventsData = x; + testState = "pass"; + }, + ex => + { + exception = ex; + testState = "fail"; + }, + () => { testState = "completed"; }); + + var modelJson = "{\"test\": \"test\"}"; + var stream = new MemoryStream(); + var writer = new StreamWriter(stream); + writer.Write(modelJson); + writer.Flush(); + stream.Position = 0; + + var putObjectArgs = new PutObjectArgs() + .WithObject("test.json") + .WithBucket(bucketName) + .WithContentType(contentType) + .WithStreamData(stream) + .WithObjectSize(stream.Length); + + await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); + Thread.Sleep(1000); + + if (testState == "pass") + { + Assert.IsTrue(rxEventsData.json.Contains("\"eventName\":\"s3:ObjectCreated:Put\"")); + new MintLogger(nameof(ListenBucketNotificationsAsync_Test3), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for no event processing", + TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + } + else if (testState == "fail") + { + throw exception; + } + else if (testState == "completed") + { + throw new Exception("Bucket notification completed without catching the event"); + } + } + catch (Exception ex) + { + new MintLogger(nameof(ListenBucketNotificationsAsync_Test3), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for no event processing", + TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + throw; + } + + finally + { + await TearDown(minio, bucketName); + if (disposable != null) + disposable.Dispose(); + } + } + + #endregion + #region Make Bucket internal static async Task MakeBucket_Test1(MinioClient minio) diff --git a/Minio.Functional.Tests/Program.cs b/Minio.Functional.Tests/Program.cs index cec7d436a..2d12f92ea 100644 --- a/Minio.Functional.Tests/Program.cs +++ b/Minio.Functional.Tests/Program.cs @@ -91,6 +91,8 @@ public static void Main(string[] args) // "Listening for bucket notification is specific only to `minio` // server endpoints". FunctionalTest.ListenBucketNotificationsAsync_Test1(minioClient).Wait(); + FunctionalTest.ListenBucketNotificationsAsync_Test2(minioClient).Wait(); + FunctionalTest.ListenBucketNotificationsAsync_Test3(minioClient).Wait(); // Check if bucket exists FunctionalTest.BucketExists_Test(minioClient).Wait(); From fd617d38e4e1b31da29a3e275d9ea5db3d95147d Mon Sep 17 00:00:00 2001 From: Ersan Date: Tue, 26 Jul 2022 01:26:38 -0700 Subject: [PATCH 2/5] A couple of format cleanups --- Minio.Functional.Tests/FunctionalTest.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index cf601500c..c88d6d4b5 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -2568,8 +2568,6 @@ static Stream ToStream(string input) return stream; } - ; - var bucketExistsArgs = new BucketExistsArgs() .WithBucket(bucketName); var found = await minio.BucketExistsAsync(bucketExistsArgs).ConfigureAwait(false); @@ -2672,12 +2670,9 @@ internal static async Task ListenBucketNotificationsAsync_Test3(MinioClient mini { var makeBucketArgs = new MakeBucketArgs() .WithBucket(bucketName); - // await minio.MakeBucketAsync(makeBucketArgs); minio.MakeBucketAsync(makeBucketArgs).Wait(); } - // Console.WriteLine($"Bucket created"); - var notificationsArgs = new ListenBucketNotificationsArgs() .WithBucket(bucketName) .WithSuffix(suffix) From 1f68384fa98f05dbacc5d112fd0202264efc04fa Mon Sep 17 00:00:00 2001 From: Ersan Date: Wed, 27 Jul 2022 00:48:29 -0700 Subject: [PATCH 3/5] More tidy up --- Minio.Functional.Tests/FunctionalTest.cs | 355 +++++++++++++++++++++++ 1 file changed, 355 insertions(+) diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index c88d6d4b5..251c24f9a 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -1352,6 +1352,361 @@ internal static async Task RemoveIncompleteUpload_Test(MinioClient minio) } } + + #region Bucket Notifications + + internal static async Task ListenBucketNotificationsAsync_Test1(MinioClient minio) + { + var startTime = DateTime.Now; + var bucketName = GetRandomName(15); + var objectName = GetRandomName(10); + var contentType = "application/octet-stream"; + IDisposable subscription = null; + var args = new Dictionary + { + { "bucketName", bucketName }, + { "objectName", objectName }, + { "contentType", contentType }, + { "size", "1KB" } + }; + try + { + await Setup_Test(minio, bucketName); + + var received = new List(); + + var eventsList = new List(); + eventsList.Add(EventType.ObjectCreatedAll); + + var listenArgs = new ListenBucketNotificationsArgs() + .WithBucket(bucketName) + .WithEvents(eventsList); + var events = minio.ListenBucketNotificationsAsync(listenArgs); + subscription = events.Subscribe( + ev => received.Add(ev), + ex => { }, + () => { } + ); + + await PutObject_Tester(minio, bucketName, objectName, null, contentType, + 0, null, rsg.GenerateStreamFromSeed(1 * KB)); + + // wait for notifications + var eventDetected = false; + for (var attempt = 0; attempt < 10; attempt++) + if (received.Count > 0) + { + // Check if there is any unexpected error returned + // and captured in the receivedJson list, like + // "NotImplemented" api error. If so, we throw an exception + // and skip running this test + if (received.Count > 1 && received[1].json.StartsWith("")) + { + // Although the attribute is called "json", + // returned data in list "received" is in xml + // format and it is an error.Here, we convert xml + // into json format. + var receivedJson = XmlStrToJsonStr(received[1].json); + + + // Cleanup the "Error" key encapsulating "receivedJson" + // data. This is required to match and convert json data + // "receivedJson" into class "ErrorResponse" + var len = "{'Error':".Length; + var trimmedFront = receivedJson.Substring(len); + var trimmedFull = trimmedFront.Substring(0, trimmedFront.Length - 1); + + var err = JsonConvert.DeserializeObject(trimmedFull); + + Exception ex = new UnexpectedMinioException(err.Message); + if (err.Code == "NotImplemented") + ex = new NotImplementedException(err.Message); + + throw ex; + } + + var notification = JsonConvert.DeserializeObject(received[0].json); + + if (notification.Records != null) + { + Assert.AreEqual(1, notification.Records.Length); + Assert.IsTrue(notification.Records[0].eventName.Contains("s3:ObjectCreated:Put")); + Assert.IsTrue( + objectName.Contains(HttpUtility.UrlDecode(notification.Records[0].s3.objectMeta.key))); + Assert.IsTrue(contentType.Contains(notification.Records[0].s3.objectMeta.contentType)); + eventDetected = true; + break; + } + } + + // subscription.Dispose(); + if (!eventDetected) + throw new UnexpectedMinioException("Failed to detect the expected bucket notification event."); + + new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for small object", + TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + } + catch (NotImplementedException ex) + { + new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for small object", + TestStatus.NA, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + } + catch (Exception ex) + { + if (ex.Message == "Listening for bucket notification is specific" + + " only to `minio` server endpoints") + { + // This is expected when bucket notification + // is requested against AWS. + // Check if endPoint is AWS + bool isAWS(string endPoint) + { + var rgx = new Regex("^s3\\.?.*\\.amazonaws\\.com", RegexOptions.IgnoreCase); + var matches = rgx.Matches(endPoint); + return matches.Count > 0; + } + + if (Environment.GetEnvironmentVariable("AWS_ENDPOINT") != null || + isAWS(Environment.GetEnvironmentVariable("SERVER_ENDPOINT"))) + // This is a PASS + new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for small object", + TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + } + else + { + new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for small object", + TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + throw; + } + } + finally + { + await TearDown(minio, bucketName); + if (subscription != null) + subscription.Dispose(); + } + } + + internal static async Task ListenBucketNotificationsAsync_Test2(MinioClient minio) + { + var startTime = DateTime.Now; + var events = new List(); + events.Add(EventType.ObjectCreatedAll); + var rxEvents = new List(); + IDisposable subscription = null; + var bucketName = GetRandomName(15); + var contentType = "application/json"; + var args = new Dictionary + { + { "bucketName", bucketName }, + { "contentType", contentType }, + { "size", "16B" } + }; + + try + { + static Stream ToStream(string input) + { + var stream = new MemoryStream(); + var writer = new StreamWriter(stream); + writer.Write(input); + writer.Flush(); + stream.Position = 0; + + return stream; + } + + var bucketExistsArgs = new BucketExistsArgs() + .WithBucket(bucketName); + var found = await minio.BucketExistsAsync(bucketExistsArgs).ConfigureAwait(false); + if (!found) + { + var makeBucketArgs = new MakeBucketArgs() + .WithBucket(bucketName); + await minio.MakeBucketAsync(makeBucketArgs).ConfigureAwait(false); + } + + void Notify(MinioNotificationRaw data) + { + var notification = JsonConvert.DeserializeObject(data.json); + if (notification is not { Records: { } }) return; + + foreach (var @event in notification.Records) rxEvents.Add(@event); + } + + var listenArgs = new ListenBucketNotificationsArgs() + .WithBucket(bucketName) + .WithEvents(events); + var observable = minio.ListenBucketNotificationsAsync(listenArgs); + + subscription = observable.Subscribe( + ev => Notify(ev), + ex => throw new Exception($"OnError: {ex.Message}"), + () => throw new Exception("STOPPED LISTENING FOR BUCKET NOTIFICATIONS\n")); + + var modelJson = "{\"test\": \"test\"}"; + await using var stream = ToStream(modelJson); + var putObjectArgs = new PutObjectArgs() + .WithObject("test.json") + .WithBucket(bucketName) + .WithContentType(contentType) + .WithStreamData(stream) + .WithObjectSize(stream.Length); + + await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); + + // Waits until the Put event is detected + // Times out if the event is not caught in 3 seconds + var timeoutDuration = 3; // seconds + var stTime = DateTime.UtcNow; + var timeout = TimeSpan.FromSeconds(timeoutDuration); + while (rxEvents.Count < 1) + { + await Task.Delay(25); + if (DateTime.UtcNow - stTime >= timeout) + throw new Exception("Timeout: while waiting for events"); + } + + foreach (var ev in rxEvents) Assert.AreEqual("s3:ObjectCreated:Put", ev.eventName); + + new MintLogger(nameof(ListenBucketNotificationsAsync_Test2), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for longer event processing", + TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + } + catch (Exception ex) + { + new MintLogger(nameof(ListenBucketNotificationsAsync_Test2), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for longer event processing", + TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + throw; + } + + finally + { + await TearDown(minio, bucketName); + if (subscription != null) + subscription.Dispose(); + } + } + + internal static async Task ListenBucketNotificationsAsync_Test3(MinioClient minio) + { + var startTime = DateTime.Now; + var events = new List(); + events.Add(EventType.ObjectCreatedAll); + var rxEventsData = new MinioNotificationRaw(""); + IDisposable disposable = null; + var bucketName = GetRandomName(15); + var suffix = ".json"; + var contentType = "application/json"; + var args = new Dictionary + { + { "bucketName", bucketName }, + { "contentType", contentType }, + { "suffix", suffix }, + { "size", "16B" } + }; + + try + { + var bucketExistsArgs = new BucketExistsArgs() + .WithBucket(bucketName); + if (!minio.BucketExistsAsync(bucketExistsArgs).Result) + { + var makeBucketArgs = new MakeBucketArgs() + .WithBucket(bucketName); + minio.MakeBucketAsync(makeBucketArgs).Wait(); + } + + var notificationsArgs = new ListenBucketNotificationsArgs() + .WithBucket(bucketName) + .WithSuffix(suffix) + .WithEvents(events); + + var notifications = minio.ListenBucketNotificationsAsync(notificationsArgs); + + var testState = "fail"; + Exception exception = null; + disposable = notifications.Subscribe( + x => + { + rxEventsData = x; + testState = "pass"; + }, + ex => + { + exception = ex; + testState = "fail"; + }, + () => { testState = "completed"; }); + + var modelJson = "{\"test\": \"test\"}"; + var stream = new MemoryStream(); + var writer = new StreamWriter(stream); + writer.Write(modelJson); + writer.Flush(); + stream.Position = 0; + + var putObjectArgs = new PutObjectArgs() + .WithObject("test.json") + .WithBucket(bucketName) + .WithContentType(contentType) + .WithStreamData(stream) + .WithObjectSize(stream.Length); + + await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); + Thread.Sleep(1000); + + if (testState == "pass") + { + Assert.IsTrue(rxEventsData.json.Contains("\"eventName\":\"s3:ObjectCreated:Put\"")); + new MintLogger(nameof(ListenBucketNotificationsAsync_Test3), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for no event processing", + TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + } + else if (testState == "fail") + { + throw exception; + } + else if (testState == "completed") + { + throw new Exception("Bucket notification completed without catching the event"); + } + } + catch (Exception ex) + { + new MintLogger(nameof(ListenBucketNotificationsAsync_Test3), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for no event processing", + TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + throw; + } + + finally + { + await TearDown(minio, bucketName); + if (disposable != null) + disposable.Dispose(); + } + } + + #endregion + #region Select Object Content internal static async Task SelectObjectContent_Test(MinioClient minio) From 17fd7a1b31c00fe497f88019867a33cd57a57b84 Mon Sep 17 00:00:00 2001 From: Ersan Date: Wed, 27 Jul 2022 00:53:28 -0700 Subject: [PATCH 4/5] Yet more cleanup --- Minio.Functional.Tests/FunctionalTest.cs | 356 ----------------------- 1 file changed, 356 deletions(-) diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index 251c24f9a..d1a939f06 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -2748,362 +2748,6 @@ internal static async Task ObjectRetentionAsync_Test1(MinioClient minio) #endregion - - #region Bucket Notifications - - internal static async Task ListenBucketNotificationsAsync_Test1(MinioClient minio) - { - var startTime = DateTime.Now; - var bucketName = GetRandomName(15); - var objectName = GetRandomName(10); - var contentType = "application/octet-stream"; - IDisposable subscription = null; - var args = new Dictionary - { - { "bucketName", bucketName }, - { "objectName", objectName }, - { "contentType", contentType }, - { "size", "1KB" } - }; - try - { - await Setup_Test(minio, bucketName); - - var received = new List(); - - var eventsList = new List(); - eventsList.Add(EventType.ObjectCreatedAll); - - var listenArgs = new ListenBucketNotificationsArgs() - .WithBucket(bucketName) - .WithEvents(eventsList); - var events = minio.ListenBucketNotificationsAsync(listenArgs); - subscription = events.Subscribe( - ev => received.Add(ev), - ex => { }, - () => { } - ); - - await PutObject_Tester(minio, bucketName, objectName, null, contentType, - 0, null, rsg.GenerateStreamFromSeed(1 * KB)); - - // wait for notifications - var eventDetected = false; - for (var attempt = 0; attempt < 10; attempt++) - if (received.Count > 0) - { - // Check if there is any unexpected error returned - // and captured in the receivedJson list, like - // "NotImplemented" api error. If so, we throw an exception - // and skip running this test - if (received.Count > 1 && received[1].json.StartsWith("")) - { - // Although the attribute is called "json", - // returned data in list "received" is in xml - // format and it is an error.Here, we convert xml - // into json format. - var receivedJson = XmlStrToJsonStr(received[1].json); - - - // Cleanup the "Error" key encapsulating "receivedJson" - // data. This is required to match and convert json data - // "receivedJson" into class "ErrorResponse" - var len = "{'Error':".Length; - var trimmedFront = receivedJson.Substring(len); - var trimmedFull = trimmedFront.Substring(0, trimmedFront.Length - 1); - - var err = JsonConvert.DeserializeObject(trimmedFull); - - Exception ex = new UnexpectedMinioException(err.Message); - if (err.Code == "NotImplemented") - ex = new NotImplementedException(err.Message); - - throw ex; - } - - var notification = JsonConvert.DeserializeObject(received[0].json); - - if (notification.Records != null) - { - Assert.AreEqual(1, notification.Records.Length); - Assert.IsTrue(notification.Records[0].eventName.Contains("s3:ObjectCreated:Put")); - Assert.IsTrue( - objectName.Contains(HttpUtility.UrlDecode(notification.Records[0].s3.objectMeta.key))); - Assert.IsTrue(contentType.Contains(notification.Records[0].s3.objectMeta.contentType)); - eventDetected = true; - break; - } - } - - // subscription.Dispose(); - if (!eventDetected) - throw new UnexpectedMinioException("Failed to detect the expected bucket notification event."); - - new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for small object", - TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); - } - catch (NotImplementedException ex) - { - new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for small object", - TestStatus.NA, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - } - catch (Exception ex) - { - if (ex.Message == "Listening for bucket notification is specific" + - " only to `minio` server endpoints") - { - // This is expected when bucket notification - // is requested against AWS. - // Check if endPoint is AWS - bool isAWS(string endPoint) - { - var rgx = new Regex("^s3\\.?.*\\.amazonaws\\.com", RegexOptions.IgnoreCase); - var matches = rgx.Matches(endPoint); - return matches.Count > 0; - } - - if (Environment.GetEnvironmentVariable("AWS_ENDPOINT") != null || - isAWS(Environment.GetEnvironmentVariable("SERVER_ENDPOINT"))) - // This is a PASS - new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for small object", - TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); - } - else - { - new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for small object", - TestStatus.FAIL, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - throw; - } - } - finally - { - await TearDown(minio, bucketName); - if (subscription != null) - subscription.Dispose(); - } - } - - - internal static async Task ListenBucketNotificationsAsync_Test2(MinioClient minio) - { - var startTime = DateTime.Now; - var events = new List(); - events.Add(EventType.ObjectCreatedAll); - var rxEvents = new List(); - IDisposable subscription = null; - var bucketName = GetRandomName(15); - var contentType = "application/json"; - var args = new Dictionary - { - { "bucketName", bucketName }, - { "contentType", contentType }, - { "size", "16B" } - }; - - try - { - static Stream ToStream(string input) - { - var stream = new MemoryStream(); - var writer = new StreamWriter(stream); - writer.Write(input); - writer.Flush(); - stream.Position = 0; - - return stream; - } - - var bucketExistsArgs = new BucketExistsArgs() - .WithBucket(bucketName); - var found = await minio.BucketExistsAsync(bucketExistsArgs).ConfigureAwait(false); - if (!found) - { - var makeBucketArgs = new MakeBucketArgs() - .WithBucket(bucketName); - await minio.MakeBucketAsync(makeBucketArgs).ConfigureAwait(false); - } - - void Notify(MinioNotificationRaw data) - { - var notification = JsonConvert.DeserializeObject(data.json); - if (notification is not { Records: { } }) return; - - foreach (var @event in notification.Records) rxEvents.Add(@event); - } - - var listenArgs = new ListenBucketNotificationsArgs() - .WithBucket(bucketName) - .WithEvents(events); - var observable = minio.ListenBucketNotificationsAsync(listenArgs); - - subscription = observable.Subscribe( - ev => Notify(ev), - ex => throw new Exception($"OnError: {ex.Message}"), - () => throw new Exception("STOPPED LISTENING FOR BUCKET NOTIFICATIONS\n")); - - var modelJson = "{\"test\": \"test\"}"; - await using var stream = ToStream(modelJson); - var putObjectArgs = new PutObjectArgs() - .WithObject("test.json") - .WithBucket(bucketName) - .WithContentType(contentType) - .WithStreamData(stream) - .WithObjectSize(stream.Length); - - await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); - - // Waits until the Put event is detected - // Times out if the event is not caught in 3 seconds - var timeoutDuration = 3; // seconds - var stTime = DateTime.UtcNow; - var timeout = TimeSpan.FromSeconds(timeoutDuration); - while (rxEvents.Count < 1) - { - await Task.Delay(25); - if (DateTime.UtcNow - stTime >= timeout) - throw new Exception("Timeout: while waiting for events"); - } - - foreach (var ev in rxEvents) Assert.AreEqual("s3:ObjectCreated:Put", ev.eventName); - - new MintLogger(nameof(ListenBucketNotificationsAsync_Test2), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for longer event processing", - TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); - } - catch (Exception ex) - { - new MintLogger(nameof(ListenBucketNotificationsAsync_Test2), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for longer event processing", - TestStatus.FAIL, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - throw; - } - - finally - { - await TearDown(minio, bucketName); - if (subscription != null) - subscription.Dispose(); - } - } - - internal static async Task ListenBucketNotificationsAsync_Test3(MinioClient minio) - { - var startTime = DateTime.Now; - var events = new List(); - events.Add(EventType.ObjectCreatedAll); - var rxEventsData = new MinioNotificationRaw(""); - IDisposable disposable = null; - var bucketName = GetRandomName(15); - var suffix = ".json"; - var contentType = "application/json"; - var args = new Dictionary - { - { "bucketName", bucketName }, - { "contentType", contentType }, - { "suffix", suffix }, - { "size", "16B" } - }; - - try - { - var bucketExistsArgs = new BucketExistsArgs() - .WithBucket(bucketName); - if (!minio.BucketExistsAsync(bucketExistsArgs).Result) - { - var makeBucketArgs = new MakeBucketArgs() - .WithBucket(bucketName); - minio.MakeBucketAsync(makeBucketArgs).Wait(); - } - - var notificationsArgs = new ListenBucketNotificationsArgs() - .WithBucket(bucketName) - .WithSuffix(suffix) - .WithEvents(events); - - var notifications = minio.ListenBucketNotificationsAsync(notificationsArgs); - - var testState = "fail"; - Exception exception = null; - disposable = notifications.Subscribe( - x => - { - rxEventsData = x; - testState = "pass"; - }, - ex => - { - exception = ex; - testState = "fail"; - }, - () => { testState = "completed"; }); - - var modelJson = "{\"test\": \"test\"}"; - var stream = new MemoryStream(); - var writer = new StreamWriter(stream); - writer.Write(modelJson); - writer.Flush(); - stream.Position = 0; - - var putObjectArgs = new PutObjectArgs() - .WithObject("test.json") - .WithBucket(bucketName) - .WithContentType(contentType) - .WithStreamData(stream) - .WithObjectSize(stream.Length); - - await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); - Thread.Sleep(1000); - - if (testState == "pass") - { - Assert.IsTrue(rxEventsData.json.Contains("\"eventName\":\"s3:ObjectCreated:Put\"")); - new MintLogger(nameof(ListenBucketNotificationsAsync_Test3), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for no event processing", - TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); - } - else if (testState == "fail") - { - throw exception; - } - else if (testState == "completed") - { - throw new Exception("Bucket notification completed without catching the event"); - } - } - catch (Exception ex) - { - new MintLogger(nameof(ListenBucketNotificationsAsync_Test3), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for no event processing", - TestStatus.FAIL, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - throw; - } - - finally - { - await TearDown(minio, bucketName); - if (disposable != null) - disposable.Dispose(); - } - } - - #endregion - #region Make Bucket internal static async Task MakeBucket_Test1(MinioClient minio) From 85df756e4d5f38a223e5f26fdb3905bb74526503 Mon Sep 17 00:00:00 2001 From: Ersan Date: Wed, 27 Jul 2022 01:19:24 -0700 Subject: [PATCH 5/5] regitlint changes --- Minio.Functional.Tests/FunctionalTest.cs | 1392 +++++++++++----------- 1 file changed, 696 insertions(+), 696 deletions(-) diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index d1a939f06..e062201bc 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -1352,620 +1352,265 @@ internal static async Task RemoveIncompleteUpload_Test(MinioClient minio) } } + #region Select Object Content - #region Bucket Notifications - - internal static async Task ListenBucketNotificationsAsync_Test1(MinioClient minio) + internal static async Task SelectObjectContent_Test(MinioClient minio) { var startTime = DateTime.Now; var bucketName = GetRandomName(15); - var objectName = GetRandomName(10); - var contentType = "application/octet-stream"; - IDisposable subscription = null; + var objectName = GetRandomObjectName(10); + var outFileName = "outFileName"; var args = new Dictionary { { "bucketName", bucketName }, { "objectName", objectName }, - { "contentType", contentType }, - { "size", "1KB" } + { "fileName", outFileName } }; try { await Setup_Test(minio, bucketName); + var csvString = new StringBuilder(); + csvString.AppendLine("Employee,Manager,Group"); + csvString.AppendLine("Employee4,Employee2,500"); + csvString.AppendLine("Employee3,Employee1,500"); + csvString.AppendLine("Employee1,,1000"); + csvString.AppendLine("Employee5,Employee1,500"); + csvString.AppendLine("Employee2,Employee1,800"); + var csvBytes = Encoding.UTF8.GetBytes(csvString.ToString()); + using (var stream = new MemoryStream(csvBytes)) + { + var putObjectArgs = new PutObjectArgs() + .WithBucket(bucketName) + .WithObject(objectName) + .WithStreamData(stream) + .WithObjectSize(stream.Length); + await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); + } - var received = new List(); - - var eventsList = new List(); - eventsList.Add(EventType.ObjectCreatedAll); - - var listenArgs = new ListenBucketNotificationsArgs() - .WithBucket(bucketName) - .WithEvents(eventsList); - var events = minio.ListenBucketNotificationsAsync(listenArgs); - subscription = events.Subscribe( - ev => received.Add(ev), - ex => { }, - () => { } - ); - - await PutObject_Tester(minio, bucketName, objectName, null, contentType, - 0, null, rsg.GenerateStreamFromSeed(1 * KB)); - - // wait for notifications - var eventDetected = false; - for (var attempt = 0; attempt < 10; attempt++) - if (received.Count > 0) + var inputSerialization = new SelectObjectInputSerialization + { + CompressionType = SelectCompressionType.NONE, + CSV = new CSVInputOptions { - // Check if there is any unexpected error returned - // and captured in the receivedJson list, like - // "NotImplemented" api error. If so, we throw an exception - // and skip running this test - if (received.Count > 1 && received[1].json.StartsWith("")) - { - // Although the attribute is called "json", - // returned data in list "received" is in xml - // format and it is an error.Here, we convert xml - // into json format. - var receivedJson = XmlStrToJsonStr(received[1].json); - - - // Cleanup the "Error" key encapsulating "receivedJson" - // data. This is required to match and convert json data - // "receivedJson" into class "ErrorResponse" - var len = "{'Error':".Length; - var trimmedFront = receivedJson.Substring(len); - var trimmedFull = trimmedFront.Substring(0, trimmedFront.Length - 1); - - var err = JsonConvert.DeserializeObject(trimmedFull); - - Exception ex = new UnexpectedMinioException(err.Message); - if (err.Code == "NotImplemented") - ex = new NotImplementedException(err.Message); - - throw ex; - } - - var notification = JsonConvert.DeserializeObject(received[0].json); - - if (notification.Records != null) - { - Assert.AreEqual(1, notification.Records.Length); - Assert.IsTrue(notification.Records[0].eventName.Contains("s3:ObjectCreated:Put")); - Assert.IsTrue( - objectName.Contains(HttpUtility.UrlDecode(notification.Records[0].s3.objectMeta.key))); - Assert.IsTrue(contentType.Contains(notification.Records[0].s3.objectMeta.contentType)); - eventDetected = true; - break; - } + FileHeaderInfo = CSVFileHeaderInfo.None, + RecordDelimiter = "\n", + FieldDelimiter = "," } - - // subscription.Dispose(); - if (!eventDetected) - throw new UnexpectedMinioException("Failed to detect the expected bucket notification event."); - - new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for small object", - TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); - } - catch (NotImplementedException ex) - { - new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for small object", - TestStatus.NA, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - } - catch (Exception ex) - { - if (ex.Message == "Listening for bucket notification is specific" + - " only to `minio` server endpoints") + }; + var outputSerialization = new SelectObjectOutputSerialization { - // This is expected when bucket notification - // is requested against AWS. - // Check if endPoint is AWS - bool isAWS(string endPoint) + CSV = new CSVOutputOptions { - var rgx = new Regex("^s3\\.?.*\\.amazonaws\\.com", RegexOptions.IgnoreCase); - var matches = rgx.Matches(endPoint); - return matches.Count > 0; + RecordDelimiter = "\n", + FieldDelimiter = "," } + }; + var selArgs = new SelectObjectContentArgs() + .WithBucket(bucketName) + .WithObject(objectName) + .WithExpressionType(QueryExpressionType.SQL) + .WithQueryExpression("select * from s3object") + .WithInputSerialization(inputSerialization) + .WithOutputSerialization(outputSerialization); + var resp = await minio.SelectObjectContentAsync(selArgs).ConfigureAwait(false); + var output = await new StreamReader(resp.Payload).ReadToEndAsync().ConfigureAwait(false); + var csvStringNoWS = Regex.Replace(csvString.ToString(), @"\s+", ""); + var outputNoWS = Regex.Replace(output, @"\s+", ""); + // Compute MD5 for a better result. + var hashedOutputBytes = MD5 + .Create() + .ComputeHash(Encoding.UTF8.GetBytes(outputNoWS)); + var outputMd5 = Convert.ToBase64String(hashedOutputBytes); + var hashedCSVBytes = MD5 + .Create() + .ComputeHash(Encoding.UTF8.GetBytes(csvStringNoWS)); + var csvMd5 = Convert.ToBase64String(hashedCSVBytes); - if (Environment.GetEnvironmentVariable("AWS_ENDPOINT") != null || - isAWS(Environment.GetEnvironmentVariable("SERVER_ENDPOINT"))) - // This is a PASS - new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for small object", - TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); - } - else - { - new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for small object", - TestStatus.FAIL, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - throw; - } + Assert.IsTrue(csvMd5.Contains(outputMd5)); + new MintLogger("SelectObjectContent_Test", selectObjectSignature, + "Tests whether SelectObjectContent passes for a select query", TestStatus.PASS, + DateTime.Now - startTime, args: args).Log(); + } + catch (Exception ex) + { + new MintLogger("SelectObjectContent_Test", selectObjectSignature, + "Tests whether SelectObjectContent passes for a select query", TestStatus.FAIL, + DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); + throw; } finally { await TearDown(minio, bucketName); - if (subscription != null) - subscription.Dispose(); + File.Delete(outFileName); } } - internal static async Task ListenBucketNotificationsAsync_Test2(MinioClient minio) + #endregion + + + #region Bucket Encryption + + internal static async Task BucketEncryptionsAsync_Test1(MinioClient minio) { var startTime = DateTime.Now; - var events = new List(); - events.Add(EventType.ObjectCreatedAll); - var rxEvents = new List(); - IDisposable subscription = null; var bucketName = GetRandomName(15); - var contentType = "application/json"; var args = new Dictionary { - { "bucketName", bucketName }, - { "contentType", contentType }, - { "size", "16B" } + { "bucketName", bucketName } }; - try { - static Stream ToStream(string input) - { - var stream = new MemoryStream(); - var writer = new StreamWriter(stream); - writer.Write(input); - writer.Flush(); - stream.Position = 0; - - return stream; - } + await Setup_Test(minio, bucketName); + } + catch (Exception ex) + { + await TearDown(minio, bucketName); + new MintLogger(nameof(BucketEncryptionsAsync_Test1), setBucketEncryptionSignature, + "Tests whether SetBucketEncryptionAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + throw; + } - var bucketExistsArgs = new BucketExistsArgs() + try + { + var encryptionArgs = new SetBucketEncryptionArgs() .WithBucket(bucketName); - var found = await minio.BucketExistsAsync(bucketExistsArgs).ConfigureAwait(false); - if (!found) - { - var makeBucketArgs = new MakeBucketArgs() - .WithBucket(bucketName); - await minio.MakeBucketAsync(makeBucketArgs).ConfigureAwait(false); - } + await minio.SetBucketEncryptionAsync(encryptionArgs).ConfigureAwait(false); + new MintLogger(nameof(BucketEncryptionsAsync_Test1), setBucketEncryptionSignature, + "Tests whether SetBucketEncryptionAsync passes", TestStatus.PASS, DateTime.Now - startTime, + args: args) + .Log(); + } + catch (NotImplementedException ex) + { + new MintLogger(nameof(BucketEncryptionsAsync_Test1), setBucketEncryptionSignature, + "Tests whether SetBucketEncryptionAsync passes", TestStatus.NA, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + } + catch (Exception ex) + { + await TearDown(minio, bucketName); + new MintLogger(nameof(BucketEncryptionsAsync_Test1), setBucketEncryptionSignature, + "Tests whether SetBucketEncryptionAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + throw; + } - void Notify(MinioNotificationRaw data) - { - var notification = JsonConvert.DeserializeObject(data.json); - if (notification is not { Records: { } }) return; - - foreach (var @event in notification.Records) rxEvents.Add(@event); - } - - var listenArgs = new ListenBucketNotificationsArgs() - .WithBucket(bucketName) - .WithEvents(events); - var observable = minio.ListenBucketNotificationsAsync(listenArgs); - - subscription = observable.Subscribe( - ev => Notify(ev), - ex => throw new Exception($"OnError: {ex.Message}"), - () => throw new Exception("STOPPED LISTENING FOR BUCKET NOTIFICATIONS\n")); - - var modelJson = "{\"test\": \"test\"}"; - await using var stream = ToStream(modelJson); - var putObjectArgs = new PutObjectArgs() - .WithObject("test.json") - .WithBucket(bucketName) - .WithContentType(contentType) - .WithStreamData(stream) - .WithObjectSize(stream.Length); - - await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); - - // Waits until the Put event is detected - // Times out if the event is not caught in 3 seconds - var timeoutDuration = 3; // seconds - var stTime = DateTime.UtcNow; - var timeout = TimeSpan.FromSeconds(timeoutDuration); - while (rxEvents.Count < 1) - { - await Task.Delay(25); - if (DateTime.UtcNow - stTime >= timeout) - throw new Exception("Timeout: while waiting for events"); - } - - foreach (var ev in rxEvents) Assert.AreEqual("s3:ObjectCreated:Put", ev.eventName); - - new MintLogger(nameof(ListenBucketNotificationsAsync_Test2), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for longer event processing", - TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + try + { + var encryptionArgs = new GetBucketEncryptionArgs() + .WithBucket(bucketName); + var config = await minio.GetBucketEncryptionAsync(encryptionArgs).ConfigureAwait(false); + Assert.IsNotNull(config); + Assert.IsNotNull(config.Rule); + Assert.IsNotNull(config.Rule.Apply); + Assert.IsTrue(config.Rule.Apply.SSEAlgorithm.Contains("AES256")); + new MintLogger(nameof(BucketEncryptionsAsync_Test1), getBucketEncryptionSignature, + "Tests whether GetBucketEncryptionAsync passes", TestStatus.PASS, DateTime.Now - startTime, + args: args) + .Log(); } - catch (Exception ex) + catch (NotImplementedException ex) { - new MintLogger(nameof(ListenBucketNotificationsAsync_Test2), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for longer event processing", - TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + new MintLogger(nameof(BucketEncryptionsAsync_Test1), getBucketEncryptionSignature, + "Tests whether GetBucketEncryptionAsync passes", TestStatus.NA, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); - throw; } - - finally + catch (Exception ex) { await TearDown(minio, bucketName); - if (subscription != null) - subscription.Dispose(); + new MintLogger(nameof(BucketEncryptionsAsync_Test1), getBucketEncryptionSignature, + "Tests whether GetBucketEncryptionAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + throw; } - } - - internal static async Task ListenBucketNotificationsAsync_Test3(MinioClient minio) - { - var startTime = DateTime.Now; - var events = new List(); - events.Add(EventType.ObjectCreatedAll); - var rxEventsData = new MinioNotificationRaw(""); - IDisposable disposable = null; - var bucketName = GetRandomName(15); - var suffix = ".json"; - var contentType = "application/json"; - var args = new Dictionary - { - { "bucketName", bucketName }, - { "contentType", contentType }, - { "suffix", suffix }, - { "size", "16B" } - }; try { - var bucketExistsArgs = new BucketExistsArgs() + var rmEncryptionArgs = new RemoveBucketEncryptionArgs() .WithBucket(bucketName); - if (!minio.BucketExistsAsync(bucketExistsArgs).Result) - { - var makeBucketArgs = new MakeBucketArgs() - .WithBucket(bucketName); - minio.MakeBucketAsync(makeBucketArgs).Wait(); - } - - var notificationsArgs = new ListenBucketNotificationsArgs() - .WithBucket(bucketName) - .WithSuffix(suffix) - .WithEvents(events); - - var notifications = minio.ListenBucketNotificationsAsync(notificationsArgs); - - var testState = "fail"; - Exception exception = null; - disposable = notifications.Subscribe( - x => - { - rxEventsData = x; - testState = "pass"; - }, - ex => - { - exception = ex; - testState = "fail"; - }, - () => { testState = "completed"; }); - - var modelJson = "{\"test\": \"test\"}"; - var stream = new MemoryStream(); - var writer = new StreamWriter(stream); - writer.Write(modelJson); - writer.Flush(); - stream.Position = 0; - - var putObjectArgs = new PutObjectArgs() - .WithObject("test.json") - .WithBucket(bucketName) - .WithContentType(contentType) - .WithStreamData(stream) - .WithObjectSize(stream.Length); - - await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); - Thread.Sleep(1000); - - if (testState == "pass") - { - Assert.IsTrue(rxEventsData.json.Contains("\"eventName\":\"s3:ObjectCreated:Put\"")); - new MintLogger(nameof(ListenBucketNotificationsAsync_Test3), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for no event processing", - TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); - } - else if (testState == "fail") + await minio.RemoveBucketEncryptionAsync(rmEncryptionArgs).ConfigureAwait(false); + var encryptionArgs = new GetBucketEncryptionArgs() + .WithBucket(bucketName); + var config = await minio.GetBucketEncryptionAsync(encryptionArgs).ConfigureAwait(false); + } + catch (NotImplementedException ex) + { + new MintLogger(nameof(BucketEncryptionsAsync_Test1), removeBucketEncryptionSignature, + "Tests whether RemoveBucketEncryptionAsync passes", TestStatus.NA, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + } + catch (Exception ex) + { + if (ex.Message.Contains("The server side encryption configuration was not found")) { - throw exception; + new MintLogger(nameof(BucketEncryptionsAsync_Test1), removeBucketEncryptionSignature, + "Tests whether RemoveBucketEncryptionAsync passes", TestStatus.PASS, DateTime.Now - startTime, + args: args).Log(); } - else if (testState == "completed") + else { - throw new Exception("Bucket notification completed without catching the event"); + new MintLogger(nameof(BucketEncryptionsAsync_Test1), removeBucketEncryptionSignature, + "Tests whether RemoveBucketEncryptionAsync passes", TestStatus.FAIL, DateTime.Now - startTime, + ex.Message, ex.ToString(), args: args).Log(); + throw; } } - catch (Exception ex) - { - new MintLogger(nameof(ListenBucketNotificationsAsync_Test3), - listenBucketNotificationsSignature, - "Tests whether ListenBucketNotifications passes for no event processing", - TestStatus.FAIL, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - throw; - } - finally { await TearDown(minio, bucketName); - if (disposable != null) - disposable.Dispose(); } } #endregion - #region Select Object Content + #region Legal Hold Status - internal static async Task SelectObjectContent_Test(MinioClient minio) + internal static async Task LegalHoldStatusAsync_Test1(MinioClient minio) { var startTime = DateTime.Now; var bucketName = GetRandomName(15); var objectName = GetRandomObjectName(10); - var outFileName = "outFileName"; var args = new Dictionary { { "bucketName", bucketName }, - { "objectName", objectName }, - { "fileName", outFileName } + { "objectName", objectName } }; try { - await Setup_Test(minio, bucketName); - var csvString = new StringBuilder(); - csvString.AppendLine("Employee,Manager,Group"); - csvString.AppendLine("Employee4,Employee2,500"); - csvString.AppendLine("Employee3,Employee1,500"); - csvString.AppendLine("Employee1,,1000"); - csvString.AppendLine("Employee5,Employee1,500"); - csvString.AppendLine("Employee2,Employee1,800"); - var csvBytes = Encoding.UTF8.GetBytes(csvString.ToString()); - using (var stream = new MemoryStream(csvBytes)) + await Setup_WithLock_Test(minio, bucketName); + } + catch (NotImplementedException ex) + { + await TearDown(minio, bucketName); + new MintLogger(nameof(LegalHoldStatusAsync_Test1), setObjectLegalHoldSignature, + "Tests whether SetObjectLegalHoldAsync passes", TestStatus.NA, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + return; + } + catch (Exception ex) + { + await TearDown(minio, bucketName); + new MintLogger(nameof(LegalHoldStatusAsync_Test1), setObjectLegalHoldSignature, + "Tests whether SetObjectLegalHoldAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + throw; + } + + try + { + using (var filestream = rsg.GenerateStreamFromSeed(1 * KB)) { var putObjectArgs = new PutObjectArgs() .WithBucket(bucketName) .WithObject(objectName) - .WithStreamData(stream) - .WithObjectSize(stream.Length); + .WithStreamData(filestream) + .WithObjectSize(filestream.Length) + .WithContentType(null); await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); } - var inputSerialization = new SelectObjectInputSerialization - { - CompressionType = SelectCompressionType.NONE, - CSV = new CSVInputOptions - { - FileHeaderInfo = CSVFileHeaderInfo.None, - RecordDelimiter = "\n", - FieldDelimiter = "," - } - }; - var outputSerialization = new SelectObjectOutputSerialization - { - CSV = new CSVOutputOptions - { - RecordDelimiter = "\n", - FieldDelimiter = "," - } - }; - var selArgs = new SelectObjectContentArgs() - .WithBucket(bucketName) - .WithObject(objectName) - .WithExpressionType(QueryExpressionType.SQL) - .WithQueryExpression("select * from s3object") - .WithInputSerialization(inputSerialization) - .WithOutputSerialization(outputSerialization); - var resp = await minio.SelectObjectContentAsync(selArgs).ConfigureAwait(false); - var output = await new StreamReader(resp.Payload).ReadToEndAsync().ConfigureAwait(false); - var csvStringNoWS = Regex.Replace(csvString.ToString(), @"\s+", ""); - var outputNoWS = Regex.Replace(output, @"\s+", ""); - // Compute MD5 for a better result. - var hashedOutputBytes = MD5 - .Create() - .ComputeHash(Encoding.UTF8.GetBytes(outputNoWS)); - var outputMd5 = Convert.ToBase64String(hashedOutputBytes); - var hashedCSVBytes = MD5 - .Create() - .ComputeHash(Encoding.UTF8.GetBytes(csvStringNoWS)); - var csvMd5 = Convert.ToBase64String(hashedCSVBytes); - - Assert.IsTrue(csvMd5.Contains(outputMd5)); - new MintLogger("SelectObjectContent_Test", selectObjectSignature, - "Tests whether SelectObjectContent passes for a select query", TestStatus.PASS, - DateTime.Now - startTime, args: args).Log(); - } - catch (Exception ex) - { - new MintLogger("SelectObjectContent_Test", selectObjectSignature, - "Tests whether SelectObjectContent passes for a select query", TestStatus.FAIL, - DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); - throw; - } - finally - { - await TearDown(minio, bucketName); - File.Delete(outFileName); - } - } - - #endregion - - - #region Bucket Encryption - - internal static async Task BucketEncryptionsAsync_Test1(MinioClient minio) - { - var startTime = DateTime.Now; - var bucketName = GetRandomName(15); - var args = new Dictionary - { - { "bucketName", bucketName } - }; - try - { - await Setup_Test(minio, bucketName); - } - catch (Exception ex) - { - await TearDown(minio, bucketName); - new MintLogger(nameof(BucketEncryptionsAsync_Test1), setBucketEncryptionSignature, - "Tests whether SetBucketEncryptionAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - throw; - } - - try - { - var encryptionArgs = new SetBucketEncryptionArgs() - .WithBucket(bucketName); - await minio.SetBucketEncryptionAsync(encryptionArgs).ConfigureAwait(false); - new MintLogger(nameof(BucketEncryptionsAsync_Test1), setBucketEncryptionSignature, - "Tests whether SetBucketEncryptionAsync passes", TestStatus.PASS, DateTime.Now - startTime, - args: args) - .Log(); - } - catch (NotImplementedException ex) - { - new MintLogger(nameof(BucketEncryptionsAsync_Test1), setBucketEncryptionSignature, - "Tests whether SetBucketEncryptionAsync passes", TestStatus.NA, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - } - catch (Exception ex) - { - await TearDown(minio, bucketName); - new MintLogger(nameof(BucketEncryptionsAsync_Test1), setBucketEncryptionSignature, - "Tests whether SetBucketEncryptionAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - throw; - } - - try - { - var encryptionArgs = new GetBucketEncryptionArgs() - .WithBucket(bucketName); - var config = await minio.GetBucketEncryptionAsync(encryptionArgs).ConfigureAwait(false); - Assert.IsNotNull(config); - Assert.IsNotNull(config.Rule); - Assert.IsNotNull(config.Rule.Apply); - Assert.IsTrue(config.Rule.Apply.SSEAlgorithm.Contains("AES256")); - new MintLogger(nameof(BucketEncryptionsAsync_Test1), getBucketEncryptionSignature, - "Tests whether GetBucketEncryptionAsync passes", TestStatus.PASS, DateTime.Now - startTime, - args: args) - .Log(); - } - catch (NotImplementedException ex) - { - new MintLogger(nameof(BucketEncryptionsAsync_Test1), getBucketEncryptionSignature, - "Tests whether GetBucketEncryptionAsync passes", TestStatus.NA, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - } - catch (Exception ex) - { - await TearDown(minio, bucketName); - new MintLogger(nameof(BucketEncryptionsAsync_Test1), getBucketEncryptionSignature, - "Tests whether GetBucketEncryptionAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - throw; - } - - try - { - var rmEncryptionArgs = new RemoveBucketEncryptionArgs() - .WithBucket(bucketName); - await minio.RemoveBucketEncryptionAsync(rmEncryptionArgs).ConfigureAwait(false); - var encryptionArgs = new GetBucketEncryptionArgs() - .WithBucket(bucketName); - var config = await minio.GetBucketEncryptionAsync(encryptionArgs).ConfigureAwait(false); - } - catch (NotImplementedException ex) - { - new MintLogger(nameof(BucketEncryptionsAsync_Test1), removeBucketEncryptionSignature, - "Tests whether RemoveBucketEncryptionAsync passes", TestStatus.NA, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - } - catch (Exception ex) - { - if (ex.Message.Contains("The server side encryption configuration was not found")) - { - new MintLogger(nameof(BucketEncryptionsAsync_Test1), removeBucketEncryptionSignature, - "Tests whether RemoveBucketEncryptionAsync passes", TestStatus.PASS, DateTime.Now - startTime, - args: args).Log(); - } - else - { - new MintLogger(nameof(BucketEncryptionsAsync_Test1), removeBucketEncryptionSignature, - "Tests whether RemoveBucketEncryptionAsync passes", TestStatus.FAIL, DateTime.Now - startTime, - ex.Message, ex.ToString(), args: args).Log(); - throw; - } - } - finally - { - await TearDown(minio, bucketName); - } - } - - #endregion - - #region Legal Hold Status - - internal static async Task LegalHoldStatusAsync_Test1(MinioClient minio) - { - var startTime = DateTime.Now; - var bucketName = GetRandomName(15); - var objectName = GetRandomObjectName(10); - var args = new Dictionary - { - { "bucketName", bucketName }, - { "objectName", objectName } - }; - try - { - await Setup_WithLock_Test(minio, bucketName); - } - catch (NotImplementedException ex) - { - await TearDown(minio, bucketName); - new MintLogger(nameof(LegalHoldStatusAsync_Test1), setObjectLegalHoldSignature, - "Tests whether SetObjectLegalHoldAsync passes", TestStatus.NA, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - return; - } - catch (Exception ex) - { - await TearDown(minio, bucketName); - new MintLogger(nameof(LegalHoldStatusAsync_Test1), setObjectLegalHoldSignature, - "Tests whether SetObjectLegalHoldAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - throw; - } - - try - { - using (var filestream = rsg.GenerateStreamFromSeed(1 * KB)) - { - var putObjectArgs = new PutObjectArgs() - .WithBucket(bucketName) - .WithObject(objectName) - .WithStreamData(filestream) - .WithObjectSize(filestream.Length) - .WithContentType(null); - await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); - } - - var legalHoldArgs = new SetObjectLegalHoldArgs() + var legalHoldArgs = new SetObjectLegalHoldArgs() .WithBucket(bucketName) .WithObject(objectName) .WithLegalHold(true); @@ -2543,207 +2188,562 @@ internal static async Task ObjectLockConfigurationAsync_Test1(MinioClient minio) { if (setLockNotImplemented || getLockNotImplemented) { - // Cannot test Remove Object Lock with Set & Get Object Lock implemented. - new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), deleteObjectLockConfigurationSignature, - "Tests whether RemoveObjectLockConfigurationAsync passes", TestStatus.NA, DateTime.Now - startTime, - "Functionality that is not implemented", "", args: args).Log(); - await TearDown(minio, bucketName); - return; + // Cannot test Remove Object Lock with Set & Get Object Lock implemented. + new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), deleteObjectLockConfigurationSignature, + "Tests whether RemoveObjectLockConfigurationAsync passes", TestStatus.NA, DateTime.Now - startTime, + "Functionality that is not implemented", "", args: args).Log(); + await TearDown(minio, bucketName); + return; + } + + var objectLockArgs = new RemoveObjectLockConfigurationArgs() + .WithBucket(bucketName); + await minio.RemoveObjectLockConfigurationAsync(objectLockArgs).ConfigureAwait(false); + var getObjectLockArgs = new GetObjectLockConfigurationArgs() + .WithBucket(bucketName); + var config = await minio.GetObjectLockConfigurationAsync(getObjectLockArgs).ConfigureAwait(false); + Assert.IsNotNull(config); + Assert.IsNull(config.Rule); + new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), deleteObjectLockConfigurationSignature, + "Tests whether RemoveObjectLockConfigurationAsync passes", TestStatus.PASS, DateTime.Now - startTime, + args: args).Log(); + } + catch (NotImplementedException ex) + { + new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), deleteObjectLockConfigurationSignature, + "Tests whether RemoveObjectLockConfigurationAsync passes", TestStatus.NA, DateTime.Now - startTime, + ex.Message, ex.ToString(), args: args).Log(); + } + catch (Exception ex) + { + new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), deleteObjectLockConfigurationSignature, + "Tests whether RemoveObjectLockConfigurationAsync passes", TestStatus.FAIL, DateTime.Now - startTime, + ex.Message, ex.ToString(), args: args).Log(); + throw; + } + finally + { + Thread.Sleep(1500); + await TearDown(minio, bucketName); + } + } + + #endregion + + + #region Object Retention + + internal static async Task ObjectRetentionAsync_Test1(MinioClient minio) + { + var startTime = DateTime.Now; + var bucketName = GetRandomName(15); + var objectName = GetRandomObjectName(10); + var args = new Dictionary + { + { "bucketName", bucketName }, + { "objectName", objectName } + }; + + try + { + await Setup_WithLock_Test(minio, bucketName); + } + catch (NotImplementedException ex) + { + await TearDown(minio, bucketName); + new MintLogger(nameof(ObjectRetentionAsync_Test1), setObjectRetentionSignature, + "Tests whether SetObjectRetentionAsync passes", TestStatus.NA, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + return; + } + catch (Exception ex) + { + await TearDown(minio, bucketName); + new MintLogger(nameof(ObjectRetentionAsync_Test1), setObjectRetentionSignature, + "Tests whether SetObjectRetentionAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + throw; + } + + try + { + var plusDays = 10; + using (var filestream = rsg.GenerateStreamFromSeed(1 * KB)) + { + var putObjectArgs = new PutObjectArgs() + .WithBucket(bucketName) + .WithObject(objectName) + .WithStreamData(filestream) + .WithObjectSize(filestream.Length) + .WithContentType(null); + await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); + } + + var untilDate = DateTime.Now.AddDays(plusDays); + var setRetentionArgs = new SetObjectRetentionArgs() + .WithBucket(bucketName) + .WithObject(objectName) + .WithRetentionMode(RetentionMode.GOVERNANCE) + .WithRetentionUntilDate(untilDate); + await minio.SetObjectRetentionAsync(setRetentionArgs).ConfigureAwait(false); + new MintLogger(nameof(ObjectRetentionAsync_Test1), setObjectRetentionSignature, + "Tests whether SetObjectRetentionAsync passes", TestStatus.PASS, DateTime.Now - startTime, + args: args) + .Log(); + } + catch (NotImplementedException ex) + { + new MintLogger(nameof(ObjectRetentionAsync_Test1), setObjectRetentionSignature, + "Tests whether SetObjectRetentionAsync passes", TestStatus.NA, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + } + catch (Exception ex) + { + await TearDown(minio, bucketName); + new MintLogger(nameof(ObjectRetentionAsync_Test1), setObjectRetentionSignature, + "Tests whether SetObjectRetentionAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + throw; + } + + try + { + var getRetentionArgs = new GetObjectRetentionArgs() + .WithBucket(bucketName) + .WithObject(objectName); + var config = await minio.GetObjectRetentionAsync(getRetentionArgs).ConfigureAwait(false); + var plusDays = 10.0; + Assert.IsNotNull(config); + Assert.AreEqual(config.Mode, RetentionMode.GOVERNANCE); + var untilDate = DateTime.Parse(config.RetainUntilDate, null, DateTimeStyles.RoundtripKind); + Assert.AreEqual(Math.Ceiling((untilDate - DateTime.Now).TotalDays), plusDays); + new MintLogger(nameof(ObjectRetentionAsync_Test1), getObjectRetentionSignature, + "Tests whether GetObjectRetentionAsync passes", TestStatus.PASS, DateTime.Now - startTime, + args: args) + .Log(); + } + catch (NotImplementedException ex) + { + new MintLogger(nameof(ObjectRetentionAsync_Test1), getObjectRetentionSignature, + "Tests whether GetObjectRetentionAsync passes", TestStatus.NA, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + } + catch (Exception ex) + { + await TearDown(minio, bucketName); + new MintLogger(nameof(ObjectRetentionAsync_Test1), getObjectRetentionSignature, + "Tests whether GetObjectRetentionAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + throw; + } + + try + { + var clearRetentionArgs = new ClearObjectRetentionArgs() + .WithBucket(bucketName) + .WithObject(objectName); + await minio.ClearObjectRetentionAsync(clearRetentionArgs).ConfigureAwait(false); + var getRetentionArgs = new GetObjectRetentionArgs() + .WithBucket(bucketName) + .WithObject(objectName); + var config = await minio.GetObjectRetentionAsync(getRetentionArgs).ConfigureAwait(false); + } + catch (NotImplementedException ex) + { + new MintLogger(nameof(ObjectRetentionAsync_Test1), clearObjectRetentionSignature, + "Tests whether ClearObjectRetentionAsync passes", TestStatus.NA, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + } + catch (Exception ex) + { + var errMsgLock = ex.Message.Contains("The specified object does not have a ObjectLock configuration"); + if (errMsgLock) + { + new MintLogger(nameof(ObjectRetentionAsync_Test1), clearObjectRetentionSignature, + "Tests whether ClearObjectRetentionAsync passes", TestStatus.PASS, DateTime.Now - startTime, + args: args).Log(); + } + else + { + new MintLogger(nameof(ObjectRetentionAsync_Test1), clearObjectRetentionSignature, + "Tests whether ClearObjectRetentionAsync passes", TestStatus.FAIL, DateTime.Now - startTime, + ex.Message, ex.ToString(), args: args).Log(); + await TearDown(minio, bucketName); + throw; + } + } + + try + { + var rmArgs = new RemoveObjectArgs() + .WithBucket(bucketName) + .WithObject(objectName); + + await minio.RemoveObjectAsync(rmArgs).ConfigureAwait(false); + await TearDown(minio, bucketName); + } + catch (Exception ex) + { + new MintLogger(nameof(ObjectRetentionAsync_Test1), clearObjectRetentionSignature, + "TearDown operation ClearObjectRetentionAsync", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + throw; + } + } + + #endregion + + + #region Bucket Notifications + + internal static async Task ListenBucketNotificationsAsync_Test1(MinioClient minio) + { + var startTime = DateTime.Now; + var bucketName = GetRandomName(15); + var objectName = GetRandomName(10); + var contentType = "application/octet-stream"; + IDisposable subscription = null; + var args = new Dictionary + { + { "bucketName", bucketName }, + { "objectName", objectName }, + { "contentType", contentType }, + { "size", "1KB" } + }; + try + { + await Setup_Test(minio, bucketName); + + var received = new List(); + + var eventsList = new List(); + eventsList.Add(EventType.ObjectCreatedAll); + + var listenArgs = new ListenBucketNotificationsArgs() + .WithBucket(bucketName) + .WithEvents(eventsList); + var events = minio.ListenBucketNotificationsAsync(listenArgs); + subscription = events.Subscribe( + ev => received.Add(ev), + ex => { }, + () => { } + ); + + await PutObject_Tester(minio, bucketName, objectName, null, contentType, + 0, null, rsg.GenerateStreamFromSeed(1 * KB)); + + // wait for notifications + var eventDetected = false; + for (var attempt = 0; attempt < 10; attempt++) + if (received.Count > 0) + { + // Check if there is any unexpected error returned + // and captured in the receivedJson list, like + // "NotImplemented" api error. If so, we throw an exception + // and skip running this test + if (received.Count > 1 && received[1].json.StartsWith("")) + { + // Although the attribute is called "json", + // returned data in list "received" is in xml + // format and it is an error.Here, we convert xml + // into json format. + var receivedJson = XmlStrToJsonStr(received[1].json); + + + // Cleanup the "Error" key encapsulating "receivedJson" + // data. This is required to match and convert json data + // "receivedJson" into class "ErrorResponse" + var len = "{'Error':".Length; + var trimmedFront = receivedJson.Substring(len); + var trimmedFull = trimmedFront.Substring(0, trimmedFront.Length - 1); + + var err = JsonConvert.DeserializeObject(trimmedFull); + + Exception ex = new UnexpectedMinioException(err.Message); + if (err.Code == "NotImplemented") + ex = new NotImplementedException(err.Message); + + throw ex; + } + + var notification = JsonConvert.DeserializeObject(received[0].json); + + if (notification.Records != null) + { + Assert.AreEqual(1, notification.Records.Length); + Assert.IsTrue(notification.Records[0].eventName.Contains("s3:ObjectCreated:Put")); + Assert.IsTrue( + objectName.Contains(HttpUtility.UrlDecode(notification.Records[0].s3.objectMeta.key))); + Assert.IsTrue(contentType.Contains(notification.Records[0].s3.objectMeta.contentType)); + eventDetected = true; + break; + } + } + + // subscription.Dispose(); + if (!eventDetected) + throw new UnexpectedMinioException("Failed to detect the expected bucket notification event."); + + new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for small object", + TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + } + catch (NotImplementedException ex) + { + new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for small object", + TestStatus.NA, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + } + catch (Exception ex) + { + if (ex.Message == "Listening for bucket notification is specific" + + " only to `minio` server endpoints") + { + // This is expected when bucket notification + // is requested against AWS. + // Check if endPoint is AWS + bool isAWS(string endPoint) + { + var rgx = new Regex("^s3\\.?.*\\.amazonaws\\.com", RegexOptions.IgnoreCase); + var matches = rgx.Matches(endPoint); + return matches.Count > 0; + } + + if (Environment.GetEnvironmentVariable("AWS_ENDPOINT") != null || + isAWS(Environment.GetEnvironmentVariable("SERVER_ENDPOINT"))) + // This is a PASS + new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for small object", + TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + } + else + { + new MintLogger(nameof(ListenBucketNotificationsAsync_Test1), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for small object", + TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); + throw; + } + } + finally + { + await TearDown(minio, bucketName); + if (subscription != null) + subscription.Dispose(); + } + } + + internal static async Task ListenBucketNotificationsAsync_Test2(MinioClient minio) + { + var startTime = DateTime.Now; + var events = new List(); + events.Add(EventType.ObjectCreatedAll); + var rxEvents = new List(); + IDisposable subscription = null; + var bucketName = GetRandomName(15); + var contentType = "application/json"; + var args = new Dictionary + { + { "bucketName", bucketName }, + { "contentType", contentType }, + { "size", "16B" } + }; + + try + { + static Stream ToStream(string input) + { + var stream = new MemoryStream(); + var writer = new StreamWriter(stream); + writer.Write(input); + writer.Flush(); + stream.Position = 0; + + return stream; + } + + var bucketExistsArgs = new BucketExistsArgs() + .WithBucket(bucketName); + var found = await minio.BucketExistsAsync(bucketExistsArgs).ConfigureAwait(false); + if (!found) + { + var makeBucketArgs = new MakeBucketArgs() + .WithBucket(bucketName); + await minio.MakeBucketAsync(makeBucketArgs).ConfigureAwait(false); + } + + void Notify(MinioNotificationRaw data) + { + var notification = JsonConvert.DeserializeObject(data.json); + if (notification is not { Records: { } }) return; + + foreach (var @event in notification.Records) rxEvents.Add(@event); + } + + var listenArgs = new ListenBucketNotificationsArgs() + .WithBucket(bucketName) + .WithEvents(events); + var observable = minio.ListenBucketNotificationsAsync(listenArgs); + + subscription = observable.Subscribe( + ev => Notify(ev), + ex => throw new Exception($"OnError: {ex.Message}"), + () => throw new Exception("STOPPED LISTENING FOR BUCKET NOTIFICATIONS\n")); + + var modelJson = "{\"test\": \"test\"}"; + await using var stream = ToStream(modelJson); + var putObjectArgs = new PutObjectArgs() + .WithObject("test.json") + .WithBucket(bucketName) + .WithContentType(contentType) + .WithStreamData(stream) + .WithObjectSize(stream.Length); + + await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); + + // Waits until the Put event is detected + // Times out if the event is not caught in 3 seconds + var timeoutDuration = 3; // seconds + var stTime = DateTime.UtcNow; + var timeout = TimeSpan.FromSeconds(timeoutDuration); + while (rxEvents.Count < 1) + { + await Task.Delay(25); + if (DateTime.UtcNow - stTime >= timeout) + throw new Exception("Timeout: while waiting for events"); } - var objectLockArgs = new RemoveObjectLockConfigurationArgs() - .WithBucket(bucketName); - await minio.RemoveObjectLockConfigurationAsync(objectLockArgs).ConfigureAwait(false); - var getObjectLockArgs = new GetObjectLockConfigurationArgs() - .WithBucket(bucketName); - var config = await minio.GetObjectLockConfigurationAsync(getObjectLockArgs).ConfigureAwait(false); - Assert.IsNotNull(config); - Assert.IsNull(config.Rule); - new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), deleteObjectLockConfigurationSignature, - "Tests whether RemoveObjectLockConfigurationAsync passes", TestStatus.PASS, DateTime.Now - startTime, - args: args).Log(); - } - catch (NotImplementedException ex) - { - new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), deleteObjectLockConfigurationSignature, - "Tests whether RemoveObjectLockConfigurationAsync passes", TestStatus.NA, DateTime.Now - startTime, - ex.Message, ex.ToString(), args: args).Log(); + foreach (var ev in rxEvents) Assert.AreEqual("s3:ObjectCreated:Put", ev.eventName); + + new MintLogger(nameof(ListenBucketNotificationsAsync_Test2), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for longer event processing", + TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); } catch (Exception ex) { - new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), deleteObjectLockConfigurationSignature, - "Tests whether RemoveObjectLockConfigurationAsync passes", TestStatus.FAIL, DateTime.Now - startTime, - ex.Message, ex.ToString(), args: args).Log(); + new MintLogger(nameof(ListenBucketNotificationsAsync_Test2), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for longer event processing", + TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + ex.ToString(), args: args).Log(); throw; } + finally { - Thread.Sleep(1500); await TearDown(minio, bucketName); + if (subscription != null) + subscription.Dispose(); } } - #endregion - - - #region Object Retention - - internal static async Task ObjectRetentionAsync_Test1(MinioClient minio) + internal static async Task ListenBucketNotificationsAsync_Test3(MinioClient minio) { var startTime = DateTime.Now; + var events = new List(); + events.Add(EventType.ObjectCreatedAll); + var rxEventsData = new MinioNotificationRaw(""); + IDisposable disposable = null; var bucketName = GetRandomName(15); - var objectName = GetRandomObjectName(10); + var suffix = ".json"; + var contentType = "application/json"; var args = new Dictionary { { "bucketName", bucketName }, - { "objectName", objectName } + { "contentType", contentType }, + { "suffix", suffix }, + { "size", "16B" } }; try { - await Setup_WithLock_Test(minio, bucketName); - } - catch (NotImplementedException ex) - { - await TearDown(minio, bucketName); - new MintLogger(nameof(ObjectRetentionAsync_Test1), setObjectRetentionSignature, - "Tests whether SetObjectRetentionAsync passes", TestStatus.NA, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - return; - } - catch (Exception ex) - { - await TearDown(minio, bucketName); - new MintLogger(nameof(ObjectRetentionAsync_Test1), setObjectRetentionSignature, - "Tests whether SetObjectRetentionAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - throw; - } - - try - { - var plusDays = 10; - using (var filestream = rsg.GenerateStreamFromSeed(1 * KB)) + var bucketExistsArgs = new BucketExistsArgs() + .WithBucket(bucketName); + if (!minio.BucketExistsAsync(bucketExistsArgs).Result) { - var putObjectArgs = new PutObjectArgs() - .WithBucket(bucketName) - .WithObject(objectName) - .WithStreamData(filestream) - .WithObjectSize(filestream.Length) - .WithContentType(null); - await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); + var makeBucketArgs = new MakeBucketArgs() + .WithBucket(bucketName); + minio.MakeBucketAsync(makeBucketArgs).Wait(); } - var untilDate = DateTime.Now.AddDays(plusDays); - var setRetentionArgs = new SetObjectRetentionArgs() + var notificationsArgs = new ListenBucketNotificationsArgs() .WithBucket(bucketName) - .WithObject(objectName) - .WithRetentionMode(RetentionMode.GOVERNANCE) - .WithRetentionUntilDate(untilDate); - await minio.SetObjectRetentionAsync(setRetentionArgs).ConfigureAwait(false); - new MintLogger(nameof(ObjectRetentionAsync_Test1), setObjectRetentionSignature, - "Tests whether SetObjectRetentionAsync passes", TestStatus.PASS, DateTime.Now - startTime, - args: args) - .Log(); - } - catch (NotImplementedException ex) - { - new MintLogger(nameof(ObjectRetentionAsync_Test1), setObjectRetentionSignature, - "Tests whether SetObjectRetentionAsync passes", TestStatus.NA, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - } - catch (Exception ex) - { - await TearDown(minio, bucketName); - new MintLogger(nameof(ObjectRetentionAsync_Test1), setObjectRetentionSignature, - "Tests whether SetObjectRetentionAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - throw; - } + .WithSuffix(suffix) + .WithEvents(events); - try - { - var getRetentionArgs = new GetObjectRetentionArgs() - .WithBucket(bucketName) - .WithObject(objectName); - var config = await minio.GetObjectRetentionAsync(getRetentionArgs).ConfigureAwait(false); - var plusDays = 10.0; - Assert.IsNotNull(config); - Assert.AreEqual(config.Mode, RetentionMode.GOVERNANCE); - var untilDate = DateTime.Parse(config.RetainUntilDate, null, DateTimeStyles.RoundtripKind); - Assert.AreEqual(Math.Ceiling((untilDate - DateTime.Now).TotalDays), plusDays); - new MintLogger(nameof(ObjectRetentionAsync_Test1), getObjectRetentionSignature, - "Tests whether GetObjectRetentionAsync passes", TestStatus.PASS, DateTime.Now - startTime, - args: args) - .Log(); - } - catch (NotImplementedException ex) - { - new MintLogger(nameof(ObjectRetentionAsync_Test1), getObjectRetentionSignature, - "Tests whether GetObjectRetentionAsync passes", TestStatus.NA, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - } - catch (Exception ex) - { - await TearDown(minio, bucketName); - new MintLogger(nameof(ObjectRetentionAsync_Test1), getObjectRetentionSignature, - "Tests whether GetObjectRetentionAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - throw; - } + var notifications = minio.ListenBucketNotificationsAsync(notificationsArgs); - try - { - var clearRetentionArgs = new ClearObjectRetentionArgs() - .WithBucket(bucketName) - .WithObject(objectName); - await minio.ClearObjectRetentionAsync(clearRetentionArgs).ConfigureAwait(false); - var getRetentionArgs = new GetObjectRetentionArgs() + var testState = "fail"; + Exception exception = null; + disposable = notifications.Subscribe( + x => + { + rxEventsData = x; + testState = "pass"; + }, + ex => + { + exception = ex; + testState = "fail"; + }, + () => { testState = "completed"; }); + + var modelJson = "{\"test\": \"test\"}"; + var stream = new MemoryStream(); + var writer = new StreamWriter(stream); + writer.Write(modelJson); + writer.Flush(); + stream.Position = 0; + + var putObjectArgs = new PutObjectArgs() + .WithObject("test.json") .WithBucket(bucketName) - .WithObject(objectName); - var config = await minio.GetObjectRetentionAsync(getRetentionArgs).ConfigureAwait(false); - } - catch (NotImplementedException ex) - { - new MintLogger(nameof(ObjectRetentionAsync_Test1), clearObjectRetentionSignature, - "Tests whether ClearObjectRetentionAsync passes", TestStatus.NA, DateTime.Now - startTime, ex.Message, - ex.ToString(), args: args).Log(); - } - catch (Exception ex) - { - var errMsgLock = ex.Message.Contains("The specified object does not have a ObjectLock configuration"); - if (errMsgLock) + .WithContentType(contentType) + .WithStreamData(stream) + .WithObjectSize(stream.Length); + + await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); + Thread.Sleep(1000); + + if (testState == "pass") { - new MintLogger(nameof(ObjectRetentionAsync_Test1), clearObjectRetentionSignature, - "Tests whether ClearObjectRetentionAsync passes", TestStatus.PASS, DateTime.Now - startTime, - args: args).Log(); + Assert.IsTrue(rxEventsData.json.Contains("\"eventName\":\"s3:ObjectCreated:Put\"")); + new MintLogger(nameof(ListenBucketNotificationsAsync_Test3), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for no event processing", + TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); } - else + else if (testState == "fail") { - new MintLogger(nameof(ObjectRetentionAsync_Test1), clearObjectRetentionSignature, - "Tests whether ClearObjectRetentionAsync passes", TestStatus.FAIL, DateTime.Now - startTime, - ex.Message, ex.ToString(), args: args).Log(); - await TearDown(minio, bucketName); - throw; + throw exception; + } + else if (testState == "completed") + { + throw new Exception("Bucket notification completed without catching the event"); } - } - - try - { - var rmArgs = new RemoveObjectArgs() - .WithBucket(bucketName) - .WithObject(objectName); - - await minio.RemoveObjectAsync(rmArgs).ConfigureAwait(false); - await TearDown(minio, bucketName); } catch (Exception ex) { - new MintLogger(nameof(ObjectRetentionAsync_Test1), clearObjectRetentionSignature, - "TearDown operation ClearObjectRetentionAsync", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, + new MintLogger(nameof(ListenBucketNotificationsAsync_Test3), + listenBucketNotificationsSignature, + "Tests whether ListenBucketNotifications passes for no event processing", + TestStatus.FAIL, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); throw; } + + finally + { + await TearDown(minio, bucketName); + if (disposable != null) + disposable.Dispose(); + } } #endregion