From fd940f81917d91fb18645657a7546b4ed49b6550 Mon Sep 17 00:00:00 2001 From: Ersan Bozduman Date: Fri, 14 Jun 2024 13:50:06 -0700 Subject: [PATCH 1/5] fixes the regression on BucketExistsAsync --- Minio/ApiEndpoints/BucketOperations.cs | 5 ----- Minio/RequestExtensions.cs | 11 ++++++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Minio/ApiEndpoints/BucketOperations.cs b/Minio/ApiEndpoints/BucketOperations.cs index 95f1cb04..436ea7ce 100644 --- a/Minio/ApiEndpoints/BucketOperations.cs +++ b/Minio/ApiEndpoints/BucketOperations.cs @@ -90,11 +90,6 @@ await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder, HttpStatusCode.NotFound != ice.ServerResponse.StatusCode) && ice.ServerResponse is not null; } - catch (Exception ex) - { - if (ex.GetType() == typeof(BucketNotFoundException)) return false; - throw; - } } /// diff --git a/Minio/RequestExtensions.cs b/Minio/RequestExtensions.cs index aa955e55..641aded2 100644 --- a/Minio/RequestExtensions.cs +++ b/Minio/RequestExtensions.cs @@ -107,10 +107,10 @@ await requestMessageBuilder.ResponseWriter(responseResult.ContentStream, cancell if (request.Method == HttpMethod.Head) { if (responseResult.Exception?.GetType().Equals(typeof(BucketNotFoundException)) == true || - path.Length == 1) + path.ToList().Count == 1) responseResult.Exception = new BucketNotFoundException(); - if (path.Length > 1) + if (path.ToList().Count > 1) { var found = await minioClient .BucketExistsAsync(new BucketExistsArgs().WithBucket(path.ToList()[0]), cancellationToken) @@ -125,7 +125,11 @@ await requestMessageBuilder.ResponseWriter(responseResult.ContentStream, cancell if (request.RequestUri.ToString().Contains("lock", StringComparison.OrdinalIgnoreCase) && request.Method == HttpMethod.Get) responseResult.Exception = new MissingObjectLockConfigurationException(); + return responseResult; } + + minioClient.HandleIfErrorResponse(responseResult, errorHandlers, startTime); + return responseResult; } catch (Exception ex) when (ex is not (OperationCanceledException or ObjectNotFoundException)) @@ -139,9 +143,6 @@ await requestMessageBuilder.ResponseWriter(responseResult.ContentStream, cancell responseResult = new ResponseResult(request, ex); return responseResult; } - - minioClient.HandleIfErrorResponse(responseResult, errorHandlers, startTime); - return responseResult; } private static Task ExecuteWithRetry(this IMinioClient minioClient, From c28c89c073f0a87516f9c0acfc5d6ca8303b8348 Mon Sep 17 00:00:00 2001 From: Ersan Bozduman Date: Tue, 18 Jun 2024 07:15:43 -0700 Subject: [PATCH 2/5] incorporates review comments --- Minio.Functional.Tests/FunctionalTest.cs | 144 ++++++++++++++++++++++- Minio.Functional.Tests/Program.cs | 2 + Minio/ApiEndpoints/BucketOperations.cs | 4 + Minio/RequestExtensions.cs | 21 ++-- 4 files changed, 157 insertions(+), 14 deletions(-) diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index e7393fd3..c47b7554 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -1,4 +1,4 @@ -/* +/* * MinIO .NET Library for Amazon S3 Compatible Cloud Storage, * (C) 2017-2021 MinIO, Inc. * @@ -4657,16 +4657,16 @@ internal static async Task GetObject_Test1(IMinioClient minio) try { await Setup_Test(minio, bucketName).ConfigureAwait(false); - - using (var filestream = rsg.GenerateStreamFromSeed(1 * MB)) + Stream strm; + await using ((strm = rsg.GenerateStreamFromSeed(1 * MB)).ConfigureAwait(false)) { - var file_write_size = filestream.Length; + var file_write_size = strm.Length; long file_read_size = 0; var putObjectArgs = new PutObjectArgs() .WithBucket(bucketName) .WithObject(objectName) - .WithStreamData(filestream) - .WithObjectSize(filestream.Length) + .WithStreamData(strm) + .WithObjectSize(strm.Length) .WithContentType(contentType); _ = await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); @@ -4775,6 +4775,138 @@ internal static async Task GetObject_Test2(IMinioClient minio) } } + internal static async Task GetObjectNegObjNotFound_Test3(IMinioClient minio) + { + var startTime = DateTime.Now; + var bucketName = GetRandomName(15); + var objectName = GetRandomObjectName(10); + string contentType = null; + var tempFileName = "tempFile-" + GetRandomName(); + var args = new Dictionary + (StringComparer.Ordinal) + { + { "bucketName", bucketName }, { "objectName", objectName }, { "contentType", contentType } + }; + try + { + await Setup_Test(minio, bucketName).ConfigureAwait(false); + // Don't Put the object, so we can hit "ObjectNotFound" exception + Stream strm; + await using ((strm = rsg.GenerateStreamFromSeed(1 * MB)).ConfigureAwait(false)) + { + var file_write_size = strm.Length; + long file_read_size = 0; + var getObjectArgs = new GetObjectArgs() + .WithBucket(bucketName) + .WithObject(objectName) + .WithCallbackStream(async (stream, cancellationToken) => + { + var fileStream = File.Create(tempFileName); + + await stream.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false); + await fileStream.DisposeAsync().ConfigureAwait(false); + + var writtenInfo = new FileInfo(tempFileName); + file_read_size = writtenInfo.Length; + + Assert.AreEqual(file_write_size, file_read_size); + File.Delete(tempFileName); + }); + _ = await minio.GetObjectAsync(getObjectArgs).ConfigureAwait(false); + } + + new MintLogger("GetObjectNegObjNotFound_Test3", getObjectSignature, + "Tests whether GetObjectAsync hits ObjectNotFoundException", + TestStatus.FAIL, DateTime.Now - startTime, "Failed to hit ObjectNotFound exception", args: args).Log(); + throw new Exception("Failed to hit ObjectNotFound exception"); + } + catch (ObjectNotFoundException) + { + new MintLogger("GetObjectNegObjNotFound_Test3", getObjectSignature, + "Tests whether GetObjectAsync hits ObjectNotFoundException", + TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + } + catch (Exception ex) + { + new MintLogger("GetObjectNegObjNotFound_Test3", getObjectSignature, + "Tests whether GetObjectAsync hits ObjectNotFoundException", + TestStatus.FAIL, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); + throw; + } + finally + { + if (File.Exists(tempFileName)) + File.Delete(tempFileName); + await TearDown(minio, bucketName).ConfigureAwait(false); + } + } + + internal static async Task GetObjectNegBcktNotFound_Test4(IMinioClient minio) + { + var startTime = DateTime.Now; + var bucketName = GetRandomName(15); + var objectName = GetRandomObjectName(10); + string contentType = null; + var tempFileName = "tempFile-" + GetRandomName(); + var args = new Dictionary + (StringComparer.Ordinal) + { + { "bucketName", bucketName }, { "objectName", objectName }, { "contentType", contentType } + }; + try + { + // No object, no bucket, so we can hit "BucketNotFoundException" with + Stream strm; + await using ((strm = rsg.GenerateStreamFromSeed(1 * MB)).ConfigureAwait(false)) + { + var file_write_size = strm.Length; + long file_read_size = 0; + var getObjectArgs = new GetObjectArgs() + .WithBucket(bucketName) + .WithObject(objectName) + .WithCallbackStream(async (stream, cancellationToken) => + { + var fileStream = File.Create(tempFileName); + + await stream.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false); + await fileStream.DisposeAsync().ConfigureAwait(false); + + var writtenInfo = new FileInfo(tempFileName); + file_read_size = writtenInfo.Length; + + Assert.AreEqual(file_write_size, file_read_size); + File.Delete(tempFileName); + }); + _ = await minio.GetObjectAsync(getObjectArgs).ConfigureAwait(false); + } + + new MintLogger("GetObjectNegBcktNotFound_Test4", getObjectSignature, + "Tests whether GetObjectAsync hits BucketNotFoundException", + TestStatus.FAIL, DateTime.Now - startTime, "Failed to hit BucketNotFoundException", args: args).Log(); + throw new Exception("Failed to hit BucketNotFoundException"); + } + catch (BucketNotFoundException) + { + new MintLogger("GetObjectNegBcktNotFound_Test4", getObjectSignature, + "Tests whether GetObjectAsync hits BucketNotFoundException", + TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + } + catch (Exception ex) + { + new MintLogger("GetObjectNegBcktNotFound_Test4", getObjectSignature, + "Tests whether GetObjectAsync hits BucketNotFoundException", + TestStatus.FAIL, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); + throw; + } + finally + { + if (File.Exists(tempFileName)) + File.Delete(tempFileName); + await TearDown(minio, bucketName).ConfigureAwait(false); + } + } + + internal static async Task GetObject_3_OffsetLength_Tests(IMinioClient minio) // 3 tests will run to check different values of offset and length parameters // when GetObject api returns part of the object as defined by the offset diff --git a/Minio.Functional.Tests/Program.cs b/Minio.Functional.Tests/Program.cs index 20297e92..65239f3f 100644 --- a/Minio.Functional.Tests/Program.cs +++ b/Minio.Functional.Tests/Program.cs @@ -151,6 +151,8 @@ public static async Task Main(string[] args) // Test GetObjectAsync function functionalTestTasks.Add(FunctionalTest.GetObject_Test1(minioClient)); functionalTestTasks.Add(FunctionalTest.GetObject_Test2(minioClient)); + functionalTestTasks.Add(FunctionalTest.GetObjectNegObjNotFound_Test3(minioClient)); + functionalTestTasks.Add(FunctionalTest.GetObjectNegBcktNotFound_Test4(minioClient)); // 3 tests will run to check different values of offset and length parameters // when GetObject api returns part of the object as defined by the offset // and length parameters. Tests will be reported as GetObject_Test3, diff --git a/Minio/ApiEndpoints/BucketOperations.cs b/Minio/ApiEndpoints/BucketOperations.cs index 436ea7ce..1a7cb5f0 100644 --- a/Minio/ApiEndpoints/BucketOperations.cs +++ b/Minio/ApiEndpoints/BucketOperations.cs @@ -90,6 +90,10 @@ await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder, HttpStatusCode.NotFound != ice.ServerResponse.StatusCode) && ice.ServerResponse is not null; } + catch (BucketNotFoundException) + { + return false; + } } /// diff --git a/Minio/RequestExtensions.cs b/Minio/RequestExtensions.cs index 641aded2..63b280cb 100644 --- a/Minio/RequestExtensions.cs +++ b/Minio/RequestExtensions.cs @@ -80,7 +80,6 @@ private static async Task ExecuteTaskCoreAsync(this IMinioClient CancellationToken cancellationToken = default) { var startTime = DateTime.Now; - var v4Authenticator = new V4Authenticator(minioClient.Config.Secure, minioClient.Config.AccessKey, minioClient.Config.SecretKey, minioClient.Config.Region, minioClient.Config.SessionToken); @@ -101,19 +100,28 @@ private static async Task ExecuteTaskCoreAsync(this IMinioClient await requestMessageBuilder.ResponseWriter(responseResult.ContentStream, cancellationToken) .ConfigureAwait(false); - var path = request.RequestUri.LocalPath.TrimStart('/').TrimEnd('/').Split('/'); + var separator = new[] { "/", " " }; + var path = request.RequestUri.LocalPath.TrimStart('/').TrimEnd('/') + .Split(separator, StringSplitOptions.RemoveEmptyEntries); if (responseResult.Response.StatusCode == HttpStatusCode.NotFound) { + if (request.RequestUri.ToString().Contains("lock", StringComparison.OrdinalIgnoreCase) && + request.Method == HttpMethod.Get) + { + responseResult.Exception = new MissingObjectLockConfigurationException(); + return responseResult; + } + if (request.Method == HttpMethod.Head) { if (responseResult.Exception?.GetType().Equals(typeof(BucketNotFoundException)) == true || - path.ToList().Count == 1) + path.Length == 1) responseResult.Exception = new BucketNotFoundException(); - if (path.ToList().Count > 1) + if (path.Length > 1) { var found = await minioClient - .BucketExistsAsync(new BucketExistsArgs().WithBucket(path.ToList()[0]), cancellationToken) + .BucketExistsAsync(new BucketExistsArgs().WithBucket(path[0]), cancellationToken) .ConfigureAwait(false); responseResult.Exception = !found ? new Exception("ThrowBucketNotFoundException") @@ -122,9 +130,6 @@ await requestMessageBuilder.ResponseWriter(responseResult.ContentStream, cancell } } - if (request.RequestUri.ToString().Contains("lock", StringComparison.OrdinalIgnoreCase) && - request.Method == HttpMethod.Get) - responseResult.Exception = new MissingObjectLockConfigurationException(); return responseResult; } From 998f1d1ff0984b0c491827fbe7f910db4b0c3625 Mon Sep 17 00:00:00 2001 From: Ersan Bozduman Date: Tue, 18 Jun 2024 07:26:02 -0700 Subject: [PATCH 3/5] incorporatess review comments-2 --- Minio/RequestExtensions.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Minio/RequestExtensions.cs b/Minio/RequestExtensions.cs index 63b280cb..52aa853a 100644 --- a/Minio/RequestExtensions.cs +++ b/Minio/RequestExtensions.cs @@ -114,8 +114,7 @@ await requestMessageBuilder.ResponseWriter(responseResult.ContentStream, cancell if (request.Method == HttpMethod.Head) { - if (responseResult.Exception?.GetType().Equals(typeof(BucketNotFoundException)) == true || - path.Length == 1) + if (responseResult.Exception is BucketNotFoundException || path.Length == 1) responseResult.Exception = new BucketNotFoundException(); if (path.Length > 1) From 465b3d40dc981703d6b5b830038f69cd61712a77 Mon Sep 17 00:00:00 2001 From: Ersan Bozduman Date: Tue, 18 Jun 2024 07:32:09 -0700 Subject: [PATCH 4/5] incorporatess review comments-3 --- Minio/RequestExtensions.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Minio/RequestExtensions.cs b/Minio/RequestExtensions.cs index 52aa853a..aaa029ad 100644 --- a/Minio/RequestExtensions.cs +++ b/Minio/RequestExtensions.cs @@ -100,9 +100,8 @@ private static async Task ExecuteTaskCoreAsync(this IMinioClient await requestMessageBuilder.ResponseWriter(responseResult.ContentStream, cancellationToken) .ConfigureAwait(false); - var separator = new[] { "/", " " }; var path = request.RequestUri.LocalPath.TrimStart('/').TrimEnd('/') - .Split(separator, StringSplitOptions.RemoveEmptyEntries); + .Split('/', StringSplitOptions.RemoveEmptyEntries); if (responseResult.Response.StatusCode == HttpStatusCode.NotFound) { if (request.RequestUri.ToString().Contains("lock", StringComparison.OrdinalIgnoreCase) && From d81bf868f8f6e42173f01ee3083f096578602c8a Mon Sep 17 00:00:00 2001 From: Ersan Bozduman Date: Tue, 18 Jun 2024 16:03:33 -0700 Subject: [PATCH 5/5] incorporates review comments-4 --- Minio.Functional.Tests/FunctionalTest.cs | 95 ++++++------------------ Minio/RequestExtensions.cs | 12 ++- 2 files changed, 29 insertions(+), 78 deletions(-) diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index c47b7554..0dc4d0f7 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -4777,136 +4777,83 @@ internal static async Task GetObject_Test2(IMinioClient minio) internal static async Task GetObjectNegObjNotFound_Test3(IMinioClient minio) { - var startTime = DateTime.Now; + var stopwatch = Stopwatch.StartNew(); var bucketName = GetRandomName(15); var objectName = GetRandomObjectName(10); - string contentType = null; - var tempFileName = "tempFile-" + GetRandomName(); var args = new Dictionary - (StringComparer.Ordinal) - { - { "bucketName", bucketName }, { "objectName", objectName }, { "contentType", contentType } - }; + (StringComparer.Ordinal) { { "bucketName", bucketName }, { "objectName", objectName } }; try { await Setup_Test(minio, bucketName).ConfigureAwait(false); // Don't Put the object, so we can hit "ObjectNotFound" exception - Stream strm; - await using ((strm = rsg.GenerateStreamFromSeed(1 * MB)).ConfigureAwait(false)) - { - var file_write_size = strm.Length; - long file_read_size = 0; - var getObjectArgs = new GetObjectArgs() - .WithBucket(bucketName) - .WithObject(objectName) - .WithCallbackStream(async (stream, cancellationToken) => - { - var fileStream = File.Create(tempFileName); - - await stream.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false); - await fileStream.DisposeAsync().ConfigureAwait(false); - - var writtenInfo = new FileInfo(tempFileName); - file_read_size = writtenInfo.Length; - - Assert.AreEqual(file_write_size, file_read_size); - File.Delete(tempFileName); - }); - _ = await minio.GetObjectAsync(getObjectArgs).ConfigureAwait(false); - } - + var getObjectArgs = new GetObjectArgs() + .WithBucket(bucketName) + .WithObject(objectName) + .WithCallbackStream(_ => throw new Exception("Should never be reached at the Callback function")); + _ = await minio.GetObjectAsync(getObjectArgs).ConfigureAwait(false); new MintLogger("GetObjectNegObjNotFound_Test3", getObjectSignature, "Tests whether GetObjectAsync hits ObjectNotFoundException", - TestStatus.FAIL, DateTime.Now - startTime, "Failed to hit ObjectNotFound exception", args: args).Log(); - throw new Exception("Failed to hit ObjectNotFound exception"); + TestStatus.FAIL, stopwatch.Elapsed, "Failed to hit ObjectNotFoundxception", args: args).Log(); + throw new Exception("Failed to hit ObjectNotFoundException"); } catch (ObjectNotFoundException) { new MintLogger("GetObjectNegObjNotFound_Test3", getObjectSignature, "Tests whether GetObjectAsync hits ObjectNotFoundException", - TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + TestStatus.PASS, stopwatch.Elapsed, args: args).Log(); } catch (Exception ex) { new MintLogger("GetObjectNegObjNotFound_Test3", getObjectSignature, "Tests whether GetObjectAsync hits ObjectNotFoundException", - TestStatus.FAIL, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); + TestStatus.FAIL, stopwatch.Elapsed, ex.Message, ex.ToString(), args: args).Log(); throw; } finally { - if (File.Exists(tempFileName)) - File.Delete(tempFileName); await TearDown(minio, bucketName).ConfigureAwait(false); } } internal static async Task GetObjectNegBcktNotFound_Test4(IMinioClient minio) { - var startTime = DateTime.Now; + var stopwatch = Stopwatch.StartNew(); var bucketName = GetRandomName(15); var objectName = GetRandomObjectName(10); - string contentType = null; - var tempFileName = "tempFile-" + GetRandomName(); var args = new Dictionary - (StringComparer.Ordinal) - { - { "bucketName", bucketName }, { "objectName", objectName }, { "contentType", contentType } - }; + (StringComparer.Ordinal) { { "bucketName", bucketName }, { "objectName", objectName } }; try { // No object, no bucket, so we can hit "BucketNotFoundException" with - Stream strm; - await using ((strm = rsg.GenerateStreamFromSeed(1 * MB)).ConfigureAwait(false)) - { - var file_write_size = strm.Length; - long file_read_size = 0; - var getObjectArgs = new GetObjectArgs() - .WithBucket(bucketName) - .WithObject(objectName) - .WithCallbackStream(async (stream, cancellationToken) => - { - var fileStream = File.Create(tempFileName); - - await stream.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false); - await fileStream.DisposeAsync().ConfigureAwait(false); - - var writtenInfo = new FileInfo(tempFileName); - file_read_size = writtenInfo.Length; - - Assert.AreEqual(file_write_size, file_read_size); - File.Delete(tempFileName); - }); - _ = await minio.GetObjectAsync(getObjectArgs).ConfigureAwait(false); - } - + var getObjectArgs = new GetObjectArgs() + .WithBucket(bucketName) + .WithObject(objectName) + .WithCallbackStream(_ => throw new Exception("Should never be reached")); + _ = await minio.GetObjectAsync(getObjectArgs).ConfigureAwait(false); new MintLogger("GetObjectNegBcktNotFound_Test4", getObjectSignature, "Tests whether GetObjectAsync hits BucketNotFoundException", - TestStatus.FAIL, DateTime.Now - startTime, "Failed to hit BucketNotFoundException", args: args).Log(); + TestStatus.FAIL, stopwatch.Elapsed, "Failed to hit BucketNotFoundException", args: args).Log(); throw new Exception("Failed to hit BucketNotFoundException"); } catch (BucketNotFoundException) { new MintLogger("GetObjectNegBcktNotFound_Test4", getObjectSignature, "Tests whether GetObjectAsync hits BucketNotFoundException", - TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + TestStatus.PASS, stopwatch.Elapsed, args: args).Log(); } catch (Exception ex) { new MintLogger("GetObjectNegBcktNotFound_Test4", getObjectSignature, "Tests whether GetObjectAsync hits BucketNotFoundException", - TestStatus.FAIL, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); + TestStatus.FAIL, stopwatch.Elapsed, ex.Message, ex.ToString(), args: args).Log(); throw; } finally { - if (File.Exists(tempFileName)) - File.Delete(tempFileName); await TearDown(minio, bucketName).ConfigureAwait(false); } } - internal static async Task GetObject_3_OffsetLength_Tests(IMinioClient minio) // 3 tests will run to check different values of offset and length parameters // when GetObject api returns part of the object as defined by the offset diff --git a/Minio/RequestExtensions.cs b/Minio/RequestExtensions.cs index aaa029ad..5cd32ec5 100644 --- a/Minio/RequestExtensions.cs +++ b/Minio/RequestExtensions.cs @@ -1,5 +1,6 @@ using System.Diagnostics.CodeAnalysis; using System.Net; +using System.Web; using Minio.Credentials; using Minio.DataModel; using Minio.DataModel.Args; @@ -104,11 +105,14 @@ await requestMessageBuilder.ResponseWriter(responseResult.ContentStream, cancell .Split('/', StringSplitOptions.RemoveEmptyEntries); if (responseResult.Response.StatusCode == HttpStatusCode.NotFound) { - if (request.RequestUri.ToString().Contains("lock", StringComparison.OrdinalIgnoreCase) && - request.Method == HttpMethod.Get) + if (request.Method == HttpMethod.Get) { - responseResult.Exception = new MissingObjectLockConfigurationException(); - return responseResult; + var q = HttpUtility.ParseQueryString(request.RequestUri.Query); + if (q.Get("object-lock") != null) + { + responseResult.Exception = new MissingObjectLockConfigurationException(); + return responseResult; + } } if (request.Method == HttpMethod.Head)