diff --git a/.gitignore b/.gitignore index 5a095ada96..2db7f5fb66 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ Cargo.lock .idea/ test.file .history +.envrc +minio \ No newline at end of file diff --git a/s3/src/bucket.rs b/s3/src/bucket.rs index 3f4cdb4daf..a2908046c7 100644 --- a/s3/src/bucket.rs +++ b/s3/src/bucket.rs @@ -1896,124 +1896,124 @@ mod test { } /// Test multi-part upload - #[ignore] - #[maybe_async::test( - feature = "sync", - async( - all( - not(feature = "sync"), - not(feature = "tokio-rustls-tls"), - feature = "with-tokio" - ), - tokio::test - ), - async( - all(not(feature = "sync"), feature = "with-async-std"), - async_std::test - ) - )] - async fn streaming_test_put_get_delete_big_object() { - init(); - let remote_path = "+stream_test_big"; - let local_path = "+stream_test_big"; - std::fs::remove_file(remote_path).unwrap_or_else(|_| {}); - let bucket = test_aws_bucket(); - let content: Vec = object(10_000_000); - - let mut file = File::create(local_path).unwrap(); - file.write_all(&content).unwrap(); - cfg_if! { - if #[cfg(feature = "with-tokio")] { - let mut reader = tokio::fs::File::open(local_path).await.unwrap(); - } else if #[cfg(feature = "with-async-std")] { - let mut reader = async_std::fs::File::open(local_path).await.unwrap(); - } else if #[cfg(feature = "sync")] { - let mut reader = File::open(local_path).unwrap(); - } - } - - let code = bucket - .put_object_stream(&mut reader, remote_path) - .await - .unwrap(); - assert_eq!(code, 200); - let mut writer = Vec::new(); - let code = bucket - .get_object_stream(remote_path, &mut writer) - .await - .unwrap(); - assert_eq!(code, 200); - assert_eq!(content, writer); - - // let (body, code) = bucket.get_object_torrent(remote_path).await.unwrap(); - // assert_eq!(code, 200); - // assert_eq!( - // body, - // [ - // 100, 56, 58, 97, 110, 110, 111, 117, 110, 99, 101, 53, 56, 58, 104, 116, 116, 112, - // 58, 47, 47, 115, 51, 45, 116, 114, 97, 99, 107, 101, 114, 46, 101, 117, 45, 99, - // 101, 110, 116, 114, 97, 108, 45, 49, 46, 97, 109, 97, 122, 111, 110, 97, 119, 115, - // 46, 99, 111, 109, 58, 54, 57, 54, 57, 47, 97, 110, 110, 111, 117, 110, 99, 101, 49, - // 51, 58, 97, 110, 110, 111, 117, 110, 99, 101, 45, 108, 105, 115, 116, 108, 108, 53, - // 56, 58, 104, 116, 116, 112, 58, 47, 47, 115, 51, 45, 116, 114, 97, 99, 107, 101, - // 114, 46, 101, 117, 45, 99, 101, 110, 116, 114, 97, 108, 45, 49, 46, 97, 109, 97, - // 122, 111, 110, 97, 119, 115, 46, 99, 111, 109, 58, 54, 57, 54, 57, 47, 97, 110, - // 110, 111, 117, 110, 99, 101, 101, 101, 52, 58, 105, 110, 102, 111, 100, 54, 58, - // 108, 101, 110, 103, 116, 104, 105, 49, 48, 48, 48, 48, 48, 48, 48, 101, 52, 58, - // 110, 97, 109, 101, 49, 54, 58, 43, 115, 116, 114, 101, 97, 109, 95, 116, 101, 115, - // 116, 95, 98, 105, 103, 49, 50, 58, 112, 105, 101, 99, 101, 32, 108, 101, 110, 103, - // 116, 104, 105, 50, 54, 50, 49, 52, 52, 101, 54, 58, 112, 105, 101, 99, 101, 115, - // 55, 56, 48, 58, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, - // 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, - // 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, - // 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, - // 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, - // 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, - // 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, - // 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, - // 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, - // 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, - // 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, - // 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, - // 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, - // 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, - // 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, - // 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, - // 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, - // 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, - // 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, - // 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, - // 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, - // 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, - // 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, - // 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, - // 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, - // 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, - // 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, - // 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, - // 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, - // 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, - // 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, - // 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, - // 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, - // 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, - // 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, - // 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, - // 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, - // 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, - // 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, - // 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, - // 96, 103, 49, 134, 201, 146, 37, 227, 84, 182, 214, 2, 98, 71, 21, 79, 174, 237, - // 155, 252, 61, 238, 62, 140, 232, 193, 49, 50, 58, 120, 45, 97, 109, 122, 45, 98, - // 117, 99, 107, 101, 116, 49, 50, 58, 114, 117, 115, 116, 45, 115, 51, 45, 116, 101, - // 115, 116, 57, 58, 120, 45, 97, 109, 122, 45, 107, 101, 121, 49, 54, 58, 43, 115, - // 116, 114, 101, 97, 109, 95, 116, 101, 115, 116, 95, 98, 105, 103, 101, 101 - // ] - // ); - let (_, code) = bucket.delete_object(local_path).await.unwrap(); - assert_eq!(code, 204); - std::fs::remove_file(local_path).unwrap_or_else(|_| {}); - } + // #[ignore] + // #[maybe_async::test( + // feature = "sync", + // async( + // all( + // not(feature = "sync"), + // not(feature = "tokio-rustls-tls"), + // feature = "with-tokio" + // ), + // tokio::test + // ), + // async( + // all(not(feature = "sync"), feature = "with-async-std"), + // async_std::test + // ) + // )] + // async fn streaming_test_put_get_delete_big_object() { + // init(); + // let remote_path = "+stream_test_big"; + // let local_path = "+stream_test_big"; + // std::fs::remove_file(remote_path).unwrap_or_else(|_| {}); + // let bucket = test_aws_bucket(); + // let content: Vec = object(10_000_000); + + // let mut file = File::create(local_path).unwrap(); + // file.write_all(&content).unwrap(); + // cfg_if! { + // if #[cfg(feature = "with-tokio")] { + // let mut reader = tokio::fs::File::open(local_path).await.unwrap(); + // } else if #[cfg(feature = "with-async-std")] { + // let mut reader = async_std::fs::File::open(local_path).await.unwrap(); + // } else if #[cfg(feature = "sync")] { + // let mut reader = File::open(local_path).unwrap(); + // } + // } + + // let code = bucket + // .put_object_stream(&mut reader, remote_path) + // .await + // .unwrap(); + // assert_eq!(code, 200); + // let mut writer = Vec::new(); + // let code = bucket + // .get_object_stream(remote_path, &mut writer) + // .await + // .unwrap(); + // assert_eq!(code, 200); + // assert_eq!(content, writer); + + // let (body, code) = bucket.get_object_torrent(remote_path).await.unwrap(); + // assert_eq!(code, 200); + // assert_eq!( + // body, + // [ + // 100, 56, 58, 97, 110, 110, 111, 117, 110, 99, 101, 53, 56, 58, 104, 116, 116, 112, + // 58, 47, 47, 115, 51, 45, 116, 114, 97, 99, 107, 101, 114, 46, 101, 117, 45, 99, + // 101, 110, 116, 114, 97, 108, 45, 49, 46, 97, 109, 97, 122, 111, 110, 97, 119, 115, + // 46, 99, 111, 109, 58, 54, 57, 54, 57, 47, 97, 110, 110, 111, 117, 110, 99, 101, 49, + // 51, 58, 97, 110, 110, 111, 117, 110, 99, 101, 45, 108, 105, 115, 116, 108, 108, 53, + // 56, 58, 104, 116, 116, 112, 58, 47, 47, 115, 51, 45, 116, 114, 97, 99, 107, 101, + // 114, 46, 101, 117, 45, 99, 101, 110, 116, 114, 97, 108, 45, 49, 46, 97, 109, 97, + // 122, 111, 110, 97, 119, 115, 46, 99, 111, 109, 58, 54, 57, 54, 57, 47, 97, 110, + // 110, 111, 117, 110, 99, 101, 101, 101, 52, 58, 105, 110, 102, 111, 100, 54, 58, + // 108, 101, 110, 103, 116, 104, 105, 49, 48, 48, 48, 48, 48, 48, 48, 101, 52, 58, + // 110, 97, 109, 101, 49, 54, 58, 43, 115, 116, 114, 101, 97, 109, 95, 116, 101, 115, + // 116, 95, 98, 105, 103, 49, 50, 58, 112, 105, 101, 99, 101, 32, 108, 101, 110, 103, + // 116, 104, 105, 50, 54, 50, 49, 52, 52, 101, 54, 58, 112, 105, 101, 99, 101, 115, + // 55, 56, 48, 58, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, + // 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, + // 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, + // 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, + // 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, + // 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, + // 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, + // 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, + // 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, + // 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, + // 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, + // 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, + // 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, + // 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, + // 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, + // 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, + // 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, + // 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, + // 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, + // 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, + // 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, + // 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, + // 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, + // 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, + // 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, + // 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, + // 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, + // 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, + // 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, + // 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, + // 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, + // 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, + // 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, + // 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, + // 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, + // 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, + // 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, + // 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, + // 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, + // 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, + // 96, 103, 49, 134, 201, 146, 37, 227, 84, 182, 214, 2, 98, 71, 21, 79, 174, 237, + // 155, 252, 61, 238, 62, 140, 232, 193, 49, 50, 58, 120, 45, 97, 109, 122, 45, 98, + // 117, 99, 107, 101, 116, 49, 50, 58, 114, 117, 115, 116, 45, 115, 51, 45, 116, 101, + // 115, 116, 57, 58, 120, 45, 97, 109, 122, 45, 107, 101, 121, 49, 54, 58, 43, 115, + // 116, 114, 101, 97, 109, 95, 116, 101, 115, 116, 95, 98, 105, 103, 101, 101 + // ] + // ); + // let (_, code) = bucket.delete_object(remote_path).await.unwrap(); + // assert_eq!(code, 204); + // std::fs::remove_file(local_path).unwrap_or_else(|_| {}); + // } /// Test streaming upload, with an object that's smaller than CHUNK_SIZE. /// This should be optimized into a plain PUT, not a multi-part upload. diff --git a/s3/src/surf_request.rs b/s3/src/surf_request.rs index f8c47a3509..d06017a8aa 100644 --- a/s3/src/surf_request.rs +++ b/s3/src/surf_request.rs @@ -65,7 +65,10 @@ impl<'a> Request for SurfRequest<'a> { ); } - let response = request.send().await.or_else(|e| Err(anyhow!("Request failed with {}", e.to_string())))?; + let response = request + .send() + .await + .map_err(|e| anyhow!("Request failed with {}", e))?; if cfg!(feature = "fail-on-err") && !response.status().is_success() { return Err(anyhow!("Request failed with code {}", response.status())); @@ -77,7 +80,10 @@ impl<'a> Request for SurfRequest<'a> { async fn response_data(&self, etag: bool) -> Result<(Vec, u16)> { let mut response = self.response().await?; let status_code = response.status(); - let body = response.body_bytes().await.or_else(|e| Err(anyhow!("Request failed with {}", e.to_string())))?; + let body = response + .body_bytes() + .await + .map_err(|e| anyhow!("Request failed with {}", e))?; let mut body_vec = Vec::new(); body_vec.extend_from_slice(&body[..]); if etag {