Skip to content

Commit

Permalink
Fixup #248
Browse files Browse the repository at this point in the history
  • Loading branch information
@durch
durch committed Feb 12, 2022
1 parent c1a174d commit 10cf754
Show file tree
Hide file tree
Showing 3 changed files with 128 additions and 120 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,5 @@ Cargo.lock
.idea/
test.file
.history
.envrc
minio
236 changes: 118 additions & 118 deletions s3/src/bucket.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1896,124 +1896,124 @@ mod test {
}

/// Test multi-part upload
#[ignore]
#[maybe_async::test(
feature = "sync",
async(
all(
not(feature = "sync"),
not(feature = "tokio-rustls-tls"),
feature = "with-tokio"
),
tokio::test
),
async(
all(not(feature = "sync"), feature = "with-async-std"),
async_std::test
)
)]
async fn streaming_test_put_get_delete_big_object() {
init();
let remote_path = "+stream_test_big";
let local_path = "+stream_test_big";
std::fs::remove_file(remote_path).unwrap_or_else(|_| {});
let bucket = test_aws_bucket();
let content: Vec<u8> = object(10_000_000);

let mut file = File::create(local_path).unwrap();
file.write_all(&content).unwrap();
cfg_if! {
if #[cfg(feature = "with-tokio")] {
let mut reader = tokio::fs::File::open(local_path).await.unwrap();
} else if #[cfg(feature = "with-async-std")] {
let mut reader = async_std::fs::File::open(local_path).await.unwrap();
} else if #[cfg(feature = "sync")] {
let mut reader = File::open(local_path).unwrap();
}
}

let code = bucket
.put_object_stream(&mut reader, remote_path)
.await
.unwrap();
assert_eq!(code, 200);
let mut writer = Vec::new();
let code = bucket
.get_object_stream(remote_path, &mut writer)
.await
.unwrap();
assert_eq!(code, 200);
assert_eq!(content, writer);

// let (body, code) = bucket.get_object_torrent(remote_path).await.unwrap();
// assert_eq!(code, 200);
// assert_eq!(
// body,
// [
// 100, 56, 58, 97, 110, 110, 111, 117, 110, 99, 101, 53, 56, 58, 104, 116, 116, 112,
// 58, 47, 47, 115, 51, 45, 116, 114, 97, 99, 107, 101, 114, 46, 101, 117, 45, 99,
// 101, 110, 116, 114, 97, 108, 45, 49, 46, 97, 109, 97, 122, 111, 110, 97, 119, 115,
// 46, 99, 111, 109, 58, 54, 57, 54, 57, 47, 97, 110, 110, 111, 117, 110, 99, 101, 49,
// 51, 58, 97, 110, 110, 111, 117, 110, 99, 101, 45, 108, 105, 115, 116, 108, 108, 53,
// 56, 58, 104, 116, 116, 112, 58, 47, 47, 115, 51, 45, 116, 114, 97, 99, 107, 101,
// 114, 46, 101, 117, 45, 99, 101, 110, 116, 114, 97, 108, 45, 49, 46, 97, 109, 97,
// 122, 111, 110, 97, 119, 115, 46, 99, 111, 109, 58, 54, 57, 54, 57, 47, 97, 110,
// 110, 111, 117, 110, 99, 101, 101, 101, 52, 58, 105, 110, 102, 111, 100, 54, 58,
// 108, 101, 110, 103, 116, 104, 105, 49, 48, 48, 48, 48, 48, 48, 48, 101, 52, 58,
// 110, 97, 109, 101, 49, 54, 58, 43, 115, 116, 114, 101, 97, 109, 95, 116, 101, 115,
// 116, 95, 98, 105, 103, 49, 50, 58, 112, 105, 101, 99, 101, 32, 108, 101, 110, 103,
// 116, 104, 105, 50, 54, 50, 49, 52, 52, 101, 54, 58, 112, 105, 101, 99, 101, 115,
// 55, 56, 48, 58, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96,
// 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1,
// 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16,
// 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60,
// 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136,
// 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206,
// 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95,
// 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85,
// 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56,
// 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65,
// 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67,
// 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157,
// 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24,
// 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103,
// 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146,
// 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201,
// 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134,
// 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49,
// 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103,
// 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96,
// 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1,
// 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16,
// 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60,
// 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136,
// 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206,
// 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95,
// 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85,
// 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56,
// 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65,
// 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67,
// 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157,
// 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24,
// 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103,
// 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146,
// 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201,
// 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134,
// 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49,
// 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103,
// 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96,
// 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1,
// 96, 103, 49, 134, 201, 146, 37, 227, 84, 182, 214, 2, 98, 71, 21, 79, 174, 237,
// 155, 252, 61, 238, 62, 140, 232, 193, 49, 50, 58, 120, 45, 97, 109, 122, 45, 98,
// 117, 99, 107, 101, 116, 49, 50, 58, 114, 117, 115, 116, 45, 115, 51, 45, 116, 101,
// 115, 116, 57, 58, 120, 45, 97, 109, 122, 45, 107, 101, 121, 49, 54, 58, 43, 115,
// 116, 114, 101, 97, 109, 95, 116, 101, 115, 116, 95, 98, 105, 103, 101, 101
// ]
// );
let (_, code) = bucket.delete_object(local_path).await.unwrap();
assert_eq!(code, 204);
std::fs::remove_file(local_path).unwrap_or_else(|_| {});
}
// #[ignore]
// #[maybe_async::test(
// feature = "sync",
// async(
// all(
// not(feature = "sync"),
// not(feature = "tokio-rustls-tls"),
// feature = "with-tokio"
// ),
// tokio::test
// ),
// async(
// all(not(feature = "sync"), feature = "with-async-std"),
// async_std::test
// )
// )]
// async fn streaming_test_put_get_delete_big_object() {
// init();
// let remote_path = "+stream_test_big";
// let local_path = "+stream_test_big";
// std::fs::remove_file(remote_path).unwrap_or_else(|_| {});
// let bucket = test_aws_bucket();
// let content: Vec<u8> = object(10_000_000);

// let mut file = File::create(local_path).unwrap();
// file.write_all(&content).unwrap();
// cfg_if! {
// if #[cfg(feature = "with-tokio")] {
// let mut reader = tokio::fs::File::open(local_path).await.unwrap();
// } else if #[cfg(feature = "with-async-std")] {
// let mut reader = async_std::fs::File::open(local_path).await.unwrap();
// } else if #[cfg(feature = "sync")] {
// let mut reader = File::open(local_path).unwrap();
// }
// }

// let code = bucket
// .put_object_stream(&mut reader, remote_path)
// .await
// .unwrap();
// assert_eq!(code, 200);
// let mut writer = Vec::new();
// let code = bucket
// .get_object_stream(remote_path, &mut writer)
// .await
// .unwrap();
// assert_eq!(code, 200);
// assert_eq!(content, writer);

// let (body, code) = bucket.get_object_torrent(remote_path).await.unwrap();
// assert_eq!(code, 200);
// assert_eq!(
// body,
// [
// 100, 56, 58, 97, 110, 110, 111, 117, 110, 99, 101, 53, 56, 58, 104, 116, 116, 112,
// 58, 47, 47, 115, 51, 45, 116, 114, 97, 99, 107, 101, 114, 46, 101, 117, 45, 99,
// 101, 110, 116, 114, 97, 108, 45, 49, 46, 97, 109, 97, 122, 111, 110, 97, 119, 115,
// 46, 99, 111, 109, 58, 54, 57, 54, 57, 47, 97, 110, 110, 111, 117, 110, 99, 101, 49,
// 51, 58, 97, 110, 110, 111, 117, 110, 99, 101, 45, 108, 105, 115, 116, 108, 108, 53,
// 56, 58, 104, 116, 116, 112, 58, 47, 47, 115, 51, 45, 116, 114, 97, 99, 107, 101,
// 114, 46, 101, 117, 45, 99, 101, 110, 116, 114, 97, 108, 45, 49, 46, 97, 109, 97,
// 122, 111, 110, 97, 119, 115, 46, 99, 111, 109, 58, 54, 57, 54, 57, 47, 97, 110,
// 110, 111, 117, 110, 99, 101, 101, 101, 52, 58, 105, 110, 102, 111, 100, 54, 58,
// 108, 101, 110, 103, 116, 104, 105, 49, 48, 48, 48, 48, 48, 48, 48, 101, 52, 58,
// 110, 97, 109, 101, 49, 54, 58, 43, 115, 116, 114, 101, 97, 109, 95, 116, 101, 115,
// 116, 95, 98, 105, 103, 49, 50, 58, 112, 105, 101, 99, 101, 32, 108, 101, 110, 103,
// 116, 104, 105, 50, 54, 50, 49, 52, 52, 101, 54, 58, 112, 105, 101, 99, 101, 115,
// 55, 56, 48, 58, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96,
// 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1,
// 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16,
// 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60,
// 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136,
// 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206,
// 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95,
// 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85,
// 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56,
// 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65,
// 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67,
// 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157,
// 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24,
// 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103,
// 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146,
// 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201,
// 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134,
// 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49,
// 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103,
// 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96,
// 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1,
// 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16,
// 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60,
// 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136,
// 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206,
// 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95,
// 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85,
// 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56,
// 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67, 65,
// 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157, 67,
// 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24, 157,
// 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103, 24,
// 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146, 103,
// 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201, 146,
// 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134, 201,
// 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49, 134,
// 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103, 49,
// 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96, 103,
// 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1, 96,
// 103, 49, 134, 201, 146, 103, 24, 157, 67, 65, 56, 85, 95, 206, 136, 60, 16, 172, 1,
// 96, 103, 49, 134, 201, 146, 37, 227, 84, 182, 214, 2, 98, 71, 21, 79, 174, 237,
// 155, 252, 61, 238, 62, 140, 232, 193, 49, 50, 58, 120, 45, 97, 109, 122, 45, 98,
// 117, 99, 107, 101, 116, 49, 50, 58, 114, 117, 115, 116, 45, 115, 51, 45, 116, 101,
// 115, 116, 57, 58, 120, 45, 97, 109, 122, 45, 107, 101, 121, 49, 54, 58, 43, 115,
// 116, 114, 101, 97, 109, 95, 116, 101, 115, 116, 95, 98, 105, 103, 101, 101
// ]
// );
// let (_, code) = bucket.delete_object(remote_path).await.unwrap();
// assert_eq!(code, 204);
// std::fs::remove_file(local_path).unwrap_or_else(|_| {});
// }

/// Test streaming upload, with an object that's smaller than CHUNK_SIZE.
/// This should be optimized into a plain PUT, not a multi-part upload.
Expand Down
10 changes: 8 additions & 2 deletions s3/src/surf_request.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,10 @@ impl<'a> Request for SurfRequest<'a> {
);
}

let response = request.send().await.or_else(|e| Err(anyhow!("Request failed with {}", e.to_string())))?;
let response = request
.send()
.await
.map_err(|e| anyhow!("Request failed with {}", e))?;

if cfg!(feature = "fail-on-err") && !response.status().is_success() {
return Err(anyhow!("Request failed with code {}", response.status()));
Expand All @@ -77,7 +80,10 @@ impl<'a> Request for SurfRequest<'a> {
async fn response_data(&self, etag: bool) -> Result<(Vec<u8>, u16)> {
let mut response = self.response().await?;
let status_code = response.status();
let body = response.body_bytes().await.or_else(|e| Err(anyhow!("Request failed with {}", e.to_string())))?;
let body = response
.body_bytes()
.await
.map_err(|e| anyhow!("Request failed with {}", e))?;
let mut body_vec = Vec::new();
body_vec.extend_from_slice(&body[..]);
if etag {
Expand Down

0 comments on commit 10cf754

Please sign in to comment.