Skip to content

Commit

Permalink
fix: remove multipart flag (#1431)
Browse files Browse the repository at this point in the history
<!-- Please make sure there is an issue that this PR is correlated to. -->

## Changes

<!-- If there are frontend changes, please include screenshots. -->
  • Loading branch information
MasterPtato committed Nov 22, 2024
1 parent 7639d46 commit d927f15
Show file tree
Hide file tree
Showing 9 changed files with 61 additions and 104 deletions.
2 changes: 1 addition & 1 deletion packages/api/actor/src/route/actors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,7 @@ pub async fn upgrade(

assert::server_for_env(&ctx, actor_id, game_id, env_id).await?;

let build_id = resolve_build_id(&ctx, game_id, body.build, body.build_tags.flatten()).await?;
let build_id = resolve_build_id(&ctx, env_id, body.build, body.build_tags.flatten()).await?;

let mut sub = ctx
.subscribe::<ds::workflows::server::UpgradeStarted>(("server_id", actor_id))
Expand Down
1 change: 0 additions & 1 deletion packages/api/actor/src/route/builds.rs
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,6 @@ pub async fn create_build(
display_name: body.name,
image_tag: Some(image_tag),
image_file: Some((*body.image_file).api_try_into()?),
multipart: multipart_upload,
kind: kind as i32,
compression: compression as i32,
})
Expand Down
31 changes: 6 additions & 25 deletions packages/api/cloud/src/route/games/builds.rs
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,6 @@ pub async fn create_build(

// TODO: Read and validate image file

let multipart_upload = body.multipart_upload.unwrap_or(false);

let kind = match body.kind {
None | Some(models::CloudGamesBuildKind::DockerImage) => {
backend::build::BuildKind::DockerImage
Expand All @@ -107,40 +105,23 @@ pub async fn create_build(
display_name: body.display_name,
image_tag: Some(body.image_tag),
image_file: Some((*body.image_file).api_try_into()?),
multipart: multipart_upload,
kind: kind as i32,
compression: compression as i32,
..Default::default()
})
.await?;

let image_presigned_request = if !multipart_upload {
Some(Box::new(
unwrap!(create_res.image_presigned_requests.first())
.clone()
.api_try_into()?,
))
} else {
None
};

let image_presigned_requests = if multipart_upload {
Some(
Ok(models::CloudGamesCreateGameBuildResponse {
build_id: unwrap_ref!(create_res.build_id).as_uuid(),
upload_id: unwrap_ref!(create_res.upload_id).as_uuid(),
image_presigned_request: None,
image_presigned_requests: Some(
create_res
.image_presigned_requests
.iter()
.cloned()
.map(ApiTryInto::api_try_into)
.collect::<GlobalResult<Vec<_>>>()?,
)
} else {
None
};

Ok(models::CloudGamesCreateGameBuildResponse {
build_id: unwrap_ref!(create_res.build_id).as_uuid(),
upload_id: unwrap_ref!(create_res.upload_id).as_uuid(),
image_presigned_request,
image_presigned_requests,
),
})
}
1 change: 0 additions & 1 deletion packages/services/build/ops/create/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,6 @@ async fn handle(
backend::upload::PrepareFile {
path: file_name,
content_length: image_file.content_length,
multipart: ctx.multipart,
..Default::default()
},
],
Expand Down
3 changes: 2 additions & 1 deletion packages/services/build/proto/create.proto
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,13 @@ import "resources/legacy/proto/backend/upload.proto";
import "resources/legacy/proto/backend/build.proto";

message Request {
reserved 6;

optional rivet.common.Uuid game_id = 1;
optional rivet.common.Uuid env_id = 10;
string display_name = 2;
optional string image_tag = 4;
optional rivet.backend.upload.PrepareFile image_file = 3;
bool multipart = 6;
rivet.backend.build.BuildKind kind = 7;
rivet.backend.build.BuildCompression compression = 8;
map<string, string> tags = 9; // JSON
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,6 @@ async fn upload_build(
backend::upload::PrepareFile {
path: unwrap!(unwrap!(build.key.file_name(), "should have file name").to_str()).to_string(),
content_length: len,
multipart: len > util::file_size::mebibytes(100),
..Default::default()
},
],
Expand Down
122 changes: 51 additions & 71 deletions packages/services/upload/ops/prepare/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,13 @@ use rivet_operation::prelude::*;
use serde_json::json;

pub const CHUNK_SIZE: u64 = util::file_size::mebibytes(100);
const MAX_UPLOAD_SIZE: u64 = util::file_size::gigabytes(10);
const MAX_MULTIPART_UPLOAD_SIZE: u64 = util::file_size::gigabytes(100);
const MAX_UPLOAD_SIZE: u64 = util::file_size::gigabytes(100);
/// Minimum size for AWS multipart file uploads.
///
/// See AWS error code `EntityTooSmall`
///
/// https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html
const MIN_MULTIPART_FILE_SIZE: u64 = util::file_size::mebibytes(5);

struct PrepareResult {
multipart: Option<MultipartUpdate>,
Expand Down Expand Up @@ -45,24 +50,11 @@ async fn handle(
.await?;

// Validate upload sizes
let total_content_length = ctx
.files
.iter()
.filter(|file| !file.multipart)
.fold(0, |acc, x| acc + x.content_length);
let total_multipart_content_length = ctx
.files
.iter()
.filter(|file| file.multipart)
.fold(0, |acc, x| acc + x.content_length);
tracing::info!(len=%ctx.files.len(), %total_content_length, %total_multipart_content_length, "file info");
let total_content_length = ctx.files.iter().fold(0, |acc, x| acc + x.content_length);
tracing::info!(len=%ctx.files.len(), %total_content_length, "file info");
ensure!(
total_content_length < MAX_UPLOAD_SIZE,
"upload size must be < 10 gb"
);
ensure!(
total_content_length < MAX_MULTIPART_UPLOAD_SIZE,
"multipart uploads must be < 100 gb"
"uploads must be < 100 gb"
);

let user_id = ctx.user_id.map(|x| x.as_uuid());
Expand Down Expand Up @@ -134,12 +126,8 @@ async fn handle(
let s3_client_internal = s3_client_internal.clone();
let s3_client_external = s3_client_external.clone();

if file.multipart {
handle_multipart_upload(s3_client_internal, s3_client_external, upload_id, file)
.boxed()
} else {
handle_upload(s3_client_external, upload_id, file).boxed()
}
handle_multipart_upload(s3_client_internal, s3_client_external, upload_id, file)
.boxed()
})
.buffer_unordered(16)
.try_collect::<Vec<_>>()
Expand Down Expand Up @@ -196,7 +184,6 @@ async fn handle(
"bucket": ctx.bucket,
"files": ctx.files.len(),
"total_content_length": total_content_length,
"total_multipart_content_length": total_multipart_content_length,
}))?),
..Default::default()
}
Expand All @@ -210,52 +197,6 @@ async fn handle(
})
}

async fn handle_upload(
s3_client_external: s3_util::Client,
upload_id: Uuid,
file: backend::upload::PrepareFile,
) -> GlobalResult<Vec<PrepareResult>> {
let fut = async move {
// Sign an upload request
let mut put_obj_builder = s3_client_external
.put_object()
.bucket(s3_client_external.bucket())
.key(format!("{}/{}", upload_id, file.path))
.content_length(file.content_length as i64);
if let Some(mime) = &file.mime {
put_obj_builder = put_obj_builder.content_type(mime.clone());
}
let presigned_upload_req = put_obj_builder
.presigned(
s3_util::aws_sdk_s3::presigning::PresigningConfig::builder()
.expires_in(Duration::from_secs(60 * 60))
.build()?,
)
.await?;

GlobalResult::Ok(backend::upload::PresignedUploadRequest {
path: file.path.clone(),
url: presigned_upload_req.uri().to_string(),
part_number: 0,
byte_offset: 0,
content_length: file.content_length,
})
}
.boxed();

Ok(vec![PrepareResult {
multipart: None,
fut,
}])
}

/// Minimum size for AWS multipart file uploads.
///
/// See AWS error code `EntityTooSmall`
///
/// https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html
const MIN_MULTIPART_FILE_SIZE: u64 = util::file_size::mebibytes(5);

async fn handle_multipart_upload(
s3_client_internal: s3_util::Client,
s3_client_external: s3_util::Client,
Expand Down Expand Up @@ -331,3 +272,42 @@ async fn handle_multipart_upload(
})
.collect::<Vec<_>>())
}

async fn handle_upload(
s3_client_external: s3_util::Client,
upload_id: Uuid,
file: backend::upload::PrepareFile,
) -> GlobalResult<Vec<PrepareResult>> {
let fut = async move {
// Sign an upload request
let mut put_obj_builder = s3_client_external
.put_object()
.bucket(s3_client_external.bucket())
.key(format!("{}/{}", upload_id, file.path))
.content_length(file.content_length as i64);
if let Some(mime) = &file.mime {
put_obj_builder = put_obj_builder.content_type(mime.clone());
}
let presigned_upload_req = put_obj_builder
.presigned(
s3_util::aws_sdk_s3::presigning::PresigningConfig::builder()
.expires_in(Duration::from_secs(60 * 60))
.build()?,
)
.await?;

GlobalResult::Ok(backend::upload::PresignedUploadRequest {
path: file.path.clone(),
url: presigned_upload_req.uri().to_string(),
part_number: 0,
byte_offset: 0,
content_length: file.content_length,
})
}
.boxed();

Ok(vec![PrepareResult {
multipart: None,
fut,
}])
}
1 change: 0 additions & 1 deletion packages/services/upload/ops/prepare/tests/integration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,6 @@ async fn multipart(ctx: TestCtx) {
.fold(0, |acc, x| {
acc + x.len()
}) as u64,
multipart: true,
..Default::default()
}],
})
Expand Down
3 changes: 1 addition & 2 deletions resources/legacy/proto/backend/upload.proto
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,11 @@ message UploadFile {
}

message PrepareFile {
reserved 4;
reserved 4, 5;

string path = 1;
optional string mime = 2;
uint64 content_length = 3;
bool multipart = 5;
}

message PresignedUploadRequest {
Expand Down

0 comments on commit d927f15

Please sign in to comment.