From e8e04f47c1d0c8b151274875be8e61478932fbd5 Mon Sep 17 00:00:00 2001 From: Daniel Szoke Date: Tue, 17 Dec 2024 17:10:50 +0100 Subject: [PATCH] test(proguard): Add chunk upload tests Add tests for chunk uploading Proguard mappings. Test the following scenarios: - Uploading a single Proguard mapping, where the file is already on the server. - Uploading a single Proguard mapping, where the file is not on the server. - Uploading two Proguard mappings, where neither is on the server. --- .../chunk_upload_needs_upload.bin | Bin 0 -> 391 bytes .../chunk_upload_two_files.bin | Bin 0 -> 731 bytes .../_fixtures/upload_proguard/mapping-2.txt | 4 + .../_fixtures/upload_proguard/mapping.txt | 4 + .../upload_proguard/get-chunk-upload.json | 11 + tests/integration/test_utils/test_manager.rs | 6 + tests/integration/upload_proguard.rs | 344 ++++++++++++++++++ 7 files changed, 369 insertions(+) create mode 100644 tests/integration/_expected_requests/upload_proguard/chunk_upload_needs_upload.bin create mode 100644 tests/integration/_expected_requests/upload_proguard/chunk_upload_two_files.bin create mode 100644 tests/integration/_fixtures/upload_proguard/mapping-2.txt create mode 100644 tests/integration/_fixtures/upload_proguard/mapping.txt create mode 100644 tests/integration/_responses/upload_proguard/get-chunk-upload.json diff --git a/tests/integration/_expected_requests/upload_proguard/chunk_upload_needs_upload.bin b/tests/integration/_expected_requests/upload_proguard/chunk_upload_needs_upload.bin new file mode 100644 index 0000000000000000000000000000000000000000..e1ca8d8b4ee94b7560671059237c465cf752602b GIT binary patch literal 391 zcmdPZ#RbZZO8rAa^GbbOBSUfn^NW%VGAm3hc)6VO^GZ_lN_1T^iwp9LGfOh_^Q;uo z@{4kHQxZ!OtrhYTb5m`V(lT>Wb-Wg^{JPL27D}62h90%7RoYg~Wn_oXq4zkY)P$$t9^Jy2T|$sfoF~ zT)bTJ-5d;H@ZYs(BiA7Z0hiyuyHazNUN~OJnmA?Bsbhznbj05CXwSB`NS69|f3Noe zA5-J}8P`@H^Y(d_aNFUxe@fVggY7Hq3*S#HaQt>s`7ztkr@ut+zkJtU*&fdvX?oSf zt*~H%f^+cN)9>G??_2!$O!c&YsoJY$tg7WEwY;mE{jGJT{0`GeKW77-!OMj!Ky`Jo K2dSWb-Wg^{JPL27D}62h90%7RoYg~Wn_oXq4zkY)P$$t9^Jy2T|$sfoF~ zT)bTJ-5d;H@ZYs(BiA7Z0hiyuyHazNUN~OJnmA?Bsbhznbj05CXwSB`NS69|f3Noe zA5-J}8P`@H^Y(d_aNFUxe@fVggY7Hq3*S#HaQt>s`7ztkr@ut+zkJtU*&fdvX?oSf zt*~H%f^+cN)9>G??_2!$O!c&YsoJY$tg7WEwY;mE{jGJT{0`GeKW77-!OMj!Ky`I# z5u~Z6#zvNAMka|CDF$Y#1{NlkCWdK-#>UC!rb(t2<|b+8md2^6=E-CRsV`cPUcnxu zTywM1?is8N?|nYwN^Zhkhu_^PVjqs}USVJKe&UDzo9z8P+9K0`r`IfvZWY@w9!0=fXJWkrk%8y1ZNf DP=ySm literal 0 HcmV?d00001 diff --git a/tests/integration/_fixtures/upload_proguard/mapping-2.txt b/tests/integration/_fixtures/upload_proguard/mapping-2.txt new file mode 100644 index 0000000000..d645356485 --- /dev/null +++ b/tests/integration/_fixtures/upload_proguard/mapping-2.txt @@ -0,0 +1,4 @@ +HelloWorld2 -> HelloWorld2: +# {"fileName":"HelloWorld2.java","id":"sourceFile"} + 1:1:void () -> + 3:4:void main(java.lang.String[]) -> main diff --git a/tests/integration/_fixtures/upload_proguard/mapping.txt b/tests/integration/_fixtures/upload_proguard/mapping.txt new file mode 100644 index 0000000000..234fe5ce21 --- /dev/null +++ b/tests/integration/_fixtures/upload_proguard/mapping.txt @@ -0,0 +1,4 @@ +HelloWorld -> HelloWorld: +# {"fileName":"HelloWorld.java","id":"sourceFile"} + 1:1:void () -> + 3:4:void main(java.lang.String[]) -> main diff --git a/tests/integration/_responses/upload_proguard/get-chunk-upload.json b/tests/integration/_responses/upload_proguard/get-chunk-upload.json new file mode 100644 index 0000000000..01fc9f7205 --- /dev/null +++ b/tests/integration/_responses/upload_proguard/get-chunk-upload.json @@ -0,0 +1,11 @@ +{ + "url": "organizations/wat-org/chunk-upload/", + "chunkSize": 8388608, + "chunksPerRequest": 64, + "maxFileSize": 2147483648, + "maxRequestSize": 33554432, + "concurrency": 8, + "hashAlgorithm": "sha1", + "compression": ["gzip"], + "accept": ["debug_files", "release_files", "pdbs", "portablepdbs", "sources", "bcsymbolmaps","proguard"] +} diff --git a/tests/integration/test_utils/test_manager.rs b/tests/integration/test_utils/test_manager.rs index a1e409730c..be0e338167 100644 --- a/tests/integration/test_utils/test_manager.rs +++ b/tests/integration/test_utils/test_manager.rs @@ -199,6 +199,12 @@ impl AssertCmdTestManager { self } + /// Set a custom environment variable for the test. + pub fn env(mut self, variable: impl AsRef, value: impl AsRef) -> Self { + self.command.env(variable, value); + self + } + /// Run the command and perform assertions. /// /// This function asserts both the mocks and the command result. diff --git a/tests/integration/upload_proguard.rs b/tests/integration/upload_proguard.rs index ee275cadde..51e471e01d 100644 --- a/tests/integration/upload_proguard.rs +++ b/tests/integration/upload_proguard.rs @@ -1,3 +1,9 @@ +use std::sync::atomic::{AtomicU8, Ordering}; +use std::{fs, str}; + +use mockito::Matcher; + +use crate::integration::test_utils::{chunk_upload, AssertCommand}; use crate::integration::{MockEndpointBuilder, TestManager}; #[test] @@ -15,3 +21,341 @@ fn command_upload_proguard() { fn command_upload_proguard_no_upload_no_auth_token() { TestManager::new().register_trycmd_test("upload_proguard/upload_proguard-no-upload.trycmd"); } + +#[test] +fn chunk_upload_already_there() { + TestManager::new() + .mock_endpoint( + MockEndpointBuilder::new("GET", "/api/0/organizations/wat-org/chunk-upload/") + .with_response_file("debug_files/get-chunk-upload.json"), + ) + .mock_endpoint( + MockEndpointBuilder::new( + "POST", + "/api/0/projects/wat-org/wat-project/files/difs/assemble/", + ) + .with_header_matcher("content-type", "application/json") + .with_matcher( + "{\ + \"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\":{\ + \"name\":\"/proguard/c038584d-c366-570c-ad1e-034fa0d194d7.txt\",\ + \"chunks\":[\"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\"]\ + }\ + }", + ) + .with_response_body( + "{\ + \"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\":{\ + \"state\":\"ok\",\ + \"detail\":null,\ + \"missingChunks\":[],\ + \"dif\":{\ + \"id\":\"12\",\ + \"uuid\":\"c038584d-c366-570c-ad1e-034fa0d194d7\",\ + \"debugId\":\"c038584d-c366-570c-ad1e-034fa0d194d7\",\ + \"codeId\":null,\ + \"cpuName\":\"any\",\ + \"objectName\":\"proguard-mapping\",\ + \"symbolType\":\"proguard\",\ + \"headers\":{\"Content-Type\":\"text/x-proguard+plain\"},\ + \"size\":155,\ + \"sha1\":\"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\",\ + \"dateCreated\":\"1776-07-04T12:00:00.000Z\",\ + \"data\":{\"features\":[\"mapping\"]}\ + }\ + }\ + }", + ), + ) + .assert_cmd([ + "upload-proguard", + "tests/integration/_fixtures/upload_proguard/mapping.txt", + ]) + .with_default_token() + .env("SENTRY_EXPERIMENTAL_PROGUARD_CHUNK_UPLOAD", "1") + .run_and_assert(AssertCommand::Success) +} + +#[test] +fn chunk_upload_needs_upload() { + const EXPECTED_CHUNKS_BOUNDARY: &str = "------------------------w2uOUUnuLEYTmQorc0ix48"; + + let call_count = AtomicU8::new(0); + let expected_chunk_body = fs::read( + "tests/integration/_expected_requests/upload_proguard/chunk_upload_needs_upload.bin", + ) + .expect("expected chunk upload request file should be readable"); + + TestManager::new() + .mock_endpoint( + MockEndpointBuilder::new("GET", "/api/0/organizations/wat-org/chunk-upload/") + .with_response_file("debug_files/get-chunk-upload.json"), + ) + .mock_endpoint( + MockEndpointBuilder::new("POST", "/api/0/organizations/wat-org/chunk-upload/") + .with_response_fn(move |request| { + let boundary = chunk_upload::boundary_from_request(request) + .expect("content-type header should be a valid multipart/form-data header"); + + let body = request.body().expect("body should be readable"); + + let chunks = chunk_upload::split_chunk_body(body, boundary) + .expect("body should be a valid multipart/form-data body"); + + let expected_chunks = chunk_upload::split_chunk_body( + &expected_chunk_body, + EXPECTED_CHUNKS_BOUNDARY, + ) + .expect("expected body is valid multipart form data"); + + // Using assert! because in case of failure, the output with assert_eq! + // is too long to be useful. + assert_eq!( + chunks, expected_chunks, + "Uploaded chunks differ from the expected chunks" + ); + + vec![] + }), + ) + .mock_endpoint( + MockEndpointBuilder::new( + "POST", + "/api/0/projects/wat-org/wat-project/files/difs/assemble/", + ) + .with_header_matcher("content-type", "application/json") + .with_matcher( + "{\ + \"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\":{\ + \"name\":\"/proguard/c038584d-c366-570c-ad1e-034fa0d194d7.txt\",\ + \"chunks\":[\"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\"]\ + }\ + }", + ) + .with_response_fn(move |_| { + match call_count.fetch_add(1, Ordering::Relaxed) { + 0 => { + // First call: The file is not found since it still needs to be uploaded. + "{\ + \"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\":{\ + \"state\":\"not_found\",\ + \"missingChunks\":[\"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\"]\ + }\ + }" + } + 1 => { + // Second call: The file has been uploaded, assemble job created. + "{\ + \"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\":{\ + \"state\":\"created\",\ + \"missingChunks\":[]\ + }\ + }" + } + 2 => { + // Third call: Assembly completed + "{\ + \"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\":{\ + \"state\":\"ok\",\ + \"detail\":null,\ + \"missingChunks\":[],\ + \"dif\":{\ + \"id\":\"12\",\ + \"uuid\":\"c038584d-c366-570c-ad1e-034fa0d194d7\",\ + \"debugId\":\"c038584d-c366-570c-ad1e-034fa0d194d7\",\ + \"codeId\":null,\ + \"cpuName\":\"any\",\ + \"objectName\":\"proguard-mapping\",\ + \"symbolType\":\"proguard\",\ + \"headers\":{\"Content-Type\":\"text/x-proguard+plain\"},\ + \"size\":155,\ + \"sha1\":\"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\",\ + \"dateCreated\":\"1776-07-04T12:00:00.000Z\",\ + \"data\":{\"features\":[\"mapping\"]}\ + }\ + }\ + }" + } + n => panic!( + "Only 3 calls to the assemble endpoint expected, but there were {}.", + n + 1 + ), + } + .into() + }) + .expect(3), + ) + .assert_cmd([ + "upload-proguard", + "tests/integration/_fixtures/upload_proguard/mapping.txt", + ]) + .with_default_token() + .env("SENTRY_EXPERIMENTAL_PROGUARD_CHUNK_UPLOAD", "1") + .run_and_assert(AssertCommand::Success) +} + +#[test] +fn chunk_upload_two_files() { + const EXPECTED_CHUNKS_BOUNDARY: &str = "------------------------HNdDRjCgjkRtu3COUTCcJV"; + + let call_count = AtomicU8::new(0); + let expected_chunk_body = + fs::read("tests/integration/_expected_requests/upload_proguard/chunk_upload_two_files.bin") + .expect("expected chunk upload request file should be readable"); + + TestManager::new() + .mock_endpoint( + MockEndpointBuilder::new("GET", "/api/0/organizations/wat-org/chunk-upload/") + .with_response_file("debug_files/get-chunk-upload.json"), + ) + .mock_endpoint( + MockEndpointBuilder::new("POST", "/api/0/organizations/wat-org/chunk-upload/") + .with_response_fn(move |request| { + let boundary = chunk_upload::boundary_from_request(request) + .expect("content-type header should be a valid multipart/form-data header"); + + let body = request.body().expect("body should be readable"); + + let chunks = chunk_upload::split_chunk_body(body, boundary) + .expect("body should be a valid multipart/form-data body"); + + let expected_chunks = chunk_upload::split_chunk_body( + &expected_chunk_body, + EXPECTED_CHUNKS_BOUNDARY, + ) + .expect("expected body is valid multipart form data"); + + // Using assert! because in case of failure, the output with assert_eq! + // is too long to be useful. + assert_eq!( + chunks, expected_chunks, + "Uploaded chunks differ from the expected chunks" + ); + + vec![] + }), + ) + .mock_endpoint( + MockEndpointBuilder::new( + "POST", + "/api/0/projects/wat-org/wat-project/files/difs/assemble/", + ) + .with_header_matcher("content-type", "application/json") + .with_matcher(Matcher::AnyOf( + // The ordering of the two files in the assemble request changes. + // Here, we match on either ordering. + [ + "{\ + \"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\":{\ + \"name\":\"/proguard/c038584d-c366-570c-ad1e-034fa0d194d7.txt\",\ + \"chunks\":[\"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\"]\ + },\ + \"e5329624a8d06e084941f133c75b5874f793ee7c\":{\ + \"name\":\"/proguard/747e1d76-509b-5225-8a5b-db7b7d4067d4.txt\",\ + \"chunks\":[\"e5329624a8d06e084941f133c75b5874f793ee7c\"]\ + }\ + }" + .into(), + "{\ + \"e5329624a8d06e084941f133c75b5874f793ee7c\":{\ + \"name\":\"/proguard/747e1d76-509b-5225-8a5b-db7b7d4067d4.txt\",\ + \"chunks\":[\"e5329624a8d06e084941f133c75b5874f793ee7c\"]\ + },\ + \"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\":{\ + \"name\":\"/proguard/c038584d-c366-570c-ad1e-034fa0d194d7.txt\",\ + \"chunks\":[\"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\"]\ + }\ + }" + .into(), + ] + .into(), + )) + .with_response_fn(move |_| { + match call_count.fetch_add(1, Ordering::Relaxed) { + 0 => { + // First call: The file is not found since it still needs to be uploaded. + "{\ + \"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\":{\ + \"state\":\"not_found\",\ + \"missingChunks\":[\"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\"]\ + },\ + \"e5329624a8d06e084941f133c75b5874f793ee7c\":{\ + \"state\":\"not_found\",\ + \"missingChunks\":[\"e5329624a8d06e084941f133c75b5874f793ee7c\"]\ + }\ + }" + } + 1 => { + // Second call: The file has been uploaded, assemble job created. + "{\ + \"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\":{\ + \"state\":\"created\",\ + \"missingChunks\":[]\ + },\ + \"e5329624a8d06e084941f133c75b5874f793ee7c\":{\ + \"state\":\"created\",\ + \"missingChunks\":[]\ + }\ + }" + } + 2 => { + // Third call: Assembly completed + "{\ + \"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\":{\ + \"state\":\"ok\",\ + \"detail\":null,\ + \"missingChunks\":[],\ + \"dif\":{\ + \"id\":\"12\",\ + \"uuid\":\"c038584d-c366-570c-ad1e-034fa0d194d7\",\ + \"debugId\":\"c038584d-c366-570c-ad1e-034fa0d194d7\",\ + \"codeId\":null,\ + \"cpuName\":\"any\",\ + \"objectName\":\"proguard-mapping\",\ + \"symbolType\":\"proguard\",\ + \"headers\":{\"Content-Type\":\"text/x-proguard+plain\"},\ + \"size\":155,\ + \"sha1\":\"297ecd9143fc2882e4b6758c1ccd13ea82930eeb\",\ + \"dateCreated\":\"1776-07-04T12:00:00.000Z\",\ + \"data\":{\"features\":[\"mapping\"]}\ + }\ + },\ + \"e5329624a8d06e084941f133c75b5874f793ee7c\":{\ + \"state\":\"ok\",\ + \"detail\":null,\ + \"missingChunks\":[],\ + \"dif\":{\ + \"id\":\"13\",\ + \"uuid\":\"747e1d76-509b-5225-8a5b-db7b7d4067d4\",\ + \"debugId\":\"747e1d76-509b-5225-8a5b-db7b7d4067d4\",\ + \"codeId\":null,\ + \"cpuName\":\"any\",\ + \"objectName\":\"proguard-mapping\",\ + \"symbolType\":\"proguard\",\ + \"headers\":{\"Content-Type\":\"text/x-proguard+plain\"},\ + \"size\":158,\ + \"sha1\":\"e5329624a8d06e084941f133c75b5874f793ee7c\",\ + \"dateCreated\":\"1776-07-04T12:00:00.000Z\",\ + \"data\":{\"features\":[\"mapping\"]}\ + }\ + }\ + }" + } + n => panic!( + "Only 3 calls to the assemble endpoint expected, but there were {}.", + n + 1 + ), + } + .into() + }) + .expect(3), + ) + .assert_cmd([ + "upload-proguard", + "tests/integration/_fixtures/upload_proguard/mapping.txt", + "tests/integration/_fixtures/upload_proguard/mapping-2.txt", + ]) + .with_default_token() + .env("SENTRY_EXPERIMENTAL_PROGUARD_CHUNK_UPLOAD", "1") + .run_and_assert(AssertCommand::Success) +}