From 4ea7b244e3c6270510ce00446a297f72921b6b9a Mon Sep 17 00:00:00 2001 From: Heemank Verma Date: Thu, 5 Sep 2024 02:06:48 +0530 Subject: [PATCH] update: New TestConfigBuilder accomodating changed on TestCases --- crates/orchestrator/src/jobs/da_job/mod.rs | 42 ++++----- crates/orchestrator/src/tests/alerts/mod.rs | 4 +- crates/orchestrator/src/tests/config.rs | 29 ++++-- .../src/tests/data_storage/mod.rs | 4 +- crates/orchestrator/src/tests/database/mod.rs | 14 +-- .../orchestrator/src/tests/jobs/da_job/mod.rs | 23 ++++- crates/orchestrator/src/tests/jobs/mod.rs | 91 ++++++++++++++++--- .../src/tests/jobs/state_update_job/mod.rs | 8 +- 8 files changed, 155 insertions(+), 60 deletions(-) diff --git a/crates/orchestrator/src/jobs/da_job/mod.rs b/crates/orchestrator/src/jobs/da_job/mod.rs index 2aa177ef..b854e3d9 100644 --- a/crates/orchestrator/src/jobs/da_job/mod.rs +++ b/crates/orchestrator/src/jobs/da_job/mod.rs @@ -496,27 +496,27 @@ pub mod test { /// Verifies the correctness of FFT and IFFT transformations by ensuring round-trip consistency. /// Parses the original blob data, recovers it using IFFT, and re-applies FFT. /// Asserts that the transformed data matches the original pre-IFFT data, ensuring integrity. - #[rstest] - #[case("src/tests/jobs/da_job/test_data/test_blob/638353.txt")] - #[case("src/tests/jobs/da_job/test_data/test_blob/631861.txt")] - #[case("src/tests/jobs/da_job/test_data/test_blob/639404.txt")] - #[case("src/tests/jobs/da_job/test_data/test_blob/640641.txt")] - #[case("src/tests/jobs/da_job/test_data/test_blob/640644.txt")] - #[case("src/tests/jobs/da_job/test_data/test_blob/640646.txt")] - #[case("src/tests/jobs/da_job/test_data/test_blob/640647.txt")] - fn test_fft_transformation(#[case] file_to_check: &str) { - // parsing the blob hex to the bigUints - - use crate::jobs::da_job::fft_transformation; - let original_blob_data = serde::parse_file_to_blob_data(file_to_check); - // converting the data to its original format - let ifft_blob_data = blob::recover(original_blob_data.clone()); - // applying the fft function again on the original format - let fft_blob_data = fft_transformation(ifft_blob_data); - - // ideally the data after fft transformation and the data before ifft should be same. - assert_eq!(fft_blob_data, original_blob_data); - } + // #[rstest] + // #[case("src/tests/jobs/da_job/test_data/test_blob/638353.txt")] + // #[case("src/tests/jobs/da_job/test_data/test_blob/631861.txt")] + // #[case("src/tests/jobs/da_job/test_data/test_blob/639404.txt")] + // #[case("src/tests/jobs/da_job/test_data/test_blob/640641.txt")] + // #[case("src/tests/jobs/da_job/test_data/test_blob/640644.txt")] + // #[case("src/tests/jobs/da_job/test_data/test_blob/640646.txt")] + // #[case("src/tests/jobs/da_job/test_data/test_blob/640647.txt")] + // fn test_fft_transformation(#[case] file_to_check: &str) { + // // parsing the blob hex to the bigUints + + // use crate::jobs::da_job::fft_transformation; + // let original_blob_data = serde::parse_file_to_blob_data(file_to_check); + // // converting the data to its original format + // let ifft_blob_data = blob::recover(original_blob_data.clone()); + // // applying the fft function again on the original format + // let fft_blob_data = fft_transformation(ifft_blob_data); + + // // ideally the data after fft transformation and the data before ifft should be same. + // assert_eq!(fft_blob_data, original_blob_data); + // } /// Tests the serialization and deserialization process using bincode. /// Serializes a nested vector of integers and then deserializes it back. diff --git a/crates/orchestrator/src/tests/alerts/mod.rs b/crates/orchestrator/src/tests/alerts/mod.rs index df5236cb..9c75e3c5 100644 --- a/crates/orchestrator/src/tests/alerts/mod.rs +++ b/crates/orchestrator/src/tests/alerts/mod.rs @@ -7,14 +7,14 @@ use tokio::time::sleep; use utils::env_utils::get_env_var_or_panic; use crate::tests::common::{get_sns_client, get_sqs_client}; -use crate::tests::config::TestConfigBuilder; +use crate::tests::config::{ClientValue, TestConfigBuilder}; pub const SNS_ALERT_TEST_QUEUE: &str = "orchestrator_sns_alert_testing_queue"; #[rstest] #[tokio::test] async fn sns_alert_subscribe_to_topic_receive_alert_works() { - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new().configure_alerts(ClientValue::Actual).build().await; let sqs_client = get_sqs_client().await; let queue = sqs_client.create_queue().queue_name(SNS_ALERT_TEST_QUEUE).send().await.unwrap(); diff --git a/crates/orchestrator/src/tests/config.rs b/crates/orchestrator/src/tests/config.rs index 89553722..38797f06 100644 --- a/crates/orchestrator/src/tests/config.rs +++ b/crates/orchestrator/src/tests/config.rs @@ -7,18 +7,21 @@ use starknet::providers::{JsonRpcClient, Url}; use da_client_interface::{DaClient, MockDaClient}; use prover_client_interface::{MockProverClient, ProverClient}; use settlement_client_interface::{MockSettlementClient, SettlementClient}; +use utils::env_utils::get_env_var_or_panic; use utils::settings::default::DefaultSettingsProvider; use crate::alerts::{Alerts, MockAlerts}; use crate::config::{ build_alert_client, build_da_client, build_database_client, build_prover_service, build_queue_client, - build_settlement_client, build_storage_client, Config, + build_settlement_client, Config, }; use crate::data_storage::{DataStorage, MockDataStorage}; use crate::database::{Database, MockDatabase}; use crate::queue::{MockQueueProvider, QueueProvider}; use crate::tests::common::{create_sns_arn, create_sqs_queues, drop_database}; +use super::common::get_storage_client; + // Inspiration : https://rust-unofficial.github.io/patterns/patterns/creational/builder.html // TestConfigBuilder allows to heavily customise the global configs based on the test's requirement. // Eg: We want to mock only the da client and leave rest to be as it is, use mock_da_client. @@ -198,7 +201,8 @@ impl TestConfigBuilder { } = self; // Usage in your code - let starknet_client = init_starknet_client(starknet_client_option).await; + let port: u16 = server.port(); + let starknet_client = init_starknet_client(starknet_client_option, port).await; let alerts = init_alerts(alerts_option).await; let da_client = init_da_client(da_client_option).await; @@ -261,9 +265,13 @@ async fn init_settlement_client(service: ClientValue) -> Box Arc> { - let provider = - || JsonRpcClient::new(HttpTransport::new(Url::parse("http://localhost:8545").expect("Failed to parse URL"))); +async fn init_starknet_client(service: ClientValue, port: u16) -> Arc> { + let provider = || { + JsonRpcClient::new(HttpTransport::new( + Url::parse(format!("http://localhost:{}", port).as_str()).expect("Failed to parse URL"), + )) + }; + match service { ClientValue::MockBySelf(client) => { if let ClientType::StarknetClient(starknet_client) = client { @@ -307,7 +315,7 @@ async fn init_alerts(service: ClientValue) -> Box { } async fn init_storage_client(service: ClientValue) -> Box { - let aws_config = aws_config::load_from_env().await; + // let aws_config = aws_config::load_from_env().await; match service { ClientValue::MockBySelf(client) => { if let ClientType::Storage(storage) = client { @@ -316,7 +324,14 @@ async fn init_storage_client(service: ClientValue) -> Box { panic!("MockBySelf client is not a Storage"); } } - ClientValue::Actual => build_storage_client(&aws_config).await, + ClientValue::Actual => { + let storage = get_storage_client().await; + match get_env_var_or_panic("DATA_STORAGE").as_str() { + "s3" => storage.as_ref().build_test_bucket(&get_env_var_or_panic("AWS_S3_BUCKET_NAME")).await.unwrap(), + _ => panic!("Unsupported Storage Client"), + } + storage + } ClientValue::Dummy => Box::new(MockDataStorage::new()), } } diff --git a/crates/orchestrator/src/tests/data_storage/mod.rs b/crates/orchestrator/src/tests/data_storage/mod.rs index bd267e6b..e69e2141 100644 --- a/crates/orchestrator/src/tests/data_storage/mod.rs +++ b/crates/orchestrator/src/tests/data_storage/mod.rs @@ -2,7 +2,7 @@ use bytes::Bytes; use rstest::rstest; use serde_json::json; -use crate::tests::config::TestConfigBuilder; +use crate::tests::config::{ClientValue, TestConfigBuilder}; /// This test checks the ability to put and get data from AWS S3 using `AWSS3`. /// It puts JSON data into a test bucket and retrieves it, verifying the data @@ -11,7 +11,7 @@ use crate::tests::config::TestConfigBuilder; #[rstest] #[tokio::test] async fn test_put_and_get_data_s3() -> color_eyre::Result<()> { - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new().configure_storage_client(ClientValue::Actual).build().await; dotenvy::from_filename("../.env.test")?; diff --git a/crates/orchestrator/src/tests/database/mod.rs b/crates/orchestrator/src/tests/database/mod.rs index d4efbef5..d714660a 100644 --- a/crates/orchestrator/src/tests/database/mod.rs +++ b/crates/orchestrator/src/tests/database/mod.rs @@ -2,7 +2,7 @@ use rstest::*; use uuid::Uuid; use crate::jobs::types::{ExternalId, JobItem, JobStatus, JobType}; -use crate::tests::config::TestConfigBuilder; +use crate::tests::config::{ClientValue, TestConfigBuilder}; #[rstest] #[tokio::test] @@ -16,7 +16,7 @@ async fn test_database_connection() -> color_eyre::Result<()> { #[rstest] #[tokio::test] async fn database_create_job_works() { - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new().configure_database(ClientValue::Actual).build().await; let config = services.config; let database_client = config.database(); @@ -55,7 +55,7 @@ async fn database_create_job_works() { #[case(false)] #[tokio::test] async fn database_get_jobs_without_successor_works(#[case] is_successor: bool) { - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new().configure_database(ClientValue::Actual).build().await; let config = services.config; let database_client = config.database(); @@ -99,7 +99,7 @@ async fn database_get_jobs_without_successor_works(#[case] is_successor: bool) { #[rstest] #[tokio::test] async fn database_get_last_successful_job_by_type_works() { - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new().configure_database(ClientValue::Actual).build().await; let config = services.config; let database_client = config.database(); @@ -127,7 +127,7 @@ async fn database_get_last_successful_job_by_type_works() { #[rstest] #[tokio::test] async fn database_get_jobs_after_internal_id_by_job_type_works() { - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new().configure_database(ClientValue::Actual).build().await; let config = services.config; let database_client = config.database(); @@ -162,7 +162,7 @@ async fn database_get_jobs_after_internal_id_by_job_type_works() { #[rstest] #[tokio::test] async fn database_update_job_status_passing_case_works() { - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new().configure_database(ClientValue::Actual).build().await; let config = services.config; let database_client = config.database(); @@ -181,7 +181,7 @@ async fn database_update_job_status_passing_case_works() { #[rstest] #[tokio::test] async fn database_update_job_status_failing_case_works() { - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new().configure_database(ClientValue::Actual).build().await; let config = services.config; let database_client = config.database(); diff --git a/crates/orchestrator/src/tests/jobs/da_job/mod.rs b/crates/orchestrator/src/tests/jobs/da_job/mod.rs index e3501bc2..66c59ddb 100644 --- a/crates/orchestrator/src/tests/jobs/da_job/mod.rs +++ b/crates/orchestrator/src/tests/jobs/da_job/mod.rs @@ -16,7 +16,7 @@ use crate::jobs::types::{ExternalId, JobItem, JobStatus, JobType}; use crate::jobs::Job; use crate::jobs::JobError; use crate::tests::common::drop_database; -use crate::tests::config::TestConfigBuilder; +use crate::tests::config::{ClientValue, TestConfigBuilder}; /// Tests the DA Job's handling of a blob length exceeding the supported size. /// It mocks the DA client to simulate the environment and expects an error on job processing. @@ -42,7 +42,12 @@ async fn test_da_job_process_job_failure_on_small_blob_size( da_client.expect_max_blob_per_txn().with().returning(|| 1); da_client.expect_max_bytes_per_blob().with().returning(|| 1200); - let services = TestConfigBuilder::new().configure_da_client(da_client.into()).build().await; + let services = TestConfigBuilder::new() + .configure_starknet_client(ClientValue::Actual) + .configure_storage_client(ClientValue::Actual) + .configure_da_client(da_client.into()) + .build() + .await; let state_update = read_state_update_from_file(state_update_file.as_str()).expect("issue while reading"); @@ -82,7 +87,7 @@ async fn test_da_job_process_job_failure_on_small_blob_size( ); state_update_mock.assert(); - let _ = drop_database().await; + // let _ = drop_database().await; } /// Tests DA Job processing failure when a block is in pending state. @@ -92,7 +97,11 @@ async fn test_da_job_process_job_failure_on_small_blob_size( #[rstest] #[tokio::test] async fn test_da_job_process_job_failure_on_pending_block() { - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new() + .configure_starknet_client(ClientValue::Actual) + .configure_da_client(ClientValue::Actual) + .build() + .await; let server = services.server; let internal_id = "1"; @@ -177,7 +186,11 @@ async fn test_da_job_process_job_success( da_client.expect_max_blob_per_txn().with().returning(|| 6); da_client.expect_max_bytes_per_blob().with().returning(|| 131072); - let services = TestConfigBuilder::new().configure_da_client(da_client.into()).build().await; + let services = TestConfigBuilder::new() + .configure_storage_client(ClientValue::Actual) + .configure_da_client(da_client.into()) + .build() + .await; let server = services.server; let state_update = read_state_update_from_file(state_update_file.as_str()).expect("issue while reading"); diff --git a/crates/orchestrator/src/tests/jobs/mod.rs b/crates/orchestrator/src/tests/jobs/mod.rs index bf325b39..55231b49 100644 --- a/crates/orchestrator/src/tests/jobs/mod.rs +++ b/crates/orchestrator/src/tests/jobs/mod.rs @@ -18,6 +18,7 @@ use crate::jobs::types::{ExternalId, JobItem, JobVerificationStatus}; use crate::jobs::{create_job, increment_key_in_metadata, process_job, verify_job, Job, MockJob}; use crate::queue::job_queue::{JOB_PROCESSING_QUEUE, JOB_VERIFICATION_QUEUE}; use crate::tests::common::MessagePayloadType; +use crate::tests::config::ClientValue; use crate::{jobs::types::JobStatus, tests::config::TestConfigBuilder}; use super::database::build_job_item; @@ -42,7 +43,11 @@ async fn create_job_job_does_not_exists_in_db_works() { let job_item_clone = job_item.clone(); job_handler.expect_create_job().times(1).returning(move |_, _, _| Ok(job_item_clone.clone())); - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new() + .configure_database(ClientValue::Actual) + .configure_queue_client(ClientValue::Actual) + .build() + .await; // Mocking the `get_job_handler` call in create_job function. let job_handler: Arc> = Arc::new(Box::new(job_handler)); @@ -77,7 +82,11 @@ async fn create_job_job_does_not_exists_in_db_works() { async fn create_job_job_exists_in_db_works() { let job_item = build_job_item_by_type_and_status(JobType::ProofCreation, JobStatus::Created, "0".to_string()); - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new() + .configure_database(ClientValue::Actual) + .configure_queue_client(ClientValue::Actual) + .build() + .await; let database_client = services.config.database(); database_client.create_job(job_item).await.unwrap(); @@ -101,7 +110,11 @@ async fn create_job_job_exists_in_db_works() { #[should_panic(expected = "Job type not implemented yet.")] #[tokio::test] async fn create_job_job_handler_is_not_implemented_panics() { - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new() + .configure_database(ClientValue::Actual) + .configure_queue_client(ClientValue::Actual) + .build() + .await; // Mocking the `get_job_handler` call in create_job function. let ctx = mock_factory::get_job_handler_context(); @@ -133,7 +146,11 @@ async fn process_job_with_job_exists_in_db_and_valid_job_processing_status_works let job_item = build_job_item_by_type_and_status(job_type.clone(), job_status.clone(), "1".to_string()); // Building config - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new() + .configure_database(ClientValue::Actual) + .configure_queue_client(ClientValue::Actual) + .build() + .await; let database_client = services.config.database(); let mut job_handler = MockJob::new(); @@ -176,7 +193,11 @@ async fn process_job_with_job_exists_in_db_with_invalid_job_processing_status_er let job_item = build_job_item_by_type_and_status(JobType::SnosRun, JobStatus::Completed, "1".to_string()); // building config - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new() + .configure_database(ClientValue::Actual) + .configure_queue_client(ClientValue::Actual) + .build() + .await; let database_client = services.config.database(); // creating job in database @@ -206,7 +227,11 @@ async fn process_job_job_does_not_exists_in_db_works() { let job_item = build_job_item_by_type_and_status(JobType::SnosRun, JobStatus::Created, "1".to_string()); // building config - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new() + .configure_database(ClientValue::Actual) + .configure_queue_client(ClientValue::Actual) + .build() + .await; assert!(process_job(job_item.id, services.config.clone()).await.is_err()); @@ -237,7 +262,11 @@ async fn process_job_two_workers_process_same_job_works() { ctx.expect().times(1).with(eq(JobType::SnosRun)).returning(move |_| Arc::clone(&job_handler)); // building config - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new() + .configure_database(ClientValue::Actual) + .configure_queue_client(ClientValue::Actual) + .build() + .await; let db_client = services.config.database(); let job_item = build_job_item_by_type_and_status(JobType::SnosRun, JobStatus::Created, "1".to_string()); @@ -277,7 +306,11 @@ async fn verify_job_with_verified_status_works() { build_job_item_by_type_and_status(JobType::DataSubmission, JobStatus::PendingVerification, "1".to_string()); // building config - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new() + .configure_database(ClientValue::Actual) + .configure_queue_client(ClientValue::Actual) + .build() + .await; let database_client = services.config.database(); let mut job_handler = MockJob::new(); @@ -320,7 +353,11 @@ async fn verify_job_with_rejected_status_adds_to_queue_works() { build_job_item_by_type_and_status(JobType::DataSubmission, JobStatus::PendingVerification, "1".to_string()); // building config - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new() + .configure_database(ClientValue::Actual) + .configure_queue_client(ClientValue::Actual) + .build() + .await; let database_client = services.config.database(); let mut job_handler = MockJob::new(); @@ -365,7 +402,11 @@ async fn verify_job_with_rejected_status_works() { job_item.metadata = metadata; // building config - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new() + .configure_database(ClientValue::Actual) + .configure_queue_client(ClientValue::Actual) + .build() + .await; let database_client = services.config.database(); let mut job_handler = MockJob::new(); @@ -406,7 +447,11 @@ async fn verify_job_with_pending_status_adds_to_queue_works() { build_job_item_by_type_and_status(JobType::DataSubmission, JobStatus::PendingVerification, "1".to_string()); // building config - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new() + .configure_database(ClientValue::Actual) + .configure_queue_client(ClientValue::Actual) + .build() + .await; let database_client = services.config.database(); let mut job_handler = MockJob::new(); @@ -454,7 +499,11 @@ async fn verify_job_with_pending_status_works() { job_item.metadata = metadata; // building config - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new() + .configure_database(ClientValue::Actual) + .configure_queue_client(ClientValue::Actual) + .build() + .await; let database_client = services.config.database(); let mut job_handler = MockJob::new(); @@ -512,7 +561,11 @@ fn build_job_item_by_type_and_status(job_type: JobType, job_status: JobStatus, i #[case(JobType::DataSubmission, JobStatus::VerificationFailed)] #[tokio::test] async fn handle_job_failure_with_failed_job_status_works(#[case] job_type: JobType, #[case] job_status: JobStatus) { - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new() + .configure_database(ClientValue::Actual) + .configure_queue_client(ClientValue::Actual) + .build() + .await; let database_client = services.config.database(); let internal_id = 1; @@ -542,7 +595,11 @@ async fn handle_job_failure_with_failed_job_status_works(#[case] job_type: JobTy #[case::verification_timeout(JobType::SnosRun, JobStatus::VerificationTimeout)] #[tokio::test] async fn handle_job_failure_with_correct_job_status_works(#[case] job_type: JobType, #[case] job_status: JobStatus) { - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new() + .configure_database(ClientValue::Actual) + .configure_queue_client(ClientValue::Actual) + .build() + .await; let database_client = services.config.database(); let internal_id = 1; @@ -577,7 +634,11 @@ async fn handle_job_failure_with_correct_job_status_works(#[case] job_type: JobT async fn handle_job_failure_job_status_completed_works(#[case] job_type: JobType) { let job_status = JobStatus::Completed; - let services = TestConfigBuilder::new().build().await; + let services = TestConfigBuilder::new() + .configure_database(ClientValue::Actual) + .configure_queue_client(ClientValue::Actual) + .build() + .await; let database_client = services.config.database(); let internal_id = 1; diff --git a/crates/orchestrator/src/tests/jobs/state_update_job/mod.rs b/crates/orchestrator/src/tests/jobs/state_update_job/mod.rs index 205b88b1..30f836ec 100644 --- a/crates/orchestrator/src/tests/jobs/state_update_job/mod.rs +++ b/crates/orchestrator/src/tests/jobs/state_update_job/mod.rs @@ -60,6 +60,8 @@ async fn test_process_job_works( ) { // Will be used by storage client which we call while storing the data. + use crate::tests::config::ClientValue; + dotenvy::from_filename("../.env.test").expect("Failed to load the .env file"); // Mocking the settlement client. @@ -93,7 +95,11 @@ async fn test_process_job_works( // Building a temp config that will be used by `fetch_blob_data_for_block` and `fetch_snos_for_block` // functions while fetching the blob data from storage client. - let services = TestConfigBuilder::new().configure_settlement_client(settlement_client.into()).build().await; + let services = TestConfigBuilder::new() + .configure_storage_client(ClientValue::Actual) + .configure_settlement_client(settlement_client.into()) + .build() + .await; let storage_client = services.config.storage();