update: parallel tests draft #1

madara-alliance · Aug 27, 2024 · dbe12a9 · dbe12a9
1 parent 081fee9
commit dbe12a9
Show file tree

Hide file tree

Showing 4 changed files with 73 additions and 12 deletions.
diff --git a/crates/orchestrator/src/tests/config.rs b/crates/orchestrator/src/tests/config.rs
@@ -42,8 +42,11 @@ pub struct TestConfigBuilder {
 
     // Storing for Data Storage client
     // These are need to be kept in scope to keep the Server alive
+    database_node: Option<ContainerAsync<Mongo>>,
     data_storage_node: Option<ContainerAsync<LocalStack>>,
     data_storage_client: Option<aws_sdk_s3::Client>,
+    queue_node: Option<ContainerAsync<LocalStack>>,
+    queue_client: Option<sqs::Client>,
 }
 
 impl Default for TestConfigBuilder {
@@ -64,8 +67,11 @@ impl TestConfigBuilder {
             queue: None,
             storage: None,
 
+            database_node: None,
             data_storage_node: None,
             data_storage_client: None,
+            queue_node: None,
+            queue_client: None,
         }
     }
 
@@ -77,6 +83,21 @@ impl TestConfigBuilder {
         self
     }
 
+    pub async fn testcontainer_sqs_data_storage(mut self, queue_name: String) -> TestConfigBuilder {
+        let (node, storage_client, client) = sqs_testcontainer_setup(queue_name).await;
+        self.queue = Some(storage_client);
+        self.queue_client = Some(client);
+        self.queue_node = Some(node);
+        self
+    }
+
+    pub async fn testcontainer_mongo_database(mut self) -> TestConfigBuilder {
+        let (node, database) = mongodb_testcontainer_setup().await;
+        self.database = Some(database);
+        self.database_node = Some(node);
+        self
+    }
+
     pub fn mock_da_client(mut self, da_client: Box<dyn DaClient>) -> TestConfigBuilder {
         self.da_client = Some(da_client);
         self
@@ -112,7 +133,16 @@ impl TestConfigBuilder {
         self
     }
 
-    pub async fn build(mut self) -> (MockServer, Option<ContainerAsync<LocalStack>>, Option<aws_sdk_s3::Client>) {
+    pub async fn build(
+        mut self,
+    ) -> (
+        MockServer,
+        Option<ContainerAsync<LocalStack>>,
+        Option<aws_sdk_s3::Client>,
+        Option<ContainerAsync<Mongo>>,
+        Option<sqs::Client>,
+        Option<ContainerAsync<LocalStack>>,
+    ) {
         dotenvy::from_filename("../.env.test").expect("Failed to load the .env file");
 
         let server = MockServer::start();
@@ -173,12 +203,18 @@ impl TestConfigBuilder {
 
         config_force_init(config).await;
 
-        (server, self.data_storage_node, self.data_storage_client)
+        (
+            server,
+            self.data_storage_node,
+            self.data_storage_client,
+            self.database_node,
+            self.queue_client,
+            self.queue_node,
+        )
     }
 }
 
 /// LocalStack (s3 and sqs) & MongoDb Setup using TestContainers ////
-
 use super::common::testcontainer_setups::LocalStack;
 use crate::data_storage::aws_s3::config::{AWSS3ConfigType, S3LocalStackConfig};
 use crate::data_storage::aws_s3::AWSS3;

diff --git a/crates/orchestrator/src/tests/data_storage/mod.rs b/crates/orchestrator/src/tests/data_storage/mod.rs
@@ -11,7 +11,7 @@ use serde_json::json;
 #[rstest]
 #[tokio::test]
 async fn test_put_and_get_data_s3() -> color_eyre::Result<()> {
-    let (_server, _localstack, _client) = TestConfigBuilder::new().testcontainer_s3_data_storage().await.build().await;
+    let services = TestConfigBuilder::new().testcontainer_s3_data_storage().await.build().await;
 
     dotenvy::from_filename("../.env.test")?;
 

diff --git a/crates/orchestrator/src/tests/database/mod.rs b/crates/orchestrator/src/tests/database/mod.rs
@@ -65,7 +65,7 @@ async fn database_get_jobs_without_successor_works(
     #[future] get_config: Guard<Arc<Config>>,
     #[case] is_successor: bool,
 ) {
-    TestConfigBuilder::new().build().await;
+    let services = TestConfigBuilder::new().testcontainer_mongo_database().await.build().await;
     let config = get_config.await;
     let database_client = config.database();
 
@@ -109,7 +109,7 @@ async fn database_get_jobs_without_successor_works(
 #[rstest]
 #[tokio::test]
 async fn database_get_last_successful_job_by_type_works(#[future] get_config: Guard<Arc<Config>>) {
-    TestConfigBuilder::new().build().await;
+    let services = TestConfigBuilder::new().testcontainer_mongo_database().await.build().await;
     let config = get_config.await;
     let database_client = config.database();
 

diff --git a/crates/orchestrator/src/tests/jobs/mod.rs b/crates/orchestrator/src/tests/jobs/mod.rs
@@ -45,7 +45,13 @@ async fn create_job_job_does_not_exists_in_db_works() {
     let job_item_clone = job_item.clone();
     job_handler.expect_create_job().times(1).returning(move |_, _, _| Ok(job_item_clone.clone()));
 
-    TestConfigBuilder::new().build().await;
+    let services = TestConfigBuilder::new()
+        .testcontainer_mongo_database()
+        .await
+        .testcontainer_sqs_data_storage(JOB_PROCESSING_QUEUE.to_string())
+        .await
+        .build()
+        .await;
     let config = config().await;
 
     // Mocking the `get_job_handler` call in create_job function.
@@ -134,7 +140,13 @@ async fn process_job_with_job_exists_in_db_and_valid_job_processing_status_works
     let job_item = build_job_item_by_type_and_status(job_type.clone(), job_status.clone(), "1".to_string());
 
     // Building config
-    TestConfigBuilder::new().build().await;
+    let services = TestConfigBuilder::new()
+        .testcontainer_mongo_database()
+        .await
+        .testcontainer_sqs_data_storage(JOB_VERIFICATION_QUEUE.to_string())
+        .await
+        .build()
+        .await;
     let config = config().await;
     let database_client = config.database();
 
@@ -209,7 +221,8 @@ async fn process_job_job_does_not_exists_in_db_works() {
     let job_item = build_job_item_by_type_and_status(JobType::SnosRun, JobStatus::Created, "1".to_string());
 
     // building config
-    TestConfigBuilder::new().build().await;
+    let services =
+        TestConfigBuilder::new().testcontainer_sqs_data_storage(JOB_VERIFICATION_QUEUE.to_string()).await.build().await;
     let config = config().await;
 
     assert!(process_job(job_item.id).await.is_err());
@@ -241,7 +254,7 @@ async fn process_job_two_workers_process_same_job_works() {
     ctx.expect().times(1).with(eq(JobType::SnosRun)).returning(move |_| Arc::clone(&job_handler));
 
     // building config
-    TestConfigBuilder::new().build().await;
+    let services = TestConfigBuilder::new().testcontainer_mongo_database().await.build().await;
     let config = config().await;
     let db_client = config.database();
 
@@ -323,7 +336,13 @@ async fn verify_job_with_rejected_status_adds_to_queue_works() {
         build_job_item_by_type_and_status(JobType::DataSubmission, JobStatus::PendingVerification, "1".to_string());
 
     // building config
-    TestConfigBuilder::new().build().await;
+    let services = TestConfigBuilder::new()
+        .testcontainer_mongo_database()
+        .await
+        .testcontainer_sqs_data_storage(JOB_PROCESSING_QUEUE.to_string())
+        .await
+        .build()
+        .await;
 
     let config = config().await;
     let database_client = config.database();
@@ -410,7 +429,13 @@ async fn verify_job_with_pending_status_adds_to_queue_works() {
         build_job_item_by_type_and_status(JobType::DataSubmission, JobStatus::PendingVerification, "1".to_string());
 
     // building config
-    TestConfigBuilder::new().build().await;
+    let services = TestConfigBuilder::new()
+        .testcontainer_mongo_database()
+        .await
+        .testcontainer_sqs_data_storage(JOB_VERIFICATION_QUEUE.to_string())
+        .await
+        .build()
+        .await;
 
     let config = config().await;
     let database_client = config.database();