update: moved testconfigbuilder to tests/config.rs & added docs , dro…

…p all collections not just jobs

CHANGELOG.md

@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 ## Changed
 
 - GitHub's coverage CI yml file for localstack and db testing.
+- Orchestrator :Moved TestConfigBuilder to `config.rs` in tests folder.
 
 ## Removed
 

crates/orchestrator/src/config.rs

@@ -183,116 +183,3 @@ pub async fn build_storage_client() -> Box<dyn DataStorage + Send + Sync> {
         _ => panic!("Unsupported Storage Client"),
     }
 }
-
-#[cfg(test)]
-use httpmock::MockServer;
-
-// Inspiration : https://rust-unofficial.github.io/patterns/patterns/creational/builder.html
-// TestConfigBuilder allows to heavily customise the global configs based on the test's requirement.
-// Eg: We want to mock only the da client and leave rest to be as it is, use mock_da_client.
-
-// TestBuilder for Config
-#[cfg(test)]
-pub struct TestConfigBuilder {
-    /// The starknet client to get data from the node
-    starknet_client: Option<Arc<JsonRpcClient<HttpTransport>>>,
-    /// The DA client to interact with the DA layer
-    da_client: Option<Box<dyn DaClient>>,
-    /// The service that produces proof and registers it onchain
-    prover_client: Option<Box<dyn ProverClient>>,
-    /// Settlement client
-    settlement_client: Option<Box<dyn SettlementClient>>,
-    /// The database client
-    database: Option<Box<dyn Database>>,
-    /// Queue client
-    queue: Option<Box<dyn QueueProvider>>,
-    /// Storage client
-    storage: Option<Box<dyn DataStorage>>,
-}
-
-#[cfg(test)]
-impl Default for TestConfigBuilder {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-#[cfg(test)]
-impl TestConfigBuilder {
-    /// Create a new config
-    pub fn new() -> TestConfigBuilder {
-        TestConfigBuilder {
-            starknet_client: None,
-            da_client: None,
-            prover_client: None,
-            settlement_client: None,
-            database: None,
-            queue: None,
-            storage: None,
-        }
-    }
-
-    pub fn mock_da_client(mut self, da_client: Box<dyn DaClient>) -> TestConfigBuilder {
-        self.da_client = Some(da_client);
-        self
-    }
-
-    pub async fn build(mut self) -> MockServer {
-        dotenvy::from_filename("../.env.test").expect("Failed to load the .env file");
-
-        // init starknet client
-        if self.starknet_client.is_none() {
-            let provider = JsonRpcClient::new(HttpTransport::new(
-                Url::parse(get_env_var_or_panic("MADARA_RPC_URL").as_str()).expect("Failed to parse URL"),
-            ));
-            self.starknet_client = Some(Arc::new(provider));
-        }
-
-        // init database
-        if self.database.is_none() {
-            self.database = Some(Box::new(MongoDb::new(MongoDbConfig::new_from_env()).await));
-        }
-
-        // init queue
-        if self.queue.is_none() {
-            self.queue = Some(Box::new(SqsQueue {}));
-        }
-
-        // init the DA client
-        if self.da_client.is_none() {
-            self.da_client = Some(build_da_client().await);
-        }
-
-        let settings_provider = DefaultSettingsProvider {};
-
-        // init the Settings client
-        if self.settlement_client.is_none() {
-            self.settlement_client = Some(build_settlement_client(&settings_provider).await);
-        }
-
-        // init the Prover client
-        if self.prover_client.is_none() {
-            self.prover_client = Some(build_prover_service(&settings_provider));
-        }
-
-        // init the storage client
-        if self.storage.is_none() {
-            self.storage = Some(build_storage_client().await);
-        }
-
-        // return config and server as tuple
-        let config = Config::new(
-            self.starknet_client.unwrap(),
-            self.da_client.unwrap(),
-            self.prover_client.unwrap(),
-            self.settlement_client.unwrap(),
-            self.database.unwrap(),
-            self.queue.unwrap(),
-            self.storage.unwrap(),
-        );
-
-        config_force_init(config).await;
-
-        MockServer::start()
-    }
-}

crates/orchestrator/src/database/mongodb/mod.rs

@@ -40,6 +40,9 @@ impl MongoDb {
         MongoDb { client }
     }
 
+    /// Mongodb client uses Arc internally, reducing the cost of clone.
+    /// Directly using clone is not recommended for libraries not using Arc internally.
+    /// Dev might want to pass an Arc manually.
     pub fn client(&self) -> Client {
         self.client.clone()
     }

crates/orchestrator/src/tests/common/mod.rs

@@ -80,7 +80,9 @@ pub fn custom_job_item(default_job_item: JobItem, #[default(String::from("0"))]
 
 pub async fn drop_database() -> color_eyre::Result<()> {
     let db_client: Client = MongoDb::new(MongoDbConfig::new_from_env()).await.client();
-    // dropping `jobs` collection.
-    db_client.database("orchestrator").collection::<JobItem>("jobs").drop(None).await?;
+    // dropping all the collection.
+    // use .collection::<JobItem>("<collection_name>")
+    // if only particular collection is to be dropped
+    db_client.database("orchestrator").drop(None).await?;
     Ok(())
 }

crates/orchestrator/src/tests/config.rs

@@ -0,0 +1,139 @@
+use std::sync::Arc;
+
+use crate::config::{
+    build_da_client, build_prover_service, build_settlement_client, build_storage_client, config_force_init, Config,
+};
+use crate::data_storage::DataStorage;
+use da_client_interface::DaClient;
+use prover_client_interface::ProverClient;
+use settlement_client_interface::SettlementClient;
+use starknet::providers::jsonrpc::HttpTransport;
+use starknet::providers::{JsonRpcClient, Url};
+use utils::env_utils::get_env_var_or_panic;
+use utils::settings::default::DefaultSettingsProvider;
+
+use crate::database::mongodb::config::MongoDbConfig;
+use crate::database::mongodb::MongoDb;
+use crate::database::{Database, DatabaseConfig};
+use crate::queue::sqs::SqsQueue;
+use crate::queue::QueueProvider;
+
+use httpmock::MockServer;
+// Inspiration : https://rust-unofficial.github.io/patterns/patterns/creational/builder.html
+// TestConfigBuilder allows to heavily customise the global configs based on the test's requirement.
+// Eg: We want to mock only the da client and leave rest to be as it is, use mock_da_client.
+
+// TestBuilder for Config
+pub struct TestConfigBuilder {
+    /// The starknet client to get data from the node
+    starknet_client: Option<Arc<JsonRpcClient<HttpTransport>>>,
+    /// The DA client to interact with the DA layer
+    da_client: Option<Box<dyn DaClient>>,
+    /// The service that produces proof and registers it onchain
+    prover_client: Option<Box<dyn ProverClient>>,
+    /// Settlement client
+    settlement_client: Option<Box<dyn SettlementClient>>,
+    /// The database client
+    database: Option<Box<dyn Database>>,
+    /// Queue client
+    queue: Option<Box<dyn QueueProvider>>,
+    /// Storage client
+    storage: Option<Box<dyn DataStorage>>,
+}
+
+impl Default for TestConfigBuilder {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl TestConfigBuilder {
+    /// Create a new config
+    pub fn new() -> TestConfigBuilder {
+        TestConfigBuilder {
+            starknet_client: None,
+            da_client: None,
+            prover_client: None,
+            settlement_client: None,
+            database: None,
+            queue: None,
+            storage: None,
+        }
+    }
+
+    pub fn mock_da_client(mut self, da_client: Box<dyn DaClient>) -> TestConfigBuilder {
+        self.da_client = Some(da_client);
+        self
+    }
+
+    pub async fn build(mut self) -> MockServer {
+        dotenvy::from_filename("../.env.test").expect("Failed to load the .env file");
+
+        let server = MockServer::start();
+
+        // init starknet client
+        if self.starknet_client.is_none() {
+            let provider = JsonRpcClient::new(HttpTransport::new(
+                Url::parse(format!("http://localhost:{}", server.port()).as_str()).expect("Failed to parse URL"),
+            ));
+            self.starknet_client = Some(Arc::new(provider));
+        }
+
+        // init database
+        if self.database.is_none() {
+            self.database = Some(Box::new(MongoDb::new(MongoDbConfig::new_from_env()).await));
+        }
+
+        // init queue
+        if self.queue.is_none() {
+            self.queue = Some(Box::new(SqsQueue {}));
+        }
+
+        // init the DA client
+        if self.da_client.is_none() {
+            self.da_client = Some(build_da_client().await);
+        }
+
+        let settings_provider = DefaultSettingsProvider {};
+
+        // init the Settings client
+        if self.settlement_client.is_none() {
+            self.settlement_client = Some(build_settlement_client(&settings_provider).await);
+        }
+
+        // init the Prover client
+        if self.prover_client.is_none() {
+            self.prover_client = Some(build_prover_service(&settings_provider));
+        }
+
+        // init the storage client
+        if self.storage.is_none() {
+            self.storage = Some(build_storage_client().await);
+            match get_env_var_or_panic("DATA_STORAGE").as_str() {
+                "s3" => self
+                    .storage
+                    .as_ref()
+                    .unwrap()
+                    .build_test_bucket(&get_env_var_or_panic("AWS_S3_BUCKET_NAME"))
+                    .await
+                    .unwrap(),
+                _ => panic!("Unsupported Storage Client"),
+            }
+        }
+
+        // return config and server as tuple
+        let config = Config::new(
+            self.starknet_client.unwrap(),
+            self.da_client.unwrap(),
+            self.prover_client.unwrap(),
+            self.settlement_client.unwrap(),
+            self.database.unwrap(),
+            self.queue.unwrap(),
+            self.storage.unwrap(),
+        );
+
+        config_force_init(config).await;
+
+        server
+    }
+}

crates/orchestrator/src/tests/data_storage/mod.rs

@@ -1,12 +1,16 @@
-use crate::config::TestConfigBuilder;
 use crate::data_storage::aws_s3::config::AWSS3Config;
 use crate::data_storage::aws_s3::AWSS3;
 use crate::data_storage::{DataStorage, DataStorageConfig};
+use crate::tests::config::TestConfigBuilder;
 use bytes::Bytes;
 use rstest::rstest;
 use serde_json::json;
 use utils::env_utils::get_env_var_or_panic;
 
+/// This test checks the ability to put and get data from AWS S3 using `AWSS3`.
+/// It puts JSON data into a test bucket and retrieves it, verifying the data
+/// matches what was originally uploaded.
+/// Dependencies: `color_eyre`, `dotenvy`, `rstest`, `tokio`, `serde_json`.
 #[rstest]
 #[tokio::test]
 async fn test_put_and_get_data_s3() -> color_eyre::Result<()> {

crates/orchestrator/src/tests/database/mod.rs

@@ -1,6 +1,7 @@
-use crate::config::{config, TestConfigBuilder};
+use crate::config::config;
 use crate::jobs::types::{ExternalId, JobItem, JobStatus, JobType};
 use crate::tests::common::drop_database;
+use crate::tests::config::TestConfigBuilder;
 use rstest::*;
 use uuid::Uuid;
 

crates/orchestrator/src/tests/mod.rs

@@ -1,3 +1,4 @@
+pub mod config;
 pub mod database;
 
 pub mod jobs;