Feat : job handler tests (#67)

* feat : added tests for increasing coverage

* feat : added mongo db tests and fixtures and updated the ci for tests

* update : removed unwanted fixtures

* update : removed unwanted fixtures

* update : added mongo db runner in ci

* update : added mongo db runner in ci

* update : added mongo db runner in ci

* update : updated with new changes and ci

* update : updated test cases for s3 client

* update : added .env.test file in the commit

* feat : added database necessary tests

* feat : added database necessary tests

* Revert "feat : added database necessary tests"

This reverts commit 65d66e6.

* Revert "feat : added database necessary tests"

This reverts commit 906a1eb.

* update: Replaced Build_Config Fixture with TestConfigBuilder

* update : config update

* update : test_put_and_get_data_s3 test

* update: moved testconfigbuilder to tests/config.rs & added docs , drop all collections not just jobs

* feat : create job test case error returning

* mock job handler

* feat : added jobs test and modified worker tests

* feat : added queue checks to tests and revamped some tests

* feat : updated tests and resolved comments

* feat : updated test config and added config type to aws s3 config

* feat : updated tests and test names

* feat : lint fixes

* feat : lint fixes

* feat : lint fixes

* chore: resolved pr comments

* Update crates/orchestrator/src/tests/database/mod.rs

Co-authored-by: 0xevolve <Artevolve@yahoo.com>

* chore : lint fixes

* feat : lint fix

* fix : coverage tests fix

* fix : test fix

* fix : updated region in localstack .env.test

* feat : updated region

* debug : added debug log to github ci

* feat : updated queue code for test fixes

* fix : sqs region fix

* debug : added debug logs for ci debugging

* feat : added override endpoint to queue url in producer and consumer

* feat : added override endpoint to queue url in producer and consumer

* fix : removed logs and refactored the code

* chore : refactor code

---------

Co-authored-by: Heemank Verma <heemankv@gmail.com>
Co-authored-by: apoorvsadana <95699312+apoorvsadana@users.noreply.github.com>
Co-authored-by: 0xevolve <Artevolve@yahoo.com>

.env.example

@@ -21,9 +21,14 @@ STARKNET_CAIRO_CORE_CONTRACT_ADDRESS=
 # MongoDB connection string
 MONGODB_CONNECTION_STRING=
 
-# SQS
+# AWS
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
+AWS_DEFAULT_REGION=
+
+# SQS
+SQS_JOB_PROCESSING_QUEUE_URL=
+SQS_JOB_VERIFICATION_QUEUE_URL=
 
 # S3
 AWS_S3_BUCKET_NAME=

.env.test

@@ -5,6 +5,9 @@ AWS_SECRET_ACCESS_KEY="AWS_SECRET_ACCESS_KEY"
 AWS_S3_BUCKET_NAME="madara-orchestrator-test-bucket"
 AWS_S3_BUCKET_REGION="us-east-1"
 AWS_ENDPOINT_URL="http://localhost.localstack.cloud:4566"
+SQS_JOB_PROCESSING_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_processing_queue"
+SQS_JOB_VERIFICATION_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_verification_queue"
+AWS_DEFAULT_REGION="localhost"
 
 ##### On chain config #####
 

CHANGELOG.md

@@ -10,13 +10,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 - Function to calculate the kzg proof of x_0.
 - Tests for updating the state.
 - Function to update the state and publish blob on ethereum in state update job.
+- Tests for job handlers in orchestrator/src/jobs/mod.rs.
 - Fixtures for testing.
+- Basic rust-toolchain support.
+- `AWS_DEFAULT_REGION="localhost"` var. in .env.test for omniqueue queue testing.
 - Added basic rust-toolchain support.
 
 ## Changed
 
 - GitHub's coverage CI yml file for localstack and db testing.
 - Orchestrator :Moved TestConfigBuilder to `config.rs` in tests folder.
+- `.env` file requires two more variables which are queue urls for processing
+  and verification.
 
 ## Removed
 

crates/orchestrator/Cargo.toml

@@ -14,10 +14,12 @@ path = "src/main.rs"
 [dependencies]
 alloy = { workspace = true }
 arc-swap = { workspace = true }
+assert_matches = "1.5.0"
 async-std = "1.12.0"
 async-trait = { workspace = true }
 aws-config = { version = "1.1.7", features = ["behavior-version-latest"] }
 aws-sdk-s3 = { version = "1.38.0", features = ["behavior-version-latest"] }
+aws-sdk-sqs = "1.36.0"
 axum = { workspace = true, features = ["macros"] }
 axum-macros = { workspace = true }
 bincode = { workspace = true }
@@ -35,7 +37,8 @@ lazy_static = { workspace = true }
 log = "0.4.21"
 majin-blob-core = { git = "https://github.com/AbdelStark/majin-blob", branch = "main" }
 majin-blob-types = { git = "https://github.com/AbdelStark/majin-blob", branch = "main" }
-mockall = "0.12.1"
+mockall = { version = "0.13.0" }
+mockall_double = "0.3.1"
 mongodb = { workspace = true, features = ["bson-uuid-1"], optional = true }
 num = { workspace = true }
 num-bigint = { workspace = true }

crates/orchestrator/src/database/mod.rs

@@ -1,6 +1,6 @@
-use ::mongodb::bson::doc;
 use std::collections::HashMap;
 
+use ::mongodb::bson::doc;
 use async_trait::async_trait;
 use color_eyre::Result;
 use mockall::automock;

crates/orchestrator/src/jobs/da_job/mod.rs

@@ -359,6 +359,7 @@ mod tests {
     use serde_json::json;
 
     use super::*;
+    // use majin_blob_types::serde;
     use crate::tests::common::init_config;
 
     #[rstest]

crates/orchestrator/src/jobs/job_handler_factory.rs

@@ -0,0 +1,65 @@
+use mockall::automock;
+
+#[automock]
+pub mod factory {
+    use std::sync::Arc;
+
+    #[allow(unused_imports)]
+    use mockall::automock;
+
+    use crate::jobs::types::JobType;
+    use crate::jobs::{da_job, proving_job, snos_job, state_update_job, Job};
+
+    /// To get the job handler
+    //         +-------------------+
+    //         |                   |
+    //         |  Arc<Box<dyn Job>>|
+    //         |                   |
+    //          +--------+----------+
+    //                   |
+    //                  |    +----------------+
+    //                  |    |                |
+    //                  +--->| Box<dyn Job>   |
+    //                  |    |                |
+    //                  |    +----------------+
+    //                  |             |
+    //                  |             |
+    //          +-------v-------+     |
+    //          |               |     |
+    //          | Closure 1     |     |
+    //          |               |     |
+    //          +---------------+     |
+    //                                |
+    //          +---------------+     |
+    //          |               |     |
+    //          | Closure x     |     |
+    //          |               |     |
+    //          +---------------+     |
+    //                                |
+    //                                |
+    //                                v
+    //                         +--------------+
+    //                         |              |
+    //                         | dyn Job      |
+    //                         | (job_handler)|
+    //                         |              |
+    //                         +--------------+
+    /// We are using Arc so that we can call the Arc::clone while testing that will point
+    /// to the same Box<dyn Job>. So when we are mocking the behaviour :
+    ///
+    /// - We create the MockJob
+    /// - We return this mocked job whenever a function calls `get_job_handler`
+    /// -  Making it an Arc allows us to return the same MockJob in multiple calls to `get_job_handler`. This is needed because `MockJob` doesn't implement Clone
+    pub async fn get_job_handler(job_type: &JobType) -> Arc<Box<dyn Job>> {
+        // Original implementation
+        let job: Box<dyn Job> = match job_type {
+            JobType::DataSubmission => Box::new(da_job::DaJob),
+            JobType::SnosRun => Box::new(snos_job::SnosJob),
+            JobType::ProofCreation => Box::new(proving_job::ProvingJob),
+            JobType::StateTransition => Box::new(state_update_job::StateUpdateJob),
+            _ => unimplemented!("Job type not implemented yet."),
+        };
+
+        Arc::new(job)
+    }
+}