From 47a2f32b3dfd34af5c18076458539dd8901ad28e Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Wed, 23 Aug 2023 22:03:30 +0200 Subject: [PATCH 01/41] proper alignment in `Makefile` --- Makefile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 08a0723d..58d232e8 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,17 @@ db_postgres: docker run --rm -d --name postgres -p 5432:5432 \ - -e POSTGRES_DB=fang \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - postgres:latest + -e POSTGRES_DB=fang \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + postgres:latest # login is root fang db_mysql: docker run --rm -d --name mysql -p 3306:3306 \ - -e MYSQL_DATABASE=fang \ - -e MYSQL_ROOT_PASSWORD=fang \ - -e TZ=UTC \ - mysql:latest + -e MYSQL_DATABASE=fang \ + -e MYSQL_ROOT_PASSWORD=fang \ + -e TZ=UTC \ + mysql:latest db_sqlite: sqlite3 fang.db "VACUUM;" From c3ae6566b266c8aedfb0f27776bb58dd094de43c Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Wed, 23 Aug 2023 22:15:31 +0200 Subject: [PATCH 02/41] use diesel cli arguments in `Makefile` --- Makefile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 58d232e8..06058c59 100644 --- a/Makefile +++ b/Makefile @@ -20,13 +20,19 @@ clippy: cargo clippy --verbose --all-targets --all-features -- -D warnings diesel_sqlite: - cd fang/sqlite_migrations && DATABASE_URL=sqlite://../../fang.db diesel migration run + diesel migration run \ + --database-url sqlite://../../fang.db \ + --migration-dir fang/sqlite_migrations diesel_postgres: - cd fang/postgres_migrations && DATABASE_URL=postgres://postgres:postgres@localhost/fang diesel migration run + diesel migration run \ + --database-url postgres://postgres:postgres@localhost/fang \ + --migration-dir fang/postgres_migrations diesel_mysql: - cd fang/mysql_migrations && DATABASE_URL=mysql://root:fang@127.0.0.1/fang diesel migration run + diesel migration run \ + --database-url mysql://root:fang@127.0.0.1/fang \ + --migration-dir fang/mysql_migrations stop_mysql: docker kill mysql From bacca91ea2cc895ba761cbcaac75141a007005a0 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Wed, 23 Aug 2023 22:42:28 +0200 Subject: [PATCH 03/41] fix mistake with diesel cli --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 06058c59..6060c31c 100644 --- a/Makefile +++ b/Makefile @@ -20,19 +20,19 @@ clippy: cargo clippy --verbose --all-targets --all-features -- -D warnings diesel_sqlite: + cd fang/sqlite_migrations && \ diesel migration run \ --database-url sqlite://../../fang.db \ - --migration-dir fang/sqlite_migrations diesel_postgres: + cd fang/postgres_migrations && \ diesel migration run \ --database-url postgres://postgres:postgres@localhost/fang \ - --migration-dir fang/postgres_migrations diesel_mysql: + cd fang/mysql_migrations && \ diesel migration run \ --database-url mysql://root:fang@127.0.0.1/fang \ - --migration-dir fang/mysql_migrations stop_mysql: docker kill mysql From 24072780176ad152fb0924a587124eb45de1d4b7 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Wed, 23 Aug 2023 22:43:38 +0200 Subject: [PATCH 04/41] better organization --- Makefile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 6060c31c..ba938d91 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ db_postgres: -e POSTGRES_PASSWORD=postgres \ postgres:latest -# login is root fang + db_mysql: docker run --rm -d --name mysql -p 3306:3306 \ -e MYSQL_DATABASE=fang \ @@ -19,11 +19,6 @@ db_sqlite: clippy: cargo clippy --verbose --all-targets --all-features -- -D warnings -diesel_sqlite: - cd fang/sqlite_migrations && \ - diesel migration run \ - --database-url sqlite://../../fang.db \ - diesel_postgres: cd fang/postgres_migrations && \ diesel migration run \ @@ -34,6 +29,11 @@ diesel_mysql: diesel migration run \ --database-url mysql://root:fang@127.0.0.1/fang \ +diesel_sqlite: + cd fang/sqlite_migrations && \ + diesel migration run \ + --database-url sqlite://../../fang.db \ + stop_mysql: docker kill mysql @@ -42,6 +42,7 @@ stop_postgres: stop_sqlite: rm fang.db + tests: DATABASE_URL=postgres://postgres:postgres@localhost/fang cargo test --all-features -- --color always --nocapture From 4d22be3e574a402405887cc1cd8dc403fa768c91 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Wed, 23 Aug 2023 22:49:57 +0200 Subject: [PATCH 05/41] user/password consistency in `Makefile` --- Makefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index ba938d91..0734fb8d 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,10 @@ db_postgres: docker run --rm -d --name postgres -p 5432:5432 \ -e POSTGRES_DB=fang \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_USER=fang \ + -e POSTGRES_PASSWORD=fang \ postgres:latest - db_mysql: docker run --rm -d --name mysql -p 3306:3306 \ -e MYSQL_DATABASE=fang \ @@ -22,7 +21,7 @@ clippy: diesel_postgres: cd fang/postgres_migrations && \ diesel migration run \ - --database-url postgres://postgres:postgres@localhost/fang \ + --database-url postgres://fang:fang@localhost/fang \ diesel_mysql: cd fang/mysql_migrations && \ From 1ae01f7d2df76a687c86bb793e61a80bc0aa67dc Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Wed, 23 Aug 2023 23:12:54 +0200 Subject: [PATCH 06/41] add `.PHONY` target --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 0734fb8d..1c62ad99 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,8 @@ +.PHONY: db_postgres db_mysql db_sqlite \ + diesel_postgres diesel_mysql diesel_sqlite \ + stop_postgres stop_mysql stop_sqlite \ + clippy tests ignored doc + db_postgres: docker run --rm -d --name postgres -p 5432:5432 \ -e POSTGRES_DB=fang \ From 75702dd0aaf529faf6c63db64e7598cd3ad6d652 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Wed, 23 Aug 2023 23:13:41 +0200 Subject: [PATCH 07/41] move `clippy` target --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1c62ad99..b01960ae 100644 --- a/Makefile +++ b/Makefile @@ -20,9 +20,6 @@ db_mysql: db_sqlite: sqlite3 fang.db "VACUUM;" -clippy: - cargo clippy --verbose --all-targets --all-features -- -D warnings - diesel_postgres: cd fang/postgres_migrations && \ diesel migration run \ @@ -47,6 +44,9 @@ stop_postgres: stop_sqlite: rm fang.db +clippy: + cargo clippy --verbose --all-targets --all-features -- -D warnings + tests: DATABASE_URL=postgres://postgres:postgres@localhost/fang cargo test --all-features -- --color always --nocapture From 150fd44ad2ebf856b35df886ba9759cbb3c178f0 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Thu, 24 Aug 2023 00:55:50 +0200 Subject: [PATCH 08/41] remove unnecessary environment variables --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b01960ae..553af4c0 100644 --- a/Makefile +++ b/Makefile @@ -48,10 +48,10 @@ clippy: cargo clippy --verbose --all-targets --all-features -- -D warnings tests: - DATABASE_URL=postgres://postgres:postgres@localhost/fang cargo test --all-features -- --color always --nocapture + cargo test --all-features -- --color always --nocapture ignored: - DATABASE_URL=postgres://postgres:postgres@localhost/fang cargo test --all-features -- --color always --nocapture --ignored + cargo test --all-features -- --color always --nocapture --ignored doc: cargo doc --open From eb6464c614ecc8c790f85316fed5e62592c34456 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Thu, 24 Aug 2023 01:15:34 +0200 Subject: [PATCH 09/41] fix problem from 4d22be3e5 --- .env | 2 +- fang/src/asynk/async_queue.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 6d0480c8..6a182add 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DATABASE_URL=postgres://postgres:postgres@localhost/fang +DATABASE_URL=postgres://fang:fang@localhost/fang diff --git a/fang/src/asynk/async_queue.rs b/fang/src/asynk/async_queue.rs index 67117ec5..61b5af59 100644 --- a/fang/src/asynk/async_queue.rs +++ b/fang/src/asynk/async_queue.rs @@ -183,7 +183,7 @@ static ASYNC_QUEUE_DB_TEST_COUNTER: Mutex = Mutex::const_new(0); impl AsyncQueue { /// Provides an AsyncQueue connected to its own DB pub async fn test() -> Self { - const BASE_URI: &str = "postgres://postgres:postgres@localhost"; + const BASE_URI: &str = "postgres://fang:fang@localhost"; let mut res = Self::builder() .max_pool_size(1_u32) .uri(format!("{}/fang", BASE_URI)) From dfac909902b3f5b27f10db6a670c6f858616bc21 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Thu, 24 Aug 2023 01:21:59 +0200 Subject: [PATCH 10/41] maybe this is better for users/passwords --- .env | 2 +- Makefile | 16 ++++++++++++---- fang/src/asynk/async_queue.rs | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 6a182add..6d0480c8 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DATABASE_URL=postgres://fang:fang@localhost/fang +DATABASE_URL=postgres://postgres:postgres@localhost/fang diff --git a/Makefile b/Makefile index 553af4c0..788c81c1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ .PHONY: db_postgres db_mysql db_sqlite \ + wait_for_postgres \ diesel_postgres diesel_mysql diesel_sqlite \ stop_postgres stop_mysql stop_sqlite \ clippy tests ignored doc @@ -6,21 +7,28 @@ db_postgres: docker run --rm -d --name postgres -p 5432:5432 \ -e POSTGRES_DB=fang \ - -e POSTGRES_USER=fang \ - -e POSTGRES_PASSWORD=fang \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ postgres:latest db_mysql: docker run --rm -d --name mysql -p 3306:3306 \ -e MYSQL_DATABASE=fang \ - -e MYSQL_ROOT_PASSWORD=fang \ + -e MYSQL_ROOT_PASSWORD=mysql \ -e TZ=UTC \ mysql:latest db_sqlite: sqlite3 fang.db "VACUUM;" -diesel_postgres: +wait_for_postgres: + @echo Waiting for Postgres server to be up and running... + while ! docker exec postgres psql -U fang --command=';' 2> /dev/null; \ + do \ + sleep 1; \ + done + +diesel_postgres: db_postgres wait_for_postgres cd fang/postgres_migrations && \ diesel migration run \ --database-url postgres://fang:fang@localhost/fang \ diff --git a/fang/src/asynk/async_queue.rs b/fang/src/asynk/async_queue.rs index 61b5af59..67117ec5 100644 --- a/fang/src/asynk/async_queue.rs +++ b/fang/src/asynk/async_queue.rs @@ -183,7 +183,7 @@ static ASYNC_QUEUE_DB_TEST_COUNTER: Mutex = Mutex::const_new(0); impl AsyncQueue { /// Provides an AsyncQueue connected to its own DB pub async fn test() -> Self { - const BASE_URI: &str = "postgres://fang:fang@localhost"; + const BASE_URI: &str = "postgres://postgres:postgres@localhost"; let mut res = Self::builder() .max_pool_size(1_u32) .uri(format!("{}/fang", BASE_URI)) From e820335a58d05fa49fcffdc81a19aabff9dcf1be Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Thu, 24 Aug 2023 12:08:49 +0200 Subject: [PATCH 11/41] fix fuck up again --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 788c81c1..a650a398 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ db_sqlite: wait_for_postgres: @echo Waiting for Postgres server to be up and running... - while ! docker exec postgres psql -U fang --command=';' 2> /dev/null; \ + while ! docker exec postgres psql -U postgres --command=';' 2> /dev/null; \ do \ sleep 1; \ done @@ -31,7 +31,7 @@ wait_for_postgres: diesel_postgres: db_postgres wait_for_postgres cd fang/postgres_migrations && \ diesel migration run \ - --database-url postgres://fang:fang@localhost/fang \ + --database-url postgres://postgres:postgres@localhost/fang \ diesel_mysql: cd fang/mysql_migrations && \ From b18b6f38094d305f4e6aec7b27f7bda7cb01aebf Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Thu, 24 Aug 2023 15:29:09 +0200 Subject: [PATCH 12/41] better organization and wait for MySQL --- Makefile | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index a650a398..eab75786 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: db_postgres db_mysql db_sqlite \ - wait_for_postgres \ + wait_for_postgres wait_for_mysql \ diesel_postgres diesel_mysql diesel_sqlite \ stop_postgres stop_mysql stop_sqlite \ clippy tests ignored doc @@ -21,28 +21,39 @@ db_mysql: db_sqlite: sqlite3 fang.db "VACUUM;" -wait_for_postgres: +wait_for_postgres: db_postgres @echo Waiting for Postgres server to be up and running... while ! docker exec postgres psql -U postgres --command=';' 2> /dev/null; \ do \ sleep 1; \ done -diesel_postgres: db_postgres wait_for_postgres +wait_for_mysql: db_mysql + @echo Waiting for MySQL server to be up and running... + while ! docker exec mysql mysqlsh --user=root --password=mysql --execute=";" 2> /dev/null; \ + do \ + sleep 1; \ + done + +diesel: diesel_postgres diesel_sqlite + +diesel_postgres: wait_for_postgres cd fang/postgres_migrations && \ diesel migration run \ - --database-url postgres://postgres:postgres@localhost/fang \ + --database-url postgres://postgres:postgres@127.0.0.1/fang \ -diesel_mysql: +diesel_mysql: wait_for_mysql cd fang/mysql_migrations && \ diesel migration run \ - --database-url mysql://root:fang@127.0.0.1/fang \ + --database-url mysql://root:mysql@127.0.0.1/fang \ -diesel_sqlite: +diesel_sqlite: db_sqlite cd fang/sqlite_migrations && \ diesel migration run \ --database-url sqlite://../../fang.db \ +stop: stop_postgres stop_mysql stop_sqlite + stop_mysql: docker kill mysql @@ -55,8 +66,9 @@ stop_sqlite: clippy: cargo clippy --verbose --all-targets --all-features -- -D warnings -tests: +tests: diesel_postgres diesel_mysql diesel_sqlite cargo test --all-features -- --color always --nocapture + $(MAKE) stop ignored: cargo test --all-features -- --color always --nocapture --ignored From a464fffd4c3e66b9b56407ddf53280a6c9b7a9b3 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Thu, 24 Aug 2023 16:25:01 +0200 Subject: [PATCH 13/41] format `README.md` --- fang/README.md | 68 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/fang/README.md b/fang/README.md index df52d776..714bf22b 100644 --- a/fang/README.md +++ b/fang/README.md @@ -8,50 +8,53 @@ Background task processing library for Rust. It uses Postgres DB as a task queue ## Key Features - Here are some of the fang's key features: - - - Async and threaded workers. - Workers can be started in threads (threaded workers) or `tokio` tasks (async workers) - - Scheduled tasks. - Tasks can be scheduled at any time in the future - - Periodic (CRON) tasks. - Tasks can be scheduled using cron expressions - - Unique tasks. - Tasks are not duplicated in the queue if they are unique - - Single-purpose workers. - Tasks are stored in a single table but workers can execute only tasks of the specific type - - Retries. - Tasks can be retried with a custom backoff mode +Here are some of the fang's key features: + +- Async and threaded workers. + Workers can be started in threads (threaded workers) or `tokio` tasks (async workers) +- Scheduled tasks. + Tasks can be scheduled at any time in the future +- Periodic (CRON) tasks. + Tasks can be scheduled using cron expressions +- Unique tasks. + Tasks are not duplicated in the queue if they are unique +- Single-purpose workers. + Tasks are stored in a single table but workers can execute only tasks of the specific type +- Retries. + Tasks can be retried with a custom backoff mode ## Installation 1. Add this to your Cargo.toml - #### the Blocking feature + ```toml [dependencies] fang = { version = "0.10.4" , features = ["blocking"], default-features = false } ``` #### the Asynk feature + ```toml [dependencies] fang = { version = "0.10.4" , features = ["asynk"], default-features = false } ``` #### the Asynk feature with derive macro + ```toml [dependencies] fang = { version = "0.10.4" , features = ["asynk", "derive-error" ], default-features = false } ``` #### All features + ```toml fang = { version = "0.10.4" } ``` -*Supports rustc 1.62+* +_Supports rustc 1.62+_ 2. Create the `fang_tasks` table in the Postgres database. The migration can be found in [the migrations directory](https://github.com/ayrat555/fang/blob/master/fang/postgres_migrations/migrations/2022-08-20-151615_create_fang_tasks/up.sql). @@ -60,11 +63,11 @@ fang = { version = "0.10.4" } ### Defining a task #### Blocking feature + Every task should implement the `fang::Runnable` trait which is used by `fang` to execute it. If you have a `CustomError`, it is recommended to implement `From`. So this way you can use [? operator](https://stackoverflow.com/questions/42917566/what-is-this-question-mark-operator-about#42921174) inside the `run` function available in `fang::Runnable` trait. - You can easily implement it with the macro `ToFangError`. This macro is only available in the feature `derive-error`. ```rust @@ -107,8 +110,8 @@ impl Runnable for MyTask { fn run(&self, _queue: &dyn Queueable) -> Result<(), FangError> { println!("the number is {}", self.number); - my_func(self.number)?; - // You can use ? operator because + my_func(self.number)?; + // You can use ? operator because // From is implemented thanks to ToFangError derive macro. Ok(()) @@ -150,11 +153,12 @@ As you can see from the example above, the trait implementation has `#[typetag:: The second parameter of the `run` function is a struct that implements `fang::Queueable`. You can re-use it to manipulate the task queue, for example, to add a new job during the current job's execution. If you don't need it, just ignore it. - #### Asynk feature + Every task should implement `fang::AsyncRunnable` trait which is used by `fang` to execute it. Be careful not to call two implementations of the AsyncRunnable trait with the same name, because it will cause a failure in the `typetag` crate. + ```rust use fang::AsyncRunnable; use fang::asynk::async_queue::AsyncQueueable; @@ -218,10 +222,10 @@ If your timezone is UTC + 2 and you want to schedule at 11:00: let expression = "0 0 9 * * * *"; ``` - ### Enqueuing a task #### the Blocking feature + To enqueue a task use `Queue::enqueue_task` ```rust @@ -238,9 +242,11 @@ use fang::Queue; ``` #### the Asynk feature + To enqueue a task use `AsyncQueueable::insert_task`. -For Postgres backend. +For Postgres backend: + ```rust use fang::asynk::async_queue::AsyncQueue; use fang::NoTls; @@ -260,7 +266,8 @@ let mut queue = AsyncQueue::builder() queue.connect(NoTls).await.unwrap(); ``` -As an easy example, we are using NoTls type. If for some reason you would like to encrypt Postgres requests, you can use [openssl](https://docs.rs/postgres-openssl/latest/postgres_openssl/) or [native-tls](https://docs.rs/postgres-native-tls/latest/postgres_native_tls/). + +As an easy example, we are using NoTls type. If for some reason you would like to encrypt Postgres requests, you can use [openssl](https://docs.rs/postgres-openssl/latest/postgres_openssl/) or [native-tls](https://docs.rs/postgres-native-tls/latest/postgres_native_tls/). ```rust // AsyncTask from the first example @@ -274,11 +281,11 @@ let task_returned = queue ### Starting workers #### the Blocking feature + Every worker runs in a separate thread. In case of panic, they are always restarted. Use `WorkerPool` to start workers. Use `WorkerPool::builder` to create your worker pool and run tasks. - ```rust use fang::WorkerPool; use fang::Queue; @@ -296,6 +303,7 @@ worker_pool.start(); ``` #### the Asynk feature + Every worker runs in a separate `tokio` task. In case of panic, they are always restarted. Use `AsyncWorkerPool` to start workers. @@ -315,7 +323,6 @@ let mut pool: AsyncWorkerPool> = AsyncWorkerPool::builder() pool.start().await; ``` - Check out: - [Simple Worker Example](https://github.com/ayrat555/fang/tree/master/fang_examples/blocking/simple_worker) - simple worker example @@ -370,8 +377,8 @@ pub struct SleepParams { If there are no tasks in the DB, a worker sleeps for `sleep_period` and each time this value increases by `sleep_step` until it reaches `max_sleep_period`. `min_sleep_period` is the initial value for `sleep_period`. All values are in seconds. - Use `set_sleep_params` to set it: + ```rust let sleep_params = SleepParams { sleep_period: Duration::from_secs(2), @@ -392,33 +399,41 @@ Set sleep params with worker pools `TypeBuilder` in both modules. 5. Create a new Pull Request ### Running tests locally + - Install diesel_cli. + ``` cargo install diesel_cli --no-default-features --features "postgres sqlite mysql" ``` + - Install docker on your machine. - Run a Postgres docker container. (See in Makefile.) + ``` make db_postgres ``` - Run the migrations + ``` make diesel_postgres ``` - Run tests + ``` make tests ``` - Run dirty//long tests, DB must be recreated afterwards. + ``` make ignored ``` - Kill the docker container + ``` make stop ``` @@ -429,7 +444,6 @@ make stop - Pepe Márquez (@pxp9) - [s1]: https://img.shields.io/crates/v/fang.svg [docs-badge]: https://img.shields.io/badge/docs-website-blue.svg [ci]: https://crates.io/crates/fang From b475dd01a5f747fad00a8782178f445c47ca2d14 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Thu, 24 Aug 2023 16:43:49 +0200 Subject: [PATCH 14/41] format `README.md` Rust code --- fang/README.md | 53 ++++++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/fang/README.md b/fang/README.md index 714bf22b..984067bb 100644 --- a/fang/README.md +++ b/fang/README.md @@ -82,21 +82,20 @@ use std::fmt::Debug; #[derive(Debug, ToFangError)] enum CustomError { - ErrorOne(String), - ErrorTwo(u32), + ErrorOne(String), + ErrorTwo(u32), } fn my_func(num : u16) -> Result<(), CustomError> { + if num == 0 { + Err(CustomError::ErrorOne("is zero".to_string())) + } - if num == 0 { - Err(CustomError::ErrorOne("is zero".to_string())) - } - - if num > 500 { - Err(CustomError::ErrorTwo(num)) - } + if num > 500 { + Err(CustomError::ErrorTwo(num)) + } - Ok(()) + Ok(()) } #[derive(Serialize, Deserialize)] @@ -120,13 +119,13 @@ impl Runnable for MyTask { // If `uniq` is set to true and the task is already in the storage, it won't be inserted again // The existing record will be returned for for any insertions operaiton fn uniq(&self) -> bool { - true + true } // This will be useful if you want to filter tasks. // the default value is `common` fn task_type(&self) -> String { - "my_task".to_string() + "my_task".to_string() } // This will be useful if you would like to schedule tasks. @@ -139,12 +138,12 @@ impl Runnable for MyTask { // the maximum number of retries. Set it to 0 to make it not retriable // the default value is 20 fn max_retries(&self) -> i32 { - 20 + 20 } // backoff mode for retries fn backoff(&self, attempt: u32) -> u32 { - u32::pow(2, attempt) + u32::pow(2, attempt) } } ``` @@ -168,7 +167,7 @@ use fang::async_trait; #[derive(Serialize, Deserialize)] #[serde(crate = "fang::serde")] struct AsyncTask { - pub number: u16, + pub number: u16, } #[typetag::serde] @@ -187,7 +186,7 @@ impl AsyncRunnable for AsyncTask { // If `uniq` is set to true and the task is already in the storage, it won't be inserted again // The existing record will be returned for for any insertions operaiton fn uniq(&self) -> bool { - true + true } // This will be useful if you would like to schedule tasks. @@ -200,12 +199,12 @@ impl AsyncRunnable for AsyncTask { // the maximum number of retries. Set it to 0 to make it not retriable // the default value is 20 fn max_retries(&self) -> i32 { - 20 + 20 } // backoff mode for retries fn backoff(&self, attempt: u32) -> u32 { - u32::pow(2, attempt) + u32::pow(2, attempt) } } ``` @@ -219,7 +218,7 @@ Example: If your timezone is UTC + 2 and you want to schedule at 11:00: ```rust - let expression = "0 0 9 * * * *"; +let expression = "0 0 9 * * * *"; ``` ### Enqueuing a task @@ -235,10 +234,9 @@ use fang::Queue; // create a fang queue - let queue = Queue::builder().connection_pool(pool).build(); - - let task_inserted = queue.insert_task(&MyTask::new(1)).unwrap(); +let queue = Queue::builder().connection_pool(pool).build(); +let task_inserted = queue.insert_task(&MyTask::new(1)).unwrap(); ``` #### the Asynk feature @@ -264,7 +262,6 @@ let mut queue = AsyncQueue::builder() // Always connect first in order to perform any operation queue.connect(NoTls).await.unwrap(); - ``` As an easy example, we are using NoTls type. If for some reason you would like to encrypt Postgres requests, you can use [openssl](https://docs.rs/postgres-openssl/latest/postgres_openssl/) or [native-tls](https://docs.rs/postgres-native-tls/latest/postgres_native_tls/). @@ -273,9 +270,9 @@ As an easy example, we are using NoTls type. If for some reason you would like t // AsyncTask from the first example let task = AsyncTask { 8 }; let task_returned = queue - .insert_task(&task as &dyn AsyncRunnable) - .await - .unwrap(); + .insert_task(&task as &dyn AsyncRunnable) + .await + .unwrap(); ``` ### Starting workers @@ -295,7 +292,7 @@ use fang::Queue; let mut worker_pool = WorkerPool::::builder() .queue(queue) .number_of_workers(3_u32) - // if you want to run tasks of the specific kind + // if you want to run tasks of the specific kind .task_type("my_task_type") .build(); @@ -316,7 +313,7 @@ use fang::asynk::async_worker_pool::AsyncWorkerPool; let mut pool: AsyncWorkerPool> = AsyncWorkerPool::builder() .number_of_workers(max_pool_size) .queue(queue.clone()) - // if you want to run tasks of the specific kind + // if you want to run tasks of the specific kind .task_type("my_task_type") .build(); From b609210b665a59d7fd956802c0bacd35034ee425 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Thu, 24 Aug 2023 18:25:55 +0200 Subject: [PATCH 15/41] update `README.md` --- fang/README.md | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/fang/README.md b/fang/README.md index 984067bb..3829283d 100644 --- a/fang/README.md +++ b/fang/README.md @@ -399,42 +399,24 @@ Set sleep params with worker pools `TypeBuilder` in both modules. - Install diesel_cli. -``` +```sh cargo install diesel_cli --no-default-features --features "postgres sqlite mysql" ``` - Install docker on your machine. -- Run a Postgres docker container. (See in Makefile.) - -``` -make db_postgres -``` - -- Run the migrations - -``` -make diesel_postgres -``` - - Run tests -``` +```sh make tests ``` -- Run dirty//long tests, DB must be recreated afterwards. +- Run dirty/long tests. -``` +```sh make ignored ``` -- Kill the docker container - -``` -make stop -``` - ## Authors - Ayrat Badykov (@ayrat555) From 107bc5d6f4fba3ccfecf01d486e1d4ac8d80fac6 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Thu, 24 Aug 2023 21:22:55 +0200 Subject: [PATCH 16/41] another probably better approach --- Makefile | 58 +++++++++++++++++++++++++++++++++++++------------- fang/README.md | 14 +++++++++++- 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index eab75786..1d1a2312 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,20 @@ -.PHONY: db_postgres db_mysql db_sqlite \ - wait_for_postgres wait_for_mysql \ - diesel_postgres diesel_mysql diesel_sqlite \ - stop_postgres stop_mysql stop_sqlite \ +.PHONY: db db_postgres db_mysql db_sqlite \ + wait_for_postgres wait_for_mysql wait_for_sqlite \ + diesel diesel_postgres diesel_mysql diesel_sqlite \ + stop stop_postgres stop_mysql stop_sqlite \ clippy tests ignored doc +.SILENT: + +db: db_postgres db_mysql db_sqlite + db_postgres: docker run --rm -d --name postgres -p 5432:5432 \ -e POSTGRES_DB=fang \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ postgres:latest + $(MAKE) diesel_postgres db_mysql: docker run --rm -d --name mysql -p 3306:3306 \ @@ -17,40 +22,62 @@ db_mysql: -e MYSQL_ROOT_PASSWORD=mysql \ -e TZ=UTC \ mysql:latest + $(MAKE) diesel_mysql db_sqlite: sqlite3 fang.db "VACUUM;" + $(MAKE) diesel_sqlite -wait_for_postgres: db_postgres +wait_for_postgres: @echo Waiting for Postgres server to be up and running... - while ! docker exec postgres psql -U postgres --command=';' 2> /dev/null; \ + while ! docker exec postgres psql -U postgres --command='' 2> /dev/null; \ do \ sleep 1; \ done -wait_for_mysql: db_mysql +wait_for_mysql: @echo Waiting for MySQL server to be up and running... - while ! docker exec mysql mysqlsh --user=root --password=mysql --execute=";" 2> /dev/null; \ + while ! docker exec mysql mysql --user=root --password=mysql --execute='' 2> /dev/null; \ + do \ + sleep 1; \ + done + +wait_for_sqlite: + @echo Waiting for SQLite DB file to be created... + while [ ! -f fang.db ]; \ do \ sleep 1; \ done -diesel: diesel_postgres diesel_sqlite +diesel: diesel_postgres diesel_mysql diesel_sqlite diesel_postgres: wait_for_postgres cd fang/postgres_migrations && \ diesel migration run \ - --database-url postgres://postgres:postgres@127.0.0.1/fang \ + --database-url postgres://postgres:postgres@127.0.0.1/fang diesel_mysql: wait_for_mysql cd fang/mysql_migrations && \ diesel migration run \ - --database-url mysql://root:mysql@127.0.0.1/fang \ + --database-url mysql://root:mysql@127.0.0.1/fang -diesel_sqlite: db_sqlite +diesel_sqlite: wait_for_sqlite cd fang/sqlite_migrations && \ diesel migration run \ - --database-url sqlite://../../fang.db \ + --database-url sqlite://../../fang.db + +clean: clean_postgres clean_mysql clean_sqlite + +clean_postgres: wait_for_postgres + docker exec postgres dropdb -U postgres fang + docker exec postgres psql -U postgres --command="CREATE DATABASE fang;" + $(MAKE) diesel_postgres + +clean_mysql: wait_for_mysql + docker exec mysql mysql --user=root --password=mysql --execute="DROP DATABASE fang; CREATE DATABASE fang;" + $(MAKE) diesel_mysql + +clean_sqlite: wait_for_sqlite stop_sqlite db_sqlite stop: stop_postgres stop_mysql stop_sqlite @@ -66,12 +93,13 @@ stop_sqlite: clippy: cargo clippy --verbose --all-targets --all-features -- -D warnings -tests: diesel_postgres diesel_mysql diesel_sqlite +tests: cargo test --all-features -- --color always --nocapture - $(MAKE) stop + $(MAKE) clean ignored: cargo test --all-features -- --color always --nocapture --ignored + $(MAKE) clean doc: cargo doc --open diff --git a/fang/README.md b/fang/README.md index 3829283d..514537d0 100644 --- a/fang/README.md +++ b/fang/README.md @@ -405,7 +405,13 @@ cargo install diesel_cli --no-default-features --features "postgres sqlite mysql - Install docker on your machine. -- Run tests +- Setup databases for testing. + +```sh +make db +``` + +- Run tests. `make db` does not need to be run in between each test cycle. ```sh make tests @@ -417,6 +423,12 @@ make tests make ignored ``` +- Take down databases. + +```sh +make stop +``` + ## Authors - Ayrat Badykov (@ayrat555) From 4c43b93c2f015a0cca7f0211e3aadd7f53bd70a8 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Thu, 24 Aug 2023 22:34:21 +0200 Subject: [PATCH 17/41] better output --- Makefile | 46 +++++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 1d1a2312..e0f8ee0c 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,22 @@ -.PHONY: db db_postgres db_mysql db_sqlite \ - wait_for_postgres wait_for_mysql wait_for_sqlite \ - diesel diesel_postgres diesel_mysql diesel_sqlite \ - stop stop_postgres stop_mysql stop_sqlite \ +DB_TARGETS = db_postgres db_mysql db_sqlite +WAIT_TARGETS = wait_for_postgres wait_for_mysql wait_for_sqlite +DIESEL_TARGETS = diesel_postgres diesel_mysql diesel_sqlite +CLEAN_TARGETS = clean_postgres clean_mysql clean_sqlite +STOP_TARGETS = stop_postgres stop_mysql stop_sqlite + +.PHONY: db $(DB_TARGETS) \ + $(WAIT_TARGETS) \ + diesel $(DIESEL_TARGETS) \ + clean $(CLEAN_TARGETS) + stop $(STOP_TARGETS) \ clippy tests ignored doc -.SILENT: +.SILENT: $(DB_TARGETS) $(WAIT_TARGETS) $(DIESEL_TARGETS) $(CLEAN_TARGETS) $(STOP_TARGETS) -db: db_postgres db_mysql db_sqlite +db: $(DB_TARGETS) db_postgres: + @echo Setting up Postgres database... docker run --rm -d --name postgres -p 5432:5432 \ -e POSTGRES_DB=fang \ -e POSTGRES_USER=postgres \ @@ -17,6 +25,7 @@ db_postgres: $(MAKE) diesel_postgres db_mysql: + @echo Setting up MySQL database... docker run --rm -d --name mysql -p 3306:3306 \ -e MYSQL_DATABASE=fang \ -e MYSQL_ROOT_PASSWORD=mysql \ @@ -25,6 +34,7 @@ db_mysql: $(MAKE) diesel_mysql db_sqlite: + @echo Setting up SQLite database... sqlite3 fang.db "VACUUM;" $(MAKE) diesel_sqlite @@ -49,45 +59,55 @@ wait_for_sqlite: sleep 1; \ done -diesel: diesel_postgres diesel_mysql diesel_sqlite +diesel: $(DIESEL_TARGETS) diesel_postgres: wait_for_postgres + @echo Running Diesel migrations on Postgres database... cd fang/postgres_migrations && \ diesel migration run \ --database-url postgres://postgres:postgres@127.0.0.1/fang diesel_mysql: wait_for_mysql + @echo Running Diesel migrations on MySQL database... cd fang/mysql_migrations && \ diesel migration run \ --database-url mysql://root:mysql@127.0.0.1/fang diesel_sqlite: wait_for_sqlite + @echo Running Diesel migrations on SQLite database... cd fang/sqlite_migrations && \ diesel migration run \ --database-url sqlite://../../fang.db -clean: clean_postgres clean_mysql clean_sqlite +clean: $(CLEAN_TARGETS) clean_postgres: wait_for_postgres + @echo Cleaning Postgres database... docker exec postgres dropdb -U postgres fang docker exec postgres psql -U postgres --command="CREATE DATABASE fang;" $(MAKE) diesel_postgres clean_mysql: wait_for_mysql + @echo Cleaning MySQL database... docker exec mysql mysql --user=root --password=mysql --execute="DROP DATABASE fang; CREATE DATABASE fang;" $(MAKE) diesel_mysql -clean_sqlite: wait_for_sqlite stop_sqlite db_sqlite +clean_sqlite: wait_for_sqlite + @echo Cleaning SQLite database... + $(MAKE) stop_sqlite db_sqlite -stop: stop_postgres stop_mysql stop_sqlite - -stop_mysql: - docker kill mysql +stop: $(STOP_TARGETS) stop_postgres: + @echo Stopping Postgres database... docker kill postgres +stop_mysql: + @echo Stopping MySQL database... + docker kill mysql + stop_sqlite: + @echo Stopping SQLite database... rm fang.db clippy: From 0b535317fca341c5f6f598304a98f4a7e7f96fa8 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Thu, 24 Aug 2023 22:52:29 +0200 Subject: [PATCH 18/41] bold echoes --- Makefile | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index e0f8ee0c..d6b9aee6 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,6 @@ +BOLD = '\033[1m' +END_BOLD = '\033[0m' + DB_TARGETS = db_postgres db_mysql db_sqlite WAIT_TARGETS = wait_for_postgres wait_for_mysql wait_for_sqlite DIESEL_TARGETS = diesel_postgres diesel_mysql diesel_sqlite @@ -16,7 +19,7 @@ STOP_TARGETS = stop_postgres stop_mysql stop_sqlite db: $(DB_TARGETS) db_postgres: - @echo Setting up Postgres database... + @echo -e $(BOLD)Setting up Postgres database...$(END_BOLD) docker run --rm -d --name postgres -p 5432:5432 \ -e POSTGRES_DB=fang \ -e POSTGRES_USER=postgres \ @@ -25,7 +28,7 @@ db_postgres: $(MAKE) diesel_postgres db_mysql: - @echo Setting up MySQL database... + @echo -e $(BOLD)Setting up MySQL database...$(END_BOLD) docker run --rm -d --name mysql -p 3306:3306 \ -e MYSQL_DATABASE=fang \ -e MYSQL_ROOT_PASSWORD=mysql \ @@ -34,26 +37,26 @@ db_mysql: $(MAKE) diesel_mysql db_sqlite: - @echo Setting up SQLite database... + @echo -e $(BOLD)Setting up SQLite database...$(END_BOLD) sqlite3 fang.db "VACUUM;" $(MAKE) diesel_sqlite wait_for_postgres: - @echo Waiting for Postgres server to be up and running... + @echo -e $(BOLD)Waiting for Postgres server to be up and running...$(END_BOLD) while ! docker exec postgres psql -U postgres --command='' 2> /dev/null; \ do \ sleep 1; \ done wait_for_mysql: - @echo Waiting for MySQL server to be up and running... + @echo -e $(BOLD)Waiting for MySQL server to be up and running...$(END_BOLD) while ! docker exec mysql mysql --user=root --password=mysql --execute='' 2> /dev/null; \ do \ sleep 1; \ done wait_for_sqlite: - @echo Waiting for SQLite DB file to be created... + @echo -e $(BOLD)Waiting for SQLite DB file to be created...$(END_BOLD) while [ ! -f fang.db ]; \ do \ sleep 1; \ @@ -62,19 +65,19 @@ wait_for_sqlite: diesel: $(DIESEL_TARGETS) diesel_postgres: wait_for_postgres - @echo Running Diesel migrations on Postgres database... + @echo -e $(BOLD)Running Diesel migrations on Postgres database...$(END_BOLD) cd fang/postgres_migrations && \ diesel migration run \ --database-url postgres://postgres:postgres@127.0.0.1/fang diesel_mysql: wait_for_mysql - @echo Running Diesel migrations on MySQL database... + @echo -e $(BOLD)Running Diesel migrations on MySQL database...$(END_BOLD) cd fang/mysql_migrations && \ diesel migration run \ --database-url mysql://root:mysql@127.0.0.1/fang diesel_sqlite: wait_for_sqlite - @echo Running Diesel migrations on SQLite database... + @echo -e $(BOLD)Running Diesel migrations on SQLite database...$(END_BOLD) cd fang/sqlite_migrations && \ diesel migration run \ --database-url sqlite://../../fang.db @@ -82,42 +85,44 @@ diesel_sqlite: wait_for_sqlite clean: $(CLEAN_TARGETS) clean_postgres: wait_for_postgres - @echo Cleaning Postgres database... + @echo -e $(BOLD)Cleaning Postgres database...$(END_BOLD) docker exec postgres dropdb -U postgres fang docker exec postgres psql -U postgres --command="CREATE DATABASE fang;" $(MAKE) diesel_postgres clean_mysql: wait_for_mysql - @echo Cleaning MySQL database... + @echo -e $(BOLD)Cleaning MySQL database...$(END_BOLD) docker exec mysql mysql --user=root --password=mysql --execute="DROP DATABASE fang; CREATE DATABASE fang;" $(MAKE) diesel_mysql clean_sqlite: wait_for_sqlite - @echo Cleaning SQLite database... + @echo -e $(BOLD)Cleaning SQLite database...$(END_BOLD) $(MAKE) stop_sqlite db_sqlite stop: $(STOP_TARGETS) stop_postgres: - @echo Stopping Postgres database... + @echo -e $(BOLD)Stopping Postgres database...$(END_BOLD) docker kill postgres stop_mysql: - @echo Stopping MySQL database... + @echo -e $(BOLD)Stopping MySQL database...$(END_BOLD) docker kill mysql stop_sqlite: - @echo Stopping SQLite database... + @echo -e $(BOLD)Stopping SQLite database...$(END_BOLD) rm fang.db clippy: cargo clippy --verbose --all-targets --all-features -- -D warnings tests: + @echo -e $(BOLD)Running tests...$(END_BOLD) cargo test --all-features -- --color always --nocapture $(MAKE) clean ignored: + @echo -e $(BOLD)Running ignored tests...$(END_BOLD) cargo test --all-features -- --color always --nocapture --ignored $(MAKE) clean From 408f1b63eef2f9aa4c647dcd2f7b172100643758 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Thu, 24 Aug 2023 22:53:18 +0200 Subject: [PATCH 19/41] update `make doc` --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d6b9aee6..1cbc5b26 100644 --- a/Makefile +++ b/Makefile @@ -127,4 +127,4 @@ ignored: $(MAKE) clean doc: - cargo doc --open + cargo doc --package fang --open From 4d7ff7fdeb3ec239f2fc1c6128d209e6a6f96dcd Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Thu, 24 Aug 2023 23:25:48 +0200 Subject: [PATCH 20/41] use variables in `Makefile` --- Makefile | 55 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 1cbc5b26..fe326eb7 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,15 @@ +POSTGRES_CONTAINER=postgres +POSTGRES_DB=fang +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres + +MYSQL_CONTAINER=mysql +MYSQL_DB=fang +MYSQL_USER=root +MYSQL_PASSWORD=mysql + +SQLITE_FILE=fang.db + BOLD = '\033[1m' END_BOLD = '\033[0m' @@ -20,44 +32,44 @@ db: $(DB_TARGETS) db_postgres: @echo -e $(BOLD)Setting up Postgres database...$(END_BOLD) - docker run --rm -d --name postgres -p 5432:5432 \ - -e POSTGRES_DB=fang \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ + docker run --rm -d --name $(POSTGRES_CONTAINER) -p 5432:5432 \ + -e POSTGRES_DB=$(POSTGRES_DB) \ + -e POSTGRES_USER=$(POSTGRES_USER) \ + -e POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) \ postgres:latest $(MAKE) diesel_postgres db_mysql: @echo -e $(BOLD)Setting up MySQL database...$(END_BOLD) - docker run --rm -d --name mysql -p 3306:3306 \ - -e MYSQL_DATABASE=fang \ - -e MYSQL_ROOT_PASSWORD=mysql \ + docker run --rm -d --name $(MYSQL_CONTAINER) -p 3306:3306 \ + -e MYSQL_DATABASE=$(MYSQL_DB) \ + -e MYSQL_ROOT_PASSWORD=$(MYSQL_PASSWORD) \ -e TZ=UTC \ mysql:latest $(MAKE) diesel_mysql db_sqlite: @echo -e $(BOLD)Setting up SQLite database...$(END_BOLD) - sqlite3 fang.db "VACUUM;" + sqlite3 $(SQLITE_FILE) "VACUUM;" $(MAKE) diesel_sqlite wait_for_postgres: @echo -e $(BOLD)Waiting for Postgres server to be up and running...$(END_BOLD) - while ! docker exec postgres psql -U postgres --command='' 2> /dev/null; \ + while ! docker exec $(POSTGRES_CONTAINER) psql -U $(POSTGRES_USER) --command='' 2> /dev/null; \ do \ sleep 1; \ done wait_for_mysql: @echo -e $(BOLD)Waiting for MySQL server to be up and running...$(END_BOLD) - while ! docker exec mysql mysql --user=root --password=mysql --execute='' 2> /dev/null; \ + while ! docker exec $(MYSQL_CONTAINER) mysql --user=$(MYSQL_USER) --password=$(MYSQL_PASSWORD) --execute='' 2> /dev/null; \ do \ sleep 1; \ done wait_for_sqlite: @echo -e $(BOLD)Waiting for SQLite DB file to be created...$(END_BOLD) - while [ ! -f fang.db ]; \ + while [ ! -f $(SQLITE_FILE) ]; \ do \ sleep 1; \ done @@ -68,31 +80,34 @@ diesel_postgres: wait_for_postgres @echo -e $(BOLD)Running Diesel migrations on Postgres database...$(END_BOLD) cd fang/postgres_migrations && \ diesel migration run \ - --database-url postgres://postgres:postgres@127.0.0.1/fang + --database-url postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@127.0.0.1/$(POSTGRES_DB) diesel_mysql: wait_for_mysql @echo -e $(BOLD)Running Diesel migrations on MySQL database...$(END_BOLD) cd fang/mysql_migrations && \ diesel migration run \ - --database-url mysql://root:mysql@127.0.0.1/fang + --database-url mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@127.0.0.1/$(MYSQL_DB) diesel_sqlite: wait_for_sqlite @echo -e $(BOLD)Running Diesel migrations on SQLite database...$(END_BOLD) cd fang/sqlite_migrations && \ diesel migration run \ - --database-url sqlite://../../fang.db + --database-url sqlite://../../$(SQLITE_FILE) clean: $(CLEAN_TARGETS) clean_postgres: wait_for_postgres @echo -e $(BOLD)Cleaning Postgres database...$(END_BOLD) - docker exec postgres dropdb -U postgres fang - docker exec postgres psql -U postgres --command="CREATE DATABASE fang;" + docker exec $(POSTGRES_CONTAINER) dropdb -U $(POSTGRES_USER) $(POSTGRES_DB) + docker exec $(POSTGRES_CONTAINER) psql -U $(POSTGRES_USER) --command="CREATE DATABASE $(POSTGRES_DB);" $(MAKE) diesel_postgres clean_mysql: wait_for_mysql @echo -e $(BOLD)Cleaning MySQL database...$(END_BOLD) - docker exec mysql mysql --user=root --password=mysql --execute="DROP DATABASE fang; CREATE DATABASE fang;" + docker exec $(MYSQL_CONTAINER) mysql \ + --user=$(MYSQL_USER) \ + --password=$(MYSQL_PASSWORD) \ + --execute="DROP DATABASE $(MYSQL_DB); CREATE DATABASE $(MYSQL_DB);" $(MAKE) diesel_mysql clean_sqlite: wait_for_sqlite @@ -103,15 +118,15 @@ stop: $(STOP_TARGETS) stop_postgres: @echo -e $(BOLD)Stopping Postgres database...$(END_BOLD) - docker kill postgres + docker kill $(POSTGRES_CONTAINER) stop_mysql: @echo -e $(BOLD)Stopping MySQL database...$(END_BOLD) - docker kill mysql + docker kill $(MYSQL_CONTAINER) stop_sqlite: @echo -e $(BOLD)Stopping SQLite database...$(END_BOLD) - rm fang.db + rm $(SQLITE_FILE) clippy: cargo clippy --verbose --all-targets --all-features -- -D warnings From 0585ad54a043d9b937e3c93fded009f1e4610560 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Thu, 24 Aug 2023 23:31:01 +0200 Subject: [PATCH 21/41] use `--migration-dir` for Diesel --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index fe326eb7..569130f0 100644 --- a/Makefile +++ b/Makefile @@ -78,21 +78,21 @@ diesel: $(DIESEL_TARGETS) diesel_postgres: wait_for_postgres @echo -e $(BOLD)Running Diesel migrations on Postgres database...$(END_BOLD) - cd fang/postgres_migrations && \ diesel migration run \ - --database-url postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@127.0.0.1/$(POSTGRES_DB) + --database-url postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@127.0.0.1/$(POSTGRES_DB) \ + --migration-dir fang/postgres_migrations/migrations diesel_mysql: wait_for_mysql @echo -e $(BOLD)Running Diesel migrations on MySQL database...$(END_BOLD) - cd fang/mysql_migrations && \ diesel migration run \ - --database-url mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@127.0.0.1/$(MYSQL_DB) + --database-url mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@127.0.0.1/$(MYSQL_DB) \ + --migration-dir fang/mysql_migrations/migrations diesel_sqlite: wait_for_sqlite @echo -e $(BOLD)Running Diesel migrations on SQLite database...$(END_BOLD) - cd fang/sqlite_migrations && \ diesel migration run \ - --database-url sqlite://../../$(SQLITE_FILE) + --database-url sqlite://$(SQLITE_FILE) \ + --migration-dir fang/sqlite_migrations/migrations clean: $(CLEAN_TARGETS) From c829c1f031e94c642937919e107558ba3f0bf87b Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Thu, 24 Aug 2023 23:37:14 +0200 Subject: [PATCH 22/41] add variables for migrations --- Makefile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 569130f0..c7cc72a5 100644 --- a/Makefile +++ b/Makefile @@ -2,13 +2,16 @@ POSTGRES_CONTAINER=postgres POSTGRES_DB=fang POSTGRES_USER=postgres POSTGRES_PASSWORD=postgres +POSTGRES_MIGRATIONS=fang/postgres_migrations/migrations MYSQL_CONTAINER=mysql MYSQL_DB=fang MYSQL_USER=root MYSQL_PASSWORD=mysql +MYSQL_MIGRATIONS=fang/mysql_migrations/migrations SQLITE_FILE=fang.db +SQLITE_MIGRATIONS=fang/sqlite_migrations/migrations BOLD = '\033[1m' END_BOLD = '\033[0m' @@ -80,19 +83,19 @@ diesel_postgres: wait_for_postgres @echo -e $(BOLD)Running Diesel migrations on Postgres database...$(END_BOLD) diesel migration run \ --database-url postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@127.0.0.1/$(POSTGRES_DB) \ - --migration-dir fang/postgres_migrations/migrations + --migration-dir $(POSTGRES_MIGRATIONS) diesel_mysql: wait_for_mysql @echo -e $(BOLD)Running Diesel migrations on MySQL database...$(END_BOLD) diesel migration run \ --database-url mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@127.0.0.1/$(MYSQL_DB) \ - --migration-dir fang/mysql_migrations/migrations + --migration-dir $(MYSQL_MIGRATIONS) diesel_sqlite: wait_for_sqlite @echo -e $(BOLD)Running Diesel migrations on SQLite database...$(END_BOLD) diesel migration run \ --database-url sqlite://$(SQLITE_FILE) \ - --migration-dir fang/sqlite_migrations/migrations + --migration-dir $(SQLITE_MIGRATIONS) clean: $(CLEAN_TARGETS) From e6704a1905c391d8f6b174a558774f3cb6cc33f0 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Thu, 24 Aug 2023 23:56:31 +0200 Subject: [PATCH 23/41] add `fang/mysql_migrations/diesel.toml` and use variables for Diesel config files --- Makefile | 21 +++++++++++++++------ fang/mysql_migrations/diesel.toml | 2 ++ 2 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 fang/mysql_migrations/diesel.toml diff --git a/Makefile b/Makefile index c7cc72a5..fff1d679 100644 --- a/Makefile +++ b/Makefile @@ -2,16 +2,22 @@ POSTGRES_CONTAINER=postgres POSTGRES_DB=fang POSTGRES_USER=postgres POSTGRES_PASSWORD=postgres -POSTGRES_MIGRATIONS=fang/postgres_migrations/migrations +POSTGRES_DIESEL_DIR=fang/postgres_migrations +POSTGRES_MIGRATIONS=$(POSTGRES_DIESEL_DIR)/migrations +POSTGRES_CONFIG=$(POSTGRES_DIESEL_DIR)/diesel.toml MYSQL_CONTAINER=mysql MYSQL_DB=fang MYSQL_USER=root MYSQL_PASSWORD=mysql -MYSQL_MIGRATIONS=fang/mysql_migrations/migrations +MYSQL_DIESEL_DIR=fang/mysql_migrations +MYSQL_MIGRATIONS=$(MYSQL_DIESEL_DIR)/migrations +MYSQL_CONFIG=$(MYSQL_DIESEL_DIR)/diesel.toml SQLITE_FILE=fang.db -SQLITE_MIGRATIONS=fang/sqlite_migrations/migrations +SQLITE_DIESEL_DIR=fang/sqlite_migrations +SQLITE_MIGRATIONS=$(SQLITE_DIESEL_DIR)/migrations +SQLITE_CONFIG=$(SQLITE_DIESEL_DIR)/diesel.toml BOLD = '\033[1m' END_BOLD = '\033[0m' @@ -83,19 +89,22 @@ diesel_postgres: wait_for_postgres @echo -e $(BOLD)Running Diesel migrations on Postgres database...$(END_BOLD) diesel migration run \ --database-url postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@127.0.0.1/$(POSTGRES_DB) \ - --migration-dir $(POSTGRES_MIGRATIONS) + --migration-dir $(POSTGRES_MIGRATIONS) \ + --config-file $(POSTGRES_CONFIG) diesel_mysql: wait_for_mysql @echo -e $(BOLD)Running Diesel migrations on MySQL database...$(END_BOLD) diesel migration run \ --database-url mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@127.0.0.1/$(MYSQL_DB) \ - --migration-dir $(MYSQL_MIGRATIONS) + --migration-dir $(MYSQL_MIGRATIONS) \ + --config-file $(MYSQL_CONFIG) diesel_sqlite: wait_for_sqlite @echo -e $(BOLD)Running Diesel migrations on SQLite database...$(END_BOLD) diesel migration run \ --database-url sqlite://$(SQLITE_FILE) \ - --migration-dir $(SQLITE_MIGRATIONS) + --migration-dir $(SQLITE_MIGRATIONS) \ + --config-file $(SQLITE_CONFIG) clean: $(CLEAN_TARGETS) diff --git a/fang/mysql_migrations/diesel.toml b/fang/mysql_migrations/diesel.toml new file mode 100644 index 00000000..ddc9bd5d --- /dev/null +++ b/fang/mysql_migrations/diesel.toml @@ -0,0 +1,2 @@ +[print_schema] +file = "../src/blocking/mysql_schema.rs" \ No newline at end of file From 94874d9c610ac0e12d01f7695e1d048fecc66a72 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Fri, 25 Aug 2023 00:29:45 +0200 Subject: [PATCH 24/41] add default goal --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index fff1d679..36097632 100644 --- a/Makefile +++ b/Makefile @@ -28,12 +28,16 @@ DIESEL_TARGETS = diesel_postgres diesel_mysql diesel_sqlite CLEAN_TARGETS = clean_postgres clean_mysql clean_sqlite STOP_TARGETS = stop_postgres stop_mysql stop_sqlite +.DEFAULT_GOAL = default + +default: db tests ignored stop + .PHONY: db $(DB_TARGETS) \ $(WAIT_TARGETS) \ diesel $(DIESEL_TARGETS) \ clean $(CLEAN_TARGETS) stop $(STOP_TARGETS) \ - clippy tests ignored doc + default clippy tests ignored doc .SILENT: $(DB_TARGETS) $(WAIT_TARGETS) $(DIESEL_TARGETS) $(CLEAN_TARGETS) $(STOP_TARGETS) From 65ea5d319a2e9557596d949bdb7353d621376617 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Fri, 25 Aug 2023 00:45:31 +0200 Subject: [PATCH 25/41] add `.env` recipe --- .env | 1 - .gitignore | 2 +- Makefile | 10 ++++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) delete mode 100644 .env diff --git a/.env b/.env deleted file mode 100644 index 6d0480c8..00000000 --- a/.env +++ /dev/null @@ -1 +0,0 @@ -DATABASE_URL=postgres://postgres:postgres@localhost/fang diff --git a/.gitignore b/.gitignore index 57b9c821..a64e3ea9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ **/target Cargo.lock -src/schema.rs docs/content/docs/CHANGELOG.md docs/content/docs/README.md fang.db +.env diff --git a/Makefile b/Makefile index 36097632..2f707901 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,8 @@ SQLITE_DIESEL_DIR=fang/sqlite_migrations SQLITE_MIGRATIONS=$(SQLITE_DIESEL_DIR)/migrations SQLITE_CONFIG=$(SQLITE_DIESEL_DIR)/diesel.toml +DATABASE_URL=postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@127.0.0.1/$(POSTGRES_DB) + BOLD = '\033[1m' END_BOLD = '\033[0m' @@ -41,6 +43,10 @@ default: db tests ignored stop .SILENT: $(DB_TARGETS) $(WAIT_TARGETS) $(DIESEL_TARGETS) $(CLEAN_TARGETS) $(STOP_TARGETS) +.env: + printf '' > .env + echo "DATABASE_URL=$(DATABASE_URL)" >> .env + db: $(DB_TARGETS) db_postgres: @@ -147,12 +153,12 @@ stop_sqlite: clippy: cargo clippy --verbose --all-targets --all-features -- -D warnings -tests: +tests: .env @echo -e $(BOLD)Running tests...$(END_BOLD) cargo test --all-features -- --color always --nocapture $(MAKE) clean -ignored: +ignored: .env @echo -e $(BOLD)Running ignored tests...$(END_BOLD) cargo test --all-features -- --color always --nocapture --ignored $(MAKE) clean From 2ed78ce84986073fd10fac5005734cc01885e04f Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Fri, 25 Aug 2023 00:57:01 +0200 Subject: [PATCH 26/41] add variables for URLs of DBs --- Makefile | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 2f707901..80528538 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,10 @@ SQLITE_DIESEL_DIR=fang/sqlite_migrations SQLITE_MIGRATIONS=$(SQLITE_DIESEL_DIR)/migrations SQLITE_CONFIG=$(SQLITE_DIESEL_DIR)/diesel.toml -DATABASE_URL=postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@127.0.0.1/$(POSTGRES_DB) +HOST=127.0.0.1 +DATABASE_URL=$(POSTGRES_URL) +POSTGRES_URL=postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(HOST)/$(POSTGRES_DB) +MYSQL_URL=mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@$(HOST)/$(MYSQL_DB) BOLD = '\033[1m' END_BOLD = '\033[0m' @@ -39,13 +42,18 @@ default: db tests ignored stop diesel $(DIESEL_TARGETS) \ clean $(CLEAN_TARGETS) stop $(STOP_TARGETS) \ - default clippy tests ignored doc + default clippy tests ignored doc .FORCE -.SILENT: $(DB_TARGETS) $(WAIT_TARGETS) $(DIESEL_TARGETS) $(CLEAN_TARGETS) $(STOP_TARGETS) +.SILENT: .env $(DB_TARGETS) $(WAIT_TARGETS) $(DIESEL_TARGETS) $(CLEAN_TARGETS) $(STOP_TARGETS) -.env: +.FORCE: + +.env: .FORCE printf '' > .env echo "DATABASE_URL=$(DATABASE_URL)" >> .env + echo "POSTGRES_URL=$(POSTGRES_URL)" >> .env + echo "MYSQL_URL=$(MYSQL_URL)" >> .env + echo "SQLITE_FILE=$(SQLITE_FILE)" >> .env db: $(DB_TARGETS) @@ -98,14 +106,14 @@ diesel: $(DIESEL_TARGETS) diesel_postgres: wait_for_postgres @echo -e $(BOLD)Running Diesel migrations on Postgres database...$(END_BOLD) diesel migration run \ - --database-url postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@127.0.0.1/$(POSTGRES_DB) \ + --database-url $(POSTGRES_URL) \ --migration-dir $(POSTGRES_MIGRATIONS) \ --config-file $(POSTGRES_CONFIG) diesel_mysql: wait_for_mysql @echo -e $(BOLD)Running Diesel migrations on MySQL database...$(END_BOLD) diesel migration run \ - --database-url mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@127.0.0.1/$(MYSQL_DB) \ + --database-url $(MYSQL_URL) \ --migration-dir $(MYSQL_MIGRATIONS) \ --config-file $(MYSQL_CONFIG) From af58b65e1d4ac72842c9d2a0650107c4f144b504 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Fri, 25 Aug 2023 01:13:31 +0200 Subject: [PATCH 27/41] format `Makefile` --- Makefile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 80528538..c87da19d 100644 --- a/Makefile +++ b/Makefile @@ -24,16 +24,16 @@ DATABASE_URL=$(POSTGRES_URL) POSTGRES_URL=postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(HOST)/$(POSTGRES_DB) MYSQL_URL=mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@$(HOST)/$(MYSQL_DB) -BOLD = '\033[1m' -END_BOLD = '\033[0m' +BOLD=\033[1m +END_BOLD=\033[0m -DB_TARGETS = db_postgres db_mysql db_sqlite -WAIT_TARGETS = wait_for_postgres wait_for_mysql wait_for_sqlite -DIESEL_TARGETS = diesel_postgres diesel_mysql diesel_sqlite -CLEAN_TARGETS = clean_postgres clean_mysql clean_sqlite -STOP_TARGETS = stop_postgres stop_mysql stop_sqlite +DB_TARGETS=db_postgres db_mysql db_sqlite +WAIT_TARGETS=wait_for_postgres wait_for_mysql wait_for_sqlite +DIESEL_TARGETS=diesel_postgres diesel_mysql diesel_sqlite +CLEAN_TARGETS=clean_postgres clean_mysql clean_sqlite +STOP_TARGETS=stop_postgres stop_mysql stop_sqlite -.DEFAULT_GOAL = default +.DEFAULT_GOAL=default default: db tests ignored stop From ced7be28abed98f09c52c53e323efd46df524b64 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Fri, 25 Aug 2023 01:17:58 +0200 Subject: [PATCH 28/41] fix fuck-up in last commit --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c87da19d..144a5bfa 100644 --- a/Makefile +++ b/Makefile @@ -24,8 +24,8 @@ DATABASE_URL=$(POSTGRES_URL) POSTGRES_URL=postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(HOST)/$(POSTGRES_DB) MYSQL_URL=mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@$(HOST)/$(MYSQL_DB) -BOLD=\033[1m -END_BOLD=\033[0m +BOLD='\033[1m' +END_BOLD='\033[0m' DB_TARGETS=db_postgres db_mysql db_sqlite WAIT_TARGETS=wait_for_postgres wait_for_mysql wait_for_sqlite From 18bccd705d6aa557f3597943625437e11da3adc3 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Fri, 25 Aug 2023 01:24:57 +0200 Subject: [PATCH 29/41] setup `Makefile` for parallel execution --- Makefile | 2 ++ fang/README.md | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 144a5bfa..dc9bc7e7 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,8 @@ default: db tests ignored stop .FORCE: +.NOTPARALLEL: default + .env: .FORCE printf '' > .env echo "DATABASE_URL=$(DATABASE_URL)" >> .env diff --git a/fang/README.md b/fang/README.md index 514537d0..6b2d79ac 100644 --- a/fang/README.md +++ b/fang/README.md @@ -408,27 +408,29 @@ cargo install diesel_cli --no-default-features --features "postgres sqlite mysql - Setup databases for testing. ```sh -make db +make -j db ``` - Run tests. `make db` does not need to be run in between each test cycle. ```sh -make tests +make -j tests ``` - Run dirty/long tests. ```sh -make ignored +make -j ignored ``` - Take down databases. ```sh -make stop +make -j stop ``` +The `-j` flag in the above examples is not necessary but highly recommended. + ## Authors - Ayrat Badykov (@ayrat555) From 3580bb42cfbd4132318c047cbba67d69045ac543 Mon Sep 17 00:00:00 2001 From: pxp9 <48651252+pxp9@users.noreply.github.com> Date: Fri, 25 Aug 2023 16:10:33 +0200 Subject: [PATCH 30/41] fix workflow --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 6e6d43a1..5f5bfd4f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -57,7 +57,7 @@ jobs: - name: Setup Postgres db working-directory: ./fang/postgres_migrations - run: diesel setup + run: diesel setup --database-url=postgres://postgres:postgres@localhost/fang - name: Change working dir working-directory: ./../.. From f6d90c3c81568bc826887a49a794a0ec6ac7756d Mon Sep 17 00:00:00 2001 From: pxp9 <48651252+pxp9@users.noreply.github.com> Date: Fri, 25 Aug 2023 16:21:37 +0200 Subject: [PATCH 31/41] fix workflow var --- .github/workflows/rust.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5f5bfd4f..a36adf33 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -7,6 +7,9 @@ on: # Check if it works with current dependencies (weekly on Wednesday 2:32 UTC) - cron: '32 2 * * 3' +env : + DATABASE_URL : postgres://postgres:postgres@localhost/fang + jobs: test: name: Test @@ -57,7 +60,7 @@ jobs: - name: Setup Postgres db working-directory: ./fang/postgres_migrations - run: diesel setup --database-url=postgres://postgres:postgres@localhost/fang + run: diesel setup - name: Change working dir working-directory: ./../.. From 85e68cf0bd2d2a32ffb131d60b1021c20c429871 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Tue, 29 Aug 2023 01:18:19 +0200 Subject: [PATCH 32/41] move `make` variables into `.env` file This commit makes the repository dependent on `.env` being both a valid `.env` file for `dotenvy` to use and being valid to include in the `Makefile`. --- .env | 25 +++++++++++++++++++++++++ .gitignore | 1 - Makefile | 41 ++++------------------------------------- 3 files changed, 29 insertions(+), 38 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 00000000..6b5095b9 --- /dev/null +++ b/.env @@ -0,0 +1,25 @@ +POSTGRES_CONTAINER=postgres +POSTGRES_DB=fang +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DIESEL_DIR=fang/postgres_migrations +POSTGRES_MIGRATIONS=${POSTGRES_DIESEL_DIR}/migrations +POSTGRES_CONFIG=${POSTGRES_DIESEL_DIR}/diesel.toml + +MYSQL_CONTAINER=mysql +MYSQL_DB=fang +MYSQL_USER=root +MYSQL_PASSWORD=mysql +MYSQL_DIESEL_DIR=fang/mysql_migrations +MYSQL_MIGRATIONS=${MYSQL_DIESEL_DIR}/migrations +MYSQL_CONFIG=${MYSQL_DIESEL_DIR}/diesel.toml + +SQLITE_FILE=fang.db +SQLITE_DIESEL_DIR=fang/sqlite_migrations +SQLITE_MIGRATIONS=${SQLITE_DIESEL_DIR}/migrations +SQLITE_CONFIG=${SQLITE_DIESEL_DIR}/diesel.toml + +HOST=127.0.0.1 +POSTGRES_URL=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${HOST}/${POSTGRES_DB} +MYSQL_URL=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@${HOST}/${MYSQL_DB} +DATABASE_URL=${POSTGRES_URL} diff --git a/.gitignore b/.gitignore index a64e3ea9..61005740 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,3 @@ Cargo.lock docs/content/docs/CHANGELOG.md docs/content/docs/README.md fang.db -.env diff --git a/Makefile b/Makefile index dc9bc7e7..23a8d3c9 100644 --- a/Makefile +++ b/Makefile @@ -1,28 +1,4 @@ -POSTGRES_CONTAINER=postgres -POSTGRES_DB=fang -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres -POSTGRES_DIESEL_DIR=fang/postgres_migrations -POSTGRES_MIGRATIONS=$(POSTGRES_DIESEL_DIR)/migrations -POSTGRES_CONFIG=$(POSTGRES_DIESEL_DIR)/diesel.toml - -MYSQL_CONTAINER=mysql -MYSQL_DB=fang -MYSQL_USER=root -MYSQL_PASSWORD=mysql -MYSQL_DIESEL_DIR=fang/mysql_migrations -MYSQL_MIGRATIONS=$(MYSQL_DIESEL_DIR)/migrations -MYSQL_CONFIG=$(MYSQL_DIESEL_DIR)/diesel.toml - -SQLITE_FILE=fang.db -SQLITE_DIESEL_DIR=fang/sqlite_migrations -SQLITE_MIGRATIONS=$(SQLITE_DIESEL_DIR)/migrations -SQLITE_CONFIG=$(SQLITE_DIESEL_DIR)/diesel.toml - -HOST=127.0.0.1 -DATABASE_URL=$(POSTGRES_URL) -POSTGRES_URL=postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(HOST)/$(POSTGRES_DB) -MYSQL_URL=mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@$(HOST)/$(MYSQL_DB) +include .env BOLD='\033[1m' END_BOLD='\033[0m' @@ -44,19 +20,10 @@ default: db tests ignored stop stop $(STOP_TARGETS) \ default clippy tests ignored doc .FORCE -.SILENT: .env $(DB_TARGETS) $(WAIT_TARGETS) $(DIESEL_TARGETS) $(CLEAN_TARGETS) $(STOP_TARGETS) - -.FORCE: +.SILENT: $(DB_TARGETS) $(WAIT_TARGETS) $(DIESEL_TARGETS) $(CLEAN_TARGETS) $(STOP_TARGETS) .NOTPARALLEL: default -.env: .FORCE - printf '' > .env - echo "DATABASE_URL=$(DATABASE_URL)" >> .env - echo "POSTGRES_URL=$(POSTGRES_URL)" >> .env - echo "MYSQL_URL=$(MYSQL_URL)" >> .env - echo "SQLITE_FILE=$(SQLITE_FILE)" >> .env - db: $(DB_TARGETS) db_postgres: @@ -163,12 +130,12 @@ stop_sqlite: clippy: cargo clippy --verbose --all-targets --all-features -- -D warnings -tests: .env +tests: @echo -e $(BOLD)Running tests...$(END_BOLD) cargo test --all-features -- --color always --nocapture $(MAKE) clean -ignored: .env +ignored: @echo -e $(BOLD)Running ignored tests...$(END_BOLD) cargo test --all-features -- --color always --nocapture --ignored $(MAKE) clean From 715494c61e619a56616669fc7bf84a77cdaaf760 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Tue, 29 Aug 2023 02:33:01 +0200 Subject: [PATCH 33/41] add quotation marks for `Makefile` substitution --- Makefile | 56 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index 23a8d3c9..ff4220b0 100644 --- a/Makefile +++ b/Makefile @@ -28,44 +28,44 @@ db: $(DB_TARGETS) db_postgres: @echo -e $(BOLD)Setting up Postgres database...$(END_BOLD) - docker run --rm -d --name $(POSTGRES_CONTAINER) -p 5432:5432 \ - -e POSTGRES_DB=$(POSTGRES_DB) \ - -e POSTGRES_USER=$(POSTGRES_USER) \ - -e POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) \ + docker run --rm -d --name "$(POSTGRES_CONTAINER)" -p 5432:5432 \ + -e POSTGRES_DB="$(POSTGRES_DB)" \ + -e POSTGRES_USER="$(POSTGRES_USER)" \ + -e POSTGRES_PASSWORD="$(POSTGRES_PASSWORD)" \ postgres:latest $(MAKE) diesel_postgres db_mysql: @echo -e $(BOLD)Setting up MySQL database...$(END_BOLD) - docker run --rm -d --name $(MYSQL_CONTAINER) -p 3306:3306 \ - -e MYSQL_DATABASE=$(MYSQL_DB) \ - -e MYSQL_ROOT_PASSWORD=$(MYSQL_PASSWORD) \ + docker run --rm -d --name "$(MYSQL_CONTAINER)" -p 3306:3306 \ + -e MYSQL_DATABASE="$(MYSQL_DB)" \ + -e MYSQL_ROOT_PASSWORD="$(MYSQL_PASSWORD)" \ -e TZ=UTC \ mysql:latest $(MAKE) diesel_mysql db_sqlite: @echo -e $(BOLD)Setting up SQLite database...$(END_BOLD) - sqlite3 $(SQLITE_FILE) "VACUUM;" + sqlite3 "$(SQLITE_FILE)" "VACUUM;" $(MAKE) diesel_sqlite wait_for_postgres: @echo -e $(BOLD)Waiting for Postgres server to be up and running...$(END_BOLD) - while ! docker exec $(POSTGRES_CONTAINER) psql -U $(POSTGRES_USER) --command='' 2> /dev/null; \ + while ! docker exec "$(POSTGRES_CONTAINER)" psql -U "$(POSTGRES_USER)" --command='' 2> /dev/null; \ do \ sleep 1; \ done wait_for_mysql: @echo -e $(BOLD)Waiting for MySQL server to be up and running...$(END_BOLD) - while ! docker exec $(MYSQL_CONTAINER) mysql --user=$(MYSQL_USER) --password=$(MYSQL_PASSWORD) --execute='' 2> /dev/null; \ + while ! docker exec "$(MYSQL_CONTAINER)" mysql --user="$(MYSQL_USER)" --password="$(MYSQL_PASSWORD)" --execute='' 2> /dev/null; \ do \ sleep 1; \ done wait_for_sqlite: @echo -e $(BOLD)Waiting for SQLite DB file to be created...$(END_BOLD) - while [ ! -f $(SQLITE_FILE) ]; \ + while [ ! -f "$(SQLITE_FILE)" ]; \ do \ sleep 1; \ done @@ -75,37 +75,37 @@ diesel: $(DIESEL_TARGETS) diesel_postgres: wait_for_postgres @echo -e $(BOLD)Running Diesel migrations on Postgres database...$(END_BOLD) diesel migration run \ - --database-url $(POSTGRES_URL) \ - --migration-dir $(POSTGRES_MIGRATIONS) \ - --config-file $(POSTGRES_CONFIG) + --database-url "$(POSTGRES_URL)" \ + --migration-dir "$(POSTGRES_MIGRATIONS)" \ + --config-file "$(POSTGRES_CONFIG)" diesel_mysql: wait_for_mysql @echo -e $(BOLD)Running Diesel migrations on MySQL database...$(END_BOLD) diesel migration run \ - --database-url $(MYSQL_URL) \ - --migration-dir $(MYSQL_MIGRATIONS) \ - --config-file $(MYSQL_CONFIG) + --database-url "$(MYSQL_URL)" \ + --migration-dir "$(MYSQL_MIGRATIONS)" \ + --config-file "$(MYSQL_CONFIG)" diesel_sqlite: wait_for_sqlite @echo -e $(BOLD)Running Diesel migrations on SQLite database...$(END_BOLD) diesel migration run \ - --database-url sqlite://$(SQLITE_FILE) \ - --migration-dir $(SQLITE_MIGRATIONS) \ - --config-file $(SQLITE_CONFIG) + --database-url sqlite://"$(SQLITE_FILE)" \ + --migration-dir "$(SQLITE_MIGRATIONS)" \ + --config-file "$(SQLITE_CONFIG)" clean: $(CLEAN_TARGETS) clean_postgres: wait_for_postgres @echo -e $(BOLD)Cleaning Postgres database...$(END_BOLD) - docker exec $(POSTGRES_CONTAINER) dropdb -U $(POSTGRES_USER) $(POSTGRES_DB) - docker exec $(POSTGRES_CONTAINER) psql -U $(POSTGRES_USER) --command="CREATE DATABASE $(POSTGRES_DB);" + docker exec "$(POSTGRES_CONTAINER)" dropdb -U "$(POSTGRES_USER)" "$(POSTGRES_DB)" + docker exec "$(POSTGRES_CONTAINER)" psql -U "$(POSTGRES_USER)" --command="CREATE DATABASE $(POSTGRES_DB);" $(MAKE) diesel_postgres clean_mysql: wait_for_mysql @echo -e $(BOLD)Cleaning MySQL database...$(END_BOLD) - docker exec $(MYSQL_CONTAINER) mysql \ - --user=$(MYSQL_USER) \ - --password=$(MYSQL_PASSWORD) \ + docker exec "$(MYSQL_CONTAINER)" mysql \ + --user="$(MYSQL_USER)" \ + --password="$(MYSQL_PASSWORD)" \ --execute="DROP DATABASE $(MYSQL_DB); CREATE DATABASE $(MYSQL_DB);" $(MAKE) diesel_mysql @@ -117,15 +117,15 @@ stop: $(STOP_TARGETS) stop_postgres: @echo -e $(BOLD)Stopping Postgres database...$(END_BOLD) - docker kill $(POSTGRES_CONTAINER) + docker kill "$(POSTGRES_CONTAINER)" stop_mysql: @echo -e $(BOLD)Stopping MySQL database...$(END_BOLD) - docker kill $(MYSQL_CONTAINER) + docker kill "$(MYSQL_CONTAINER)" stop_sqlite: @echo -e $(BOLD)Stopping SQLite database...$(END_BOLD) - rm $(SQLITE_FILE) + rm "$(SQLITE_FILE)" clippy: cargo clippy --verbose --all-targets --all-features -- -D warnings From bd0faa43a8b21730eb2be04c6b5372f05df26ce6 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Tue, 29 Aug 2023 02:51:01 +0200 Subject: [PATCH 34/41] solve race condition in `clean_sqlite` --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ff4220b0..188a538b 100644 --- a/Makefile +++ b/Makefile @@ -111,7 +111,8 @@ clean_mysql: wait_for_mysql clean_sqlite: wait_for_sqlite @echo -e $(BOLD)Cleaning SQLite database...$(END_BOLD) - $(MAKE) stop_sqlite db_sqlite + $(MAKE) stop_sqlite + $(MAKE) db_sqlite stop: $(STOP_TARGETS) From 078dafb0cad91bdb6a1069278b2d3e727b2307ec Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Tue, 29 Aug 2023 02:52:41 +0200 Subject: [PATCH 35/41] fix `README.md` --- fang/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fang/README.md b/fang/README.md index 6b2d79ac..59acda4d 100644 --- a/fang/README.md +++ b/fang/README.md @@ -405,6 +405,8 @@ cargo install diesel_cli --no-default-features --features "postgres sqlite mysql - Install docker on your machine. +- Install SQLite 3 on your machine. + - Setup databases for testing. ```sh From 5051eeb41de5fa133ac06bab4928759fa6c7fff2 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Tue, 29 Aug 2023 14:29:36 +0200 Subject: [PATCH 36/41] stupid problems require stupid solutions --- Makefile | 53 ++++++++++++++++------------------------------------- 1 file changed, 16 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index 188a538b..9df39f23 100644 --- a/Makefile +++ b/Makefile @@ -49,59 +49,38 @@ db_sqlite: sqlite3 "$(SQLITE_FILE)" "VACUUM;" $(MAKE) diesel_sqlite -wait_for_postgres: - @echo -e $(BOLD)Waiting for Postgres server to be up and running...$(END_BOLD) - while ! docker exec "$(POSTGRES_CONTAINER)" psql -U "$(POSTGRES_USER)" --command='' 2> /dev/null; \ - do \ - sleep 1; \ - done - -wait_for_mysql: - @echo -e $(BOLD)Waiting for MySQL server to be up and running...$(END_BOLD) - while ! docker exec "$(MYSQL_CONTAINER)" mysql --user="$(MYSQL_USER)" --password="$(MYSQL_PASSWORD)" --execute='' 2> /dev/null; \ - do \ - sleep 1; \ - done - -wait_for_sqlite: - @echo -e $(BOLD)Waiting for SQLite DB file to be created...$(END_BOLD) - while [ ! -f "$(SQLITE_FILE)" ]; \ - do \ - sleep 1; \ - done - diesel: $(DIESEL_TARGETS) -diesel_postgres: wait_for_postgres +diesel_postgres: @echo -e $(BOLD)Running Diesel migrations on Postgres database...$(END_BOLD) - diesel migration run \ - --database-url "$(POSTGRES_URL)" \ - --migration-dir "$(POSTGRES_MIGRATIONS)" \ - --config-file "$(POSTGRES_CONFIG)" + while ! diesel migration run --database-url "$(POSTGRES_URL)" --migration-dir "$(POSTGRES_MIGRATIONS)" --config-file "$(POSTGRES_CONFIG)" 2> /dev/null; \ + do \ + :; \ + done diesel_mysql: wait_for_mysql @echo -e $(BOLD)Running Diesel migrations on MySQL database...$(END_BOLD) - diesel migration run \ - --database-url "$(MYSQL_URL)" \ - --migration-dir "$(MYSQL_MIGRATIONS)" \ - --config-file "$(MYSQL_CONFIG)" + while ! diesel migration run --database-url "$(MYSQL_URL)" --migration-dir "$(MYSQL_MIGRATIONS)" --config-file "$(MYSQL_CONFIG)" 2> /dev/null; \ + do \ + :; \ + done diesel_sqlite: wait_for_sqlite @echo -e $(BOLD)Running Diesel migrations on SQLite database...$(END_BOLD) - diesel migration run \ - --database-url sqlite://"$(SQLITE_FILE)" \ - --migration-dir "$(SQLITE_MIGRATIONS)" \ - --config-file "$(SQLITE_CONFIG)" + while ! diesel migration run --database-url sqlite://"$(SQLITE_FILE)" --migration-dir "$(SQLITE_MIGRATIONS)" --config-file "$(SQLITE_CONFIG)" 2> /dev/null; \ + do \ + :; \ + done clean: $(CLEAN_TARGETS) -clean_postgres: wait_for_postgres +clean_postgres: @echo -e $(BOLD)Cleaning Postgres database...$(END_BOLD) docker exec "$(POSTGRES_CONTAINER)" dropdb -U "$(POSTGRES_USER)" "$(POSTGRES_DB)" docker exec "$(POSTGRES_CONTAINER)" psql -U "$(POSTGRES_USER)" --command="CREATE DATABASE $(POSTGRES_DB);" $(MAKE) diesel_postgres -clean_mysql: wait_for_mysql +clean_mysql: @echo -e $(BOLD)Cleaning MySQL database...$(END_BOLD) docker exec "$(MYSQL_CONTAINER)" mysql \ --user="$(MYSQL_USER)" \ @@ -109,7 +88,7 @@ clean_mysql: wait_for_mysql --execute="DROP DATABASE $(MYSQL_DB); CREATE DATABASE $(MYSQL_DB);" $(MAKE) diesel_mysql -clean_sqlite: wait_for_sqlite +clean_sqlite: @echo -e $(BOLD)Cleaning SQLite database...$(END_BOLD) $(MAKE) stop_sqlite $(MAKE) db_sqlite From 7ad3b070946b050d033bfdc6ca6e6911ea694373 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Tue, 29 Aug 2023 15:18:57 +0200 Subject: [PATCH 37/41] remove unnecessary clean --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 9df39f23..90a9f2e4 100644 --- a/Makefile +++ b/Makefile @@ -113,7 +113,6 @@ clippy: tests: @echo -e $(BOLD)Running tests...$(END_BOLD) cargo test --all-features -- --color always --nocapture - $(MAKE) clean ignored: @echo -e $(BOLD)Running ignored tests...$(END_BOLD) From abfd17dc312d2e62029881803279c36fee6dfd21 Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Tue, 29 Aug 2023 15:46:50 +0200 Subject: [PATCH 38/41] add `sleep`s --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 90a9f2e4..5bf650d9 100644 --- a/Makefile +++ b/Makefile @@ -55,21 +55,21 @@ diesel_postgres: @echo -e $(BOLD)Running Diesel migrations on Postgres database...$(END_BOLD) while ! diesel migration run --database-url "$(POSTGRES_URL)" --migration-dir "$(POSTGRES_MIGRATIONS)" --config-file "$(POSTGRES_CONFIG)" 2> /dev/null; \ do \ - :; \ + sleep 1; \ done -diesel_mysql: wait_for_mysql +diesel_mysql: @echo -e $(BOLD)Running Diesel migrations on MySQL database...$(END_BOLD) while ! diesel migration run --database-url "$(MYSQL_URL)" --migration-dir "$(MYSQL_MIGRATIONS)" --config-file "$(MYSQL_CONFIG)" 2> /dev/null; \ do \ - :; \ + sleep 1; \ done -diesel_sqlite: wait_for_sqlite +diesel_sqlite: @echo -e $(BOLD)Running Diesel migrations on SQLite database...$(END_BOLD) while ! diesel migration run --database-url sqlite://"$(SQLITE_FILE)" --migration-dir "$(SQLITE_MIGRATIONS)" --config-file "$(SQLITE_CONFIG)" 2> /dev/null; \ do \ - :; \ + sleep 1; \ done clean: $(CLEAN_TARGETS) From fc1b0c7463074e4eb90c452eea65709c72d843fe Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Tue, 29 Aug 2023 23:10:51 +0200 Subject: [PATCH 39/41] add docker image versions as a `.env` and `Makefile` variable --- .env | 2 ++ Makefile | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 6b5095b9..95fd800e 100644 --- a/.env +++ b/.env @@ -1,4 +1,5 @@ POSTGRES_CONTAINER=postgres +POSTGRES_VERSION=latest POSTGRES_DB=fang POSTGRES_USER=postgres POSTGRES_PASSWORD=postgres @@ -7,6 +8,7 @@ POSTGRES_MIGRATIONS=${POSTGRES_DIESEL_DIR}/migrations POSTGRES_CONFIG=${POSTGRES_DIESEL_DIR}/diesel.toml MYSQL_CONTAINER=mysql +MYSQL_VERSION=latest MYSQL_DB=fang MYSQL_USER=root MYSQL_PASSWORD=mysql diff --git a/Makefile b/Makefile index 5bf650d9..fc340091 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ db_postgres: -e POSTGRES_DB="$(POSTGRES_DB)" \ -e POSTGRES_USER="$(POSTGRES_USER)" \ -e POSTGRES_PASSWORD="$(POSTGRES_PASSWORD)" \ - postgres:latest + postgres:"$(POSTGRES_VERSION)" $(MAKE) diesel_postgres db_mysql: @@ -41,7 +41,7 @@ db_mysql: -e MYSQL_DATABASE="$(MYSQL_DB)" \ -e MYSQL_ROOT_PASSWORD="$(MYSQL_PASSWORD)" \ -e TZ=UTC \ - mysql:latest + mysql:"$(MYSQL_VERSION)" $(MAKE) diesel_mysql db_sqlite: From 882dca8f8df373a7fc49cb5c5757dbcd9c2df286 Mon Sep 17 00:00:00 2001 From: pxp9 <48651252+pxp9@users.noreply.github.com> Date: Wed, 30 Aug 2023 09:45:43 +0200 Subject: [PATCH 40/41] latest databases but without latest --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 95fd800e..43ea1787 100644 --- a/.env +++ b/.env @@ -1,5 +1,5 @@ POSTGRES_CONTAINER=postgres -POSTGRES_VERSION=latest +POSTGRES_VERSION=15.4 POSTGRES_DB=fang POSTGRES_USER=postgres POSTGRES_PASSWORD=postgres @@ -8,7 +8,7 @@ POSTGRES_MIGRATIONS=${POSTGRES_DIESEL_DIR}/migrations POSTGRES_CONFIG=${POSTGRES_DIESEL_DIR}/diesel.toml MYSQL_CONTAINER=mysql -MYSQL_VERSION=latest +MYSQL_VERSION=8.1 MYSQL_DB=fang MYSQL_USER=root MYSQL_PASSWORD=mysql From 200b9e88f7af0e1a7dff6c213adb3c5f1639cf3d Mon Sep 17 00:00:00 2001 From: Dopplerian Date: Wed, 30 Aug 2023 21:23:52 +0200 Subject: [PATCH 41/41] small clarification in `README.md` --- fang/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fang/README.md b/fang/README.md index 59acda4d..d4b1e092 100644 --- a/fang/README.md +++ b/fang/README.md @@ -431,7 +431,7 @@ make -j ignored make -j stop ``` -The `-j` flag in the above examples is not necessary but highly recommended. +The `-j` flag in the above examples enables parallelism for `make`, is not necessary but highly recommended. ## Authors