From 2054f08908495f92703de9225bbe1a72cd2f1185 Mon Sep 17 00:00:00 2001 From: Razvan Vacaru Date: Fri, 17 Sep 2021 16:53:44 +0200 Subject: [PATCH 1/6] Fix setup_db.sh by waiting for pg_isready success return. Fixes #3876 --- test/setup_db.sh | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/test/setup_db.sh b/test/setup_db.sh index b52c3c487d8..e3eaf54ebeb 100644 --- a/test/setup_db.sh +++ b/test/setup_db.sh @@ -1,6 +1,5 @@ #!/bin/bash set -x - env | grep '^PG' # If you want to run this script for your own postgresql (run with @@ -30,16 +29,9 @@ if [[ -n $CIRCLECI ]]; then connect_circle fi -createdb dbt -psql -c "CREATE ROLE root WITH PASSWORD 'password';" -psql -c "ALTER ROLE root WITH LOGIN;" -psql -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;" - -psql -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;" -psql -c "ALTER ROLE noaccess WITH LOGIN;" -psql -c "GRANT CONNECT ON DATABASE dbt TO noaccess;" - -psql -c 'CREATE DATABASE "dbtMixedCase";' -psql -c 'GRANT CREATE, CONNECT ON DATABASE "dbtMixedCase" TO root WITH GRANT OPTION;' +until pg_isready -h ${PGHOST} -p ${PGPORT} -U ${PGUSER}; do + echo "Waiting for postgres to be ready..." + sleep 2; +done; set +x From 05ea00e90ae76a863e0096d1eb98bb2fa6649840 Mon Sep 17 00:00:00 2001 From: Razvan Vacaru Date: Fri, 17 Sep 2021 17:19:02 +0200 Subject: [PATCH 2/6] restored noaccess and dbtMixedCase creation and updated changelog and contributing md files --- CHANGELOG.md | 2 ++ CONTRIBUTING.md | 2 -- test/setup_db.sh | 7 +++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07a31529799..61c67e838b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,7 @@ Contributors: - Introduce a macro, `get_where_subquery`, for tests that use `where` config. Alias filtering subquery as `dbt_subquery` instead of resource identifier ([#3857](https://github.com/dbt-labs/dbt/issues/3857), [#3859](https://github.com/dbt-labs/dbt/issues/3859)) - Use group by column_name in accepted_values test for compatibility with most database engines ([#3905](https://github.com/dbt-labs/dbt/issues/3905), [#3906](https://github.com/dbt-labs/dbt/pull/3906)) - Separated table vs view configuration for BigQuery since some configuration is not possible to set for tables vs views. ([#3682](https://github.com/dbt-labs/dbt/issues/3682), [#3691](https://github.com/dbt-labs/dbt/issues/3682)) +- Wait for postgres docker container to be ready in `setup_db.sh`. ([#3876](https://github.com/dbt-labs/dbt/pull/3908)) ### Under the hood @@ -102,6 +103,7 @@ Contributors: - [@Kayrnt](https://github.com/Kayrnt) ([3707](https://github.com/dbt-labs/dbt/pull/3707)) - [@TeddyCr](https://github.com/TeddyCr) ([#3448](https://github.com/dbt-labs/dbt/pull/3865)) - [@sdebruyn](https://github.com/sdebruyn) ([#3906](https://github.com/dbt-labs/dbt/pull/3906)) +- [@rvacaru](https://github.com/rvacaru) ([#3908](https://github.com/dbt-labs/dbt/pull/3908)) ## dbt 0.21.0b2 (August 19, 2021) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1df6eef182f..23219dbaa5d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -174,8 +174,6 @@ docker-compose up -d database PGHOST=localhost PGUSER=root PGPASSWORD=password PGDATABASE=postgres bash test/setup_db.sh ``` -Note that you may need to run the previous command twice as it does not currently wait for the database to be running before attempting to run commands against it. This will be fixed with [#3876](https://github.com/dbt-labs/dbt/issues/3876). - `dbt` uses test credentials specified in a `test.env` file in the root of the repository for non-Postgres databases. This `test.env` file is git-ignored, but please be _extra_ careful to never check in credentials or other sensitive information when developing against `dbt`. To create your `test.env` file, copy the provided sample file, then supply your relevant credentials. This step is only required to use non-Postgres databases. ``` diff --git a/test/setup_db.sh b/test/setup_db.sh index e3eaf54ebeb..8e9cfa009ca 100644 --- a/test/setup_db.sh +++ b/test/setup_db.sh @@ -34,4 +34,11 @@ until pg_isready -h ${PGHOST} -p ${PGPORT} -U ${PGUSER}; do sleep 2; done; +psql -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;" +psql -c "ALTER ROLE noaccess WITH LOGIN;" +psql -c "GRANT CONNECT ON DATABASE dbt TO noaccess;" + +psql -c 'CREATE DATABASE "dbtMixedCase";' +psql -c 'GRANT CREATE, CONNECT ON DATABASE "dbtMixedCase" TO root WITH GRANT OPTION;' + set +x From 839ed70be8364a4a24f18ef258a0fda8130847c1 Mon Sep 17 00:00:00 2001 From: Razvan Vacaru Date: Fri, 17 Sep 2021 17:31:00 +0200 Subject: [PATCH 3/6] restored root auth commands --- test/setup_db.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/setup_db.sh b/test/setup_db.sh index 8e9cfa009ca..5d948ac92e0 100644 --- a/test/setup_db.sh +++ b/test/setup_db.sh @@ -34,6 +34,8 @@ until pg_isready -h ${PGHOST} -p ${PGPORT} -U ${PGUSER}; do sleep 2; done; +psql -c "CREATE ROLE root WITH PASSWORD 'password';" +psql -c "ALTER ROLE root WITH LOGIN;" psql -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;" psql -c "ALTER ROLE noaccess WITH LOGIN;" psql -c "GRANT CONNECT ON DATABASE dbt TO noaccess;" From d0db8e9e680d6d2a9efd0622b39b091c520b6a6e Mon Sep 17 00:00:00 2001 From: Razvan Vacaru Date: Fri, 17 Sep 2021 17:36:32 +0200 Subject: [PATCH 4/6] restored creation of dbt schema, aparently this is needed even if docker compose also creates it... --- test/setup_db.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/setup_db.sh b/test/setup_db.sh index 5d948ac92e0..acf386a99a9 100644 --- a/test/setup_db.sh +++ b/test/setup_db.sh @@ -34,8 +34,11 @@ until pg_isready -h ${PGHOST} -p ${PGPORT} -U ${PGUSER}; do sleep 2; done; +createdb dbt psql -c "CREATE ROLE root WITH PASSWORD 'password';" psql -c "ALTER ROLE root WITH LOGIN;" +psql -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;" + psql -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;" psql -c "ALTER ROLE noaccess WITH LOGIN;" psql -c "GRANT CONNECT ON DATABASE dbt TO noaccess;" From 25bfc6bca6e72dffed907f2b5eef3e93d073af3f Mon Sep 17 00:00:00 2001 From: Razvan Vacaru Date: Tue, 12 Oct 2021 09:50:29 +0200 Subject: [PATCH 5/6] pr comments: avoid infinite loop and quote variables --- test/setup_db.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/setup_db.sh b/test/setup_db.sh index acf386a99a9..b8b869e143b 100644 --- a/test/setup_db.sh +++ b/test/setup_db.sh @@ -29,7 +29,11 @@ if [[ -n $CIRCLECI ]]; then connect_circle fi -until pg_isready -h ${PGHOST} -p ${PGPORT} -U ${PGUSER}; do +for i in {1..10}; do + if $(pg_isready -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}") ; then + break + fi + echo "Waiting for postgres to be ready..." sleep 2; done; From 84672893b664fc98d8eebb5256d02be553a6b40e Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Wed, 20 Oct 2021 18:01:21 +0200 Subject: [PATCH 6/6] Update changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c662b1d91d5..8a97675f412 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,12 @@ - Fix intermittent errors in partial parsing tests ([#4060](https://github.com/dbt-labs/dbt-core/issues/4060), [#4068](https://github.com/dbt-labs/dbt-core/pull/4068)) - Make finding disabled nodes more consistent ([#4069](https://github.com/dbt-labs/dbt-core/issues/4069), [#4073](https://github.com/dbt-labas/dbt-core/pull/4073)) - Remove connection from `render_with_context` during parsing, thereby removing misleading log message ([#3137](https://github.com/dbt-labs/dbt-core/issues/3137), [#4062](https://github.com/dbt-labas/dbt-core/pull/4062)) +- Wait for postgres docker container to be ready in `setup_db.sh`. ([#3876](https://github.com/dbt-labs/dbt-core/issues/3876), [#3908](https://github.com/dbt-labs/dbt-core/pull/3908)) Contributors: - [@sungchun12](https://github.com/sungchun12) ([#4017](https://github.com/dbt-labs/dbt/pull/4017)) - [@matt-winkler](https://github.com/matt-winkler) ([#4017](https://github.com/dbt-labs/dbt/pull/4017)) +- [@rvacaru](https://github.com/rvacaru) ([#3908](https://github.com/dbt-labs/dbt/pull/3908)) ## dbt-core 1.0.0b1 (October 11, 2021) @@ -110,7 +112,6 @@ Contributors: - Introduce a macro, `get_where_subquery`, for tests that use `where` config. Alias filtering subquery as `dbt_subquery` instead of resource identifier ([#3857](https://github.com/dbt-labs/dbt-core/issues/3857), [#3859](https://github.com/dbt-labs/dbt-core/issues/3859)) - Use group by column_name in accepted_values test for compatibility with most database engines ([#3905](https://github.com/dbt-labs/dbt-core/issues/3905), [#3906](https://github.com/dbt-labs/dbt-core/pull/3906)) - Separated table vs view configuration for BigQuery since some configuration is not possible to set for tables vs views. ([#3682](https://github.com/dbt-labs/dbt-core/issues/3682), [#3691](https://github.com/dbt-labs/dbt-core/issues/3682)) -- Wait for postgres docker container to be ready in `setup_db.sh`. ([#3876](https://github.com/dbt-labs/dbt/pull/3908)) ### Under the hood @@ -136,7 +137,6 @@ Contributors: - [@Kayrnt](https://github.com/Kayrnt) ([3707](https://github.com/dbt-labs/dbt-core/pull/3707)) - [@TeddyCr](https://github.com/TeddyCr) ([#3448](https://github.com/dbt-labs/dbt-core/pull/3865)) - [@sdebruyn](https://github.com/sdebruyn) ([#3906](https://github.com/dbt-labs/dbt-core/pull/3906)) -- [@rvacaru](https://github.com/rvacaru) ([#3908](https://github.com/dbt-labs/dbt/pull/3908)) ## dbt 0.21.0b2 (August 19, 2021)