Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Switch to CircleCI workflow #316

Merged
merged 6 commits into from
Jan 6, 2021
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Use workflows, more threads, fail fast, pg first
  • Loading branch information
jtcohen6 committed Jan 6, 2021
commit d91c386ca02a1e507146f037e42ac73116667fdd
64 changes: 46 additions & 18 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -1,23 +1,15 @@

version: 2
version: 2.1

jobs:
build:

integration-postgres:
docker:
- image: circleci/python:3.6.3-stretch
- image: circleci/postgres:9.6.5-alpine-ram

steps:
- checkout

- run:
run: setup_creds
command: |
echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json

- restore_cache:
key: deps2-{{ .Branch }}

- run:
name: "Run Tests - Postgres"
environment:
@@ -27,22 +19,58 @@ jobs:
POSTGRES_TEST_PORT: 5432
POSTGRES_TEST_DBNAME: circle_test
command: ./run_test.sh postgres
- store_artifacts:
path: ./logs

integration-redshift:
docker:
- image: circleci/python:3.6.3-stretch
steps:
- checkout
- run:
name: "Run Tests - Redshift"
command: ./run_test.sh redshift
- store_artifacts:
path: ./logs

integration-snowflake:
docker:
- image: circleci/python:3.6.3-stretch
steps:
- checkout
- run:
name: "Run Tests - Snowflake"
command: ./run_test.sh snowflake

- store_artifacts:
path: ./logs

integration-bigquery:
environment:
BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
docker:
- image: circleci/python:3.6.3-stretch
steps:
- checkout
- run:
name: "Set up credentials"
command: echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json
- run:
name: "Run Tests - BigQuery"
environment:
BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
command: ./run_test.sh bigquery
- store_artifacts:
path: ./logs

- save_cache:
key: deps1-{{ .Branch }}
paths:
- "venv"
workflows:
version: 2
test-all:
jobs:
- integration-postgres
- integration-redshift:
requires:
- integration-postgres
- integration-snowflake:
requires:
- integration-postgres
- integration-bigquery:
requires:
- integration-postgres
8 changes: 4 additions & 4 deletions integration_tests/ci/sample.profiles.yml
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ integration_tests:
port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}"
dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}"
schema: dbt_utils_integration_tests_postgres
threads: 1
threads: 5

redshift:
type: redshift
@@ -27,15 +27,15 @@ integration_tests:
dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}"
port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}"
schema: dbt_utils_integration_tests_redshift
threads: 1
threads: 5

bigquery:
type: bigquery
method: service-account
keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}"
project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}"
schema: dbt_utils_integration_tests_bigquery
threads: 1
threads: 10

snowflake:
type: snowflake
@@ -46,4 +46,4 @@ integration_tests:
database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}"
warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}"
schema: dbt_utils_integration_tests_snowflake
threads: 1
threads: 10
4 changes: 2 additions & 2 deletions run_test.sh
Original file line number Diff line number Diff line change
@@ -24,5 +24,5 @@ if [[ ! -z $3 ]]; then _seeds="--select $3 --full-refresh"; fi

dbt deps --target $1
dbt seed --target $1 $_seeds
dbt run --target $1 $_models
dbt test --target $1 $_models
dbt run -x --target $1 $_models
dbt test -x --target $1 $_models