diff --git a/.circleci/config.yml b/.circleci/config.yml index 65bdd894..230824ac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,23 +1,15 @@ -version: 2 +version: 2.1 jobs: - build: + + integration-postgres: docker: - image: circleci/python:3.6.3-stretch - image: circleci/postgres:9.6.5-alpine-ram steps: - checkout - - - run: - run: setup_creds - command: | - echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json - - - restore_cache: - key: deps2-{{ .Branch }} - - run: name: "Run Tests - Postgres" environment: @@ -27,22 +19,58 @@ jobs: POSTGRES_TEST_PORT: 5432 POSTGRES_TEST_DBNAME: circle_test command: ./run_test.sh postgres + - store_artifacts: + path: ./logs + integration-redshift: + docker: + - image: circleci/python:3.6.3-stretch + steps: + - checkout - run: name: "Run Tests - Redshift" command: ./run_test.sh redshift + - store_artifacts: + path: ./logs + integration-snowflake: + docker: + - image: circleci/python:3.6.3-stretch + steps: + - checkout - run: name: "Run Tests - Snowflake" command: ./run_test.sh snowflake - + - store_artifacts: + path: ./logs + + integration-bigquery: + environment: + BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" + docker: + - image: circleci/python:3.6.3-stretch + steps: + - checkout + - run: + name: "Set up credentials" + command: echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json - run: name: "Run Tests - BigQuery" - environment: - BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" command: ./run_test.sh bigquery + - store_artifacts: + path: ./logs - - save_cache: - key: deps1-{{ .Branch }} - paths: - - "venv" +workflows: + version: 2 + test-all: + jobs: + - integration-postgres + - integration-redshift: + requires: + - integration-postgres + - integration-snowflake: + requires: + - integration-postgres + - integration-bigquery: + requires: + - integration-postgres diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 7a026e2c..d3a61793 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -17,7 +17,7 @@ integration_tests: port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}" dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}" schema: dbt_utils_integration_tests_postgres - threads: 1 + threads: 5 redshift: type: redshift @@ -27,7 +27,7 @@ integration_tests: dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}" port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}" schema: dbt_utils_integration_tests_redshift - threads: 1 + threads: 5 bigquery: type: bigquery @@ -35,7 +35,7 @@ integration_tests: keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}" project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}" schema: dbt_utils_integration_tests_bigquery - threads: 1 + threads: 10 snowflake: type: snowflake @@ -46,4 +46,4 @@ integration_tests: database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}" warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}" schema: dbt_utils_integration_tests_snowflake - threads: 1 + threads: 10 diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index 9b80b025..5ca4da73 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -142,7 +142,13 @@ from {{tmp_relation.include(schema=False)}} ); {%- endcall %} - {%- set rows_inserted = (load_result('main-' ~ i)['status'].split(" "))[2] | int -%} + {% set result = load_result('main-' ~ i) %} + {% if 'response' in result.keys() %} {# added in v0.19.0 #} + {% set rows_inserted = result['response']['rows_affected'] %} + {% else %} {# older versions #} + {% set rows_inserted = result['status'].split(" ")[2] | int %} + {% endif %} + {%- set sum_rows_inserted = loop_vars['sum_rows_inserted'] + rows_inserted -%} {%- if loop_vars.update({'sum_rows_inserted': sum_rows_inserted}) %} {% endif -%} @@ -165,7 +171,7 @@ {%- set status_string = "INSERT " ~ loop_vars['sum_rows_inserted'] -%} - {% call noop_statement(name='main', status=status_string) -%} + {% call noop_statement('main', status_string) -%} -- no-op {%- endcall %} diff --git a/run_test.sh b/run_test.sh index 3614924e..80089ebf 100755 --- a/run_test.sh +++ b/run_test.sh @@ -24,5 +24,8 @@ if [[ ! -z $3 ]]; then _seeds="--select $3 --full-refresh"; fi dbt deps --target $1 dbt seed --target $1 $_seeds -dbt run --target $1 $_models -dbt test --target $1 $_models +if [ $1 == 'redshift' ]; then + dbt run -x -m test_insert_by_period --full-refresh --target redshift +fi +dbt run -x --target $1 $_models +dbt test -x --target $1 $_models