From f499a22de17c08c95edc4ca69414887c9298a35e Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 11 Sep 2018 14:30:19 -0400 Subject: [PATCH 1/4] Make union work in bigquery, fix tests --- .circleci/config.yml | 10 +++++++++- integration_tests/Makefile | 16 ++++++++-------- .../data/sql/data_union_expected.csv | 6 ++++++ .../data/sql/data_union_table_1.csv | 4 ++++ .../data/sql/data_union_table_2.csv | 3 +++ integration_tests/models/sql/schema.yml | 5 +++++ .../models/sql/test_get_column_values.sql | 1 + integration_tests/models/sql/test_union.sql | 6 ++++++ macros/cross_db_utils/literal.sql | 8 ++++++++ macros/sql/union.sql | 4 ++-- 10 files changed, 52 insertions(+), 11 deletions(-) create mode 100644 integration_tests/data/sql/data_union_expected.csv create mode 100644 integration_tests/data/sql/data_union_table_1.csv create mode 100644 integration_tests/data/sql/data_union_table_2.csv create mode 100644 integration_tests/models/sql/test_union.sql create mode 100644 macros/cross_db_utils/literal.sql diff --git a/.circleci/config.yml b/.circleci/config.yml index c9bd5579d32..c3052a832fb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,15 @@ jobs: command: | python3 -m venv venv . venv/bin/activate - pip install dbt + + # Temporary fix for 0.11.0 bug -- 0.11.1 is yet unreleased + # TODO : Make this use latest from pip once 0.11.1 is out + git clone https://github.com/fishtown-analytics/dbt.git + cd dbt + git checkout dev/lucretia-mott + pip install -e . + cd .. + mkdir -p ~/.dbt cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml diff --git a/integration_tests/Makefile b/integration_tests/Makefile index 243f960fade..ffa967cffe9 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -1,22 +1,22 @@ test-postgres: - dbt seed --target postgres - dbt run --target postgres + dbt seed --target postgres --full-refresh + dbt run --target postgres --full-refresh dbt test --target postgres test-redshift: - dbt seed --target redshift - dbt run --target redshift + dbt seed --target redshift --full-refresh + dbt run --target redshift --full-refresh dbt test --target redshift test-snowflake: - dbt seed --target snowflake - dbt run --target snowflake + dbt seed --target snowflake --full-refresh + dbt run --target snowflake --full-refresh dbt test --target snowflake test-bigquery: - dbt seed --target bigquery - dbt run --target bigquery + dbt seed --target bigquery --full-refresh + dbt run --target bigquery --full-refresh dbt test --target bigquery test-all: test-postgres test-redshift test-snowflake test-bigquery diff --git a/integration_tests/data/sql/data_union_expected.csv b/integration_tests/data/sql/data_union_expected.csv new file mode 100644 index 00000000000..59ebe14e206 --- /dev/null +++ b/integration_tests/data/sql/data_union_expected.csv @@ -0,0 +1,6 @@ +_dbt_source_table,id,name,favorite_color +"""integration_tests"".""data_union_table_1""",1,"drew", +"""integration_tests"".""data_union_table_1""",2,"bob", +"""integration_tests"".""data_union_table_1""",3,"alice", +"""integration_tests"".""data_union_table_2""",1,,"green" +"""integration_tests"".""data_union_table_2""",2,,"pink" diff --git a/integration_tests/data/sql/data_union_table_1.csv b/integration_tests/data/sql/data_union_table_1.csv new file mode 100644 index 00000000000..3f33321ceb2 --- /dev/null +++ b/integration_tests/data/sql/data_union_table_1.csv @@ -0,0 +1,4 @@ +id,name +1,drew +2,bob +3,alice diff --git a/integration_tests/data/sql/data_union_table_2.csv b/integration_tests/data/sql/data_union_table_2.csv new file mode 100644 index 00000000000..6f2fe9c3f56 --- /dev/null +++ b/integration_tests/data/sql/data_union_table_2.csv @@ -0,0 +1,3 @@ +id,favorite_color +1,green +2,pink diff --git a/integration_tests/models/sql/schema.yml b/integration_tests/models/sql/schema.yml index ec89f16fcd4..fa5dc7337a0 100644 --- a/integration_tests/models/sql/schema.yml +++ b/integration_tests/models/sql/schema.yml @@ -56,3 +56,8 @@ test_surrogate_key: constraints: assert_equal: - {actual: actual, expected: expected} + +test_union: + constraints: + dbt_utils.equality: + - ref('data_union_expected') diff --git a/integration_tests/models/sql/test_get_column_values.sql b/integration_tests/models/sql/test_get_column_values.sql index 47f45bb9005..25f3cbc6eda 100644 --- a/integration_tests/models/sql/test_get_column_values.sql +++ b/integration_tests/models/sql/test_get_column_values.sql @@ -3,6 +3,7 @@ select + {% set columns = columns if columns is iterable else [] %} {% for column in columns -%} sum(case when field = '{{ column }}' then 1 else 0 end) as count_{{ column }} diff --git a/integration_tests/models/sql/test_union.sql b/integration_tests/models/sql/test_union.sql new file mode 100644 index 00000000000..28207d05800 --- /dev/null +++ b/integration_tests/models/sql/test_union.sql @@ -0,0 +1,6 @@ + +{{ dbt_utils.union_tables([ + ref('data_union_table_1'), + ref('data_union_table_2')] +) }} + diff --git a/macros/cross_db_utils/literal.sql b/macros/cross_db_utils/literal.sql new file mode 100644 index 00000000000..c3f3e4eeba9 --- /dev/null +++ b/macros/cross_db_utils/literal.sql @@ -0,0 +1,8 @@ + +{% macro string_literal(value) %} + {{ adapter_macro('dbt_utils.string_literal', value) }} +{% endmacro %} + +{% macro default__string_literal(value) -%} + '{{ value }}' +{%- endmacro %} diff --git a/macros/sql/union.sql b/macros/sql/union.sql index 59eb022d536..d79aafd7077 100644 --- a/macros/sql/union.sql +++ b/macros/sql/union.sql @@ -56,7 +56,7 @@ ( select - '{{ table }}'::text as _dbt_source_table, + {{ dbt_utils.safe_cast(dbt_utils.string_literal(table), dbt_utils.type_string()) }} as _dbt_source_table, {% for col_name in ordered_column_names -%} @@ -64,7 +64,7 @@ {%- set col_type = column_override.get(col.column, col.data_type) %} {%- set col_name = adapter.quote(col_name) if col_name in table_columns[table] else 'null' %} - {{ col_name }}::{{ col_type }} as {{ col.quoted }} {% if not loop.last %},{% endif %} + {{ dbt_utils.safe_cast(col_name, col_type) }} as {{ col.quoted }} {% if not loop.last %},{% endif %} {%- endfor %} from {{ table }} From e5513c828a545ca7f3afcd847d4f46aeeeacaf04 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Fri, 14 Sep 2018 15:35:53 -0400 Subject: [PATCH 2/4] Use full-refresh for tests --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c3052a832fb..afb8e234802 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,8 +41,8 @@ jobs: . venv/bin/activate cd integration_tests dbt deps - dbt seed - dbt run + dbt seed --full-refresh + dbt run --full-refresh dbt test - save_cache: From 6a3e80ce0418e24d0802f51e7a8afababef03005 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Fri, 14 Sep 2018 15:52:50 -0400 Subject: [PATCH 3/4] fix for different dev/ci schema names --- integration_tests/data/sql/data_union_expected.csv | 12 ++++++------ integration_tests/models/sql/test_union.sql | 10 ++++++---- integration_tests/models/sql/test_union_base.sql | 6 ++++++ 3 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 integration_tests/models/sql/test_union_base.sql diff --git a/integration_tests/data/sql/data_union_expected.csv b/integration_tests/data/sql/data_union_expected.csv index 59ebe14e206..5a22f3dfbf3 100644 --- a/integration_tests/data/sql/data_union_expected.csv +++ b/integration_tests/data/sql/data_union_expected.csv @@ -1,6 +1,6 @@ -_dbt_source_table,id,name,favorite_color -"""integration_tests"".""data_union_table_1""",1,"drew", -"""integration_tests"".""data_union_table_1""",2,"bob", -"""integration_tests"".""data_union_table_1""",3,"alice", -"""integration_tests"".""data_union_table_2""",1,,"green" -"""integration_tests"".""data_union_table_2""",2,,"pink" +id,name,favorite_color +1,"drew", +2,"bob", +3,"alice", +1,,"green" +2,,"pink" diff --git a/integration_tests/models/sql/test_union.sql b/integration_tests/models/sql/test_union.sql index 28207d05800..69836833c8c 100644 --- a/integration_tests/models/sql/test_union.sql +++ b/integration_tests/models/sql/test_union.sql @@ -1,6 +1,8 @@ -{{ dbt_utils.union_tables([ - ref('data_union_table_1'), - ref('data_union_table_2')] -) }} +select + id, + name, + favorite_color + +from {{ ref('test_union_base') }} diff --git a/integration_tests/models/sql/test_union_base.sql b/integration_tests/models/sql/test_union_base.sql new file mode 100644 index 00000000000..28207d05800 --- /dev/null +++ b/integration_tests/models/sql/test_union_base.sql @@ -0,0 +1,6 @@ + +{{ dbt_utils.union_tables([ + ref('data_union_table_1'), + ref('data_union_table_2')] +) }} + From bf998055c51ddc09400ff385c7a61b3ee0c8bddf Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Mon, 1 Oct 2018 14:43:13 -0400 Subject: [PATCH 4/4] use latest dbt --- .circleci/config.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index afb8e234802..81c699c4f64 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,13 +19,7 @@ jobs: python3 -m venv venv . venv/bin/activate - # Temporary fix for 0.11.0 bug -- 0.11.1 is yet unreleased - # TODO : Make this use latest from pip once 0.11.1 is out - git clone https://github.com/fishtown-analytics/dbt.git - cd dbt - git checkout dev/lucretia-mott - pip install -e . - cd .. + pip install dbt mkdir -p ~/.dbt cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml