diff --git a/.circleci/config.yml b/.circleci/config.yml index c9bd5579d32..81c699c4f64 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,9 @@ jobs: command: | python3 -m venv venv . venv/bin/activate + pip install dbt + mkdir -p ~/.dbt cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml @@ -33,8 +35,8 @@ jobs: . venv/bin/activate cd integration_tests dbt deps - dbt seed - dbt run + dbt seed --full-refresh + dbt run --full-refresh dbt test - save_cache: diff --git a/integration_tests/Makefile b/integration_tests/Makefile index 243f960fade..ffa967cffe9 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -1,22 +1,22 @@ test-postgres: - dbt seed --target postgres - dbt run --target postgres + dbt seed --target postgres --full-refresh + dbt run --target postgres --full-refresh dbt test --target postgres test-redshift: - dbt seed --target redshift - dbt run --target redshift + dbt seed --target redshift --full-refresh + dbt run --target redshift --full-refresh dbt test --target redshift test-snowflake: - dbt seed --target snowflake - dbt run --target snowflake + dbt seed --target snowflake --full-refresh + dbt run --target snowflake --full-refresh dbt test --target snowflake test-bigquery: - dbt seed --target bigquery - dbt run --target bigquery + dbt seed --target bigquery --full-refresh + dbt run --target bigquery --full-refresh dbt test --target bigquery test-all: test-postgres test-redshift test-snowflake test-bigquery diff --git a/integration_tests/data/sql/data_union_expected.csv b/integration_tests/data/sql/data_union_expected.csv new file mode 100644 index 00000000000..5a22f3dfbf3 --- /dev/null +++ b/integration_tests/data/sql/data_union_expected.csv @@ -0,0 +1,6 @@ +id,name,favorite_color +1,"drew", +2,"bob", +3,"alice", +1,,"green" +2,,"pink" diff --git a/integration_tests/data/sql/data_union_table_1.csv b/integration_tests/data/sql/data_union_table_1.csv new file mode 100644 index 00000000000..3f33321ceb2 --- /dev/null +++ b/integration_tests/data/sql/data_union_table_1.csv @@ -0,0 +1,4 @@ +id,name +1,drew +2,bob +3,alice diff --git a/integration_tests/data/sql/data_union_table_2.csv b/integration_tests/data/sql/data_union_table_2.csv new file mode 100644 index 00000000000..6f2fe9c3f56 --- /dev/null +++ b/integration_tests/data/sql/data_union_table_2.csv @@ -0,0 +1,3 @@ +id,favorite_color +1,green +2,pink diff --git a/integration_tests/models/sql/schema.yml b/integration_tests/models/sql/schema.yml index ec89f16fcd4..fa5dc7337a0 100644 --- a/integration_tests/models/sql/schema.yml +++ b/integration_tests/models/sql/schema.yml @@ -56,3 +56,8 @@ test_surrogate_key: constraints: assert_equal: - {actual: actual, expected: expected} + +test_union: + constraints: + dbt_utils.equality: + - ref('data_union_expected') diff --git a/integration_tests/models/sql/test_get_column_values.sql b/integration_tests/models/sql/test_get_column_values.sql index 47f45bb9005..25f3cbc6eda 100644 --- a/integration_tests/models/sql/test_get_column_values.sql +++ b/integration_tests/models/sql/test_get_column_values.sql @@ -3,6 +3,7 @@ select + {% set columns = columns if columns is iterable else [] %} {% for column in columns -%} sum(case when field = '{{ column }}' then 1 else 0 end) as count_{{ column }} diff --git a/integration_tests/models/sql/test_union.sql b/integration_tests/models/sql/test_union.sql new file mode 100644 index 00000000000..69836833c8c --- /dev/null +++ b/integration_tests/models/sql/test_union.sql @@ -0,0 +1,8 @@ + +select + id, + name, + favorite_color + +from {{ ref('test_union_base') }} + diff --git a/integration_tests/models/sql/test_union_base.sql b/integration_tests/models/sql/test_union_base.sql new file mode 100644 index 00000000000..28207d05800 --- /dev/null +++ b/integration_tests/models/sql/test_union_base.sql @@ -0,0 +1,6 @@ + +{{ dbt_utils.union_tables([ + ref('data_union_table_1'), + ref('data_union_table_2')] +) }} + diff --git a/macros/cross_db_utils/literal.sql b/macros/cross_db_utils/literal.sql new file mode 100644 index 00000000000..c3f3e4eeba9 --- /dev/null +++ b/macros/cross_db_utils/literal.sql @@ -0,0 +1,8 @@ + +{% macro string_literal(value) %} + {{ adapter_macro('dbt_utils.string_literal', value) }} +{% endmacro %} + +{% macro default__string_literal(value) -%} + '{{ value }}' +{%- endmacro %} diff --git a/macros/sql/union.sql b/macros/sql/union.sql index 59eb022d536..d79aafd7077 100644 --- a/macros/sql/union.sql +++ b/macros/sql/union.sql @@ -56,7 +56,7 @@ ( select - '{{ table }}'::text as _dbt_source_table, + {{ dbt_utils.safe_cast(dbt_utils.string_literal(table), dbt_utils.type_string()) }} as _dbt_source_table, {% for col_name in ordered_column_names -%} @@ -64,7 +64,7 @@ {%- set col_type = column_override.get(col.column, col.data_type) %} {%- set col_name = adapter.quote(col_name) if col_name in table_columns[table] else 'null' %} - {{ col_name }}::{{ col_type }} as {{ col.quoted }} {% if not loop.last %},{% endif %} + {{ dbt_utils.safe_cast(col_name, col_type) }} as {{ col.quoted }} {% if not loop.last %},{% endif %} {%- endfor %} from {{ table }}