From 892d111dbca871a52032c0528b0a0ecfe9ff300f Mon Sep 17 00:00:00 2001 From: jeremyyeo Date: Fri, 13 May 2022 10:29:49 +1200 Subject: [PATCH 1/3] make case insensitive --- macros/sql/union.sql | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/macros/sql/union.sql b/macros/sql/union.sql index 0d4ebccb..74bbd988 100644 --- a/macros/sql/union.sql +++ b/macros/sql/union.sql @@ -17,6 +17,20 @@ {%- set relation_columns = {} -%} {%- set column_superset = {} -%} + {%- set all_excludes = [] -%} + {%- set all_includes = [] -%} + + {%- if exclude -%} + {%- for exc in exclude -%} + {%- do all_excludes.append(exc | lower) -%} + {%- endfor -%} + {%- endif -%} + + {%- if include -%} + {%- for inc in include -%} + {%- do all_includes.append(inc | lower) -%} + {%- endfor -%} + {%- endif -%} {%- for relation in relations -%} @@ -28,10 +42,10 @@ {%- for col in cols -%} {#- If an exclude list was provided and the column is in the list, do nothing -#} - {%- if exclude and col.column in exclude -%} + {%- if exclude and col.column | lower in all_excludes -%} {#- If an include list was provided and the column is not in the list, do nothing -#} - {%- elif include and col.column not in include -%} + {%- elif include and col.column | lower not in all_includes -%} {#- Otherwise add the column to the column superset -#} {%- else -%} From 854d33467ea6fe560121f9f26209dd7a6cf1a5e9 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Fri, 13 May 2022 14:02:39 -0600 Subject: [PATCH 2/3] Integration test when excluding columns from union_relations --- .../data/sql/data_union_exclude_expected.csv | 6 ++++++ integration_tests/models/sql/schema.yml | 5 +++++ integration_tests/models/sql/test_union_exclude.sql | 4 ++++ integration_tests/models/sql/test_union_exclude_base.sql | 8 ++++++++ 4 files changed, 23 insertions(+) create mode 100644 integration_tests/data/sql/data_union_exclude_expected.csv create mode 100644 integration_tests/models/sql/test_union_exclude.sql create mode 100644 integration_tests/models/sql/test_union_exclude_base.sql diff --git a/integration_tests/data/sql/data_union_exclude_expected.csv b/integration_tests/data/sql/data_union_exclude_expected.csv new file mode 100644 index 00000000..3283e5c6 --- /dev/null +++ b/integration_tests/data/sql/data_union_exclude_expected.csv @@ -0,0 +1,6 @@ +id,favorite_color,favorite_number +1,,pi +2,,e +3,,4 +1,"green",7 +2,"pink",13 diff --git a/integration_tests/models/sql/schema.yml b/integration_tests/models/sql/schema.yml index e5927fbf..b8f66411 100644 --- a/integration_tests/models/sql/schema.yml +++ b/integration_tests/models/sql/schema.yml @@ -163,6 +163,11 @@ models: tests: - dbt_utils.not_constant + - name: test_union_exclude + tests: + - dbt_utils.equality: + compare_model: ref('data_union_exclude_expected') + - name: test_get_relations_by_pattern tests: - dbt_utils.equality: diff --git a/integration_tests/models/sql/test_union_exclude.sql b/integration_tests/models/sql/test_union_exclude.sql new file mode 100644 index 00000000..f3bbbec1 --- /dev/null +++ b/integration_tests/models/sql/test_union_exclude.sql @@ -0,0 +1,4 @@ +select + {{ dbt_utils.star(ref("test_union_exclude_base"), except=["_dbt_source_relation"]) }} + +from {{ ref("test_union_exclude_base") }} diff --git a/integration_tests/models/sql/test_union_exclude_base.sql b/integration_tests/models/sql/test_union_exclude_base.sql new file mode 100644 index 00000000..dc005a8e --- /dev/null +++ b/integration_tests/models/sql/test_union_exclude_base.sql @@ -0,0 +1,8 @@ + +{{ dbt_utils.union_relations( + relations=[ + ref('data_union_table_1'), + ref('data_union_table_2'), + ], + exclude=['name'] +) }} From a5fcc7cbd79933021610fe56bf3cce0caf986fec Mon Sep 17 00:00:00 2001 From: jeremyyeo Date: Mon, 16 May 2022 21:43:47 +1200 Subject: [PATCH 3/3] modify test with casing --- CHANGELOG.md | 2 ++ integration_tests/models/sql/schema.yml | 7 ++++++- integration_tests/models/sql/test_union_exclude.sql | 4 ---- ...ude_base.sql => test_union_exclude_base_lowercase.sql} | 0 .../models/sql/test_union_exclude_base_uppercase.sql | 8 ++++++++ .../models/sql/test_union_exclude_lowercase.sql | 4 ++++ .../models/sql/test_union_exclude_uppercase.sql | 4 ++++ 7 files changed, 24 insertions(+), 5 deletions(-) delete mode 100644 integration_tests/models/sql/test_union_exclude.sql rename integration_tests/models/sql/{test_union_exclude_base.sql => test_union_exclude_base_lowercase.sql} (100%) create mode 100644 integration_tests/models/sql/test_union_exclude_base_uppercase.sql create mode 100644 integration_tests/models/sql/test_union_exclude_lowercase.sql create mode 100644 integration_tests/models/sql/test_union_exclude_uppercase.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index f72afbd4..805f9ee5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ## Fixes - Enable a negative part_number for `split_part()` ([#557](https://github.com/dbt-labs/dbt-utils/issues/557), [#559](https://github.com/dbt-labs/dbt-utils/pull/559)) +- Make `exclude` case insensitive for `union_relations()` ([#578](https://github.com/dbt-labs/dbt-utils/issues/557), [#587](https://github.com/dbt-labs/dbt-utils/issues/587)) ## Quality of life - Documentation about listagg macro ([#544](https://github.com/dbt-labs/dbt-utils/issues/544), [#560](https://github.com/dbt-labs/dbt-utils/pull/560)) @@ -21,6 +22,7 @@ - [@clausherther](https://github.com/clausherther) (#555) - [@epapineau](https://github.com/epapineau) (#583) - [@b-per](https://github.com/b-per) (#559) +- [@dbeatty10](https://github.com/dbeatty10), [@jeremyyeo](https://github.com/jeremyyeo) (#587) # dbt-utils v0.8.4 ## Fixes diff --git a/integration_tests/models/sql/schema.yml b/integration_tests/models/sql/schema.yml index b8f66411..df10ccac 100644 --- a/integration_tests/models/sql/schema.yml +++ b/integration_tests/models/sql/schema.yml @@ -163,7 +163,12 @@ models: tests: - dbt_utils.not_constant - - name: test_union_exclude + - name: test_union_exclude_lowercase + tests: + - dbt_utils.equality: + compare_model: ref('data_union_exclude_expected') + + - name: test_union_exclude_uppercase tests: - dbt_utils.equality: compare_model: ref('data_union_exclude_expected') diff --git a/integration_tests/models/sql/test_union_exclude.sql b/integration_tests/models/sql/test_union_exclude.sql deleted file mode 100644 index f3bbbec1..00000000 --- a/integration_tests/models/sql/test_union_exclude.sql +++ /dev/null @@ -1,4 +0,0 @@ -select - {{ dbt_utils.star(ref("test_union_exclude_base"), except=["_dbt_source_relation"]) }} - -from {{ ref("test_union_exclude_base") }} diff --git a/integration_tests/models/sql/test_union_exclude_base.sql b/integration_tests/models/sql/test_union_exclude_base_lowercase.sql similarity index 100% rename from integration_tests/models/sql/test_union_exclude_base.sql rename to integration_tests/models/sql/test_union_exclude_base_lowercase.sql diff --git a/integration_tests/models/sql/test_union_exclude_base_uppercase.sql b/integration_tests/models/sql/test_union_exclude_base_uppercase.sql new file mode 100644 index 00000000..af3dc3fa --- /dev/null +++ b/integration_tests/models/sql/test_union_exclude_base_uppercase.sql @@ -0,0 +1,8 @@ + +{{ dbt_utils.union_relations( + relations=[ + ref('data_union_table_1'), + ref('data_union_table_2'), + ], + exclude=['NAME'] +) }} diff --git a/integration_tests/models/sql/test_union_exclude_lowercase.sql b/integration_tests/models/sql/test_union_exclude_lowercase.sql new file mode 100644 index 00000000..95efc06a --- /dev/null +++ b/integration_tests/models/sql/test_union_exclude_lowercase.sql @@ -0,0 +1,4 @@ +select + {{ dbt_utils.star(ref("test_union_exclude_base_lowercase"), except=["_dbt_source_relation"]) }} + +from {{ ref("test_union_exclude_base_lowercase") }} diff --git a/integration_tests/models/sql/test_union_exclude_uppercase.sql b/integration_tests/models/sql/test_union_exclude_uppercase.sql new file mode 100644 index 00000000..3e0ab4f9 --- /dev/null +++ b/integration_tests/models/sql/test_union_exclude_uppercase.sql @@ -0,0 +1,4 @@ +select + {{ dbt_utils.star(ref("test_union_exclude_base_uppercase"), except=["_DBT_SOURCE_RELATION"]) }} + +from {{ ref("test_union_exclude_base_uppercase") }}