Skip to content

Commit

Permalink
Fix quoting on columns for seeds/incremental models
Browse files Browse the repository at this point in the history
Added tests
  • Loading branch information
Jacob Beck committed Nov 4, 2019
1 parent 670c26b commit ffcf7af
Show file tree
Hide file tree
Showing 6 changed files with 73 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@


{% macro common_get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}
{%- set dest_cols_csv = dest_columns | map(attribute="name") | join(', ') -%}

{%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}

{% if unique_key is not none %}
delete from {{ target }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
{%- for col_name in agate_table.column_names -%}
{%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}
{%- set type = column_override.get(col_name, inferred_type) -%}
{{ col_name | string }} {{ type }} {%- if not loop.last -%}, {%- endif -%}
{{ adapter.quote(col_name | string) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}
{%- endfor -%}
)
{% endset %}
Expand All @@ -46,8 +46,19 @@
{% endmacro %}


{% macro get_quoted_csv(column_names) %}
{% set quoted = [] %}
{% for col in column_names -%}
{%- do quoted.append(adapter.quote(col)) -%}
{%- endfor %}

{%- set dest_cols_csv = quoted | join(', ') -%}
{{ return(dest_cols_csv) }}
{% endmacro %}


{% macro basic_load_csv_rows(model, batch_size, agate_table) %}
{% set cols_sql = ", ".join(agate_table.column_names) %}
{% set cols_sql = get_quoted_csv(agate_table.column_names) %}
{% set bindings = [] %}

{% set statements = [] %}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
is provided, then this macro will do a proper merge instead.
#}

{%- set dest_cols_csv = dest_columns | map(attribute="name") | join(', ') -%}
{%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute='name')) -%}

{%- if unique_key is none -%}

Expand Down
4 changes: 4 additions & 0 deletions test/integration/052_column_quoting/data/seed.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
"col_A","col_B"
1,2
3,4
5,6
16 changes: 16 additions & 0 deletions test/integration/052_column_quoting/models/model.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{% set col_a = '"col_A"' %}
{% set col_b = '"col_B"' %}
{% if adapter.type() == 'bigquery' %}
{% set col_a = '`col_A`' %}
{% set col_b = '`col_B`' %}
{% endif %}

{{config(
materialized = 'incremental',
unique_key = col_a,
incremental_strategy = "delete+insert"
)}}

select
{{ col_a }}, {{ col_b }}
from {{ref('seed')}}
37 changes: 37 additions & 0 deletions test/integration/052_column_quoting/test_column_quotes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
from test.integration.base import DBTIntegrationTest, use_profile
import os


class TestColumnQuoting(DBTIntegrationTest):
@property
def schema(self):
return 'dbt_column_quoting_052'

@staticmethod
def dir(value):
return os.path.normpath(value)

@property
def models(self):
return self.dir('models')

def _run_columnn_quotes(self):
self.run_dbt(['seed'])
self.run_dbt()
self.run_dbt()

@use_profile('postgres')
def test_postgres_column_quotes(self):
self._run_columnn_quotes()

@use_profile('redshift')
def test_redshift_column_quotes(self):
self._run_columnn_quotes()

@use_profile('snowflake')
def test_snowflake_column_quotes(self):
self._run_columnn_quotes()

@use_profile('bigquery')
def test_bigquery_column_quotes(self):
self._run_columnn_quotes()

0 comments on commit ffcf7af

Please sign in to comment.