Updated generate_model_yaml macro to correctly handle nested bigquery… (

#54) * Updated generate_model_yaml macro to correctly handle nested bigquery fields * Updated changelog.md * Update macros/generate_model_yaml.sql * Integration tests Co-authored-by: Bob De Schutter <bob.deschutter@digitalswat.be> Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Co-authored-by: Doug Beatty <doug.beatty@dbtlabs.com>
dbt-labs · Jun 23, 2022 · e982a11 · e982a11
1 parent 046eb72
commit e982a11
Show file tree

Hide file tree

Showing 10 changed files with 114 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,10 +1,14 @@
 # Unreleased
+
 ## New features
 - Add support for importing descriptions from columns with the same names in upstream models. It is available by setting the parameter `upstream_descriptions` to `True` in `generate_model_yaml` ([#61](https://github.com/dbt-labs/dbt-codegen/pull/61))
 - Add support for including description placeholders for the source and table, which changes the behavior of `generate_source` when `include_descriptions` is set to `True`. Previous logic only created description placeholders for the columns. 
 - Add optional `name` arg to `generate_source`
 - Add optional `table_names` arg to `generate_source` (#50 @rahulj51)
 
+## Fixes
+- generate_model_yaml now correctly handles nested bigquery fields (#27)
+
 # dbt-codegen v0.6.0
 
 This release creates breaking changes to the `generate_source.sql` macro.

diff --git a/integration_tests/models/model_struct.sql b/integration_tests/models/model_struct.sql
@@ -0,0 +1,24 @@
+{% if target.type == "bigquery" %}
+
+    {#--- This exists to test the BigQuery-specific behavior reqeusted in #27 -#}
+    select
+        STRUCT(
+            source,
+            medium,
+            source_medium
+        ) as analytics,
+        col_x
+    from {{ ref('data__campaign_analytics') }}
+
+{% else %}
+
+    {#--- This enables mimicking the BigQuery behavior for other adapters -#}
+    select
+        analytics,
+        source,
+        medium,
+        source_medium,
+        col_x
+    from {{ ref('data__campaign_analytics') }}
+
+{% endif %}
diff --git a/integration_tests/seeds/data__campaign_analytics.csv b/integration_tests/seeds/data__campaign_analytics.csv
@@ -0,0 +1,4 @@
+source,medium,source_medium,analytics,col_x
+source_1,medium_a,1a,,x
+source_2,medium_b,2b,,x
+source_3,medium_c,3c,,x
diff --git a/integration_tests/tests/test_generate_model_struct_yaml.sql b/integration_tests/tests/test_generate_model_struct_yaml.sql
@@ -0,0 +1,42 @@
+{% set raw_schema = generate_schema_name('raw_data') %}
+
+-- test all args
+{% set actual_source_yaml = codegen.generate_source(
+    database_name=target.database,
+    schema_name='codegen_integration_tests__data_source_schema',
+    table_names=['codegen_integration_tests__data_source_table_nested_array'],
+    generate_columns=True,
+    include_descriptions=True
+) %}
+
+{% set actual_source_yaml = codegen.generate_model_yaml(
+    model_name='model_struct'
+  )
+%}
+
+
+{% set expected_source_yaml %}
+version: 2
+
+models:
+  - name: model_struct
+    description: ""
+    columns:
+      - name: analytics
+        description: ""
+
+      - name: source
+        description: ""
+
+      - name: medium
+        description: ""
+
+      - name: source_medium
+        description: ""
+
+      - name: col_x
+        description: ""
+
+{% endset %}
+
+{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }}
diff --git a/integration_tests/tests/test_generate_source.sql b/integration_tests/tests/test_generate_source.sql
@@ -12,6 +12,7 @@ sources:
     tables:
       - name: data__a_relation
       - name: data__b_relation
+      - name: data__campaign_analytics
 {% endset %}
 
 

diff --git a/integration_tests/tests/test_generate_source_all_args.sql b/integration_tests/tests/test_generate_source_all_args.sql
@@ -36,6 +36,20 @@ sources:
           - name: col_b
             description: ""
 
+      - name: data__campaign_analytics
+        description: ""
+        columns:
+          - name: source
+            description: ""
+          - name: medium
+            description: ""
+          - name: source_medium
+            description: ""
+          - name: analytics
+            description: ""
+          - name: col_x
+            description: ""
+
 {% endset %}
 
 {{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }}
diff --git a/integration_tests/tests/test_generate_source_exclude.sql b/integration_tests/tests/test_generate_source_exclude.sql
@@ -11,6 +11,7 @@ sources:
   - name: {{ raw_schema | trim | lower}}
     tables:
       - name: data__b_relation
+      - name: data__campaign_analytics
 {% endset %}
 
 

diff --git a/integration_tests/tests/test_generate_source_table_descriptions.sql b/integration_tests/tests/test_generate_source_table_descriptions.sql
@@ -15,6 +15,8 @@ sources:
         description: ""
       - name: data__b_relation
         description: ""
+      - name: data__campaign_analytics
+        description: ""
 {% endset %}
 
 

diff --git a/integration_tests/tests/test_generate_source_table_name.sql b/integration_tests/tests/test_generate_source_table_name.sql
@@ -13,6 +13,7 @@ sources:
     tables:
       - name: data__a_relation
       - name: data__b_relation
+      - name: data__campaign_analytics
 
 {% endset %}
 

diff --git a/macros/generate_model_yaml.sql b/macros/generate_model_yaml.sql
@@ -1,3 +1,22 @@
+{% macro generate_column_yaml(column, model_yaml, column_desc_dict, parent_column_name="") %}
+    {% if parent_column_name %}
+        {% set column_name = parent_column_name ~ "." ~ column.name %}
+    {% else %}
+        {% set column_name = column.name %}
+    {% endif %}
+
+    {% do model_yaml.append('      - name: ' ~ column.name | lower ) %}
+    {% do model_yaml.append('        description: "' ~ column_desc_dict.get(column.name | lower,'') ~ '"') %}
+    {% do model_yaml.append('') %}
+
+    {% if column.fields|length > 0 %}
+        {% for child_column in column.fields %}
+            {% set model_yaml = codegen.generate_column_yaml(child_column, model_yaml, column_desc_dict, parent_column_name=column_name) %}
+        {% endfor %}
+    {% endif %}
+    {% do return(model_yaml) %}
+{% endmacro %}
+
 {% macro generate_model_yaml(model_name, upstream_descriptions=False) %}
 
 {% set model_yaml=[] %}
@@ -14,9 +33,7 @@
 {%- set columns = adapter.get_columns_in_relation(relation) -%}
 
 {% for column in columns %}
-    {% do model_yaml.append('      - name: ' ~ column.name | lower ) %}
-    {% do model_yaml.append('        description: "' ~ column_desc_dict.get(column.name | lower,'') ~ '"') %}
-    {% do model_yaml.append('') %}
+    {% set model_yaml = codegen.generate_column_yaml(column, model_yaml, column_desc_dict) %}
 {% endfor %}
 
 {% if execute %}
@@ -27,4 +44,4 @@
 
 {% endif %}
 
-{% endmacro %}
+{% endmacro %}