diff --git a/CHANGELOG.md b/CHANGELOG.md index 0baacc6..7190e62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,14 @@ # Unreleased + ## New features - Add support for importing descriptions from columns with the same names in upstream models. It is available by setting the parameter `upstream_descriptions` to `True` in `generate_model_yaml` ([#61](https://github.com/dbt-labs/dbt-codegen/pull/61)) - Add support for including description placeholders for the source and table, which changes the behavior of `generate_source` when `include_descriptions` is set to `True`. Previous logic only created description placeholders for the columns. - Add optional `name` arg to `generate_source` - Add optional `table_names` arg to `generate_source` (#50 @rahulj51) +## Fixes +- generate_model_yaml now correctly handles nested bigquery fields (#27) + # dbt-codegen v0.6.0 This release creates breaking changes to the `generate_source.sql` macro. diff --git a/integration_tests/models/model_struct.sql b/integration_tests/models/model_struct.sql new file mode 100644 index 0000000..9d256b7 --- /dev/null +++ b/integration_tests/models/model_struct.sql @@ -0,0 +1,24 @@ +{% if target.type == "bigquery" %} + + {#--- This exists to test the BigQuery-specific behavior reqeusted in #27 -#} + select + STRUCT( + source, + medium, + source_medium + ) as analytics, + col_x + from {{ ref('data__campaign_analytics') }} + +{% else %} + + {#--- This enables mimicking the BigQuery behavior for other adapters -#} + select + analytics, + source, + medium, + source_medium, + col_x + from {{ ref('data__campaign_analytics') }} + +{% endif %} diff --git a/integration_tests/seeds/data__campaign_analytics.csv b/integration_tests/seeds/data__campaign_analytics.csv new file mode 100644 index 0000000..3f55b64 --- /dev/null +++ b/integration_tests/seeds/data__campaign_analytics.csv @@ -0,0 +1,4 @@ +source,medium,source_medium,analytics,col_x +source_1,medium_a,1a,,x +source_2,medium_b,2b,,x +source_3,medium_c,3c,,x diff --git a/integration_tests/tests/test_generate_model_struct_yaml.sql b/integration_tests/tests/test_generate_model_struct_yaml.sql new file mode 100644 index 0000000..9670f11 --- /dev/null +++ b/integration_tests/tests/test_generate_model_struct_yaml.sql @@ -0,0 +1,42 @@ +{% set raw_schema = generate_schema_name('raw_data') %} + +-- test all args +{% set actual_source_yaml = codegen.generate_source( + database_name=target.database, + schema_name='codegen_integration_tests__data_source_schema', + table_names=['codegen_integration_tests__data_source_table_nested_array'], + generate_columns=True, + include_descriptions=True +) %} + +{% set actual_source_yaml = codegen.generate_model_yaml( + model_name='model_struct' + ) +%} + + +{% set expected_source_yaml %} +version: 2 + +models: + - name: model_struct + description: "" + columns: + - name: analytics + description: "" + + - name: source + description: "" + + - name: medium + description: "" + + - name: source_medium + description: "" + + - name: col_x + description: "" + +{% endset %} + +{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }} diff --git a/integration_tests/tests/test_generate_source.sql b/integration_tests/tests/test_generate_source.sql index 2b32059..ce48bd6 100644 --- a/integration_tests/tests/test_generate_source.sql +++ b/integration_tests/tests/test_generate_source.sql @@ -12,6 +12,7 @@ sources: tables: - name: data__a_relation - name: data__b_relation + - name: data__campaign_analytics {% endset %} diff --git a/integration_tests/tests/test_generate_source_all_args.sql b/integration_tests/tests/test_generate_source_all_args.sql index e134069..e60faf8 100644 --- a/integration_tests/tests/test_generate_source_all_args.sql +++ b/integration_tests/tests/test_generate_source_all_args.sql @@ -36,6 +36,20 @@ sources: - name: col_b description: "" + - name: data__campaign_analytics + description: "" + columns: + - name: source + description: "" + - name: medium + description: "" + - name: source_medium + description: "" + - name: analytics + description: "" + - name: col_x + description: "" + {% endset %} {{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }} diff --git a/integration_tests/tests/test_generate_source_exclude.sql b/integration_tests/tests/test_generate_source_exclude.sql index fcfd7a8..e1bd85d 100644 --- a/integration_tests/tests/test_generate_source_exclude.sql +++ b/integration_tests/tests/test_generate_source_exclude.sql @@ -11,6 +11,7 @@ sources: - name: {{ raw_schema | trim | lower}} tables: - name: data__b_relation + - name: data__campaign_analytics {% endset %} diff --git a/integration_tests/tests/test_generate_source_table_descriptions.sql b/integration_tests/tests/test_generate_source_table_descriptions.sql index b10dbbd..a8a1922 100644 --- a/integration_tests/tests/test_generate_source_table_descriptions.sql +++ b/integration_tests/tests/test_generate_source_table_descriptions.sql @@ -15,6 +15,8 @@ sources: description: "" - name: data__b_relation description: "" + - name: data__campaign_analytics + description: "" {% endset %} diff --git a/integration_tests/tests/test_generate_source_table_name.sql b/integration_tests/tests/test_generate_source_table_name.sql index 6045595..968f7e3 100644 --- a/integration_tests/tests/test_generate_source_table_name.sql +++ b/integration_tests/tests/test_generate_source_table_name.sql @@ -13,6 +13,7 @@ sources: tables: - name: data__a_relation - name: data__b_relation + - name: data__campaign_analytics {% endset %} diff --git a/macros/generate_model_yaml.sql b/macros/generate_model_yaml.sql index 12d55ed..f00f910 100644 --- a/macros/generate_model_yaml.sql +++ b/macros/generate_model_yaml.sql @@ -1,3 +1,22 @@ +{% macro generate_column_yaml(column, model_yaml, column_desc_dict, parent_column_name="") %} + {% if parent_column_name %} + {% set column_name = parent_column_name ~ "." ~ column.name %} + {% else %} + {% set column_name = column.name %} + {% endif %} + + {% do model_yaml.append(' - name: ' ~ column.name | lower ) %} + {% do model_yaml.append(' description: "' ~ column_desc_dict.get(column.name | lower,'') ~ '"') %} + {% do model_yaml.append('') %} + + {% if column.fields|length > 0 %} + {% for child_column in column.fields %} + {% set model_yaml = codegen.generate_column_yaml(child_column, model_yaml, column_desc_dict, parent_column_name=column_name) %} + {% endfor %} + {% endif %} + {% do return(model_yaml) %} +{% endmacro %} + {% macro generate_model_yaml(model_name, upstream_descriptions=False) %} {% set model_yaml=[] %} @@ -14,9 +33,7 @@ {%- set columns = adapter.get_columns_in_relation(relation) -%} {% for column in columns %} - {% do model_yaml.append(' - name: ' ~ column.name | lower ) %} - {% do model_yaml.append(' description: "' ~ column_desc_dict.get(column.name | lower,'') ~ '"') %} - {% do model_yaml.append('') %} + {% set model_yaml = codegen.generate_column_yaml(column, model_yaml, column_desc_dict) %} {% endfor %} {% if execute %} @@ -27,4 +44,4 @@ {% endif %} -{% endmacro %} +{% endmacro %} \ No newline at end of file