From 046eb72a1f5b2f4d8463caaaeeea70c3fc5cbdce Mon Sep 17 00:00:00 2001 From: Rahul Jain Date: Thu, 23 Jun 2022 00:01:53 +0200 Subject: [PATCH] Source for specific tables (#51) * Addig another parameter to pass a list of tables to generate_source * Addig another parameter to pass a list of tables to generate_source * Updated README * Updated changelog * Apply suggestions from code review * Update integration_tests/tests/test_generate_source_some_tables.sql Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> --- CHANGELOG.md | 1 + README.md | 3 +- .../test_generate_source_some_tables.sql | 30 +++++++++++++++++++ macros/generate_source.sql | 6 +++- 4 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 integration_tests/tests/test_generate_source_some_tables.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index be37aa6..0baacc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - Add support for importing descriptions from columns with the same names in upstream models. It is available by setting the parameter `upstream_descriptions` to `True` in `generate_model_yaml` ([#61](https://github.com/dbt-labs/dbt-codegen/pull/61)) - Add support for including description placeholders for the source and table, which changes the behavior of `generate_source` when `include_descriptions` is set to `True`. Previous logic only created description placeholders for the columns. - Add optional `name` arg to `generate_source` +- Add optional `table_names` arg to `generate_source` (#50 @rahulj51) # dbt-codegen v0.6.0 diff --git a/README.md b/README.md index 4f0d811..d31f437 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ which you can then paste into a schema file. * `schema_name` (required): The schema name that contains your source data * `database_name` (optional, default=target.database): The database that your source data is in. +* `table_names` (optional, default=none): A list of tables that you want to generate the source definitions for. * `generate_columns` (optional, default=False): Whether you want to add the column names to your source definition. * `include_descriptions` (optional, default=False): Whether you want to add @@ -47,7 +48,7 @@ or ``` # for multiple arguments, use the dict syntax -$ dbt run-operation generate_source --args '{"schema_name": "jaffle_shop", "database_name": "raw"}' +$ dbt run-operation generate_source --args '{"schema_name": "jaffle_shop", "database_name": "raw", "table_names":["table_1", "table_2"]}' ``` 2. The YAML for the source will be logged to the command line diff --git a/integration_tests/tests/test_generate_source_some_tables.sql b/integration_tests/tests/test_generate_source_some_tables.sql new file mode 100644 index 0000000..3d6fae6 --- /dev/null +++ b/integration_tests/tests/test_generate_source_some_tables.sql @@ -0,0 +1,30 @@ +{% set raw_schema = generate_schema_name('raw_data') %} + +-- test all args +{% set actual_source_yaml = codegen.generate_source( + schema_name=raw_schema, + database_name=target.database, + table_names=['data__a_relation'], + generate_columns=True, + include_descriptions=True +) %} + + +{% set expected_source_yaml %} +version: 2 + +sources: + - name: {{ raw_schema | trim }} + description: "" + tables: + - name: data__a_relation + description: "" + columns: + - name: col_a + description: "" + - name: col_b + description: "" + +{% endset %} + +{{ assert_equal (actual_source_yaml | trim, expected_source_yaml | trim) }} diff --git a/macros/generate_source.sql b/macros/generate_source.sql index aa90c25..366b4b8 100644 --- a/macros/generate_source.sql +++ b/macros/generate_source.sql @@ -15,7 +15,7 @@ --- -{% macro generate_source(schema_name, database_name=target.database, generate_columns=False, include_descriptions=False, table_pattern='%', exclude='', name=schema_name) %} +{% macro generate_source(schema_name, database_name=target.database, generate_columns=False, include_descriptions=False, table_pattern='%', exclude='', name=schema_name, table_names=None) %} {% set sources_yaml=[] %} {% do sources_yaml.append('version: 2') %} @@ -37,7 +37,11 @@ {% do sources_yaml.append(' tables:') %} +{% if table_names is none %} {% set tables=codegen.get_tables_in_schema(schema_name, database_name, table_pattern, exclude) %} +{% else %} +{% set tables = table_names %} +{% endif %} {% for table in tables %} {% do sources_yaml.append(' - name: ' ~ table | lower ) %}