From 7156cc5c1da9da2dae9d7f2fd4d524e39773bb61 Mon Sep 17 00:00:00 2001 From: Peter Webb Date: Tue, 27 Jun 2023 17:40:27 -0400 Subject: [PATCH] Add Partial Parsing Support for Semantic Models (#7964) * CT-2711: Add partial parsing support for semantic models * CT-2711: Fix typo identified in code review --- .../unreleased/Features-20230627-123123.yaml | 6 ++++ core/dbt/parser/partial.py | 23 ++++++++++++++- .../test_semantic_model_parsing.py | 28 +++++++++++++++++-- 3 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 .changes/unreleased/Features-20230627-123123.yaml diff --git a/.changes/unreleased/Features-20230627-123123.yaml b/.changes/unreleased/Features-20230627-123123.yaml new file mode 100644 index 00000000000..39f12e9ce80 --- /dev/null +++ b/.changes/unreleased/Features-20230627-123123.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add partial parsing support for semantic models +time: 2023-06-27T12:31:23.953018-04:00 +custom: + Author: peterallenwebb + Issue: "7897" diff --git a/core/dbt/parser/partial.py b/core/dbt/parser/partial.py index 8a70ba7525a..edcae83574a 100644 --- a/core/dbt/parser/partial.py +++ b/core/dbt/parser/partial.py @@ -417,6 +417,14 @@ def schedule_nodes_for_parsing(self, unique_ids): self._schedule_for_parsing( "metrics", metric, metric.name, self.delete_schema_metric ) + elif unique_id in self.saved_manifest.semantic_models: + semantic_model = self.saved_manifest.semantic_models[unique_id] + self._schedule_for_parsing( + "semantic_models", + semantic_model, + semantic_model.name, + self.delete_schema_semantic_model, + ) elif unique_id in self.saved_manifest.macros: macro = self.saved_manifest.macros[unique_id] file_id = macro.file_id @@ -600,7 +608,7 @@ def handle_schema_file_changes(self, schema_file, saved_yaml_dict, new_yaml_dict env_var_changes = self.env_vars_changed_schema_files[schema_file.file_id] # models, seeds, snapshots, analyses - for dict_key in ["models", "seeds", "snapshots", "analyses", "semantic_models"]: + for dict_key in ["models", "seeds", "snapshots", "analyses"]: key_diff = self.get_diff_for(dict_key, saved_yaml_dict, new_yaml_dict) if key_diff["changed"]: for elem in key_diff["changed"]: @@ -662,6 +670,7 @@ def handle_change(key: str, delete: Callable): handle_change("exposures", self.delete_schema_exposure) handle_change("metrics", self.delete_schema_metric) handle_change("groups", self.delete_schema_group) + handle_change("semantic_models", self.delete_schema_semantic_model) def _handle_element_change( self, schema_file, saved_yaml_dict, new_yaml_dict, env_var_changes, dict_key: str, delete @@ -874,6 +883,18 @@ def delete_schema_metric(self, schema_file, metric_dict): elif unique_id in self.saved_manifest.disabled: self.delete_disabled(unique_id, schema_file.file_id) + def delete_schema_semantic_model(self, schema_file, semantic_model_dict): + semantic_model_name = semantic_model_dict["name"] + semantic_models = schema_file.semantic_models.copy() + for unique_id in semantic_models: + if unique_id in self.saved_manifest.semantic_models: + semantic_model = self.saved_manifest.semantic_models[unique_id] + if semantic_model.name == semantic_model_name: + self.saved_manifest.semantic_models.pop(unique_id) + schema_file.semantic_models.remove(unique_id) + elif unique_id in self.saved_manifest.disabled: + self.delete_disabled(unique_id, schema_file.file_id) + def get_schema_element(self, elem_list, elem_name): for element in elem_list: if "name" in element and element["name"] == elem_name: diff --git a/tests/functional/semantic_models/test_semantic_model_parsing.py b/tests/functional/semantic_models/test_semantic_model_parsing.py index 149b609b75f..0384f907f10 100644 --- a/tests/functional/semantic_models/test_semantic_model_parsing.py +++ b/tests/functional/semantic_models/test_semantic_model_parsing.py @@ -55,6 +55,11 @@ expr: user_id """ +schema_without_semantic_model_yml = """models: + - name: fct_revenue + description: This is the model fct_revenue. It should be able to use doc blocks +""" + fct_revenue_sql = """select 1 as id, 10 as user_id, @@ -85,12 +90,12 @@ def test_semantic_model_parsing(self, project): ) assert len(semantic_model.measures) == 5 - @pytest.mark.skip("Restore this test when partial parsing is implemented.") - def test_semantic_model_partial_parsing(self, project): + def test_semantic_model_changed_partial_parsing(self, project): # First, use the default schema.yml to define our semantic model, and # run the dbt parse command runner = dbtRunner() result = runner.invoke(["parse"]) + assert result.success # Next, modify the default schema.yml to change a detail of the semantic # model. @@ -99,8 +104,27 @@ def test_semantic_model_partial_parsing(self, project): # Now, run the dbt parse command again. result = runner.invoke(["parse"]) + assert result.success # Finally, verify that the manifest reflects the partially parsed change manifest = result.result semantic_model = manifest.semantic_models["semanticmodel.test.revenue"] assert semantic_model.dimensions[0].type_params.time_granularity == TimeGranularity.WEEK + + def test_semantic_model_deleted_partial_parsing(self, project): + # First, use the default schema.yml to define our semantic model, and + # run the dbt parse command + runner = dbtRunner() + result = runner.invoke(["parse"]) + assert result.success + assert "semanticmodel.test.revenue" in result.result.semantic_models + + # Next, modify the default schema.yml to remove the semantic model. + write_file(schema_without_semantic_model_yml, project.project_root, "models", "schema.yml") + + # Now, run the dbt parse command again. + result = runner.invoke(["parse"]) + assert result.success + + # Finally, verify that the manifest reflects the deletion + assert "semanticmodel.test.revenue" not in result.result.semantic_models