From e771ff5199e04d85783820e8ac06e05a2d7b7c73 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Thu, 22 Jun 2023 14:56:13 -0400 Subject: [PATCH] CT-2719: Rename the semantic_nodes collection on the manifest to semantic_models --- core/dbt/compilation.py | 4 +-- core/dbt/contracts/files.py | 2 +- core/dbt/contracts/graph/manifest.py | 30 +++++++++---------- core/dbt/contracts/graph/manifest_upgrade.py | 4 +-- core/dbt/parser/manifest.py | 2 +- schemas/dbt/manifest/v10.json | 6 ++-- .../functional/artifacts/expected_manifest.py | 6 ++-- tests/functional/artifacts/test_artifacts.py | 2 +- .../test_semantic_model_parsing.py | 6 ++-- tests/unit/test_manifest.py | 24 +++++++-------- 10 files changed, 43 insertions(+), 43 deletions(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 52c4e4dddd1..55c999b2b08 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -179,8 +179,8 @@ def link_node(self, node: GraphMemberNode, manifest: Manifest): def link_graph(self, manifest: Manifest): for source in manifest.sources.values(): self.add_node(source.unique_id) - for semantic_node in manifest.semantic_nodes.values(): - self.add_node(semantic_node.unique_id) + for semantic_model in manifest.semantic_models.values(): + self.add_node(semantic_model.unique_id) for node in manifest.nodes.values(): self.link_node(node, manifest) diff --git a/core/dbt/contracts/files.py b/core/dbt/contracts/files.py index f4f23617133..f54533c38c1 100644 --- a/core/dbt/contracts/files.py +++ b/core/dbt/contracts/files.py @@ -228,7 +228,7 @@ class SchemaSourceFile(BaseSourceFile): groups: List[str] = field(default_factory=list) # node patches contain models, seeds, snapshots, analyses ndp: List[str] = field(default_factory=list) - semantic_nodes: List[str] = field(default_factory=list) + semantic_models: List[str] = field(default_factory=list) # any macro patches in this file by macro unique_id. mcp: Dict[str, str] = field(default_factory=dict) # any source patches in this file. The entries are package, name pairs diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 7fa870f5366..92f82dff4b8 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -701,7 +701,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin): disabled: MutableMapping[str, List[GraphMemberNode]] = field(default_factory=dict) env_vars: MutableMapping[str, str] = field(default_factory=dict) publications: MutableMapping[str, PublicationConfig] = field(default_factory=dict) - semantic_nodes: MutableMapping[str, SemanticModel] = field(default_factory=dict) + semantic_models: MutableMapping[str, SemanticModel] = field(default_factory=dict) _doc_lookup: Optional[DocLookup] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} @@ -753,8 +753,8 @@ def build_flat_graph(self): "metrics": {k: v.to_dict(omit_none=False) for k, v in self.metrics.items()}, "nodes": {k: v.to_dict(omit_none=False) for k, v in self.nodes.items()}, "sources": {k: v.to_dict(omit_none=False) for k, v in self.sources.items()}, - "semantic_nodes": { - k: v.to_dict(omit_none=False) for k, v in self.semantic_nodes.items() + "semantic_models": { + k: v.to_dict(omit_none=False) for k, v in self.semantic_models.items() }, } @@ -816,7 +816,7 @@ def get_resource_fqns(self) -> Mapping[str, PathSet]: self.nodes.values(), self.sources.values(), self.metrics.values(), - self.semantic_nodes.values(), + self.semantic_models.values(), ) for resource in all_resources: resource_type_plural = resource.resource_type.pluralize() @@ -852,7 +852,7 @@ def deepcopy(self): files={k: _deepcopy(v) for k, v in self.files.items()}, state_check=_deepcopy(self.state_check), publications={k: _deepcopy(v) for k, v in self.publications.items()}, - semantic_nodes={k: _deepcopy(v) for k, v in self.semantic_nodes.items()}, + semantic_models={k: _deepcopy(v) for k, v in self.semantic_models.items()}, ) copy.build_flat_graph() return copy @@ -864,7 +864,7 @@ def build_parent_and_child_maps(self): self.sources.values(), self.exposures.values(), self.metrics.values(), - self.semantic_nodes.values(), + self.semantic_models.values(), ) ) forward_edges, backward_edges = build_node_edges(edge_members) @@ -911,7 +911,7 @@ def writable_manifest(self) -> "WritableManifest": child_map=self.child_map, parent_map=self.parent_map, group_map=self.group_map, - semantic_nodes=self.semantic_nodes, + semantic_models=self.semantic_models, ) def write(self, path): @@ -928,8 +928,8 @@ def expect(self, unique_id: str) -> GraphMemberNode: return self.exposures[unique_id] elif unique_id in self.metrics: return self.metrics[unique_id] - elif unique_id in self.semantic_nodes: - return self.semantic_nodes[unique_id] + elif unique_id in self.semantic_models: + return self.semantic_models[unique_id] else: # something terrible has happened raise dbt.exceptions.DbtInternalError( @@ -988,7 +988,7 @@ def analysis_lookup(self) -> AnalysisLookup: def pydantic_semantic_manifest(self) -> PydanticSemanticManifest: pydantic_semantic_manifest = PydanticSemanticManifest(metrics=[], semantic_models=[]) - for semantic_model in self.semantic_nodes.values(): + for semantic_model in self.semantic_models.values(): pydantic_semantic_manifest.semantic_models.append( PydanticSemanticModel.parse_obj(semantic_model.to_dict()) ) @@ -1267,9 +1267,9 @@ def add_doc(self, source_file: SourceFile, doc: Documentation): source_file.docs.append(doc.unique_id) def add_semantic_model(self, source_file: SchemaSourceFile, semantic_model: SemanticModel): - _check_duplicates(semantic_model, self.semantic_nodes) - self.semantic_nodes[semantic_model.unique_id] = semantic_model - source_file.semantic_nodes.append(semantic_model.unique_id) + _check_duplicates(semantic_model, self.semantic_models) + self.semantic_models[semantic_model.unique_id] = semantic_model + source_file.semantic_models.append(semantic_model.unique_id) # end of methods formerly in ParseResult @@ -1298,7 +1298,7 @@ def __reduce_ex__(self, protocol): self.disabled, self.env_vars, self.publications, - self.semantic_nodes, + self.semantic_models, self._doc_lookup, self._source_lookup, self._ref_lookup, @@ -1368,7 +1368,7 @@ class WritableManifest(ArtifactMixin): description="A mapping from group names to their nodes", ) ) - semantic_nodes: Mapping[UniqueID, SemanticModel] = field( + semantic_models: Mapping[UniqueID, SemanticModel] = field( metadata=dict(description=("The semantic models defined in the dbt project")) ) metadata: ManifestMetadata = field( diff --git a/core/dbt/contracts/graph/manifest_upgrade.py b/core/dbt/contracts/graph/manifest_upgrade.py index 644c081e3fb..fb979ec09a4 100644 --- a/core/dbt/contracts/graph/manifest_upgrade.py +++ b/core/dbt/contracts/graph/manifest_upgrade.py @@ -102,6 +102,6 @@ def upgrade_manifest_json(manifest: dict, manifest_schema_version: int) -> dict: if "root_path" in doc_content: del doc_content["root_path"] doc_content["resource_type"] = "doc" - if "semantic_nodes" not in manifest: - manifest["semantic_nodes"] = {} + if "semantic_models" not in manifest: + manifest["semantic_models"] = {} return manifest diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 96030aea79b..797b2534215 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -1103,7 +1103,7 @@ def process_refs(self, current_project: str): if metric.created_at < self.started_at: continue _process_refs(self.manifest, current_project, metric) - for semantic_model in self.manifest.semantic_nodes.values(): + for semantic_model in self.manifest.semantic_models.values(): if semantic_model.created_at < self.started_at: continue _process_refs(self.manifest, current_project, semantic_model) diff --git a/schemas/dbt/manifest/v10.json b/schemas/dbt/manifest/v10.json index 4f0e2ce4d80..e57a55f4531 100644 --- a/schemas/dbt/manifest/v10.json +++ b/schemas/dbt/manifest/v10.json @@ -10,7 +10,7 @@ "metrics", "groups", "selectors", - "semantic_nodes" + "semantic_models" ], "properties": { "metadata": { @@ -203,7 +203,7 @@ ], "description": "A mapping from group names to their nodes" }, - "semantic_nodes": { + "semantic_models": { "type": "object", "additionalProperties": { "$ref": "#/definitions/SemanticModel" @@ -212,7 +212,7 @@ } }, "additionalProperties": false, - "description": "WritableManifest(metadata: dbt.contracts.graph.manifest.ManifestMetadata, nodes: Mapping[str, Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode]], sources: Mapping[str, dbt.contracts.graph.nodes.SourceDefinition], macros: Mapping[str, dbt.contracts.graph.nodes.Macro], docs: Mapping[str, dbt.contracts.graph.nodes.Documentation], exposures: Mapping[str, dbt.contracts.graph.nodes.Exposure], metrics: Mapping[str, dbt.contracts.graph.nodes.Metric], groups: Mapping[str, dbt.contracts.graph.nodes.Group], selectors: Mapping[str, Any], disabled: Union[Mapping[str, List[Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode, dbt.contracts.graph.nodes.SourceDefinition, dbt.contracts.graph.nodes.Exposure, dbt.contracts.graph.nodes.Metric]]], NoneType], parent_map: Union[Dict[str, List[str]], NoneType], child_map: Union[Dict[str, List[str]], NoneType], group_map: Union[Dict[str, List[str]], NoneType], semantic_nodes: Mapping[str, dbt.contracts.graph.nodes.SemanticModel])", + "description": "WritableManifest(metadata: dbt.contracts.graph.manifest.ManifestMetadata, nodes: Mapping[str, Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode]], sources: Mapping[str, dbt.contracts.graph.nodes.SourceDefinition], macros: Mapping[str, dbt.contracts.graph.nodes.Macro], docs: Mapping[str, dbt.contracts.graph.nodes.Documentation], exposures: Mapping[str, dbt.contracts.graph.nodes.Exposure], metrics: Mapping[str, dbt.contracts.graph.nodes.Metric], groups: Mapping[str, dbt.contracts.graph.nodes.Group], selectors: Mapping[str, Any], disabled: Union[Mapping[str, List[Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode, dbt.contracts.graph.nodes.SourceDefinition, dbt.contracts.graph.nodes.Exposure, dbt.contracts.graph.nodes.Metric]]], NoneType], parent_map: Union[Dict[str, List[str]], NoneType], child_map: Union[Dict[str, List[str]], NoneType], group_map: Union[Dict[str, List[str]], NoneType], semantic_models: Mapping[str, dbt.contracts.graph.nodes.SemanticModel])", "definitions": { "ManifestMetadata": { "type": "object", diff --git a/tests/functional/artifacts/expected_manifest.py b/tests/functional/artifacts/expected_manifest.py index a96d584b243..d6729342543 100644 --- a/tests/functional/artifacts/expected_manifest.py +++ b/tests/functional/artifacts/expected_manifest.py @@ -886,7 +886,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "doc.test.macro_arg_info": ANY, }, "disabled": {}, - "semantic_nodes": {}, + "semantic_models": {}, } @@ -1445,7 +1445,7 @@ def expected_references_manifest(project): ], } }, - "semantic_nodes": {}, + "semantic_models": {}, } @@ -1924,5 +1924,5 @@ def expected_versions_manifest(project): }, "disabled": {}, "macros": {}, - "semantic_nodes": {}, + "semantic_models": {}, } diff --git a/tests/functional/artifacts/test_artifacts.py b/tests/functional/artifacts/test_artifacts.py index 5f33441b73d..db0f72fd5cb 100644 --- a/tests/functional/artifacts/test_artifacts.py +++ b/tests/functional/artifacts/test_artifacts.py @@ -468,7 +468,7 @@ def verify_manifest(project, expected_manifest, start_time, manifest_schema_path "disabled", "exposures", "selectors", - "semantic_nodes", + "semantic_models", } assert set(manifest.keys()) == manifest_keys diff --git a/tests/functional/semantic_models/test_semantic_model_parsing.py b/tests/functional/semantic_models/test_semantic_model_parsing.py index 18b9b07d235..879c19aeadd 100644 --- a/tests/functional/semantic_models/test_semantic_model_parsing.py +++ b/tests/functional/semantic_models/test_semantic_model_parsing.py @@ -58,8 +58,8 @@ def test_semantic_model_parsing(self, project): assert result.success assert isinstance(result.result, Manifest) manifest = result.result - assert len(manifest.semantic_nodes) == 1 - semantic_model = manifest.semantic_nodes["semanticmodel.test.revenue"] + assert len(manifest.semantic_models) == 1 + semantic_model = manifest.semantic_models["semanticmodel.test.revenue"] assert semantic_model.node_relation.alias == "fct_revenue" assert ( semantic_model.node_relation.relation_name @@ -83,5 +83,5 @@ def test_semantic_model_partial_parsing(self, project): # Finally, verify that the manifest reflects the partially parsed change manifest = result.result - semantic_model = manifest.semantic_nodes["semanticmodel.test.revenue"] + semantic_model = manifest.semantic_models["semanticmodel.test.revenue"] assert semantic_model.dimensions[0].type_params.time_granularity == TimeGranularity.WEEK diff --git a/tests/unit/test_manifest.py b/tests/unit/test_manifest.py index 6e611e96bc0..8ccd25cef39 100644 --- a/tests/unit/test_manifest.py +++ b/tests/unit/test_manifest.py @@ -336,7 +336,7 @@ def setUp(self): ), } - self.semantic_nodes = {} + self.semantic_models = {} for exposure in self.exposures.values(): exposure.validate(exposure.to_dict(omit_none=True)) @@ -366,7 +366,7 @@ def test_no_nodes(self): metrics={}, selectors={}, metadata=ManifestMetadata(generated_at=datetime.utcnow()), - semantic_nodes={}, + semantic_models={}, ) invocation_id = dbt.events.functions.EVENT_MANAGER.invocation_id @@ -392,7 +392,7 @@ def test_no_nodes(self): }, "docs": {}, "disabled": {}, - "semantic_nodes": {}, + "semantic_models": {}, }, ) @@ -476,7 +476,7 @@ def test_build_flat_graph(self): flat_metrics = flat_graph["metrics"] flat_nodes = flat_graph["nodes"] flat_sources = flat_graph["sources"] - flat_semantic_nodes = flat_graph["semantic_nodes"] + flat_semantic_models = flat_graph["semantic_models"] self.assertEqual( set(flat_graph), set( @@ -486,7 +486,7 @@ def test_build_flat_graph(self): "nodes", "sources", "metrics", - "semantic_nodes", + "semantic_models", ] ), ) @@ -495,7 +495,7 @@ def test_build_flat_graph(self): self.assertEqual(set(flat_metrics), set(self.metrics)) self.assertEqual(set(flat_nodes), set(self.nested_nodes)) self.assertEqual(set(flat_sources), set(self.sources)) - self.assertEqual(set(flat_semantic_nodes), set(self.semantic_nodes)) + self.assertEqual(set(flat_semantic_models), set(self.semantic_models)) for node in flat_nodes.values(): self.assertEqual(frozenset(node), REQUIRED_PARSED_NODE_KEYS) @@ -542,7 +542,7 @@ def test_no_nodes_with_metadata(self, mock_user): metadata=metadata, files={}, exposures={}, - semantic_nodes={}, + semantic_models={}, ) self.assertEqual( @@ -571,7 +571,7 @@ def test_no_nodes_with_metadata(self, mock_user): "env": {ENV_KEY_NAME: "value"}, }, "disabled": {}, - "semantic_nodes": {}, + "semantic_models": {}, }, ) @@ -919,7 +919,7 @@ def test_no_nodes(self): metadata=metadata, files={}, exposures={}, - semantic_nodes={}, + semantic_models={}, ) self.assertEqual( manifest.writable_manifest().to_dict(omit_none=True), @@ -943,7 +943,7 @@ def test_no_nodes(self): }, "docs": {}, "disabled": {}, - "semantic_nodes": {}, + "semantic_models": {}, }, ) @@ -1011,7 +1011,7 @@ def test_build_flat_graph(self): selectors={}, files={}, exposures={}, - semantic_nodes={}, + semantic_models={}, ) manifest.build_flat_graph() flat_graph = manifest.flat_graph @@ -1025,7 +1025,7 @@ def test_build_flat_graph(self): "metrics", "nodes", "sources", - "semantic_nodes", + "semantic_models", ] ), )