Skip to content

Commit

Permalink
partial parsing for groups
Browse files Browse the repository at this point in the history
  • Loading branch information
@MichelleArk
MichelleArk committed Feb 13, 2023
1 parent 24ec6a2 commit 4911ccc
Show file tree
Hide file tree
Showing 4 changed files with 120 additions and 28 deletions.
1 change: 1 addition & 0 deletions core/dbt/contracts/files.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ class SchemaSourceFile(BaseSourceFile):
sources: List[str] = field(default_factory=list)
exposures: List[str] = field(default_factory=list)
metrics: List[str] = field(default_factory=list)
groups: List[str] = field(default_factory=list)
# node patches contain models, seeds, snapshots, analyses
ndp: List[str] = field(default_factory=list)
# any macro patches in this file by macro unique_id.
Expand Down
2 changes: 2 additions & 0 deletions core/dbt/contracts/graph/manifest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1074,6 +1074,8 @@ def add_node(self, source_file: AnySourceFile, node: ManifestNode, test_from=Non
source_file.metrics.append(node.unique_id)
if isinstance(node, Exposure):
source_file.exposures.append(node.unique_id)
if isinstance(node, Group):
source_file.groups.append(node.unique_id)
else:
source_file.nodes.append(node.unique_id)

Expand Down
36 changes: 36 additions & 0 deletions core/dbt/parser/partial.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -746,6 +746,29 @@ def handle_schema_file_changes(self, schema_file, saved_yaml_dict, new_yaml_dict
self.delete_schema_metric(schema_file, elem)
self.merge_patch(schema_file, dict_key, elem)

# groups
dict_key = "groups"
group_diff = self.get_diff_for("groups", saved_yaml_dict, new_yaml_dict)
if group_diff["changed"]:
for group in group_diff["changed"]:
self.delete_schema_group(schema_file, group)
self.merge_patch(schema_file, dict_key, group)
if group_diff["deleted"]:
for metric in group_diff["deleted"]:
self.delete_schema_group(schema_file, group)
if group_diff["added"]:
for metric in group_diff["added"]:
self.merge_patch(schema_file, dict_key, group)
# Handle schema file updates due to env_var changes
if dict_key in env_var_changes and dict_key in new_yaml_dict:
for name in env_var_changes[dict_key]:
if name in group_diff["changed_or_deleted_names"]:
continue
elem = self.get_schema_element(new_yaml_dict[dict_key], name)
if elem:
self.delete_schema_group(schema_file, elem)
self.merge_patch(schema_file, dict_key, elem)

# Take a "section" of the schema file yaml dictionary from saved and new schema files
# and determine which parts have changed
def get_diff_for(self, key, saved_yaml_dict, new_yaml_dict):
Expand Down Expand Up @@ -903,6 +926,19 @@ def delete_schema_exposure(self, schema_file, exposure_dict):
elif unique_id in self.saved_manifest.disabled:
self.delete_disabled(unique_id, schema_file.file_id)

# groups are created only from schema files, so just delete the group
def delete_schema_group(self, schema_file, group_dict):
group_name = group_dict["name"]
groups = schema_file.groups.copy()
for unique_id in groups:
if unique_id in self.saved_manifest.groups:
group = self.saved_manifest.groups[unique_id]
if group.name == group_name:
self.deleted_manifest.groups[unique_id] = self.saved_manifest.groups.pop(
unique_id
)
schema_file.groups.remove(unique_id)

# metrics are created only from schema files, but also can be referred to by other nodes
def delete_schema_metric(self, schema_file, metric_dict):
metric_name = metric_dict["name"]
Expand Down
109 changes: 81 additions & 28 deletions schemas/dbt/manifest/v8.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
"docs",
"exposures",
"metrics",
"groups",
"selectors"
],
"properties": {
Expand Down Expand Up @@ -85,6 +86,13 @@
},
"description": "The metrics defined in the dbt project and its dependencies"
},
"groups": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/Group"
},
"description": "The groups defined in the dbt project"
},
"selectors": {
"type": "object",
"description": "The selectors defined in selectors.yml"
Expand Down Expand Up @@ -173,7 +181,7 @@
}
},
"additionalProperties": false,
"description": "WritableManifest(metadata: dbt.contracts.graph.manifest.ManifestMetadata, nodes: Mapping[str, Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode]], sources: Mapping[str, dbt.contracts.graph.nodes.SourceDefinition], macros: Mapping[str, dbt.contracts.graph.nodes.Macro], docs: Mapping[str, dbt.contracts.graph.nodes.Documentation], exposures: Mapping[str, dbt.contracts.graph.nodes.Exposure], metrics: Mapping[str, dbt.contracts.graph.nodes.Metric], selectors: Mapping[str, Any], disabled: Optional[Mapping[str, List[Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode, dbt.contracts.graph.nodes.SourceDefinition]]]], parent_map: Optional[Dict[str, List[str]]], child_map: Optional[Dict[str, List[str]]])",
"description": "WritableManifest(metadata: dbt.contracts.graph.manifest.ManifestMetadata, nodes: Mapping[str, Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode]], sources: Mapping[str, dbt.contracts.graph.nodes.SourceDefinition], macros: Mapping[str, dbt.contracts.graph.nodes.Macro], docs: Mapping[str, dbt.contracts.graph.nodes.Documentation], exposures: Mapping[str, dbt.contracts.graph.nodes.Exposure], metrics: Mapping[str, dbt.contracts.graph.nodes.Metric], groups: Mapping[str, dbt.contracts.graph.nodes.Group], selectors: Mapping[str, Any], disabled: Optional[Mapping[str, List[Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode, dbt.contracts.graph.nodes.SourceDefinition]]]], parent_map: Optional[Dict[str, List[str]]], child_map: Optional[Dict[str, List[str]]])",
"definitions": {
"ManifestMetadata": {
"type": "object",
Expand All @@ -190,7 +198,7 @@
"generated_at": {
"type": "string",
"format": "date-time",
"default": "2023-02-09T23:46:55.265899Z"
"default": "2023-02-13T17:41:47.731826Z"
},
"invocation_id": {
"oneOf": [
Expand All @@ -201,7 +209,7 @@
"type": "null"
}
],
"default": "e6a9b266-203d-4fec-93af-fb8f55423a6b"
"default": "2084ee8a-3962-4f49-99a1-4c06387a6e57"
},
"env": {
"type": "object",
Expand Down Expand Up @@ -406,7 +414,7 @@
},
"created_at": {
"type": "number",
"default": 1675986415.2675269
"default": 1676310107.733252
},
"config_call_dict": {
"type": "object",
Expand Down Expand Up @@ -953,7 +961,7 @@
},
"created_at": {
"type": "number",
"default": 1675986415.2685802
"default": 1676310107.734257
},
"config_call_dict": {
"type": "object",
Expand Down Expand Up @@ -1312,7 +1320,7 @@
},
"created_at": {
"type": "number",
"default": 1675986415.269182
"default": 1676310107.734807
},
"config_call_dict": {
"type": "object",
Expand Down Expand Up @@ -1569,7 +1577,7 @@
},
"created_at": {
"type": "number",
"default": 1675986415.2698119
"default": 1676310107.73539
},
"config_call_dict": {
"type": "object",
Expand Down Expand Up @@ -1816,7 +1824,7 @@
},
"created_at": {
"type": "number",
"default": 1675986415.2704
"default": 1676310107.735925
},
"config_call_dict": {
"type": "object",
Expand Down Expand Up @@ -2063,7 +2071,7 @@
},
"created_at": {
"type": "number",
"default": 1675986415.270981
"default": 1676310107.7364511
},
"config_call_dict": {
"type": "object",
Expand Down Expand Up @@ -2306,7 +2314,7 @@
},
"created_at": {
"type": "number",
"default": 1675986415.2716632
"default": 1676310107.7370791
},
"config_call_dict": {
"type": "object",
Expand Down Expand Up @@ -2577,7 +2585,7 @@
},
"created_at": {
"type": "number",
"default": 1675986415.272834
"default": 1676310107.738108
},
"config_call_dict": {
"type": "object",
Expand Down Expand Up @@ -3030,7 +3038,7 @@
},
"created_at": {
"type": "number",
"default": 1675986415.27388
"default": 1676310107.7390342
},
"config_call_dict": {
"type": "object",
Expand Down Expand Up @@ -3416,7 +3424,7 @@
},
"created_at": {
"type": "number",
"default": 1675986415.2750158
"default": 1676310107.740042
}
},
"additionalProperties": false,
Expand Down Expand Up @@ -3531,7 +3539,7 @@
"generated_at": {
"type": "string",
"format": "date-time",
"default": "2023-02-09T23:46:55.263337Z"
"default": "2023-02-13T17:41:47.729692Z"
},
"invocation_id": {
"oneOf": [
Expand All @@ -3542,7 +3550,7 @@
"type": "null"
}
],
"default": "e6a9b266-203d-4fec-93af-fb8f55423a6b"
"default": "2084ee8a-3962-4f49-99a1-4c06387a6e57"
},
"env": {
"type": "object",
Expand Down Expand Up @@ -3895,7 +3903,7 @@
},
"created_at": {
"type": "number",
"default": 1675986415.275425
"default": 1676310107.7403321
},
"supported_languages": {
"oneOf": [
Expand Down Expand Up @@ -4037,7 +4045,7 @@
]
},
"owner": {
"$ref": "#/definitions/ExposureOwner"
"$ref": "#/definitions/Owner"
},
"description": {
"type": "string",
Expand Down Expand Up @@ -4138,20 +4146,25 @@
},
"created_at": {
"type": "number",
"default": 1675986415.27617
"default": 1676310107.741005
}
},
"additionalProperties": false,
"description": "Exposure(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], type: dbt.contracts.graph.unparsed.ExposureType, owner: dbt.contracts.graph.unparsed.ExposureOwner, description: str = '', label: Optional[str] = None, maturity: Optional[dbt.contracts.graph.unparsed.MaturityType] = None, meta: Dict[str, Any] = <factory>, tags: List[str] = <factory>, config: dbt.contracts.graph.model_config.ExposureConfig = <factory>, unrendered_config: Dict[str, Any] = <factory>, url: Optional[str] = None, depends_on: dbt.contracts.graph.nodes.DependsOn = <factory>, refs: List[List[str]] = <factory>, sources: List[List[str]] = <factory>, metrics: List[List[str]] = <factory>, created_at: float = <factory>)"
"description": "Exposure(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], type: dbt.contracts.graph.unparsed.ExposureType, owner: dbt.contracts.graph.unparsed.Owner, description: str = '', label: Optional[str] = None, maturity: Optional[dbt.contracts.graph.unparsed.MaturityType] = None, meta: Dict[str, Any] = <factory>, tags: List[str] = <factory>, config: dbt.contracts.graph.model_config.ExposureConfig = <factory>, unrendered_config: Dict[str, Any] = <factory>, url: Optional[str] = None, depends_on: dbt.contracts.graph.nodes.DependsOn = <factory>, refs: List[List[str]] = <factory>, sources: List[List[str]] = <factory>, metrics: List[List[str]] = <factory>, created_at: float = <factory>)"
},
"ExposureOwner": {
"Owner": {
"type": "object",
"required": [
"email"
],
"required": [],
"properties": {
"email": {
"type": "string"
"oneOf": [
{
"type": "string"
},
{
"type": "null"
}
]
},
"name": {
"oneOf": [
Expand All @@ -4164,8 +4177,8 @@
]
}
},
"additionalProperties": false,
"description": "ExposureOwner(email: str, name: Optional[str] = None)"
"additionalProperties": true,
"description": "Owner(_extra: Dict[str, Any] = <factory>, email: Optional[str] = None, name: Optional[str] = None)"
},
"ExposureConfig": {
"type": "object",
Expand Down Expand Up @@ -4355,7 +4368,7 @@
},
"created_at": {
"type": "number",
"default": 1675986415.2768772
"default": 1676310107.741645
}
},
"additionalProperties": false,
Expand Down Expand Up @@ -4427,8 +4440,48 @@
},
"additionalProperties": true,
"description": "MetricConfig(_extra: Dict[str, Any] = <factory>, enabled: bool = True)"
},
"Group": {
"type": "object",
"required": [
"name",
"resource_type",
"package_name",
"path",
"original_file_path",
"unique_id",
"owner"
],
"properties": {
"name": {
"type": "string"
},
"resource_type": {
"type": "string",
"enum": [
"group"
]
},
"package_name": {
"type": "string"
},
"path": {
"type": "string"
},
"original_file_path": {
"type": "string"
},
"unique_id": {
"type": "string"
},
"owner": {
"$ref": "#/definitions/Owner"
}
},
"additionalProperties": false,
"description": "Group(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, owner: dbt.contracts.graph.unparsed.Owner)"
}
},
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://schemas.getdbt.com/dbt/manifest/v8.json"
}
}

0 comments on commit 4911ccc

Please sign in to comment.