Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Unskip and rename test_expression_metric #8578

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
1ac2f04
Add docstrings to `contracts/graph/metrics.py` functions to document …
QMalcolm Sep 5, 2023
b84b575
Add typing to `reverse_dag_parsing` and update function to work on 1.…
QMalcolm Sep 6, 2023
a92e5aa
Add typing to `parent_metrics` and `parent_metrics_names`
QMalcolm Sep 6, 2023
c8389f9
Add typing to `base_metric_dependency` and `derived_metric_dependency…
QMalcolm Sep 6, 2023
8f0c5c6
Simplify implementations of `basic_metric_dependency` and `derived_me…
QMalcolm Sep 6, 2023
4e062cb
Add typing to `ResolvedMetricReference` initialization
QMalcolm Sep 6, 2023
4dfe539
Add typing to `derived_metric_dependency_graph`
QMalcolm Sep 6, 2023
4a11c9f
Simplify conditional controls in `ResolvedMetricReference` functions
QMalcolm Sep 6, 2023
41c8ad0
Don't recurse on over `depends_on` for non-derived metrics in `revers…
QMalcolm Sep 6, 2023
b476f28
Simplify `parent_metrics_names` by having it call `parent_metrics`
QMalcolm Sep 6, 2023
84c5091
Unskip `TestMetricHelperFunctions.test_derived_metric` and update fix…
QMalcolm Sep 6, 2023
b073cff
Add changie doc for metric helper function updates
QMalcolm Sep 6, 2023
711ca98
Get manifest in `test_derived_metric` from the parse dbt_run invocation
QMalcolm Sep 6, 2023
70091df
Remove `Relation` a intiatlization attribute for `ResolvedMetricRefer…
QMalcolm Sep 6, 2023
da99e59
Add return typing to class `__` functions of `ResolvedMetricReference`
QMalcolm Sep 6, 2023
f80b4e1
Move from `manifest.metrics.get` to `manifest.expect` in metric helpers
QMalcolm Sep 7, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/unreleased/Fixes-20230906-120212.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixes
body: Update metric helper functions to work with new semantic layer metrics
time: 2023-09-06T12:02:12.156534-07:00
custom:
Author: QMalcolm
Issue: "8134"
2 changes: 1 addition & 1 deletion core/dbt/context/providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -618,7 +618,7 @@
target_package=target_package,
)

return ResolvedMetricReference(target_metric, self.manifest, self.Relation)
return ResolvedMetricReference(target_metric, self.manifest)

Check warning on line 621 in core/dbt/context/providers.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/context/providers.py#L621

Added line #L621 was not covered by tests


# `var` implementations.
Expand Down
87 changes: 46 additions & 41 deletions core/dbt/contracts/graph/metrics.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,11 @@
from dbt.node_types import NodeType
from dbt.contracts.graph.manifest import Manifest, Metric
from dbt_semantic_interfaces.type_enums import MetricType

from typing import Any, Dict, Iterator, List


DERIVED_METRICS = [MetricType.DERIVED, MetricType.RATIO]
BASE_METRICS = [MetricType.SIMPLE, MetricType.CUMULATIVE]


class MetricReference(object):
Expand All @@ -17,76 +24,74 @@
for working with metrics (ie. __str__ and templating functions)
"""

def __init__(self, node, manifest, Relation):
def __init__(self, node: Metric, manifest: Manifest) -> None:
super().__init__(node.name, node.package_name)
self.node = node
self.manifest = manifest
self.Relation = Relation

def __getattr__(self, key):
def __getattr__(self, key) -> Any:
return getattr(self.node, key)

def __str__(self):
def __str__(self) -> str:
return f"{self.node.name}"

@classmethod
def parent_metrics(cls, metric_node, manifest):
def parent_metrics(cls, metric_node: Metric, manifest: Manifest) -> Iterator[Metric]:
"""For a given metric, yeilds all upstream metrics."""
yield metric_node

for parent_unique_id in metric_node.depends_on.nodes:
node = manifest.metrics.get(parent_unique_id)
if node and node.resource_type == NodeType.Metric:
node = manifest.expect(parent_unique_id)
if isinstance(node, Metric):

Check warning on line 45 in core/dbt/contracts/graph/metrics.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/contracts/graph/metrics.py#L44-L45

Added lines #L44 - L45 were not covered by tests
yield from cls.parent_metrics(node, manifest)

@classmethod
def parent_metrics_names(cls, metric_node, manifest):
yield metric_node.name

for parent_unique_id in metric_node.depends_on.nodes:
node = manifest.metrics.get(parent_unique_id)
if node and node.resource_type == NodeType.Metric:
yield from cls.parent_metrics_names(node, manifest)
def parent_metrics_names(cls, metric_node: Metric, manifest: Manifest) -> Iterator[str]:
"""For a given metric, yeilds all upstream metric names"""
for metric in cls.parent_metrics(metric_node, manifest):
yield metric.name

Check warning on line 52 in core/dbt/contracts/graph/metrics.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/contracts/graph/metrics.py#L51-L52

Added lines #L51 - L52 were not covered by tests

@classmethod
def reverse_dag_parsing(cls, metric_node, manifest, metric_depth_count):
if metric_node.calculation_method == "derived":
def reverse_dag_parsing(
cls, metric_node: Metric, manifest: Manifest, metric_depth_count: int
) -> Iterator[Dict[str, int]]:
"""For the given metric, yeilds dictionaries having {<metric_name>: <depth_from_initial_metric} of upstream derived metrics.

This function is intended as a helper function for other metric helper functions.
"""
if metric_node.type in DERIVED_METRICS:

Check warning on line 62 in core/dbt/contracts/graph/metrics.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/contracts/graph/metrics.py#L62

Added line #L62 was not covered by tests
yield {metric_node.name: metric_depth_count}
metric_depth_count = metric_depth_count + 1

for parent_unique_id in metric_node.depends_on.nodes:
node = manifest.metrics.get(parent_unique_id)
if (
node
and node.resource_type == NodeType.Metric
and node.calculation_method == "derived"
):
yield from cls.reverse_dag_parsing(node, manifest, metric_depth_count)
for parent_unique_id in metric_node.depends_on.nodes:
node = manifest.expect(parent_unique_id)
if isinstance(node, Metric):
yield from cls.reverse_dag_parsing(node, manifest, metric_depth_count + 1)

Check warning on line 68 in core/dbt/contracts/graph/metrics.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/contracts/graph/metrics.py#L65-L68

Added lines #L65 - L68 were not covered by tests

def full_metric_dependency(self):
"""Returns a unique list of all upstream metric names."""
to_return = list(set(self.parent_metrics_names(self.node, self.manifest)))
return to_return

def base_metric_dependency(self):
def base_metric_dependency(self) -> List[str]:
"""Returns a unique list of names for all upstream non-derived metrics."""
in_scope_metrics = list(self.parent_metrics(self.node, self.manifest))
base_metrics = {

Check warning on line 78 in core/dbt/contracts/graph/metrics.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/contracts/graph/metrics.py#L78

Added line #L78 was not covered by tests
metric.name for metric in in_scope_metrics if metric.type not in DERIVED_METRICS
}

to_return = []
for metric in in_scope_metrics:
if metric.calculation_method != "derived" and metric.name not in to_return:
to_return.append(metric.name)

return to_return
return list(base_metrics)

Check warning on line 82 in core/dbt/contracts/graph/metrics.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/contracts/graph/metrics.py#L82

Added line #L82 was not covered by tests

def derived_metric_dependency(self):
def derived_metric_dependency(self) -> List[str]:
"""Returns a unique list of names for all upstream derived metrics."""
in_scope_metrics = list(self.parent_metrics(self.node, self.manifest))
derived_metrics = {

Check warning on line 87 in core/dbt/contracts/graph/metrics.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/contracts/graph/metrics.py#L87

Added line #L87 was not covered by tests
metric.name for metric in in_scope_metrics if metric.type in DERIVED_METRICS
}

to_return = []
for metric in in_scope_metrics:
if metric.calculation_method == "derived" and metric.name not in to_return:
to_return.append(metric.name)

return to_return
return list(derived_metrics)

Check warning on line 91 in core/dbt/contracts/graph/metrics.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/contracts/graph/metrics.py#L91

Added line #L91 was not covered by tests

def derived_metric_dependency_depth(self):
def derived_metric_dependency_depth(self) -> List[Dict[str, int]]:
"""Returns a list of {<metric_name>: <depth_from_initial_metric>} for all upstream metrics."""
metric_depth_count = 1
to_return = list(self.reverse_dag_parsing(self.node, self.manifest, metric_depth_count))

Expand Down
24 changes: 14 additions & 10 deletions tests/functional/metrics/test_metric_helper_functions.py
Original file line number Diff line number Diff line change
@@ -1,34 +1,38 @@
import pytest

from dbt.tests.util import run_dbt, get_manifest
from dbt.tests.util import run_dbt
from dbt.contracts.graph.manifest import Manifest
from dbt.contracts.graph.metrics import ResolvedMetricReference

from tests.functional.metrics.fixtures import models_people_sql, basic_metrics_yml
from tests.functional.metrics.fixtures import (
models_people_sql,
basic_metrics_yml,
semantic_model_people_yml,
metricflow_time_spine_sql,
)


class TestMetricHelperFunctions:
@pytest.fixture(scope="class")
def models(self):
return {
"metrics.yml": basic_metrics_yml,
"semantic_people.yml": semantic_model_people_yml,
"metricflow_time_spine.sql": metricflow_time_spine_sql,
"people.sql": models_people_sql,
}

@pytest.mark.skip(
"TODO reactivate after we begin property hydrating metric `depends_on` and `refs`"
)
def test_expression_metric(
def test_derived_metric(
self,
project,
):

# initial parse
run_dbt(["compile"])
manifest = run_dbt(["parse"])
assert isinstance(manifest, Manifest)

# make sure all the metrics are in the manifest
manifest = get_manifest(project.project_root)
parsed_metric = manifest.metrics["metric.test.average_tenure_plus_one"]
testing_metric = ResolvedMetricReference(parsed_metric, manifest, None)
testing_metric = ResolvedMetricReference(parsed_metric, manifest)

full_metric_dependency = set(testing_metric.full_metric_dependency())
expected_full_metric_dependency = set(
Expand Down