Skip to content

Commit

Permalink
WIP - read everything to make sure it looks right, maybe split up com…
Browse files Browse the repository at this point in the history 
…mits
  • Loading branch information
@courtneyholcomb
courtneyholcomb committed Nov 15, 2024
1 parent b990eeb commit 528c8dc
Show file tree
Hide file tree
Showing 21 changed files with 3,039 additions and 3,434 deletions.
10 changes: 10 additions & 0 deletions metricflow-semantics/metricflow_semantics/specs/metric_spec.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,3 +49,13 @@ def has_time_offset(self) -> bool: # noqa: D102
def without_offset(self) -> MetricSpec:
"""Represents the metric spec with any time offsets removed."""
return MetricSpec(element_name=self.element_name, filter_spec_set=self.filter_spec_set, alias=self.alias)

def with_new_filter_spec_set(self, new_filter_spec_set: WhereFilterSpecSet) -> MetricSpec:
"""Represents the metric spec with a new filter spec set."""
return MetricSpec(
element_name=self.element_name,
filter_spec_set=new_filter_spec_set,
alias=self.alias,
offset_window=self.offset_window,
offset_to_grain=self.offset_to_grain,
)
10 changes: 10 additions & 0 deletions metricflow-semantics/metricflow_semantics/specs/where_filter/where_filter_spec_set.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,13 @@ def merge(self, other: WhereFilterSpecSet) -> WhereFilterSpecSet:
metric_level_filter_specs=self.metric_level_filter_specs + other.metric_level_filter_specs,
query_level_filter_specs=self.query_level_filter_specs + other.query_level_filter_specs,
)

def replace_query_level_filter_specs(
self, new_query_level_filter_specs: Tuple[WhereFilterSpec, ...]
) -> WhereFilterSpecSet:
"""Return the same set but with the query level filter specs replaced."""
return WhereFilterSpecSet(
measure_level_filter_specs=self.measure_level_filter_specs,
metric_level_filter_specs=self.metric_level_filter_specs,
query_level_filter_specs=new_query_level_filter_specs,
)
2 changes: 1 addition & 1 deletion ...cs/metricflow_semantics/test_helpers/semantic_manifest_yamls/simple_manifest/metrics.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -724,7 +724,7 @@ metric:
description: bookings metric offset twice with tiered filters
type: derived
type_params:
expr: bookings
expr: bookings_offset_once
metrics:
- name: bookings_offset_once
offset_window: 1 month
Expand Down
4 changes: 3 additions & 1 deletion metricflow/dataflow/builder/builder_cache.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
from __future__ import annotations

from dataclasses import dataclass
from typing import Optional
from typing import Optional, Sequence

from metricflow_semantics.collection_helpers.lru_cache import LruCache
from metricflow_semantics.specs.linkable_spec_set import LinkableSpecSet
from metricflow_semantics.specs.metric_spec import MetricSpec
from metricflow_semantics.specs.where_filter.where_filter_spec import WhereFilterSpec
from metricflow_semantics.specs.where_filter.where_filter_transform import WhereSpecFactory

from metricflow.dataflow.builder.measure_spec_properties import MeasureSpecProperties
Expand Down Expand Up @@ -39,6 +40,7 @@ class BuildAnyMetricOutputNodeParameterSet:
filter_spec_factory: WhereSpecFactory
predicate_pushdown_state: PredicatePushdownState
for_group_by_source_node: bool
time_spine_filter_specs: Sequence[WhereFilterSpec] = ()


class DataflowPlanBuilderCache:
Expand Down
187 changes: 95 additions & 92 deletions metricflow/dataflow/builder/dataflow_plan_builder.py
View file

Large diffs are not rendered by default.

729 changes: 313 additions & 416 deletions ...yPlan/DuckDB/test_offset_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql
View file

Large diffs are not rendered by default.

51 changes: 25 additions & 26 deletions ...DB/test_offset_metric_with_custom_granularity_filter_not_in_group_by__plan0_optimized.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,37 +7,36 @@ SELECT
metric_time__day
, bookings_5_days_ago AS bookings_5_day_lag
FROM (
-- Constrain Output with WHERE
-- Join to Time Spine Dataset
-- Pass Only Elements: ['bookings', 'metric_time__day']
-- Aggregate Measures
-- Compute Metrics via Expressions
SELECT
metric_time__day
, SUM(bookings) AS bookings_5_days_ago
subq_11.metric_time__day AS metric_time__day
, SUM(subq_10.bookings) AS bookings_5_days_ago
FROM (
-- Join to Time Spine Dataset
-- Join to Custom Granularity Dataset
-- Filter Time Spine
SELECT
subq_13.ds AS metric_time__day
, subq_11.bookings AS bookings
, subq_14.martian_day AS metric_time__martian_day
FROM ***************************.mf_time_spine subq_13
INNER JOIN (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
metric_time__day
FROM (
-- Time Spine
SELECT
DATE_TRUNC('day', ds) AS metric_time__day
, 1 AS bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_11
ON
subq_13.ds - INTERVAL 5 day = subq_11.metric_time__day
LEFT OUTER JOIN
***************************.mf_time_spine subq_14
ON
subq_13.ds = subq_14.ds
) subq_15
WHERE metric_time__martian_day = '2020-01-01'
ds AS metric_time__day
, martian_day AS metric_time__martian_day
FROM ***************************.mf_time_spine subq_12
) subq_13
WHERE metric_time__martian_day = '2020-01-01'
) subq_11
INNER JOIN (
-- Read Elements From Semantic Model 'bookings_source'
-- Metric Time Dimension 'ds'
SELECT
DATE_TRUNC('day', ds) AS metric_time__day
, 1 AS bookings
FROM ***************************.fct_bookings bookings_source_src_28000
) subq_10
ON
subq_11.metric_time__day - INTERVAL 5 day = subq_10.metric_time__day
GROUP BY
metric_time__day
) subq_19
subq_11.metric_time__day
) subq_17
145 changes: 32 additions & 113 deletions ...Plan/test_offset_to_grain_metric_filter_and_query_have_different_granularities__dfp_0.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,54 +59,13 @@ docstring:
<ComputeMetricsNode>
<!-- description = 'Compute Metrics via Expressions' -->
<!-- node_id = NodeId(id_str='cm_0') -->
<!-- metric_spec = -->
<!-- MetricSpec( -->
<!-- element_name='bookings', -->
<!-- filter_spec_set=WhereFilterSpecSet( -->
<!-- query_level_filter_specs=( -->
<!-- WhereFilterSpec( -->
<!-- where_sql="metric_time__day = '2020-01-01'", -->
<!-- bind_parameters=SqlBindParameterSet(), -->
<!-- linkable_element_unions=( -->
<!-- LinkableElementUnion( -->
<!-- linkable_dimension=LinkableDimension( -->
<!-- properties=( -->
<!-- METRIC_TIME, -->
<!-- ), -->
<!-- defined_in_semantic_model=SemanticModelReference( -->
<!-- semantic_model_name='bookings_source', -->
<!-- ), -->
<!-- element_name='metric_time', -->
<!-- dimension_type=TIME, -->
<!-- join_path=SemanticModelJoinPath( -->
<!-- left_semantic_model_reference=SemanticModelReference( -->
<!-- semantic_model_name='bookings_source', -->
<!-- ), -->
<!-- ), -->
<!-- time_granularity=ExpandedTimeGranularity( -->
<!-- name='day', -->
<!-- base_granularity=DAY, -->
<!-- ), -->
<!-- ), -->
<!-- ), -->
<!-- ), -->
<!-- linkable_spec_set=LinkableSpecSet( -->
<!-- time_dimension_specs=( -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity( -->
<!-- name='day', -->
<!-- base_granularity=DAY, -->
<!-- ), -->
<!-- ), -->
<!-- ), -->
<!-- ), -->
<!-- ), -->
<!-- ), -->
<!-- ), -->
<!-- alias='bookings_start_of_month', -->
<!-- offset_to_grain=MONTH, -->
<!-- ) -->
<!-- metric_spec = -->
<!-- MetricSpec( -->
<!-- element_name='bookings', -->
<!-- filter_spec_set=WhereFilterSpecSet(), -->
<!-- alias='bookings_start_of_month', -->
<!-- offset_to_grain=MONTH, -->
<!-- ) -->
<AggregateMeasuresNode>
<!-- description = 'Aggregate Measures' -->
<!-- node_id = NodeId(id_str='am_0') -->
Expand All @@ -120,71 +79,31 @@ docstring:
<!-- time_granularity=ExpandedTimeGranularity(name='month', base_granularity=MONTH), -->
<!-- ) -->
<!-- distinct = False -->
<WhereConstraintNode>
<!-- description = 'Constrain Output with WHERE' -->
<!-- node_id = NodeId(id_str='wcc_0') -->
<!-- where_condition = -->
<!-- WhereFilterSpec( -->
<!-- where_sql="metric_time__day = '2020-01-01'", -->
<!-- bind_parameters=SqlBindParameterSet(), -->
<!-- linkable_element_unions=( -->
<!-- LinkableElementUnion( -->
<!-- linkable_dimension=LinkableDimension( -->
<!-- properties=(METRIC_TIME,), -->
<!-- defined_in_semantic_model=SemanticModelReference( -->
<!-- semantic_model_name='bookings_source', -->
<!-- ), -->
<!-- element_name='metric_time', -->
<!-- dimension_type=TIME, -->
<!-- join_path=SemanticModelJoinPath( -->
<!-- left_semantic_model_reference=SemanticModelReference( -->
<!-- semantic_model_name='bookings_source', -->
<!-- ), -->
<!-- ), -->
<!-- time_granularity=ExpandedTimeGranularity( -->
<!-- name='day', -->
<!-- base_granularity=DAY, -->
<!-- ), -->
<!-- ), -->
<!-- ), -->
<!-- ), -->
<!-- linkable_spec_set=LinkableSpecSet( -->
<!-- time_dimension_specs=( -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity( -->
<!-- name='day', -->
<!-- base_granularity=DAY, -->
<!-- ), -->
<!-- ), -->
<!-- ), -->
<!-- ), -->
<!-- ) -->
<JoinToTimeSpineNode>
<!-- description = 'Join to Time Spine Dataset' -->
<!-- node_id = NodeId(id_str='jts_0') -->
<!-- requested_agg_time_dimension_specs = -->
<!-- ( -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity(name='month', base_granularity=MONTH), -->
<!-- ), -->
<!-- ) -->
<!-- use_custom_agg_time_dimension = False -->
<!-- join_type = INNER -->
<!-- offset_to_grain = MONTH -->
<MetricTimeDimensionTransformNode>
<!-- description = "Metric Time Dimension 'ds'" -->
<!-- node_id = NodeId(id_str='sma_28009') -->
<!-- aggregation_time_dimension = 'ds' -->
<ReadSqlSourceNode>
<!-- description = "Read From SemanticModelDataSet('bookings_source')" -->
<!-- node_id = NodeId(id_str='rss_28020') -->
<!-- data_set = SemanticModelDataSet('bookings_source') -->
</ReadSqlSourceNode>
</MetricTimeDimensionTransformNode>
</JoinToTimeSpineNode>
</WhereConstraintNode>
<JoinToTimeSpineNode>
<!-- description = 'Join to Time Spine Dataset' -->
<!-- node_id = NodeId(id_str='jts_0') -->
<!-- requested_agg_time_dimension_specs = -->
<!-- ( -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity(name='month', base_granularity=MONTH), -->
<!-- ), -->
<!-- ) -->
<!-- use_custom_agg_time_dimension = False -->
<!-- join_type = INNER -->
<!-- offset_to_grain = MONTH -->
<!-- time_spine_filters = ["metric_time__day = '2020-01-01'",] -->
<MetricTimeDimensionTransformNode>
<!-- description = "Metric Time Dimension 'ds'" -->
<!-- node_id = NodeId(id_str='sma_28009') -->
<!-- aggregation_time_dimension = 'ds' -->
<ReadSqlSourceNode>
<!-- description = "Read From SemanticModelDataSet('bookings_source')" -->
<!-- node_id = NodeId(id_str='rss_28020') -->
<!-- data_set = SemanticModelDataSet('bookings_source') -->
</ReadSqlSourceNode>
</MetricTimeDimensionTransformNode>
</JoinToTimeSpineNode>
</FilterElementsNode>
</AggregateMeasuresNode>
</ComputeMetricsNode>
Expand Down
Loading

0 comments on commit 528c8dc

Please sign in to comment.