-
Notifications
You must be signed in to change notification settings - Fork 263
fix(dbt): substitute DERIVED metric references in a single pass #353
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -644,6 +644,139 @@ def test_derived_metric_uses_alias_for_substitution(self) -> None: | |
| profit_ossie = next(m for m in _ossie_metrics(result) if m.name == "profit") | ||
| assert profit_ossie.expression.dialects[0].expression == "SUM(orders.amount) - SUM(orders.cost_amount)" | ||
|
|
||
| def test_derived_metric_does_not_re_expand_an_inlined_reference(self) -> None: | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I suggest to add a test for a DERIVED metric whose That's the case where the dict-based
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Added two tests in 6952a43: one that rejects the same reference listed twice with differing filters, and one that accepts a redundant identical duplicate (revenue + revenue). Went with reject rather than asserting last-wins ordering. |
||
| """A reference is substituted once, even when its name appears in already-inlined text. | ||
|
|
||
| `gross` inlines to `SUM(orders.net)`, which contains the name of the second | ||
| input metric. Substituting references one at a time would expand `net` inside | ||
| that text as well. | ||
| """ | ||
| sm = semantic_model_with_guaranteed_meta( | ||
| name="orders", | ||
| measures=[ | ||
| _measure("gross", agg=AggregationType.SUM, expr="net"), | ||
| _measure("net", agg=AggregationType.SUM, expr="net_amount"), | ||
| ], | ||
| ) | ||
| gross_m = _simple_metric("gross", "gross") | ||
| net_m = _simple_metric("net", "net") | ||
| margin = PydanticMetric( | ||
| name="margin", | ||
| description=None, | ||
| type=MetricType.DERIVED, | ||
| type_params=PydanticMetricTypeParams( | ||
| expr="gross - net", | ||
| metrics=[ | ||
| PydanticMetricInput(name="gross"), | ||
| PydanticMetricInput(name="net"), | ||
| ], | ||
| ), | ||
| filter=None, | ||
| metadata=default_meta(), | ||
| config=None, | ||
| ) | ||
| result = ( | ||
| MSIToOssieConverter().convert(_manifest(semantic_models=[sm], metrics=[gross_m, net_m, margin])).output | ||
| ) | ||
|
|
||
| margin_ossie = next(m for m in _ossie_metrics(result) if m.name == "margin") | ||
| assert margin_ossie.expression.dialects[0].expression == "SUM(orders.net) - SUM(orders.net_amount)" | ||
|
|
||
| def test_derived_metric_preserves_backslashes_from_a_filter(self) -> None: | ||
| """Backslashes in an inlined expression survive substitution verbatim. | ||
|
|
||
| A resolved expression is inserted as a literal, not as a `re.sub` replacement | ||
| template, so an escape sequence such as `\\b` in a filter's SQL is not | ||
| reinterpreted (`\\b` would otherwise become a backspace character). | ||
| """ | ||
| sm = semantic_model_with_guaranteed_meta( | ||
| name="orders", | ||
| measures=[_measure("revenue", agg=AggregationType.SUM, expr="amount")], | ||
| ) | ||
| revenue_m = _simple_metric("revenue", "revenue") | ||
| revenue_m.filter = _filter(r"{{ Dimension('order__path') }} LIKE 'a\b'") | ||
| scaled = PydanticMetric( | ||
| name="scaled", | ||
| description=None, | ||
| type=MetricType.DERIVED, | ||
| type_params=PydanticMetricTypeParams( | ||
| expr="revenue * 2", | ||
| metrics=[PydanticMetricInput(name="revenue")], | ||
| ), | ||
| filter=None, | ||
| metadata=default_meta(), | ||
| config=None, | ||
| ) | ||
| result = MSIToOssieConverter().convert(_manifest(semantic_models=[sm], metrics=[revenue_m, scaled])).output | ||
|
|
||
| revenue_ossie = next(m for m in _ossie_metrics(result) if m.name == "revenue") | ||
| scaled_ossie = next(m for m in _ossie_metrics(result) if m.name == "scaled") | ||
| assert revenue_ossie.expression.dialects[0].expression == ( | ||
| r"SUM(CASE WHEN order__path LIKE 'a\b' THEN orders.amount END)" | ||
| ) | ||
| assert scaled_ossie.expression.dialects[0].expression == ( | ||
| r"SUM(CASE WHEN order__path LIKE 'a\b' THEN orders.amount END) * 2" | ||
| ) | ||
|
|
||
| def test_derived_metric_rejects_a_reference_listed_twice_with_differing_filters(self) -> None: | ||
| """An input metric listed twice under one reference, resolving differently, is ambiguous. | ||
|
|
||
| MetricFlow accepts this shape — `DerivedMetricRule._validate_alias_collision` | ||
| only compares entries that set an alias — so the converter has to reject it | ||
| rather than silently pick one of the two filters. | ||
| """ | ||
| sm = semantic_model_with_guaranteed_meta( | ||
| name="orders", | ||
| measures=[_measure("revenue", agg=AggregationType.SUM, expr="amount")], | ||
| ) | ||
| revenue_m = _simple_metric("revenue", "revenue") | ||
| both = PydanticMetric( | ||
| name="both", | ||
| description=None, | ||
| type=MetricType.DERIVED, | ||
| type_params=PydanticMetricTypeParams( | ||
| expr="revenue", | ||
| metrics=[ | ||
| PydanticMetricInput(name="revenue", filter=_filter("{{ Dimension('order__region') }} = 'EU'")), | ||
| PydanticMetricInput(name="revenue", filter=_filter("{{ Dimension('order__region') }} = 'US'")), | ||
| ], | ||
| ), | ||
| filter=None, | ||
| metadata=default_meta(), | ||
| config=None, | ||
| ) | ||
| with pytest.raises(ValueError, match="listed more than once"): | ||
| MSIToOssieConverter().convert(_manifest(semantic_models=[sm], metrics=[revenue_m, both])) | ||
|
|
||
| def test_derived_metric_accepts_a_reference_listed_twice_resolving_identically(self) -> None: | ||
| """A redundant duplicate is not ambiguous: both occurrences resolve to the same SQL.""" | ||
| sm = semantic_model_with_guaranteed_meta( | ||
| name="orders", | ||
| measures=[_measure("revenue", agg=AggregationType.SUM, expr="amount")], | ||
| ) | ||
| revenue_m = _simple_metric("revenue", "revenue") | ||
| doubled = PydanticMetric( | ||
| name="doubled", | ||
| description=None, | ||
| type=MetricType.DERIVED, | ||
| type_params=PydanticMetricTypeParams( | ||
| expr="revenue + revenue", | ||
| metrics=[ | ||
| PydanticMetricInput(name="revenue"), | ||
| PydanticMetricInput(name="revenue"), | ||
| ], | ||
| ), | ||
| filter=None, | ||
| metadata=default_meta(), | ||
| config=None, | ||
| ) | ||
| result = ( | ||
| MSIToOssieConverter().convert(_manifest(semantic_models=[sm], metrics=[revenue_m, doubled])).output | ||
| ) | ||
|
|
||
| doubled_ossie = next(m for m in _ossie_metrics(result) if m.name == "doubled") | ||
| assert doubled_ossie.expression.dialects[0].expression == "SUM(orders.amount) + SUM(orders.amount)" | ||
|
|
||
| def test_derived_metric_nested(self, snapshot: SnapshotAssertion) -> None: | ||
| sm = semantic_model_with_guaranteed_meta( | ||
| name="orders", | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think that collecting into
replacements[ref] = resolvedchanges the collision behavior from "first entry wins" to "last entry wins". I'm not sure it's addressed anywhere.if a derived metric lists the same input metric twice with different per-input filters and no alias, this dict collapses to one entry and both occurrences get F2's SQL. This input shape isn't rejected upstream: MetricFlow's
DerivedMetricRule._validate_alias_collisiononly compares entries that have analiasset, so two unaliased duplicates sail through validation.Worth either erroring on a duplicate unaliased ref here, or confirming this silent overwrite is intentional (and so it should be documented).
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanx @jbonofre for the review. Good catch — silent last-wins wasn’t intentional.
MetricFlowdoesn’t reject this shape (DerivedMetricRule._validate_alias_collisiononly compares aliased entries), and with a single token in expr neither resolution is more correct. Raised aValueErrorwhen the same reference resolves differently, and pointed at distinct aliases as the fix. Identical duplicates stay accepted since they’re redundant, not ambiguous. Covered in 6952a43.