From ae2121d5e1cedd15211948ff1e954e6e5c1885c7 Mon Sep 17 00:00:00 2001 From: ota2000 <16278388+OTA2000@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:09:26 +0900 Subject: [PATCH 01/33] Add Lightdash converter Bidirectional converter between Ossie documents and Lightdash semantic definitions (dbt schema.yml meta blocks): datasets/fields/metrics/ relationships map to models/columns/meta.metrics/meta.joins, and Lightdash presentation attributes round-trip through custom_extensions with vendor_name "lightdash". Follows the apache-ossie- uv packaging layout and ships tests off the TPC-DS example, including a structural round-trip. --- converters/lightdash/.gitignore | 2 + converters/lightdash/README.md | 84 +++++ converters/lightdash/pyproject.toml | 61 ++++ .../lightdash/src/ossie_lightdash/__init__.py | 32 ++ .../lightdash/src/ossie_lightdash/cli.py | 97 ++++++ .../src/ossie_lightdash/converter_issues.py | 54 +++ .../src/ossie_lightdash/expression_utils.py | 127 +++++++ .../src/ossie_lightdash/lightdash_to_osi.py | 312 ++++++++++++++++++ .../src/ossie_lightdash/osi_to_lightdash.py | 225 +++++++++++++ converters/lightdash/tests/__init__.py | 16 + .../lightdash/tests/test_lightdash_to_osi.py | 187 +++++++++++ .../lightdash/tests/test_osi_to_lightdash.py | 200 +++++++++++ .../lightdash/tests/test_tpcds_roundtrip.py | 121 +++++++ converters/lightdash/uv.lock | 304 +++++++++++++++++ 14 files changed, 1822 insertions(+) create mode 100644 converters/lightdash/.gitignore create mode 100644 converters/lightdash/README.md create mode 100644 converters/lightdash/pyproject.toml create mode 100644 converters/lightdash/src/ossie_lightdash/__init__.py create mode 100644 converters/lightdash/src/ossie_lightdash/cli.py create mode 100644 converters/lightdash/src/ossie_lightdash/converter_issues.py create mode 100644 converters/lightdash/src/ossie_lightdash/expression_utils.py create mode 100644 converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py create mode 100644 converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py create mode 100644 converters/lightdash/tests/__init__.py create mode 100644 converters/lightdash/tests/test_lightdash_to_osi.py create mode 100644 converters/lightdash/tests/test_osi_to_lightdash.py create mode 100644 converters/lightdash/tests/test_tpcds_roundtrip.py create mode 100644 converters/lightdash/uv.lock diff --git a/converters/lightdash/.gitignore b/converters/lightdash/.gitignore new file mode 100644 index 00000000..a230a78a --- /dev/null +++ b/converters/lightdash/.gitignore @@ -0,0 +1,2 @@ +.venv/ +__pycache__/ diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md new file mode 100644 index 00000000..85200921 --- /dev/null +++ b/converters/lightdash/README.md @@ -0,0 +1,84 @@ + + +# Apache Ossie <> Lightdash converter + +Bidirectional converter between Ossie documents and +[Lightdash](https://github.com/lightdash/lightdash) semantic definitions. +Lightdash reads its semantic layer from dbt `schema.yml` files: dimensions and +metrics are declared per column (and per model) under `meta`. This converter +translates between that shape and Ossie. + +- **Export** (`osi_to_lightdash`): Ossie document → a dbt `schema.yml`-shaped + dictionary with Lightdash `meta` blocks, ready to merge into a dbt project. +- **Import** (`lightdash_to_osi`): a Lightdash-flavoured `schema.yml` → an + Ossie document, as a migration path for teams with an existing installed + base of Lightdash metrics. + +``` +ossie-lightdash export semantic_model.yaml schema.yml +ossie-lightdash import schema.yml semantic_model.json --database analytics_db --schema marts +``` + +## Mapping + +| Ossie | Lightdash (dbt meta) | +| ----- | -------------------- | +| `dataset` | dbt model (`name` = table part of `source`) | +| `dataset.source` | assembled on import from `--database` / `--schema` / model name | +| `field` (no `dimension`) | plain column entry | +| `field` with `dimension` | `columns[].meta.dimension` (`is_time` ↔ `type: date/timestamp`; an empty `dimension: {}` marks a categorical dimension) | +| `field.label` / `.description` | `meta.dimension.label` / column `description` | +| `field.expression` (≠ column name) | `meta.dimension.sql` (`dataset.col` ↔ `${TABLE}.col`) | +| `metric` with single-aggregation expression (`SUM(ds.col)`, `COUNT(DISTINCT ds.col)`, ...) | column-level `meta.metrics.` with a typed metric (`sum`, `count_distinct`, ...) | +| `metric` with any other single-dataset expression | model-level `meta.metrics.` with `type: number` + `sql` | +| `relationship` | `meta.joins` (`sql_on` built from / parsed into column pairs) | +| Lightdash presentation attributes (`label`, `format`, `round`, `compact`, `group_label`, `hidden`, `percentile`, ...) | `custom_extensions` with `vendor_name: "lightdash"`; on export the extension data is overlaid onto the generated definition and always wins | + +Expressions are written under the `ANSI_SQL` dialect. Warehouse-specific +dialects (e.g. `BIGQUERY`) can be added once the surrounding tooling resolves +them. + +## Recommended source shape for dbt-native flows + +If the Ossie documents are also consumed by dbt's native OSI parsing, prefer +importing **without** `--database` (i.e. `schema.table` sources): the database +is usually environment-dependent in dbt projects, and a database-less source +keeps one document valid across environments (see +[dbt-core#15649](https://github.com/dbt-labs/dbt-core/issues/15649)). +Omitting `--schema` as well is reported as a `SOURCE_UNQUALIFIED` issue. + +## Known limitations + +- **Cross-dataset metrics are dropped on export** (with a + `CROSS_DATASET_METRIC_DROPPED` issue): a Lightdash model metric cannot + reference other tables. +- **Percentile metrics** keep `type` / `percentile` in the `lightdash` + extension (Ossie expressions cannot express them faithfully) and re-export + as model-level metrics. +- **`primary_key` / `unique_keys` are not exported** — Lightdash has no + corresponding concept — and consequently cannot be reconstructed on import. +- **`ai_context` is not carried** into Lightdash meta. +- **Standalone Lightdash YAML projects** (Lightdash without dbt) are not + supported yet; the converter targets the dbt-meta flavour. +- Custom extensions from other vendors are ignored on export (reported as + `FOREIGN_EXTENSION_IGNORED`); they remain untouched in the Ossie document. +- Documents are emitted at the current in-repo spec version. Note that + dbt-core 1.12's native OSI parsing accepts spec versions `0.1.0` / `0.1.1` + only. diff --git a/converters/lightdash/pyproject.toml b/converters/lightdash/pyproject.toml new file mode 100644 index 00000000..ec94ddcd --- /dev/null +++ b/converters/lightdash/pyproject.toml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "pytest>=8.0", +] + +[project] +name = "apache-ossie-lightdash" +version = "0.1.0.dev0" +description = "Lightdash (dbt meta semantic definitions) <> Apache Ossie converter" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] +requires-python = ">=3.11" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Lightdash" +] +dependencies = [ + "apache-ossie>=0.2.0.dev0", + "PyYAML>=6.0", +] + +[project.scripts] +ossie-lightdash = "ossie_lightdash.cli:main" + +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_lightdash"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.uv.sources] +# apache-ossie is not yet published to PyPI; resolve it from the in-repo +# package for now. Remove this block once apache-ossie published to PyPI. +apache-ossie = { path = "../../python", editable = true} diff --git a/converters/lightdash/src/ossie_lightdash/__init__.py b/converters/lightdash/src/ossie_lightdash/__init__.py new file mode 100644 index 00000000..afeea775 --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/__init__.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from ossie_lightdash.converter_issues import ( + ConverterIssue, + ConverterIssueType, + ConverterResult, +) +from ossie_lightdash.lightdash_to_osi import LightdashToOSIConverter +from ossie_lightdash.osi_to_lightdash import OSIToLightdashConverter + +__all__ = [ + "ConverterIssue", + "ConverterIssueType", + "ConverterResult", + "LightdashToOSIConverter", + "OSIToLightdashConverter", +] diff --git a/converters/lightdash/src/ossie_lightdash/cli.py b/converters/lightdash/src/ossie_lightdash/cli.py new file mode 100644 index 00000000..b42bd3c3 --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/cli.py @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Command line interface for the Ossie <> Lightdash converter.""" + +import argparse +import json +import sys +from pathlib import Path + +import yaml + +from ossie import OSIDocument +from ossie_lightdash.lightdash_to_osi import LightdashToOSIConverter +from ossie_lightdash.osi_to_lightdash import OSIToLightdashConverter + + +def _read_document(path: Path) -> OSIDocument: + text = path.read_text(encoding="utf-8") + if path.suffix == ".json": + return OSIDocument.model_validate_json(text) + return OSIDocument.model_validate(yaml.safe_load(text)) + + +def _print_issues(issues) -> None: + for issue in issues: + print(f"[{issue.issue_type.value}] {issue.element_name}", file=sys.stderr) + + +def main() -> int: + parser = argparse.ArgumentParser(prog="ossie-lightdash") + subparsers = parser.add_subparsers(dest="command", required=True) + + export_parser = subparsers.add_parser( + "export", help="Ossie document (.json/.yaml) -> Lightdash dbt schema.yml" + ) + export_parser.add_argument("input", type=Path) + export_parser.add_argument("output", type=Path) + + import_parser = subparsers.add_parser( + "import", help="Lightdash dbt schema.yml -> Ossie document (.json/.yaml)" + ) + import_parser.add_argument("input", type=Path) + import_parser.add_argument("output", type=Path) + import_parser.add_argument("--database", default=None) + import_parser.add_argument("--schema", default=None) + import_parser.add_argument( + "--semantic-model-name", default="lightdash_semantic_model" + ) + + args = parser.parse_args() + + if args.command == "export": + result = OSIToLightdashConverter().convert(_read_document(args.input)) + args.output.write_text( + yaml.safe_dump(result.output, sort_keys=False, allow_unicode=True), + encoding="utf-8", + ) + else: + schema_yml = yaml.safe_load(args.input.read_text(encoding="utf-8")) + result = LightdashToOSIConverter().convert( + schema_yml, + database=args.database, + schema=args.schema, + semantic_model_name=args.semantic_model_name, + ) + document = result.output.model_dump(by_alias=True, exclude_none=True) + if args.output.suffix == ".json": + args.output.write_text( + json.dumps(document, indent=2, ensure_ascii=False), encoding="utf-8" + ) + else: + args.output.write_text( + yaml.safe_dump(document, sort_keys=False, allow_unicode=True), + encoding="utf-8", + ) + + _print_issues(result.issues) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py new file mode 100644 index 00000000..ffc3a5bb --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from dataclasses import dataclass +from enum import Enum +from typing import Generic, List, TypeVar + + +class ConverterIssueType(Enum): + """Identifies the kind of information loss that occurred during conversion.""" + + # Import: the dataset source could not be qualified with a schema/database. + SOURCE_UNQUALIFIED = "SOURCE_UNQUALIFIED" + # Import: a join's sql_on could not be parsed into column pairs. + JOIN_SQL_UNPARSED = "JOIN_SQL_UNPARSED" + # Export: a metric references more than one dataset, which a Lightdash + # model metric cannot express. + CROSS_DATASET_METRIC_DROPPED = "CROSS_DATASET_METRIC_DROPPED" + # Export: a custom extension from another vendor cannot be carried into + # Lightdash meta (it remains in the OSI document itself). + FOREIGN_EXTENSION_IGNORED = "FOREIGN_EXTENSION_IGNORED" + + +@dataclass(frozen=True) +class ConverterIssue: + """Records a single instance of information loss during conversion.""" + + issue_type: ConverterIssueType + element_name: str + + +T = TypeVar("T") + + +@dataclass(frozen=True) +class ConverterResult(Generic[T]): + """Return value of a converter's convert() method, pairing the output with any conversion issues.""" + + output: T + issues: List[ConverterIssue] diff --git a/converters/lightdash/src/ossie_lightdash/expression_utils.py b/converters/lightdash/src/ossie_lightdash/expression_utils.py new file mode 100644 index 00000000..b8473c4e --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/expression_utils.py @@ -0,0 +1,127 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Small expression helpers shared by both conversion directions. + +Lightdash SQL snippets reference columns as ``${TABLE}.column`` (and joined +tables as ``${other_table.column}``); OSI expressions reference them as +``dataset.column``. These helpers translate between the two spellings and +recognise the single-aggregation shapes that map onto Lightdash's typed +metrics. +""" + +import re +from typing import Optional, Tuple + +# Aggregations that translate to a typed Lightdash metric. Anything else is +# exported as a `number` metric with raw SQL. +_AGG_TO_LIGHTDASH_TYPE = { + "SUM": "sum", + "MIN": "min", + "MAX": "max", + "AVG": "average", + "AVERAGE": "average", + "MEDIAN": "median", + "COUNT": "count", +} + +_LIGHTDASH_TYPE_TO_AGG = { + "sum": "SUM", + "min": "MIN", + "max": "MAX", + "average": "AVG", + "median": "MEDIAN", + "count": "COUNT", +} + +_SIMPLE_AGG_RE = re.compile( + r"^\s*(?P[A-Za-z_]+)\s*\(\s*(?PDISTINCT\s+)?(?P[A-Za-z_][\w.]*)\s*\)\s*$", + re.IGNORECASE, +) + +_COLUMN_REF_RE = re.compile(r"^[A-Za-z_]\w*$") + + +def parse_simple_aggregation(expression: str) -> Optional[Tuple[str, str]]: + """Parse ``AGG(qualifier.column)`` into a (lightdash_type, column_ref) pair. + + Returns None when the expression is anything more complex than a single + aggregation over a single column reference. + """ + match = _SIMPLE_AGG_RE.match(expression) + if not match: + return None + func = match.group("func").upper() + inner = match.group("inner") + if match.group("distinct"): + if func != "COUNT": + return None + return ("count_distinct", inner) + lightdash_type = _AGG_TO_LIGHTDASH_TYPE.get(func) + if lightdash_type is None: + return None + return (lightdash_type, inner) + + +def build_aggregation(lightdash_type: str, dataset: str, column: str) -> Optional[str]: + """Build the OSI expression for a typed Lightdash metric, if it has one.""" + if lightdash_type == "count_distinct": + return f"COUNT(DISTINCT {dataset}.{column})" + agg = _LIGHTDASH_TYPE_TO_AGG.get(lightdash_type) + if agg is None: + return None + return f"{agg}({dataset}.{column})" + + +def strip_qualifier(column_ref: str) -> str: + """Return the bare column name of a possibly ``qualifier.column`` reference.""" + return column_ref.rsplit(".", 1)[-1] + + +def qualifier_of(column_ref: str) -> Optional[str]: + """Return the qualifier of a ``qualifier.column`` reference, if present.""" + if "." in column_ref: + return column_ref.rsplit(".", 1)[0] + return None + + +def osi_sql_to_lightdash(expression: str, dataset: str) -> str: + """Rewrite ``dataset.column`` references into Lightdash's ``${TABLE}.column``.""" + return re.sub( + rf"\b{re.escape(dataset)}\.(\w+)", + r"${TABLE}.\1", + expression, + ) + + +def lightdash_sql_to_osi(sql: str, dataset: str) -> str: + """Rewrite Lightdash's ``${TABLE}.column`` references into ``dataset.column``.""" + return sql.replace("${TABLE}.", f"{dataset}.") + + +def is_bare_column(reference: str) -> bool: + """True when the reference is a plain column name without qualifier or SQL.""" + return bool(_COLUMN_REF_RE.match(reference)) + + +def referenced_datasets(expression: str, dataset_names: set) -> set: + """Return which of the given dataset names an OSI expression references.""" + found = set() + for match in re.finditer(r"([A-Za-z_]\w*)\.\w+", expression): + if match.group(1) in dataset_names: + found.add(match.group(1)) + return found diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py new file mode 100644 index 00000000..a1f2eb74 --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py @@ -0,0 +1,312 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert Lightdash semantic definitions into an OSI document. + +The input is a dbt ``schema.yml``-shaped dictionary whose ``meta`` blocks +carry Lightdash dimensions, metrics and joins. Structural information becomes +first-class OSI vocabulary (datasets, fields, metrics, relationships); +Lightdash presentation attributes without OSI vocabulary (``format``, +``round``, ``group_label``, ``hidden``, ...) are preserved in +``custom_extensions`` entries with ``vendor_name: "lightdash"`` so that the +export direction can reproduce them exactly. +""" + +import json +import re +from typing import Any, Dict, List, Optional, Tuple + +from ossie import ( + OSICustomExtension, + OSIDataset, + OSIDialect, + OSIDialectExpression, + OSIDimension, + OSIDocument, + OSIExpression, + OSIField, + OSIMetric, + OSIRelationship, + OSISemanticModel, +) + +from ossie_lightdash.converter_issues import ( + ConverterIssue, + ConverterIssueType, + ConverterResult, +) +from ossie_lightdash.expression_utils import ( + build_aggregation, + lightdash_sql_to_osi, +) + +LIGHTDASH_VENDOR_NAME = "lightdash" + +# Keys that are structurally encoded in OSI vocabulary and therefore must NOT +# be duplicated into the extension (a stale copy would win on export). +# ``type`` stays in the extension only for metric types whose semantics OSI +# expressions cannot express faithfully (currently ``percentile``). +_STRUCTURAL_METRIC_KEYS = {"sql", "description"} +_STRUCTURAL_DIMENSION_KEYS = {"label", "sql"} + +_TIME_DIMENSION_TYPES = {"date", "timestamp"} + +_JOIN_PAIR_RE = re.compile( + r"\$\{(\w+)\.(\w+)\}\s*=\s*\$\{(\w+)\.(\w+)\}", +) + + +def _ansi(expression: str) -> OSIExpression: + return OSIExpression( + dialects=[ + OSIDialectExpression(dialect=OSIDialect.ANSI_SQL, expression=expression) + ] + ) + + +def _lightdash_extension(data: Dict[str, Any]) -> List[OSICustomExtension]: + if not data: + return [] + return [ + OSICustomExtension( + vendor_name=LIGHTDASH_VENDOR_NAME, + data=json.dumps(data, ensure_ascii=False, sort_keys=True), + ) + ] + + +class LightdashToOSIConverter: + """Converts a Lightdash-flavoured dbt schema.yml dict into an OSIDocument.""" + + def convert( + self, + schema_yml: Dict[str, Any], + *, + database: Optional[str] = None, + schema: Optional[str] = None, + semantic_model_name: str = "lightdash_semantic_model", + ) -> ConverterResult[OSIDocument]: + issues: List[ConverterIssue] = [] + datasets: List[OSIDataset] = [] + metrics: List[OSIMetric] = [] + relationships: List[OSIRelationship] = [] + + for model in schema_yml.get("models") or []: + dataset, model_metrics, model_relationships = self._convert_model( + model, database=database, schema=schema, issues=issues + ) + datasets.append(dataset) + metrics.extend(model_metrics) + relationships.extend(model_relationships) + + document = OSIDocument( + version="0.2.0.dev0", + semantic_model=[ + OSISemanticModel( + name=semantic_model_name, + datasets=datasets, + metrics=metrics or None, + relationships=relationships or None, + ) + ], + ) + return ConverterResult(output=document, issues=issues) + + def _convert_model( + self, + model: Dict[str, Any], + *, + database: Optional[str], + schema: Optional[str], + issues: List[ConverterIssue], + ) -> Tuple[OSIDataset, List[OSIMetric], List[OSIRelationship]]: + name = model["name"] + source = ".".join(part for part in [database, schema, name] if part) + if schema is None: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.SOURCE_UNQUALIFIED, + element_name=name, + ) + ) + + fields: List[OSIField] = [] + metrics: List[OSIMetric] = [] + for column in model.get("columns") or []: + field, column_metrics = self._convert_column(column, dataset_name=name) + fields.append(field) + metrics.extend(column_metrics) + + model_meta = model.get("meta") or {} + for metric_name, definition in (model_meta.get("metrics") or {}).items(): + metrics.append( + self._convert_sql_metric(metric_name, definition, dataset_name=name) + ) + + relationships = self._convert_joins( + model_meta.get("joins") or [], from_model=name, issues=issues + ) + + dataset = OSIDataset( + name=name, + source=source, + description=model.get("description"), + fields=fields or None, + ) + return dataset, metrics, relationships + + def _convert_column( + self, column: Dict[str, Any], *, dataset_name: str + ) -> Tuple[OSIField, List[OSIMetric]]: + column_name = column["name"] + meta = column.get("meta") or {} + dimension_meta = meta.get("dimension") + + expression = column_name + dimension: Optional[OSIDimension] = None + label: Optional[str] = None + extension_data: Dict[str, Any] = {} + if dimension_meta is not None: + label = dimension_meta.get("label") + if dimension_meta.get("sql"): + expression = lightdash_sql_to_osi(dimension_meta["sql"], dataset_name) + dimension = OSIDimension( + is_time=dimension_meta.get("type") in _TIME_DIMENSION_TYPES + ) + extension_data = { + key: value + for key, value in dimension_meta.items() + if key not in _STRUCTURAL_DIMENSION_KEYS + } + + field = OSIField( + name=column_name, + expression=_ansi(expression), + dimension=dimension, + label=label, + description=column.get("description"), + custom_extensions=_lightdash_extension(extension_data) or None, + ) + + metrics = [ + self._convert_column_metric( + metric_name, definition, dataset_name=dataset_name, column=column_name + ) + for metric_name, definition in (meta.get("metrics") or {}).items() + ] + return field, metrics + + def _convert_column_metric( + self, + metric_name: str, + definition: Dict[str, Any], + *, + dataset_name: str, + column: str, + ) -> OSIMetric: + lightdash_type = definition.get("type", "number") + expression = build_aggregation(lightdash_type, dataset_name, column) + keep_type_in_extension = False + if lightdash_type == "count_distinct": + pass + elif expression is None: + # number metrics carry their own SQL; percentile and friends have + # no faithful OSI expression, so their type stays in the extension. + sql = definition.get("sql") + if sql: + expression = lightdash_sql_to_osi(sql, dataset_name) + else: + expression = f"{dataset_name}.{column}" + keep_type_in_extension = True + + return self._build_metric( + metric_name, + definition, + expression=expression, + keep_type_in_extension=keep_type_in_extension, + ) + + def _convert_sql_metric( + self, metric_name: str, definition: Dict[str, Any], *, dataset_name: str + ) -> OSIMetric: + sql = definition.get("sql") or "" + expression = lightdash_sql_to_osi(sql, dataset_name) + return self._build_metric( + metric_name, definition, expression=expression, keep_type_in_extension=False + ) + + @staticmethod + def _build_metric( + metric_name: str, + definition: Dict[str, Any], + *, + expression: str, + keep_type_in_extension: bool, + ) -> OSIMetric: + excluded = set(_STRUCTURAL_METRIC_KEYS) + if not keep_type_in_extension: + excluded.add("type") + extension_data = { + key: value for key, value in definition.items() if key not in excluded + } + return OSIMetric( + name=metric_name, + expression=_ansi(expression), + description=definition.get("description"), + custom_extensions=_lightdash_extension(extension_data) or None, + ) + + @staticmethod + def _convert_joins( + joins: List[Dict[str, Any]], + *, + from_model: str, + issues: List[ConverterIssue], + ) -> List[OSIRelationship]: + relationships: List[OSIRelationship] = [] + for join in joins: + to_model = join.get("join") + pairs = _JOIN_PAIR_RE.findall(join.get("sql_on") or "") + from_columns: List[str] = [] + to_columns: List[str] = [] + for left_table, left_column, right_table, right_column in pairs: + if left_table == from_model and right_table == to_model: + from_columns.append(left_column) + to_columns.append(right_column) + elif left_table == to_model and right_table == from_model: + from_columns.append(right_column) + to_columns.append(left_column) + if not to_model or not from_columns: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.JOIN_SQL_UNPARSED, + element_name=f"{from_model} -> {to_model or ''}", + ) + ) + continue + relationships.append( + OSIRelationship.model_validate( + { + "name": f"{from_model}_to_{to_model}", + "from": from_model, + "to": to_model, + "from_columns": from_columns, + "to_columns": to_columns, + } + ) + ) + return relationships diff --git a/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py b/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py new file mode 100644 index 00000000..7585e2f9 --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py @@ -0,0 +1,225 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert an OSI document into Lightdash semantic definitions. + +The output is a dbt ``schema.yml``-shaped dictionary whose ``meta`` blocks +carry Lightdash dimensions, metrics and joins, ready to be merged into a dbt +project that Lightdash reads. Lightdash-specific presentation attributes that +have no OSI vocabulary round-trip through ``custom_extensions`` entries with +``vendor_name: "lightdash"``; their keys are overlaid onto the generated +definitions and therefore always win. +""" + +import json +from typing import Any, Dict, List, Optional + +from ossie import OSIDataset, OSIDialect, OSIDocument, OSIMetric, OSISemanticModel + +from ossie_lightdash.converter_issues import ( + ConverterIssue, + ConverterIssueType, + ConverterResult, +) +from ossie_lightdash.expression_utils import ( + osi_sql_to_lightdash, + parse_simple_aggregation, + qualifier_of, + referenced_datasets, + strip_qualifier, +) + +LIGHTDASH_VENDOR_NAME = "lightdash" + + +def _pick_expression(osi_expression: Any, dialect: OSIDialect) -> str: + """Return the expression for the preferred dialect (fallback: first available).""" + for dialect_expression in osi_expression.dialects: + if dialect_expression.dialect is dialect: + return dialect_expression.expression + return osi_expression.dialects[0].expression if osi_expression.dialects else "" + + +def _lightdash_extension_data(element: Any, issues: List[ConverterIssue]) -> Dict[str, Any]: + """Return the ``lightdash`` vendor extension data of an OSI element, if any.""" + data: Dict[str, Any] = {} + for extension in element.custom_extensions or []: + if extension.vendor_name == LIGHTDASH_VENDOR_NAME: + try: + data.update(json.loads(extension.data)) + except (TypeError, ValueError): + pass + else: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.FOREIGN_EXTENSION_IGNORED, + element_name=getattr(element, "name", ""), + ) + ) + return data + + +def _model_name_for(dataset: OSIDataset) -> str: + """A Lightdash table is addressed by its dbt model name = the source's table part.""" + return dataset.source.rsplit(".", 1)[-1] + + +class OSIToLightdashConverter: + """Converts an OSIDocument into a Lightdash-flavoured dbt schema.yml dict.""" + + def __init__(self, dialect: OSIDialect = OSIDialect.ANSI_SQL) -> None: + self._dialect = dialect + + def convert(self, document: OSIDocument) -> ConverterResult[Dict[str, Any]]: + issues: List[ConverterIssue] = [] + models: List[Dict[str, Any]] = [] + for semantic_model in document.semantic_model: + models.extend(self._convert_semantic_model(semantic_model, issues)) + return ConverterResult(output={"version": 2, "models": models}, issues=issues) + + def _convert_semantic_model( + self, semantic_model: OSISemanticModel, issues: List[ConverterIssue] + ) -> List[Dict[str, Any]]: + datasets = semantic_model.datasets or [] + dataset_names = {dataset.name for dataset in datasets} + model_name_by_dataset = { + dataset.name: _model_name_for(dataset) for dataset in datasets + } + + models_by_dataset: Dict[str, Dict[str, Any]] = {} + columns_by_dataset: Dict[str, Dict[str, Dict[str, Any]]] = {} + for dataset in datasets: + model, columns = self._convert_dataset(dataset, issues) + models_by_dataset[dataset.name] = model + columns_by_dataset[dataset.name] = columns + + for metric in semantic_model.metrics or []: + self._convert_metric( + metric, + dataset_names, + models_by_dataset, + columns_by_dataset, + issues, + ) + + for relationship in semantic_model.relationships or []: + from_model = models_by_dataset.get(relationship.from_dataset) + to_model_name = model_name_by_dataset.get(relationship.to) + from_model_name = model_name_by_dataset.get(relationship.from_dataset) + if from_model is None or to_model_name is None: + continue + sql_on = " AND ".join( + f"${{{from_model_name}.{from_column}}} = ${{{to_model_name}.{to_column}}}" + for from_column, to_column in zip( + relationship.from_columns, relationship.to_columns + ) + ) + joins = from_model.setdefault("meta", {}).setdefault("joins", []) + joins.append({"join": to_model_name, "sql_on": sql_on}) + + return [models_by_dataset[dataset.name] for dataset in datasets] + + def _convert_dataset( + self, dataset: OSIDataset, issues: List[ConverterIssue] + ) -> tuple: + columns_by_name: Dict[str, Dict[str, Any]] = {} + for field in dataset.fields or []: + column: Dict[str, Any] = {"name": field.name} + if field.description: + column["description"] = field.description + + dimension: Dict[str, Any] = {} + if field.label: + dimension["label"] = field.label + if field.dimension is not None and field.dimension.is_time: + dimension["type"] = "date" + expression = _pick_expression(field.expression, self._dialect) + if expression and expression != field.name: + dimension["sql"] = osi_sql_to_lightdash(expression, dataset.name) + dimension.update(_lightdash_extension_data(field, issues)) + # An empty dict still marks dimension-ness: a field OSI declares as a + # categorical dimension must not degrade to a plain column on export, + # or the import direction could not reconstruct it. + if dimension or field.dimension is not None: + column["meta"] = {"dimension": dimension} + columns_by_name[field.name] = column + + model: Dict[str, Any] = {"name": _model_name_for(dataset)} + if dataset.description: + model["description"] = dataset.description + model["columns"] = list(columns_by_name.values()) + return model, columns_by_name + + def _convert_metric( + self, + metric: OSIMetric, + dataset_names: set, + models_by_dataset: Dict[str, Dict[str, Any]], + columns_by_dataset: Dict[str, Dict[str, Dict[str, Any]]], + issues: List[ConverterIssue], + ) -> None: + expression = _pick_expression(metric.expression, self._dialect) + extension_data = _lightdash_extension_data(metric, issues) + + target_dataset = self._resolve_target_dataset(expression, dataset_names) + if target_dataset is None: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.CROSS_DATASET_METRIC_DROPPED, + element_name=metric.name, + ) + ) + return + + definition: Dict[str, Any] = {} + if metric.description: + definition["description"] = metric.description + + target_column: Optional[str] = None + parsed = parse_simple_aggregation(expression) + if parsed is not None: + lightdash_type, column_ref = parsed + qualifier = qualifier_of(column_ref) + if qualifier in (None, target_dataset): + target_column = strip_qualifier(column_ref) + definition["type"] = lightdash_type + if target_column is None or target_column not in columns_by_dataset[target_dataset]: + definition["type"] = extension_data.get("type", "number") + definition["sql"] = osi_sql_to_lightdash(expression, target_dataset) + target_column = None + + definition.update(extension_data) + + if target_column is not None: + column = columns_by_dataset[target_dataset][target_column] + metrics = ( + column.setdefault("meta", {}).setdefault("metrics", {}) + ) + metrics[metric.name] = definition + else: + model = models_by_dataset[target_dataset] + metrics = model.setdefault("meta", {}).setdefault("metrics", {}) + metrics[metric.name] = definition + + @staticmethod + def _resolve_target_dataset(expression: str, dataset_names: set) -> Optional[str]: + referenced = referenced_datasets(expression, dataset_names) + if len(referenced) == 1: + return next(iter(referenced)) + if len(referenced) == 0 and len(dataset_names) == 1: + return next(iter(dataset_names)) + return None diff --git a/converters/lightdash/tests/__init__.py b/converters/lightdash/tests/__init__.py new file mode 100644 index 00000000..13a83393 --- /dev/null +++ b/converters/lightdash/tests/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/converters/lightdash/tests/test_lightdash_to_osi.py b/converters/lightdash/tests/test_lightdash_to_osi.py new file mode 100644 index 00000000..0cbea090 --- /dev/null +++ b/converters/lightdash/tests/test_lightdash_to_osi.py @@ -0,0 +1,187 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +from ossie_lightdash import ConverterIssueType, LightdashToOSIConverter + +SCHEMA_YML = { + "version": 2, + "models": [ + { + "name": "orders", + "description": "One row per order", + "meta": { + "joins": [ + { + "join": "customers", + "sql_on": "${orders.customer_id} = ${customers.customer_id}", + } + ], + "metrics": { + "conversion_rate": { + "type": "number", + "label": "Conversion rate", + "format": "percent", + "round": 1, + "sql": "SUM(${TABLE}.completed_count) / NULLIF(SUM(${TABLE}.total_count), 0)", + } + }, + }, + "columns": [ + { + "name": "order_date", + "description": "Date the order was placed", + "meta": {"dimension": {"label": "Order date", "type": "date"}}, + }, + { + "name": "status", + "meta": {"dimension": {"label": "Status", "type": "string"}}, + }, + { + "name": "amount", + "description": "Order amount", + "meta": { + "metrics": { + "total_amount": { + "type": "sum", + "label": "Total amount", + "format": "usd", + }, + "median_amount": {"type": "median"}, + "p90_amount": {"type": "percentile", "percentile": 90}, + } + }, + }, + {"name": "completed_count"}, + {"name": "total_count"}, + { + "name": "customer_id", + "meta": { + "metrics": { + "unique_customers": {"type": "count_distinct"}, + } + }, + }, + ], + }, + { + "name": "customers", + "columns": [{"name": "customer_id"}], + }, + ], +} + + +def _metric(document, name): + return next(m for m in document.semantic_model[0].metrics if m.name == name) + + +def _lightdash_data(element): + for extension in element.custom_extensions or []: + if extension.vendor_name == "lightdash": + return json.loads(extension.data) + return {} + + +class TestLightdashToOSI: + def test_dataset_source_is_qualified(self): + result = LightdashToOSIConverter().convert( + SCHEMA_YML, database="analytics_db", schema="marts" + ) + dataset = result.output.semantic_model[0].datasets[0] + assert dataset.source == "analytics_db.marts.orders" + assert not any( + issue.issue_type is ConverterIssueType.SOURCE_UNQUALIFIED + for issue in result.issues + ) + + def test_missing_schema_is_reported(self): + result = LightdashToOSIConverter().convert(SCHEMA_YML) + dataset = result.output.semantic_model[0].datasets[0] + assert dataset.source == "orders" + assert any( + issue.issue_type is ConverterIssueType.SOURCE_UNQUALIFIED + for issue in result.issues + ) + + def test_time_dimension(self): + result = LightdashToOSIConverter().convert(SCHEMA_YML, schema="marts") + field = result.output.semantic_model[0].datasets[0].fields[0] + assert field.name == "order_date" + assert field.label == "Order date" + assert field.dimension is not None and field.dimension.is_time + assert field.description == "Date the order was placed" + + def test_typed_metric_becomes_aggregation_expression(self): + result = LightdashToOSIConverter().convert(SCHEMA_YML, schema="marts") + metric = _metric(result.output, "total_amount") + assert metric.expression.dialects[0].expression == "SUM(orders.amount)" + assert _lightdash_data(metric) == {"label": "Total amount", "format": "usd"} + + def test_count_distinct_metric(self): + result = LightdashToOSIConverter().convert(SCHEMA_YML, schema="marts") + metric = _metric(result.output, "unique_customers") + assert ( + metric.expression.dialects[0].expression + == "COUNT(DISTINCT orders.customer_id)" + ) + + def test_percentile_metric_keeps_type_in_extension(self): + result = LightdashToOSIConverter().convert(SCHEMA_YML, schema="marts") + metric = _metric(result.output, "p90_amount") + assert _lightdash_data(metric) == {"type": "percentile", "percentile": 90} + + def test_sql_metric_expression_is_rewritten(self): + result = LightdashToOSIConverter().convert(SCHEMA_YML, schema="marts") + metric = _metric(result.output, "conversion_rate") + assert ( + metric.expression.dialects[0].expression + == "SUM(orders.completed_count) / NULLIF(SUM(orders.total_count), 0)" + ) + assert _lightdash_data(metric) == { + "label": "Conversion rate", + "format": "percent", + "round": 1, + } + + def test_join_becomes_relationship(self): + result = LightdashToOSIConverter().convert(SCHEMA_YML, schema="marts") + relationship = result.output.semantic_model[0].relationships[0] + assert relationship.from_dataset == "orders" + assert relationship.to == "customers" + assert relationship.from_columns == ["customer_id"] + assert relationship.to_columns == ["customer_id"] + + def test_unparseable_join_is_reported(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": { + "joins": [{"join": "customers", "sql_on": "1 = 1"}], + }, + "columns": [], + } + ] + } + result = LightdashToOSIConverter().convert(schema_yml, schema="marts") + assert result.output.semantic_model[0].relationships is None + assert any( + issue.issue_type is ConverterIssueType.JOIN_SQL_UNPARSED + for issue in result.issues + ) diff --git a/converters/lightdash/tests/test_osi_to_lightdash.py b/converters/lightdash/tests/test_osi_to_lightdash.py new file mode 100644 index 00000000..e791c082 --- /dev/null +++ b/converters/lightdash/tests/test_osi_to_lightdash.py @@ -0,0 +1,200 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +from ossie import ( + OSICustomExtension, + OSIDataset, + OSIDialect, + OSIDialectExpression, + OSIDimension, + OSIDocument, + OSIExpression, + OSIField, + OSIMetric, + OSIRelationship, + OSISemanticModel, +) + +from ossie_lightdash import ConverterIssueType, OSIToLightdashConverter + + +def _ansi(expression: str) -> OSIExpression: + return OSIExpression( + dialects=[ + OSIDialectExpression(dialect=OSIDialect.ANSI_SQL, expression=expression) + ] + ) + + +def _document() -> OSIDocument: + orders = OSIDataset( + name="orders", + source="analytics_db.marts.orders", + description="One row per order", + fields=[ + OSIField( + name="order_date", + expression=_ansi("order_date"), + dimension=OSIDimension(is_time=True), + label="Order date", + ), + OSIField( + name="status", + expression=_ansi("status"), + dimension=OSIDimension(is_time=False), + ), + OSIField(name="amount", expression=_ansi("amount")), + OSIField(name="customer_id", expression=_ansi("customer_id")), + ], + ) + customers = OSIDataset( + name="customers", + source="analytics_db.marts.customers", + fields=[OSIField(name="customer_id", expression=_ansi("customer_id"))], + ) + metrics = [ + OSIMetric( + name="total_amount", + expression=_ansi("SUM(orders.amount)"), + description="Sum of order amounts", + custom_extensions=[ + OSICustomExtension( + vendor_name="lightdash", + data=json.dumps({"label": "Total amount", "format": "usd"}), + ) + ], + ), + OSIMetric( + name="conversion_rate", + expression=_ansi( + "SUM(orders.completed_count) / NULLIF(SUM(orders.total_count), 0)" + ), + custom_extensions=[ + OSICustomExtension( + vendor_name="lightdash", + data=json.dumps({"format": "percent", "round": 1}), + ) + ], + ), + OSIMetric( + name="cross_dataset", + expression=_ansi("SUM(orders.amount) / COUNT(customers.customer_id)"), + ), + OSIMetric( + name="foreign_vendor_metric", + expression=_ansi("SUM(orders.amount)"), + custom_extensions=[ + OSICustomExtension(vendor_name="somebi", data='{"x": 1}') + ], + ), + ] + relationships = [ + OSIRelationship.model_validate( + { + "name": "orders_to_customers", + "from": "orders", + "to": "customers", + "from_columns": ["customer_id"], + "to_columns": ["customer_id"], + } + ) + ] + return OSIDocument( + version="0.2.0.dev0", + semantic_model=[ + OSISemanticModel( + name="sales", + datasets=[orders, customers], + metrics=metrics, + relationships=relationships, + ) + ], + ) + + +def _model(output, name): + return next(m for m in output["models"] if m["name"] == name) + + +def _column(model, name): + return next(c for c in model["columns"] if c["name"] == name) + + +class TestOSIToLightdash: + def test_time_dimension_exports_date_type(self): + result = OSIToLightdashConverter().convert(_document()) + column = _column(_model(result.output, "orders"), "order_date") + assert column["meta"]["dimension"] == {"label": "Order date", "type": "date"} + + def test_categorical_dimension_keeps_dimension_marker(self): + result = OSIToLightdashConverter().convert(_document()) + column = _column(_model(result.output, "orders"), "status") + assert column["meta"]["dimension"] == {} + + def test_plain_field_has_no_dimension_meta(self): + result = OSIToLightdashConverter().convert(_document()) + column = _column(_model(result.output, "orders"), "amount") + assert "dimension" not in column.get("meta", {}) + + def test_simple_aggregation_becomes_column_metric(self): + result = OSIToLightdashConverter().convert(_document()) + column = _column(_model(result.output, "orders"), "amount") + metric = column["meta"]["metrics"]["total_amount"] + assert metric["type"] == "sum" + assert metric["label"] == "Total amount" + assert metric["format"] == "usd" + assert metric["description"] == "Sum of order amounts" + assert "sql" not in metric + + def test_complex_expression_becomes_model_metric(self): + result = OSIToLightdashConverter().convert(_document()) + metric = _model(result.output, "orders")["meta"]["metrics"]["conversion_rate"] + assert metric["type"] == "number" + assert ( + metric["sql"] + == "SUM(${TABLE}.completed_count) / NULLIF(SUM(${TABLE}.total_count), 0)" + ) + assert metric["format"] == "percent" + assert metric["round"] == 1 + + def test_cross_dataset_metric_is_dropped_with_issue(self): + result = OSIToLightdashConverter().convert(_document()) + assert any( + issue.issue_type is ConverterIssueType.CROSS_DATASET_METRIC_DROPPED + and issue.element_name == "cross_dataset" + for issue in result.issues + ) + + def test_foreign_extension_is_reported(self): + result = OSIToLightdashConverter().convert(_document()) + assert any( + issue.issue_type is ConverterIssueType.FOREIGN_EXTENSION_IGNORED + and issue.element_name == "foreign_vendor_metric" + for issue in result.issues + ) + + def test_relationship_becomes_join(self): + result = OSIToLightdashConverter().convert(_document()) + joins = _model(result.output, "orders")["meta"]["joins"] + assert joins == [ + { + "join": "customers", + "sql_on": "${orders.customer_id} = ${customers.customer_id}", + } + ] diff --git a/converters/lightdash/tests/test_tpcds_roundtrip.py b/converters/lightdash/tests/test_tpcds_roundtrip.py new file mode 100644 index 00000000..8538aaa0 --- /dev/null +++ b/converters/lightdash/tests/test_tpcds_roundtrip.py @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Round-trip the in-repo TPC-DS example through the Lightdash converter. + +Ossie -> Lightdash schema.yml -> Ossie must preserve the structural core: +datasets, fields (and their dimension-ness), single-dataset metrics and +relationships. Cross-dataset metrics are the documented exception: Lightdash +model metrics cannot reference other tables, so the export direction drops +them with a CROSS_DATASET_METRIC_DROPPED issue. +""" + +from pathlib import Path + +import yaml + +from ossie import OSIDocument + +from ossie_lightdash import ( + ConverterIssueType, + LightdashToOSIConverter, + OSIToLightdashConverter, +) + +TPCDS_PATH = Path(__file__).parent / ".." / ".." / ".." / "examples" / "tpcds_semantic_model.yaml" + + +def _load_tpcds() -> OSIDocument: + return OSIDocument.model_validate(yaml.safe_load(TPCDS_PATH.read_text())) + + +def _roundtrip(): + original = _load_tpcds() + exported = OSIToLightdashConverter().convert(original) + reimported = LightdashToOSIConverter().convert( + exported.output, + database="tpcds", + schema="public", + semantic_model_name=original.semantic_model[0].name, + ) + return original, exported, reimported + + +class TestTpcdsRoundtrip: + def test_datasets_and_sources_are_preserved(self): + original, _, reimported = _roundtrip() + original_sources = { + dataset.name: dataset.source + for dataset in original.semantic_model[0].datasets + } + roundtripped_sources = { + dataset.name: dataset.source + for dataset in reimported.output.semantic_model[0].datasets + } + assert roundtripped_sources == original_sources + + def test_fields_and_dimension_flags_are_preserved(self): + original, _, reimported = _roundtrip() + for original_dataset, roundtripped_dataset in zip( + original.semantic_model[0].datasets, + reimported.output.semantic_model[0].datasets, + ): + original_fields = { + field.name: (field.dimension is not None and field.dimension.is_time, + field.dimension is not None) + for field in original_dataset.fields or [] + } + roundtripped_fields = { + field.name: (field.dimension is not None and field.dimension.is_time, + field.dimension is not None) + for field in roundtripped_dataset.fields or [] + } + assert roundtripped_fields == original_fields + + def test_single_dataset_metrics_survive_with_expressions(self): + original, exported, reimported = _roundtrip() + dropped = { + issue.element_name + for issue in exported.issues + if issue.issue_type is ConverterIssueType.CROSS_DATASET_METRIC_DROPPED + } + original_metrics = { + metric.name: metric.expression.dialects[0].expression + for metric in original.semantic_model[0].metrics or [] + if metric.name not in dropped + } + roundtripped_metrics = { + metric.name: metric.expression.dialects[0].expression + for metric in reimported.output.semantic_model[0].metrics or [] + } + assert set(roundtripped_metrics) == set(original_metrics) + for name, expression in original_metrics.items(): + assert roundtripped_metrics[name].replace(" ", "") == expression.replace( + " ", "" + ), name + + def test_relationships_are_preserved(self): + original, _, reimported = _roundtrip() + original_edges = { + (r.from_dataset, r.to, tuple(r.from_columns), tuple(r.to_columns)) + for r in original.semantic_model[0].relationships or [] + } + roundtripped_edges = { + (r.from_dataset, r.to, tuple(r.from_columns), tuple(r.to_columns)) + for r in reimported.output.semantic_model[0].relationships or [] + } + assert roundtripped_edges == original_edges diff --git a/converters/lightdash/uv.lock b/converters/lightdash/uv.lock new file mode 100644 index 00000000..682505de --- /dev/null +++ b/converters/lightdash/uv.lock @@ -0,0 +1,304 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "apache-ossie" +version = "0.2.0.dev0" +source = { editable = "../../python" } +dependencies = [ + { name = "pydantic" }, + { name = "pyyaml" }, +] + +[package.metadata] +requires-dist = [ + { name = "pydantic", specifier = ">=2.0" }, + { name = "pyyaml", specifier = ">=6.0" }, +] + +[[package]] +name = "apache-ossie-lightdash" +version = "0.1.0.dev0" +source = { editable = "." } +dependencies = [ + { name = "apache-ossie" }, + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "apache-ossie", editable = "../../python" }, + { name = "pyyaml", specifier = ">=6.0" }, +] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8.0" }] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] From 57808858a8c94c4105aa9c5df4f99d913259e21a Mon Sep 17 00:00:00 2001 From: ota2000 <16278388+OTA2000@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:15:38 +0900 Subject: [PATCH 02/33] Self-review: add CI workflow, joined-table references, percentile fidelity - Add converter-lightdash-ci.yml mirroring the other converters (tests were otherwise not exercised by CI) - Rewrite ${other_table.column} references into cross-dataset references on import, not just ${TABLE}.column - Keep inexpressible metric types (percentile) in the extension even when the metric carries custom SQL, on both column-level and model-level metrics - Drop the redundant per-converter .gitignore (covered by the root one) and unused helpers; document the not-yet-carried model-level meta keys in the README --- .github/workflows/converter-lightdash-ci.yml | 63 +++++++++++++++++++ converters/lightdash/.gitignore | 2 - converters/lightdash/README.md | 3 + .../src/ossie_lightdash/expression_utils.py | 14 ++--- .../src/ossie_lightdash/lightdash_to_osi.py | 27 +++++--- .../lightdash/tests/test_lightdash_to_osi.py | 50 +++++++++++++++ 6 files changed, 140 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/converter-lightdash-ci.yml delete mode 100644 converters/lightdash/.gitignore diff --git a/.github/workflows/converter-lightdash-ci.yml b/.github/workflows/converter-lightdash-ci.yml new file mode 100644 index 00000000..802e78e7 --- /dev/null +++ b/.github/workflows/converter-lightdash-ci.yml @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Converters Lightdash CI + +on: + push: + branches: [ "main" ] + paths: + - 'converters/lightdash/**' + - '.github/**' + pull_request: + branches: [ "main" ] + paths: + - 'converters/lightdash/**' + - '.github/**' + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" + + - name: Sync dependencies + working-directory: converters/lightdash + run: | + uv sync + + - name: Unit Tests + working-directory: converters/lightdash + run: | + uv run pytest diff --git a/converters/lightdash/.gitignore b/converters/lightdash/.gitignore deleted file mode 100644 index a230a78a..00000000 --- a/converters/lightdash/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.venv/ -__pycache__/ diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 85200921..15d245db 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -75,6 +75,9 @@ Omitting `--schema` as well is reported as a `SOURCE_UNQUALIFIED` issue. - **`primary_key` / `unique_keys` are not exported** — Lightdash has no corresponding concept — and consequently cannot be reconstructed on import. - **`ai_context` is not carried** into Lightdash meta. +- **Model-level Lightdash meta beyond `metrics` and `joins`** (`label`, + `group_details`, `sql_filter`, `order_fields_by`, column + `additional_dimensions`, ...) is not carried yet. - **Standalone Lightdash YAML projects** (Lightdash without dbt) are not supported yet; the converter targets the dbt-meta flavour. - Custom extensions from other vendors are ignored on export (reported as diff --git a/converters/lightdash/src/ossie_lightdash/expression_utils.py b/converters/lightdash/src/ossie_lightdash/expression_utils.py index b8473c4e..275abe0e 100644 --- a/converters/lightdash/src/ossie_lightdash/expression_utils.py +++ b/converters/lightdash/src/ossie_lightdash/expression_utils.py @@ -53,9 +53,6 @@ re.IGNORECASE, ) -_COLUMN_REF_RE = re.compile(r"^[A-Za-z_]\w*$") - - def parse_simple_aggregation(expression: str) -> Optional[Tuple[str, str]]: """Parse ``AGG(qualifier.column)`` into a (lightdash_type, column_ref) pair. @@ -109,13 +106,14 @@ def osi_sql_to_lightdash(expression: str, dataset: str) -> str: def lightdash_sql_to_osi(sql: str, dataset: str) -> str: - """Rewrite Lightdash's ``${TABLE}.column`` references into ``dataset.column``.""" - return sql.replace("${TABLE}.", f"{dataset}.") + """Rewrite Lightdash column references into OSI ``dataset.column`` references. + ``${TABLE}.column`` refers to the current model; ``${other_table.column}`` + refers to a joined model and becomes a cross-dataset reference. + """ + rewritten = sql.replace("${TABLE}.", f"{dataset}.") + return re.sub(r"\$\{(\w+)\.(\w+)\}", r"\1.\2", rewritten) -def is_bare_column(reference: str) -> bool: - """True when the reference is a plain column name without qualifier or SQL.""" - return bool(_COLUMN_REF_RE.match(reference)) def referenced_datasets(expression: str, dataset_names: set) -> set: diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py index a1f2eb74..227dba27 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py @@ -78,6 +78,18 @@ def _ansi(expression: str) -> OSIExpression: ) +def _type_needs_extension(lightdash_type: str) -> bool: + """True for metric types an OSI expression cannot encode faithfully. + + ``number`` is fully described by its SQL and typed aggregations are + recovered by parsing the expression, so only the remaining types + (currently ``percentile``) must survive inside the extension. + """ + if lightdash_type == "number": + return False + return build_aggregation(lightdash_type, "_", "_") is None + + def _lightdash_extension(data: Dict[str, Any]) -> List[OSICustomExtension]: if not data: return [] @@ -220,24 +232,18 @@ def _convert_column_metric( ) -> OSIMetric: lightdash_type = definition.get("type", "number") expression = build_aggregation(lightdash_type, dataset_name, column) - keep_type_in_extension = False - if lightdash_type == "count_distinct": - pass - elif expression is None: - # number metrics carry their own SQL; percentile and friends have - # no faithful OSI expression, so their type stays in the extension. + if expression is None: sql = definition.get("sql") if sql: expression = lightdash_sql_to_osi(sql, dataset_name) else: expression = f"{dataset_name}.{column}" - keep_type_in_extension = True return self._build_metric( metric_name, definition, expression=expression, - keep_type_in_extension=keep_type_in_extension, + keep_type_in_extension=_type_needs_extension(lightdash_type), ) def _convert_sql_metric( @@ -246,7 +252,10 @@ def _convert_sql_metric( sql = definition.get("sql") or "" expression = lightdash_sql_to_osi(sql, dataset_name) return self._build_metric( - metric_name, definition, expression=expression, keep_type_in_extension=False + metric_name, + definition, + expression=expression, + keep_type_in_extension=_type_needs_extension(definition.get("type", "number")), ) @staticmethod diff --git a/converters/lightdash/tests/test_lightdash_to_osi.py b/converters/lightdash/tests/test_lightdash_to_osi.py index 0cbea090..483252e6 100644 --- a/converters/lightdash/tests/test_lightdash_to_osi.py +++ b/converters/lightdash/tests/test_lightdash_to_osi.py @@ -167,6 +167,56 @@ def test_join_becomes_relationship(self): assert relationship.from_columns == ["customer_id"] assert relationship.to_columns == ["customer_id"] + def test_percentile_with_sql_keeps_type_in_extension(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": { + "metrics": { + "p90_custom": { + "type": "percentile", + "percentile": 90, + "sql": "${TABLE}.amount - ${TABLE}.discount", + } + } + }, + "columns": [], + } + ] + } + result = LightdashToOSIConverter().convert(schema_yml, schema="marts") + metric = _metric(result.output, "p90_custom") + assert ( + metric.expression.dialects[0].expression + == "orders.amount - orders.discount" + ) + assert _lightdash_data(metric) == {"type": "percentile", "percentile": 90} + + def test_joined_table_references_become_cross_dataset(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": { + "metrics": { + "orders_per_customer": { + "type": "number", + "sql": "COUNT(${TABLE}.order_id) / COUNT(DISTINCT ${customers.customer_id})", + } + } + }, + "columns": [], + } + ] + } + result = LightdashToOSIConverter().convert(schema_yml, schema="marts") + metric = _metric(result.output, "orders_per_customer") + assert ( + metric.expression.dialects[0].expression + == "COUNT(orders.order_id) / COUNT(DISTINCT customers.customer_id)" + ) + def test_unparseable_join_is_reported(self): schema_yml = { "models": [ From 80c775d5865888b1942e3e0a15a54aa22903a153 Mon Sep 17 00:00:00 2001 From: ota2000 <16278388+OTA2000@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:21:49 +0900 Subject: [PATCH 03/33] Address review feedback: protect structural keys, validate relationship columns - Drop structural keys (sql/label on dimensions, sql/description on metrics) from the extension overlay on export, so a hand-authored extension can never override the OSI-derived definition; metric type stays overridable as the documented channel for inexpressible types - Skip relationships whose from_columns/to_columns lengths differ, reporting RELATIONSHIP_COLUMNS_MISMATCHED instead of silently truncating the join - Document that dataset names are not preserved when they differ from the source table name --- converters/lightdash/README.md | 9 +++- .../src/ossie_lightdash/converter_issues.py | 3 ++ .../src/ossie_lightdash/osi_to_lightdash.py | 36 +++++++++++++-- .../lightdash/tests/test_osi_to_lightdash.py | 44 +++++++++++++++++++ 4 files changed, 88 insertions(+), 4 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 15d245db..fac5fd95 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -49,7 +49,7 @@ ossie-lightdash import schema.yml semantic_model.json --database analytics_db -- | `metric` with single-aggregation expression (`SUM(ds.col)`, `COUNT(DISTINCT ds.col)`, ...) | column-level `meta.metrics.` with a typed metric (`sum`, `count_distinct`, ...) | | `metric` with any other single-dataset expression | model-level `meta.metrics.` with `type: number` + `sql` | | `relationship` | `meta.joins` (`sql_on` built from / parsed into column pairs) | -| Lightdash presentation attributes (`label`, `format`, `round`, `compact`, `group_label`, `hidden`, `percentile`, ...) | `custom_extensions` with `vendor_name: "lightdash"`; on export the extension data is overlaid onto the generated definition and always wins | +| Lightdash presentation attributes (`label`, `format`, `round`, `compact`, `group_label`, `hidden`, `percentile`, ...) | `custom_extensions` with `vendor_name: "lightdash"`; on export the extension data is overlaid onto the generated definition (structural keys — `sql`/`label` on dimensions, `sql`/`description` on metrics — are protected and cannot be overridden) | Expressions are written under the `ANSI_SQL` dialect. Warehouse-specific dialects (e.g. `BIGQUERY`) can be added once the surrounding tooling resolves @@ -74,6 +74,13 @@ Omitting `--schema` as well is reported as a `SOURCE_UNQUALIFIED` issue. as model-level metrics. - **`primary_key` / `unique_keys` are not exported** — Lightdash has no corresponding concept — and consequently cannot be reconstructed on import. +- **`dataset.name` is not preserved when it differs from the source table + name**: the dbt model is named after the table part of `source`, and the + import direction derives dataset names from model names. References inside + expressions and relationships are rewritten consistently, but a + name-stable round-trip is not guaranteed. +- **Relationships with mismatched `from_columns` / `to_columns` lengths are + skipped on export** with a `RELATIONSHIP_COLUMNS_MISMATCHED` issue. - **`ai_context` is not carried** into Lightdash meta. - **Model-level Lightdash meta beyond `metrics` and `joins`** (`label`, `group_details`, `sql_filter`, `order_fields_by`, column diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py index ffc3a5bb..8fd8ab79 100644 --- a/converters/lightdash/src/ossie_lightdash/converter_issues.py +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -30,6 +30,9 @@ class ConverterIssueType(Enum): # Export: a metric references more than one dataset, which a Lightdash # model metric cannot express. CROSS_DATASET_METRIC_DROPPED = "CROSS_DATASET_METRIC_DROPPED" + # Export: a relationship's from_columns/to_columns differ in length, so a + # correct sql_on cannot be built. + RELATIONSHIP_COLUMNS_MISMATCHED = "RELATIONSHIP_COLUMNS_MISMATCHED" # Export: a custom extension from another vendor cannot be carried into # Lightdash meta (it remains in the OSI document itself). FOREIGN_EXTENSION_IGNORED = "FOREIGN_EXTENSION_IGNORED" diff --git a/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py b/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py index 7585e2f9..13b15a42 100644 --- a/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py +++ b/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py @@ -22,7 +22,9 @@ project that Lightdash reads. Lightdash-specific presentation attributes that have no OSI vocabulary round-trip through ``custom_extensions`` entries with ``vendor_name: "lightdash"``; their keys are overlaid onto the generated -definitions and therefore always win. +definitions and win for presentation attributes, while structural keys +(``sql``/``label`` on dimensions, ``sql``/``description`` on metrics) are +protected so they can never override the OSI-derived definition. """ import json @@ -45,6 +47,14 @@ LIGHTDASH_VENDOR_NAME = "lightdash" +# Structural keys are owned by OSI vocabulary (the import direction never puts +# them into the extension); dropping them here keeps a hand-authored extension +# from overriding the OSI-derived definition. ``type`` stays overridable on +# metrics: it is the documented channel for types OSI expressions cannot +# express (e.g. percentile). +_PROTECTED_DIMENSION_KEYS = {"sql", "label"} +_PROTECTED_METRIC_KEYS = {"sql", "description"} + def _pick_expression(osi_expression: Any, dialect: OSIDialect) -> str: """Return the expression for the preferred dialect (fallback: first available).""" @@ -122,6 +132,14 @@ def _convert_semantic_model( from_model_name = model_name_by_dataset.get(relationship.from_dataset) if from_model is None or to_model_name is None: continue + if len(relationship.from_columns) != len(relationship.to_columns): + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.RELATIONSHIP_COLUMNS_MISMATCHED, + element_name=relationship.name, + ) + ) + continue sql_on = " AND ".join( f"${{{from_model_name}.{from_column}}} = ${{{to_model_name}.{to_column}}}" for from_column, to_column in zip( @@ -150,7 +168,13 @@ def _convert_dataset( expression = _pick_expression(field.expression, self._dialect) if expression and expression != field.name: dimension["sql"] = osi_sql_to_lightdash(expression, dataset.name) - dimension.update(_lightdash_extension_data(field, issues)) + dimension.update( + { + key: value + for key, value in _lightdash_extension_data(field, issues).items() + if key not in _PROTECTED_DIMENSION_KEYS + } + ) # An empty dict still marks dimension-ness: a field OSI declares as a # categorical dimension must not degrade to a plain column on export, # or the import direction could not reconstruct it. @@ -202,7 +226,13 @@ def _convert_metric( definition["sql"] = osi_sql_to_lightdash(expression, target_dataset) target_column = None - definition.update(extension_data) + definition.update( + { + key: value + for key, value in extension_data.items() + if key not in _PROTECTED_METRIC_KEYS + } + ) if target_column is not None: column = columns_by_dataset[target_dataset][target_column] diff --git a/converters/lightdash/tests/test_osi_to_lightdash.py b/converters/lightdash/tests/test_osi_to_lightdash.py index e791c082..c8c671c6 100644 --- a/converters/lightdash/tests/test_osi_to_lightdash.py +++ b/converters/lightdash/tests/test_osi_to_lightdash.py @@ -189,6 +189,50 @@ def test_foreign_extension_is_reported(self): for issue in result.issues ) + def test_extension_cannot_override_structural_keys(self): + document = _document() + tampered = document.model_copy(deep=True) + metric = tampered.semantic_model[0].metrics[0].model_copy( + update={ + "custom_extensions": [ + OSICustomExtension( + vendor_name="lightdash", + data=json.dumps( + {"label": "Total amount", "sql": "1 + 1", "description": "stale"} + ), + ) + ] + } + ) + tampered.semantic_model[0].metrics[0] = metric + result = OSIToLightdashConverter().convert(tampered) + column = _column(_model(result.output, "orders"), "amount") + exported = column["meta"]["metrics"]["total_amount"] + assert exported["label"] == "Total amount" + assert "sql" not in exported + assert exported["description"] == "Sum of order amounts" + + def test_mismatched_relationship_columns_are_skipped(self): + document = _document() + tampered = document.model_copy(deep=True) + relationship = OSIRelationship.model_validate( + { + "name": "broken", + "from": "orders", + "to": "customers", + "from_columns": ["customer_id", "order_id"], + "to_columns": ["customer_id"], + } + ) + tampered.semantic_model[0].relationships[0] = relationship + result = OSIToLightdashConverter().convert(tampered) + assert "joins" not in _model(result.output, "orders").get("meta", {}) + assert any( + issue.issue_type is ConverterIssueType.RELATIONSHIP_COLUMNS_MISMATCHED + and issue.element_name == "broken" + for issue in result.issues + ) + def test_relationship_becomes_join(self): result = OSIToLightdashConverter().convert(_document()) joins = _model(result.output, "orders")["meta"]["joins"] From bb09df3133d44c10a8a1fe2ac69104059fe7ebed Mon Sep 17 00:00:00 2001 From: ota2000 <16278388+OTA2000@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:30:20 +0900 Subject: [PATCH 04/33] Address review feedback: report invalid extension JSON, skip sql-less model metrics, add [tool.uv] - Surface unparseable lightdash extension data as an EXTENSION_DATA_INVALID issue instead of silently dropping it - Skip model-level metrics without sql on import, reporting METRIC_SQL_MISSING rather than emitting an empty OSI expression - Add the [tool.uv] block (required-version, default-groups) used by the other converters so uv sync reliably installs the dev group in CI --- converters/lightdash/pyproject.toml | 6 ++++++ .../src/ossie_lightdash/converter_issues.py | 6 ++++++ .../src/ossie_lightdash/lightdash_to_osi.py | 11 +++++++++-- .../src/ossie_lightdash/osi_to_lightdash.py | 7 ++++++- .../lightdash/tests/test_lightdash_to_osi.py | 18 ++++++++++++++++++ .../lightdash/tests/test_osi_to_lightdash.py | 18 ++++++++++++++++++ 6 files changed, 63 insertions(+), 3 deletions(-) diff --git a/converters/lightdash/pyproject.toml b/converters/lightdash/pyproject.toml index ec94ddcd..0d7a2090 100644 --- a/converters/lightdash/pyproject.toml +++ b/converters/lightdash/pyproject.toml @@ -55,6 +55,12 @@ packages = ["src/ossie_lightdash"] [tool.pytest.ini_options] testpaths = ["tests"] +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev" +] + [tool.uv.sources] # apache-ossie is not yet published to PyPI; resolve it from the in-repo # package for now. Remove this block once apache-ossie published to PyPI. diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py index 8fd8ab79..97a5c210 100644 --- a/converters/lightdash/src/ossie_lightdash/converter_issues.py +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -33,6 +33,12 @@ class ConverterIssueType(Enum): # Export: a relationship's from_columns/to_columns differ in length, so a # correct sql_on cannot be built. RELATIONSHIP_COLUMNS_MISMATCHED = "RELATIONSHIP_COLUMNS_MISMATCHED" + # Export: a `lightdash` extension whose data is not valid JSON cannot be + # applied; its presentation attributes are lost. + EXTENSION_DATA_INVALID = "EXTENSION_DATA_INVALID" + # Import: a model-level metric without `sql` has no expressible OSI + # expression and is skipped. + METRIC_SQL_MISSING = "METRIC_SQL_MISSING" # Export: a custom extension from another vendor cannot be carried into # Lightdash meta (it remains in the OSI document itself). FOREIGN_EXTENSION_IGNORED = "FOREIGN_EXTENSION_IGNORED" diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py index 227dba27..7fd415bc 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py @@ -165,6 +165,14 @@ def _convert_model( model_meta = model.get("meta") or {} for metric_name, definition in (model_meta.get("metrics") or {}).items(): + if not definition.get("sql"): + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.METRIC_SQL_MISSING, + element_name=metric_name, + ) + ) + continue metrics.append( self._convert_sql_metric(metric_name, definition, dataset_name=name) ) @@ -249,8 +257,7 @@ def _convert_column_metric( def _convert_sql_metric( self, metric_name: str, definition: Dict[str, Any], *, dataset_name: str ) -> OSIMetric: - sql = definition.get("sql") or "" - expression = lightdash_sql_to_osi(sql, dataset_name) + expression = lightdash_sql_to_osi(definition["sql"], dataset_name) return self._build_metric( metric_name, definition, diff --git a/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py b/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py index 13b15a42..e9860175 100644 --- a/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py +++ b/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py @@ -72,7 +72,12 @@ def _lightdash_extension_data(element: Any, issues: List[ConverterIssue]) -> Dic try: data.update(json.loads(extension.data)) except (TypeError, ValueError): - pass + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.EXTENSION_DATA_INVALID, + element_name=getattr(element, "name", ""), + ) + ) else: issues.append( ConverterIssue( diff --git a/converters/lightdash/tests/test_lightdash_to_osi.py b/converters/lightdash/tests/test_lightdash_to_osi.py index 483252e6..b8d95522 100644 --- a/converters/lightdash/tests/test_lightdash_to_osi.py +++ b/converters/lightdash/tests/test_lightdash_to_osi.py @@ -217,6 +217,24 @@ def test_joined_table_references_become_cross_dataset(self): == "COUNT(orders.order_id) / COUNT(DISTINCT customers.customer_id)" ) + def test_model_metric_without_sql_is_skipped(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": {"metrics": {"broken_metric": {"type": "number"}}}, + "columns": [], + } + ] + } + result = LightdashToOSIConverter().convert(schema_yml, schema="marts") + assert result.output.semantic_model[0].metrics is None + assert any( + issue.issue_type is ConverterIssueType.METRIC_SQL_MISSING + and issue.element_name == "broken_metric" + for issue in result.issues + ) + def test_unparseable_join_is_reported(self): schema_yml = { "models": [ diff --git a/converters/lightdash/tests/test_osi_to_lightdash.py b/converters/lightdash/tests/test_osi_to_lightdash.py index c8c671c6..1f48fb06 100644 --- a/converters/lightdash/tests/test_osi_to_lightdash.py +++ b/converters/lightdash/tests/test_osi_to_lightdash.py @@ -233,6 +233,24 @@ def test_mismatched_relationship_columns_are_skipped(self): for issue in result.issues ) + def test_invalid_extension_json_is_reported(self): + document = _document() + tampered = document.model_copy(deep=True) + metric = tampered.semantic_model[0].metrics[0].model_copy( + update={ + "custom_extensions": [ + OSICustomExtension(vendor_name="lightdash", data="{not json") + ] + } + ) + tampered.semantic_model[0].metrics[0] = metric + result = OSIToLightdashConverter().convert(tampered) + assert any( + issue.issue_type is ConverterIssueType.EXTENSION_DATA_INVALID + and issue.element_name == "total_amount" + for issue in result.issues + ) + def test_relationship_becomes_join(self): result = OSIToLightdashConverter().convert(_document()) joins = _model(result.output, "orders")["meta"]["joins"] From 41b8192e8c7c706cd49e64aa968fbeec40665097 Mon Sep 17 00:00:00 2001 From: ota2000 <16278388+OTA2000@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:38:29 +0900 Subject: [PATCH 05/33] Align CI workflow path filter with the repo-wide update Mirror the change from the recent CI actions path cleanup: trigger on this workflow's own file instead of the whole .github tree. --- .github/workflows/converter-lightdash-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/converter-lightdash-ci.yml b/.github/workflows/converter-lightdash-ci.yml index 802e78e7..462d954b 100644 --- a/.github/workflows/converter-lightdash-ci.yml +++ b/.github/workflows/converter-lightdash-ci.yml @@ -24,12 +24,12 @@ on: branches: [ "main" ] paths: - 'converters/lightdash/**' - - '.github/**' + - '.github/workflows/converter-lightdash-ci.yml' pull_request: branches: [ "main" ] paths: - 'converters/lightdash/**' - - '.github/**' + - '.github/workflows/converter-lightdash-ci.yml' jobs: build: From 049a5289c96c4d343579fa7abc8a2f70687e5819 Mon Sep 17 00:00:00 2001 From: ota2000 <16278388+OTA2000@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:52:14 +0900 Subject: [PATCH 06/33] Adopt the datatype field and is_time role-marker semantics Spec change #113 added `datatype` to fields and metrics and reframed `dimension.is_time` as a role marker rather than a type flag, which broke the TPC-DS round-trip: the converter inferred `is_time` from the Lightdash type and wrote `false` where the source left it unset. - Map Lightdash dimension types to and from `datatype`, so the type is carried in standard vocabulary instead of a vendor extension - Leave `is_time` unset on import (Lightdash has no equivalent marker) and report TIME_ROLE_NOT_REPRESENTABLE when a non-temporal field is flagged as a time axis on export - Emit a Lightdash type only for fields that are dimensions, so a measure-only field is not turned into a dimension by a round-trip - Document that datatypes round-trip by category rather than exact type --- converters/lightdash/README.md | 12 ++- .../src/ossie_lightdash/converter_issues.py | 3 + .../src/ossie_lightdash/datatype_utils.py | 76 +++++++++++++++++++ .../src/ossie_lightdash/lightdash_to_osi.py | 13 ++-- .../src/ossie_lightdash/osi_to_lightdash.py | 27 ++++++- .../lightdash/tests/test_lightdash_to_osi.py | 17 ++++- .../lightdash/tests/test_tpcds_roundtrip.py | 51 +++++++++++-- 7 files changed, 184 insertions(+), 15 deletions(-) create mode 100644 converters/lightdash/src/ossie_lightdash/datatype_utils.py diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index fac5fd95..0466646e 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -43,7 +43,9 @@ ossie-lightdash import schema.yml semantic_model.json --database analytics_db -- | `dataset` | dbt model (`name` = table part of `source`) | | `dataset.source` | assembled on import from `--database` / `--schema` / model name | | `field` (no `dimension`) | plain column entry | -| `field` with `dimension` | `columns[].meta.dimension` (`is_time` ↔ `type: date/timestamp`; an empty `dimension: {}` marks a categorical dimension) | +| `field` with `dimension` | `columns[].meta.dimension` (an empty `dimension: {}` marks a dimension with no extra attributes) | +| `field.datatype` | `meta.dimension.type` (`String`→`string`, `Integer`/`Decimal`/`Float`→`number`, `Date`→`date`, `DateTime`/`DateTimeTz`→`timestamp`, `Boolean`→`boolean`, `Time`/`Opaque`→`string`); on import `number` maps back to `Decimal` | +| `field.dimension.is_time` | *not carried* — it is a role marker (a field can be a time axis without a temporal datatype, e.g. a year stored as `Integer`) and Lightdash has no equivalent | | `field.label` / `.description` | `meta.dimension.label` / column `description` | | `field.expression` (≠ column name) | `meta.dimension.sql` (`dataset.col` ↔ `${TABLE}.col`) | | `metric` with single-aggregation expression (`SUM(ds.col)`, `COUNT(DISTINCT ds.col)`, ...) | column-level `meta.metrics.` with a typed metric (`sum`, `count_distinct`, ...) | @@ -81,6 +83,14 @@ Omitting `--schema` as well is reported as a `SOURCE_UNQUALIFIED` issue. name-stable round-trip is not guaranteed. - **Relationships with mismatched `from_columns` / `to_columns` lengths are skipped on export** with a `RELATIONSHIP_COLUMNS_MISMATCHED` issue. +- **Datatypes round-trip by category, not by exact type**: Lightdash types are + coarser than Ossie datatypes, so `Integer` comes back as `Decimal` and + `DateTimeTz` as `DateTime`. +- **A measure-only field (no `dimension`) loses its `datatype`**: Lightdash + carries types on dimensions only, so there is nowhere to put it. +- **`dimension.is_time` is not carried**, and a field whose datatype is not + temporal but is flagged as a time axis is reported with a + `TIME_ROLE_NOT_REPRESENTABLE` issue on export. - **`ai_context` is not carried** into Lightdash meta. - **Model-level Lightdash meta beyond `metrics` and `joins`** (`label`, `group_details`, `sql_filter`, `order_fields_by`, column diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py index 97a5c210..87c6cf5f 100644 --- a/converters/lightdash/src/ossie_lightdash/converter_issues.py +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -39,6 +39,9 @@ class ConverterIssueType(Enum): # Import: a model-level metric without `sql` has no expressible OSI # expression and is skipped. METRIC_SQL_MISSING = "METRIC_SQL_MISSING" + # Export: a field is marked as a time-axis role (`dimension.is_time`) that + # its datatype does not already imply; Lightdash has no equivalent marker. + TIME_ROLE_NOT_REPRESENTABLE = "TIME_ROLE_NOT_REPRESENTABLE" # Export: a custom extension from another vendor cannot be carried into # Lightdash meta (it remains in the OSI document itself). FOREIGN_EXTENSION_IGNORED = "FOREIGN_EXTENSION_IGNORED" diff --git a/converters/lightdash/src/ossie_lightdash/datatype_utils.py b/converters/lightdash/src/ossie_lightdash/datatype_utils.py new file mode 100644 index 00000000..21e4c4d2 --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/datatype_utils.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Translation between Ossie datatypes and Lightdash dimension types. + +Ossie carries a portable logical datatype on every field; Lightdash has a +coarser set of dimension types. The mapping is therefore lossy in one +direction: several Ossie numeric and temporal types collapse onto a single +Lightdash type, so a round-trip preserves the *category* of a datatype but not +always its exact member (e.g. ``Integer`` comes back as ``Decimal``). +""" + +from typing import Optional + +from ossie import OSIDataType + +# Ossie datatype -> Lightdash dimension type. +_DATATYPE_TO_LIGHTDASH = { + OSIDataType.STRING: "string", + OSIDataType.INTEGER: "number", + OSIDataType.DECIMAL: "number", + OSIDataType.FLOAT: "number", + OSIDataType.BOOLEAN: "boolean", + OSIDataType.DATE: "date", + OSIDataType.DATE_TIME: "timestamp", + OSIDataType.DATE_TIME_TZ: "timestamp", + # Lightdash has no time-of-day dimension type; a string keeps the value + # visible rather than dropping the column. + OSIDataType.TIME: "string", + OSIDataType.OPAQUE: "string", +} + +# Lightdash dimension type -> Ossie datatype. Numeric widths are not expressed +# in Lightdash, so `number` maps to the widest exact type. +_LIGHTDASH_TO_DATATYPE = { + "string": OSIDataType.STRING, + "number": OSIDataType.DECIMAL, + "boolean": OSIDataType.BOOLEAN, + "date": OSIDataType.DATE, + "timestamp": OSIDataType.DATE_TIME, +} + +_TEMPORAL = {OSIDataType.DATE, OSIDataType.TIME, OSIDataType.DATE_TIME, OSIDataType.DATE_TIME_TZ} + + +def datatype_to_lightdash_type(datatype: Optional[OSIDataType]) -> Optional[str]: + """Return the Lightdash dimension type for an Ossie datatype, if any.""" + if datatype is None: + return None + return _DATATYPE_TO_LIGHTDASH.get(datatype) + + +def lightdash_type_to_datatype(lightdash_type: Optional[str]) -> Optional[OSIDataType]: + """Return the Ossie datatype for a Lightdash dimension type, if any.""" + if lightdash_type is None: + return None + return _LIGHTDASH_TO_DATATYPE.get(lightdash_type) + + +def is_temporal(datatype: Optional[OSIDataType]) -> bool: + """True when the datatype represents a point in time.""" + return datatype in _TEMPORAL diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py index 7fd415bc..6c300da8 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py @@ -49,6 +49,7 @@ ConverterIssueType, ConverterResult, ) +from ossie_lightdash.datatype_utils import lightdash_type_to_datatype from ossie_lightdash.expression_utils import ( build_aggregation, lightdash_sql_to_osi, @@ -63,8 +64,6 @@ _STRUCTURAL_METRIC_KEYS = {"sql", "description"} _STRUCTURAL_DIMENSION_KEYS = {"label", "sql"} -_TIME_DIMENSION_TYPES = {"date", "timestamp"} - _JOIN_PAIR_RE = re.compile( r"\$\{(\w+)\.(\w+)\}\s*=\s*\$\{(\w+)\.(\w+)\}", ) @@ -198,15 +197,18 @@ def _convert_column( expression = column_name dimension: Optional[OSIDimension] = None + datatype = None label: Optional[str] = None extension_data: Dict[str, Any] = {} if dimension_meta is not None: label = dimension_meta.get("label") if dimension_meta.get("sql"): expression = lightdash_sql_to_osi(dimension_meta["sql"], dataset_name) - dimension = OSIDimension( - is_time=dimension_meta.get("type") in _TIME_DIMENSION_TYPES - ) + datatype = lightdash_type_to_datatype(dimension_meta.get("type")) + # `is_time` is a role marker in Ossie, not a type: Lightdash has no + # equivalent, so it is left unset rather than inferred from the type + # (the type itself is carried by `datatype`). + dimension = OSIDimension() extension_data = { key: value for key, value in dimension_meta.items() @@ -217,6 +219,7 @@ def _convert_column( name=column_name, expression=_ansi(expression), dimension=dimension, + datatype=datatype, label=label, description=column.get("description"), custom_extensions=_lightdash_extension(extension_data) or None, diff --git a/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py b/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py index e9860175..4fb017ea 100644 --- a/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py +++ b/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py @@ -37,6 +37,7 @@ ConverterIssueType, ConverterResult, ) +from ossie_lightdash.datatype_utils import datatype_to_lightdash_type, is_temporal from ossie_lightdash.expression_utils import ( osi_sql_to_lightdash, parse_simple_aggregation, @@ -168,8 +169,30 @@ def _convert_dataset( dimension: Dict[str, Any] = {} if field.label: dimension["label"] = field.label - if field.dimension is not None and field.dimension.is_time: - dimension["type"] = "date" + if field.dimension is not None: + # Only dimension fields carry a Lightdash type: emitting one for + # a measure-only field would turn it into a dimension on import. + lightdash_type = datatype_to_lightdash_type(field.datatype) + if lightdash_type is not None: + dimension["type"] = lightdash_type + elif field.dimension.is_time: + # No datatype to translate, but the field is declared as a + # time axis: `date` is the closest Lightdash type. + dimension["type"] = "date" + if ( + field.dimension is not None + and field.dimension.is_time + and not is_temporal(field.datatype) + and field.datatype is not None + ): + # A non-temporal datatype flagged as a time axis (e.g. a year + # stored as Integer) has no Lightdash equivalent. + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.TIME_ROLE_NOT_REPRESENTABLE, + element_name=field.name, + ) + ) expression = _pick_expression(field.expression, self._dialect) if expression and expression != field.name: dimension["sql"] = osi_sql_to_lightdash(expression, dataset.name) diff --git a/converters/lightdash/tests/test_lightdash_to_osi.py b/converters/lightdash/tests/test_lightdash_to_osi.py index b8d95522..6082dcf0 100644 --- a/converters/lightdash/tests/test_lightdash_to_osi.py +++ b/converters/lightdash/tests/test_lightdash_to_osi.py @@ -17,6 +17,8 @@ import json +from ossie import OSIDataType + from ossie_lightdash import ConverterIssueType, LightdashToOSIConverter SCHEMA_YML = { @@ -124,8 +126,21 @@ def test_time_dimension(self): field = result.output.semantic_model[0].datasets[0].fields[0] assert field.name == "order_date" assert field.label == "Order date" - assert field.dimension is not None and field.dimension.is_time assert field.description == "Date the order was placed" + assert field.dimension is not None + # The Lightdash type becomes a datatype; `is_time` is an Ossie role + # marker with no Lightdash source, so it stays unset. + assert field.datatype is OSIDataType.DATE + assert field.dimension.is_time is None + + def test_dimension_types_become_datatypes(self): + result = LightdashToOSIConverter().convert(SCHEMA_YML, schema="marts") + by_name = { + field.name: field + for field in result.output.semantic_model[0].datasets[0].fields + } + assert by_name["status"].datatype is OSIDataType.STRING + assert by_name["order_date"].datatype is OSIDataType.DATE def test_typed_metric_becomes_aggregation_expression(self): result = LightdashToOSIConverter().convert(SCHEMA_YML, schema="marts") diff --git a/converters/lightdash/tests/test_tpcds_roundtrip.py b/converters/lightdash/tests/test_tpcds_roundtrip.py index 8538aaa0..9dfe97fe 100644 --- a/converters/lightdash/tests/test_tpcds_roundtrip.py +++ b/converters/lightdash/tests/test_tpcds_roundtrip.py @@ -28,7 +28,7 @@ import yaml -from ossie import OSIDocument +from ossie import OSIDataType, OSIDocument from ossie_lightdash import ( ConverterIssueType, @@ -68,24 +68,63 @@ def test_datasets_and_sources_are_preserved(self): } assert roundtripped_sources == original_sources - def test_fields_and_dimension_flags_are_preserved(self): + def test_fields_and_dimension_markers_are_preserved(self): original, _, reimported = _roundtrip() for original_dataset, roundtripped_dataset in zip( original.semantic_model[0].datasets, reimported.output.semantic_model[0].datasets, ): original_fields = { - field.name: (field.dimension is not None and field.dimension.is_time, - field.dimension is not None) + field.name: field.dimension is not None for field in original_dataset.fields or [] } roundtripped_fields = { - field.name: (field.dimension is not None and field.dimension.is_time, - field.dimension is not None) + field.name: field.dimension is not None for field in roundtripped_dataset.fields or [] } assert roundtripped_fields == original_fields + def test_datatype_categories_survive_the_round_trip(self): + """Lightdash types are coarser than Ossie datatypes, so a round-trip + preserves the category (temporal / numeric / string / boolean) rather + than the exact member (e.g. Integer comes back as Decimal).""" + categories = { + OSIDataType.STRING: "string", + OSIDataType.BOOLEAN: "boolean", + OSIDataType.INTEGER: "number", + OSIDataType.DECIMAL: "number", + OSIDataType.FLOAT: "number", + OSIDataType.DATE: "date", + OSIDataType.DATE_TIME: "timestamp", + OSIDataType.DATE_TIME_TZ: "timestamp", + } + original, _, reimported = _roundtrip() + for original_dataset, roundtripped_dataset in zip( + original.semantic_model[0].datasets, + reimported.output.semantic_model[0].datasets, + ): + roundtripped_by_name = { + field.name: field for field in roundtripped_dataset.fields or [] + } + for field in original_dataset.fields or []: + # Measure-only fields have no Lightdash dimension to carry a + # type, so their datatype is not expected to survive. + if field.datatype not in categories or field.dimension is None: + continue + roundtripped = roundtripped_by_name[field.name] + assert categories.get(roundtripped.datatype) == categories[ + field.datatype + ], field.name + + def test_time_role_marker_is_not_carried_into_lightdash(self): + """`is_time` is an Ossie role marker with no Lightdash equivalent, so the + import direction leaves it unset instead of inferring one.""" + _, _, reimported = _roundtrip() + for dataset in reimported.output.semantic_model[0].datasets: + for field in dataset.fields or []: + if field.dimension is not None: + assert field.dimension.is_time is None, field.name + def test_single_dataset_metrics_survive_with_expressions(self): original, exported, reimported = _roundtrip() dropped = { From c6d8ce28761fe96c885cd4df7a54f868c327e41f Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 10:41:57 +0100 Subject: [PATCH 07/33] refactor(lightdash): rename OSI references to Ossie after upstream rename Upstream #288 renamed the Python models (OSIDocument -> OssieDocument, ...) and the osi_to_ / _to_osi module convention. Apply the same rename to the Lightdash converter so it imports and runs against main. --- converters/lightdash/README.md | 4 +- .../lightdash/src/ossie_lightdash/__init__.py | 8 +- .../lightdash/src/ossie_lightdash/cli.py | 16 +-- .../src/ossie_lightdash/converter_issues.py | 4 +- .../src/ossie_lightdash/datatype_utils.py | 40 +++---- .../src/ossie_lightdash/expression_utils.py | 8 +- ...htdash_to_osi.py => lightdash_to_ossie.py} | 90 ++++++++-------- ..._to_lightdash.py => ossie_to_lightdash.py} | 36 +++---- ...h_to_osi.py => test_lightdash_to_ossie.py} | 38 +++---- ...ightdash.py => test_ossie_to_lightdash.py} | 100 +++++++++--------- .../lightdash/tests/test_tpcds_roundtrip.py | 30 +++--- 11 files changed, 187 insertions(+), 187 deletions(-) rename converters/lightdash/src/ossie_lightdash/{lightdash_to_osi.py => lightdash_to_ossie.py} (85%) rename converters/lightdash/src/ossie_lightdash/{osi_to_lightdash.py => ossie_to_lightdash.py} (90%) rename converters/lightdash/tests/{test_lightdash_to_osi.py => test_lightdash_to_ossie.py} (87%) rename converters/lightdash/tests/{test_osi_to_lightdash.py => test_ossie_to_lightdash.py} (77%) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 0466646e..793b9f0f 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -25,9 +25,9 @@ Lightdash reads its semantic layer from dbt `schema.yml` files: dimensions and metrics are declared per column (and per model) under `meta`. This converter translates between that shape and Ossie. -- **Export** (`osi_to_lightdash`): Ossie document → a dbt `schema.yml`-shaped +- **Export** (`ossie_to_lightdash`): Ossie document → a dbt `schema.yml`-shaped dictionary with Lightdash `meta` blocks, ready to merge into a dbt project. -- **Import** (`lightdash_to_osi`): a Lightdash-flavoured `schema.yml` → an +- **Import** (`lightdash_to_ossie`): a Lightdash-flavoured `schema.yml` → an Ossie document, as a migration path for teams with an existing installed base of Lightdash metrics. diff --git a/converters/lightdash/src/ossie_lightdash/__init__.py b/converters/lightdash/src/ossie_lightdash/__init__.py index afeea775..e3cd0de7 100644 --- a/converters/lightdash/src/ossie_lightdash/__init__.py +++ b/converters/lightdash/src/ossie_lightdash/__init__.py @@ -20,13 +20,13 @@ ConverterIssueType, ConverterResult, ) -from ossie_lightdash.lightdash_to_osi import LightdashToOSIConverter -from ossie_lightdash.osi_to_lightdash import OSIToLightdashConverter +from ossie_lightdash.lightdash_to_ossie import LightdashToOssieConverter +from ossie_lightdash.ossie_to_lightdash import OssieToLightdashConverter __all__ = [ "ConverterIssue", "ConverterIssueType", "ConverterResult", - "LightdashToOSIConverter", - "OSIToLightdashConverter", + "LightdashToOssieConverter", + "OssieToLightdashConverter", ] diff --git a/converters/lightdash/src/ossie_lightdash/cli.py b/converters/lightdash/src/ossie_lightdash/cli.py index b42bd3c3..35d4d5b3 100644 --- a/converters/lightdash/src/ossie_lightdash/cli.py +++ b/converters/lightdash/src/ossie_lightdash/cli.py @@ -24,16 +24,16 @@ import yaml -from ossie import OSIDocument -from ossie_lightdash.lightdash_to_osi import LightdashToOSIConverter -from ossie_lightdash.osi_to_lightdash import OSIToLightdashConverter +from ossie import OssieDocument +from ossie_lightdash.lightdash_to_ossie import LightdashToOssieConverter +from ossie_lightdash.ossie_to_lightdash import OssieToLightdashConverter -def _read_document(path: Path) -> OSIDocument: +def _read_document(path: Path) -> OssieDocument: text = path.read_text(encoding="utf-8") if path.suffix == ".json": - return OSIDocument.model_validate_json(text) - return OSIDocument.model_validate(yaml.safe_load(text)) + return OssieDocument.model_validate_json(text) + return OssieDocument.model_validate(yaml.safe_load(text)) def _print_issues(issues) -> None: @@ -65,14 +65,14 @@ def main() -> int: args = parser.parse_args() if args.command == "export": - result = OSIToLightdashConverter().convert(_read_document(args.input)) + result = OssieToLightdashConverter().convert(_read_document(args.input)) args.output.write_text( yaml.safe_dump(result.output, sort_keys=False, allow_unicode=True), encoding="utf-8", ) else: schema_yml = yaml.safe_load(args.input.read_text(encoding="utf-8")) - result = LightdashToOSIConverter().convert( + result = LightdashToOssieConverter().convert( schema_yml, database=args.database, schema=args.schema, diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py index 87c6cf5f..4ae439dc 100644 --- a/converters/lightdash/src/ossie_lightdash/converter_issues.py +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -36,14 +36,14 @@ class ConverterIssueType(Enum): # Export: a `lightdash` extension whose data is not valid JSON cannot be # applied; its presentation attributes are lost. EXTENSION_DATA_INVALID = "EXTENSION_DATA_INVALID" - # Import: a model-level metric without `sql` has no expressible OSI + # Import: a model-level metric without `sql` has no expressible Ossie # expression and is skipped. METRIC_SQL_MISSING = "METRIC_SQL_MISSING" # Export: a field is marked as a time-axis role (`dimension.is_time`) that # its datatype does not already imply; Lightdash has no equivalent marker. TIME_ROLE_NOT_REPRESENTABLE = "TIME_ROLE_NOT_REPRESENTABLE" # Export: a custom extension from another vendor cannot be carried into - # Lightdash meta (it remains in the OSI document itself). + # Lightdash meta (it remains in the Ossie document itself). FOREIGN_EXTENSION_IGNORED = "FOREIGN_EXTENSION_IGNORED" diff --git a/converters/lightdash/src/ossie_lightdash/datatype_utils.py b/converters/lightdash/src/ossie_lightdash/datatype_utils.py index 21e4c4d2..40de1076 100644 --- a/converters/lightdash/src/ossie_lightdash/datatype_utils.py +++ b/converters/lightdash/src/ossie_lightdash/datatype_utils.py @@ -26,51 +26,51 @@ from typing import Optional -from ossie import OSIDataType +from ossie import OssieDataType # Ossie datatype -> Lightdash dimension type. _DATATYPE_TO_LIGHTDASH = { - OSIDataType.STRING: "string", - OSIDataType.INTEGER: "number", - OSIDataType.DECIMAL: "number", - OSIDataType.FLOAT: "number", - OSIDataType.BOOLEAN: "boolean", - OSIDataType.DATE: "date", - OSIDataType.DATE_TIME: "timestamp", - OSIDataType.DATE_TIME_TZ: "timestamp", + OssieDataType.STRING: "string", + OssieDataType.INTEGER: "number", + OssieDataType.DECIMAL: "number", + OssieDataType.FLOAT: "number", + OssieDataType.BOOLEAN: "boolean", + OssieDataType.DATE: "date", + OssieDataType.DATE_TIME: "timestamp", + OssieDataType.DATE_TIME_TZ: "timestamp", # Lightdash has no time-of-day dimension type; a string keeps the value # visible rather than dropping the column. - OSIDataType.TIME: "string", - OSIDataType.OPAQUE: "string", + OssieDataType.TIME: "string", + OssieDataType.OPAQUE: "string", } # Lightdash dimension type -> Ossie datatype. Numeric widths are not expressed # in Lightdash, so `number` maps to the widest exact type. _LIGHTDASH_TO_DATATYPE = { - "string": OSIDataType.STRING, - "number": OSIDataType.DECIMAL, - "boolean": OSIDataType.BOOLEAN, - "date": OSIDataType.DATE, - "timestamp": OSIDataType.DATE_TIME, + "string": OssieDataType.STRING, + "number": OssieDataType.DECIMAL, + "boolean": OssieDataType.BOOLEAN, + "date": OssieDataType.DATE, + "timestamp": OssieDataType.DATE_TIME, } -_TEMPORAL = {OSIDataType.DATE, OSIDataType.TIME, OSIDataType.DATE_TIME, OSIDataType.DATE_TIME_TZ} +_TEMPORAL = {OssieDataType.DATE, OssieDataType.TIME, OssieDataType.DATE_TIME, OssieDataType.DATE_TIME_TZ} -def datatype_to_lightdash_type(datatype: Optional[OSIDataType]) -> Optional[str]: +def datatype_to_lightdash_type(datatype: Optional[OssieDataType]) -> Optional[str]: """Return the Lightdash dimension type for an Ossie datatype, if any.""" if datatype is None: return None return _DATATYPE_TO_LIGHTDASH.get(datatype) -def lightdash_type_to_datatype(lightdash_type: Optional[str]) -> Optional[OSIDataType]: +def lightdash_type_to_datatype(lightdash_type: Optional[str]) -> Optional[OssieDataType]: """Return the Ossie datatype for a Lightdash dimension type, if any.""" if lightdash_type is None: return None return _LIGHTDASH_TO_DATATYPE.get(lightdash_type) -def is_temporal(datatype: Optional[OSIDataType]) -> bool: +def is_temporal(datatype: Optional[OssieDataType]) -> bool: """True when the datatype represents a point in time.""" return datatype in _TEMPORAL diff --git a/converters/lightdash/src/ossie_lightdash/expression_utils.py b/converters/lightdash/src/ossie_lightdash/expression_utils.py index 275abe0e..571b1032 100644 --- a/converters/lightdash/src/ossie_lightdash/expression_utils.py +++ b/converters/lightdash/src/ossie_lightdash/expression_utils.py @@ -18,7 +18,7 @@ """Small expression helpers shared by both conversion directions. Lightdash SQL snippets reference columns as ``${TABLE}.column`` (and joined -tables as ``${other_table.column}``); OSI expressions reference them as +tables as ``${other_table.column}``); Ossie expressions reference them as ``dataset.column``. These helpers translate between the two spellings and recognise the single-aggregation shapes that map onto Lightdash's typed metrics. @@ -75,7 +75,7 @@ def parse_simple_aggregation(expression: str) -> Optional[Tuple[str, str]]: def build_aggregation(lightdash_type: str, dataset: str, column: str) -> Optional[str]: - """Build the OSI expression for a typed Lightdash metric, if it has one.""" + """Build the Ossie expression for a typed Lightdash metric, if it has one.""" if lightdash_type == "count_distinct": return f"COUNT(DISTINCT {dataset}.{column})" agg = _LIGHTDASH_TYPE_TO_AGG.get(lightdash_type) @@ -106,7 +106,7 @@ def osi_sql_to_lightdash(expression: str, dataset: str) -> str: def lightdash_sql_to_osi(sql: str, dataset: str) -> str: - """Rewrite Lightdash column references into OSI ``dataset.column`` references. + """Rewrite Lightdash column references into Ossie ``dataset.column`` references. ``${TABLE}.column`` refers to the current model; ``${other_table.column}`` refers to a joined model and becomes a cross-dataset reference. @@ -117,7 +117,7 @@ def lightdash_sql_to_osi(sql: str, dataset: str) -> str: def referenced_datasets(expression: str, dataset_names: set) -> set: - """Return which of the given dataset names an OSI expression references.""" + """Return which of the given dataset names an Ossie expression references.""" found = set() for match in re.finditer(r"([A-Za-z_]\w*)\.\w+", expression): if match.group(1) in dataset_names: diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py similarity index 85% rename from converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py rename to converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py index 6c300da8..bab9e5e8 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_osi.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py @@ -15,12 +15,12 @@ # specific language governing permissions and limitations # under the License. -"""Convert Lightdash semantic definitions into an OSI document. +"""Convert Lightdash semantic definitions into an Ossie document. The input is a dbt ``schema.yml``-shaped dictionary whose ``meta`` blocks carry Lightdash dimensions, metrics and joins. Structural information becomes -first-class OSI vocabulary (datasets, fields, metrics, relationships); -Lightdash presentation attributes without OSI vocabulary (``format``, +first-class Ossie vocabulary (datasets, fields, metrics, relationships); +Lightdash presentation attributes without Ossie vocabulary (``format``, ``round``, ``group_label``, ``hidden``, ...) are preserved in ``custom_extensions`` entries with ``vendor_name: "lightdash"`` so that the export direction can reproduce them exactly. @@ -31,17 +31,17 @@ from typing import Any, Dict, List, Optional, Tuple from ossie import ( - OSICustomExtension, - OSIDataset, - OSIDialect, - OSIDialectExpression, - OSIDimension, - OSIDocument, - OSIExpression, - OSIField, - OSIMetric, - OSIRelationship, - OSISemanticModel, + OssieCustomExtension, + OssieDataset, + OssieDialect, + OssieDialectExpression, + OssieDimension, + OssieDocument, + OssieExpression, + OssieField, + OssieMetric, + OssieRelationship, + OssieSemanticModel, ) from ossie_lightdash.converter_issues import ( @@ -57,9 +57,9 @@ LIGHTDASH_VENDOR_NAME = "lightdash" -# Keys that are structurally encoded in OSI vocabulary and therefore must NOT +# Keys that are structurally encoded in Ossie vocabulary and therefore must NOT # be duplicated into the extension (a stale copy would win on export). -# ``type`` stays in the extension only for metric types whose semantics OSI +# ``type`` stays in the extension only for metric types whose semantics Ossie # expressions cannot express faithfully (currently ``percentile``). _STRUCTURAL_METRIC_KEYS = {"sql", "description"} _STRUCTURAL_DIMENSION_KEYS = {"label", "sql"} @@ -69,16 +69,16 @@ ) -def _ansi(expression: str) -> OSIExpression: - return OSIExpression( +def _ansi(expression: str) -> OssieExpression: + return OssieExpression( dialects=[ - OSIDialectExpression(dialect=OSIDialect.ANSI_SQL, expression=expression) + OssieDialectExpression(dialect=OssieDialect.ANSI_SQL, expression=expression) ] ) def _type_needs_extension(lightdash_type: str) -> bool: - """True for metric types an OSI expression cannot encode faithfully. + """True for metric types an Ossie expression cannot encode faithfully. ``number`` is fully described by its SQL and typed aggregations are recovered by parsing the expression, so only the remaining types @@ -89,19 +89,19 @@ def _type_needs_extension(lightdash_type: str) -> bool: return build_aggregation(lightdash_type, "_", "_") is None -def _lightdash_extension(data: Dict[str, Any]) -> List[OSICustomExtension]: +def _lightdash_extension(data: Dict[str, Any]) -> List[OssieCustomExtension]: if not data: return [] return [ - OSICustomExtension( + OssieCustomExtension( vendor_name=LIGHTDASH_VENDOR_NAME, data=json.dumps(data, ensure_ascii=False, sort_keys=True), ) ] -class LightdashToOSIConverter: - """Converts a Lightdash-flavoured dbt schema.yml dict into an OSIDocument.""" +class LightdashToOssieConverter: + """Converts a Lightdash-flavoured dbt schema.yml dict into an OssieDocument.""" def convert( self, @@ -110,11 +110,11 @@ def convert( database: Optional[str] = None, schema: Optional[str] = None, semantic_model_name: str = "lightdash_semantic_model", - ) -> ConverterResult[OSIDocument]: + ) -> ConverterResult[OssieDocument]: issues: List[ConverterIssue] = [] - datasets: List[OSIDataset] = [] - metrics: List[OSIMetric] = [] - relationships: List[OSIRelationship] = [] + datasets: List[OssieDataset] = [] + metrics: List[OssieMetric] = [] + relationships: List[OssieRelationship] = [] for model in schema_yml.get("models") or []: dataset, model_metrics, model_relationships = self._convert_model( @@ -124,10 +124,10 @@ def convert( metrics.extend(model_metrics) relationships.extend(model_relationships) - document = OSIDocument( + document = OssieDocument( version="0.2.0.dev0", semantic_model=[ - OSISemanticModel( + OssieSemanticModel( name=semantic_model_name, datasets=datasets, metrics=metrics or None, @@ -144,7 +144,7 @@ def _convert_model( database: Optional[str], schema: Optional[str], issues: List[ConverterIssue], - ) -> Tuple[OSIDataset, List[OSIMetric], List[OSIRelationship]]: + ) -> Tuple[OssieDataset, List[OssieMetric], List[OssieRelationship]]: name = model["name"] source = ".".join(part for part in [database, schema, name] if part) if schema is None: @@ -155,8 +155,8 @@ def _convert_model( ) ) - fields: List[OSIField] = [] - metrics: List[OSIMetric] = [] + fields: List[OssieField] = [] + metrics: List[OssieMetric] = [] for column in model.get("columns") or []: field, column_metrics = self._convert_column(column, dataset_name=name) fields.append(field) @@ -180,7 +180,7 @@ def _convert_model( model_meta.get("joins") or [], from_model=name, issues=issues ) - dataset = OSIDataset( + dataset = OssieDataset( name=name, source=source, description=model.get("description"), @@ -190,13 +190,13 @@ def _convert_model( def _convert_column( self, column: Dict[str, Any], *, dataset_name: str - ) -> Tuple[OSIField, List[OSIMetric]]: + ) -> Tuple[OssieField, List[OssieMetric]]: column_name = column["name"] meta = column.get("meta") or {} dimension_meta = meta.get("dimension") expression = column_name - dimension: Optional[OSIDimension] = None + dimension: Optional[OssieDimension] = None datatype = None label: Optional[str] = None extension_data: Dict[str, Any] = {} @@ -208,14 +208,14 @@ def _convert_column( # `is_time` is a role marker in Ossie, not a type: Lightdash has no # equivalent, so it is left unset rather than inferred from the type # (the type itself is carried by `datatype`). - dimension = OSIDimension() + dimension = OssieDimension() extension_data = { key: value for key, value in dimension_meta.items() if key not in _STRUCTURAL_DIMENSION_KEYS } - field = OSIField( + field = OssieField( name=column_name, expression=_ansi(expression), dimension=dimension, @@ -240,7 +240,7 @@ def _convert_column_metric( *, dataset_name: str, column: str, - ) -> OSIMetric: + ) -> OssieMetric: lightdash_type = definition.get("type", "number") expression = build_aggregation(lightdash_type, dataset_name, column) if expression is None: @@ -259,7 +259,7 @@ def _convert_column_metric( def _convert_sql_metric( self, metric_name: str, definition: Dict[str, Any], *, dataset_name: str - ) -> OSIMetric: + ) -> OssieMetric: expression = lightdash_sql_to_osi(definition["sql"], dataset_name) return self._build_metric( metric_name, @@ -275,14 +275,14 @@ def _build_metric( *, expression: str, keep_type_in_extension: bool, - ) -> OSIMetric: + ) -> OssieMetric: excluded = set(_STRUCTURAL_METRIC_KEYS) if not keep_type_in_extension: excluded.add("type") extension_data = { key: value for key, value in definition.items() if key not in excluded } - return OSIMetric( + return OssieMetric( name=metric_name, expression=_ansi(expression), description=definition.get("description"), @@ -295,8 +295,8 @@ def _convert_joins( *, from_model: str, issues: List[ConverterIssue], - ) -> List[OSIRelationship]: - relationships: List[OSIRelationship] = [] + ) -> List[OssieRelationship]: + relationships: List[OssieRelationship] = [] for join in joins: to_model = join.get("join") pairs = _JOIN_PAIR_RE.findall(join.get("sql_on") or "") @@ -318,7 +318,7 @@ def _convert_joins( ) continue relationships.append( - OSIRelationship.model_validate( + OssieRelationship.model_validate( { "name": f"{from_model}_to_{to_model}", "from": from_model, diff --git a/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py similarity index 90% rename from converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py rename to converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py index 4fb017ea..8bbff9de 100644 --- a/converters/lightdash/src/ossie_lightdash/osi_to_lightdash.py +++ b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py @@ -15,22 +15,22 @@ # specific language governing permissions and limitations # under the License. -"""Convert an OSI document into Lightdash semantic definitions. +"""Convert an Ossie document into Lightdash semantic definitions. The output is a dbt ``schema.yml``-shaped dictionary whose ``meta`` blocks carry Lightdash dimensions, metrics and joins, ready to be merged into a dbt project that Lightdash reads. Lightdash-specific presentation attributes that -have no OSI vocabulary round-trip through ``custom_extensions`` entries with +have no Ossie vocabulary round-trip through ``custom_extensions`` entries with ``vendor_name: "lightdash"``; their keys are overlaid onto the generated definitions and win for presentation attributes, while structural keys (``sql``/``label`` on dimensions, ``sql``/``description`` on metrics) are -protected so they can never override the OSI-derived definition. +protected so they can never override the Ossie-derived definition. """ import json from typing import Any, Dict, List, Optional -from ossie import OSIDataset, OSIDialect, OSIDocument, OSIMetric, OSISemanticModel +from ossie import OssieDataset, OssieDialect, OssieDocument, OssieMetric, OssieSemanticModel from ossie_lightdash.converter_issues import ( ConverterIssue, @@ -48,16 +48,16 @@ LIGHTDASH_VENDOR_NAME = "lightdash" -# Structural keys are owned by OSI vocabulary (the import direction never puts +# Structural keys are owned by Ossie vocabulary (the import direction never puts # them into the extension); dropping them here keeps a hand-authored extension -# from overriding the OSI-derived definition. ``type`` stays overridable on -# metrics: it is the documented channel for types OSI expressions cannot +# from overriding the Ossie-derived definition. ``type`` stays overridable on +# metrics: it is the documented channel for types Ossie expressions cannot # express (e.g. percentile). _PROTECTED_DIMENSION_KEYS = {"sql", "label"} _PROTECTED_METRIC_KEYS = {"sql", "description"} -def _pick_expression(osi_expression: Any, dialect: OSIDialect) -> str: +def _pick_expression(osi_expression: Any, dialect: OssieDialect) -> str: """Return the expression for the preferred dialect (fallback: first available).""" for dialect_expression in osi_expression.dialects: if dialect_expression.dialect is dialect: @@ -66,7 +66,7 @@ def _pick_expression(osi_expression: Any, dialect: OSIDialect) -> str: def _lightdash_extension_data(element: Any, issues: List[ConverterIssue]) -> Dict[str, Any]: - """Return the ``lightdash`` vendor extension data of an OSI element, if any.""" + """Return the ``lightdash`` vendor extension data of an Ossie element, if any.""" data: Dict[str, Any] = {} for extension in element.custom_extensions or []: if extension.vendor_name == LIGHTDASH_VENDOR_NAME: @@ -89,18 +89,18 @@ def _lightdash_extension_data(element: Any, issues: List[ConverterIssue]) -> Dic return data -def _model_name_for(dataset: OSIDataset) -> str: +def _model_name_for(dataset: OssieDataset) -> str: """A Lightdash table is addressed by its dbt model name = the source's table part.""" return dataset.source.rsplit(".", 1)[-1] -class OSIToLightdashConverter: - """Converts an OSIDocument into a Lightdash-flavoured dbt schema.yml dict.""" +class OssieToLightdashConverter: + """Converts an OssieDocument into a Lightdash-flavoured dbt schema.yml dict.""" - def __init__(self, dialect: OSIDialect = OSIDialect.ANSI_SQL) -> None: + def __init__(self, dialect: OssieDialect = OssieDialect.ANSI_SQL) -> None: self._dialect = dialect - def convert(self, document: OSIDocument) -> ConverterResult[Dict[str, Any]]: + def convert(self, document: OssieDocument) -> ConverterResult[Dict[str, Any]]: issues: List[ConverterIssue] = [] models: List[Dict[str, Any]] = [] for semantic_model in document.semantic_model: @@ -108,7 +108,7 @@ def convert(self, document: OSIDocument) -> ConverterResult[Dict[str, Any]]: return ConverterResult(output={"version": 2, "models": models}, issues=issues) def _convert_semantic_model( - self, semantic_model: OSISemanticModel, issues: List[ConverterIssue] + self, semantic_model: OssieSemanticModel, issues: List[ConverterIssue] ) -> List[Dict[str, Any]]: datasets = semantic_model.datasets or [] dataset_names = {dataset.name for dataset in datasets} @@ -158,7 +158,7 @@ def _convert_semantic_model( return [models_by_dataset[dataset.name] for dataset in datasets] def _convert_dataset( - self, dataset: OSIDataset, issues: List[ConverterIssue] + self, dataset: OssieDataset, issues: List[ConverterIssue] ) -> tuple: columns_by_name: Dict[str, Dict[str, Any]] = {} for field in dataset.fields or []: @@ -203,7 +203,7 @@ def _convert_dataset( if key not in _PROTECTED_DIMENSION_KEYS } ) - # An empty dict still marks dimension-ness: a field OSI declares as a + # An empty dict still marks dimension-ness: a field Ossie declares as a # categorical dimension must not degrade to a plain column on export, # or the import direction could not reconstruct it. if dimension or field.dimension is not None: @@ -218,7 +218,7 @@ def _convert_dataset( def _convert_metric( self, - metric: OSIMetric, + metric: OssieMetric, dataset_names: set, models_by_dataset: Dict[str, Dict[str, Any]], columns_by_dataset: Dict[str, Dict[str, Dict[str, Any]]], diff --git a/converters/lightdash/tests/test_lightdash_to_osi.py b/converters/lightdash/tests/test_lightdash_to_ossie.py similarity index 87% rename from converters/lightdash/tests/test_lightdash_to_osi.py rename to converters/lightdash/tests/test_lightdash_to_ossie.py index 6082dcf0..1f9e9beb 100644 --- a/converters/lightdash/tests/test_lightdash_to_osi.py +++ b/converters/lightdash/tests/test_lightdash_to_ossie.py @@ -17,9 +17,9 @@ import json -from ossie import OSIDataType +from ossie import OssieDataType -from ossie_lightdash import ConverterIssueType, LightdashToOSIConverter +from ossie_lightdash import ConverterIssueType, LightdashToOssieConverter SCHEMA_YML = { "version": 2, @@ -100,9 +100,9 @@ def _lightdash_data(element): return {} -class TestLightdashToOSI: +class TestLightdashToOssie: def test_dataset_source_is_qualified(self): - result = LightdashToOSIConverter().convert( + result = LightdashToOssieConverter().convert( SCHEMA_YML, database="analytics_db", schema="marts" ) dataset = result.output.semantic_model[0].datasets[0] @@ -113,7 +113,7 @@ def test_dataset_source_is_qualified(self): ) def test_missing_schema_is_reported(self): - result = LightdashToOSIConverter().convert(SCHEMA_YML) + result = LightdashToOssieConverter().convert(SCHEMA_YML) dataset = result.output.semantic_model[0].datasets[0] assert dataset.source == "orders" assert any( @@ -122,7 +122,7 @@ def test_missing_schema_is_reported(self): ) def test_time_dimension(self): - result = LightdashToOSIConverter().convert(SCHEMA_YML, schema="marts") + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") field = result.output.semantic_model[0].datasets[0].fields[0] assert field.name == "order_date" assert field.label == "Order date" @@ -130,26 +130,26 @@ def test_time_dimension(self): assert field.dimension is not None # The Lightdash type becomes a datatype; `is_time` is an Ossie role # marker with no Lightdash source, so it stays unset. - assert field.datatype is OSIDataType.DATE + assert field.datatype is OssieDataType.DATE assert field.dimension.is_time is None def test_dimension_types_become_datatypes(self): - result = LightdashToOSIConverter().convert(SCHEMA_YML, schema="marts") + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") by_name = { field.name: field for field in result.output.semantic_model[0].datasets[0].fields } - assert by_name["status"].datatype is OSIDataType.STRING - assert by_name["order_date"].datatype is OSIDataType.DATE + assert by_name["status"].datatype is OssieDataType.STRING + assert by_name["order_date"].datatype is OssieDataType.DATE def test_typed_metric_becomes_aggregation_expression(self): - result = LightdashToOSIConverter().convert(SCHEMA_YML, schema="marts") + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") metric = _metric(result.output, "total_amount") assert metric.expression.dialects[0].expression == "SUM(orders.amount)" assert _lightdash_data(metric) == {"label": "Total amount", "format": "usd"} def test_count_distinct_metric(self): - result = LightdashToOSIConverter().convert(SCHEMA_YML, schema="marts") + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") metric = _metric(result.output, "unique_customers") assert ( metric.expression.dialects[0].expression @@ -157,12 +157,12 @@ def test_count_distinct_metric(self): ) def test_percentile_metric_keeps_type_in_extension(self): - result = LightdashToOSIConverter().convert(SCHEMA_YML, schema="marts") + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") metric = _metric(result.output, "p90_amount") assert _lightdash_data(metric) == {"type": "percentile", "percentile": 90} def test_sql_metric_expression_is_rewritten(self): - result = LightdashToOSIConverter().convert(SCHEMA_YML, schema="marts") + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") metric = _metric(result.output, "conversion_rate") assert ( metric.expression.dialects[0].expression @@ -175,7 +175,7 @@ def test_sql_metric_expression_is_rewritten(self): } def test_join_becomes_relationship(self): - result = LightdashToOSIConverter().convert(SCHEMA_YML, schema="marts") + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") relationship = result.output.semantic_model[0].relationships[0] assert relationship.from_dataset == "orders" assert relationship.to == "customers" @@ -200,7 +200,7 @@ def test_percentile_with_sql_keeps_type_in_extension(self): } ] } - result = LightdashToOSIConverter().convert(schema_yml, schema="marts") + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") metric = _metric(result.output, "p90_custom") assert ( metric.expression.dialects[0].expression @@ -225,7 +225,7 @@ def test_joined_table_references_become_cross_dataset(self): } ] } - result = LightdashToOSIConverter().convert(schema_yml, schema="marts") + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") metric = _metric(result.output, "orders_per_customer") assert ( metric.expression.dialects[0].expression @@ -242,7 +242,7 @@ def test_model_metric_without_sql_is_skipped(self): } ] } - result = LightdashToOSIConverter().convert(schema_yml, schema="marts") + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") assert result.output.semantic_model[0].metrics is None assert any( issue.issue_type is ConverterIssueType.METRIC_SQL_MISSING @@ -262,7 +262,7 @@ def test_unparseable_join_is_reported(self): } ] } - result = LightdashToOSIConverter().convert(schema_yml, schema="marts") + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") assert result.output.semantic_model[0].relationships is None assert any( issue.issue_type is ConverterIssueType.JOIN_SQL_UNPARSED diff --git a/converters/lightdash/tests/test_osi_to_lightdash.py b/converters/lightdash/tests/test_ossie_to_lightdash.py similarity index 77% rename from converters/lightdash/tests/test_osi_to_lightdash.py rename to converters/lightdash/tests/test_ossie_to_lightdash.py index 1f48fb06..2fcb6cf8 100644 --- a/converters/lightdash/tests/test_osi_to_lightdash.py +++ b/converters/lightdash/tests/test_ossie_to_lightdash.py @@ -18,94 +18,94 @@ import json from ossie import ( - OSICustomExtension, - OSIDataset, - OSIDialect, - OSIDialectExpression, - OSIDimension, - OSIDocument, - OSIExpression, - OSIField, - OSIMetric, - OSIRelationship, - OSISemanticModel, + OssieCustomExtension, + OssieDataset, + OssieDialect, + OssieDialectExpression, + OssieDimension, + OssieDocument, + OssieExpression, + OssieField, + OssieMetric, + OssieRelationship, + OssieSemanticModel, ) -from ossie_lightdash import ConverterIssueType, OSIToLightdashConverter +from ossie_lightdash import ConverterIssueType, OssieToLightdashConverter -def _ansi(expression: str) -> OSIExpression: - return OSIExpression( +def _ansi(expression: str) -> OssieExpression: + return OssieExpression( dialects=[ - OSIDialectExpression(dialect=OSIDialect.ANSI_SQL, expression=expression) + OssieDialectExpression(dialect=OssieDialect.ANSI_SQL, expression=expression) ] ) -def _document() -> OSIDocument: - orders = OSIDataset( +def _document() -> OssieDocument: + orders = OssieDataset( name="orders", source="analytics_db.marts.orders", description="One row per order", fields=[ - OSIField( + OssieField( name="order_date", expression=_ansi("order_date"), - dimension=OSIDimension(is_time=True), + dimension=OssieDimension(is_time=True), label="Order date", ), - OSIField( + OssieField( name="status", expression=_ansi("status"), - dimension=OSIDimension(is_time=False), + dimension=OssieDimension(is_time=False), ), - OSIField(name="amount", expression=_ansi("amount")), - OSIField(name="customer_id", expression=_ansi("customer_id")), + OssieField(name="amount", expression=_ansi("amount")), + OssieField(name="customer_id", expression=_ansi("customer_id")), ], ) - customers = OSIDataset( + customers = OssieDataset( name="customers", source="analytics_db.marts.customers", - fields=[OSIField(name="customer_id", expression=_ansi("customer_id"))], + fields=[OssieField(name="customer_id", expression=_ansi("customer_id"))], ) metrics = [ - OSIMetric( + OssieMetric( name="total_amount", expression=_ansi("SUM(orders.amount)"), description="Sum of order amounts", custom_extensions=[ - OSICustomExtension( + OssieCustomExtension( vendor_name="lightdash", data=json.dumps({"label": "Total amount", "format": "usd"}), ) ], ), - OSIMetric( + OssieMetric( name="conversion_rate", expression=_ansi( "SUM(orders.completed_count) / NULLIF(SUM(orders.total_count), 0)" ), custom_extensions=[ - OSICustomExtension( + OssieCustomExtension( vendor_name="lightdash", data=json.dumps({"format": "percent", "round": 1}), ) ], ), - OSIMetric( + OssieMetric( name="cross_dataset", expression=_ansi("SUM(orders.amount) / COUNT(customers.customer_id)"), ), - OSIMetric( + OssieMetric( name="foreign_vendor_metric", expression=_ansi("SUM(orders.amount)"), custom_extensions=[ - OSICustomExtension(vendor_name="somebi", data='{"x": 1}') + OssieCustomExtension(vendor_name="somebi", data='{"x": 1}') ], ), ] relationships = [ - OSIRelationship.model_validate( + OssieRelationship.model_validate( { "name": "orders_to_customers", "from": "orders", @@ -115,10 +115,10 @@ def _document() -> OSIDocument: } ) ] - return OSIDocument( + return OssieDocument( version="0.2.0.dev0", semantic_model=[ - OSISemanticModel( + OssieSemanticModel( name="sales", datasets=[orders, customers], metrics=metrics, @@ -136,24 +136,24 @@ def _column(model, name): return next(c for c in model["columns"] if c["name"] == name) -class TestOSIToLightdash: +class TestOssieToLightdash: def test_time_dimension_exports_date_type(self): - result = OSIToLightdashConverter().convert(_document()) + result = OssieToLightdashConverter().convert(_document()) column = _column(_model(result.output, "orders"), "order_date") assert column["meta"]["dimension"] == {"label": "Order date", "type": "date"} def test_categorical_dimension_keeps_dimension_marker(self): - result = OSIToLightdashConverter().convert(_document()) + result = OssieToLightdashConverter().convert(_document()) column = _column(_model(result.output, "orders"), "status") assert column["meta"]["dimension"] == {} def test_plain_field_has_no_dimension_meta(self): - result = OSIToLightdashConverter().convert(_document()) + result = OssieToLightdashConverter().convert(_document()) column = _column(_model(result.output, "orders"), "amount") assert "dimension" not in column.get("meta", {}) def test_simple_aggregation_becomes_column_metric(self): - result = OSIToLightdashConverter().convert(_document()) + result = OssieToLightdashConverter().convert(_document()) column = _column(_model(result.output, "orders"), "amount") metric = column["meta"]["metrics"]["total_amount"] assert metric["type"] == "sum" @@ -163,7 +163,7 @@ def test_simple_aggregation_becomes_column_metric(self): assert "sql" not in metric def test_complex_expression_becomes_model_metric(self): - result = OSIToLightdashConverter().convert(_document()) + result = OssieToLightdashConverter().convert(_document()) metric = _model(result.output, "orders")["meta"]["metrics"]["conversion_rate"] assert metric["type"] == "number" assert ( @@ -174,7 +174,7 @@ def test_complex_expression_becomes_model_metric(self): assert metric["round"] == 1 def test_cross_dataset_metric_is_dropped_with_issue(self): - result = OSIToLightdashConverter().convert(_document()) + result = OssieToLightdashConverter().convert(_document()) assert any( issue.issue_type is ConverterIssueType.CROSS_DATASET_METRIC_DROPPED and issue.element_name == "cross_dataset" @@ -182,7 +182,7 @@ def test_cross_dataset_metric_is_dropped_with_issue(self): ) def test_foreign_extension_is_reported(self): - result = OSIToLightdashConverter().convert(_document()) + result = OssieToLightdashConverter().convert(_document()) assert any( issue.issue_type is ConverterIssueType.FOREIGN_EXTENSION_IGNORED and issue.element_name == "foreign_vendor_metric" @@ -195,7 +195,7 @@ def test_extension_cannot_override_structural_keys(self): metric = tampered.semantic_model[0].metrics[0].model_copy( update={ "custom_extensions": [ - OSICustomExtension( + OssieCustomExtension( vendor_name="lightdash", data=json.dumps( {"label": "Total amount", "sql": "1 + 1", "description": "stale"} @@ -205,7 +205,7 @@ def test_extension_cannot_override_structural_keys(self): } ) tampered.semantic_model[0].metrics[0] = metric - result = OSIToLightdashConverter().convert(tampered) + result = OssieToLightdashConverter().convert(tampered) column = _column(_model(result.output, "orders"), "amount") exported = column["meta"]["metrics"]["total_amount"] assert exported["label"] == "Total amount" @@ -215,7 +215,7 @@ def test_extension_cannot_override_structural_keys(self): def test_mismatched_relationship_columns_are_skipped(self): document = _document() tampered = document.model_copy(deep=True) - relationship = OSIRelationship.model_validate( + relationship = OssieRelationship.model_validate( { "name": "broken", "from": "orders", @@ -225,7 +225,7 @@ def test_mismatched_relationship_columns_are_skipped(self): } ) tampered.semantic_model[0].relationships[0] = relationship - result = OSIToLightdashConverter().convert(tampered) + result = OssieToLightdashConverter().convert(tampered) assert "joins" not in _model(result.output, "orders").get("meta", {}) assert any( issue.issue_type is ConverterIssueType.RELATIONSHIP_COLUMNS_MISMATCHED @@ -239,12 +239,12 @@ def test_invalid_extension_json_is_reported(self): metric = tampered.semantic_model[0].metrics[0].model_copy( update={ "custom_extensions": [ - OSICustomExtension(vendor_name="lightdash", data="{not json") + OssieCustomExtension(vendor_name="lightdash", data="{not json") ] } ) tampered.semantic_model[0].metrics[0] = metric - result = OSIToLightdashConverter().convert(tampered) + result = OssieToLightdashConverter().convert(tampered) assert any( issue.issue_type is ConverterIssueType.EXTENSION_DATA_INVALID and issue.element_name == "total_amount" @@ -252,7 +252,7 @@ def test_invalid_extension_json_is_reported(self): ) def test_relationship_becomes_join(self): - result = OSIToLightdashConverter().convert(_document()) + result = OssieToLightdashConverter().convert(_document()) joins = _model(result.output, "orders")["meta"]["joins"] assert joins == [ { diff --git a/converters/lightdash/tests/test_tpcds_roundtrip.py b/converters/lightdash/tests/test_tpcds_roundtrip.py index 9dfe97fe..dc7e663a 100644 --- a/converters/lightdash/tests/test_tpcds_roundtrip.py +++ b/converters/lightdash/tests/test_tpcds_roundtrip.py @@ -28,25 +28,25 @@ import yaml -from ossie import OSIDataType, OSIDocument +from ossie import OssieDataType, OssieDocument from ossie_lightdash import ( ConverterIssueType, - LightdashToOSIConverter, - OSIToLightdashConverter, + LightdashToOssieConverter, + OssieToLightdashConverter, ) TPCDS_PATH = Path(__file__).parent / ".." / ".." / ".." / "examples" / "tpcds_semantic_model.yaml" -def _load_tpcds() -> OSIDocument: - return OSIDocument.model_validate(yaml.safe_load(TPCDS_PATH.read_text())) +def _load_tpcds() -> OssieDocument: + return OssieDocument.model_validate(yaml.safe_load(TPCDS_PATH.read_text())) def _roundtrip(): original = _load_tpcds() - exported = OSIToLightdashConverter().convert(original) - reimported = LightdashToOSIConverter().convert( + exported = OssieToLightdashConverter().convert(original) + reimported = LightdashToOssieConverter().convert( exported.output, database="tpcds", schema="public", @@ -89,14 +89,14 @@ def test_datatype_categories_survive_the_round_trip(self): preserves the category (temporal / numeric / string / boolean) rather than the exact member (e.g. Integer comes back as Decimal).""" categories = { - OSIDataType.STRING: "string", - OSIDataType.BOOLEAN: "boolean", - OSIDataType.INTEGER: "number", - OSIDataType.DECIMAL: "number", - OSIDataType.FLOAT: "number", - OSIDataType.DATE: "date", - OSIDataType.DATE_TIME: "timestamp", - OSIDataType.DATE_TIME_TZ: "timestamp", + OssieDataType.STRING: "string", + OssieDataType.BOOLEAN: "boolean", + OssieDataType.INTEGER: "number", + OssieDataType.DECIMAL: "number", + OssieDataType.FLOAT: "number", + OssieDataType.DATE: "date", + OssieDataType.DATE_TIME: "timestamp", + OssieDataType.DATE_TIME_TZ: "timestamp", } original, _, reimported = _roundtrip() for original_dataset, roundtripped_dataset in zip( From 1611644630e86d77226969e055b96f7ebaf64f5e Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 10:42:41 +0100 Subject: [PATCH 08/33] fix(lightdash): serialize enums when importing to a YAML document The import command dumped the pydantic model without mode="json", so a .yaml target raised RepresenterError on OssieDialect. Take argv in main() and cover both output formats with a CLI test. --- .../lightdash/src/ossie_lightdash/cli.py | 7 +-- converters/lightdash/tests/test_cli.py | 45 +++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 converters/lightdash/tests/test_cli.py diff --git a/converters/lightdash/src/ossie_lightdash/cli.py b/converters/lightdash/src/ossie_lightdash/cli.py index 35d4d5b3..e1d2704b 100644 --- a/converters/lightdash/src/ossie_lightdash/cli.py +++ b/converters/lightdash/src/ossie_lightdash/cli.py @@ -21,6 +21,7 @@ import json import sys from pathlib import Path +from typing import List, Optional import yaml @@ -41,7 +42,7 @@ def _print_issues(issues) -> None: print(f"[{issue.issue_type.value}] {issue.element_name}", file=sys.stderr) -def main() -> int: +def main(argv: Optional[List[str]] = None) -> int: parser = argparse.ArgumentParser(prog="ossie-lightdash") subparsers = parser.add_subparsers(dest="command", required=True) @@ -62,7 +63,7 @@ def main() -> int: "--semantic-model-name", default="lightdash_semantic_model" ) - args = parser.parse_args() + args = parser.parse_args(argv) if args.command == "export": result = OssieToLightdashConverter().convert(_read_document(args.input)) @@ -78,7 +79,7 @@ def main() -> int: schema=args.schema, semantic_model_name=args.semantic_model_name, ) - document = result.output.model_dump(by_alias=True, exclude_none=True) + document = result.output.model_dump(mode="json", by_alias=True, exclude_none=True) if args.output.suffix == ".json": args.output.write_text( json.dumps(document, indent=2, ensure_ascii=False), encoding="utf-8" diff --git a/converters/lightdash/tests/test_cli.py b/converters/lightdash/tests/test_cli.py new file mode 100644 index 00000000..01a5e35a --- /dev/null +++ b/converters/lightdash/tests/test_cli.py @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Command line round trips through both output formats.""" + +from pathlib import Path + +import pytest +import yaml + +from ossie import OssieDocument +from ossie_lightdash.cli import main + +TPCDS_PATH = Path(__file__).parent / ".." / ".." / ".." / "examples" / "tpcds_semantic_model.yaml" + + +@pytest.mark.parametrize("suffix", [".yaml", ".json"]) +def test_import_writes_a_loadable_document(tmp_path, suffix): + schema_yml = tmp_path / "schema.yml" + assert main(["export", str(TPCDS_PATH), str(schema_yml)]) == 0 + + document_path = tmp_path / f"semantic_model{suffix}" + assert main(["import", str(schema_yml), str(document_path), "--schema", "public"]) == 0 + + text = document_path.read_text(encoding="utf-8") + if suffix == ".json": + document = OssieDocument.model_validate_json(text) + else: + document = OssieDocument.model_validate(yaml.safe_load(text)) + assert {dataset.name for dataset in document.semantic_model[0].datasets} == { + "store_sales", "date_dim", "customer", "item", "store" + } From 8d6281c2853dfd69734712863eb27263a76aa597 Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 10:54:45 +0100 Subject: [PATCH 09/33] feat(lightdash): keep joins, references and aggregations intact on import Real Lightdash projects alias most joins, reference sibling fields as ${column}, other metrics as ${metric} and use typed metrics over custom SQL. The importer matched joins on the model name only, left every non-${TABLE} reference in the Ossie expression, ignored the sql of typed metrics and emitted percentile and distinct metrics as a bare column. - resolve join aliases when parsing sql_on; stash the alias and other join attributes in the relationship's lightdash extension and restore them on export, aliasing repeated dataset pairs so Lightdash can compile them - rewrite ${column}, ${TABLE} and ${alias.column} references; inline ${metric} references; skip elements that use parameters or user attributes with an EXPRESSION_NOT_PORTABLE issue - express every typed metric as an aggregation over its operand, including SUM(DISTINCT ...), AVG(DISTINCT ...) and PERCENTILE_CONT ... WITHIN GROUP, and parse those shapes back into typed metrics on export On the 37-model jaffle-shop project this takes relationships from 1 of 21 to 21 of 21 and removes every unrewritten reference from the document. --- converters/lightdash/README.md | 21 +- .../src/ossie_lightdash/converter_issues.py | 11 +- .../src/ossie_lightdash/expression_utils.py | 218 ++++++++++---- .../src/ossie_lightdash/lightdash_to_ossie.py | 280 ++++++++++++------ .../src/ossie_lightdash/ossie_to_lightdash.py | 88 ++++-- .../tests/test_lightdash_to_ossie.py | 206 ++++++++++++- .../tests/test_ossie_to_lightdash.py | 86 ++++++ 7 files changed, 727 insertions(+), 183 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 793b9f0f..676a9987 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -48,10 +48,12 @@ ossie-lightdash import schema.yml semantic_model.json --database analytics_db -- | `field.dimension.is_time` | *not carried* — it is a role marker (a field can be a time axis without a temporal datatype, e.g. a year stored as `Integer`) and Lightdash has no equivalent | | `field.label` / `.description` | `meta.dimension.label` / column `description` | | `field.expression` (≠ column name) | `meta.dimension.sql` (`dataset.col` ↔ `${TABLE}.col`) | -| `metric` with single-aggregation expression (`SUM(ds.col)`, `COUNT(DISTINCT ds.col)`, ...) | column-level `meta.metrics.` with a typed metric (`sum`, `count_distinct`, ...) | +| `metric` that is one aggregation over a column (`SUM(ds.col)`, `COUNT(DISTINCT ds.col)`, `SUM(DISTINCT ds.col)`, `PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY ds.col)`, ...) | column-level `meta.metrics.` with a typed metric (`sum`, `count_distinct`, `sum_distinct`, `percentile` + `percentile: 90`, ...) | +| `metric` that is one aggregation over any other expression (`AVG(CASE WHEN ds.status = 'done' THEN 1 ELSE 0 END)`) | model-level `meta.metrics.` with the typed metric and the operand as `sql` | | `metric` with any other single-dataset expression | model-level `meta.metrics.` with `type: number` + `sql` | -| `relationship` | `meta.joins` (`sql_on` built from / parsed into column pairs) | -| Lightdash presentation attributes (`label`, `format`, `round`, `compact`, `group_label`, `hidden`, `percentile`, ...) | `custom_extensions` with `vendor_name: "lightdash"`; on export the extension data is overlaid onto the generated definition (structural keys — `sql`/`label` on dimensions, `sql`/`description` on metrics — are protected and cannot be overridden) | +| `relationship` | `meta.joins` (`sql_on` built from / parsed into column pairs); the `alias` and other join attributes (`relationship`, `type`, `fields`, ...) travel in the relationship's `lightdash` extension, and a dataset joined more than once from the same model is aliased on export | +| `${TABLE}.col`, `${col}`, `${other_model.col}` in Lightdash SQL | `dataset.col` / `other_model.col`; `${alias.col}` is flattened onto the aliased model, `${metric}` is replaced by that metric's expression | +| Lightdash presentation attributes (`label`, `format`, `round`, `compact`, `group_label`, `hidden`, ...) | `custom_extensions` with `vendor_name: "lightdash"`; on export the extension data is overlaid onto the generated definition (structural keys — `sql`/`label` on dimensions, `sql`/`description` on metrics and `type`/`percentile` on metrics whose expression is a recognised aggregation, `join`/`sql_on` on joins — are protected and cannot be overridden) | Expressions are written under the `ANSI_SQL` dialect. Warehouse-specific dialects (e.g. `BIGQUERY`) can be added once the surrounding tooling resolves @@ -71,9 +73,16 @@ Omitting `--schema` as well is reported as a `SOURCE_UNQUALIFIED` issue. - **Cross-dataset metrics are dropped on export** (with a `CROSS_DATASET_METRIC_DROPPED` issue): a Lightdash model metric cannot reference other tables. -- **Percentile metrics** keep `type` / `percentile` in the `lightdash` - extension (Ossie expressions cannot express them faithfully) and re-export - as model-level metrics. +- **Parameter and user-attribute references** (`${lightdash.parameters.x}`, + `${ld.user.email}`) have no Ossie form: a dimension or metric whose SQL uses + them is skipped on import with an `EXPRESSION_NOT_PORTABLE` issue. +- **Metric-to-metric references** (`${other_metric}`) are inlined on import + (`METRIC_REFERENCE_INLINED`), since Ossie metrics cannot reference each + other; the export direction does not reconstruct the reference. +- **References through a join alias** (`${sold_date.year}`) are rewritten to + the joined dataset (`date_dim.year`) with an `ALIAS_REFERENCE_FLATTENED` + issue: Ossie has no aliases, so which of several joins to the same dataset + was meant is not preserved in the expression. - **`primary_key` / `unique_keys` are not exported** — Lightdash has no corresponding concept — and consequently cannot be reconstructed on import. - **`dataset.name` is not preserved when it differs from the source table diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py index 4ae439dc..d29b36a0 100644 --- a/converters/lightdash/src/ossie_lightdash/converter_issues.py +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -14,7 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - from dataclasses import dataclass from enum import Enum from typing import Generic, List, TypeVar @@ -45,6 +44,16 @@ class ConverterIssueType(Enum): # Export: a custom extension from another vendor cannot be carried into # Lightdash meta (it remains in the Ossie document itself). FOREIGN_EXTENSION_IGNORED = "FOREIGN_EXTENSION_IGNORED" + # Import: an expression references project parameters or user attributes + # (`${lightdash.parameters.x}`, `${ld.user.x}`), which have no Ossie form; + # the element is skipped. + EXPRESSION_NOT_PORTABLE = "EXPRESSION_NOT_PORTABLE" + # Import: a `${metric}` reference was replaced by that metric's expression, + # since Ossie metrics cannot reference each other. + METRIC_REFERENCE_INLINED = "METRIC_REFERENCE_INLINED" + # Import: a `${alias.column}` reference to an aliased join was rewritten to + # the joined dataset; Ossie has no join aliases, so the join path is lost. + ALIAS_REFERENCE_FLATTENED = "ALIAS_REFERENCE_FLATTENED" @dataclass(frozen=True) diff --git a/converters/lightdash/src/ossie_lightdash/expression_utils.py b/converters/lightdash/src/ossie_lightdash/expression_utils.py index 571b1032..8411616b 100644 --- a/converters/lightdash/src/ossie_lightdash/expression_utils.py +++ b/converters/lightdash/src/ossie_lightdash/expression_utils.py @@ -14,22 +14,44 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Expression helpers shared by both conversion directions. -"""Small expression helpers shared by both conversion directions. - -Lightdash SQL snippets reference columns as ``${TABLE}.column`` (and joined -tables as ``${other_table.column}``); Ossie expressions reference them as +Lightdash SQL snippets reference the current model as ``${TABLE}.column``, +sibling fields as ``${column}`` or ``${metric}``, joined models as +``${other_table.column}`` and project parameters or user attributes as +``${lightdash.…}`` / ``${ld.…}``. Ossie expressions reference columns as ``dataset.column``. These helpers translate between the two spellings and -recognise the single-aggregation shapes that map onto Lightdash's typed -metrics. +recognise the aggregation shapes that map onto Lightdash's typed metrics. """ import re -from typing import Optional, Tuple +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional + +_REFERENCE_RE = re.compile(r"\$\{([^}]+)\}") +_NON_PORTABLE_REFERENCE_RE = re.compile(r"\$\{(?:lightdash|ld)\.[^}]*\}") +_COLUMN_REFERENCE_RE = re.compile(r"^[A-Za-z_]\w*(?:\.\w+)?$") -# Aggregations that translate to a typed Lightdash metric. Anything else is -# exported as a `number` metric with raw SQL. -_AGG_TO_LIGHTDASH_TYPE = { +# Lightdash metric types that are encoded as an aggregation in the Ossie +# expression. Anything else is exported as a `number` metric with raw SQL. +_AGGREGATE_FUNCTIONS = { + "sum": "SUM", + "min": "MIN", + "max": "MAX", + "average": "AVG", + "median": "MEDIAN", + "count": "COUNT", +} +_DISTINCT_AGGREGATE_FUNCTIONS = { + "count_distinct": "COUNT", + "sum_distinct": "SUM", + "average_distinct": "AVG", +} +AGGREGATE_TYPES = frozenset( + {*_AGGREGATE_FUNCTIONS, *_DISTINCT_AGGREGATE_FUNCTIONS, "percentile"} +) + +_FUNCTION_TO_TYPE = { "SUM": "sum", "MIN": "min", "MAX": "max", @@ -38,50 +60,99 @@ "MEDIAN": "median", "COUNT": "count", } - -_LIGHTDASH_TYPE_TO_AGG = { - "sum": "SUM", - "min": "MIN", - "max": "MAX", - "average": "AVG", - "median": "MEDIAN", - "count": "COUNT", +_DISTINCT_FUNCTION_TO_TYPE = { + "COUNT": "count_distinct", + "SUM": "sum_distinct", + "AVG": "average_distinct", + "AVERAGE": "average_distinct", } -_SIMPLE_AGG_RE = re.compile( - r"^\s*(?P[A-Za-z_]+)\s*\(\s*(?PDISTINCT\s+)?(?P[A-Za-z_][\w.]*)\s*\)\s*$", - re.IGNORECASE, +_CALL_RE = re.compile(r"^\s*(?P[A-Za-z_]+)\s*\((?P.*)\)\s*$", re.DOTALL) +_DISTINCT_RE = re.compile(r"^DISTINCT\s+(?P.+)$", re.IGNORECASE | re.DOTALL) +_PERCENTILE_RE = re.compile( + r"^\s*PERCENTILE_CONT\s*\(\s*(?P[0-9]*\.?[0-9]+)\s*\)\s*" + r"WITHIN\s+GROUP\s*\(\s*ORDER\s+BY\s+(?P.+?)\s*\)\s*$", + re.IGNORECASE | re.DOTALL, ) -def parse_simple_aggregation(expression: str) -> Optional[Tuple[str, str]]: - """Parse ``AGG(qualifier.column)`` into a (lightdash_type, column_ref) pair. - Returns None when the expression is anything more complex than a single - aggregation over a single column reference. +@dataclass(frozen=True) +class ParsedAggregation: + lightdash_type: str + inner: str + percentile: Optional[float] = None + + +def _is_single_call_body(body: str) -> bool: + """True when the parentheses in ``body`` are balanced, i.e. the outer + parentheses around it belong to one function call.""" + depth = 0 + for char in body: + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth < 0: + return False + return depth == 0 + + +def parse_aggregation(expression: str) -> Optional[ParsedAggregation]: + """Recognise an expression that is exactly one aggregation over an operand. + + ``SUM(x)``, ``COUNT(DISTINCT x)``, ``SUM(DISTINCT x)`` and + ``PERCENTILE_CONT(p) WITHIN GROUP (ORDER BY x)`` map onto Lightdash's typed + metrics; the operand may be a bare column reference or any expression. + Returns None for anything else (``COUNT(*)``, arithmetic between + aggregations, unknown functions). """ - match = _SIMPLE_AGG_RE.match(expression) - if not match: + percentile_match = _PERCENTILE_RE.match(expression) + if percentile_match: + percentile = float(percentile_match.group("fraction")) * 100 + if percentile.is_integer(): + percentile = int(percentile) + return ParsedAggregation( + "percentile", percentile_match.group("inner").strip(), percentile + ) + + match = _CALL_RE.match(expression) + if match is None or not _is_single_call_body(match.group("body")): return None - func = match.group("func").upper() - inner = match.group("inner") - if match.group("distinct"): - if func != "COUNT": - return None - return ("count_distinct", inner) - lightdash_type = _AGG_TO_LIGHTDASH_TYPE.get(func) - if lightdash_type is None: + function = match.group("func").upper() + inner = match.group("body").strip() + distinct = _DISTINCT_RE.match(inner) + if distinct: + lightdash_type = _DISTINCT_FUNCTION_TO_TYPE.get(function) + inner = distinct.group("inner").strip() + else: + lightdash_type = _FUNCTION_TO_TYPE.get(function) + if lightdash_type is None or not inner or inner == "*": return None - return (lightdash_type, inner) + return ParsedAggregation(lightdash_type, inner) + +def build_aggregation( + lightdash_type: str, inner: str, percentile: Optional[float] = None +) -> Optional[str]: + """Build the Ossie expression for a typed Lightdash metric over ``inner``. -def build_aggregation(lightdash_type: str, dataset: str, column: str) -> Optional[str]: - """Build the Ossie expression for a typed Lightdash metric, if it has one.""" - if lightdash_type == "count_distinct": - return f"COUNT(DISTINCT {dataset}.{column})" - agg = _LIGHTDASH_TYPE_TO_AGG.get(lightdash_type) - if agg is None: + Returns None for metric types that are not aggregations (``number``, + ``string``, ``boolean``, ...). + """ + if lightdash_type == "percentile": + fraction = (50 if percentile is None else percentile) / 100 + return f"PERCENTILE_CONT({fraction:g}) WITHIN GROUP (ORDER BY {inner})" + if lightdash_type in _DISTINCT_AGGREGATE_FUNCTIONS: + return f"{_DISTINCT_AGGREGATE_FUNCTIONS[lightdash_type]}(DISTINCT {inner})" + function = _AGGREGATE_FUNCTIONS.get(lightdash_type) + if function is None: return None - return f"{agg}({dataset}.{column})" + return f"{function}({inner})" + + +def is_column_reference(expression: str) -> bool: + """True for a bare ``column`` or ``qualifier.column`` reference.""" + return _COLUMN_REFERENCE_RE.match(expression) is not None def strip_qualifier(column_ref: str) -> str: @@ -96,7 +167,7 @@ def qualifier_of(column_ref: str) -> Optional[str]: return None -def osi_sql_to_lightdash(expression: str, dataset: str) -> str: +def ossie_sql_to_lightdash(expression: str, dataset: str) -> str: """Rewrite ``dataset.column`` references into Lightdash's ``${TABLE}.column``.""" return re.sub( rf"\b{re.escape(dataset)}\.(\w+)", @@ -105,15 +176,60 @@ def osi_sql_to_lightdash(expression: str, dataset: str) -> str: ) -def lightdash_sql_to_osi(sql: str, dataset: str) -> str: - """Rewrite Lightdash column references into Ossie ``dataset.column`` references. +def has_non_portable_reference(sql: str) -> bool: + """True when the SQL references project parameters or user attributes.""" + return _NON_PORTABLE_REFERENCE_RE.search(sql) is not None - ``${TABLE}.column`` refers to the current model; ``${other_table.column}`` - refers to a joined model and becomes a cross-dataset reference. - """ - rewritten = sql.replace("${TABLE}.", f"{dataset}.") - return re.sub(r"\$\{(\w+)\.(\w+)\}", r"\1.\2", rewritten) +@dataclass +class RewriteResult: + expression: str + inlined_metrics: List[str] = field(default_factory=list) + flattened_aliases: List[str] = field(default_factory=list) + + +def lightdash_sql_to_ossie( + sql: str, + dataset: str, + *, + aliases: Optional[Dict[str, str]] = None, + resolve_metric: Optional[Callable[[str], Optional[str]]] = None, +) -> RewriteResult: + """Rewrite Lightdash references into Ossie ``dataset.column`` references. + + ``${TABLE}.column`` and bare ``${column}`` refer to the current model; + ``${other_table.column}`` refers to a joined model and becomes a + cross-dataset reference; ``${alias.column}`` is flattened onto the joined + model; ``${metric}`` is replaced by that metric's expression when + ``resolve_metric`` knows it. Parameter and user-attribute references are + left untouched: callers check ``has_non_portable_reference`` first. + """ + alias_map = aliases or {} + result = RewriteResult(expression="") + + def replace(match: "re.Match[str]") -> str: + reference = match.group(1) + if _NON_PORTABLE_REFERENCE_RE.match(match.group(0)): + return match.group(0) + if reference == "TABLE": + return dataset + if reference.startswith("TABLE."): + return f"{dataset}.{reference[len('TABLE.'):]}" + if "." in reference: + table, column = reference.split(".", 1) + if table in alias_map: + result.flattened_aliases.append(table) + return f"{alias_map[table]}.{column}" + return reference + if resolve_metric is not None: + resolved = resolve_metric(reference) + if resolved is not None: + result.inlined_metrics.append(reference) + return f"({resolved})" + return f"{dataset}.{reference}" + + result.expression = _REFERENCE_RE.sub(replace, sql) + return result def referenced_datasets(expression: str, dataset_names: set) -> set: diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py index bab9e5e8..e8e9ae27 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py @@ -14,7 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - """Convert Lightdash semantic definitions into an Ossie document. The input is a dbt ``schema.yml``-shaped dictionary whose ``meta`` blocks @@ -28,7 +27,7 @@ import json import re -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple from ossie import ( OssieCustomExtension, @@ -51,18 +50,19 @@ ) from ossie_lightdash.datatype_utils import lightdash_type_to_datatype from ossie_lightdash.expression_utils import ( + AGGREGATE_TYPES, build_aggregation, - lightdash_sql_to_osi, + has_non_portable_reference, + lightdash_sql_to_ossie, ) LIGHTDASH_VENDOR_NAME = "lightdash" # Keys that are structurally encoded in Ossie vocabulary and therefore must NOT # be duplicated into the extension (a stale copy would win on export). -# ``type`` stays in the extension only for metric types whose semantics Ossie -# expressions cannot express faithfully (currently ``percentile``). _STRUCTURAL_METRIC_KEYS = {"sql", "description"} _STRUCTURAL_DIMENSION_KEYS = {"label", "sql"} +_STRUCTURAL_JOIN_KEYS = {"join", "sql_on"} _JOIN_PAIR_RE = re.compile( r"\$\{(\w+)\.(\w+)\}\s*=\s*\$\{(\w+)\.(\w+)\}", @@ -77,18 +77,6 @@ def _ansi(expression: str) -> OssieExpression: ) -def _type_needs_extension(lightdash_type: str) -> bool: - """True for metric types an Ossie expression cannot encode faithfully. - - ``number`` is fully described by its SQL and typed aggregations are - recovered by parsing the expression, so only the remaining types - (currently ``percentile``) must survive inside the extension. - """ - if lightdash_type == "number": - return False - return build_aggregation(lightdash_type, "_", "_") is None - - def _lightdash_extension(data: Dict[str, Any]) -> List[OssieCustomExtension]: if not data: return [] @@ -100,6 +88,103 @@ def _lightdash_extension(data: Dict[str, Any]) -> List[OssieCustomExtension]: ] +def _unique_name(base: str, used: Set[str]) -> str: + name = base + suffix = 2 + while name in used: + name = f"{base}_{suffix}" + suffix += 1 + used.add(name) + return name + + +class _ModelContext: + """Expression rewriting for one model: alias resolution, metric inlining + and non-portable reference detection, with the issues they raise.""" + + def __init__( + self, + dataset_name: str, + aliases: Dict[str, str], + definitions: Dict[str, Tuple[Dict[str, Any], Optional[str]]], + issues: List[ConverterIssue], + ) -> None: + self.dataset_name = dataset_name + self.aliases = aliases + self.definitions = definitions + self.issues = issues + self._expressions: Dict[str, Optional[str]] = {} + self._resolving: Set[str] = set() + + def rewrite(self, sql: str, element_name: str) -> Optional[str]: + """Rewrite Lightdash SQL into an Ossie expression, or None (with an + issue) when it references parameters or user attributes.""" + if has_non_portable_reference(sql): + self.issues.append( + ConverterIssue( + issue_type=ConverterIssueType.EXPRESSION_NOT_PORTABLE, + element_name=element_name, + ) + ) + return None + result = lightdash_sql_to_ossie( + sql, + self.dataset_name, + aliases=self.aliases, + resolve_metric=self.metric_expression, + ) + for _ in result.inlined_metrics: + self.issues.append( + ConverterIssue( + issue_type=ConverterIssueType.METRIC_REFERENCE_INLINED, + element_name=element_name, + ) + ) + for _ in result.flattened_aliases: + self.issues.append( + ConverterIssue( + issue_type=ConverterIssueType.ALIAS_REFERENCE_FLATTENED, + element_name=element_name, + ) + ) + return result.expression + + def metric_expression(self, name: str) -> Optional[str]: + """The Ossie expression of one of this model's metrics, or None when the + name is not a metric, the metric is not portable, or it references + itself.""" + if name not in self.definitions: + return None + if name in self._expressions: + return self._expressions[name] + if name in self._resolving: + return None + self._resolving.add(name) + definition, column = self.definitions[name] + expression = self._build_expression(name, definition, column) + self._resolving.discard(name) + self._expressions[name] = expression + return expression + + def _build_expression( + self, name: str, definition: Dict[str, Any], column: Optional[str] + ) -> Optional[str]: + sql = definition.get("sql") + if sql: + inner = self.rewrite(sql, name) + if inner is None: + return None + elif column is not None: + inner = f"{self.dataset_name}.{column}" + else: + return None + lightdash_type = definition.get("type", "number") + return ( + build_aggregation(lightdash_type, inner, definition.get("percentile")) + or inner + ) + + class LightdashToOssieConverter: """Converts a Lightdash-flavoured dbt schema.yml dict into an OssieDocument.""" @@ -115,10 +200,15 @@ def convert( datasets: List[OssieDataset] = [] metrics: List[OssieMetric] = [] relationships: List[OssieRelationship] = [] + relationship_names: Set[str] = set() for model in schema_yml.get("models") or []: dataset, model_metrics, model_relationships = self._convert_model( - model, database=database, schema=schema, issues=issues + model, + database=database, + schema=schema, + issues=issues, + relationship_names=relationship_names, ) datasets.append(dataset) metrics.extend(model_metrics) @@ -144,6 +234,7 @@ def _convert_model( database: Optional[str], schema: Optional[str], issues: List[ConverterIssue], + relationship_names: Set[str], ) -> Tuple[OssieDataset, List[OssieMetric], List[OssieRelationship]]: name = model["name"] source = ".".join(part for part in [database, schema, name] if part) @@ -155,29 +246,43 @@ def _convert_model( ) ) + model_meta = model.get("meta") or {} + joins = model_meta.get("joins") or [] + aliases = { + join["alias"]: join["join"] + for join in joins + if join.get("alias") and join.get("join") + } + + # Metric definitions are collected before any SQL is rewritten so that + # `${metric}` references can be inlined. + definitions: Dict[str, Tuple[Dict[str, Any], Optional[str]]] = {} + for column in model.get("columns") or []: + column_meta = column.get("meta") or {} + for metric_name, definition in (column_meta.get("metrics") or {}).items(): + definitions[metric_name] = (definition, column["name"]) + for metric_name, definition in (model_meta.get("metrics") or {}).items(): + definitions[metric_name] = (definition, None) + + context = _ModelContext(name, aliases, definitions, issues) + fields: List[OssieField] = [] - metrics: List[OssieMetric] = [] for column in model.get("columns") or []: - field, column_metrics = self._convert_column(column, dataset_name=name) - fields.append(field) - metrics.extend(column_metrics) + field = self._convert_column(column, context) + if field is not None: + fields.append(field) - model_meta = model.get("meta") or {} - for metric_name, definition in (model_meta.get("metrics") or {}).items(): - if not definition.get("sql"): - issues.append( - ConverterIssue( - issue_type=ConverterIssueType.METRIC_SQL_MISSING, - element_name=metric_name, - ) - ) - continue - metrics.append( - self._convert_sql_metric(metric_name, definition, dataset_name=name) - ) + metrics: List[OssieMetric] = [] + for metric_name, (definition, column_name) in definitions.items(): + metric = self._convert_metric(metric_name, definition, column_name, context) + if metric is not None: + metrics.append(metric) relationships = self._convert_joins( - model_meta.get("joins") or [], from_model=name, issues=issues + joins, + from_model=name, + issues=issues, + relationship_names=relationship_names, ) dataset = OssieDataset( @@ -189,11 +294,10 @@ def _convert_model( return dataset, metrics, relationships def _convert_column( - self, column: Dict[str, Any], *, dataset_name: str - ) -> Tuple[OssieField, List[OssieMetric]]: + self, column: Dict[str, Any], context: _ModelContext + ) -> Optional[OssieField]: column_name = column["name"] - meta = column.get("meta") or {} - dimension_meta = meta.get("dimension") + dimension_meta = (column.get("meta") or {}).get("dimension") expression = column_name dimension: Optional[OssieDimension] = None @@ -203,7 +307,10 @@ def _convert_column( if dimension_meta is not None: label = dimension_meta.get("label") if dimension_meta.get("sql"): - expression = lightdash_sql_to_osi(dimension_meta["sql"], dataset_name) + rewritten = context.rewrite(dimension_meta["sql"], column_name) + if rewritten is None: + return None + expression = rewritten datatype = lightdash_type_to_datatype(dimension_meta.get("type")) # `is_time` is a role marker in Ossie, not a type: Lightdash has no # equivalent, so it is left unset rather than inferred from the type @@ -215,7 +322,7 @@ def _convert_column( if key not in _STRUCTURAL_DIMENSION_KEYS } - field = OssieField( + return OssieField( name=column_name, expression=_ansi(expression), dimension=dimension, @@ -225,60 +332,34 @@ def _convert_column( custom_extensions=_lightdash_extension(extension_data) or None, ) - metrics = [ - self._convert_column_metric( - metric_name, definition, dataset_name=dataset_name, column=column_name - ) - for metric_name, definition in (meta.get("metrics") or {}).items() - ] - return field, metrics - - def _convert_column_metric( - self, + @staticmethod + def _convert_metric( metric_name: str, definition: Dict[str, Any], - *, - dataset_name: str, - column: str, - ) -> OssieMetric: - lightdash_type = definition.get("type", "number") - expression = build_aggregation(lightdash_type, dataset_name, column) + column: Optional[str], + context: _ModelContext, + ) -> Optional[OssieMetric]: + if column is None and not definition.get("sql"): + context.issues.append( + ConverterIssue( + issue_type=ConverterIssueType.METRIC_SQL_MISSING, + element_name=metric_name, + ) + ) + return None + expression = context.metric_expression(metric_name) if expression is None: - sql = definition.get("sql") - if sql: - expression = lightdash_sql_to_osi(sql, dataset_name) - else: - expression = f"{dataset_name}.{column}" - - return self._build_metric( - metric_name, - definition, - expression=expression, - keep_type_in_extension=_type_needs_extension(lightdash_type), - ) - - def _convert_sql_metric( - self, metric_name: str, definition: Dict[str, Any], *, dataset_name: str - ) -> OssieMetric: - expression = lightdash_sql_to_osi(definition["sql"], dataset_name) - return self._build_metric( - metric_name, - definition, - expression=expression, - keep_type_in_extension=_type_needs_extension(definition.get("type", "number")), - ) + return None - @staticmethod - def _build_metric( - metric_name: str, - definition: Dict[str, Any], - *, - expression: str, - keep_type_in_extension: bool, - ) -> OssieMetric: + # Typed aggregations (and their percentile) are recovered from the + # expression on export; only types an expression cannot encode + # (`boolean`, `string`, `date`, ...) travel in the extension. + lightdash_type = definition.get("type", "number") excluded = set(_STRUCTURAL_METRIC_KEYS) - if not keep_type_in_extension: + if lightdash_type == "number" or lightdash_type in AGGREGATE_TYPES: excluded.add("type") + if lightdash_type == "percentile": + excluded.add("percentile") extension_data = { key: value for key, value in definition.items() if key not in excluded } @@ -295,18 +376,21 @@ def _convert_joins( *, from_model: str, issues: List[ConverterIssue], + relationship_names: Set[str], ) -> List[OssieRelationship]: relationships: List[OssieRelationship] = [] for join in joins: to_model = join.get("join") + # An aliased join is referenced by its alias in `sql_on`. + reference = join.get("alias") or to_model pairs = _JOIN_PAIR_RE.findall(join.get("sql_on") or "") from_columns: List[str] = [] to_columns: List[str] = [] for left_table, left_column, right_table, right_column in pairs: - if left_table == from_model and right_table == to_model: + if left_table == from_model and right_table == reference: from_columns.append(left_column) to_columns.append(right_column) - elif left_table == to_model and right_table == from_model: + elif left_table == reference and right_table == from_model: from_columns.append(right_column) to_columns.append(left_column) if not to_model or not from_columns: @@ -317,14 +401,24 @@ def _convert_joins( ) ) continue + # The alias and any other join attributes (type, relationship, + # fields, ...) have no Ossie vocabulary and travel in the extension. + extras = { + key: value + for key, value in join.items() + if key not in _STRUCTURAL_JOIN_KEYS + } relationships.append( OssieRelationship.model_validate( { - "name": f"{from_model}_to_{to_model}", + "name": _unique_name( + f"{from_model}_to_{reference}", relationship_names + ), "from": from_model, "to": to_model, "from_columns": from_columns, "to_columns": to_columns, + "custom_extensions": _lightdash_extension(extras) or None, } ) ) diff --git a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py index 8bbff9de..ea948b41 100644 --- a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py +++ b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py @@ -14,7 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - """Convert an Ossie document into Lightdash semantic definitions. The output is a dbt ``schema.yml``-shaped dictionary whose ``meta`` blocks @@ -23,12 +22,13 @@ have no Ossie vocabulary round-trip through ``custom_extensions`` entries with ``vendor_name: "lightdash"``; their keys are overlaid onto the generated definitions and win for presentation attributes, while structural keys -(``sql``/``label`` on dimensions, ``sql``/``description`` on metrics) are -protected so they can never override the Ossie-derived definition. +(``sql``/``label`` on dimensions, ``sql``/``description`` on metrics, +``join``/``sql_on`` on joins) are protected so they can never override the +Ossie-derived definition. """ import json -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Set, Tuple from ossie import OssieDataset, OssieDialect, OssieDocument, OssieMetric, OssieSemanticModel @@ -39,8 +39,9 @@ ) from ossie_lightdash.datatype_utils import datatype_to_lightdash_type, is_temporal from ossie_lightdash.expression_utils import ( - osi_sql_to_lightdash, - parse_simple_aggregation, + is_column_reference, + ossie_sql_to_lightdash, + parse_aggregation, qualifier_of, referenced_datasets, strip_qualifier, @@ -51,18 +52,20 @@ # Structural keys are owned by Ossie vocabulary (the import direction never puts # them into the extension); dropping them here keeps a hand-authored extension # from overriding the Ossie-derived definition. ``type`` stays overridable on -# metrics: it is the documented channel for types Ossie expressions cannot -# express (e.g. percentile). +# metrics whose expression is not a recognised aggregation: it is the channel +# for types an expression cannot express (``boolean``, ``string``, ...). _PROTECTED_DIMENSION_KEYS = {"sql", "label"} _PROTECTED_METRIC_KEYS = {"sql", "description"} +_PROTECTED_AGGREGATION_KEYS = _PROTECTED_METRIC_KEYS | {"type", "percentile"} +_PROTECTED_JOIN_KEYS = {"join", "sql_on", "alias"} -def _pick_expression(osi_expression: Any, dialect: OssieDialect) -> str: +def _pick_expression(ossie_expression: Any, dialect: OssieDialect) -> str: """Return the expression for the preferred dialect (fallback: first available).""" - for dialect_expression in osi_expression.dialects: + for dialect_expression in ossie_expression.dialects: if dialect_expression.dialect is dialect: return dialect_expression.expression - return osi_expression.dialects[0].expression if osi_expression.dialects else "" + return ossie_expression.dialects[0].expression if ossie_expression.dialects else "" def _lightdash_extension_data(element: Any, issues: List[ConverterIssue]) -> Dict[str, Any]: @@ -132,6 +135,7 @@ def _convert_semantic_model( issues, ) + joined_pairs: Set[Tuple[str, str]] = set() for relationship in semantic_model.relationships or []: from_model = models_by_dataset.get(relationship.from_dataset) to_model_name = model_name_by_dataset.get(relationship.to) @@ -146,14 +150,35 @@ def _convert_semantic_model( ) ) continue + extension_data = _lightdash_extension_data(relationship, issues) + # Lightdash refuses to join the same table twice without an alias; + # a stashed alias is restored, otherwise the relationship name + # aliases every repeat of a dataset pair. + pair = (relationship.from_dataset, relationship.to) + alias = extension_data.get("alias") + if alias is None and pair in joined_pairs: + alias = relationship.name + joined_pairs.add(pair) + join_reference = alias or to_model_name sql_on = " AND ".join( - f"${{{from_model_name}.{from_column}}} = ${{{to_model_name}.{to_column}}}" + f"${{{from_model_name}.{from_column}}} = ${{{join_reference}.{to_column}}}" for from_column, to_column in zip( relationship.from_columns, relationship.to_columns ) ) + join: Dict[str, Any] = {"join": to_model_name} + if alias: + join["alias"] = alias + join["sql_on"] = sql_on + join.update( + { + key: value + for key, value in extension_data.items() + if key not in _PROTECTED_JOIN_KEYS + } + ) joins = from_model.setdefault("meta", {}).setdefault("joins", []) - joins.append({"join": to_model_name, "sql_on": sql_on}) + joins.append(join) return [models_by_dataset[dataset.name] for dataset in datasets] @@ -195,7 +220,7 @@ def _convert_dataset( ) expression = _pick_expression(field.expression, self._dialect) if expression and expression != field.name: - dimension["sql"] = osi_sql_to_lightdash(expression, dataset.name) + dimension["sql"] = ossie_sql_to_lightdash(expression, dataset.name) dimension.update( { key: value @@ -241,32 +266,41 @@ def _convert_metric( if metric.description: definition["description"] = metric.description + # A single aggregation becomes a typed metric: on the column when the + # operand is one of the dataset's columns, otherwise on the model with + # the operand as `sql`. Anything else is a `number` metric with raw SQL. target_column: Optional[str] = None - parsed = parse_simple_aggregation(expression) + parsed = parse_aggregation(expression) if parsed is not None: - lightdash_type, column_ref = parsed - qualifier = qualifier_of(column_ref) - if qualifier in (None, target_dataset): - target_column = strip_qualifier(column_ref) - definition["type"] = lightdash_type - if target_column is None or target_column not in columns_by_dataset[target_dataset]: + definition["type"] = parsed.lightdash_type + inner = parsed.inner + if ( + is_column_reference(inner) + and qualifier_of(inner) in (None, target_dataset) + and strip_qualifier(inner) in columns_by_dataset[target_dataset] + ): + target_column = strip_qualifier(inner) + else: + definition["sql"] = ossie_sql_to_lightdash(inner, target_dataset) + if parsed.percentile is not None: + definition["percentile"] = parsed.percentile + protected = _PROTECTED_AGGREGATION_KEYS + else: definition["type"] = extension_data.get("type", "number") - definition["sql"] = osi_sql_to_lightdash(expression, target_dataset) - target_column = None + definition["sql"] = ossie_sql_to_lightdash(expression, target_dataset) + protected = _PROTECTED_METRIC_KEYS definition.update( { key: value for key, value in extension_data.items() - if key not in _PROTECTED_METRIC_KEYS + if key not in protected } ) if target_column is not None: column = columns_by_dataset[target_dataset][target_column] - metrics = ( - column.setdefault("meta", {}).setdefault("metrics", {}) - ) + metrics = column.setdefault("meta", {}).setdefault("metrics", {}) metrics[metric.name] = definition else: model = models_by_dataset[target_dataset] diff --git a/converters/lightdash/tests/test_lightdash_to_ossie.py b/converters/lightdash/tests/test_lightdash_to_ossie.py index 1f9e9beb..8adb18bc 100644 --- a/converters/lightdash/tests/test_lightdash_to_ossie.py +++ b/converters/lightdash/tests/test_lightdash_to_ossie.py @@ -156,10 +156,14 @@ def test_count_distinct_metric(self): == "COUNT(DISTINCT orders.customer_id)" ) - def test_percentile_metric_keeps_type_in_extension(self): + def test_percentile_metric_becomes_percentile_cont(self): result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") metric = _metric(result.output, "p90_amount") - assert _lightdash_data(metric) == {"type": "percentile", "percentile": 90} + assert ( + metric.expression.dialects[0].expression + == "PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY orders.amount)" + ) + assert _lightdash_data(metric) == {} def test_sql_metric_expression_is_rewritten(self): result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") @@ -182,7 +186,7 @@ def test_join_becomes_relationship(self): assert relationship.from_columns == ["customer_id"] assert relationship.to_columns == ["customer_id"] - def test_percentile_with_sql_keeps_type_in_extension(self): + def test_percentile_with_sql_orders_by_the_expression(self): schema_yml = { "models": [ { @@ -204,9 +208,9 @@ def test_percentile_with_sql_keeps_type_in_extension(self): metric = _metric(result.output, "p90_custom") assert ( metric.expression.dialects[0].expression - == "orders.amount - orders.discount" + == "PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY orders.amount - orders.discount)" ) - assert _lightdash_data(metric) == {"type": "percentile", "percentile": 90} + assert _lightdash_data(metric) == {} def test_joined_table_references_become_cross_dataset(self): schema_yml = { @@ -268,3 +272,195 @@ def test_unparseable_join_is_reported(self): issue.issue_type is ConverterIssueType.JOIN_SQL_UNPARSED for issue in result.issues ) + + def test_typed_metric_with_sql_aggregates_the_expression(self): + schema_yml = { + "models": [ + { + "name": "work_orders", + "columns": [ + { + "name": "status", + "meta": { + "metrics": { + "completion_rate": { + "type": "average", + "sql": "CASE WHEN ${status} = 'Completed' THEN 1 ELSE 0 END", + }, + "distinct_total": {"type": "sum_distinct"}, + } + }, + } + ], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + assert ( + _metric(result.output, "completion_rate").expression.dialects[0].expression + == "AVG(CASE WHEN work_orders.status = 'Completed' THEN 1 ELSE 0 END)" + ) + distinct_total = _metric(result.output, "distinct_total") + assert ( + distinct_total.expression.dialects[0].expression + == "SUM(DISTINCT work_orders.status)" + ) + assert _lightdash_data(distinct_total) == {} + + def test_metric_reference_is_inlined(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": { + "metrics": { + "amount_per_customer": { + "type": "number", + "sql": "${total_amount} / NULLIF(${unique_customers}, 0)", + } + } + }, + "columns": [ + {"name": "amount", "meta": {"metrics": {"total_amount": {"type": "sum"}}}}, + { + "name": "customer_id", + "meta": {"metrics": {"unique_customers": {"type": "count_distinct"}}}, + }, + ], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + assert ( + _metric(result.output, "amount_per_customer").expression.dialects[0].expression + == "(SUM(orders.amount)) / NULLIF((COUNT(DISTINCT orders.customer_id)), 0)" + ) + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.METRIC_REFERENCE_INLINED + ] == ["amount_per_customer", "amount_per_customer"] + + def test_bare_field_references_resolve_to_the_dataset(self): + schema_yml = { + "models": [ + { + "name": "customers", + "columns": [ + {"name": "first_name", "meta": {"dimension": {"type": "string"}}}, + { + "name": "full_name", + "meta": { + "dimension": { + "type": "string", + "sql": "${first_name} || ' ' || ${TABLE}.last_name", + } + }, + }, + { + "name": "order_count", + "meta": { + "dimension": { + "type": "number", + "sql": "(SELECT COUNT(*) FROM orders WHERE orders.customer_id = ${TABLE}.customer_id)", + } + }, + }, + ], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + by_name = { + field.name: field.expression.dialects[0].expression + for field in result.output.semantic_model[0].datasets[0].fields + } + assert by_name["full_name"] == "customers.first_name || ' ' || customers.last_name" + assert by_name["order_count"] == ( + "(SELECT COUNT(*) FROM orders WHERE orders.customer_id = customers.customer_id)" + ) + + def test_parameter_references_skip_the_element(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": { + "metrics": { + "my_orders": { + "type": "number", + "sql": "SUM(CASE WHEN ${TABLE}.owner = ${ld.user.email} THEN 1 END)", + } + } + }, + "columns": [ + { + "name": "is_recent", + "meta": { + "dimension": { + "type": "boolean", + "sql": "${TABLE}.order_date >= ${lightdash.parameters.start_date}", + } + }, + }, + {"name": "order_date", "meta": {"dimension": {"type": "date"}}}, + ], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + dataset = result.output.semantic_model[0].datasets[0] + assert [field.name for field in dataset.fields] == ["order_date"] + assert result.output.semantic_model[0].metrics is None + assert sorted( + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.EXPRESSION_NOT_PORTABLE + ) == ["is_recent", "my_orders"] + + def test_aliased_joins_become_relationships(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": { + "joins": [ + { + "join": "date_dim", + "alias": "sold_date", + "sql_on": "${orders.sold_date_id} = ${sold_date.date_id}", + "relationship": "many-to-one", + }, + { + "join": "date_dim", + "alias": "return_date", + "sql_on": "${orders.return_date_id} = ${return_date.date_id}", + }, + ] + }, + "columns": [ + { + "name": "sold_year", + "meta": {"dimension": {"type": "number", "sql": "${sold_date.year}"}}, + } + ], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + relationships = result.output.semantic_model[0].relationships + assert [(r.name, r.to, r.from_columns, r.to_columns) for r in relationships] == [ + ("orders_to_sold_date", "date_dim", ["sold_date_id"], ["date_id"]), + ("orders_to_return_date", "date_dim", ["return_date_id"], ["date_id"]), + ] + assert _lightdash_data(relationships[0]) == { + "alias": "sold_date", + "relationship": "many-to-one", + } + field = result.output.semantic_model[0].datasets[0].fields[0] + assert field.expression.dialects[0].expression == "date_dim.year" + assert any( + issue.issue_type is ConverterIssueType.ALIAS_REFERENCE_FLATTENED + and issue.element_name == "sold_year" + for issue in result.issues + ) diff --git a/converters/lightdash/tests/test_ossie_to_lightdash.py b/converters/lightdash/tests/test_ossie_to_lightdash.py index 2fcb6cf8..117bacac 100644 --- a/converters/lightdash/tests/test_ossie_to_lightdash.py +++ b/converters/lightdash/tests/test_ossie_to_lightdash.py @@ -92,6 +92,18 @@ def _document() -> OssieDocument: ) ], ), + OssieMetric( + name="p90_amount", + expression=_ansi("PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY orders.amount)"), + ), + OssieMetric( + name="distinct_amount", + expression=_ansi("SUM(DISTINCT orders.amount)"), + ), + OssieMetric( + name="completed_rate", + expression=_ansi("AVG(CASE WHEN orders.status = 'completed' THEN 1 ELSE 0 END)"), + ), OssieMetric( name="cross_dataset", expression=_ansi("SUM(orders.amount) / COUNT(customers.customer_id)"), @@ -260,3 +272,77 @@ def test_relationship_becomes_join(self): "sql_on": "${orders.customer_id} = ${customers.customer_id}", } ] + + def test_percentile_cont_becomes_percentile_metric(self): + result = OssieToLightdashConverter().convert(_document()) + column = _column(_model(result.output, "orders"), "amount") + assert column["meta"]["metrics"]["p90_amount"] == { + "type": "percentile", + "percentile": 90, + } + assert column["meta"]["metrics"]["distinct_amount"] == {"type": "sum_distinct"} + + def test_aggregation_over_expression_becomes_typed_model_metric(self): + result = OssieToLightdashConverter().convert(_document()) + metric = _model(result.output, "orders")["meta"]["metrics"]["completed_rate"] + assert metric == { + "type": "average", + "sql": "CASE WHEN ${TABLE}.status = 'completed' THEN 1 ELSE 0 END", + } + + def test_repeated_relationship_gets_an_alias(self): + document = _document() + tampered = document.model_copy(deep=True) + tampered.semantic_model[0].relationships.append( + OssieRelationship.model_validate( + { + "name": "orders_to_referrer", + "from": "orders", + "to": "customers", + "from_columns": ["amount"], + "to_columns": ["customer_id"], + } + ) + ) + result = OssieToLightdashConverter().convert(tampered) + assert _model(result.output, "orders")["meta"]["joins"] == [ + { + "join": "customers", + "sql_on": "${orders.customer_id} = ${customers.customer_id}", + }, + { + "join": "customers", + "alias": "orders_to_referrer", + "sql_on": "${orders.amount} = ${orders_to_referrer.customer_id}", + }, + ] + + def test_join_alias_and_attributes_restore_from_extension(self): + document = _document() + tampered = document.model_copy(deep=True) + tampered.semantic_model[0].relationships[0] = OssieRelationship.model_validate( + { + "name": "orders_to_customers", + "from": "orders", + "to": "customers", + "from_columns": ["customer_id"], + "to_columns": ["customer_id"], + "custom_extensions": [ + { + "vendor_name": "lightdash", + "data": json.dumps( + {"alias": "buyer", "relationship": "many-to-one", "sql_on": "1 = 1"} + ), + } + ], + } + ) + result = OssieToLightdashConverter().convert(tampered) + assert _model(result.output, "orders")["meta"]["joins"] == [ + { + "join": "customers", + "alias": "buyer", + "sql_on": "${orders.customer_id} = ${buyer.customer_id}", + "relationship": "many-to-one", + } + ] From 35e378b0bced811d339e9ca6182b4a84fd40619d Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 10:58:23 +0100 Subject: [PATCH 10/33] feat(lightdash): honour dialects and export metrics across declared joins - export prefers --dialect, then ANSI_SQL, and reports DIALECT_UNAVAILABLE instead of silently taking whatever dialect comes first - import labels expressions with --dialect, since Lightdash SQL is written for the project's warehouse rather than ANSI - a metric spanning several datasets is hosted on the dataset that joins all the others directly, with ${joined_model.column} references, instead of being dropped; TPC-DS's two cross-dataset metrics now export - field expressions resolve joined datasets the same way and report FIELD_REFERENCE_UNJOINED when no join exists --- converters/lightdash/README.md | 28 ++-- .../lightdash/src/ossie_lightdash/cli.py | 20 ++- .../src/ossie_lightdash/converter_issues.py | 6 + .../src/ossie_lightdash/expression_utils.py | 27 +++- .../src/ossie_lightdash/lightdash_to_ossie.py | 22 ++- .../src/ossie_lightdash/ossie_to_lightdash.py | 147 ++++++++++++++---- .../tests/test_lightdash_to_ossie.py | 11 +- .../tests/test_ossie_to_lightdash.py | 105 ++++++++++++- 8 files changed, 305 insertions(+), 61 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 676a9987..371ec3bf 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -32,8 +32,8 @@ translates between that shape and Ossie. base of Lightdash metrics. ``` -ossie-lightdash export semantic_model.yaml schema.yml -ossie-lightdash import schema.yml semantic_model.json --database analytics_db --schema marts +ossie-lightdash export semantic_model.yaml schema.yml --dialect BIGQUERY +ossie-lightdash import schema.yml semantic_model.json --database analytics_db --schema marts --dialect BIGQUERY ``` ## Mapping @@ -50,14 +50,19 @@ ossie-lightdash import schema.yml semantic_model.json --database analytics_db -- | `field.expression` (≠ column name) | `meta.dimension.sql` (`dataset.col` ↔ `${TABLE}.col`) | | `metric` that is one aggregation over a column (`SUM(ds.col)`, `COUNT(DISTINCT ds.col)`, `SUM(DISTINCT ds.col)`, `PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY ds.col)`, ...) | column-level `meta.metrics.` with a typed metric (`sum`, `count_distinct`, `sum_distinct`, `percentile` + `percentile: 90`, ...) | | `metric` that is one aggregation over any other expression (`AVG(CASE WHEN ds.status = 'done' THEN 1 ELSE 0 END)`) | model-level `meta.metrics.` with the typed metric and the operand as `sql` | -| `metric` with any other single-dataset expression | model-level `meta.metrics.` with `type: number` + `sql` | +| `metric` with any other expression | model-level `meta.metrics.` with `type: number` + `sql`, on the dataset that joins every other dataset the expression references (`${joined_model.col}`) | | `relationship` | `meta.joins` (`sql_on` built from / parsed into column pairs); the `alias` and other join attributes (`relationship`, `type`, `fields`, ...) travel in the relationship's `lightdash` extension, and a dataset joined more than once from the same model is aliased on export | | `${TABLE}.col`, `${col}`, `${other_model.col}` in Lightdash SQL | `dataset.col` / `other_model.col`; `${alias.col}` is flattened onto the aliased model, `${metric}` is replaced by that metric's expression | | Lightdash presentation attributes (`label`, `format`, `round`, `compact`, `group_label`, `hidden`, ...) | `custom_extensions` with `vendor_name: "lightdash"`; on export the extension data is overlaid onto the generated definition (structural keys — `sql`/`label` on dimensions, `sql`/`description` on metrics and `type`/`percentile` on metrics whose expression is a recognised aggregation, `join`/`sql_on` on joins — are protected and cannot be overridden) | -Expressions are written under the `ANSI_SQL` dialect. Warehouse-specific -dialects (e.g. `BIGQUERY`) can be added once the surrounding tooling resolves -them. +## Dialects + +Lightdash SQL is written for the project's warehouse, so `import --dialect` +labels the emitted expressions with that warehouse's Ossie dialect +(`BIGQUERY`, `SNOWFLAKE`, `DATABRICKS`); warehouses without an Ossie dialect +(Postgres, Redshift, ...) keep the default `ANSI_SQL`. `export --dialect` +prefers that dialect, falls back to `ANSI_SQL`, and takes the first available +dialect with a `DIALECT_UNAVAILABLE` issue when an expression offers neither. ## Recommended source shape for dbt-native flows @@ -70,9 +75,14 @@ Omitting `--schema` as well is reported as a `SOURCE_UNQUALIFIED` issue. ## Known limitations -- **Cross-dataset metrics are dropped on export** (with a - `CROSS_DATASET_METRIC_DROPPED` issue): a Lightdash model metric cannot - reference other tables. +- **A metric spanning datasets none of which joins all the others is + dropped on export** (`CROSS_DATASET_METRIC_DROPPED`): Lightdash resolves + `${other.column}` only through the joins the hosting model declares, never + transitively. A field expression referencing an unjoined dataset is emitted + as-is with a `FIELD_REFERENCE_UNJOINED` issue. +- **A dataset joined more than once** is referenced through its first join + when an expression names it (`date_dim.year` → `${date_dim.year}` rather + than the aliased second join). - **Parameter and user-attribute references** (`${lightdash.parameters.x}`, `${ld.user.email}`) have no Ossie form: a dimension or metric whose SQL uses them is skipped on import with an `EXPRESSION_NOT_PORTABLE` issue. diff --git a/converters/lightdash/src/ossie_lightdash/cli.py b/converters/lightdash/src/ossie_lightdash/cli.py index e1d2704b..ac3ae0a4 100644 --- a/converters/lightdash/src/ossie_lightdash/cli.py +++ b/converters/lightdash/src/ossie_lightdash/cli.py @@ -25,7 +25,7 @@ import yaml -from ossie import OssieDocument +from ossie import OssieDialect, OssieDocument from ossie_lightdash.lightdash_to_ossie import LightdashToOssieConverter from ossie_lightdash.ossie_to_lightdash import OssieToLightdashConverter @@ -51,6 +51,12 @@ def main(argv: Optional[List[str]] = None) -> int: ) export_parser.add_argument("input", type=Path) export_parser.add_argument("output", type=Path) + export_parser.add_argument( + "--dialect", + choices=[dialect.name for dialect in OssieDialect], + default=OssieDialect.ANSI_SQL.name, + help="preferred expression dialect (falls back to ANSI_SQL)", + ) import_parser = subparsers.add_parser( "import", help="Lightdash dbt schema.yml -> Ossie document (.json/.yaml)" @@ -62,18 +68,26 @@ def main(argv: Optional[List[str]] = None) -> int: import_parser.add_argument( "--semantic-model-name", default="lightdash_semantic_model" ) + import_parser.add_argument( + "--dialect", + choices=[dialect.name for dialect in OssieDialect], + default=OssieDialect.ANSI_SQL.name, + help="dialect the Lightdash SQL is written in (the project's warehouse)", + ) args = parser.parse_args(argv) if args.command == "export": - result = OssieToLightdashConverter().convert(_read_document(args.input)) + result = OssieToLightdashConverter(OssieDialect[args.dialect]).convert( + _read_document(args.input) + ) args.output.write_text( yaml.safe_dump(result.output, sort_keys=False, allow_unicode=True), encoding="utf-8", ) else: schema_yml = yaml.safe_load(args.input.read_text(encoding="utf-8")) - result = LightdashToOssieConverter().convert( + result = LightdashToOssieConverter(OssieDialect[args.dialect]).convert( schema_yml, database=args.database, schema=args.schema, diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py index d29b36a0..fd22fa7f 100644 --- a/converters/lightdash/src/ossie_lightdash/converter_issues.py +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -54,6 +54,12 @@ class ConverterIssueType(Enum): # Import: a `${alias.column}` reference to an aliased join was rewritten to # the joined dataset; Ossie has no join aliases, so the join path is lost. ALIAS_REFERENCE_FLATTENED = "ALIAS_REFERENCE_FLATTENED" + # Export: neither the requested dialect nor ANSI_SQL is available for an + # expression; the first available dialect is used instead. + DIALECT_UNAVAILABLE = "DIALECT_UNAVAILABLE" + # Export: a field expression references a dataset the field's dataset does + # not join, so Lightdash cannot resolve the reference. + FIELD_REFERENCE_UNJOINED = "FIELD_REFERENCE_UNJOINED" @dataclass(frozen=True) diff --git a/converters/lightdash/src/ossie_lightdash/expression_utils.py b/converters/lightdash/src/ossie_lightdash/expression_utils.py index 8411616b..867557d8 100644 --- a/converters/lightdash/src/ossie_lightdash/expression_utils.py +++ b/converters/lightdash/src/ossie_lightdash/expression_utils.py @@ -167,13 +167,26 @@ def qualifier_of(column_ref: str) -> Optional[str]: return None -def ossie_sql_to_lightdash(expression: str, dataset: str) -> str: - """Rewrite ``dataset.column`` references into Lightdash's ``${TABLE}.column``.""" - return re.sub( - rf"\b{re.escape(dataset)}\.(\w+)", - r"${TABLE}.\1", - expression, - ) +def ossie_sql_to_lightdash( + expression: str, dataset: str, references: Optional[Dict[str, str]] = None +) -> str: + """Rewrite Ossie column references into Lightdash references. + + ``dataset.column`` becomes ``${TABLE}.column``; a dataset in ``references`` + (joined from ``dataset``) becomes ``${.column}``, where the + join reference is the joined model's name or its alias. + """ + names = {dataset: None, **(references or {})} + + def replace(match: "re.Match[str]") -> str: + name, column = match.group(1), match.group(2) + reference = names[name] + if reference is None: + return f"${{TABLE}}.{column}" + return f"${{{reference}.{column}}}" + + alternatives = "|".join(re.escape(name) for name in names) + return re.sub(rf"\b({alternatives})\.(\w+)", replace, expression) def has_non_portable_reference(sql: str) -> bool: diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py index e8e9ae27..aced3e87 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py @@ -69,11 +69,9 @@ ) -def _ansi(expression: str) -> OssieExpression: +def _expression(expression: str, dialect: OssieDialect) -> OssieExpression: return OssieExpression( - dialects=[ - OssieDialectExpression(dialect=OssieDialect.ANSI_SQL, expression=expression) - ] + dialects=[OssieDialectExpression(dialect=dialect, expression=expression)] ) @@ -186,7 +184,15 @@ def _build_expression( class LightdashToOssieConverter: - """Converts a Lightdash-flavoured dbt schema.yml dict into an OssieDocument.""" + """Converts a Lightdash-flavoured dbt schema.yml dict into an OssieDocument. + + Lightdash SQL is written for the project's warehouse; ``dialect`` labels the + emitted expressions accordingly (``ANSI_SQL`` when the warehouse has no + Ossie dialect, e.g. Postgres or Redshift). + """ + + def __init__(self, dialect: OssieDialect = OssieDialect.ANSI_SQL) -> None: + self._dialect = dialect def convert( self, @@ -324,7 +330,7 @@ def _convert_column( return OssieField( name=column_name, - expression=_ansi(expression), + expression=_expression(expression, self._dialect), dimension=dimension, datatype=datatype, label=label, @@ -332,8 +338,8 @@ def _convert_column( custom_extensions=_lightdash_extension(extension_data) or None, ) - @staticmethod def _convert_metric( + self, metric_name: str, definition: Dict[str, Any], column: Optional[str], @@ -365,7 +371,7 @@ def _convert_metric( } return OssieMetric( name=metric_name, - expression=_ansi(expression), + expression=_expression(expression, self._dialect), description=definition.get("description"), custom_extensions=_lightdash_extension(extension_data) or None, ) diff --git a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py index ea948b41..e605b8ee 100644 --- a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py +++ b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py @@ -30,7 +30,14 @@ import json from typing import Any, Dict, List, Optional, Set, Tuple -from ossie import OssieDataset, OssieDialect, OssieDocument, OssieMetric, OssieSemanticModel +from ossie import ( + OssieDataset, + OssieDialect, + OssieDocument, + OssieExpression, + OssieMetric, + OssieSemanticModel, +) from ossie_lightdash.converter_issues import ( ConverterIssue, @@ -59,13 +66,10 @@ _PROTECTED_AGGREGATION_KEYS = _PROTECTED_METRIC_KEYS | {"type", "percentile"} _PROTECTED_JOIN_KEYS = {"join", "sql_on", "alias"} - -def _pick_expression(ossie_expression: Any, dialect: OssieDialect) -> str: - """Return the expression for the preferred dialect (fallback: first available).""" - for dialect_expression in ossie_expression.dialects: - if dialect_expression.dialect is dialect: - return dialect_expression.expression - return ossie_expression.dialects[0].expression if ossie_expression.dialects else "" +# Joins a dataset declares, keyed by the joined dataset: the name Lightdash SQL +# uses to reference it (the joined model, or its alias). The first join to a +# dataset wins when it is joined more than once. +JoinReferences = Dict[str, str] def _lightdash_extension_data(element: Any, issues: List[ConverterIssue]) -> Dict[str, Any]: @@ -98,7 +102,12 @@ def _model_name_for(dataset: OssieDataset) -> str: class OssieToLightdashConverter: - """Converts an OssieDocument into a Lightdash-flavoured dbt schema.yml dict.""" + """Converts an OssieDocument into a Lightdash-flavoured dbt schema.yml dict. + + ``dialect`` is the expression dialect to prefer (the project's warehouse); + ``ANSI_SQL`` is the fallback, and an expression offering neither is taken + from its first dialect with a ``DIALECT_UNAVAILABLE`` issue. + """ def __init__(self, dialect: OssieDialect = OssieDialect.ANSI_SQL) -> None: self._dialect = dialect @@ -110,6 +119,26 @@ def convert(self, document: OssieDocument) -> ConverterResult[Dict[str, Any]]: models.extend(self._convert_semantic_model(semantic_model, issues)) return ConverterResult(output={"version": 2, "models": models}, issues=issues) + def _pick_expression( + self, expression: OssieExpression, element_name: str, issues: List[ConverterIssue] + ) -> str: + by_dialect = { + dialect_expression.dialect: dialect_expression.expression + for dialect_expression in expression.dialects + } + for dialect in (self._dialect, OssieDialect.ANSI_SQL): + if dialect in by_dialect: + return by_dialect[dialect] + if not expression.dialects: + return "" + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.DIALECT_UNAVAILABLE, + element_name=element_name, + ) + ) + return expression.dialects[0].expression + def _convert_semantic_model( self, semantic_model: OssieSemanticModel, issues: List[ConverterIssue] ) -> List[Dict[str, Any]]: @@ -119,28 +148,49 @@ def _convert_semantic_model( dataset.name: _model_name_for(dataset) for dataset in datasets } + # Joins are planned first: field and metric expressions may reference + # other datasets only through the joins their own dataset declares. + joins_by_dataset, references_by_dataset = self._plan_joins( + semantic_model, model_name_by_dataset, issues + ) + models_by_dataset: Dict[str, Dict[str, Any]] = {} columns_by_dataset: Dict[str, Dict[str, Dict[str, Any]]] = {} for dataset in datasets: - model, columns = self._convert_dataset(dataset, issues) + model, columns = self._convert_dataset( + dataset, dataset_names, references_by_dataset.get(dataset.name, {}), issues + ) models_by_dataset[dataset.name] = model columns_by_dataset[dataset.name] = columns for metric in semantic_model.metrics or []: self._convert_metric( metric, - dataset_names, + [dataset.name for dataset in datasets], models_by_dataset, columns_by_dataset, + references_by_dataset, issues, ) + for dataset_name, joins in joins_by_dataset.items(): + models_by_dataset[dataset_name].setdefault("meta", {})["joins"] = joins + + return [models_by_dataset[dataset.name] for dataset in datasets] + + def _plan_joins( + self, + semantic_model: OssieSemanticModel, + model_name_by_dataset: Dict[str, str], + issues: List[ConverterIssue], + ) -> Tuple[Dict[str, List[Dict[str, Any]]], Dict[str, JoinReferences]]: + joins_by_dataset: Dict[str, List[Dict[str, Any]]] = {} + references_by_dataset: Dict[str, JoinReferences] = {} joined_pairs: Set[Tuple[str, str]] = set() for relationship in semantic_model.relationships or []: - from_model = models_by_dataset.get(relationship.from_dataset) - to_model_name = model_name_by_dataset.get(relationship.to) from_model_name = model_name_by_dataset.get(relationship.from_dataset) - if from_model is None or to_model_name is None: + to_model_name = model_name_by_dataset.get(relationship.to) + if from_model_name is None or to_model_name is None: continue if len(relationship.from_columns) != len(relationship.to_columns): issues.append( @@ -177,13 +227,18 @@ def _convert_semantic_model( if key not in _PROTECTED_JOIN_KEYS } ) - joins = from_model.setdefault("meta", {}).setdefault("joins", []) - joins.append(join) - - return [models_by_dataset[dataset.name] for dataset in datasets] + joins_by_dataset.setdefault(relationship.from_dataset, []).append(join) + references_by_dataset.setdefault(relationship.from_dataset, {}).setdefault( + relationship.to, join_reference + ) + return joins_by_dataset, references_by_dataset def _convert_dataset( - self, dataset: OssieDataset, issues: List[ConverterIssue] + self, + dataset: OssieDataset, + dataset_names: Set[str], + references: JoinReferences, + issues: List[ConverterIssue], ) -> tuple: columns_by_name: Dict[str, Dict[str, Any]] = {} for field in dataset.fields or []: @@ -218,9 +273,22 @@ def _convert_dataset( element_name=field.name, ) ) - expression = _pick_expression(field.expression, self._dialect) + expression = self._pick_expression(field.expression, field.name, issues) if expression and expression != field.name: - dimension["sql"] = ossie_sql_to_lightdash(expression, dataset.name) + unjoined = referenced_datasets(expression, dataset_names) - { + dataset.name, + *references, + } + if unjoined: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.FIELD_REFERENCE_UNJOINED, + element_name=field.name, + ) + ) + dimension["sql"] = ossie_sql_to_lightdash( + expression, dataset.name, references + ) dimension.update( { key: value @@ -244,15 +312,18 @@ def _convert_dataset( def _convert_metric( self, metric: OssieMetric, - dataset_names: set, + dataset_names: List[str], models_by_dataset: Dict[str, Dict[str, Any]], columns_by_dataset: Dict[str, Dict[str, Dict[str, Any]]], + references_by_dataset: Dict[str, JoinReferences], issues: List[ConverterIssue], ) -> None: - expression = _pick_expression(metric.expression, self._dialect) + expression = self._pick_expression(metric.expression, metric.name, issues) extension_data = _lightdash_extension_data(metric, issues) - target_dataset = self._resolve_target_dataset(expression, dataset_names) + target_dataset = self._resolve_target_dataset( + expression, dataset_names, references_by_dataset + ) if target_dataset is None: issues.append( ConverterIssue( @@ -261,6 +332,7 @@ def _convert_metric( ) ) return + references = references_by_dataset.get(target_dataset, {}) definition: Dict[str, Any] = {} if metric.description: @@ -281,13 +353,13 @@ def _convert_metric( ): target_column = strip_qualifier(inner) else: - definition["sql"] = ossie_sql_to_lightdash(inner, target_dataset) + definition["sql"] = ossie_sql_to_lightdash(inner, target_dataset, references) if parsed.percentile is not None: definition["percentile"] = parsed.percentile protected = _PROTECTED_AGGREGATION_KEYS else: definition["type"] = extension_data.get("type", "number") - definition["sql"] = ossie_sql_to_lightdash(expression, target_dataset) + definition["sql"] = ossie_sql_to_lightdash(expression, target_dataset, references) protected = _PROTECTED_METRIC_KEYS definition.update( @@ -308,10 +380,25 @@ def _convert_metric( metrics[metric.name] = definition @staticmethod - def _resolve_target_dataset(expression: str, dataset_names: set) -> Optional[str]: - referenced = referenced_datasets(expression, dataset_names) + def _resolve_target_dataset( + expression: str, + dataset_names: List[str], + references_by_dataset: Dict[str, JoinReferences], + ) -> Optional[str]: + """The dataset whose model hosts the metric. + + A metric spanning several datasets lives on the one that joins all the + others directly: Lightdash resolves ``${other.column}`` only against + the joins the hosting model declares, never transitively. + """ + referenced = referenced_datasets(expression, set(dataset_names)) + if len(referenced) == 0: + return dataset_names[0] if len(dataset_names) == 1 else None if len(referenced) == 1: return next(iter(referenced)) - if len(referenced) == 0 and len(dataset_names) == 1: - return next(iter(dataset_names)) + for name in dataset_names: + if name in referenced and referenced - {name} <= set( + references_by_dataset.get(name, {}) + ): + return name return None diff --git a/converters/lightdash/tests/test_lightdash_to_ossie.py b/converters/lightdash/tests/test_lightdash_to_ossie.py index 8adb18bc..a49c5401 100644 --- a/converters/lightdash/tests/test_lightdash_to_ossie.py +++ b/converters/lightdash/tests/test_lightdash_to_ossie.py @@ -17,7 +17,7 @@ import json -from ossie import OssieDataType +from ossie import OssieDataType, OssieDialect from ossie_lightdash import ConverterIssueType, LightdashToOssieConverter @@ -464,3 +464,12 @@ def test_aliased_joins_become_relationships(self): and issue.element_name == "sold_year" for issue in result.issues ) + + def test_expressions_carry_the_warehouse_dialect(self): + result = LightdashToOssieConverter(OssieDialect.BIGQUERY).convert( + SCHEMA_YML, schema="marts" + ) + metric = _metric(result.output, "total_amount") + assert [d.dialect for d in metric.expression.dialects] == [OssieDialect.BIGQUERY] + field = result.output.semantic_model[0].datasets[0].fields[0] + assert field.expression.dialects[0].dialect is OssieDialect.BIGQUERY diff --git a/converters/lightdash/tests/test_ossie_to_lightdash.py b/converters/lightdash/tests/test_ossie_to_lightdash.py index 117bacac..9fb62522 100644 --- a/converters/lightdash/tests/test_ossie_to_lightdash.py +++ b/converters/lightdash/tests/test_ossie_to_lightdash.py @@ -185,14 +185,113 @@ def test_complex_expression_becomes_model_metric(self): assert metric["format"] == "percent" assert metric["round"] == 1 - def test_cross_dataset_metric_is_dropped_with_issue(self): + def test_metric_over_a_joined_dataset_lives_on_the_joining_model(self): result = OssieToLightdashConverter().convert(_document()) - assert any( + metric = _model(result.output, "orders")["meta"]["metrics"]["cross_dataset"] + assert metric == { + "type": "number", + "sql": "SUM(${TABLE}.amount) / COUNT(${customers.customer_id})", + } + assert not any( issue.issue_type is ConverterIssueType.CROSS_DATASET_METRIC_DROPPED - and issue.element_name == "cross_dataset" for issue in result.issues ) + def test_metric_over_an_unjoined_dataset_is_dropped_with_issue(self): + document = _document() + tampered = document.model_copy(deep=True) + # customers does not join orders, so no model can host the metric. + tampered.semantic_model[0] = tampered.semantic_model[0].model_copy( + update={ + "relationships": [], + "metrics": [ + *tampered.semantic_model[0].metrics, + OssieMetric( + name="reversed", + expression=_ansi("COUNT(customers.customer_id) / SUM(orders.amount)"), + ), + ], + } + ) + result = OssieToLightdashConverter().convert(tampered) + assert "metrics" not in _model(result.output, "customers").get("meta", {}) + assert { + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.CROSS_DATASET_METRIC_DROPPED + } == {"cross_dataset", "reversed"} + + def test_field_references_resolve_through_declared_joins(self): + document = _document() + tampered = document.model_copy(deep=True) + orders = tampered.semantic_model[0].datasets[0] + orders.fields.append( + OssieField( + name="customer_key", + expression=_ansi("UPPER(customers.customer_id)"), + dimension=OssieDimension(), + ) + ) + customers = tampered.semantic_model[0].datasets[1] + customers.fields.append( + OssieField( + name="last_order_status", + expression=_ansi("orders.status"), + dimension=OssieDimension(), + ) + ) + result = OssieToLightdashConverter().convert(tampered) + joined = _column(_model(result.output, "orders"), "customer_key") + assert joined["meta"]["dimension"]["sql"] == "UPPER(${customers.customer_id})" + unjoined = _column(_model(result.output, "customers"), "last_order_status") + assert unjoined["meta"]["dimension"]["sql"] == "orders.status" + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.FIELD_REFERENCE_UNJOINED + ] == ["last_order_status"] + + def test_preferred_dialect_falls_back_to_ansi_then_reports(self): + document = _document() + tampered = document.model_copy(deep=True) + orders = tampered.semantic_model[0].datasets[0] + orders.fields[1] = OssieField( + name="status", + expression=OssieExpression( + dialects=[ + OssieDialectExpression(dialect=OssieDialect.ANSI_SQL, expression="status"), + OssieDialectExpression( + dialect=OssieDialect.BIGQUERY, expression="LOWER(status)" + ), + ] + ), + dimension=OssieDimension(), + ) + orders.fields.append( + OssieField( + name="snowflake_only", + expression=OssieExpression( + dialects=[ + OssieDialectExpression( + dialect=OssieDialect.SNOWFLAKE, expression="status::VARCHAR" + ) + ] + ), + dimension=OssieDimension(), + ) + ) + result = OssieToLightdashConverter(OssieDialect.BIGQUERY).convert(tampered) + model = _model(result.output, "orders") + assert _column(model, "status")["meta"]["dimension"]["sql"] == "LOWER(status)" + assert ( + _column(model, "snowflake_only")["meta"]["dimension"]["sql"] == "status::VARCHAR" + ) + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.DIALECT_UNAVAILABLE + ] == ["snowflake_only"] + def test_foreign_extension_is_reported(self): result = OssieToLightdashConverter().convert(_document()) assert any( From 770d6023b2f9d262d356b6a24f0f6a6e4e139701 Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 11:02:00 +0100 Subject: [PATCH 11/33] feat(lightdash): map primary keys, AI hints, cardinality and time roles Lightdash has vocabulary for more of the Ossie model than the README claimed: - dataset.primary_key <-> meta.primary_key (string or composite list) - ai_context <-> ai_hint on models, dimensions and metrics; multi-line instructions become a list of hints, structured synonyms and examples are rendered as extra hints on export - exported joins carry relationship: many-to-one, the cardinality an Ossie relationship defines, unless a stashed Lightdash value overrides it - time_intervals: OFF <-> an explicit is_time: false on a temporal column - metric datatype is derived from the aggregation and column type on import A label or ai_context on a measure-only field is now dropped with a FIELD_ATTRIBUTE_NOT_REPRESENTABLE issue instead of turning the field into a dimension on export. --- converters/lightdash/README.md | 20 +++-- .../src/ossie_lightdash/converter_issues.py | 4 + .../src/ossie_lightdash/datatype_utils.py | 35 +++++++++ .../src/ossie_lightdash/lightdash_to_ossie.py | 55 ++++++++++++-- .../src/ossie_lightdash/ossie_to_lightdash.py | 64 +++++++++++++++- .../tests/test_lightdash_to_ossie.py | 52 ++++++++++++- .../tests/test_ossie_to_lightdash.py | 75 +++++++++++++++++++ 7 files changed, 284 insertions(+), 21 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 371ec3bf..803c0f89 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -42,16 +42,19 @@ ossie-lightdash import schema.yml semantic_model.json --database analytics_db -- | ----- | -------------------- | | `dataset` | dbt model (`name` = table part of `source`) | | `dataset.source` | assembled on import from `--database` / `--schema` / model name | +| `dataset.primary_key` | model `meta.primary_key` (a single key as a string, a composite key as a list) | +| `ai_context` on datasets, fields and metrics | `ai_hint` on models, dimensions and metrics; a multi-line instruction is a list of hints, and the synonyms / examples of the structured form are rendered as extra hints on export | | `field` (no `dimension`) | plain column entry | | `field` with `dimension` | `columns[].meta.dimension` (an empty `dimension: {}` marks a dimension with no extra attributes) | | `field.datatype` | `meta.dimension.type` (`String`→`string`, `Integer`/`Decimal`/`Float`→`number`, `Date`→`date`, `DateTime`/`DateTimeTz`→`timestamp`, `Boolean`→`boolean`, `Time`/`Opaque`→`string`); on import `number` maps back to `Decimal` | -| `field.dimension.is_time` | *not carried* — it is a role marker (a field can be a time axis without a temporal datatype, e.g. a year stored as `Integer`) and Lightdash has no equivalent | +| `field.dimension.is_time` | `time_intervals: OFF` ↔ an explicit `is_time: false` on a temporal column; otherwise not carried — a non-temporal time axis (e.g. a year stored as `Integer`) has no Lightdash equivalent | | `field.label` / `.description` | `meta.dimension.label` / column `description` | | `field.expression` (≠ column name) | `meta.dimension.sql` (`dataset.col` ↔ `${TABLE}.col`) | +| `metric.datatype` | derived on import: `Integer` for counts, `Decimal` for numeric aggregates over a `number` column, the column's type for `min`/`max`, the declared type for `boolean`/`string`/`date`/`timestamp` metrics | | `metric` that is one aggregation over a column (`SUM(ds.col)`, `COUNT(DISTINCT ds.col)`, `SUM(DISTINCT ds.col)`, `PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY ds.col)`, ...) | column-level `meta.metrics.` with a typed metric (`sum`, `count_distinct`, `sum_distinct`, `percentile` + `percentile: 90`, ...) | | `metric` that is one aggregation over any other expression (`AVG(CASE WHEN ds.status = 'done' THEN 1 ELSE 0 END)`) | model-level `meta.metrics.` with the typed metric and the operand as `sql` | | `metric` with any other expression | model-level `meta.metrics.` with `type: number` + `sql`, on the dataset that joins every other dataset the expression references (`${joined_model.col}`) | -| `relationship` | `meta.joins` (`sql_on` built from / parsed into column pairs); the `alias` and other join attributes (`relationship`, `type`, `fields`, ...) travel in the relationship's `lightdash` extension, and a dataset joined more than once from the same model is aliased on export | +| `relationship` | `meta.joins` (`sql_on` built from / parsed into column pairs, `relationship: many-to-one` unless a stashed one says otherwise); the `alias` and other join attributes (`relationship`, `type`, `fields`, ...) travel in the relationship's `lightdash` extension, and a dataset joined more than once from the same model is aliased on export | | `${TABLE}.col`, `${col}`, `${other_model.col}` in Lightdash SQL | `dataset.col` / `other_model.col`; `${alias.col}` is flattened onto the aliased model, `${metric}` is replaced by that metric's expression | | Lightdash presentation attributes (`label`, `format`, `round`, `compact`, `group_label`, `hidden`, ...) | `custom_extensions` with `vendor_name: "lightdash"`; on export the extension data is overlaid onto the generated definition (structural keys — `sql`/`label` on dimensions, `sql`/`description` on metrics and `type`/`percentile` on metrics whose expression is a recognised aggregation, `join`/`sql_on` on joins — are protected and cannot be overridden) | @@ -93,8 +96,11 @@ Omitting `--schema` as well is reported as a `SOURCE_UNQUALIFIED` issue. the joined dataset (`date_dim.year`) with an `ALIAS_REFERENCE_FLATTENED` issue: Ossie has no aliases, so which of several joins to the same dataset was meant is not preserved in the expression. -- **`primary_key` / `unique_keys` are not exported** — Lightdash has no - corresponding concept — and consequently cannot be reconstructed on import. +- **`unique_keys` are not exported** — Lightdash has no corresponding + concept — and consequently cannot be reconstructed on import. +- **A label or `ai_context` on a measure-only field is dropped on export** + (`FIELD_ATTRIBUTE_NOT_REPRESENTABLE`): Lightdash keeps both on dimensions + only, and writing them would turn the field into a dimension. - **`dataset.name` is not preserved when it differs from the source table name**: the dbt model is named after the table part of `source`, and the import direction derives dataset names from model names. References inside @@ -107,10 +113,8 @@ Omitting `--schema` as well is reported as a `SOURCE_UNQUALIFIED` issue. `DateTimeTz` as `DateTime`. - **A measure-only field (no `dimension`) loses its `datatype`**: Lightdash carries types on dimensions only, so there is nowhere to put it. -- **`dimension.is_time` is not carried**, and a field whose datatype is not - temporal but is flagged as a time axis is reported with a - `TIME_ROLE_NOT_REPRESENTABLE` issue on export. -- **`ai_context` is not carried** into Lightdash meta. +- **A non-temporal time axis** (`is_time: true` on an `Integer` year) is + reported with a `TIME_ROLE_NOT_REPRESENTABLE` issue on export. - **Model-level Lightdash meta beyond `metrics` and `joins`** (`label`, `group_details`, `sql_filter`, `order_fields_by`, column `additional_dimensions`, ...) is not carried yet. diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py index fd22fa7f..813e496f 100644 --- a/converters/lightdash/src/ossie_lightdash/converter_issues.py +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -60,6 +60,10 @@ class ConverterIssueType(Enum): # Export: a field expression references a dataset the field's dataset does # not join, so Lightdash cannot resolve the reference. FIELD_REFERENCE_UNJOINED = "FIELD_REFERENCE_UNJOINED" + # Export: a label or AI context on a measure-only field has no Lightdash + # home (column meta exists only for dimensions) and is dropped rather than + # turning the field into a dimension. + FIELD_ATTRIBUTE_NOT_REPRESENTABLE = "FIELD_ATTRIBUTE_NOT_REPRESENTABLE" @dataclass(frozen=True) diff --git a/converters/lightdash/src/ossie_lightdash/datatype_utils.py b/converters/lightdash/src/ossie_lightdash/datatype_utils.py index 40de1076..bd432c18 100644 --- a/converters/lightdash/src/ossie_lightdash/datatype_utils.py +++ b/converters/lightdash/src/ossie_lightdash/datatype_utils.py @@ -71,6 +71,41 @@ def lightdash_type_to_datatype(lightdash_type: Optional[str]) -> Optional[OssieD return _LIGHTDASH_TO_DATATYPE.get(lightdash_type) +_COUNT_TYPES = {"count", "count_distinct"} +_NUMERIC_AGGREGATES = { + "sum", "sum_distinct", "average", "average_distinct", "median", "percentile" +} +_ORDER_AGGREGATES = {"min", "max"} +_VALUE_TYPES = { + "boolean": OssieDataType.BOOLEAN, + "string": OssieDataType.STRING, + "date": OssieDataType.DATE, + "timestamp": OssieDataType.DATE_TIME, +} + + +def metric_datatype( + lightdash_type: str, column_type: Optional[str] +) -> Optional[OssieDataType]: + """The Ossie datatype of a Lightdash metric's result, when it is implied. + + Counts are integers; numeric aggregates over a `number` column are + decimals; min/max keep the column's type; value-typed metrics (`boolean`, + `string`, `date`, `timestamp`) declare their own type. Anything else + (`number` with arbitrary SQL, aggregates over untyped columns) is left + unset rather than guessed. + """ + if lightdash_type in _COUNT_TYPES: + return OssieDataType.INTEGER + if lightdash_type in _VALUE_TYPES: + return _VALUE_TYPES[lightdash_type] + if lightdash_type in _NUMERIC_AGGREGATES and column_type == "number": + return OssieDataType.DECIMAL + if lightdash_type in _ORDER_AGGREGATES: + return lightdash_type_to_datatype(column_type) + return None + + def is_temporal(datatype: Optional[OssieDataType]) -> bool: """True when the datatype represents a point in time.""" return datatype in _TEMPORAL diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py index aced3e87..014354ce 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py @@ -48,7 +48,7 @@ ConverterIssueType, ConverterResult, ) -from ossie_lightdash.datatype_utils import lightdash_type_to_datatype +from ossie_lightdash.datatype_utils import lightdash_type_to_datatype, metric_datatype from ossie_lightdash.expression_utils import ( AGGREGATE_TYPES, build_aggregation, @@ -60,8 +60,8 @@ # Keys that are structurally encoded in Ossie vocabulary and therefore must NOT # be duplicated into the extension (a stale copy would win on export). -_STRUCTURAL_METRIC_KEYS = {"sql", "description"} -_STRUCTURAL_DIMENSION_KEYS = {"label", "sql"} +_STRUCTURAL_METRIC_KEYS = {"sql", "description", "ai_hint"} +_STRUCTURAL_DIMENSION_KEYS = {"label", "sql", "ai_hint"} _STRUCTURAL_JOIN_KEYS = {"join", "sql_on"} _JOIN_PAIR_RE = re.compile( @@ -86,6 +86,23 @@ def _lightdash_extension(data: Dict[str, Any]) -> List[OssieCustomExtension]: ] +def _ai_context(ai_hint: Any) -> Optional[str]: + """Lightdash `ai_hint` (a string or a list of strings) as Ossie `ai_context`.""" + if isinstance(ai_hint, list): + return "\n".join(str(hint) for hint in ai_hint) or None + if isinstance(ai_hint, str): + return ai_hint or None + return None + + +def _primary_key(primary_key: Any) -> Optional[List[str]]: + if isinstance(primary_key, str): + return [primary_key] + if isinstance(primary_key, list) and primary_key: + return [str(column) for column in primary_key] + return None + + def _unique_name(base: str, used: Set[str]) -> str: name = base suffix = 2 @@ -111,6 +128,7 @@ def __init__( self.aliases = aliases self.definitions = definitions self.issues = issues + self.column_types: Dict[str, str] = {} self._expressions: Dict[str, Optional[str]] = {} self._resolving: Set[str] = set() @@ -271,6 +289,10 @@ def _convert_model( definitions[metric_name] = (definition, None) context = _ModelContext(name, aliases, definitions, issues) + for column in model.get("columns") or []: + dimension_meta = (column.get("meta") or {}).get("dimension") or {} + if dimension_meta.get("type"): + context.column_types[column["name"]] = dimension_meta["type"] fields: List[OssieField] = [] for column in model.get("columns") or []: @@ -295,6 +317,8 @@ def _convert_model( name=name, source=source, description=model.get("description"), + primary_key=_primary_key(model_meta.get("primary_key")), + ai_context=_ai_context(model_meta.get("ai_hint")), fields=fields or None, ) return dataset, metrics, relationships @@ -309,23 +333,32 @@ def _convert_column( dimension: Optional[OssieDimension] = None datatype = None label: Optional[str] = None + ai_context: Optional[str] = None extension_data: Dict[str, Any] = {} if dimension_meta is not None: label = dimension_meta.get("label") + ai_context = _ai_context(dimension_meta.get("ai_hint")) if dimension_meta.get("sql"): rewritten = context.rewrite(dimension_meta["sql"], column_name) if rewritten is None: return None expression = rewritten datatype = lightdash_type_to_datatype(dimension_meta.get("type")) - # `is_time` is a role marker in Ossie, not a type: Lightdash has no - # equivalent, so it is left unset rather than inferred from the type - # (the type itself is carried by `datatype`). - dimension = OssieDimension() + # `is_time` is a role marker in Ossie, not a type. Lightdash's only + # role marker is `time_intervals: OFF`, which withdraws a temporal + # column from the time axis; otherwise `is_time` is left unset so + # the datatype decides. + excluded = set(_STRUCTURAL_DIMENSION_KEYS) + time_intervals = dimension_meta.get("time_intervals") + if time_intervals is False or time_intervals == "OFF": + dimension = OssieDimension(is_time=False) + excluded.add("time_intervals") + else: + dimension = OssieDimension() extension_data = { key: value for key, value in dimension_meta.items() - if key not in _STRUCTURAL_DIMENSION_KEYS + if key not in excluded } return OssieField( @@ -335,6 +368,7 @@ def _convert_column( datatype=datatype, label=label, description=column.get("description"), + ai_context=ai_context, custom_extensions=_lightdash_extension(extension_data) or None, ) @@ -372,7 +406,12 @@ def _convert_metric( return OssieMetric( name=metric_name, expression=_expression(expression, self._dialect), + datatype=metric_datatype( + lightdash_type, + context.column_types.get(column) if not definition.get("sql") else None, + ), description=definition.get("description"), + ai_context=_ai_context(definition.get("ai_hint")), custom_extensions=_lightdash_extension(extension_data) or None, ) diff --git a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py index e605b8ee..4fb5ea3e 100644 --- a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py +++ b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py @@ -61,8 +61,8 @@ # from overriding the Ossie-derived definition. ``type`` stays overridable on # metrics whose expression is not a recognised aggregation: it is the channel # for types an expression cannot express (``boolean``, ``string``, ...). -_PROTECTED_DIMENSION_KEYS = {"sql", "label"} -_PROTECTED_METRIC_KEYS = {"sql", "description"} +_PROTECTED_DIMENSION_KEYS = {"sql", "label", "ai_hint"} +_PROTECTED_METRIC_KEYS = {"sql", "description", "ai_hint"} _PROTECTED_AGGREGATION_KEYS = _PROTECTED_METRIC_KEYS | {"type", "percentile"} _PROTECTED_JOIN_KEYS = {"join", "sql_on", "alias"} @@ -96,6 +96,26 @@ def _lightdash_extension_data(element: Any, issues: List[ConverterIssue]) -> Dic return data +def _ai_hint(ai_context: Any) -> Any: + """Ossie `ai_context` as a Lightdash `ai_hint`: one line stays a string, a + multi-line instruction becomes a list, and synonyms and examples of the + structured form are rendered as extra hints.""" + if ai_context is None: + return None + if isinstance(ai_context, str): + hints = ai_context.split("\n") + else: + hints = (ai_context.instructions or "").split("\n") + if ai_context.synonyms: + hints.append("Also known as: " + ", ".join(ai_context.synonyms)) + if ai_context.examples: + hints.append("Example questions: " + "; ".join(ai_context.examples)) + hints = [hint for hint in hints if hint] + if not hints: + return None + return hints[0] if len(hints) == 1 else hints + + def _model_name_for(dataset: OssieDataset) -> str: """A Lightdash table is addressed by its dbt model name = the source's table part.""" return dataset.source.rsplit(".", 1)[-1] @@ -216,10 +236,13 @@ def _plan_joins( relationship.from_columns, relationship.to_columns ) ) + # An Ossie relationship always runs from the many side to the one + # side; a stashed Lightdash `relationship` overrides it below. join: Dict[str, Any] = {"join": to_model_name} if alias: join["alias"] = alias join["sql_on"] = sql_on + join["relationship"] = "many-to-one" join.update( { key: value @@ -247,8 +270,21 @@ def _convert_dataset( column["description"] = field.description dimension: Dict[str, Any] = {} - if field.label: - dimension["label"] = field.label + ai_hint = _ai_hint(field.ai_context) + if field.dimension is None and (field.label or ai_hint is not None): + # Lightdash keeps labels and AI hints on dimensions only; + # writing them would turn a measure-only field into one. + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.FIELD_ATTRIBUTE_NOT_REPRESENTABLE, + element_name=field.name, + ) + ) + elif field.dimension is not None: + if field.label: + dimension["label"] = field.label + if ai_hint is not None: + dimension["ai_hint"] = ai_hint if field.dimension is not None: # Only dimension fields carry a Lightdash type: emitting one for # a measure-only field would turn it into a dimension on import. @@ -273,6 +309,14 @@ def _convert_dataset( element_name=field.name, ) ) + if ( + field.dimension is not None + and field.dimension.is_time is False + and is_temporal(field.datatype) + ): + # Explicitly withdrawn from the time axis: Lightdash's role + # marker for that is `time_intervals: OFF`. + dimension["time_intervals"] = "OFF" expression = self._pick_expression(field.expression, field.name, issues) if expression and expression != field.name: unjoined = referenced_datasets(expression, dataset_names) - { @@ -306,6 +350,15 @@ def _convert_dataset( model: Dict[str, Any] = {"name": _model_name_for(dataset)} if dataset.description: model["description"] = dataset.description + meta: Dict[str, Any] = {} + if dataset.primary_key: + keys = list(dataset.primary_key) + meta["primary_key"] = keys[0] if len(keys) == 1 else keys + ai_hint = _ai_hint(dataset.ai_context) + if ai_hint is not None: + meta["ai_hint"] = ai_hint + if meta: + model["meta"] = meta model["columns"] = list(columns_by_name.values()) return model, columns_by_name @@ -337,6 +390,9 @@ def _convert_metric( definition: Dict[str, Any] = {} if metric.description: definition["description"] = metric.description + ai_hint = _ai_hint(metric.ai_context) + if ai_hint is not None: + definition["ai_hint"] = ai_hint # A single aggregation becomes a typed metric: on the column when the # operand is one of the dataset's columns, otherwise on the model with diff --git a/converters/lightdash/tests/test_lightdash_to_ossie.py b/converters/lightdash/tests/test_lightdash_to_ossie.py index a49c5401..c6ac8526 100644 --- a/converters/lightdash/tests/test_lightdash_to_ossie.py +++ b/converters/lightdash/tests/test_lightdash_to_ossie.py @@ -28,6 +28,8 @@ "name": "orders", "description": "One row per order", "meta": { + "primary_key": "order_id", + "ai_hint": ["Orders placed in the web shop.", "One row per order."], "joins": [ { "join": "customers", @@ -52,18 +54,39 @@ }, { "name": "status", - "meta": {"dimension": {"label": "Status", "type": "string"}}, + "meta": { + "dimension": { + "label": "Status", + "type": "string", + "ai_hint": "Order lifecycle stage.", + } + }, + }, + { + "name": "updated_at", + "meta": { + "dimension": {"type": "timestamp", "time_intervals": "OFF"} + }, + }, + { + "name": "shipped_at", + "meta": { + "dimension": {"type": "timestamp", "time_intervals": ["DAY", "MONTH"]} + }, }, { "name": "amount", "description": "Order amount", "meta": { + "dimension": {"type": "number"}, "metrics": { "total_amount": { "type": "sum", "label": "Total amount", "format": "usd", + "ai_hint": "Revenue before refunds.", }, + "latest_amount": {"type": "max"}, "median_amount": {"type": "median"}, "p90_amount": {"type": "percentile", "percentile": 90}, } @@ -83,6 +106,7 @@ }, { "name": "customers", + "meta": {"primary_key": ["customer_id", "region"]}, "columns": [{"name": "customer_id"}], }, ], @@ -132,6 +156,14 @@ def test_time_dimension(self): # marker with no Lightdash source, so it stays unset. assert field.datatype is OssieDataType.DATE assert field.dimension.is_time is None + withdrawn = result.output.semantic_model[0].datasets[0].fields[2] + assert withdrawn.name == "updated_at" + assert withdrawn.dimension.is_time is False + assert _lightdash_data(withdrawn) == {"type": "timestamp"} + # A custom interval list is not a role marker: it stays in the extension. + custom = result.output.semantic_model[0].datasets[0].fields[3] + assert custom.dimension.is_time is None + assert _lightdash_data(custom) == {"type": "timestamp", "time_intervals": ["DAY", "MONTH"]} def test_dimension_types_become_datatypes(self): result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") @@ -147,6 +179,7 @@ def test_typed_metric_becomes_aggregation_expression(self): metric = _metric(result.output, "total_amount") assert metric.expression.dialects[0].expression == "SUM(orders.amount)" assert _lightdash_data(metric) == {"label": "Total amount", "format": "usd"} + assert metric.ai_context == "Revenue before refunds." def test_count_distinct_metric(self): result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") @@ -473,3 +506,20 @@ def test_expressions_carry_the_warehouse_dialect(self): assert [d.dialect for d in metric.expression.dialects] == [OssieDialect.BIGQUERY] field = result.output.semantic_model[0].datasets[0].fields[0] assert field.expression.dialects[0].dialect is OssieDialect.BIGQUERY + + def test_primary_key_and_ai_hint_become_dataset_attributes(self): + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") + orders, customers = result.output.semantic_model[0].datasets + assert orders.primary_key == ["order_id"] + assert orders.ai_context == "Orders placed in the web shop.\nOne row per order." + assert customers.primary_key == ["customer_id", "region"] + status = next(field for field in orders.fields if field.name == "status") + assert status.ai_context == "Order lifecycle stage." + assert _lightdash_data(status) == {"type": "string"} + + def test_metric_datatypes_follow_the_aggregation(self): + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") + assert _metric(result.output, "unique_customers").datatype is OssieDataType.INTEGER + assert _metric(result.output, "total_amount").datatype is OssieDataType.DECIMAL + assert _metric(result.output, "latest_amount").datatype is OssieDataType.DECIMAL + assert _metric(result.output, "conversion_rate").datatype is None diff --git a/converters/lightdash/tests/test_ossie_to_lightdash.py b/converters/lightdash/tests/test_ossie_to_lightdash.py index 9fb62522..290000fc 100644 --- a/converters/lightdash/tests/test_ossie_to_lightdash.py +++ b/converters/lightdash/tests/test_ossie_to_lightdash.py @@ -18,7 +18,9 @@ import json from ossie import ( + OssieAIContextObject, OssieCustomExtension, + OssieDataType, OssieDataset, OssieDialect, OssieDialectExpression, @@ -369,6 +371,7 @@ def test_relationship_becomes_join(self): { "join": "customers", "sql_on": "${orders.customer_id} = ${customers.customer_id}", + "relationship": "many-to-one", } ] @@ -408,11 +411,13 @@ def test_repeated_relationship_gets_an_alias(self): { "join": "customers", "sql_on": "${orders.customer_id} = ${customers.customer_id}", + "relationship": "many-to-one", }, { "join": "customers", "alias": "orders_to_referrer", "sql_on": "${orders.amount} = ${orders_to_referrer.customer_id}", + "relationship": "many-to-one", }, ] @@ -445,3 +450,73 @@ def test_join_alias_and_attributes_restore_from_extension(self): "relationship": "many-to-one", } ] + + def test_primary_key_and_ai_context_become_model_meta(self): + document = _document() + tampered = document.model_copy(deep=True) + datasets = tampered.semantic_model[0].datasets + datasets[0] = datasets[0].model_copy( + update={ + "primary_key": ["order_id"], + "ai_context": "Orders placed in the web shop.\nOne row per order.", + } + ) + datasets[1] = datasets[1].model_copy( + update={ + "primary_key": ["customer_id", "region"], + "ai_context": OssieAIContextObject( + instructions="Customer master data.", + synonyms=("clients", "buyers"), + ), + } + ) + result = OssieToLightdashConverter().convert(tampered) + orders_meta = _model(result.output, "orders")["meta"] + assert orders_meta["primary_key"] == "order_id" + assert orders_meta["ai_hint"] == [ + "Orders placed in the web shop.", + "One row per order.", + ] + customers_meta = _model(result.output, "customers")["meta"] + assert customers_meta["primary_key"] == ["customer_id", "region"] + assert customers_meta["ai_hint"] == [ + "Customer master data.", + "Also known as: clients, buyers", + ] + + def test_field_and_metric_ai_context_become_ai_hints(self): + document = _document() + tampered = document.model_copy(deep=True) + orders = tampered.semantic_model[0].datasets[0] + orders.fields[1] = orders.fields[1].model_copy(update={"ai_context": "Order lifecycle stage."}) + # A measure-only field cannot carry the hint without becoming a dimension. + orders.fields[2] = orders.fields[2].model_copy(update={"ai_context": "Gross amount."}) + metrics = tampered.semantic_model[0].metrics + metrics[0] = metrics[0].model_copy(update={"ai_context": "Revenue before refunds."}) + result = OssieToLightdashConverter().convert(tampered) + model = _model(result.output, "orders") + assert _column(model, "status")["meta"]["dimension"]["ai_hint"] == "Order lifecycle stage." + assert "dimension" not in _column(model, "amount")["meta"] + assert _column(model, "amount")["meta"]["metrics"]["total_amount"]["ai_hint"] == ( + "Revenue before refunds." + ) + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.FIELD_ATTRIBUTE_NOT_REPRESENTABLE + ] == ["amount"] + + def test_time_axis_withdrawn_becomes_time_intervals_off(self): + document = _document() + tampered = document.model_copy(deep=True) + orders = tampered.semantic_model[0].datasets[0] + orders.fields[0] = orders.fields[0].model_copy( + update={"datatype": OssieDataType.DATE, "dimension": OssieDimension(is_time=False)} + ) + result = OssieToLightdashConverter().convert(tampered) + column = _column(_model(result.output, "orders"), "order_date") + assert column["meta"]["dimension"] == { + "label": "Order date", + "type": "date", + "time_intervals": "OFF", + } From b3a88b5b1d68ca82e214adc7465b4c1e65beafed Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 11:02:45 +0100 Subject: [PATCH 12/33] fix(lightdash): treat Liquid templating as non-portable on import Lightdash evaluates {% ... %} / {{ ... }} tags and bare ld.parameters, ld.query and ld.user references at query time; a dimension or metric using them is skipped with EXPRESSION_NOT_PORTABLE like ${ld....} references. --- converters/lightdash/README.md | 6 ++++-- .../src/ossie_lightdash/expression_utils.py | 13 +++++++++++-- .../lightdash/tests/test_lightdash_to_ossie.py | 11 ++++++++++- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 803c0f89..ddcb22c8 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -87,8 +87,10 @@ Omitting `--schema` as well is reported as a `SOURCE_UNQUALIFIED` issue. when an expression names it (`date_dim.year` → `${date_dim.year}` rather than the aliased second join). - **Parameter and user-attribute references** (`${lightdash.parameters.x}`, - `${ld.user.email}`) have no Ossie form: a dimension or metric whose SQL uses - them is skipped on import with an `EXPRESSION_NOT_PORTABLE` issue. + `${ld.user.email}`) and **Liquid templating** (`{% if ld.query.filters … %}`) + are evaluated by Lightdash at query time and have no Ossie form: a dimension + or metric whose SQL uses them is skipped on import with an + `EXPRESSION_NOT_PORTABLE` issue. - **Metric-to-metric references** (`${other_metric}`) are inlined on import (`METRIC_REFERENCE_INLINED`), since Ossie metrics cannot reference each other; the export direction does not reconstruct the reference. diff --git a/converters/lightdash/src/ossie_lightdash/expression_utils.py b/converters/lightdash/src/ossie_lightdash/expression_utils.py index 867557d8..a11e23fc 100644 --- a/converters/lightdash/src/ossie_lightdash/expression_utils.py +++ b/converters/lightdash/src/ossie_lightdash/expression_utils.py @@ -30,6 +30,11 @@ _REFERENCE_RE = re.compile(r"\$\{([^}]+)\}") _NON_PORTABLE_REFERENCE_RE = re.compile(r"\$\{(?:lightdash|ld)\.[^}]*\}") +# Liquid tags and bare `ld.parameters.x` / `ld.query.filters` / `ld.user.x` +# references inside them are evaluated by Lightdash at query time. +_NON_PORTABLE_SYNTAX_RE = re.compile( + r"\{%|\{\{|(? str: def has_non_portable_reference(sql: str) -> bool: - """True when the SQL references project parameters or user attributes.""" - return _NON_PORTABLE_REFERENCE_RE.search(sql) is not None + """True when the SQL depends on Lightdash query-time evaluation: project + parameters, user attributes, or Liquid templating.""" + return ( + _NON_PORTABLE_REFERENCE_RE.search(sql) is not None + or _NON_PORTABLE_SYNTAX_RE.search(sql) is not None + ) @dataclass diff --git a/converters/lightdash/tests/test_lightdash_to_ossie.py b/converters/lightdash/tests/test_lightdash_to_ossie.py index c6ac8526..140ef57b 100644 --- a/converters/lightdash/tests/test_lightdash_to_ossie.py +++ b/converters/lightdash/tests/test_lightdash_to_ossie.py @@ -436,6 +436,15 @@ def test_parameter_references_skip_the_element(self): } }, }, + { + "name": "status_label", + "meta": { + "dimension": { + "type": "string", + "sql": "{% if ld.query.filters contains 'orders.status' %} 'filtered' {% else %} ${TABLE}.status {% endif %}", + } + }, + }, {"name": "order_date", "meta": {"dimension": {"type": "date"}}}, ], } @@ -449,7 +458,7 @@ def test_parameter_references_skip_the_element(self): issue.element_name for issue in result.issues if issue.issue_type is ConverterIssueType.EXPRESSION_NOT_PORTABLE - ) == ["is_recent", "my_orders"] + ) == ["is_recent", "my_orders", "status_label"] def test_aliased_joins_become_relationships(self): schema_yml = { From caa00b287a9360ff1fcdb3414d46d77dae6e92a7 Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 11:08:23 +0100 Subject: [PATCH 13/33] feat(lightdash): read Lightdash meta from config.meta dbt 1.10+ nests meta under config; Lightdash merges meta and config.meta with config.meta winning. The importer only read the top-level key and saw an empty project when every model used the new placement. Export gains --meta-under-config to write that placement. --- converters/lightdash/README.md | 7 +++ .../lightdash/src/ossie_lightdash/cli.py | 11 +++-- .../src/ossie_lightdash/lightdash_to_ossie.py | 29 +++++++++++-- .../src/ossie_lightdash/ossie_to_lightdash.py | 20 ++++++++- .../tests/test_lightdash_to_ossie.py | 43 +++++++++++++++++++ .../tests/test_ossie_to_lightdash.py | 9 ++++ 6 files changed, 111 insertions(+), 8 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index ddcb22c8..152d9db6 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -67,6 +67,13 @@ labels the emitted expressions with that warehouse's Ossie dialect prefers that dialect, falls back to `ANSI_SQL`, and takes the first available dialect with a `DIALECT_UNAVAILABLE` issue when an expression offers neither. +## Where the meta lives + +dbt 1.10+ places `meta` under `config:`; Lightdash reads both the top-level +`meta` and `config.meta` and lets `config.meta` win, and so does the import +direction. Export writes top-level `meta` by default; pass +`--meta-under-config` to emit the `config.meta` placement instead. + ## Recommended source shape for dbt-native flows If the Ossie documents are also consumed by dbt's native OSI parsing, prefer diff --git a/converters/lightdash/src/ossie_lightdash/cli.py b/converters/lightdash/src/ossie_lightdash/cli.py index ac3ae0a4..066732da 100644 --- a/converters/lightdash/src/ossie_lightdash/cli.py +++ b/converters/lightdash/src/ossie_lightdash/cli.py @@ -57,6 +57,11 @@ def main(argv: Optional[List[str]] = None) -> int: default=OssieDialect.ANSI_SQL.name, help="preferred expression dialect (falls back to ANSI_SQL)", ) + export_parser.add_argument( + "--meta-under-config", + action="store_true", + help="write Lightdash meta under `config:` (dbt 1.10+) instead of top-level `meta:`", + ) import_parser = subparsers.add_parser( "import", help="Lightdash dbt schema.yml -> Ossie document (.json/.yaml)" @@ -78,9 +83,9 @@ def main(argv: Optional[List[str]] = None) -> int: args = parser.parse_args(argv) if args.command == "export": - result = OssieToLightdashConverter(OssieDialect[args.dialect]).convert( - _read_document(args.input) - ) + result = OssieToLightdashConverter( + OssieDialect[args.dialect], meta_under_config=args.meta_under_config + ).convert(_read_document(args.input)) args.output.write_text( yaml.safe_dump(result.output, sort_keys=False, allow_unicode=True), encoding="utf-8", diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py index 014354ce..f2f524b5 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py @@ -103,6 +103,27 @@ def _primary_key(primary_key: Any) -> Optional[List[str]]: return None +def _merge_meta(base: Any, override: Any) -> Dict[str, Any]: + """Deep-merge two meta blocks; keys in ``override`` win.""" + merged: Dict[str, Any] = dict(base) if isinstance(base, dict) else {} + if isinstance(override, dict): + for key, value in override.items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = _merge_meta(merged[key], value) + else: + merged[key] = value + return merged + + +def lightdash_meta(node: Dict[str, Any]) -> Dict[str, Any]: + """The Lightdash meta of a dbt model or column. + + dbt 1.10+ moved ``meta`` under ``config``; Lightdash reads both and lets + ``config.meta`` win, so the converter merges them the same way. + """ + return _merge_meta(node.get("meta"), (node.get("config") or {}).get("meta")) + + def _unique_name(base: str, used: Set[str]) -> str: name = base suffix = 2 @@ -270,7 +291,7 @@ def _convert_model( ) ) - model_meta = model.get("meta") or {} + model_meta = lightdash_meta(model) joins = model_meta.get("joins") or [] aliases = { join["alias"]: join["join"] @@ -282,7 +303,7 @@ def _convert_model( # `${metric}` references can be inlined. definitions: Dict[str, Tuple[Dict[str, Any], Optional[str]]] = {} for column in model.get("columns") or []: - column_meta = column.get("meta") or {} + column_meta = lightdash_meta(column) for metric_name, definition in (column_meta.get("metrics") or {}).items(): definitions[metric_name] = (definition, column["name"]) for metric_name, definition in (model_meta.get("metrics") or {}).items(): @@ -290,7 +311,7 @@ def _convert_model( context = _ModelContext(name, aliases, definitions, issues) for column in model.get("columns") or []: - dimension_meta = (column.get("meta") or {}).get("dimension") or {} + dimension_meta = lightdash_meta(column).get("dimension") or {} if dimension_meta.get("type"): context.column_types[column["name"]] = dimension_meta["type"] @@ -327,7 +348,7 @@ def _convert_column( self, column: Dict[str, Any], context: _ModelContext ) -> Optional[OssieField]: column_name = column["name"] - dimension_meta = (column.get("meta") or {}).get("dimension") + dimension_meta = lightdash_meta(column).get("dimension") expression = column_name dimension: Optional[OssieDimension] = None diff --git a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py index 4fb5ea3e..0aeb85fa 100644 --- a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py +++ b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py @@ -116,6 +116,13 @@ def _ai_hint(ai_context: Any) -> Any: return hints[0] if len(hints) == 1 else hints +def _nest_meta_under_config(node: Dict[str, Any]) -> None: + """Move a node's ``meta`` under ``config`` (the dbt 1.10+ placement).""" + meta = node.pop("meta", None) + if meta: + node["config"] = {"meta": meta} + + def _model_name_for(dataset: OssieDataset) -> str: """A Lightdash table is addressed by its dbt model name = the source's table part.""" return dataset.source.rsplit(".", 1)[-1] @@ -129,14 +136,25 @@ class OssieToLightdashConverter: from its first dialect with a ``DIALECT_UNAVAILABLE`` issue. """ - def __init__(self, dialect: OssieDialect = OssieDialect.ANSI_SQL) -> None: + def __init__( + self, + dialect: OssieDialect = OssieDialect.ANSI_SQL, + *, + meta_under_config: bool = False, + ) -> None: self._dialect = dialect + self._meta_under_config = meta_under_config def convert(self, document: OssieDocument) -> ConverterResult[Dict[str, Any]]: issues: List[ConverterIssue] = [] models: List[Dict[str, Any]] = [] for semantic_model in document.semantic_model: models.extend(self._convert_semantic_model(semantic_model, issues)) + if self._meta_under_config: + for model in models: + _nest_meta_under_config(model) + for column in model.get("columns") or []: + _nest_meta_under_config(column) return ConverterResult(output={"version": 2, "models": models}, issues=issues) def _pick_expression( diff --git a/converters/lightdash/tests/test_lightdash_to_ossie.py b/converters/lightdash/tests/test_lightdash_to_ossie.py index 140ef57b..28db1de7 100644 --- a/converters/lightdash/tests/test_lightdash_to_ossie.py +++ b/converters/lightdash/tests/test_lightdash_to_ossie.py @@ -532,3 +532,46 @@ def test_metric_datatypes_follow_the_aggregation(self): assert _metric(result.output, "total_amount").datatype is OssieDataType.DECIMAL assert _metric(result.output, "latest_amount").datatype is OssieDataType.DECIMAL assert _metric(result.output, "conversion_rate").datatype is None + + def test_config_meta_is_read_and_wins_over_meta(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": {"primary_key": "legacy_id"}, + "config": { + "meta": { + "primary_key": "order_id", + "joins": [ + { + "join": "customers", + "sql_on": "${orders.customer_id} = ${customers.customer_id}", + } + ], + } + }, + "columns": [ + { + "name": "amount", + "meta": {"dimension": {"type": "number", "label": "Amount"}}, + "config": { + "meta": { + "dimension": {"label": "Order amount"}, + "metrics": {"total_amount": {"type": "sum"}}, + } + }, + } + ], + }, + {"name": "customers", "columns": [{"name": "customer_id"}]}, + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + orders = result.output.semantic_model[0].datasets[0] + assert orders.primary_key == ["order_id"] + assert orders.fields[0].label == "Order amount" + assert orders.fields[0].datatype is OssieDataType.DECIMAL + assert _metric(result.output, "total_amount").expression.dialects[0].expression == ( + "SUM(orders.amount)" + ) + assert result.output.semantic_model[0].relationships[0].to == "customers" diff --git a/converters/lightdash/tests/test_ossie_to_lightdash.py b/converters/lightdash/tests/test_ossie_to_lightdash.py index 290000fc..ee1c3ed8 100644 --- a/converters/lightdash/tests/test_ossie_to_lightdash.py +++ b/converters/lightdash/tests/test_ossie_to_lightdash.py @@ -520,3 +520,12 @@ def test_time_axis_withdrawn_becomes_time_intervals_off(self): "type": "date", "time_intervals": "OFF", } + + def test_meta_can_be_placed_under_config(self): + result = OssieToLightdashConverter(meta_under_config=True).convert(_document()) + model = _model(result.output, "orders") + assert "meta" not in model + assert model["config"]["meta"]["joins"][0]["join"] == "customers" + column = _column(model, "order_date") + assert "meta" not in column + assert column["config"]["meta"]["dimension"]["type"] == "date" From 17580a9a2d009d419c4ddd732f34b66b4de44330 Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 11:28:08 +0100 Subject: [PATCH 14/33] feat(lightdash): name Ossie metrics by their Lightdash field id Lightdash scopes metric names per model, Ossie per semantic model, so any real project (count, total_events, unique_users on many models) produced an invalid document. The Ossie name is now Lightdash's own field id, _, which is stable regardless of what other models define and matches the id users see in the API and URLs. The bare name travels in the lightdash extension and is restored on export; an Ossie metric without a stash exports under its name minus a _ prefix. Names that still collide after qualification are suffixed with a METRIC_NAME_COLLISION issue. --- converters/lightdash/README.md | 7 +++ .../src/ossie_lightdash/converter_issues.py | 3 + .../src/ossie_lightdash/lightdash_to_ossie.py | 26 ++++++++- .../src/ossie_lightdash/ossie_to_lightdash.py | 17 +++++- .../tests/test_lightdash_to_ossie.py | 56 +++++++++++++++++-- .../tests/test_ossie_to_lightdash.py | 29 ++++++++++ .../lightdash/tests/test_tpcds_roundtrip.py | 15 ++++- 7 files changed, 141 insertions(+), 12 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 152d9db6..3614c742 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -50,6 +50,7 @@ ossie-lightdash import schema.yml semantic_model.json --database analytics_db -- | `field.dimension.is_time` | `time_intervals: OFF` ↔ an explicit `is_time: false` on a temporal column; otherwise not carried — a non-temporal time axis (e.g. a year stored as `Integer`) has no Lightdash equivalent | | `field.label` / `.description` | `meta.dimension.label` / column `description` | | `field.expression` (≠ column name) | `meta.dimension.sql` (`dataset.col` ↔ `${TABLE}.col`) | +| `metric.name` | Lightdash scopes metric names per model, Ossie per semantic model, so the Ossie name is Lightdash's own field id `_` (`orders_total_amount`); the bare name is stashed in the extension and restored on export. An Ossie metric with no stash exports under its name minus a `_` prefix, if it has one | | `metric.datatype` | derived on import: `Integer` for counts, `Decimal` for numeric aggregates over a `number` column, the column's type for `min`/`max`, the declared type for `boolean`/`string`/`date`/`timestamp` metrics | | `metric` that is one aggregation over a column (`SUM(ds.col)`, `COUNT(DISTINCT ds.col)`, `SUM(DISTINCT ds.col)`, `PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY ds.col)`, ...) | column-level `meta.metrics.` with a typed metric (`sum`, `count_distinct`, `sum_distinct`, `percentile` + `percentile: 90`, ...) | | `metric` that is one aggregation over any other expression (`AVG(CASE WHEN ds.status = 'done' THEN 1 ELSE 0 END)`) | model-level `meta.metrics.` with the typed metric and the operand as `sql` | @@ -98,6 +99,12 @@ Omitting `--schema` as well is reported as a `SOURCE_UNQUALIFIED` issue. are evaluated by Lightdash at query time and have no Ossie form: a dimension or metric whose SQL uses them is skipped on import with an `EXPRESSION_NOT_PORTABLE` issue. +- **Metric names are normalised on the first round trip**: an Ossie metric + named `total_sales` on `store_sales` comes back as `store_sales_total_sales` + after Lightdash → Ossie, and stays stable from then on. A name that still + collides after qualification (model `orders` + metric `x_total` vs model + `orders_x` + metric `total`) is suffixed with a `METRIC_NAME_COLLISION` + issue. - **Metric-to-metric references** (`${other_metric}`) are inlined on import (`METRIC_REFERENCE_INLINED`), since Ossie metrics cannot reference each other; the export direction does not reconstruct the reference. diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py index 813e496f..ddb16389 100644 --- a/converters/lightdash/src/ossie_lightdash/converter_issues.py +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -64,6 +64,9 @@ class ConverterIssueType(Enum): # home (column meta exists only for dimensions) and is dropped rather than # turning the field into a dimension. FIELD_ATTRIBUTE_NOT_REPRESENTABLE = "FIELD_ATTRIBUTE_NOT_REPRESENTABLE" + # Import: two metrics still share a name after qualification with their + # model name; the later one is suffixed. + METRIC_NAME_COLLISION = "METRIC_NAME_COLLISION" @dataclass(frozen=True) diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py index f2f524b5..06d1fc81 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py @@ -60,7 +60,7 @@ # Keys that are structurally encoded in Ossie vocabulary and therefore must NOT # be duplicated into the extension (a stale copy would win on export). -_STRUCTURAL_METRIC_KEYS = {"sql", "description", "ai_hint"} +_STRUCTURAL_METRIC_KEYS = {"sql", "description", "ai_hint", "name"} _STRUCTURAL_DIMENSION_KEYS = {"label", "sql", "ai_hint"} _STRUCTURAL_JOIN_KEYS = {"join", "sql_on"} @@ -246,6 +246,7 @@ def convert( metrics: List[OssieMetric] = [] relationships: List[OssieRelationship] = [] relationship_names: Set[str] = set() + metric_names: Set[str] = set() for model in schema_yml.get("models") or []: dataset, model_metrics, model_relationships = self._convert_model( @@ -254,6 +255,7 @@ def convert( schema=schema, issues=issues, relationship_names=relationship_names, + metric_names=metric_names, ) datasets.append(dataset) metrics.extend(model_metrics) @@ -280,6 +282,7 @@ def _convert_model( schema: Optional[str], issues: List[ConverterIssue], relationship_names: Set[str], + metric_names: Set[str], ) -> Tuple[OssieDataset, List[OssieMetric], List[OssieRelationship]]: name = model["name"] source = ".".join(part for part in [database, schema, name] if part) @@ -323,7 +326,9 @@ def _convert_model( metrics: List[OssieMetric] = [] for metric_name, (definition, column_name) in definitions.items(): - metric = self._convert_metric(metric_name, definition, column_name, context) + metric = self._convert_metric( + metric_name, definition, column_name, context, metric_names + ) if metric is not None: metrics.append(metric) @@ -399,6 +404,7 @@ def _convert_metric( definition: Dict[str, Any], column: Optional[str], context: _ModelContext, + metric_names: Set[str], ) -> Optional[OssieMetric]: if column is None and not definition.get("sql"): context.issues.append( @@ -424,8 +430,22 @@ def _convert_metric( extension_data = { key: value for key, value in definition.items() if key not in excluded } + # Lightdash scopes metric names per model; Ossie scopes them per + # semantic model. The Ossie name is Lightdash's own field id, + # `_`, and the bare name travels in the extension so + # export restores it exactly. + extension_data["name"] = metric_name + qualified = f"{context.dataset_name}_{metric_name}" + ossie_name = _unique_name(qualified, metric_names) + if ossie_name != qualified: + context.issues.append( + ConverterIssue( + issue_type=ConverterIssueType.METRIC_NAME_COLLISION, + element_name=metric_name, + ) + ) return OssieMetric( - name=metric_name, + name=ossie_name, expression=_expression(expression, self._dialect), datatype=metric_datatype( lightdash_type, diff --git a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py index 0aeb85fa..8f28f09a 100644 --- a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py +++ b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py @@ -62,7 +62,7 @@ # metrics whose expression is not a recognised aggregation: it is the channel # for types an expression cannot express (``boolean``, ``string``, ...). _PROTECTED_DIMENSION_KEYS = {"sql", "label", "ai_hint"} -_PROTECTED_METRIC_KEYS = {"sql", "description", "ai_hint"} +_PROTECTED_METRIC_KEYS = {"sql", "description", "ai_hint", "name"} _PROTECTED_AGGREGATION_KEYS = _PROTECTED_METRIC_KEYS | {"type", "percentile"} _PROTECTED_JOIN_KEYS = {"join", "sql_on", "alias"} @@ -444,14 +444,25 @@ def _convert_metric( } ) + # The Lightdash name is the stashed one, else the Ossie name with the + # `_` field-id prefix removed, else the Ossie name as is. + lightdash_name = extension_data.get("name") + if not isinstance(lightdash_name, str) or not lightdash_name: + prefix = f"{target_dataset}_" + lightdash_name = ( + metric.name[len(prefix):] + if metric.name.startswith(prefix) and len(metric.name) > len(prefix) + else metric.name + ) + if target_column is not None: column = columns_by_dataset[target_dataset][target_column] metrics = column.setdefault("meta", {}).setdefault("metrics", {}) - metrics[metric.name] = definition + metrics[lightdash_name] = definition else: model = models_by_dataset[target_dataset] metrics = model.setdefault("meta", {}).setdefault("metrics", {}) - metrics[metric.name] = definition + metrics[lightdash_name] = definition @staticmethod def _resolve_target_dataset( diff --git a/converters/lightdash/tests/test_lightdash_to_ossie.py b/converters/lightdash/tests/test_lightdash_to_ossie.py index 28db1de7..d2225942 100644 --- a/converters/lightdash/tests/test_lightdash_to_ossie.py +++ b/converters/lightdash/tests/test_lightdash_to_ossie.py @@ -113,17 +113,28 @@ } -def _metric(document, name): - return next(m for m in document.semantic_model[0].metrics if m.name == name) - - -def _lightdash_data(element): +def _raw_lightdash_data(element): for extension in element.custom_extensions or []: if extension.vendor_name == "lightdash": return json.loads(extension.data) return {} +def _metric(document, name): + """Look a metric up by its Lightdash name (stashed in the extension).""" + return next( + m + for m in document.semantic_model[0].metrics + if _raw_lightdash_data(m).get("name") == name + ) + + +def _lightdash_data(element): + data = _raw_lightdash_data(element) + data.pop("name", None) + return data + + class TestLightdashToOssie: def test_dataset_source_is_qualified(self): result = LightdashToOssieConverter().convert( @@ -575,3 +586,38 @@ def test_config_meta_is_read_and_wins_over_meta(self): "SUM(orders.amount)" ) assert result.output.semantic_model[0].relationships[0].to == "customers" + + def test_metric_names_are_qualified_with_the_model(self): + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") + metric = _metric(result.output, "total_amount") + assert metric.name == "orders_total_amount" + assert _raw_lightdash_data(metric)["name"] == "total_amount" + assert [m.name for m in result.output.semantic_model[0].metrics][:3] == [ + "orders_total_amount", + "orders_latest_amount", + "orders_median_amount", + ] + + def test_qualified_names_that_still_collide_are_suffixed(self): + schema_yml = { + "models": [ + { + "name": "orders", + "columns": [{"name": "amount", "meta": {"metrics": {"x_total": {"type": "sum"}}}}], + }, + { + "name": "orders_x", + "columns": [{"name": "amount", "meta": {"metrics": {"total": {"type": "sum"}}}}], + }, + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + assert [m.name for m in result.output.semantic_model[0].metrics] == [ + "orders_x_total", + "orders_x_total_2", + ] + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.METRIC_NAME_COLLISION + ] == ["total"] diff --git a/converters/lightdash/tests/test_ossie_to_lightdash.py b/converters/lightdash/tests/test_ossie_to_lightdash.py index ee1c3ed8..b44b1c8c 100644 --- a/converters/lightdash/tests/test_ossie_to_lightdash.py +++ b/converters/lightdash/tests/test_ossie_to_lightdash.py @@ -529,3 +529,32 @@ def test_meta_can_be_placed_under_config(self): column = _column(model, "order_date") assert "meta" not in column assert column["config"]["meta"]["dimension"]["type"] == "date" + + def test_lightdash_metric_name_comes_from_the_stash_then_the_prefix(self): + document = _document() + tampered = document.model_copy(deep=True) + tampered.semantic_model[0] = tampered.semantic_model[0].model_copy( + update={ + "metrics": [ + OssieMetric( + name="orders_total_amount", + expression=_ansi("SUM(orders.amount)"), + custom_extensions=[ + OssieCustomExtension( + vendor_name="lightdash", + data=json.dumps({"name": "total_amount", "label": "Total"}), + ) + ], + ), + OssieMetric(name="orders_max_amount", expression=_ansi("MAX(orders.amount)")), + OssieMetric(name="min_amount", expression=_ansi("MIN(orders.amount)")), + ] + } + ) + result = OssieToLightdashConverter().convert(tampered) + metrics = _column(_model(result.output, "orders"), "amount")["meta"]["metrics"] + assert metrics == { + "total_amount": {"type": "sum", "label": "Total"}, + "max_amount": {"type": "max"}, + "min_amount": {"type": "min"}, + } diff --git a/converters/lightdash/tests/test_tpcds_roundtrip.py b/converters/lightdash/tests/test_tpcds_roundtrip.py index dc7e663a..9d9c796b 100644 --- a/converters/lightdash/tests/test_tpcds_roundtrip.py +++ b/converters/lightdash/tests/test_tpcds_roundtrip.py @@ -24,6 +24,7 @@ them with a CROSS_DATASET_METRIC_DROPPED issue. """ +import json from pathlib import Path import yaml @@ -39,6 +40,13 @@ TPCDS_PATH = Path(__file__).parent / ".." / ".." / ".." / "examples" / "tpcds_semantic_model.yaml" +def _lightdash_name(metric) -> str: + for extension in metric.custom_extensions or []: + if extension.vendor_name == "lightdash": + return json.loads(extension.data)["name"] + return metric.name + + def _load_tpcds() -> OssieDocument: return OssieDocument.model_validate(yaml.safe_load(TPCDS_PATH.read_text())) @@ -137,11 +145,16 @@ def test_single_dataset_metrics_survive_with_expressions(self): for metric in original.semantic_model[0].metrics or [] if metric.name not in dropped } + # Re-imported metrics carry Lightdash's `_` id as their + # Ossie name; the original name is the stashed Lightdash name. roundtripped_metrics = { - metric.name: metric.expression.dialects[0].expression + _lightdash_name(metric): metric.expression.dialects[0].expression for metric in reimported.output.semantic_model[0].metrics or [] } assert set(roundtripped_metrics) == set(original_metrics) + assert "store_sales_total_sales" in { + metric.name for metric in reimported.output.semantic_model[0].metrics + } for name, expression in original_metrics.items(): assert roundtripped_metrics[name].replace(" ", "") == expression.replace( " ", "" From 65e34b0cdb58c2f301926161952e4fb9cfbee5da Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 11:29:23 +0100 Subject: [PATCH 15/33] fix(lightdash): import seeds as datasets and skip joins to unknown targets Lightdash treats dbt seeds as models, so a schema file's seeds: entries become datasets too. A join whose target is not in the input would leave a dangling relationship (the validator rejects it); it is skipped with a JOIN_TARGET_UNKNOWN issue instead. --- converters/lightdash/README.md | 7 +++++ .../src/ossie_lightdash/converter_issues.py | 3 ++ .../src/ossie_lightdash/lightdash_to_ossie.py | 18 +++++++++++- .../tests/test_lightdash_to_ossie.py | 29 ++++++++++++++++++- 4 files changed, 55 insertions(+), 2 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 3614c742..c80454c0 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -68,6 +68,13 @@ labels the emitted expressions with that warehouse's Ossie dialect prefers that dialect, falls back to `ANSI_SQL`, and takes the first available dialect with a `DIALECT_UNAVAILABLE` issue when an expression offers neither. +## Input shape + +`import` reads the `models:` and `seeds:` entries of a dbt schema file (seeds +are tables to Lightdash too). A join whose target is not among them is skipped +with a `JOIN_TARGET_UNKNOWN` issue, since Ossie relationships may only +reference datasets in the document. + ## Where the meta lives dbt 1.10+ places `meta` under `config:`; Lightdash reads both the top-level diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py index ddb16389..cba2864b 100644 --- a/converters/lightdash/src/ossie_lightdash/converter_issues.py +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -67,6 +67,9 @@ class ConverterIssueType(Enum): # Import: two metrics still share a name after qualification with their # model name; the later one is suffixed. METRIC_NAME_COLLISION = "METRIC_NAME_COLLISION" + # Import: a join targets a model that is not in the input, so the + # relationship would reference an unknown dataset and is skipped. + JOIN_TARGET_UNKNOWN = "JOIN_TARGET_UNKNOWN" @dataclass(frozen=True) diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py index 06d1fc81..51a7dee0 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py @@ -248,7 +248,9 @@ def convert( relationship_names: Set[str] = set() metric_names: Set[str] = set() - for model in schema_yml.get("models") or []: + # Seeds are tables to Lightdash just like models. + nodes = [*(schema_yml.get("models") or []), *(schema_yml.get("seeds") or [])] + for model in nodes: dataset, model_metrics, model_relationships = self._convert_model( model, database=database, @@ -261,6 +263,20 @@ def convert( metrics.extend(model_metrics) relationships.extend(model_relationships) + dataset_names = {dataset.name for dataset in datasets} + known_relationships: List[OssieRelationship] = [] + for relationship in relationships: + if relationship.to in dataset_names: + known_relationships.append(relationship) + else: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.JOIN_TARGET_UNKNOWN, + element_name=f"{relationship.from_dataset} -> {relationship.to}", + ) + ) + relationships = known_relationships + document = OssieDocument( version="0.2.0.dev0", semantic_model=[ diff --git a/converters/lightdash/tests/test_lightdash_to_ossie.py b/converters/lightdash/tests/test_lightdash_to_ossie.py index d2225942..4a3cce87 100644 --- a/converters/lightdash/tests/test_lightdash_to_ossie.py +++ b/converters/lightdash/tests/test_lightdash_to_ossie.py @@ -497,7 +497,8 @@ def test_aliased_joins_become_relationships(self): "meta": {"dimension": {"type": "number", "sql": "${sold_date.year}"}}, } ], - } + }, + {"name": "date_dim", "columns": [{"name": "date_id"}, {"name": "year"}]}, ] } result = LightdashToOssieConverter().convert(schema_yml, schema="marts") @@ -621,3 +622,29 @@ def test_qualified_names_that_still_collide_are_suffixed(self): for issue in result.issues if issue.issue_type is ConverterIssueType.METRIC_NAME_COLLISION ] == ["total"] + + def test_seeds_are_datasets_and_joins_to_missing_models_are_skipped(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": { + "joins": [ + {"join": "order_statuses", "sql_on": "${orders.status_id} = ${order_statuses.id}"}, + {"join": "not_in_this_file", "sql_on": "${orders.x} = ${not_in_this_file.x}"}, + ] + }, + "columns": [{"name": "status_id"}], + } + ], + "seeds": [{"name": "order_statuses", "columns": [{"name": "id"}]}], + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + semantic_model = result.output.semantic_model[0] + assert [dataset.name for dataset in semantic_model.datasets] == ["orders", "order_statuses"] + assert [relationship.to for relationship in semantic_model.relationships] == ["order_statuses"] + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.JOIN_TARGET_UNKNOWN + ] == ["orders -> not_in_this_file"] From 0e382dcfaa1930203affa405b1eeb22d95911cfe Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 11:31:47 +0100 Subject: [PATCH 16/33] fix(lightdash): qualify bare column names and remember a metric's model Lightdash SQL may name a column of the current model without ${TABLE} (SUM(budget_use)); the Ossie expression then carried no dataset qualifier and export could not place the metric in a multi-dataset document (67 of 390 jaffle-shop metrics were dropped). Bare identifiers that name one of the model's columns are now qualified, outside string literals and never when called as a function, and the hosting model is stashed in the extension so an expression that names no dataset at all is still placed. --- converters/lightdash/README.md | 1 + .../src/ossie_lightdash/expression_utils.py | 29 ++++++++++++++ .../src/ossie_lightdash/lightdash_to_ossie.py | 9 ++++- .../src/ossie_lightdash/ossie_to_lightdash.py | 8 +++- .../tests/test_lightdash_to_ossie.py | 40 +++++++++++++++++++ .../tests/test_ossie_to_lightdash.py | 30 ++++++++++++++ 6 files changed, 115 insertions(+), 2 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index c80454c0..7e7606d4 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -56,6 +56,7 @@ ossie-lightdash import schema.yml semantic_model.json --database analytics_db -- | `metric` that is one aggregation over any other expression (`AVG(CASE WHEN ds.status = 'done' THEN 1 ELSE 0 END)`) | model-level `meta.metrics.` with the typed metric and the operand as `sql` | | `metric` with any other expression | model-level `meta.metrics.` with `type: number` + `sql`, on the dataset that joins every other dataset the expression references (`${joined_model.col}`) | | `relationship` | `meta.joins` (`sql_on` built from / parsed into column pairs, `relationship: many-to-one` unless a stashed one says otherwise); the `alias` and other join attributes (`relationship`, `type`, `fields`, ...) travel in the relationship's `lightdash` extension, and a dataset joined more than once from the same model is aliased on export | +| bare column names in Lightdash SQL (`SUM(budget_use)`) | qualified with the dataset when they name one of the model's columns (outside string literals, not when called as a function); the hosting model is also stashed as `model` so an expression that names no dataset can still be placed on export | | `${TABLE}.col`, `${col}`, `${other_model.col}` in Lightdash SQL | `dataset.col` / `other_model.col`; `${alias.col}` is flattened onto the aliased model, `${metric}` is replaced by that metric's expression | | Lightdash presentation attributes (`label`, `format`, `round`, `compact`, `group_label`, `hidden`, ...) | `custom_extensions` with `vendor_name: "lightdash"`; on export the extension data is overlaid onto the generated definition (structural keys — `sql`/`label` on dimensions, `sql`/`description` on metrics and `type`/`percentile` on metrics whose expression is a recognised aggregation, `join`/`sql_on` on joins — are protected and cannot be overridden) | diff --git a/converters/lightdash/src/ossie_lightdash/expression_utils.py b/converters/lightdash/src/ossie_lightdash/expression_utils.py index a11e23fc..09692757 100644 --- a/converters/lightdash/src/ossie_lightdash/expression_utils.py +++ b/converters/lightdash/src/ossie_lightdash/expression_utils.py @@ -254,6 +254,35 @@ def replace(match: "re.Match[str]") -> str: return result +_STRING_LITERAL_RE = re.compile(r"('(?:[^']|'')*')") + + +def qualify_bare_columns(sql: str, dataset: str, columns) -> str: + """Prefix bare references to the model's own columns with the dataset name. + + Lightdash lets SQL name a column of the current model without ``${TABLE}`` + (``SUM(budget_use)``); Ossie consumers need ``dataset.column``. Only + identifiers that name one of ``columns`` are touched, never inside string + literals, never when already qualified, never when followed by ``(``. + """ + lookup = {column.lower(): column for column in columns} + if not lookup: + return sql + pattern = re.compile( + r"(? str: + return f"{dataset}.{match.group(1)}" + + parts = _STRING_LITERAL_RE.split(sql) + return "".join( + part if index % 2 else pattern.sub(replace, part) + for index, part in enumerate(parts) + ) + + def referenced_datasets(expression: str, dataset_names: set) -> set: """Return which of the given dataset names an Ossie expression references.""" found = set() diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py index 51a7dee0..d19d93ab 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py @@ -54,13 +54,14 @@ build_aggregation, has_non_portable_reference, lightdash_sql_to_ossie, + qualify_bare_columns, ) LIGHTDASH_VENDOR_NAME = "lightdash" # Keys that are structurally encoded in Ossie vocabulary and therefore must NOT # be duplicated into the extension (a stale copy would win on export). -_STRUCTURAL_METRIC_KEYS = {"sql", "description", "ai_hint", "name"} +_STRUCTURAL_METRIC_KEYS = {"sql", "description", "ai_hint", "name", "model"} _STRUCTURAL_DIMENSION_KEYS = {"label", "sql", "ai_hint"} _STRUCTURAL_JOIN_KEYS = {"join", "sql_on"} @@ -150,6 +151,7 @@ def __init__( self.definitions = definitions self.issues = issues self.column_types: Dict[str, str] = {} + self.column_names: List[str] = [] self._expressions: Dict[str, Optional[str]] = {} self._resolving: Set[str] = set() @@ -170,6 +172,9 @@ def rewrite(self, sql: str, element_name: str) -> Optional[str]: aliases=self.aliases, resolve_metric=self.metric_expression, ) + result.expression = qualify_bare_columns( + result.expression, self.dataset_name, self.column_names + ) for _ in result.inlined_metrics: self.issues.append( ConverterIssue( @@ -329,6 +334,7 @@ def _convert_model( definitions[metric_name] = (definition, None) context = _ModelContext(name, aliases, definitions, issues) + context.column_names = [column["name"] for column in model.get("columns") or []] for column in model.get("columns") or []: dimension_meta = lightdash_meta(column).get("dimension") or {} if dimension_meta.get("type"): @@ -451,6 +457,7 @@ def _convert_metric( # `_`, and the bare name travels in the extension so # export restores it exactly. extension_data["name"] = metric_name + extension_data["model"] = context.dataset_name qualified = f"{context.dataset_name}_{metric_name}" ossie_name = _unique_name(qualified, metric_names) if ossie_name != qualified: diff --git a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py index 8f28f09a..710d2b4b 100644 --- a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py +++ b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py @@ -62,7 +62,7 @@ # metrics whose expression is not a recognised aggregation: it is the channel # for types an expression cannot express (``boolean``, ``string``, ...). _PROTECTED_DIMENSION_KEYS = {"sql", "label", "ai_hint"} -_PROTECTED_METRIC_KEYS = {"sql", "description", "ai_hint", "name"} +_PROTECTED_METRIC_KEYS = {"sql", "description", "ai_hint", "name", "model"} _PROTECTED_AGGREGATION_KEYS = _PROTECTED_METRIC_KEYS | {"type", "percentile"} _PROTECTED_JOIN_KEYS = {"join", "sql_on", "alias"} @@ -395,6 +395,12 @@ def _convert_metric( target_dataset = self._resolve_target_dataset( expression, dataset_names, references_by_dataset ) + # An expression that names no dataset at all can still be placed when + # the stash says which model it came from. + stashed_model = extension_data.get("model") + if target_dataset is None and stashed_model in dataset_names: + if not referenced_datasets(expression, set(dataset_names)): + target_dataset = stashed_model if target_dataset is None: issues.append( ConverterIssue( diff --git a/converters/lightdash/tests/test_lightdash_to_ossie.py b/converters/lightdash/tests/test_lightdash_to_ossie.py index 4a3cce87..104b9a4a 100644 --- a/converters/lightdash/tests/test_lightdash_to_ossie.py +++ b/converters/lightdash/tests/test_lightdash_to_ossie.py @@ -132,6 +132,7 @@ def _metric(document, name): def _lightdash_data(element): data = _raw_lightdash_data(element) data.pop("name", None) + data.pop("model", None) return data @@ -593,6 +594,7 @@ def test_metric_names_are_qualified_with_the_model(self): metric = _metric(result.output, "total_amount") assert metric.name == "orders_total_amount" assert _raw_lightdash_data(metric)["name"] == "total_amount" + assert _raw_lightdash_data(metric)["model"] == "orders" assert [m.name for m in result.output.semantic_model[0].metrics][:3] == [ "orders_total_amount", "orders_latest_amount", @@ -648,3 +650,41 @@ def test_seeds_are_datasets_and_joins_to_missing_models_are_skipped(self): for issue in result.issues if issue.issue_type is ConverterIssueType.JOIN_TARGET_UNKNOWN ] == ["orders -> not_in_this_file"] + + def test_bare_column_names_in_sql_are_qualified(self): + schema_yml = { + "models": [ + { + "name": "budgets", + "meta": { + "metrics": { + "use_percentage": { + "type": "number", + "sql": "SUM(budget_use) / NULLIF(SUM(budget_total), 0)", + }, + "mobile_share": { + "type": "number", + "sql": "SUM(CASE WHEN device_type = 'device_type' THEN 1 END) / COUNT(*)", + }, + } + }, + "columns": [ + {"name": "budget_use"}, + {"name": "budget_total"}, + {"name": "device_type"}, + {"name": "count", "meta": {"dimension": {"sql": "COUNT(budget_use) OVER ()"}}}, + ], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + assert _metric(result.output, "use_percentage").expression.dialects[0].expression == ( + "SUM(budgets.budget_use) / NULLIF(SUM(budgets.budget_total), 0)" + ) + # The literal is untouched; the column reference is qualified. + assert _metric(result.output, "mobile_share").expression.dialects[0].expression == ( + "SUM(CASE WHEN budgets.device_type = 'device_type' THEN 1 END) / COUNT(*)" + ) + # A column named like a function is not qualified when called. + fields = {f.name: f.expression.dialects[0].expression for f in result.output.semantic_model[0].datasets[0].fields} + assert fields["count"] == "COUNT(budgets.budget_use) OVER ()" diff --git a/converters/lightdash/tests/test_ossie_to_lightdash.py b/converters/lightdash/tests/test_ossie_to_lightdash.py index b44b1c8c..61ef6fc9 100644 --- a/converters/lightdash/tests/test_ossie_to_lightdash.py +++ b/converters/lightdash/tests/test_ossie_to_lightdash.py @@ -558,3 +558,33 @@ def test_lightdash_metric_name_comes_from_the_stash_then_the_prefix(self): "max_amount": {"type": "max"}, "min_amount": {"type": "min"}, } + + def test_unqualified_metric_is_placed_on_the_stashed_model(self): + document = _document() + tampered = document.model_copy(deep=True) + tampered.semantic_model[0] = tampered.semantic_model[0].model_copy( + update={ + "metrics": [ + OssieMetric( + name="customers_row_count", + expression=_ansi("COUNT(*)"), + custom_extensions=[ + OssieCustomExtension( + vendor_name="lightdash", + data=json.dumps({"name": "row_count", "model": "customers"}), + ) + ], + ), + OssieMetric(name="unplaceable", expression=_ansi("COUNT(*)")), + ] + } + ) + result = OssieToLightdashConverter().convert(tampered) + assert _model(result.output, "customers")["meta"]["metrics"] == { + "row_count": {"type": "number", "sql": "COUNT(*)"} + } + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.CROSS_DATASET_METRIC_DROPPED + ] == ["unplaceable"] From d3dafc927f39953547a8ade71d682296271fc6f8 Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 11:39:36 +0100 Subject: [PATCH 17/33] feat(lightdash): stash what Ossie cannot express on the dataset and field Lightdash explores compose joins per base model and carry model meta with no Ossie vocabulary; both were dropped silently. Now: - a join through another joined model (${projects.org_id} = ${organizations.org_id} on the queries explore) derives the edge between the two models it names, unless that model declares it itself, and the join is stashed verbatim on the dataset's lightdash extension; expression joins and joins with extra conditions are stashed the same way - export restores stashed joins, replacing the generated join to the same target and alias, and lets field and metric expressions resolve through them - model meta without Ossie vocabulary (label, hidden, sql_filter, group_details, default_time_dimension, required_filters, ...) and column meta outside dimension/metrics (additional_dimensions, ...) travel in the dataset's and field's extension and come back as they were - aliased joins on the same columns stay distinct relationships; pair and side order in sql_on no longer force a stash On the analytics project every one of 140 joins and all 60 models' meta now round-trip; jaffle-shop and demo-f1 likewise. --- converters/lightdash/README.md | 24 ++- .../src/ossie_lightdash/converter_issues.py | 7 +- .../src/ossie_lightdash/lightdash_to_ossie.py | 204 +++++++++++++----- .../src/ossie_lightdash/ossie_to_lightdash.py | 51 ++++- .../tests/test_lightdash_to_ossie.py | 118 ++++++++++ .../tests/test_ossie_to_lightdash.py | 79 +++++++ 6 files changed, 421 insertions(+), 62 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 7e7606d4..15e9289a 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -42,7 +42,7 @@ ossie-lightdash import schema.yml semantic_model.json --database analytics_db -- | ----- | -------------------- | | `dataset` | dbt model (`name` = table part of `source`) | | `dataset.source` | assembled on import from `--database` / `--schema` / model name | -| `dataset.primary_key` | model `meta.primary_key` (a single key as a string, a composite key as a list) | +| `dataset.primary_key` | model `meta.primary_key` (a single key exports as a string, a composite key as a list) | | `ai_context` on datasets, fields and metrics | `ai_hint` on models, dimensions and metrics; a multi-line instruction is a list of hints, and the synonyms / examples of the structured form are rendered as extra hints on export | | `field` (no `dimension`) | plain column entry | | `field` with `dimension` | `columns[].meta.dimension` (an empty `dimension: {}` marks a dimension with no extra attributes) | @@ -56,6 +56,8 @@ ossie-lightdash import schema.yml semantic_model.json --database analytics_db -- | `metric` that is one aggregation over any other expression (`AVG(CASE WHEN ds.status = 'done' THEN 1 ELSE 0 END)`) | model-level `meta.metrics.` with the typed metric and the operand as `sql` | | `metric` with any other expression | model-level `meta.metrics.` with `type: number` + `sql`, on the dataset that joins every other dataset the expression references (`${joined_model.col}`) | | `relationship` | `meta.joins` (`sql_on` built from / parsed into column pairs, `relationship: many-to-one` unless a stashed one says otherwise); the `alias` and other join attributes (`relationship`, `type`, `fields`, ...) travel in the relationship's `lightdash` extension, and a dataset joined more than once from the same model is aliased on export | +| a join Ossie cannot reproduce — chained through another joined model (`${projects.org_id} = ${organizations.org_id}` on the `queries` explore), an expression join (`LOWER(a) = b`), extra conditions | the column pairs still become relationships (a chained pair derives the edge between the two models it names, unless that model declares it itself); the join is stashed verbatim on the dataset's `lightdash` extension and restored on export, replacing the generated join to the same target and alias. Pair order and side order (`${T.y} = ${M.x}`) are not semantic and need no stash | +| model meta without Ossie vocabulary (`label`, `hidden`, `sql_filter`, `group_details`, `default_time_dimension`, `required_filters`, `order_fields_by`, ...) and column meta outside `dimension` / `metrics` (`additional_dimensions`, ...) | stashed on the dataset's / field's `lightdash` extension (the latter under `column_meta`) and restored on export; invisible to other consumers | | bare column names in Lightdash SQL (`SUM(budget_use)`) | qualified with the dataset when they name one of the model's columns (outside string literals, not when called as a function); the hosting model is also stashed as `model` so an expression that names no dataset can still be placed on export | | `${TABLE}.col`, `${col}`, `${other_model.col}` in Lightdash SQL | `dataset.col` / `other_model.col`; `${alias.col}` is flattened onto the aliased model, `${metric}` is replaced by that metric's expression | | Lightdash presentation attributes (`label`, `format`, `round`, `compact`, `group_label`, `hidden`, ...) | `custom_extensions` with `vendor_name: "lightdash"`; on export the extension data is overlaid onto the generated definition (structural keys — `sql`/`label` on dimensions, `sql`/`description` on metrics and `type`/`percentile` on metrics whose expression is a recognised aggregation, `join`/`sql_on` on joins — are protected and cannot be overridden) | @@ -92,6 +94,18 @@ keeps one document valid across environments (see [dbt-core#15649](https://github.com/dbt-labs/dbt-core/issues/15649)). Omitting `--schema` as well is reported as a `SOURCE_UNQUALIFIED` issue. +## Issues + +Every loss or approximation is reported as a `ConverterIssue`: on import +`JOIN_STASHED` / `JOIN_SQL_UNPARSED` (a join kept for Lightdash only), +`JOIN_TARGET_UNKNOWN`, `EXPRESSION_NOT_PORTABLE`, `METRIC_REFERENCE_INLINED`, +`ALIAS_REFERENCE_FLATTENED`, `METRIC_NAME_COLLISION`, `SOURCE_UNQUALIFIED`, +`METRIC_SQL_MISSING`; on export `CROSS_DATASET_METRIC_DROPPED`, +`FIELD_REFERENCE_UNJOINED`, `FIELD_ATTRIBUTE_NOT_REPRESENTABLE`, +`TIME_ROLE_NOT_REPRESENTABLE`, `DIALECT_UNAVAILABLE`, +`RELATIONSHIP_COLUMNS_MISMATCHED`, `EXTENSION_DATA_INVALID`, +`FOREIGN_EXTENSION_IGNORED`. + ## Known limitations - **A metric spanning datasets none of which joins all the others is @@ -139,9 +153,11 @@ Omitting `--schema` as well is reported as a `SOURCE_UNQUALIFIED` issue. carries types on dimensions only, so there is nowhere to put it. - **A non-temporal time axis** (`is_time: true` on an `Integer` year) is reported with a `TIME_ROLE_NOT_REPRESENTABLE` issue on export. -- **Model-level Lightdash meta beyond `metrics` and `joins`** (`label`, - `group_details`, `sql_filter`, `order_fields_by`, column - `additional_dimensions`, ...) is not carried yet. +- **Stashed meta is Lightdash-only.** Model meta without Ossie vocabulary and + joins Ossie cannot reproduce round-trip exactly through the dataset's + extension, but other consumers do not see them; `sql_filter` in particular + restricts every Lightdash query while the Ossie dataset does not (a query + `source` would carry it, not done yet). - **Standalone Lightdash YAML projects** (Lightdash without dbt) are not supported yet; the converter targets the dbt-meta flavour. - Custom extensions from other vendors are ignored on export (reported as diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py index cba2864b..d7da9a70 100644 --- a/converters/lightdash/src/ossie_lightdash/converter_issues.py +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -24,8 +24,13 @@ class ConverterIssueType(Enum): # Import: the dataset source could not be qualified with a schema/database. SOURCE_UNQUALIFIED = "SOURCE_UNQUALIFIED" - # Import: a join's sql_on could not be parsed into column pairs. + # Import: a join's sql_on yields no column pairs (expression join); it is + # stashed on the dataset for Lightdash but has no relationship. JOIN_SQL_UNPARSED = "JOIN_SQL_UNPARSED" + # Import: a join Ossie relationships cannot reproduce exactly (chained + # through another joined model, extra conditions) is stashed verbatim on + # the dataset; any column pairs it contains still become relationships. + JOIN_STASHED = "JOIN_STASHED" # Export: a metric references more than one dataset, which a Lightdash # model metric cannot express. CROSS_DATASET_METRIC_DROPPED = "CROSS_DATASET_METRIC_DROPPED" diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py index d19d93ab..e3e46a4e 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py @@ -64,12 +64,36 @@ _STRUCTURAL_METRIC_KEYS = {"sql", "description", "ai_hint", "name", "model"} _STRUCTURAL_DIMENSION_KEYS = {"label", "sql", "ai_hint"} _STRUCTURAL_JOIN_KEYS = {"join", "sql_on"} +# Model meta with Ossie vocabulary; everything else is stashed on the dataset. +_HANDLED_MODEL_KEYS = {"metrics", "joins", "primary_key", "ai_hint"} +_HANDLED_COLUMN_KEYS = {"dimension", "metrics"} _JOIN_PAIR_RE = re.compile( r"\$\{(\w+)\.(\w+)\}\s*=\s*\$\{(\w+)\.(\w+)\}", ) +class _Edge: + """A relationship derived from a join, before de-duplication.""" + + def __init__(self, from_model, to_model, from_columns, to_columns, reference, extras): + self.from_model = from_model + self.to_model = to_model + self.from_columns = from_columns + self.to_columns = to_columns + self.reference = reference + self.extras = extras + + @property + def columns_key(self): + return (self.from_model, self.to_model, tuple(self.from_columns), tuple(self.to_columns)) + + @property + def key(self): + # An aliased join is a distinct relationship even on the same columns. + return (*self.columns_key, self.reference) + + def _expression(expression: str, dialect: OssieDialect) -> OssieExpression: return OssieExpression( dialects=[OssieDialectExpression(dialect=dialect, expression=expression)] @@ -252,35 +276,59 @@ def convert( relationships: List[OssieRelationship] = [] relationship_names: Set[str] = set() metric_names: Set[str] = set() + direct_edges: List[_Edge] = [] + derived_edges: List[_Edge] = [] # Seeds are tables to Lightdash just like models. nodes = [*(schema_yml.get("models") or []), *(schema_yml.get("seeds") or [])] for model in nodes: - dataset, model_metrics, model_relationships = self._convert_model( + dataset, model_metrics, model_direct, model_derived = self._convert_model( model, database=database, schema=schema, issues=issues, - relationship_names=relationship_names, metric_names=metric_names, ) datasets.append(dataset) metrics.extend(model_metrics) - relationships.extend(model_relationships) + direct_edges.extend(model_direct) + derived_edges.extend(model_derived) + # Edges a model declares itself come first, so an edge that a chained + # join merely passes through does not shadow the declared one. dataset_names = {dataset.name for dataset in datasets} - known_relationships: List[OssieRelationship] = [] - for relationship in relationships: - if relationship.to in dataset_names: - known_relationships.append(relationship) - else: + seen_edges: Set[tuple] = set() + seen_columns: Set[tuple] = set() + for edge in [*direct_edges, *derived_edges]: + if edge.to_model not in dataset_names or edge.from_model not in dataset_names: issues.append( ConverterIssue( issue_type=ConverterIssueType.JOIN_TARGET_UNKNOWN, - element_name=f"{relationship.from_dataset} -> {relationship.to}", + element_name=f"{edge.from_model} -> {edge.to_model}", ) ) - relationships = known_relationships + continue + derived = edge in derived_edges + # A derived edge is redundant with any declared edge on the same + # columns; a declared edge is redundant only with an identical one. + if (derived and edge.columns_key in seen_columns) or edge.key in seen_edges: + continue + seen_edges.add(edge.key) + seen_columns.add(edge.columns_key) + relationships.append( + OssieRelationship.model_validate( + { + "name": _unique_name( + f"{edge.from_model}_to_{edge.reference}", relationship_names + ), + "from": edge.from_model, + "to": edge.to_model, + "from_columns": edge.from_columns, + "to_columns": edge.to_columns, + "custom_extensions": _lightdash_extension(edge.extras) or None, + } + ) + ) document = OssieDocument( version="0.2.0.dev0", @@ -302,9 +350,8 @@ def _convert_model( database: Optional[str], schema: Optional[str], issues: List[ConverterIssue], - relationship_names: Set[str], metric_names: Set[str], - ) -> Tuple[OssieDataset, List[OssieMetric], List[OssieRelationship]]: + ) -> Tuple[OssieDataset, List[OssieMetric], List[_Edge], List[_Edge]]: name = model["name"] source = ".".join(part for part in [database, schema, name] if part) if schema is None: @@ -354,13 +401,18 @@ def _convert_model( if metric is not None: metrics.append(metric) - relationships = self._convert_joins( - joins, - from_model=name, - issues=issues, - relationship_names=relationship_names, + direct_edges, derived_edges, stashed_joins = self._convert_joins( + joins, from_model=name, aliases=aliases, issues=issues ) + # Model meta without Ossie vocabulary, and joins Ossie cannot reproduce + # exactly, travel on the dataset so export restores the explore as is. + stash = { + key: value for key, value in model_meta.items() if key not in _HANDLED_MODEL_KEYS + } + if stashed_joins: + stash["joins"] = stashed_joins + dataset = OssieDataset( name=name, source=source, @@ -368,14 +420,16 @@ def _convert_model( primary_key=_primary_key(model_meta.get("primary_key")), ai_context=_ai_context(model_meta.get("ai_hint")), fields=fields or None, + custom_extensions=_lightdash_extension(stash) or None, ) - return dataset, metrics, relationships + return dataset, metrics, direct_edges, derived_edges def _convert_column( self, column: Dict[str, Any], context: _ModelContext ) -> Optional[OssieField]: column_name = column["name"] - dimension_meta = lightdash_meta(column).get("dimension") + column_meta = lightdash_meta(column) + dimension_meta = column_meta.get("dimension") expression = column_name dimension: Optional[OssieDimension] = None @@ -408,6 +462,11 @@ def _convert_column( for key, value in dimension_meta.items() if key not in excluded } + column_extras = { + key: value for key, value in column_meta.items() if key not in _HANDLED_COLUMN_KEYS + } + if column_extras: + extension_data["column_meta"] = column_extras return OssieField( name=column_name, @@ -484,51 +543,86 @@ def _convert_joins( joins: List[Dict[str, Any]], *, from_model: str, + aliases: Dict[str, str], issues: List[ConverterIssue], - relationship_names: Set[str], - ) -> List[OssieRelationship]: - relationships: List[OssieRelationship] = [] + ) -> Tuple[List[_Edge], List[_Edge], List[Dict[str, Any]]]: + """Relationships and stashed joins for one model's explore. + + A pair ``${M.x} = ${T.y}`` on model M's join to T is a direct edge + M -> T. A pair through another model already joined in the explore, + ``${A.x} = ${T.y}``, is a chained join: the edge A -> T is derived and + the join itself is stashed verbatim, since Ossie relationships cannot + say which explore includes it. A join whose ``sql_on`` the export + direction would not rebuild identically (extra conditions, expression + joins) is stashed the same way. + """ + direct: List[_Edge] = [] + derived: List[_Edge] = [] + stashed: List[Dict[str, Any]] = [] + joined = {from_model} for join in joins: to_model = join.get("join") - # An aliased join is referenced by its alias in `sql_on`. + if not to_model: + stashed.append(join) + continue reference = join.get("alias") or to_model + joined.add(reference) pairs = _JOIN_PAIR_RE.findall(join.get("sql_on") or "") - from_columns: List[str] = [] - to_columns: List[str] = [] + direct_columns: Tuple[List[str], List[str]] = ([], []) + chained: Dict[str, Tuple[List[str], List[str]]] = {} for left_table, left_column, right_table, right_column in pairs: - if left_table == from_model and right_table == reference: - from_columns.append(left_column) - to_columns.append(right_column) - elif left_table == reference and right_table == from_model: - from_columns.append(right_column) - to_columns.append(left_column) - if not to_model or not from_columns: + if left_table == reference: + other, other_column, target_column = right_table, right_column, left_column + elif right_table == reference: + other, other_column, target_column = left_table, left_column, right_column + else: + continue + if other == from_model: + direct_columns[0].append(other_column) + direct_columns[1].append(target_column) + elif other in joined: + columns = chained.setdefault(other, ([], [])) + columns[0].append(other_column) + columns[1].append(target_column) + extras = { + key: value for key, value in join.items() if key not in _STRUCTURAL_JOIN_KEYS + } + if direct_columns[0]: + direct.append( + _Edge(from_model, to_model, *direct_columns, reference, extras) + ) + for other, (other_columns, target_columns) in chained.items(): + derived.append( + _Edge(aliases.get(other, other), to_model, other_columns, target_columns, to_model, {}) + ) + # Pair order and side order are not semantic: `${T.y} = ${M.x}` + # rebuilds as `${M.x} = ${T.y}` without needing a stash. + rebuilt_pairs = { + (from_model, left, reference, right) for left, right in zip(*direct_columns) + } + original_pairs = { + (a, b, c, d) if a == from_model else (c, d, a, b) + for a, b, c, d in pairs + } + residue = _JOIN_PAIR_RE.sub("", join.get("sql_on") or "") + reproducible = ( + original_pairs == rebuilt_pairs + and not residue.replace("AND", "").replace("and", "").strip() + ) + if not direct_columns[0] and not chained: issues.append( ConverterIssue( issue_type=ConverterIssueType.JOIN_SQL_UNPARSED, - element_name=f"{from_model} -> {to_model or ''}", + element_name=f"{from_model} -> {to_model}", ) ) - continue - # The alias and any other join attributes (type, relationship, - # fields, ...) have no Ossie vocabulary and travel in the extension. - extras = { - key: value - for key, value in join.items() - if key not in _STRUCTURAL_JOIN_KEYS - } - relationships.append( - OssieRelationship.model_validate( - { - "name": _unique_name( - f"{from_model}_to_{reference}", relationship_names - ), - "from": from_model, - "to": to_model, - "from_columns": from_columns, - "to_columns": to_columns, - "custom_extensions": _lightdash_extension(extras) or None, - } + stashed.append(join) + elif chained or not reproducible: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.JOIN_STASHED, + element_name=f"{from_model} -> {to_model}", + ) ) - ) - return relationships + stashed.append(join) + return direct, derived, stashed diff --git a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py index 710d2b4b..3d745bd6 100644 --- a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py +++ b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py @@ -61,10 +61,12 @@ # from overriding the Ossie-derived definition. ``type`` stays overridable on # metrics whose expression is not a recognised aggregation: it is the channel # for types an expression cannot express (``boolean``, ``string``, ...). -_PROTECTED_DIMENSION_KEYS = {"sql", "label", "ai_hint"} +_PROTECTED_DIMENSION_KEYS = {"sql", "label", "ai_hint", "column_meta"} _PROTECTED_METRIC_KEYS = {"sql", "description", "ai_hint", "name", "model"} _PROTECTED_AGGREGATION_KEYS = _PROTECTED_METRIC_KEYS | {"type", "percentile"} _PROTECTED_JOIN_KEYS = {"join", "sql_on", "alias"} +# Dataset-level stash keys that map to Ossie vocabulary or are handled apart. +_PROTECTED_MODEL_KEYS = {"joins", "metrics", "primary_key", "ai_hint"} # Joins a dataset declares, keyed by the joined dataset: the name Lightdash SQL # uses to reference it (the joined model, or its alias). The first join to a @@ -194,10 +196,45 @@ def _convert_semantic_model( models_by_dataset: Dict[str, Dict[str, Any]] = {} columns_by_dataset: Dict[str, Dict[str, Dict[str, Any]]] = {} + stash_by_dataset: Dict[str, Dict[str, Any]] = {} + for dataset in datasets: + stash = _lightdash_extension_data(dataset, issues) + stash_by_dataset[dataset.name] = stash + # A stashed join (chained, expression, extra conditions) is + # restored verbatim, replacing the generated join to the same + # target and alias, and its target becomes referenceable. + joins = joins_by_dataset.setdefault(dataset.name, []) + references = references_by_dataset.setdefault(dataset.name, {}) + for stashed_join in stash.get("joins") or []: + if not isinstance(stashed_join, dict) or not stashed_join.get("join"): + continue + key = (stashed_join.get("join"), stashed_join.get("alias")) + for index, join in enumerate(joins): + if (join.get("join"), join.get("alias")) == key: + joins[index] = stashed_join + break + else: + joins.append(stashed_join) + references.setdefault( + stashed_join["join"], stashed_join.get("alias") or stashed_join["join"] + ) + if not joins: + del joins_by_dataset[dataset.name] + for dataset in datasets: model, columns = self._convert_dataset( dataset, dataset_names, references_by_dataset.get(dataset.name, {}), issues ) + meta = model.setdefault("meta", {}) + meta.update( + { + key: value + for key, value in stash_by_dataset[dataset.name].items() + if key not in _PROTECTED_MODEL_KEYS + } + ) + if not meta: + del model["meta"] models_by_dataset[dataset.name] = model columns_by_dataset[dataset.name] = columns @@ -351,10 +388,11 @@ def _convert_dataset( dimension["sql"] = ossie_sql_to_lightdash( expression, dataset.name, references ) + field_data = _lightdash_extension_data(field, issues) dimension.update( { key: value - for key, value in _lightdash_extension_data(field, issues).items() + for key, value in field_data.items() if key not in _PROTECTED_DIMENSION_KEYS } ) @@ -363,6 +401,15 @@ def _convert_dataset( # or the import direction could not reconstruct it. if dimension or field.dimension is not None: column["meta"] = {"dimension": dimension} + column_meta = field_data.get("column_meta") + if isinstance(column_meta, dict) and column_meta: + column.setdefault("meta", {}).update( + { + key: value + for key, value in column_meta.items() + if key not in ("dimension", "metrics") + } + ) columns_by_name[field.name] = column model: Dict[str, Any] = {"name": _model_name_for(dataset)} diff --git a/converters/lightdash/tests/test_lightdash_to_ossie.py b/converters/lightdash/tests/test_lightdash_to_ossie.py index 104b9a4a..e3048d32 100644 --- a/converters/lightdash/tests/test_lightdash_to_ossie.py +++ b/converters/lightdash/tests/test_lightdash_to_ossie.py @@ -688,3 +688,121 @@ def test_bare_column_names_in_sql_are_qualified(self): # A column named like a function is not qualified when called. fields = {f.name: f.expression.dialects[0].expression for f in result.output.semantic_model[0].datasets[0].fields} assert fields["count"] == "COUNT(budgets.budget_use) OVER ()" + + def test_chained_and_expression_joins_are_stashed_and_edges_derived(self): + schema_yml = { + "models": [ + { + "name": "queries", + "meta": { + "sql_filter": "${TABLE}.deleted = false", + "group_details": {"usage": {"label": "Usage"}}, + "joins": [ + {"join": "projects", "sql_on": "${queries.project_id} = ${projects.project_id}"}, + { + "join": "organizations", + "sql_on": "${projects.organization_id} = ${organizations.organization_id}", + }, + { + "join": "users", + "sql_on": "LOWER(${queries.user_email}) = ${users.email}", + }, + { + "join": "warehouses", + "sql_on": "${queries.warehouse_id} = ${warehouses.id} AND ${warehouses.active}", + }, + ], + }, + "columns": [{"name": "project_id"}], + }, + { + "name": "projects", + "meta": { + "joins": [ + { + "join": "organizations", + "sql_on": "${projects.organization_id} = ${organizations.organization_id}", + "relationship": "many-to-one", + } + ] + }, + "columns": [{"name": "project_id"}, {"name": "organization_id"}], + }, + {"name": "organizations", "columns": [{"name": "organization_id"}]}, + {"name": "users", "columns": [{"name": "email"}]}, + {"name": "warehouses", "columns": [{"name": "id"}, {"name": "active"}]}, + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + semantic_model = result.output.semantic_model[0] + edges = [(r.from_dataset, r.to, r.from_columns, r.to_columns) for r in semantic_model.relationships] + # The chained edge projects -> organizations is declared on projects + # too; the declared one wins and carries its extras. + assert edges == [ + ("queries", "projects", ["project_id"], ["project_id"]), + ("queries", "warehouses", ["warehouse_id"], ["id"]), + ("projects", "organizations", ["organization_id"], ["organization_id"]), + ] + assert _raw_lightdash_data(semantic_model.relationships[2]) == {"relationship": "many-to-one"} + stash = _raw_lightdash_data(semantic_model.datasets[0]) + assert stash["sql_filter"] == "${TABLE}.deleted = false" + assert stash["group_details"] == {"usage": {"label": "Usage"}} + assert [join["join"] for join in stash["joins"]] == ["organizations", "users", "warehouses"] + assert { + (issue.issue_type.value, issue.element_name) + for issue in result.issues + if issue.issue_type in (ConverterIssueType.JOIN_STASHED, ConverterIssueType.JOIN_SQL_UNPARSED) + } == { + ("JOIN_STASHED", "queries -> organizations"), + ("JOIN_SQL_UNPARSED", "queries -> users"), + ("JOIN_STASHED", "queries -> warehouses"), + } + + def test_chained_join_without_a_declared_edge_derives_one(self): + schema_yml = { + "models": [ + { + "name": "users", + "meta": { + "joins": [ + {"join": "roles", "sql_on": "${users.id} = ${roles.user_id}"}, + {"join": "projects", "sql_on": "${roles.project_id} = ${projects.id}"}, + ] + }, + "columns": [{"name": "id"}], + }, + {"name": "roles", "columns": [{"name": "user_id"}, {"name": "project_id"}]}, + {"name": "projects", "columns": [{"name": "id"}]}, + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + edges = [(r.from_dataset, r.to) for r in result.output.semantic_model[0].relationships] + assert edges == [("users", "roles"), ("roles", "projects")] + + def test_column_meta_outside_dimension_is_stashed(self): + schema_yml = { + "models": [ + { + "name": "orders", + "columns": [ + { + "name": "amount", + "meta": { + "dimension": {"type": "number"}, + "additional_dimensions": {"amount_bucket": {"type": "string", "sql": "CASE WHEN ${TABLE}.amount > 100 THEN 'big' ELSE 'small' END"}}, + }, + } + ], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + field = result.output.semantic_model[0].datasets[0].fields[0] + assert _lightdash_data(field) == { + "type": "number", + "column_meta": { + "additional_dimensions": { + "amount_bucket": {"type": "string", "sql": "CASE WHEN ${TABLE}.amount > 100 THEN 'big' ELSE 'small' END"} + } + }, + } diff --git a/converters/lightdash/tests/test_ossie_to_lightdash.py b/converters/lightdash/tests/test_ossie_to_lightdash.py index 61ef6fc9..16540f54 100644 --- a/converters/lightdash/tests/test_ossie_to_lightdash.py +++ b/converters/lightdash/tests/test_ossie_to_lightdash.py @@ -588,3 +588,82 @@ def test_unqualified_metric_is_placed_on_the_stashed_model(self): for issue in result.issues if issue.issue_type is ConverterIssueType.CROSS_DATASET_METRIC_DROPPED ] == ["unplaceable"] + + def test_stashed_joins_and_meta_restore_the_explore(self): + document = _document() + tampered = document.model_copy(deep=True) + datasets = tampered.semantic_model[0].datasets + datasets[0] = datasets[0].model_copy( + update={ + "custom_extensions": [ + OssieCustomExtension( + vendor_name="lightdash", + data=json.dumps( + { + "sql_filter": "${TABLE}.deleted = false", + "joins": [ + { + "join": "customers", + "sql_on": "${orders.customer_id} = ${customers.customer_id} AND ${customers.active}", + }, + {"join": "regions", "sql_on": "${customers.region_id} = ${regions.id}"}, + ], + } + ), + ) + ] + } + ) + datasets[1] = datasets[1].model_copy( + update={ + "fields": [ + datasets[1].fields[0].model_copy( + update={ + "custom_extensions": [ + OssieCustomExtension( + vendor_name="lightdash", + data=json.dumps({"column_meta": {"additional_dimensions": {"id_prefix": {"type": "string", "sql": "LEFT(${TABLE}.customer_id, 2)"}}}}), + ) + ] + } + ) + ] + } + ) + regions = OssieDataset( + name="regions", + source="analytics_db.marts.regions", + fields=[OssieField(name="id", expression=_ansi("id"))], + ) + tampered.semantic_model[0] = tampered.semantic_model[0].model_copy( + update={ + "datasets": [*datasets, regions], + "metrics": [ + OssieMetric( + name="regional_spread", + expression=_ansi("COUNT(orders.customer_id) / COUNT(DISTINCT regions.id)"), + ), + ], + } + ) + result = OssieToLightdashConverter().convert(tampered) + orders = _model(result.output, "orders") + # The stashed join replaces the generated one to the same target and + # the chained join is appended; the model meta comes back as is. + assert orders["meta"]["joins"] == [ + { + "join": "customers", + "sql_on": "${orders.customer_id} = ${customers.customer_id} AND ${customers.active}", + }, + {"join": "regions", "sql_on": "${customers.region_id} = ${regions.id}"}, + ] + assert orders["meta"]["sql_filter"] == "${TABLE}.deleted = false" + # A metric over the chained target resolves through the stashed join. + assert orders["meta"]["metrics"]["regional_spread"] == { + "type": "number", + "sql": "COUNT(${TABLE}.customer_id) / COUNT(DISTINCT ${regions.id})", + } + customer_id = _column(_model(result.output, "customers"), "customer_id") + assert customer_id["meta"]["additional_dimensions"] == { + "id_prefix": {"type": "string", "sql": "LEFT(${TABLE}.customer_id, 2)"} + } From d737c0291f27d9766cbce660547e30bdacc45f85 Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 11:59:07 +0100 Subject: [PATCH 18/33] feat(lightdash)!: every column is a dimension unless hidden Lightdash treats every dbt column as a dimension by default; only meta.dimension.hidden withdraws it from grouping. The converter read the presence of meta.dimension as the dimension marker instead, so a real project exported most of its columns as measure-only fields (5,481 of 7,435 on the Lightdash analytics project). Import now emits dimension: {} for every column that is not hidden, and no dimension for a hidden one. Export writes a measure-only Ossie field as a hidden dimension, keeping its type, label and AI hint, and writes nothing for a dimension with nothing else to say. FIELD_ATTRIBUTE_NOT_REPRESENTABLE is gone: labels and hints always have a home now. --- converters/lightdash/README.md | 12 ++--- .../src/ossie_lightdash/converter_issues.py | 4 -- .../src/ossie_lightdash/lightdash_to_ossie.py | 11 +++-- .../src/ossie_lightdash/ossie_to_lightdash.py | 47 +++++++------------ .../tests/test_lightdash_to_ossie.py | 23 +++++++++ .../tests/test_ossie_to_lightdash.py | 26 +++++----- 6 files changed, 64 insertions(+), 59 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 15e9289a..2aff3995 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -44,8 +44,8 @@ ossie-lightdash import schema.yml semantic_model.json --database analytics_db -- | `dataset.source` | assembled on import from `--database` / `--schema` / model name | | `dataset.primary_key` | model `meta.primary_key` (a single key exports as a string, a composite key as a list) | | `ai_context` on datasets, fields and metrics | `ai_hint` on models, dimensions and metrics; a multi-line instruction is a list of hints, and the synonyms / examples of the structured form are rendered as extra hints on export | -| `field` (no `dimension`) | plain column entry | -| `field` with `dimension` | `columns[].meta.dimension` (an empty `dimension: {}` marks a dimension with no extra attributes) | +| `field` with `dimension` | a column: every dbt column is a Lightdash dimension by default, so a dimension with nothing else to say is a plain column entry with no `meta` | +| `field` without `dimension` (measure-only) | `columns[].meta.dimension.hidden: true` — a hidden column is the closest Lightdash comes to a field that is not for grouping; it keeps its `type`, `label` and `ai_hint` | | `field.datatype` | `meta.dimension.type` (`String`→`string`, `Integer`/`Decimal`/`Float`→`number`, `Date`→`date`, `DateTime`/`DateTimeTz`→`timestamp`, `Boolean`→`boolean`, `Time`/`Opaque`→`string`); on import `number` maps back to `Decimal` | | `field.dimension.is_time` | `time_intervals: OFF` ↔ an explicit `is_time: false` on a temporal column; otherwise not carried — a non-temporal time axis (e.g. a year stored as `Integer`) has no Lightdash equivalent | | `field.label` / `.description` | `meta.dimension.label` / column `description` | @@ -101,8 +101,7 @@ Every loss or approximation is reported as a `ConverterIssue`: on import `JOIN_TARGET_UNKNOWN`, `EXPRESSION_NOT_PORTABLE`, `METRIC_REFERENCE_INLINED`, `ALIAS_REFERENCE_FLATTENED`, `METRIC_NAME_COLLISION`, `SOURCE_UNQUALIFIED`, `METRIC_SQL_MISSING`; on export `CROSS_DATASET_METRIC_DROPPED`, -`FIELD_REFERENCE_UNJOINED`, `FIELD_ATTRIBUTE_NOT_REPRESENTABLE`, -`TIME_ROLE_NOT_REPRESENTABLE`, `DIALECT_UNAVAILABLE`, +`FIELD_REFERENCE_UNJOINED`, `TIME_ROLE_NOT_REPRESENTABLE`, `DIALECT_UNAVAILABLE`, `RELATIONSHIP_COLUMNS_MISMATCHED`, `EXTENSION_DATA_INVALID`, `FOREIGN_EXTENSION_IGNORED`. @@ -136,9 +135,6 @@ Every loss or approximation is reported as a `ConverterIssue`: on import was meant is not preserved in the expression. - **`unique_keys` are not exported** — Lightdash has no corresponding concept — and consequently cannot be reconstructed on import. -- **A label or `ai_context` on a measure-only field is dropped on export** - (`FIELD_ATTRIBUTE_NOT_REPRESENTABLE`): Lightdash keeps both on dimensions - only, and writing them would turn the field into a dimension. - **`dataset.name` is not preserved when it differs from the source table name**: the dbt model is named after the table part of `source`, and the import direction derives dataset names from model names. References inside @@ -149,8 +145,6 @@ Every loss or approximation is reported as a `ConverterIssue`: on import - **Datatypes round-trip by category, not by exact type**: Lightdash types are coarser than Ossie datatypes, so `Integer` comes back as `Decimal` and `DateTimeTz` as `DateTime`. -- **A measure-only field (no `dimension`) loses its `datatype`**: Lightdash - carries types on dimensions only, so there is nowhere to put it. - **A non-temporal time axis** (`is_time: true` on an `Integer` year) is reported with a `TIME_ROLE_NOT_REPRESENTABLE` issue on export. - **Stashed meta is Lightdash-only.** Model meta without Ossie vocabulary and diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py index d7da9a70..bc305f55 100644 --- a/converters/lightdash/src/ossie_lightdash/converter_issues.py +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -65,10 +65,6 @@ class ConverterIssueType(Enum): # Export: a field expression references a dataset the field's dataset does # not join, so Lightdash cannot resolve the reference. FIELD_REFERENCE_UNJOINED = "FIELD_REFERENCE_UNJOINED" - # Export: a label or AI context on a measure-only field has no Lightdash - # home (column meta exists only for dimensions) and is dropped rather than - # turning the field into a dimension. - FIELD_ATTRIBUTE_NOT_REPRESENTABLE = "FIELD_ATTRIBUTE_NOT_REPRESENTABLE" # Import: two metrics still share a name after qualification with their # model name; the later one is suffixed. METRIC_NAME_COLLISION = "METRIC_NAME_COLLISION" diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py index e3e46a4e..367eb010 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py @@ -62,7 +62,7 @@ # Keys that are structurally encoded in Ossie vocabulary and therefore must NOT # be duplicated into the extension (a stale copy would win on export). _STRUCTURAL_METRIC_KEYS = {"sql", "description", "ai_hint", "name", "model"} -_STRUCTURAL_DIMENSION_KEYS = {"label", "sql", "ai_hint"} +_STRUCTURAL_DIMENSION_KEYS = {"label", "sql", "ai_hint", "hidden"} _STRUCTURAL_JOIN_KEYS = {"join", "sql_on"} # Model meta with Ossie vocabulary; everything else is stashed on the dataset. _HANDLED_MODEL_KEYS = {"metrics", "joins", "primary_key", "ai_hint"} @@ -432,11 +432,14 @@ def _convert_column( dimension_meta = column_meta.get("dimension") expression = column_name - dimension: Optional[OssieDimension] = None datatype = None label: Optional[str] = None ai_context: Optional[str] = None extension_data: Dict[str, Any] = {} + # Every dbt column is a Lightdash dimension unless it is hidden; a + # hidden column is the closest Lightdash comes to a measure-only field. + hidden = bool((dimension_meta or {}).get("hidden")) + dimension: Optional[OssieDimension] = None if hidden else OssieDimension() if dimension_meta is not None: label = dimension_meta.get("label") ai_context = _ai_context(dimension_meta.get("ai_hint")) @@ -452,11 +455,9 @@ def _convert_column( # the datatype decides. excluded = set(_STRUCTURAL_DIMENSION_KEYS) time_intervals = dimension_meta.get("time_intervals") - if time_intervals is False or time_intervals == "OFF": + if not hidden and (time_intervals is False or time_intervals == "OFF"): dimension = OssieDimension(is_time=False) excluded.add("time_intervals") - else: - dimension = OssieDimension() extension_data = { key: value for key, value in dimension_meta.items() diff --git a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py index 3d745bd6..8614002c 100644 --- a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py +++ b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py @@ -61,7 +61,7 @@ # from overriding the Ossie-derived definition. ``type`` stays overridable on # metrics whose expression is not a recognised aggregation: it is the channel # for types an expression cannot express (``boolean``, ``string``, ...). -_PROTECTED_DIMENSION_KEYS = {"sql", "label", "ai_hint", "column_meta"} +_PROTECTED_DIMENSION_KEYS = {"sql", "label", "ai_hint", "hidden", "column_meta"} _PROTECTED_METRIC_KEYS = {"sql", "description", "ai_hint", "name", "model"} _PROTECTED_AGGREGATION_KEYS = _PROTECTED_METRIC_KEYS | {"type", "percentile"} _PROTECTED_JOIN_KEYS = {"join", "sql_on", "alias"} @@ -325,31 +325,22 @@ def _convert_dataset( column["description"] = field.description dimension: Dict[str, Any] = {} + if field.dimension is None: + # Every Lightdash column is a dimension; a measure-only Ossie + # field is a hidden one. + dimension["hidden"] = True + if field.label: + dimension["label"] = field.label ai_hint = _ai_hint(field.ai_context) - if field.dimension is None and (field.label or ai_hint is not None): - # Lightdash keeps labels and AI hints on dimensions only; - # writing them would turn a measure-only field into one. - issues.append( - ConverterIssue( - issue_type=ConverterIssueType.FIELD_ATTRIBUTE_NOT_REPRESENTABLE, - element_name=field.name, - ) - ) - elif field.dimension is not None: - if field.label: - dimension["label"] = field.label - if ai_hint is not None: - dimension["ai_hint"] = ai_hint - if field.dimension is not None: - # Only dimension fields carry a Lightdash type: emitting one for - # a measure-only field would turn it into a dimension on import. - lightdash_type = datatype_to_lightdash_type(field.datatype) - if lightdash_type is not None: - dimension["type"] = lightdash_type - elif field.dimension.is_time: - # No datatype to translate, but the field is declared as a - # time axis: `date` is the closest Lightdash type. - dimension["type"] = "date" + if ai_hint is not None: + dimension["ai_hint"] = ai_hint + lightdash_type = datatype_to_lightdash_type(field.datatype) + if lightdash_type is not None: + dimension["type"] = lightdash_type + elif field.dimension is not None and field.dimension.is_time: + # No datatype to translate, but the field is declared as a + # time axis: `date` is the closest Lightdash type. + dimension["type"] = "date" if ( field.dimension is not None and field.dimension.is_time @@ -396,10 +387,8 @@ def _convert_dataset( if key not in _PROTECTED_DIMENSION_KEYS } ) - # An empty dict still marks dimension-ness: a field Ossie declares as a - # categorical dimension must not degrade to a plain column on export, - # or the import direction could not reconstruct it. - if dimension or field.dimension is not None: + # A dimension with nothing to say is Lightdash's default: no meta. + if dimension: column["meta"] = {"dimension": dimension} column_meta = field_data.get("column_meta") if isinstance(column_meta, dict) and column_meta: diff --git a/converters/lightdash/tests/test_lightdash_to_ossie.py b/converters/lightdash/tests/test_lightdash_to_ossie.py index e3048d32..c9bb00b7 100644 --- a/converters/lightdash/tests/test_lightdash_to_ossie.py +++ b/converters/lightdash/tests/test_lightdash_to_ossie.py @@ -806,3 +806,26 @@ def test_column_meta_outside_dimension_is_stashed(self): } }, } + + def test_every_column_is_a_dimension_unless_hidden(self): + schema_yml = { + "models": [ + { + "name": "orders", + "columns": [ + {"name": "plain"}, + {"name": "shown", "meta": {"dimension": {"hidden": False, "type": "string"}}}, + {"name": "hidden_key", "meta": {"dimension": {"hidden": True, "type": "number", "label": "Key"}}}, + ], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + fields = {f.name: f for f in result.output.semantic_model[0].datasets[0].fields} + assert fields["plain"].dimension is not None + assert fields["shown"].dimension is not None + assert _lightdash_data(fields["shown"]) == {"type": "string"} + assert fields["hidden_key"].dimension is None + assert fields["hidden_key"].datatype is OssieDataType.DECIMAL + assert fields["hidden_key"].label == "Key" + assert _lightdash_data(fields["hidden_key"]) == {"type": "number"} diff --git a/converters/lightdash/tests/test_ossie_to_lightdash.py b/converters/lightdash/tests/test_ossie_to_lightdash.py index 16540f54..663dd6b2 100644 --- a/converters/lightdash/tests/test_ossie_to_lightdash.py +++ b/converters/lightdash/tests/test_ossie_to_lightdash.py @@ -156,15 +156,16 @@ def test_time_dimension_exports_date_type(self): column = _column(_model(result.output, "orders"), "order_date") assert column["meta"]["dimension"] == {"label": "Order date", "type": "date"} - def test_categorical_dimension_keeps_dimension_marker(self): + def test_dimension_with_nothing_to_say_needs_no_meta(self): + # Every Lightdash column is a dimension by default. result = OssieToLightdashConverter().convert(_document()) column = _column(_model(result.output, "orders"), "status") - assert column["meta"]["dimension"] == {} + assert "meta" not in column - def test_plain_field_has_no_dimension_meta(self): + def test_measure_only_field_becomes_a_hidden_dimension(self): result = OssieToLightdashConverter().convert(_document()) column = _column(_model(result.output, "orders"), "amount") - assert "dimension" not in column.get("meta", {}) + assert column["meta"]["dimension"] == {"hidden": True} def test_simple_aggregation_becomes_column_metric(self): result = OssieToLightdashConverter().convert(_document()) @@ -489,22 +490,23 @@ def test_field_and_metric_ai_context_become_ai_hints(self): tampered = document.model_copy(deep=True) orders = tampered.semantic_model[0].datasets[0] orders.fields[1] = orders.fields[1].model_copy(update={"ai_context": "Order lifecycle stage."}) - # A measure-only field cannot carry the hint without becoming a dimension. - orders.fields[2] = orders.fields[2].model_copy(update={"ai_context": "Gross amount."}) + orders.fields[2] = orders.fields[2].model_copy( + update={"ai_context": "Gross amount.", "datatype": OssieDataType.DECIMAL} + ) metrics = tampered.semantic_model[0].metrics metrics[0] = metrics[0].model_copy(update={"ai_context": "Revenue before refunds."}) result = OssieToLightdashConverter().convert(tampered) model = _model(result.output, "orders") assert _column(model, "status")["meta"]["dimension"]["ai_hint"] == "Order lifecycle stage." - assert "dimension" not in _column(model, "amount")["meta"] + # A hidden dimension keeps its hint and its type. + assert _column(model, "amount")["meta"]["dimension"] == { + "hidden": True, + "ai_hint": "Gross amount.", + "type": "number", + } assert _column(model, "amount")["meta"]["metrics"]["total_amount"]["ai_hint"] == ( "Revenue before refunds." ) - assert [ - issue.element_name - for issue in result.issues - if issue.issue_type is ConverterIssueType.FIELD_ATTRIBUTE_NOT_REPRESENTABLE - ] == ["amount"] def test_time_axis_withdrawn_becomes_time_intervals_off(self): document = _document() From c159ccc3645e25b1aadccec4b30c16814c8583fe Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 11:59:43 +0100 Subject: [PATCH 19/33] feat(lightdash): report filters that other consumers cannot see A metric's filters and a model's sql_filter / sql_where / required_filters travel in the lightdash extension and round-trip, but every other consumer of the document sees the unfiltered aggregate or the unrestricted dataset. Both are now reported on import (METRIC_FILTER_NOT_PORTABLE, ROW_FILTER_NOT_PORTABLE) instead of passing silently. --- converters/lightdash/README.md | 13 +++++-- .../src/ossie_lightdash/converter_issues.py | 6 +++ .../src/ossie_lightdash/lightdash_to_ossie.py | 16 ++++++++ .../tests/test_lightdash_to_ossie.py | 37 +++++++++++++++++++ 4 files changed, 68 insertions(+), 4 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 2aff3995..11b958be 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -98,7 +98,8 @@ Omitting `--schema` as well is reported as a `SOURCE_UNQUALIFIED` issue. Every loss or approximation is reported as a `ConverterIssue`: on import `JOIN_STASHED` / `JOIN_SQL_UNPARSED` (a join kept for Lightdash only), -`JOIN_TARGET_UNKNOWN`, `EXPRESSION_NOT_PORTABLE`, `METRIC_REFERENCE_INLINED`, +`JOIN_TARGET_UNKNOWN`, `METRIC_FILTER_NOT_PORTABLE`, `ROW_FILTER_NOT_PORTABLE`, +`EXPRESSION_NOT_PORTABLE`, `METRIC_REFERENCE_INLINED`, `ALIAS_REFERENCE_FLATTENED`, `METRIC_NAME_COLLISION`, `SOURCE_UNQUALIFIED`, `METRIC_SQL_MISSING`; on export `CROSS_DATASET_METRIC_DROPPED`, `FIELD_REFERENCE_UNJOINED`, `TIME_ROLE_NOT_REPRESENTABLE`, `DIALECT_UNAVAILABLE`, @@ -149,9 +150,13 @@ Every loss or approximation is reported as a `ConverterIssue`: on import reported with a `TIME_ROLE_NOT_REPRESENTABLE` issue on export. - **Stashed meta is Lightdash-only.** Model meta without Ossie vocabulary and joins Ossie cannot reproduce round-trip exactly through the dataset's - extension, but other consumers do not see them; `sql_filter` in particular - restricts every Lightdash query while the Ossie dataset does not (a query - `source` would carry it, not done yet). + extension, but other consumers do not see them. Two of these change query + results rather than presentation and are therefore reported: a metric's + `filters` (`METRIC_FILTER_NOT_PORTABLE`: the Ossie expression is the + unfiltered aggregate) and a model's `sql_filter` / `required_filters` + (`ROW_FILTER_NOT_PORTABLE`: the Ossie dataset is unrestricted). Encoding + metric filters as `CASE WHEN` and `sql_filter` as a query `source` is the + planned fix. - **Standalone Lightdash YAML projects** (Lightdash without dbt) are not supported yet; the converter targets the dbt-meta flavour. - Custom extensions from other vendors are ignored on export (reported as diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py index bc305f55..c06e054b 100644 --- a/converters/lightdash/src/ossie_lightdash/converter_issues.py +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -68,6 +68,12 @@ class ConverterIssueType(Enum): # Import: two metrics still share a name after qualification with their # model name; the later one is suffixed. METRIC_NAME_COLLISION = "METRIC_NAME_COLLISION" + # Import: a metric's `filters` are kept for Lightdash only; the Ossie + # expression is unfiltered, so other consumers compute a different number. + METRIC_FILTER_NOT_PORTABLE = "METRIC_FILTER_NOT_PORTABLE" + # Import: a model's `sql_filter` / `sql_where` / `required_filters` are kept + # for Lightdash only; the Ossie dataset is unrestricted for other consumers. + ROW_FILTER_NOT_PORTABLE = "ROW_FILTER_NOT_PORTABLE" # Import: a join targets a model that is not in the input, so the # relationship would reference an unknown dataset and is skipped. JOIN_TARGET_UNKNOWN = "JOIN_TARGET_UNKNOWN" diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py index 367eb010..afb5ba4b 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py @@ -67,6 +67,8 @@ # Model meta with Ossie vocabulary; everything else is stashed on the dataset. _HANDLED_MODEL_KEYS = {"metrics", "joins", "primary_key", "ai_hint"} _HANDLED_COLUMN_KEYS = {"dimension", "metrics"} +# Model meta that changes query results, not just presentation. +_ROW_FILTER_KEYS = ("sql_filter", "sql_where", "required_filters") _JOIN_PAIR_RE = re.compile( r"\$\{(\w+)\.(\w+)\}\s*=\s*\$\{(\w+)\.(\w+)\}", @@ -412,6 +414,13 @@ def _convert_model( } if stashed_joins: stash["joins"] = stashed_joins + if any(model_meta.get(key) for key in _ROW_FILTER_KEYS): + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.ROW_FILTER_NOT_PORTABLE, + element_name=name, + ) + ) dataset = OssieDataset( name=name, @@ -499,6 +508,13 @@ def _convert_metric( expression = context.metric_expression(metric_name) if expression is None: return None + if definition.get("filters"): + context.issues.append( + ConverterIssue( + issue_type=ConverterIssueType.METRIC_FILTER_NOT_PORTABLE, + element_name=metric_name, + ) + ) # Typed aggregations (and their percentile) are recovered from the # expression on export; only types an expression cannot encode diff --git a/converters/lightdash/tests/test_lightdash_to_ossie.py b/converters/lightdash/tests/test_lightdash_to_ossie.py index c9bb00b7..1849e4eb 100644 --- a/converters/lightdash/tests/test_lightdash_to_ossie.py +++ b/converters/lightdash/tests/test_lightdash_to_ossie.py @@ -829,3 +829,40 @@ def test_every_column_is_a_dimension_unless_hidden(self): assert fields["hidden_key"].datatype is OssieDataType.DECIMAL assert fields["hidden_key"].label == "Key" assert _lightdash_data(fields["hidden_key"]) == {"type": "number"} + + def test_filters_that_other_consumers_cannot_see_are_reported(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": {"sql_filter": "${TABLE}.deleted = false"}, + "columns": [ + { + "name": "amount", + "meta": { + "metrics": { + "paid_amount": {"type": "sum", "filters": [{"status": "paid"}]}, + "total_amount": {"type": "sum"}, + } + }, + } + ], + }, + {"name": "customers", "meta": {"required_filters": [{"region": "EU"}]}, "columns": []}, + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + # The values are stashed for Lightdash, and the loss for everyone else is reported. + paid = _metric(result.output, "paid_amount") + assert paid.expression.dialects[0].expression == "SUM(orders.amount)" + assert _lightdash_data(paid) == {"filters": [{"status": "paid"}]} + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.METRIC_FILTER_NOT_PORTABLE + ] == ["paid_amount"] + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.ROW_FILTER_NOT_PORTABLE + ] == ["orders", "customers"] From 70d78cd283578ce72be6a9a916b71985aa92ba46 Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 12:00:17 +0100 Subject: [PATCH 20/33] feat: register LIGHTDASH as a well-known vendor Add LIGHTDASH to the spec's vendor-name table, the schema examples, the Python OssieVendor enum and the converters README, and make the Lightdash converter emit the registered token. Documents written with the earlier lowercase name are still read. --- converters/README.md | 1 + converters/lightdash/README.md | 2 +- .../src/ossie_lightdash/lightdash_to_ossie.py | 4 +- .../src/ossie_lightdash/ossie_to_lightdash.py | 8 ++- .../tests/test_lightdash_to_ossie.py | 2 +- .../tests/test_ossie_to_lightdash.py | 64 ++++++++++++++++--- .../lightdash/tests/test_tpcds_roundtrip.py | 2 +- core-spec/ossie-schema.json | 2 +- core-spec/spec.md | 1 + core-spec/spec.yaml | 2 +- python/src/ossie/models.py | 1 + 11 files changed, 69 insertions(+), 20 deletions(-) diff --git a/converters/README.md b/converters/README.md index 417d7663..4151fea5 100644 --- a/converters/README.md +++ b/converters/README.md @@ -76,6 +76,7 @@ The Ossie specification currently defines extensions for the following vendors: | `OMNI` | Omni semantic model | | `WISDOM` | WisdomAI domain | | `NVIDIA_GSF` | NVIDIA Generative Semantic Fabric standalone YAML | +| `LIGHTDASH` | Lightdash semantic layer (dbt `meta`) | Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 11b958be..2bd844ac 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -60,7 +60,7 @@ ossie-lightdash import schema.yml semantic_model.json --database analytics_db -- | model meta without Ossie vocabulary (`label`, `hidden`, `sql_filter`, `group_details`, `default_time_dimension`, `required_filters`, `order_fields_by`, ...) and column meta outside `dimension` / `metrics` (`additional_dimensions`, ...) | stashed on the dataset's / field's `lightdash` extension (the latter under `column_meta`) and restored on export; invisible to other consumers | | bare column names in Lightdash SQL (`SUM(budget_use)`) | qualified with the dataset when they name one of the model's columns (outside string literals, not when called as a function); the hosting model is also stashed as `model` so an expression that names no dataset can still be placed on export | | `${TABLE}.col`, `${col}`, `${other_model.col}` in Lightdash SQL | `dataset.col` / `other_model.col`; `${alias.col}` is flattened onto the aliased model, `${metric}` is replaced by that metric's expression | -| Lightdash presentation attributes (`label`, `format`, `round`, `compact`, `group_label`, `hidden`, ...) | `custom_extensions` with `vendor_name: "lightdash"`; on export the extension data is overlaid onto the generated definition (structural keys — `sql`/`label` on dimensions, `sql`/`description` on metrics and `type`/`percentile` on metrics whose expression is a recognised aggregation, `join`/`sql_on` on joins — are protected and cannot be overridden) | +| Lightdash presentation attributes (`label`, `format`, `round`, `compact`, `group_label`, `hidden`, ...) | `custom_extensions` with `vendor_name: LIGHTDASH` (the registered vendor token; the lowercase name of earlier documents is still read); on export the extension data is overlaid onto the generated definition (structural keys — `sql`/`label` on dimensions, `sql`/`description` on metrics and `type`/`percentile` on metrics whose expression is a recognised aggregation, `join`/`sql_on` on joins — are protected and cannot be overridden) | ## Dialects diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py index afb5ba4b..2b236d70 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py @@ -21,7 +21,7 @@ first-class Ossie vocabulary (datasets, fields, metrics, relationships); Lightdash presentation attributes without Ossie vocabulary (``format``, ``round``, ``group_label``, ``hidden``, ...) are preserved in -``custom_extensions`` entries with ``vendor_name: "lightdash"`` so that the +``custom_extensions`` entries with ``vendor_name: LIGHTDASH`` so that the export direction can reproduce them exactly. """ @@ -57,7 +57,7 @@ qualify_bare_columns, ) -LIGHTDASH_VENDOR_NAME = "lightdash" +LIGHTDASH_VENDOR_NAME = "LIGHTDASH" # Keys that are structurally encoded in Ossie vocabulary and therefore must NOT # be duplicated into the extension (a stale copy would win on export). diff --git a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py index 8614002c..3d18023c 100644 --- a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py +++ b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py @@ -20,7 +20,7 @@ carry Lightdash dimensions, metrics and joins, ready to be merged into a dbt project that Lightdash reads. Lightdash-specific presentation attributes that have no Ossie vocabulary round-trip through ``custom_extensions`` entries with -``vendor_name: "lightdash"``; their keys are overlaid onto the generated +``vendor_name: LIGHTDASH``; their keys are overlaid onto the generated definitions and win for presentation attributes, while structural keys (``sql``/``label`` on dimensions, ``sql``/``description`` on metrics, ``join``/``sql_on`` on joins) are protected so they can never override the @@ -54,7 +54,7 @@ strip_qualifier, ) -LIGHTDASH_VENDOR_NAME = "lightdash" +LIGHTDASH_VENDOR_NAME = "LIGHTDASH" # Structural keys are owned by Ossie vocabulary (the import direction never puts # them into the extension); dropping them here keeps a hand-authored extension @@ -78,7 +78,9 @@ def _lightdash_extension_data(element: Any, issues: List[ConverterIssue]) -> Dic """Return the ``lightdash`` vendor extension data of an Ossie element, if any.""" data: Dict[str, Any] = {} for extension in element.custom_extensions or []: - if extension.vendor_name == LIGHTDASH_VENDOR_NAME: + # The registered token is LIGHTDASH; documents written before the + # registration used the lowercase name. + if extension.vendor_name.upper() == LIGHTDASH_VENDOR_NAME: try: data.update(json.loads(extension.data)) except (TypeError, ValueError): diff --git a/converters/lightdash/tests/test_lightdash_to_ossie.py b/converters/lightdash/tests/test_lightdash_to_ossie.py index 1849e4eb..06e294ff 100644 --- a/converters/lightdash/tests/test_lightdash_to_ossie.py +++ b/converters/lightdash/tests/test_lightdash_to_ossie.py @@ -115,7 +115,7 @@ def _raw_lightdash_data(element): for extension in element.custom_extensions or []: - if extension.vendor_name == "lightdash": + if extension.vendor_name.upper() == "LIGHTDASH": return json.loads(extension.data) return {} diff --git a/converters/lightdash/tests/test_ossie_to_lightdash.py b/converters/lightdash/tests/test_ossie_to_lightdash.py index 663dd6b2..f92cf304 100644 --- a/converters/lightdash/tests/test_ossie_to_lightdash.py +++ b/converters/lightdash/tests/test_ossie_to_lightdash.py @@ -33,7 +33,11 @@ OssieSemanticModel, ) -from ossie_lightdash import ConverterIssueType, OssieToLightdashConverter +from ossie_lightdash import ( + ConverterIssueType, + LightdashToOssieConverter, + OssieToLightdashConverter, +) def _ansi(expression: str) -> OssieExpression: @@ -77,7 +81,7 @@ def _document() -> OssieDocument: description="Sum of order amounts", custom_extensions=[ OssieCustomExtension( - vendor_name="lightdash", + vendor_name="LIGHTDASH", data=json.dumps({"label": "Total amount", "format": "usd"}), ) ], @@ -89,7 +93,7 @@ def _document() -> OssieDocument: ), custom_extensions=[ OssieCustomExtension( - vendor_name="lightdash", + vendor_name="LIGHTDASH", data=json.dumps({"format": "percent", "round": 1}), ) ], @@ -303,6 +307,46 @@ def test_foreign_extension_is_reported(self): for issue in result.issues ) + def test_extension_vendor_name_is_the_registered_token(self): + result = LightdashToOssieConverter().convert( + {"models": [{"name": "t", "columns": [{"name": "c", "meta": {"dimension": {"format": "usd"}}}]}]}, + schema="s", + ) + field = result.output.semantic_model[0].datasets[0].fields[0] + assert field.custom_extensions[0].vendor_name == "LIGHTDASH" + # Documents written before the registration used the lowercase name. + legacy = OssieToLightdashConverter().convert( + result.output.model_copy( + update={ + "semantic_model": [ + result.output.semantic_model[0].model_copy( + update={ + "datasets": [ + result.output.semantic_model[0].datasets[0].model_copy( + update={ + "fields": [ + field.model_copy( + update={ + "custom_extensions": [ + OssieCustomExtension(vendor_name="lightdash", data=field.custom_extensions[0].data) + ] + } + ) + ] + } + ) + ] + } + ) + ] + } + ) + ) + assert _column(_model(legacy.output, "t"), "c")["meta"]["dimension"]["format"] == "usd" + assert not any( + issue.issue_type is ConverterIssueType.FOREIGN_EXTENSION_IGNORED for issue in legacy.issues + ) + def test_extension_cannot_override_structural_keys(self): document = _document() tampered = document.model_copy(deep=True) @@ -310,7 +354,7 @@ def test_extension_cannot_override_structural_keys(self): update={ "custom_extensions": [ OssieCustomExtension( - vendor_name="lightdash", + vendor_name="LIGHTDASH", data=json.dumps( {"label": "Total amount", "sql": "1 + 1", "description": "stale"} ), @@ -353,7 +397,7 @@ def test_invalid_extension_json_is_reported(self): metric = tampered.semantic_model[0].metrics[0].model_copy( update={ "custom_extensions": [ - OssieCustomExtension(vendor_name="lightdash", data="{not json") + OssieCustomExtension(vendor_name="LIGHTDASH", data="{not json") ] } ) @@ -434,7 +478,7 @@ def test_join_alias_and_attributes_restore_from_extension(self): "to_columns": ["customer_id"], "custom_extensions": [ { - "vendor_name": "lightdash", + "vendor_name": "LIGHTDASH", "data": json.dumps( {"alias": "buyer", "relationship": "many-to-one", "sql_on": "1 = 1"} ), @@ -543,7 +587,7 @@ def test_lightdash_metric_name_comes_from_the_stash_then_the_prefix(self): expression=_ansi("SUM(orders.amount)"), custom_extensions=[ OssieCustomExtension( - vendor_name="lightdash", + vendor_name="LIGHTDASH", data=json.dumps({"name": "total_amount", "label": "Total"}), ) ], @@ -572,7 +616,7 @@ def test_unqualified_metric_is_placed_on_the_stashed_model(self): expression=_ansi("COUNT(*)"), custom_extensions=[ OssieCustomExtension( - vendor_name="lightdash", + vendor_name="LIGHTDASH", data=json.dumps({"name": "row_count", "model": "customers"}), ) ], @@ -599,7 +643,7 @@ def test_stashed_joins_and_meta_restore_the_explore(self): update={ "custom_extensions": [ OssieCustomExtension( - vendor_name="lightdash", + vendor_name="LIGHTDASH", data=json.dumps( { "sql_filter": "${TABLE}.deleted = false", @@ -623,7 +667,7 @@ def test_stashed_joins_and_meta_restore_the_explore(self): update={ "custom_extensions": [ OssieCustomExtension( - vendor_name="lightdash", + vendor_name="LIGHTDASH", data=json.dumps({"column_meta": {"additional_dimensions": {"id_prefix": {"type": "string", "sql": "LEFT(${TABLE}.customer_id, 2)"}}}}), ) ] diff --git a/converters/lightdash/tests/test_tpcds_roundtrip.py b/converters/lightdash/tests/test_tpcds_roundtrip.py index 9d9c796b..22695ca7 100644 --- a/converters/lightdash/tests/test_tpcds_roundtrip.py +++ b/converters/lightdash/tests/test_tpcds_roundtrip.py @@ -42,7 +42,7 @@ def _lightdash_name(metric) -> str: for extension in metric.custom_extensions or []: - if extension.vendor_name == "lightdash": + if extension.vendor_name.upper() == "LIGHTDASH": return json.loads(extension.data)["name"] return metric.name diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index c87af723..a1efdb43 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -28,7 +28,7 @@ }, "Vendor": { "type": "string", - "examples": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM"], + "examples": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM", "LIGHTDASH"], "description": "Vendor name for custom extensions. Any string value is accepted." }, "AIContext": { diff --git a/core-spec/spec.md b/core-spec/spec.md index 2850edfe..8b33452a 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -447,6 +447,7 @@ The following are well-known examples: | `GOODDATA` | GoodData-specific attributes | | `HONEYDEW` | Honeydew-specific attributes | | `WISDOM` | WisdomAI-specific attributes | +| `LIGHTDASH` | Lightdash-specific attributes | ### Examples diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 4678b004..93109ad6 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -55,7 +55,7 @@ datatypes: - "Opaque" # Known type outside the portable vocabulary # Vendor name for custom extensions (free-form string) -# Examples: "COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM" +# Examples: "COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM", "LIGHTDASH" vendor_name: string diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index e782275c..d3188bdb 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -71,6 +71,7 @@ class OssieVendor(str, Enum): GOODDATA = "GOODDATA" SEMANTIDO = "SEMANTIDO" WISDOM = "WISDOM" + LIGHTDASH = "LIGHTDASH" class OssieAIContextObject(BaseModel): From 356f9c66ec6b5576198499db085a49b2ed376315 Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 12:01:55 +0100 Subject: [PATCH 21/33] docs(lightdash): explain fidelity and unavoidable losses Restructure the README around usage, the Python API, the mapping, and a section on why each loss is unavoidable (name scope, what a dimension is, joins versus relationships, query-time evaluation, types), followed by what is kept for Lightdash only, approximated, and not carried. --- converters/lightdash/README.md | 197 +++++++++++++++++++++++---------- 1 file changed, 136 insertions(+), 61 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 2bd844ac..82b41fd1 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -28,14 +28,41 @@ translates between that shape and Ossie. - **Export** (`ossie_to_lightdash`): Ossie document → a dbt `schema.yml`-shaped dictionary with Lightdash `meta` blocks, ready to merge into a dbt project. - **Import** (`lightdash_to_ossie`): a Lightdash-flavoured `schema.yml` → an - Ossie document, as a migration path for teams with an existing installed - base of Lightdash metrics. + Ossie document, for teams adopting Ossie as the source of truth for + definitions they already maintain in Lightdash. + +Lightdash-only attributes travel in `custom_extensions` under the registered +vendor token `LIGHTDASH`, so Lightdash → Ossie → Lightdash restores the +project exactly while every other consumer works from the core vocabulary. + +## Usage ``` -ossie-lightdash export semantic_model.yaml schema.yml --dialect BIGQUERY +ossie-lightdash export semantic_model.yaml schema.yml --dialect BIGQUERY [--meta-under-config] ossie-lightdash import schema.yml semantic_model.json --database analytics_db --schema marts --dialect BIGQUERY ``` +Issues (anything lost or approximated, see below) are printed to stderr as +`[ISSUE_TYPE] element`. + +### Python API + +```python +from ossie import OssieDialect +from ossie_lightdash import LightdashToOssieConverter, OssieToLightdashConverter + +result = LightdashToOssieConverter(OssieDialect.BIGQUERY).convert( + schema_yml, database="analytics_db", schema="marts" +) +result.output # OssieDocument +result.issues # [ConverterIssue(issue_type, element_name), ...] + +exported = OssieToLightdashConverter(OssieDialect.BIGQUERY).convert(result.output) +exported.output # {"version": 2, "models": [...]} +``` + +Requires Python 3.11+ and the in-repo `apache-ossie` package (`../../python`). + ## Mapping | Ossie | Lightdash (dbt meta) | @@ -94,6 +121,100 @@ keeps one document valid across environments (see [dbt-core#15649](https://github.com/dbt-labs/dbt-core/issues/15649)). Omitting `--schema` as well is reported as a `SOURCE_UNQUALIFIED` issue. +## Fidelity and unavoidable losses + +The two models disagree in five places. Each disagreement is handled the same +way: carry what has vocabulary on both sides, keep the rest on the Lightdash +side of the document, and report whatever another consumer of the document +will not see. + +**Scope of metric names.** Lightdash scopes metric names per model, Ossie per +semantic model, and every real Lightdash project has a `count` on several +models. The Ossie name is therefore Lightdash's own field id, +`_`, always rather than only on collision: a metric's name must +not change because a metric was added to a different model, and the field id +is what Lightdash users already see in its API and URLs. The bare name is +stashed and restored, so Lightdash → Ossie → Lightdash is exact; an Ossie +document not written by Lightdash sees its metric names normalised once +(`total_sales` on `store_sales` becomes `store_sales_total_sales`). + +**What a dimension is.** Every dbt column is a Lightdash dimension unless it +is hidden; an Ossie field is a dimension only when it says so. `hidden: true` +and "no `dimension`" are the two ends of one mapping. The cost is that a +hidden column cannot carry Ossie's `is_time` role, and Lightdash has no way to +say "a dimension that is not for grouping but is not a measure either"; neither +side has such a thing today. + +**Joins versus relationships.** Ossie's relationships form one graph. A +Lightdash explore is a base model plus an explicit list of joins, which may +reach a table through another one, join on an expression, or add conditions. +The graph part becomes relationships (a chained join derives the edge between +the two models it names); the explore part is stashed verbatim on the dataset +and restored, so the explore comes back exactly while other consumers see the +graph. Because Lightdash never resolves `${other.column}` transitively, a +metric spanning datasets is hosted on the model that joins every referenced +dataset directly, and dropped with an issue when no such model exists. + +**Query-time evaluation.** Lightdash evaluates project parameters, user +attributes and Liquid templating when a query runs. No downstream consumer +can, so a dimension or metric whose SQL depends on them is skipped on import +and reported rather than shipped as SQL that is not SQL. Metric-to-metric +references are inlined for the same reason: Ossie metrics cannot reference +each other. + +**Types.** Lightdash's `number` covers Ossie's `Integer`, `Decimal` and +`Float`, and its `timestamp` covers `DateTime` and `DateTimeTz`, so datatypes +round-trip by category rather than by exact member. A column with no authored +`type` leaves without a datatype, which the spec allows; Lightdash learns +those types from the warehouse, not from the YAML. + +### Kept for Lightdash only + +Stashed in the `LIGHTDASH` extension and restored on export, invisible to +other consumers: presentation attributes of dimensions and metrics (`format`, +`round`, `compact`, `groups`, `urls`, `show_underlying_values`, ...); model +meta without Ossie vocabulary (`label`, `hidden`, `group_details`, +`default_time_dimension`, `order_fields_by`, ...); column meta outside +`dimension` / `metrics` (`additional_dimensions`); join attributes (`alias`, +`type`, `fields`, ...); and joins Ossie cannot reproduce. + +Two of these change query results rather than presentation and are therefore +reported: a metric's `filters` (`METRIC_FILTER_NOT_PORTABLE` — the Ossie +expression is the unfiltered aggregate) and a model's `sql_filter` / +`sql_where` / `required_filters` (`ROW_FILTER_NOT_PORTABLE` — the Ossie +dataset is unrestricted). Encoding metric filters as `CASE WHEN` and +`sql_filter` as a query `source` is the planned fix. + +### Approximated + +- A dataset joined more than once is referenced through its first join when + an expression names it (`date_dim.year` → `${date_dim.year}`, not the + aliased second join); a `${alias.column}` reference is flattened onto the + joined dataset with an `ALIAS_REFERENCE_FLATTENED` issue. +- A name that still collides after qualification (model `orders` + metric + `x_total` versus model `orders_x` + metric `total`) is suffixed with a + `METRIC_NAME_COLLISION` issue. +- `ai_context` synonyms and examples of the structured form are rendered as + extra `ai_hint` lines on export; on import `ai_hint` becomes a plain + instruction string. +- A single-column `primary_key` exports as a string, a composite one as a list. + +### Not carried + +- `unique_keys` — Lightdash has no corresponding concept. +- `ai_context` and custom extensions on relationships. +- `dataset.name` when it differs from the source table name: the dbt model is + named after the table part of `source`. +- Relationships with mismatched `from_columns` / `to_columns` lengths + (`RELATIONSHIP_COLUMNS_MISMATCHED`), and relationships to datasets missing + from the document. +- Custom extensions of other vendors (`FOREIGN_EXTENSION_IGNORED`); they + remain untouched in the Ossie document. +- Standalone Lightdash YAML projects (Lightdash without dbt) — the converter + targets the dbt-meta flavour. +- Documents are emitted at the in-repo spec version; dbt-core 1.12's native + Ossie parsing accepts `0.1.0` / `0.1.1` only. + ## Issues Every loss or approximation is reported as a `ConverterIssue`: on import @@ -106,61 +227,15 @@ Every loss or approximation is reported as a `ConverterIssue`: on import `RELATIONSHIP_COLUMNS_MISMATCHED`, `EXTENSION_DATA_INVALID`, `FOREIGN_EXTENSION_IGNORED`. -## Known limitations - -- **A metric spanning datasets none of which joins all the others is - dropped on export** (`CROSS_DATASET_METRIC_DROPPED`): Lightdash resolves - `${other.column}` only through the joins the hosting model declares, never - transitively. A field expression referencing an unjoined dataset is emitted - as-is with a `FIELD_REFERENCE_UNJOINED` issue. -- **A dataset joined more than once** is referenced through its first join - when an expression names it (`date_dim.year` → `${date_dim.year}` rather - than the aliased second join). -- **Parameter and user-attribute references** (`${lightdash.parameters.x}`, - `${ld.user.email}`) and **Liquid templating** (`{% if ld.query.filters … %}`) - are evaluated by Lightdash at query time and have no Ossie form: a dimension - or metric whose SQL uses them is skipped on import with an - `EXPRESSION_NOT_PORTABLE` issue. -- **Metric names are normalised on the first round trip**: an Ossie metric - named `total_sales` on `store_sales` comes back as `store_sales_total_sales` - after Lightdash → Ossie, and stays stable from then on. A name that still - collides after qualification (model `orders` + metric `x_total` vs model - `orders_x` + metric `total`) is suffixed with a `METRIC_NAME_COLLISION` - issue. -- **Metric-to-metric references** (`${other_metric}`) are inlined on import - (`METRIC_REFERENCE_INLINED`), since Ossie metrics cannot reference each - other; the export direction does not reconstruct the reference. -- **References through a join alias** (`${sold_date.year}`) are rewritten to - the joined dataset (`date_dim.year`) with an `ALIAS_REFERENCE_FLATTENED` - issue: Ossie has no aliases, so which of several joins to the same dataset - was meant is not preserved in the expression. -- **`unique_keys` are not exported** — Lightdash has no corresponding - concept — and consequently cannot be reconstructed on import. -- **`dataset.name` is not preserved when it differs from the source table - name**: the dbt model is named after the table part of `source`, and the - import direction derives dataset names from model names. References inside - expressions and relationships are rewritten consistently, but a - name-stable round-trip is not guaranteed. -- **Relationships with mismatched `from_columns` / `to_columns` lengths are - skipped on export** with a `RELATIONSHIP_COLUMNS_MISMATCHED` issue. -- **Datatypes round-trip by category, not by exact type**: Lightdash types are - coarser than Ossie datatypes, so `Integer` comes back as `Decimal` and - `DateTimeTz` as `DateTime`. -- **A non-temporal time axis** (`is_time: true` on an `Integer` year) is - reported with a `TIME_ROLE_NOT_REPRESENTABLE` issue on export. -- **Stashed meta is Lightdash-only.** Model meta without Ossie vocabulary and - joins Ossie cannot reproduce round-trip exactly through the dataset's - extension, but other consumers do not see them. Two of these change query - results rather than presentation and are therefore reported: a metric's - `filters` (`METRIC_FILTER_NOT_PORTABLE`: the Ossie expression is the - unfiltered aggregate) and a model's `sql_filter` / `required_filters` - (`ROW_FILTER_NOT_PORTABLE`: the Ossie dataset is unrestricted). Encoding - metric filters as `CASE WHEN` and `sql_filter` as a query `source` is the - planned fix. -- **Standalone Lightdash YAML projects** (Lightdash without dbt) are not - supported yet; the converter targets the dbt-meta flavour. -- Custom extensions from other vendors are ignored on export (reported as - `FOREIGN_EXTENSION_IGNORED`); they remain untouched in the Ossie document. -- Documents are emitted at the current in-repo spec version. Note that - dbt-core 1.12's native OSI parsing accepts spec versions `0.1.0` / `0.1.1` - only. +## Development + +``` +uv sync +uv run pytest +``` + +Beyond the unit tests and the TPC-DS round trip, the converter is exercised +against real Lightdash projects (the public jaffle-shop demo and two +production projects on BigQuery): every model's meta and every join survive +Lightdash → Ossie → Lightdash, every uniquely named metric returns with its +expression unchanged, and the documents pass `validation/validate.py`. From bcfb095cbbe59aa2f7d776be420d617aa8e92513 Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 14:03:57 +0100 Subject: [PATCH 22/33] feat(lightdash): export Lightdash's dbt-free model files by default An Ossie document now becomes a deployable Lightdash project without a dbt project around it: export writes lightdash/models/.yml per dataset (type: model, sql_from = the dataset's source, typed dimensions with their own sql, metrics and joins in place) plus a starter lightdash.config.yml whose warehouse type follows --dialect or --warehouse. Model meta the dbt flavour has to stash becomes ordinary top-level keys. --format dbt-meta keeps the single schema.yml output for dbt projects. lightdash compile accepts the exported TPC-DS project: 5 explores, 0 errors. --- converters/lightdash/README.md | 58 +++++++++--- .../lightdash/src/ossie_lightdash/cli.py | 89 ++++++++++++++++-- .../src/ossie_lightdash/converter_issues.py | 6 ++ .../src/ossie_lightdash/ossie_to_lightdash.py | 92 +++++++++++++++++-- converters/lightdash/tests/test_cli.py | 25 ++++- .../tests/test_ossie_to_lightdash.py | 44 +++++++++ 6 files changed, 284 insertions(+), 30 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 82b41fd1..743aa4ad 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -25,10 +25,13 @@ Lightdash reads its semantic layer from dbt `schema.yml` files: dimensions and metrics are declared per column (and per model) under `meta`. This converter translates between that shape and Ossie. -- **Export** (`ossie_to_lightdash`): Ossie document → a dbt `schema.yml`-shaped - dictionary with Lightdash `meta` blocks, ready to merge into a dbt project. -- **Import** (`lightdash_to_ossie`): a Lightdash-flavoured `schema.yml` → an - Ossie document, for teams adopting Ossie as the source of truth for +- **Export** (`ossie_to_lightdash`): Ossie document → a Lightdash project. + The default output is Lightdash's own dbt-free model files (`type: model`, + `sql_from`, typed dimensions), which `lightdash deploy` reads as they are; + `--format dbt-meta` produces one dbt `schema.yml` with Lightdash `meta` + blocks instead, for projects that keep their definitions in dbt. +- **Import** (`lightdash_to_ossie`): a Lightdash-flavoured dbt `schema.yml` → + an Ossie document, for teams adopting Ossie as the source of truth for definitions they already maintain in Lightdash. Lightdash-only attributes travel in `custom_extensions` under the registered @@ -38,10 +41,24 @@ project exactly while every other consumer works from the core vocabulary. ## Usage ``` -ossie-lightdash export semantic_model.yaml schema.yml --dialect BIGQUERY [--meta-under-config] +# Ossie -> a deployable Lightdash project (no dbt needed) +ossie-lightdash export semantic_model.yaml my-project --dialect BIGQUERY +cd my-project && lightdash deploy + +# Ossie -> one dbt schema.yml with Lightdash meta, for a dbt project +ossie-lightdash export semantic_model.yaml schema.yml --format dbt-meta --dialect BIGQUERY [--meta-under-config] + +# Lightdash dbt meta -> Ossie ossie-lightdash import schema.yml semantic_model.json --database analytics_db --schema marts --dialect BIGQUERY ``` +The default export writes `my-project/lightdash/models/.yml`, one file +per dataset, and a starter `my-project/lightdash.config.yml` whose +`warehouse.type` is derived from `--dialect` (`BIGQUERY`, `SNOWFLAKE`, +`DATABRICKS`) or given with `--warehouse`; an existing config is left alone. +Each dataset's `source` becomes the model's `sql_from` verbatim, a table +reference or a query. + Issues (anything lost or approximated, see below) are printed to stderr as `[ISSUE_TYPE] element`. @@ -57,14 +74,24 @@ result = LightdashToOssieConverter(OssieDialect.BIGQUERY).convert( result.output # OssieDocument result.issues # [ConverterIssue(issue_type, element_name), ...] -exported = OssieToLightdashConverter(OssieDialect.BIGQUERY).convert(result.output) -exported.output # {"version": 2, "models": [...]} +models = OssieToLightdashConverter(OssieDialect.BIGQUERY).convert_models(document) +models.output # [{"type": "model", "name": ..., "sql_from": ..., "dimensions": [...]}, ...] + +exported = OssieToLightdashConverter(OssieDialect.BIGQUERY).convert(document) +exported.output # {"version": 2, "models": [...]} (dbt-meta flavour) ``` Requires Python 3.11+ and the in-repo `apache-ossie` package (`../../python`). ## Mapping +The table describes the dbt-meta flavour; the model-file flavour is the same +mapping with three differences: `dataset.source` is the model's `sql_from`, +every dimension carries its own `type` and `sql` (a field without a datatype +gets `string` with a `DIMENSION_TYPE_DEFAULTED` issue), and model meta that +the dbt flavour has to stash (`sql_filter`, `group_details`, +`default_time_dimension`, ...) are ordinary top-level keys of the model file. + | Ossie | Lightdash (dbt meta) | | ----- | -------------------- | | `dataset` | dbt model (`name` = table part of `source`) | @@ -170,8 +197,9 @@ those types from the warehouse, not from the YAML. ### Kept for Lightdash only -Stashed in the `LIGHTDASH` extension and restored on export, invisible to -other consumers: presentation attributes of dimensions and metrics (`format`, +Stashed in the `LIGHTDASH` extension and restored on export (as `meta` in the +dbt flavour, as top-level keys in the model-file flavour), invisible to other +consumers: presentation attributes of dimensions and metrics (`format`, `round`, `compact`, `groups`, `urls`, `show_underlying_values`, ...); model meta without Ossie vocabulary (`label`, `hidden`, `group_details`, `default_time_dimension`, `order_fields_by`, ...); column meta outside @@ -210,8 +238,8 @@ dataset is unrestricted). Encoding metric filters as `CASE WHEN` and from the document. - Custom extensions of other vendors (`FOREIGN_EXTENSION_IGNORED`); they remain untouched in the Ossie document. -- Standalone Lightdash YAML projects (Lightdash without dbt) — the converter - targets the dbt-meta flavour. +- Import reads the dbt-meta flavour only; Lightdash model files are not yet + read back. - Documents are emitted at the in-repo spec version; dbt-core 1.12's native Ossie parsing accepts `0.1.0` / `0.1.1` only. @@ -223,7 +251,8 @@ Every loss or approximation is reported as a `ConverterIssue`: on import `EXPRESSION_NOT_PORTABLE`, `METRIC_REFERENCE_INLINED`, `ALIAS_REFERENCE_FLATTENED`, `METRIC_NAME_COLLISION`, `SOURCE_UNQUALIFIED`, `METRIC_SQL_MISSING`; on export `CROSS_DATASET_METRIC_DROPPED`, -`FIELD_REFERENCE_UNJOINED`, `TIME_ROLE_NOT_REPRESENTABLE`, `DIALECT_UNAVAILABLE`, +`FIELD_REFERENCE_UNJOINED`, `TIME_ROLE_NOT_REPRESENTABLE`, +`DIMENSION_TYPE_DEFAULTED`, `COLUMN_META_NOT_REPRESENTABLE`, `DIALECT_UNAVAILABLE`, `RELATIONSHIP_COLUMNS_MISMATCHED`, `EXTENSION_DATA_INVALID`, `FOREIGN_EXTENSION_IGNORED`. @@ -234,8 +263,9 @@ uv sync uv run pytest ``` -Beyond the unit tests and the TPC-DS round trip, the converter is exercised -against real Lightdash projects (the public jaffle-shop demo and two +The exported TPC-DS project compiles with `lightdash compile` (5 explores, 0 +errors). Beyond the unit tests and the TPC-DS round trip, the converter is +exercised against real Lightdash projects (the public jaffle-shop demo and two production projects on BigQuery): every model's meta and every join survive Lightdash → Ossie → Lightdash, every uniquely named metric returns with its expression unchanged, and the documents pass `validation/validate.py`. diff --git a/converters/lightdash/src/ossie_lightdash/cli.py b/converters/lightdash/src/ossie_lightdash/cli.py index 066732da..3c0c226f 100644 --- a/converters/lightdash/src/ossie_lightdash/cli.py +++ b/converters/lightdash/src/ossie_lightdash/cli.py @@ -37,6 +37,49 @@ def _read_document(path: Path) -> OssieDocument: return OssieDocument.model_validate(yaml.safe_load(text)) +# Ossie dialects that name a Lightdash warehouse type. +_WAREHOUSE_BY_DIALECT = { + OssieDialect.BIGQUERY: "bigquery", + OssieDialect.SNOWFLAKE: "snowflake", + OssieDialect.DATABRICKS: "databricks", +} + + +def _write_lightdash_project( + models, output: Path, *, name: str, warehouse: Optional[str] +) -> None: + """Write one model file per dataset plus a starter lightdash.config.yml. + + Files go to ``/lightdash/models/.yml``, the layout + ``lightdash deploy`` looks for; an existing config is left alone. + """ + models_dir = output / "lightdash" / "models" + models_dir.mkdir(parents=True, exist_ok=True) + for model in models: + (models_dir / f"{model['name']}.yml").write_text( + yaml.safe_dump(model, sort_keys=False, allow_unicode=True), encoding="utf-8" + ) + config = output / "lightdash.config.yml" + if not config.exists(): + config.write_text( + yaml.safe_dump( + { + "name": name, + "version": "1.0", + "warehouse": {"type": warehouse or "CHANGE_ME"}, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + if warehouse is None: + print( + "lightdash.config.yml: set warehouse.type (pass --warehouse, or a " + "--dialect Lightdash knows: BIGQUERY, SNOWFLAKE, DATABRICKS)", + file=sys.stderr, + ) + + def _print_issues(issues) -> None: for issue in issues: print(f"[{issue.issue_type.value}] {issue.element_name}", file=sys.stderr) @@ -47,10 +90,28 @@ def main(argv: Optional[List[str]] = None) -> int: subparsers = parser.add_subparsers(dest="command", required=True) export_parser = subparsers.add_parser( - "export", help="Ossie document (.json/.yaml) -> Lightdash dbt schema.yml" + "export", + help="Ossie document (.json/.yaml) -> Lightdash model files, or a dbt schema.yml", ) export_parser.add_argument("input", type=Path) - export_parser.add_argument("output", type=Path) + export_parser.add_argument( + "output", + type=Path, + help="project directory (lightdash-yml) or schema file (dbt-meta)", + ) + export_parser.add_argument( + "--format", + choices=["lightdash-yml", "dbt-meta"], + default="lightdash-yml", + help="lightdash-yml: Lightdash's dbt-free model files, deployable as they are " + "(default); dbt-meta: one dbt schema.yml with Lightdash meta blocks", + ) + export_parser.add_argument( + "--warehouse", + default=None, + help="warehouse.type for the generated lightdash.config.yml " + "(default: derived from --dialect when possible)", + ) export_parser.add_argument( "--dialect", choices=[dialect.name for dialect in OssieDialect], @@ -83,13 +144,23 @@ def main(argv: Optional[List[str]] = None) -> int: args = parser.parse_args(argv) if args.command == "export": - result = OssieToLightdashConverter( - OssieDialect[args.dialect], meta_under_config=args.meta_under_config - ).convert(_read_document(args.input)) - args.output.write_text( - yaml.safe_dump(result.output, sort_keys=False, allow_unicode=True), - encoding="utf-8", - ) + dialect = OssieDialect[args.dialect] + document = _read_document(args.input) + converter = OssieToLightdashConverter(dialect, meta_under_config=args.meta_under_config) + if args.format == "lightdash-yml": + result = converter.convert_models(document) + _write_lightdash_project( + result.output, + args.output, + name=document.semantic_model[0].name if document.semantic_model else "ossie", + warehouse=args.warehouse or _WAREHOUSE_BY_DIALECT.get(dialect), + ) + else: + result = converter.convert(document) + args.output.write_text( + yaml.safe_dump(result.output, sort_keys=False, allow_unicode=True), + encoding="utf-8", + ) else: schema_yml = yaml.safe_load(args.input.read_text(encoding="utf-8")) result = LightdashToOssieConverter(OssieDialect[args.dialect]).convert( diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py index c06e054b..3685871e 100644 --- a/converters/lightdash/src/ossie_lightdash/converter_issues.py +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -74,6 +74,12 @@ class ConverterIssueType(Enum): # Import: a model's `sql_filter` / `sql_where` / `required_filters` are kept # for Lightdash only; the Ossie dataset is unrestricted for other consumers. ROW_FILTER_NOT_PORTABLE = "ROW_FILTER_NOT_PORTABLE" + # Export (Lightdash YAML): a dimension needs a type and the field has no + # datatype, so `string` is assumed. + DIMENSION_TYPE_DEFAULTED = "DIMENSION_TYPE_DEFAULTED" + # Export (Lightdash YAML): stashed column meta other than + # `additional_dimensions` has no place on a YAML dimension. + COLUMN_META_NOT_REPRESENTABLE = "COLUMN_META_NOT_REPRESENTABLE" # Import: a join targets a model that is not in the input, so the # relationship would reference an unknown dataset and is skipped. JOIN_TARGET_UNKNOWN = "JOIN_TARGET_UNKNOWN" diff --git a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py index 3d18023c..42466dea 100644 --- a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py +++ b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py @@ -16,9 +16,11 @@ # under the License. """Convert an Ossie document into Lightdash semantic definitions. -The output is a dbt ``schema.yml``-shaped dictionary whose ``meta`` blocks -carry Lightdash dimensions, metrics and joins, ready to be merged into a dbt -project that Lightdash reads. Lightdash-specific presentation attributes that +``convert`` produces a dbt ``schema.yml``-shaped dictionary whose ``meta`` +blocks carry Lightdash dimensions, metrics and joins, ready to be merged into +a dbt project that Lightdash reads. ``convert_models`` produces Lightdash's +own dbt-free model files (``type: model``, ``sql_from``, typed dimensions), +which ``lightdash deploy`` reads directly. Lightdash-specific presentation attributes that have no Ossie vocabulary round-trip through ``custom_extensions`` entries with ``vendor_name: LIGHTDASH``; their keys are overlaid onto the generated definitions and win for presentation attributes, while structural keys @@ -67,6 +69,10 @@ _PROTECTED_JOIN_KEYS = {"join", "sql_on", "alias"} # Dataset-level stash keys that map to Ossie vocabulary or are handled apart. _PROTECTED_MODEL_KEYS = {"joins", "metrics", "primary_key", "ai_hint"} +# Order of the top-level keys in a Lightdash model file; anything else +# (stashed model meta) follows in source order. +_MODEL_KEY_ORDER = ("type", "name", "label", "description", "sql_from", "primary_key", "ai_hint") +_DIMENSION_KEY_ORDER = ("name", "type", "label", "description", "sql", "hidden") # Joins a dataset declares, keyed by the joined dataset: the name Lightdash SQL # uses to reference it (the joined model, or its alias). The first join to a @@ -120,6 +126,64 @@ def _ai_hint(ai_context: Any) -> Any: return hints[0] if len(hints) == 1 else hints +def _ordered(entries: Dict[str, Any], order: Tuple[str, ...]) -> Dict[str, Any]: + """Return ``entries`` with the keys in ``order`` first, the rest as they came.""" + return { + **{key: entries[key] for key in order if key in entries}, + **{key: value for key, value in entries.items() if key not in order}, + } + + +def _to_lightdash_model( + dataset: OssieDataset, model: Dict[str, Any], issues: List[ConverterIssue] +) -> Dict[str, Any]: + """Reshape a dbt-flavoured model into a Lightdash model file.""" + meta = dict(model.get("meta") or {}) + out: Dict[str, Any] = {"type": "model", "name": model["name"]} + if model.get("description"): + out["description"] = model["description"] + out["sql_from"] = dataset.source + joins = meta.pop("joins", None) + model_metrics = meta.pop("metrics", None) + out.update(meta) + if joins: + out["joins"] = joins + if model_metrics: + out["metrics"] = model_metrics + + dimensions: List[Dict[str, Any]] = [] + for column in model.get("columns") or []: + column_meta = column.get("meta") or {} + dimension = dict(column_meta.get("dimension") or {}) + dimension["name"] = column["name"] + if column.get("description") and "description" not in dimension: + dimension["description"] = column["description"] + if not dimension.get("type"): + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.DIMENSION_TYPE_DEFAULTED, + element_name=f"{model['name']}.{column['name']}", + ) + ) + dimension["type"] = "string" + if not dimension.get("sql"): + dimension["sql"] = f"${{TABLE}}.{column['name']}" + if column_meta.get("metrics"): + dimension["metrics"] = column_meta["metrics"] + if column_meta.get("additional_dimensions"): + dimension["additional_dimensions"] = column_meta["additional_dimensions"] + if any(key not in ("dimension", "metrics", "additional_dimensions") for key in column_meta): + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.COLUMN_META_NOT_REPRESENTABLE, + element_name=f"{model['name']}.{column['name']}", + ) + ) + dimensions.append(_ordered(dimension, _DIMENSION_KEY_ORDER)) + out["dimensions"] = dimensions + return _ordered(out, _MODEL_KEY_ORDER) + + def _nest_meta_under_config(node: Dict[str, Any]) -> None: """Move a node's ``meta`` under ``config`` (the dbt 1.10+ placement).""" meta = node.pop("meta", None) @@ -150,10 +214,11 @@ def __init__( self._meta_under_config = meta_under_config def convert(self, document: OssieDocument) -> ConverterResult[Dict[str, Any]]: + """Ossie document → dbt ``schema.yml`` dict with Lightdash ``meta``.""" issues: List[ConverterIssue] = [] models: List[Dict[str, Any]] = [] for semantic_model in document.semantic_model: - models.extend(self._convert_semantic_model(semantic_model, issues)) + models.extend(model for _, model in self._convert_semantic_model(semantic_model, issues)) if self._meta_under_config: for model in models: _nest_meta_under_config(model) @@ -161,6 +226,21 @@ def convert(self, document: OssieDocument) -> ConverterResult[Dict[str, Any]]: _nest_meta_under_config(column) return ConverterResult(output={"version": 2, "models": models}, issues=issues) + def convert_models(self, document: OssieDocument) -> ConverterResult[List[Dict[str, Any]]]: + """Ossie document → Lightdash model files (one dict per dataset). + + The dbt-free format has a home for everything the dbt flavour has to + stash: ``sql_from`` is the dataset's ``source`` verbatim, model meta + becomes top-level keys, and every dimension carries its own ``type`` + and ``sql``. + """ + issues: List[ConverterIssue] = [] + models: List[Dict[str, Any]] = [] + for semantic_model in document.semantic_model: + for dataset, model in self._convert_semantic_model(semantic_model, issues): + models.append(_to_lightdash_model(dataset, model, issues)) + return ConverterResult(output=models, issues=issues) + def _pick_expression( self, expression: OssieExpression, element_name: str, issues: List[ConverterIssue] ) -> str: @@ -183,7 +263,7 @@ def _pick_expression( def _convert_semantic_model( self, semantic_model: OssieSemanticModel, issues: List[ConverterIssue] - ) -> List[Dict[str, Any]]: + ) -> List[Tuple[OssieDataset, Dict[str, Any]]]: datasets = semantic_model.datasets or [] dataset_names = {dataset.name for dataset in datasets} model_name_by_dataset = { @@ -253,7 +333,7 @@ def _convert_semantic_model( for dataset_name, joins in joins_by_dataset.items(): models_by_dataset[dataset_name].setdefault("meta", {})["joins"] = joins - return [models_by_dataset[dataset.name] for dataset in datasets] + return [(dataset, models_by_dataset[dataset.name]) for dataset in datasets] def _plan_joins( self, diff --git a/converters/lightdash/tests/test_cli.py b/converters/lightdash/tests/test_cli.py index 01a5e35a..b60cda04 100644 --- a/converters/lightdash/tests/test_cli.py +++ b/converters/lightdash/tests/test_cli.py @@ -30,7 +30,7 @@ @pytest.mark.parametrize("suffix", [".yaml", ".json"]) def test_import_writes_a_loadable_document(tmp_path, suffix): schema_yml = tmp_path / "schema.yml" - assert main(["export", str(TPCDS_PATH), str(schema_yml)]) == 0 + assert main(["export", str(TPCDS_PATH), str(schema_yml), "--format", "dbt-meta"]) == 0 document_path = tmp_path / f"semantic_model{suffix}" assert main(["import", str(schema_yml), str(document_path), "--schema", "public"]) == 0 @@ -43,3 +43,26 @@ def test_import_writes_a_loadable_document(tmp_path, suffix): assert {dataset.name for dataset in document.semantic_model[0].datasets} == { "store_sales", "date_dim", "customer", "item", "store" } + +def test_export_writes_a_lightdash_project(tmp_path): + project = tmp_path / "project" + assert main(["export", str(TPCDS_PATH), str(project), "--dialect", "BIGQUERY"]) == 0 + files = sorted(p.name for p in (project / "lightdash" / "models").iterdir()) + assert files == ["customer.yml", "date_dim.yml", "item.yml", "store.yml", "store_sales.yml"] + model = yaml.safe_load((project / "lightdash" / "models" / "store_sales.yml").read_text()) + assert model["type"] == "model" + assert model["sql_from"] == "tpcds.public.store_sales" + assert all({"name", "type", "sql"} <= set(d) for d in model["dimensions"]) + config = yaml.safe_load((project / "lightdash.config.yml").read_text()) + assert config["warehouse"] == {"type": "bigquery"} + assert config["name"] == "tpcds_retail_model" + # A second export leaves an existing config alone. + (project / "lightdash.config.yml").write_text("name: mine\nversion: '1.0'\nwarehouse:\n type: postgres\n") + assert main(["export", str(TPCDS_PATH), str(project)]) == 0 + assert yaml.safe_load((project / "lightdash.config.yml").read_text())["name"] == "mine" + + +def test_export_dbt_meta_still_writes_one_schema_file(tmp_path): + schema_yml = tmp_path / "schema.yml" + assert main(["export", str(TPCDS_PATH), str(schema_yml), "--format", "dbt-meta"]) == 0 + assert yaml.safe_load(schema_yml.read_text())["version"] == 2 diff --git a/converters/lightdash/tests/test_ossie_to_lightdash.py b/converters/lightdash/tests/test_ossie_to_lightdash.py index f92cf304..70786423 100644 --- a/converters/lightdash/tests/test_ossie_to_lightdash.py +++ b/converters/lightdash/tests/test_ossie_to_lightdash.py @@ -713,3 +713,47 @@ def test_stashed_joins_and_meta_restore_the_explore(self): assert customer_id["meta"]["additional_dimensions"] == { "id_prefix": {"type": "string", "sql": "LEFT(${TABLE}.customer_id, 2)"} } + + def test_lightdash_model_files_are_deployable_as_they_are(self): + document = _document() + tampered = document.model_copy(deep=True) + datasets = tampered.semantic_model[0].datasets + datasets[0] = datasets[0].model_copy( + update={ + "primary_key": ["order_id"], + "custom_extensions": [ + OssieCustomExtension( + vendor_name="LIGHTDASH", + data=json.dumps({"sql_filter": "${TABLE}.deleted = false", "label": "Orders"}), + ) + ], + } + ) + result = OssieToLightdashConverter().convert_models(tampered) + orders = next(m for m in result.output if m["name"] == "orders") + assert list(orders)[:4] == ["type", "name", "label", "description"] + assert orders["type"] == "model" + assert orders["sql_from"] == "analytics_db.marts.orders" + assert orders["primary_key"] == "order_id" + assert orders["sql_filter"] == "${TABLE}.deleted = false" + assert orders["joins"][0]["join"] == "customers" + assert orders["metrics"]["conversion_rate"]["type"] == "number" + dimensions = {d["name"]: d for d in orders["dimensions"]} + # Every dimension carries its own type and sql; a measure-only field + # is a hidden one; column metrics sit under their dimension. + assert dimensions["order_date"] == { + "name": "order_date", + "type": "date", + "label": "Order date", + "sql": "${TABLE}.order_date", + } + assert dimensions["amount"]["hidden"] is True + assert dimensions["amount"]["sql"] == "${TABLE}.amount" + assert dimensions["amount"]["metrics"]["total_amount"]["type"] == "sum" + # No datatype on `status`: the type is assumed and reported. + assert dimensions["status"]["type"] == "string" + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.DIMENSION_TYPE_DEFAULTED + ] == ["orders.status", "orders.amount", "orders.customer_id", "customers.customer_id"] From 819f00e91fa876e2918f59a7dd19dbc06108895b Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 14:16:15 +0100 Subject: [PATCH 23/33] feat(lightdash): import a whole dbt project directory dbt projects keep one schema YAML per model, so import now accepts a directory as well as a file: it walks it in sorted order, merges every list-valued models: and seeds: entry, and ignores target/, dbt_packages/ and dbt_project.yml. load_schema is exported for the Python API. --- converters/lightdash/README.md | 9 ++- .../lightdash/src/ossie_lightdash/__init__.py | 2 + .../lightdash/src/ossie_lightdash/cli.py | 10 +++- .../src/ossie_lightdash/dbt_project.py | 58 +++++++++++++++++++ converters/lightdash/tests/test_cli.py | 20 +++++++ 5 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 converters/lightdash/src/ossie_lightdash/dbt_project.py diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 743aa4ad..f9180234 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -48,8 +48,8 @@ cd my-project && lightdash deploy # Ossie -> one dbt schema.yml with Lightdash meta, for a dbt project ossie-lightdash export semantic_model.yaml schema.yml --format dbt-meta --dialect BIGQUERY [--meta-under-config] -# Lightdash dbt meta -> Ossie -ossie-lightdash import schema.yml semantic_model.json --database analytics_db --schema marts --dialect BIGQUERY +# Lightdash dbt meta -> Ossie (a schema file, or a whole dbt project directory) +ossie-lightdash import path/to/dbt semantic_model.json --database analytics_db --schema marts --dialect BIGQUERY ``` The default export writes `my-project/lightdash/models/.yml`, one file @@ -128,7 +128,10 @@ dialect with a `DIALECT_UNAVAILABLE` issue when an expression offers neither. ## Input shape `import` reads the `models:` and `seeds:` entries of a dbt schema file (seeds -are tables to Lightdash too). A join whose target is not among them is skipped +are tables to Lightdash too), or of every YAML file under a directory: point it +at the dbt project root and it walks `models/`, `seeds/` and the rest in sorted +order, ignoring `target/`, `dbt_packages/` and `dbt_project.yml`. A join whose +target is not among them is skipped with a `JOIN_TARGET_UNKNOWN` issue, since Ossie relationships may only reference datasets in the document. diff --git a/converters/lightdash/src/ossie_lightdash/__init__.py b/converters/lightdash/src/ossie_lightdash/__init__.py index e3cd0de7..9e0848b9 100644 --- a/converters/lightdash/src/ossie_lightdash/__init__.py +++ b/converters/lightdash/src/ossie_lightdash/__init__.py @@ -20,6 +20,7 @@ ConverterIssueType, ConverterResult, ) +from ossie_lightdash.dbt_project import load_schema from ossie_lightdash.lightdash_to_ossie import LightdashToOssieConverter from ossie_lightdash.ossie_to_lightdash import OssieToLightdashConverter @@ -29,4 +30,5 @@ "ConverterResult", "LightdashToOssieConverter", "OssieToLightdashConverter", + "load_schema", ] diff --git a/converters/lightdash/src/ossie_lightdash/cli.py b/converters/lightdash/src/ossie_lightdash/cli.py index 3c0c226f..d4792595 100644 --- a/converters/lightdash/src/ossie_lightdash/cli.py +++ b/converters/lightdash/src/ossie_lightdash/cli.py @@ -26,6 +26,7 @@ import yaml from ossie import OssieDialect, OssieDocument +from ossie_lightdash.dbt_project import load_schema from ossie_lightdash.lightdash_to_ossie import LightdashToOssieConverter from ossie_lightdash.ossie_to_lightdash import OssieToLightdashConverter @@ -125,9 +126,12 @@ def main(argv: Optional[List[str]] = None) -> int: ) import_parser = subparsers.add_parser( - "import", help="Lightdash dbt schema.yml -> Ossie document (.json/.yaml)" + "import", + help="Lightdash dbt schema.yml, or a dbt project directory -> Ossie document (.json/.yaml)", + ) + import_parser.add_argument( + "input", type=Path, help="a schema file, or a directory walked for models: and seeds:" ) - import_parser.add_argument("input", type=Path) import_parser.add_argument("output", type=Path) import_parser.add_argument("--database", default=None) import_parser.add_argument("--schema", default=None) @@ -162,7 +166,7 @@ def main(argv: Optional[List[str]] = None) -> int: encoding="utf-8", ) else: - schema_yml = yaml.safe_load(args.input.read_text(encoding="utf-8")) + schema_yml = load_schema(args.input) result = LightdashToOssieConverter(OssieDialect[args.dialect]).convert( schema_yml, database=args.database, diff --git a/converters/lightdash/src/ossie_lightdash/dbt_project.py b/converters/lightdash/src/ossie_lightdash/dbt_project.py new file mode 100644 index 00000000..f0376741 --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/dbt_project.py @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Read Lightdash-flavoured dbt schema definitions from a file or a project. + +dbt lets a project spread its ``models:`` and ``seeds:`` entries over any +number of YAML files, so ``import`` accepts a directory as well as a single +schema file and merges everything it finds. +""" + +from pathlib import Path +from typing import Any, Dict, List + +import yaml + +# Directories dbt generates; nothing in them is authored schema. +_SKIPPED_DIRS = {"target", "dbt_packages", "logs", ".git", "node_modules"} + + +def load_schema(path: Path) -> Dict[str, Any]: + """Return ``{"version": 2, "models": [...], "seeds": [...]}`` for ``path``. + + A file is read as is. A directory is walked in sorted order; every + ``.yml`` / ``.yaml`` file contributes its list-valued ``models:`` and + ``seeds:`` entries (``dbt_project.yml`` has a dict-valued ``models:`` and + is skipped by that rule), and generated directories are ignored. + """ + if path.is_file(): + return yaml.safe_load(path.read_text(encoding="utf-8")) or {} + + models: List[Dict[str, Any]] = [] + seeds: List[Dict[str, Any]] = [] + for file in sorted(path.rglob("*.y*ml")): + if file.suffix not in (".yml", ".yaml"): + continue + if _SKIPPED_DIRS & set(file.relative_to(path).parts[:-1]): + continue + document = yaml.safe_load(file.read_text(encoding="utf-8")) or {} + if not isinstance(document, dict): + continue + if isinstance(document.get("models"), list): + models.extend(document["models"]) + if isinstance(document.get("seeds"), list): + seeds.extend(document["seeds"]) + return {"version": 2, "models": models, "seeds": seeds} diff --git a/converters/lightdash/tests/test_cli.py b/converters/lightdash/tests/test_cli.py index b60cda04..0bede2f4 100644 --- a/converters/lightdash/tests/test_cli.py +++ b/converters/lightdash/tests/test_cli.py @@ -66,3 +66,23 @@ def test_export_dbt_meta_still_writes_one_schema_file(tmp_path): schema_yml = tmp_path / "schema.yml" assert main(["export", str(TPCDS_PATH), str(schema_yml), "--format", "dbt-meta"]) == 0 assert yaml.safe_load(schema_yml.read_text())["version"] == 2 + +def test_import_reads_a_whole_dbt_project(tmp_path): + project = tmp_path / "dbt" + (project / "models" / "marts").mkdir(parents=True) + (project / "target").mkdir() + (project / "dbt_project.yml").write_text("name: p\nmodels:\n p:\n +materialized: table\n") + (project / "models" / "orders.yml").write_text( + "models:\n - name: orders\n columns:\n - name: amount\n meta:\n metrics:\n total: {type: sum}\n" + ) + (project / "models" / "marts" / "customers.yaml").write_text( + "models:\n - name: customers\n columns:\n - name: id\n" + ) + (project / "data.yml").write_text("seeds:\n - name: statuses\n columns:\n - name: code\n") + (project / "target" / "stale.yml").write_text("models:\n - name: stale\n") + + document_path = tmp_path / "model.yaml" + assert main(["import", str(project), str(document_path), "--schema", "marts"]) == 0 + document = OssieDocument.model_validate(yaml.safe_load(document_path.read_text())) + assert [d.name for d in document.semantic_model[0].datasets] == ["customers", "orders", "statuses"] + assert document.semantic_model[0].metrics[0].name == "orders_total" From f28caf14762a9a5135159f64389665f2aeb3048c Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 14:23:31 +0100 Subject: [PATCH 24/33] feat(lightdash): say what was written and explain each issue The CLI printed nothing on success and bare issue codes. It now reports what it wrote (model files and config, or datasets, metrics and relationships), explains each issue type the first time it appears, and closes with a count. Everything goes to stderr, issues first, like the other converters; stdout stays clean. --- .../lightdash/src/ossie_lightdash/cli.py | 24 +++++++++++++++++- .../src/ossie_lightdash/converter_issues.py | 25 +++++++++++++++++++ converters/lightdash/tests/test_cli.py | 12 ++++++++- 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/converters/lightdash/src/ossie_lightdash/cli.py b/converters/lightdash/src/ossie_lightdash/cli.py index d4792595..f855141a 100644 --- a/converters/lightdash/src/ossie_lightdash/cli.py +++ b/converters/lightdash/src/ossie_lightdash/cli.py @@ -26,6 +26,7 @@ import yaml from ossie import OssieDialect, OssieDocument +from ossie_lightdash.converter_issues import ISSUE_EXPLANATIONS from ossie_lightdash.dbt_project import load_schema from ossie_lightdash.lightdash_to_ossie import LightdashToOssieConverter from ossie_lightdash.ossie_to_lightdash import OssieToLightdashConverter @@ -82,8 +83,16 @@ def _write_lightdash_project( def _print_issues(issues) -> None: + """One line per issue, with the explanation on its first occurrence.""" + explained = set() for issue in issues: - print(f"[{issue.issue_type.value}] {issue.element_name}", file=sys.stderr) + line = f"[{issue.issue_type.value}] {issue.element_name}" + if issue.issue_type not in explained: + explained.add(issue.issue_type) + line += f" -- {ISSUE_EXPLANATIONS.get(issue.issue_type, '')}".rstrip(" -") + print(line, file=sys.stderr) + if issues: + print(f"{len(issues)} issue(s); everything else converted cleanly.", file=sys.stderr) def main(argv: Optional[List[str]] = None) -> int: @@ -159,12 +168,17 @@ def main(argv: Optional[List[str]] = None) -> int: name=document.semantic_model[0].name if document.semantic_model else "ossie", warehouse=args.warehouse or _WAREHOUSE_BY_DIALECT.get(dialect), ) + summary = ( + f"Wrote {len(result.output)} model file(s) to {args.output / 'lightdash' / 'models'}" + f" and {args.output / 'lightdash.config.yml'}; run `lightdash compile` there." + ) else: result = converter.convert(document) args.output.write_text( yaml.safe_dump(result.output, sort_keys=False, allow_unicode=True), encoding="utf-8", ) + summary = f"Wrote {len(result.output['models'])} model(s) to {args.output}." else: schema_yml = load_schema(args.input) result = LightdashToOssieConverter(OssieDialect[args.dialect]).convert( @@ -173,6 +187,12 @@ def main(argv: Optional[List[str]] = None) -> int: schema=args.schema, semantic_model_name=args.semantic_model_name, ) + semantic_model = result.output.semantic_model[0] + summary = ( + f"Wrote {len(semantic_model.datasets or [])} dataset(s), " + f"{len(semantic_model.metrics or [])} metric(s), " + f"{len(semantic_model.relationships or [])} relationship(s) to {args.output}." + ) document = result.output.model_dump(mode="json", by_alias=True, exclude_none=True) if args.output.suffix == ".json": args.output.write_text( @@ -184,7 +204,9 @@ def main(argv: Optional[List[str]] = None) -> int: encoding="utf-8", ) + # Issues first, then what was written, all on stderr like the other converters. _print_issues(result.issues) + print(summary, file=sys.stderr) return 0 diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py index 3685871e..9ff00237 100644 --- a/converters/lightdash/src/ossie_lightdash/converter_issues.py +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -85,6 +85,31 @@ class ConverterIssueType(Enum): JOIN_TARGET_UNKNOWN = "JOIN_TARGET_UNKNOWN" +# One line per issue type, for people reading the CLI output. +ISSUE_EXPLANATIONS = { + ConverterIssueType.SOURCE_UNQUALIFIED: "no --schema given, so the dataset source is just the model name", + ConverterIssueType.JOIN_SQL_UNPARSED: "no column pair in sql_on; kept for Lightdash only, no relationship", + ConverterIssueType.JOIN_STASHED: "chained join, expression join or extra conditions; kept verbatim for Lightdash, relationships derived from the column pairs", + ConverterIssueType.CROSS_DATASET_METRIC_DROPPED: "no single model joins every dataset the expression references", + ConverterIssueType.RELATIONSHIP_COLUMNS_MISMATCHED: "from_columns and to_columns differ in length; skipped", + ConverterIssueType.EXTENSION_DATA_INVALID: "LIGHTDASH extension data is not valid JSON; its attributes are lost", + ConverterIssueType.METRIC_SQL_MISSING: "model-level metric without sql; skipped", + ConverterIssueType.TIME_ROLE_NOT_REPRESENTABLE: "is_time on a non-date type (e.g. an integer year); Lightdash has no such marker, the column is a plain dimension", + ConverterIssueType.FOREIGN_EXTENSION_IGNORED: "another vendor's extension; left untouched in the Ossie document", + ConverterIssueType.EXPRESSION_NOT_PORTABLE: "SQL uses parameters, user attributes or Liquid, which only Lightdash can evaluate; skipped", + ConverterIssueType.METRIC_REFERENCE_INLINED: "${metric} reference replaced by that metric's expression", + ConverterIssueType.ALIAS_REFERENCE_FLATTENED: "${alias.column} now points at the joined model; which join was meant is lost", + ConverterIssueType.DIALECT_UNAVAILABLE: "neither --dialect nor ANSI_SQL offered; first available dialect used", + ConverterIssueType.FIELD_REFERENCE_UNJOINED: "expression names a dataset this model does not join; emitted as is", + ConverterIssueType.METRIC_NAME_COLLISION: "still a duplicate after _; suffixed", + ConverterIssueType.METRIC_FILTER_NOT_PORTABLE: "filters kept for Lightdash only; other tools see the unfiltered aggregate", + ConverterIssueType.ROW_FILTER_NOT_PORTABLE: "sql_filter / required_filters kept for Lightdash only; other tools see all rows", + ConverterIssueType.DIMENSION_TYPE_DEFAULTED: "no datatype on the field; dimension type set to string", + ConverterIssueType.COLUMN_META_NOT_REPRESENTABLE: "stashed column meta other than additional_dimensions has no place on a model-file dimension; dropped", + ConverterIssueType.JOIN_TARGET_UNKNOWN: "join to a model that is not in the input; skipped", +} + + @dataclass(frozen=True) class ConverterIssue: """Records a single instance of information loss during conversion.""" diff --git a/converters/lightdash/tests/test_cli.py b/converters/lightdash/tests/test_cli.py index 0bede2f4..ecd6f922 100644 --- a/converters/lightdash/tests/test_cli.py +++ b/converters/lightdash/tests/test_cli.py @@ -44,9 +44,19 @@ def test_import_writes_a_loadable_document(tmp_path, suffix): "store_sales", "date_dim", "customer", "item", "store" } -def test_export_writes_a_lightdash_project(tmp_path): +def test_export_writes_a_lightdash_project(tmp_path, capsys): project = tmp_path / "project" assert main(["export", str(TPCDS_PATH), str(project), "--dialect", "BIGQUERY"]) == 0 + captured = capsys.readouterr() + # Like the other converters: stdout stays clean, stderr carries the report. + assert captured.out == "" + assert captured.err.splitlines()[-1].startswith("Wrote 5 model file(s) to ") + # The one loss is named, explained once, and counted. + assert captured.err.splitlines()[:2] == [ + "[TIME_ROLE_NOT_REPRESENTABLE] d_year -- is_time on a non-date type (e.g. an integer year); " + "Lightdash has no such marker, the column is a plain dimension", + "1 issue(s); everything else converted cleanly.", + ] files = sorted(p.name for p in (project / "lightdash" / "models").iterdir()) assert files == ["customer.yml", "date_dim.yml", "item.yml", "store.yml", "store_sales.yml"] model = yaml.safe_load((project / "lightdash" / "models" / "store_sales.yml").read_text()) From 9cd03e81585d43b20b07ebe85c3d8e1f4cdda640 Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 14:40:14 +0100 Subject: [PATCH 25/33] feat(lightdash): take column types from a dbt catalog on import Lightdash learns most dimension types from the warehouse, not from YAML, so a Lightdash project imports with most fields untyped and the model-file export has to assume string for them. --catalog reads the catalog.json that dbt docs generate writes and fills datatype for every column without an authored type, reducing the physical type to Ossie's vocabulary (INT64 -> Integer, NUMBER(12,2) -> Decimal, TIMESTAMP_TZ -> DateTimeTz). Authored types win. A model missing from the catalog is reported, which is how a stale catalog shows up. On demo-f1 this takes the model-file export from 78 assumed types to 1. --- converters/lightdash/README.md | 21 +++- .../lightdash/src/ossie_lightdash/catalog.py | 113 ++++++++++++++++++ .../lightdash/src/ossie_lightdash/cli.py | 9 ++ .../src/ossie_lightdash/converter_issues.py | 4 + .../src/ossie_lightdash/lightdash_to_ossie.py | 36 +++++- converters/lightdash/tests/test_cli.py | 16 +++ .../tests/test_lightdash_to_ossie.py | 49 ++++++++ 7 files changed, 242 insertions(+), 6 deletions(-) create mode 100644 converters/lightdash/src/ossie_lightdash/catalog.py diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index f9180234..4e92ad06 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -49,9 +49,19 @@ cd my-project && lightdash deploy ossie-lightdash export semantic_model.yaml schema.yml --format dbt-meta --dialect BIGQUERY [--meta-under-config] # Lightdash dbt meta -> Ossie (a schema file, or a whole dbt project directory) -ossie-lightdash import path/to/dbt semantic_model.json --database analytics_db --schema marts --dialect BIGQUERY +ossie-lightdash import path/to/dbt semantic_model.json --database analytics_db --schema marts --dialect BIGQUERY \ + --catalog path/to/dbt/target/catalog.json ``` +`--catalog` takes the `catalog.json` that `dbt docs generate` writes: the +warehouse's real column types fill in `datatype` for every column without an +authored `type` (authored types win), reduced to Ossie's vocabulary +(`INT64` → `Integer`, `NUMBER(12,2)` → `Decimal`, `TIMESTAMP_TZ` → +`DateTimeTz`, ...). Lightdash learns most of its types from the warehouse +rather than from YAML, so without a catalog most fields leave untyped. A +model the catalog does not know is reported (`CATALOG_MODEL_MISSING`), which +is also how a stale catalog shows. + The default export writes `my-project/lightdash/models/.yml`, one file per dataset, and a starter `my-project/lightdash.config.yml` whose `warehouse.type` is derived from `--dialect` (`BIGQUERY`, `SNOWFLAKE`, @@ -194,9 +204,10 @@ each other. **Types.** Lightdash's `number` covers Ossie's `Integer`, `Decimal` and `Float`, and its `timestamp` covers `DateTime` and `DateTimeTz`, so datatypes -round-trip by category rather than by exact member. A column with no authored -`type` leaves without a datatype, which the spec allows; Lightdash learns -those types from the warehouse, not from the YAML. +authored in YAML round-trip by category rather than by exact member. A column +with no authored `type` leaves without a datatype unless `--catalog` supplies +the warehouse's, in which case the exact member is known; Lightdash itself +learns those types from the warehouse, not from the YAML. ### Kept for Lightdash only @@ -255,7 +266,7 @@ Every loss or approximation is reported as a `ConverterIssue`: on import `ALIAS_REFERENCE_FLATTENED`, `METRIC_NAME_COLLISION`, `SOURCE_UNQUALIFIED`, `METRIC_SQL_MISSING`; on export `CROSS_DATASET_METRIC_DROPPED`, `FIELD_REFERENCE_UNJOINED`, `TIME_ROLE_NOT_REPRESENTABLE`, -`DIMENSION_TYPE_DEFAULTED`, `COLUMN_META_NOT_REPRESENTABLE`, `DIALECT_UNAVAILABLE`, +`DIMENSION_TYPE_DEFAULTED`, `COLUMN_META_NOT_REPRESENTABLE`, `CATALOG_MODEL_MISSING`, `DIALECT_UNAVAILABLE`, `RELATIONSHIP_COLUMNS_MISMATCHED`, `EXTENSION_DATA_INVALID`, `FOREIGN_EXTENSION_IGNORED`. diff --git a/converters/lightdash/src/ossie_lightdash/catalog.py b/converters/lightdash/src/ossie_lightdash/catalog.py new file mode 100644 index 00000000..d9bae66a --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/catalog.py @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Column types from a dbt ``catalog.json``. + +``dbt docs generate`` records the warehouse's real type for every column of +every model. Lightdash learns most dimension types from the warehouse rather +than from YAML, so a Lightdash project usually has few authored types; the +catalog fills the gaps with the physical type reduced to Ossie's logical +vocabulary. +""" + +import json +import re +from pathlib import Path +from typing import Dict, Optional + +from ossie import OssieDataType + +# Column types keyed by model name, then by lower-cased column name. +Catalog = Dict[str, Dict[str, str]] + +_TYPE_RE = re.compile(r"^\s*([A-Za-z_][A-Za-z_0-9 ]*?)\s*(?:\(\s*(\d+)\s*(?:,\s*(\d+))?\s*\))?\s*$") + +_EXACT = { + "INT": OssieDataType.INTEGER, + "INTEGER": OssieDataType.INTEGER, + "INT64": OssieDataType.INTEGER, + "BIGINT": OssieDataType.INTEGER, + "SMALLINT": OssieDataType.INTEGER, + "TINYINT": OssieDataType.INTEGER, + "INT2": OssieDataType.INTEGER, + "INT4": OssieDataType.INTEGER, + "INT8": OssieDataType.INTEGER, + "FLOAT": OssieDataType.FLOAT, + "FLOAT4": OssieDataType.FLOAT, + "FLOAT8": OssieDataType.FLOAT, + "FLOAT64": OssieDataType.FLOAT, + "DOUBLE": OssieDataType.FLOAT, + "DOUBLE PRECISION": OssieDataType.FLOAT, + "REAL": OssieDataType.FLOAT, + "STRING": OssieDataType.STRING, + "VARCHAR": OssieDataType.STRING, + "NVARCHAR": OssieDataType.STRING, + "CHAR": OssieDataType.STRING, + "CHARACTER": OssieDataType.STRING, + "CHARACTER VARYING": OssieDataType.STRING, + "TEXT": OssieDataType.STRING, + "BOOL": OssieDataType.BOOLEAN, + "BOOLEAN": OssieDataType.BOOLEAN, + "DATE": OssieDataType.DATE, + "DATETIME": OssieDataType.DATE_TIME, + "TIMESTAMP": OssieDataType.DATE_TIME, + "TIMESTAMP_NTZ": OssieDataType.DATE_TIME, + "TIMESTAMP WITHOUT TIME ZONE": OssieDataType.DATE_TIME, + "TIMESTAMPTZ": OssieDataType.DATE_TIME_TZ, + "TIMESTAMP_TZ": OssieDataType.DATE_TIME_TZ, + "TIMESTAMP_LTZ": OssieDataType.DATE_TIME_TZ, + "TIMESTAMP WITH TIME ZONE": OssieDataType.DATE_TIME_TZ, + "TIME": OssieDataType.TIME, +} +# Exact-decimal families: integral when the scale is 0 (or, for NUMBER, when +# Snowflake's default NUMBER(38,0) is spelled without arguments). +_DECIMAL_FAMILY = {"NUMERIC", "BIGNUMERIC", "DECIMAL", "NUMBER"} + + +def warehouse_type_to_datatype(warehouse_type: str) -> Optional[OssieDataType]: + """Reduce a physical column type to an Ossie datatype, or None when the + type is outside the portable vocabulary (arrays, structs, JSON, ...).""" + match = _TYPE_RE.match(warehouse_type or "") + if match is None: + return None + base = " ".join(match.group(1).upper().split()) + scale = match.group(3) + if base in _DECIMAL_FAMILY: + if scale is not None: + return OssieDataType.INTEGER if int(scale) == 0 else OssieDataType.DECIMAL + return OssieDataType.INTEGER if base == "NUMBER" else OssieDataType.DECIMAL + return _EXACT.get(base) + + +def load_catalog(path: Path) -> Catalog: + """Read a dbt ``catalog.json`` into ``{model: {column: type}}``. + + Models and seeds are keyed by their dbt name (the last segment of the + node's unique id); column names are lower-cased because warehouses differ + in how they report case. + """ + document = json.loads(Path(path).read_text(encoding="utf-8")) + catalog: Catalog = {} + for unique_id, node in (document.get("nodes") or {}).items(): + kind = unique_id.split(".", 1)[0] + if kind not in ("model", "seed"): + continue + name = unique_id.rsplit(".", 1)[-1] + catalog[name] = { + column_name.lower(): (column.get("type") or "") + for column_name, column in (node.get("columns") or {}).items() + } + return catalog diff --git a/converters/lightdash/src/ossie_lightdash/cli.py b/converters/lightdash/src/ossie_lightdash/cli.py index f855141a..bfc51571 100644 --- a/converters/lightdash/src/ossie_lightdash/cli.py +++ b/converters/lightdash/src/ossie_lightdash/cli.py @@ -26,6 +26,7 @@ import yaml from ossie import OssieDialect, OssieDocument +from ossie_lightdash.catalog import load_catalog from ossie_lightdash.converter_issues import ISSUE_EXPLANATIONS from ossie_lightdash.dbt_project import load_schema from ossie_lightdash.lightdash_to_ossie import LightdashToOssieConverter @@ -153,6 +154,13 @@ def main(argv: Optional[List[str]] = None) -> int: default=OssieDialect.ANSI_SQL.name, help="dialect the Lightdash SQL is written in (the project's warehouse)", ) + import_parser.add_argument( + "--catalog", + type=Path, + default=None, + help="dbt target/catalog.json (from `dbt docs generate`): warehouse column " + "types fill in datatypes for columns without an authored type", + ) args = parser.parse_args(argv) @@ -186,6 +194,7 @@ def main(argv: Optional[List[str]] = None) -> int: database=args.database, schema=args.schema, semantic_model_name=args.semantic_model_name, + catalog=load_catalog(args.catalog) if args.catalog else None, ) semantic_model = result.output.semantic_model[0] summary = ( diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py index 9ff00237..5536efea 100644 --- a/converters/lightdash/src/ossie_lightdash/converter_issues.py +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -80,6 +80,9 @@ class ConverterIssueType(Enum): # Export (Lightdash YAML): stashed column meta other than # `additional_dimensions` has no place on a YAML dimension. COLUMN_META_NOT_REPRESENTABLE = "COLUMN_META_NOT_REPRESENTABLE" + # Import: --catalog was given but has no entry for the model, so its + # columns get no types from it. + CATALOG_MODEL_MISSING = "CATALOG_MODEL_MISSING" # Import: a join targets a model that is not in the input, so the # relationship would reference an unknown dataset and is skipped. JOIN_TARGET_UNKNOWN = "JOIN_TARGET_UNKNOWN" @@ -107,6 +110,7 @@ class ConverterIssueType(Enum): ConverterIssueType.DIMENSION_TYPE_DEFAULTED: "no datatype on the field; dimension type set to string", ConverterIssueType.COLUMN_META_NOT_REPRESENTABLE: "stashed column meta other than additional_dimensions has no place on a model-file dimension; dropped", ConverterIssueType.JOIN_TARGET_UNKNOWN: "join to a model that is not in the input; skipped", + ConverterIssueType.CATALOG_MODEL_MISSING: "not in the catalog (stale, or never built); its columns get no types from it", } diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py index 2b236d70..008fccd7 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py @@ -48,7 +48,12 @@ ConverterIssueType, ConverterResult, ) -from ossie_lightdash.datatype_utils import lightdash_type_to_datatype, metric_datatype +from ossie_lightdash.catalog import Catalog, warehouse_type_to_datatype +from ossie_lightdash.datatype_utils import ( + datatype_to_lightdash_type, + lightdash_type_to_datatype, + metric_datatype, +) from ossie_lightdash.expression_utils import ( AGGREGATE_TYPES, build_aggregation, @@ -178,6 +183,8 @@ def __init__( self.issues = issues self.column_types: Dict[str, str] = {} self.column_names: List[str] = [] + # Datatypes from --catalog, consulted when a column has no authored type. + self.catalog_datatypes: Dict[str, "OssieDataType"] = {} self._expressions: Dict[str, Optional[str]] = {} self._resolving: Set[str] = set() @@ -271,6 +278,7 @@ def convert( database: Optional[str] = None, schema: Optional[str] = None, semantic_model_name: str = "lightdash_semantic_model", + catalog: Optional[Catalog] = None, ) -> ConverterResult[OssieDocument]: issues: List[ConverterIssue] = [] datasets: List[OssieDataset] = [] @@ -290,6 +298,7 @@ def convert( schema=schema, issues=issues, metric_names=metric_names, + catalog=catalog, ) datasets.append(dataset) metrics.extend(model_metrics) @@ -353,6 +362,7 @@ def _convert_model( schema: Optional[str], issues: List[ConverterIssue], metric_names: Set[str], + catalog: Optional[Catalog] = None, ) -> Tuple[OssieDataset, List[OssieMetric], List[_Edge], List[_Edge]]: name = model["name"] source = ".".join(part for part in [database, schema, name] if part) @@ -384,10 +394,30 @@ def _convert_model( context = _ModelContext(name, aliases, definitions, issues) context.column_names = [column["name"] for column in model.get("columns") or []] + if catalog is not None: + if name in catalog: + for column_name, warehouse_type in catalog[name].items(): + datatype = warehouse_type_to_datatype(warehouse_type) + if datatype is not None: + context.catalog_datatypes[column_name] = datatype + else: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.CATALOG_MODEL_MISSING, + element_name=name, + ) + ) for column in model.get("columns") or []: dimension_meta = lightdash_meta(column).get("dimension") or {} + # Authored types are intent and win; the catalog fills the gaps. if dimension_meta.get("type"): context.column_types[column["name"]] = dimension_meta["type"] + elif column["name"].lower() in context.catalog_datatypes: + lightdash_type = datatype_to_lightdash_type( + context.catalog_datatypes[column["name"].lower()] + ) + if lightdash_type is not None: + context.column_types[column["name"]] = lightdash_type fields: List[OssieField] = [] for column in model.get("columns") or []: @@ -449,6 +479,8 @@ def _convert_column( # hidden column is the closest Lightdash comes to a measure-only field. hidden = bool((dimension_meta or {}).get("hidden")) dimension: Optional[OssieDimension] = None if hidden else OssieDimension() + if dimension_meta is None: + datatype = context.catalog_datatypes.get(column_name.lower()) if dimension_meta is not None: label = dimension_meta.get("label") ai_context = _ai_context(dimension_meta.get("ai_hint")) @@ -458,6 +490,8 @@ def _convert_column( return None expression = rewritten datatype = lightdash_type_to_datatype(dimension_meta.get("type")) + if datatype is None: + datatype = context.catalog_datatypes.get(column_name.lower()) # `is_time` is a role marker in Ossie, not a type. Lightdash's only # role marker is `time_intervals: OFF`, which withdraws a temporal # column from the time axis; otherwise `is_time` is left unset so diff --git a/converters/lightdash/tests/test_cli.py b/converters/lightdash/tests/test_cli.py index ecd6f922..fca916c1 100644 --- a/converters/lightdash/tests/test_cli.py +++ b/converters/lightdash/tests/test_cli.py @@ -16,6 +16,7 @@ # under the License. """Command line round trips through both output formats.""" +import json from pathlib import Path import pytest @@ -96,3 +97,18 @@ def test_import_reads_a_whole_dbt_project(tmp_path): document = OssieDocument.model_validate(yaml.safe_load(document_path.read_text())) assert [d.name for d in document.semantic_model[0].datasets] == ["customers", "orders", "statuses"] assert document.semantic_model[0].metrics[0].name == "orders_total" + +def test_import_takes_types_from_a_dbt_catalog(tmp_path): + schema_yml = tmp_path / "schema.yml" + schema_yml.write_text("models:\n - name: races\n columns:\n - name: race_date\n - name: laps\n") + catalog = tmp_path / "catalog.json" + catalog.write_text(json.dumps({ + "nodes": { + "model.demo.races": {"columns": {"race_date": {"type": "DATE"}, "LAPS": {"type": "INT64"}}} + } + })) + out = tmp_path / "model.yaml" + assert main(["import", str(schema_yml), str(out), "--schema", "marts", "--catalog", str(catalog)]) == 0 + fields = {f["name"]: f for f in yaml.safe_load(out.read_text())["semantic_model"][0]["datasets"][0]["fields"]} + assert fields["race_date"]["datatype"] == "Date" + assert fields["laps"]["datatype"] == "Integer" diff --git a/converters/lightdash/tests/test_lightdash_to_ossie.py b/converters/lightdash/tests/test_lightdash_to_ossie.py index 06e294ff..3b06b6c2 100644 --- a/converters/lightdash/tests/test_lightdash_to_ossie.py +++ b/converters/lightdash/tests/test_lightdash_to_ossie.py @@ -866,3 +866,52 @@ def test_filters_that_other_consumers_cannot_see_are_reported(self): for issue in result.issues if issue.issue_type is ConverterIssueType.ROW_FILTER_NOT_PORTABLE ] == ["orders", "customers"] + + def test_catalog_types_fill_the_gaps_but_never_override_authored_types(self): + from ossie_lightdash.catalog import warehouse_type_to_datatype + + assert warehouse_type_to_datatype("INT64") is OssieDataType.INTEGER + assert warehouse_type_to_datatype("NUMBER(38,0)") is OssieDataType.INTEGER + assert warehouse_type_to_datatype("NUMBER(12,2)") is OssieDataType.DECIMAL + assert warehouse_type_to_datatype("NUMERIC") is OssieDataType.DECIMAL + assert warehouse_type_to_datatype("character varying(255)") is OssieDataType.STRING + assert warehouse_type_to_datatype("TIMESTAMP_TZ(9)") is OssieDataType.DATE_TIME_TZ + assert warehouse_type_to_datatype("ARRAY") is None + + schema_yml = { + "models": [ + { + "name": "results", + "columns": [ + {"name": "points", "meta": {"dimension": {"type": "string"}}}, + {"name": "position", "meta": {"dimension": {"label": "Position"}}}, + {"name": "race_date"}, + {"name": "payload"}, + {"name": "laps", "meta": {"metrics": {"total_laps": {"type": "sum"}}}}, + ], + }, + {"name": "orphan", "columns": [{"name": "id"}]}, + ] + } + catalog = { + "results": { + "points": "FLOAT64", + "position": "INT64", + "race_date": "DATE", + "payload": "STRUCT", + "laps": "INT64", + } + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts", catalog=catalog) + fields = {f.name: f for f in result.output.semantic_model[0].datasets[0].fields} + assert fields["points"].datatype is OssieDataType.STRING # authored wins + assert fields["position"].datatype is OssieDataType.INTEGER # gap filled + assert fields["race_date"].datatype is OssieDataType.DATE # no meta at all + assert fields["payload"].datatype is None # outside the vocabulary + # A catalog type also feeds the metric datatype. + assert _metric(result.output, "total_laps").datatype is OssieDataType.DECIMAL + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.CATALOG_MODEL_MISSING + ] == ["orphan"] From 93f2b9e1641c6010b3ddf40c5ea0ec959898c1d5 Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Wed, 2 Sep 2026 14:44:12 +0100 Subject: [PATCH 26/33] docs(lightdash): installation without a checkout --- converters/lightdash/README.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index 4e92ad06..f72f84f3 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -38,6 +38,20 @@ Lightdash-only attributes travel in `custom_extensions` under the registered vendor token `LIGHTDASH`, so Lightdash → Ossie → Lightdash restores the project exactly while every other consumer works from the core vocabulary. +## Installation + +The core `apache-ossie` package is not on PyPI yet, so install both from git +(no checkout needed; `pipx` works the same way): + +```bash +pip install "apache-ossie @ git+https://github.com/apache/ossie@main#subdirectory=python" +pip install "apache-ossie-lightdash @ git+https://github.com/apache/ossie@main#subdirectory=converters/lightdash" +``` + +From a checkout of this directory, `uv sync` (or `pip install -e .`) does the +same and picks up the in-repo core package. Once both packages are published, +`pip install apache-ossie-lightdash` is all that is needed. Python 3.11+. + ## Usage ``` @@ -91,8 +105,6 @@ exported = OssieToLightdashConverter(OssieDialect.BIGQUERY).convert(document) exported.output # {"version": 2, "models": [...]} (dbt-meta flavour) ``` -Requires Python 3.11+ and the in-repo `apache-ossie` package (`../../python`). - ## Mapping The table describes the dbt-meta flavour; the model-file flavour is the same From 25f9bdfe5315d8b14d3ae135c78d82a27f10537e Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Thu, 3 Sep 2026 08:26:59 +0100 Subject: [PATCH 27/33] feat(lightdash): -i/--input and -o/--output like the other converters The positional form still works. --- converters/lightdash/README.md | 11 ++++--- .../lightdash/src/ossie_lightdash/cli.py | 33 ++++++++++++++----- converters/lightdash/tests/test_cli.py | 9 +++-- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index f72f84f3..d38420ff 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -56,14 +56,14 @@ same and picks up the in-repo core package. Once both packages are published, ``` # Ossie -> a deployable Lightdash project (no dbt needed) -ossie-lightdash export semantic_model.yaml my-project --dialect BIGQUERY +ossie-lightdash export -i semantic_model.yaml -o my-project --dialect BIGQUERY cd my-project && lightdash deploy # Ossie -> one dbt schema.yml with Lightdash meta, for a dbt project -ossie-lightdash export semantic_model.yaml schema.yml --format dbt-meta --dialect BIGQUERY [--meta-under-config] +ossie-lightdash export -i semantic_model.yaml -o schema.yml --format dbt-meta --dialect BIGQUERY [--meta-under-config] # Lightdash dbt meta -> Ossie (a schema file, or a whole dbt project directory) -ossie-lightdash import path/to/dbt semantic_model.json --database analytics_db --schema marts --dialect BIGQUERY \ +ossie-lightdash import -i path/to/dbt -o semantic_model.json --database analytics_db --schema marts --dialect BIGQUERY \ --catalog path/to/dbt/target/catalog.json ``` @@ -83,8 +83,9 @@ per dataset, and a starter `my-project/lightdash.config.yml` whose Each dataset's `source` becomes the model's `sql_from` verbatim, a table reference or a query. -Issues (anything lost or approximated, see below) are printed to stderr as -`[ISSUE_TYPE] element`. +`-i/--input` and `-o/--output` follow the other converters; the two +positional forms (`export in out`) work too. Issues (anything lost or +approximated, see below) are printed to stderr as `[ISSUE_TYPE] element`. ### Python API diff --git a/converters/lightdash/src/ossie_lightdash/cli.py b/converters/lightdash/src/ossie_lightdash/cli.py index bfc51571..c79cfbbc 100644 --- a/converters/lightdash/src/ossie_lightdash/cli.py +++ b/converters/lightdash/src/ossie_lightdash/cli.py @@ -83,6 +83,22 @@ def _write_lightdash_project( ) +def _add_io_arguments(parser: argparse.ArgumentParser, input_help: str, output_help: str) -> None: + """``-i/--input`` and ``-o/--output`` like the other converters; the two + positionals are still accepted.""" + parser.add_argument("-i", "--input", dest="input_flag", metavar="INPUT", type=Path, help=input_help) + parser.add_argument("-o", "--output", dest="output_flag", metavar="OUTPUT", type=Path, help=output_help) + parser.add_argument("input", nargs="?", type=Path, help=argparse.SUPPRESS) + parser.add_argument("output", nargs="?", type=Path, help=argparse.SUPPRESS) + + +def _resolve_io(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: + args.input = args.input_flag or args.input + args.output = args.output_flag or args.output + if args.input is None or args.output is None: + parser.error("both --input and --output are required") + + def _print_issues(issues) -> None: """One line per issue, with the explanation on its first occurrence.""" explained = set() @@ -104,11 +120,10 @@ def main(argv: Optional[List[str]] = None) -> int: "export", help="Ossie document (.json/.yaml) -> Lightdash model files, or a dbt schema.yml", ) - export_parser.add_argument("input", type=Path) - export_parser.add_argument( - "output", - type=Path, - help="project directory (lightdash-yml) or schema file (dbt-meta)", + _add_io_arguments( + export_parser, + "Ossie document (.json or .yaml)", + "project directory (lightdash-yml) or schema file (dbt-meta)", ) export_parser.add_argument( "--format", @@ -139,10 +154,11 @@ def main(argv: Optional[List[str]] = None) -> int: "import", help="Lightdash dbt schema.yml, or a dbt project directory -> Ossie document (.json/.yaml)", ) - import_parser.add_argument( - "input", type=Path, help="a schema file, or a directory walked for models: and seeds:" + _add_io_arguments( + import_parser, + "a dbt schema file, or a directory walked for models: and seeds:", + "Ossie document to write (.json or .yaml)", ) - import_parser.add_argument("output", type=Path) import_parser.add_argument("--database", default=None) import_parser.add_argument("--schema", default=None) import_parser.add_argument( @@ -163,6 +179,7 @@ def main(argv: Optional[List[str]] = None) -> int: ) args = parser.parse_args(argv) + _resolve_io(export_parser if args.command == "export" else import_parser, args) if args.command == "export": dialect = OssieDialect[args.dialect] diff --git a/converters/lightdash/tests/test_cli.py b/converters/lightdash/tests/test_cli.py index fca916c1..521f5a8c 100644 --- a/converters/lightdash/tests/test_cli.py +++ b/converters/lightdash/tests/test_cli.py @@ -47,7 +47,7 @@ def test_import_writes_a_loadable_document(tmp_path, suffix): def test_export_writes_a_lightdash_project(tmp_path, capsys): project = tmp_path / "project" - assert main(["export", str(TPCDS_PATH), str(project), "--dialect", "BIGQUERY"]) == 0 + assert main(["export", "-i", str(TPCDS_PATH), "-o", str(project), "--dialect", "BIGQUERY"]) == 0 captured = capsys.readouterr() # Like the other converters: stdout stays clean, stderr carries the report. assert captured.out == "" @@ -93,7 +93,7 @@ def test_import_reads_a_whole_dbt_project(tmp_path): (project / "target" / "stale.yml").write_text("models:\n - name: stale\n") document_path = tmp_path / "model.yaml" - assert main(["import", str(project), str(document_path), "--schema", "marts"]) == 0 + assert main(["import", "--input", str(project), "--output", str(document_path), "--schema", "marts"]) == 0 document = OssieDocument.model_validate(yaml.safe_load(document_path.read_text())) assert [d.name for d in document.semantic_model[0].datasets] == ["customers", "orders", "statuses"] assert document.semantic_model[0].metrics[0].name == "orders_total" @@ -112,3 +112,8 @@ def test_import_takes_types_from_a_dbt_catalog(tmp_path): fields = {f["name"]: f for f in yaml.safe_load(out.read_text())["semantic_model"][0]["datasets"][0]["fields"]} assert fields["race_date"]["datatype"] == "Date" assert fields["laps"]["datatype"] == "Integer" + +def test_input_and_output_are_required(capsys): + with pytest.raises(SystemExit): + main(["export", "-i", str(TPCDS_PATH)]) + assert "both --input and --output are required" in capsys.readouterr().err From 4caef39f20eb03a39d514f50ac07aa62aadcb89c Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Thu, 3 Sep 2026 08:40:30 +0100 Subject: [PATCH 28/33] fix(lightdash): replace the placeholder config on the next export A config written without a warehouse type carried CHANGE_ME and was then never rewritten, so re-running with --dialect fixed the models but left lightdash compile refusing the placeholder. The config is now rewritten while it still holds the placeholder and kept once a real type is in it. --- .../lightdash/src/ossie_lightdash/cli.py | 58 +++++++++++++------ converters/lightdash/tests/test_cli.py | 11 +++- 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/converters/lightdash/src/ossie_lightdash/cli.py b/converters/lightdash/src/ossie_lightdash/cli.py index c79cfbbc..22f72a77 100644 --- a/converters/lightdash/src/ossie_lightdash/cli.py +++ b/converters/lightdash/src/ossie_lightdash/cli.py @@ -48,6 +48,24 @@ def _read_document(path: Path) -> OssieDocument: } +_PLACEHOLDER = "CHANGE_ME" + + +def _existing_warehouse_type(config: Path) -> Optional[str]: + """warehouse.type of an existing config; None when there is no config or + no readable type in it.""" + if not config.exists(): + return None + try: + document = yaml.safe_load(config.read_text(encoding="utf-8")) or {} + except yaml.YAMLError: + return None + warehouse = document.get("warehouse") if isinstance(document, dict) else None + if isinstance(warehouse, dict) and isinstance(warehouse.get("type"), str): + return warehouse["type"] + return None + + def _write_lightdash_project( models, output: Path, *, name: str, warehouse: Optional[str] ) -> None: @@ -62,25 +80,31 @@ def _write_lightdash_project( (models_dir / f"{model['name']}.yml").write_text( yaml.safe_dump(model, sort_keys=False, allow_unicode=True), encoding="utf-8" ) + # The config is rewritten while it still carries the placeholder and kept + # once a real warehouse type is in it, whether we or the user put it there. config = output / "lightdash.config.yml" - if not config.exists(): - config.write_text( - yaml.safe_dump( - { - "name": name, - "version": "1.0", - "warehouse": {"type": warehouse or "CHANGE_ME"}, - }, - sort_keys=False, - ), - encoding="utf-8", + existing = _existing_warehouse_type(config) + if existing is not None and existing != _PLACEHOLDER: + print(f"{config}: kept (warehouse.type is {existing}).", file=sys.stderr) + return + config.write_text( + yaml.safe_dump( + { + "name": name, + "version": "1.0", + "warehouse": {"type": warehouse or _PLACEHOLDER}, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + if warehouse is None: + print( + f"{config}: warehouse.type is {_PLACEHOLDER}; pass --warehouse, or a " + "--dialect Lightdash knows (BIGQUERY, SNOWFLAKE, DATABRICKS), and " + "re-run. lightdash compile will refuse the placeholder.", + file=sys.stderr, ) - if warehouse is None: - print( - "lightdash.config.yml: set warehouse.type (pass --warehouse, or a " - "--dialect Lightdash knows: BIGQUERY, SNOWFLAKE, DATABRICKS)", - file=sys.stderr, - ) def _add_io_arguments(parser: argparse.ArgumentParser, input_help: str, output_help: str) -> None: diff --git a/converters/lightdash/tests/test_cli.py b/converters/lightdash/tests/test_cli.py index 521f5a8c..0a1b806c 100644 --- a/converters/lightdash/tests/test_cli.py +++ b/converters/lightdash/tests/test_cli.py @@ -67,12 +67,21 @@ def test_export_writes_a_lightdash_project(tmp_path, capsys): config = yaml.safe_load((project / "lightdash.config.yml").read_text()) assert config["warehouse"] == {"type": "bigquery"} assert config["name"] == "tpcds_retail_model" - # A second export leaves an existing config alone. + # A config that someone has given a real warehouse type is kept ... (project / "lightdash.config.yml").write_text("name: mine\nversion: '1.0'\nwarehouse:\n type: postgres\n") assert main(["export", str(TPCDS_PATH), str(project)]) == 0 assert yaml.safe_load((project / "lightdash.config.yml").read_text())["name"] == "mine" +def test_placeholder_config_is_replaced_on_the_next_run(tmp_path): + project = tmp_path / "project" + # ... but the placeholder we wrote ourselves is not. + assert main(["export", "-i", str(TPCDS_PATH), "-o", str(project)]) == 0 + assert yaml.safe_load((project / "lightdash.config.yml").read_text())["warehouse"] == {"type": "CHANGE_ME"} + assert main(["export", "-i", str(TPCDS_PATH), "-o", str(project), "--dialect", "BIGQUERY"]) == 0 + assert yaml.safe_load((project / "lightdash.config.yml").read_text())["warehouse"] == {"type": "bigquery"} + + def test_export_dbt_meta_still_writes_one_schema_file(tmp_path): schema_yml = tmp_path / "schema.yml" assert main(["export", str(TPCDS_PATH), str(schema_yml), "--format", "dbt-meta"]) == 0 From 90dc48233c8b349b479d8545ba84dd9d806b9d62 Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Thu, 3 Sep 2026 08:58:43 +0100 Subject: [PATCH 29/33] feat(lightdash): report issues as readable blocks, --verbose for every element A real project produced hundreds of one-line issues. The report is now one block per issue type: a header with the count, the explanation, and the affected elements wrapped to a readable width (the first few, or all of them with -v/--verbose). --- .../lightdash/src/ossie_lightdash/cli.py | 46 +++++++++++++++---- converters/lightdash/tests/test_cli.py | 30 ++++++++++-- 2 files changed, 61 insertions(+), 15 deletions(-) diff --git a/converters/lightdash/src/ossie_lightdash/cli.py b/converters/lightdash/src/ossie_lightdash/cli.py index 22f72a77..cd5766aa 100644 --- a/converters/lightdash/src/ossie_lightdash/cli.py +++ b/converters/lightdash/src/ossie_lightdash/cli.py @@ -110,6 +110,7 @@ def _write_lightdash_project( def _add_io_arguments(parser: argparse.ArgumentParser, input_help: str, output_help: str) -> None: """``-i/--input`` and ``-o/--output`` like the other converters; the two positionals are still accepted.""" + parser.add_argument("-v", "--verbose", action="store_true", help="list every affected element, not just the first few per issue type") parser.add_argument("-i", "--input", dest="input_flag", metavar="INPUT", type=Path, help=input_help) parser.add_argument("-o", "--output", dest="output_flag", metavar="OUTPUT", type=Path, help=output_help) parser.add_argument("input", nargs="?", type=Path, help=argparse.SUPPRESS) @@ -123,17 +124,42 @@ def _resolve_io(parser: argparse.ArgumentParser, args: argparse.Namespace) -> No parser.error("both --input and --output are required") -def _print_issues(issues) -> None: - """One line per issue, with the explanation on its first occurrence.""" - explained = set() +_SHOWN_PER_TYPE = 8 +_WRAP = 88 + + +def _wrap(names, indent: str = " ") -> str: + lines, line = [], indent + for name in names: + piece = name + ", " + if len(line) + len(piece) > _WRAP and line.strip(): + lines.append(line.rstrip(", ")) + line = indent + line += piece + lines.append(line.rstrip(", ")) + return "\n".join(lines) + + +def _print_issues(issues, verbose: bool = False) -> None: + """A short block per issue type: header with the count, the explanation, + then the affected elements (the first few, or all of them with + ``verbose``).""" + by_type: "dict" = {} for issue in issues: - line = f"[{issue.issue_type.value}] {issue.element_name}" - if issue.issue_type not in explained: - explained.add(issue.issue_type) - line += f" -- {ISSUE_EXPLANATIONS.get(issue.issue_type, '')}".rstrip(" -") - print(line, file=sys.stderr) + by_type.setdefault(issue.issue_type, []).append(issue.element_name) + for issue_type, elements in by_type.items(): + unique = list(dict.fromkeys(elements)) + noun = "element" if len(unique) == 1 else "elements" + print(f"{issue_type.value} ({len(unique)} {noun})", file=sys.stderr) + print(f" {ISSUE_EXPLANATIONS.get(issue_type, '')}", file=sys.stderr) + shown = unique if verbose else unique[:_SHOWN_PER_TYPE] + print(_wrap(shown), file=sys.stderr) + if len(unique) > len(shown): + print(f" ... and {len(unique) - len(shown)} more", file=sys.stderr) + print(file=sys.stderr) if issues: - print(f"{len(issues)} issue(s); everything else converted cleanly.", file=sys.stderr) + hint = "" if verbose else " Pass --verbose to list every element." + print(f"{len(issues)} issue(s); everything else converted cleanly.{hint}", file=sys.stderr) def main(argv: Optional[List[str]] = None) -> int: @@ -255,7 +281,7 @@ def main(argv: Optional[List[str]] = None) -> int: ) # Issues first, then what was written, all on stderr like the other converters. - _print_issues(result.issues) + _print_issues(result.issues, verbose=args.verbose) print(summary, file=sys.stderr) return 0 diff --git a/converters/lightdash/tests/test_cli.py b/converters/lightdash/tests/test_cli.py index 0a1b806c..02da335d 100644 --- a/converters/lightdash/tests/test_cli.py +++ b/converters/lightdash/tests/test_cli.py @@ -52,11 +52,14 @@ def test_export_writes_a_lightdash_project(tmp_path, capsys): # Like the other converters: stdout stays clean, stderr carries the report. assert captured.out == "" assert captured.err.splitlines()[-1].startswith("Wrote 5 model file(s) to ") - # The one loss is named, explained once, and counted. - assert captured.err.splitlines()[:2] == [ - "[TIME_ROLE_NOT_REPRESENTABLE] d_year -- is_time on a non-date type (e.g. an integer year); " - "Lightdash has no such marker, the column is a plain dimension", - "1 issue(s); everything else converted cleanly.", + # The one loss is named, explained, and counted. + assert captured.err.splitlines()[:5] == [ + "TIME_ROLE_NOT_REPRESENTABLE (1 element)", + " is_time on a non-date type (e.g. an integer year); Lightdash has no such marker, " + "the column is a plain dimension", + " d_year", + "", + "1 issue(s); everything else converted cleanly. Pass --verbose to list every element.", ] files = sorted(p.name for p in (project / "lightdash" / "models").iterdir()) assert files == ["customer.yml", "date_dim.yml", "item.yml", "store.yml", "store_sales.yml"] @@ -126,3 +129,20 @@ def test_input_and_output_are_required(capsys): with pytest.raises(SystemExit): main(["export", "-i", str(TPCDS_PATH)]) assert "both --input and --output are required" in capsys.readouterr().err + +def test_issues_are_grouped_by_type_unless_verbose(tmp_path, capsys): + schema_yml = tmp_path / "schema.yml" + columns = "".join( + f" - name: c{i}\n meta:\n metrics:\n m{i}: {{type: sum, filters: [{{x: y}}]}}\n" + for i in range(11) + ) + schema_yml.write_text("models:\n - name: t\n columns:\n" + columns) + assert main(["import", "-i", str(schema_yml), "-o", str(tmp_path / "a.yaml"), "--schema", "s"]) == 0 + grouped = capsys.readouterr().err.splitlines() + assert grouped[0] == "METRIC_FILTER_NOT_PORTABLE (11 elements)" + assert grouped[2] == " m0, m1, m2, m3, m4, m5, m6, m7" + assert grouped[3] == " ... and 3 more" + assert main(["import", "-i", str(schema_yml), "-o", str(tmp_path / "b.yaml"), "--schema", "s", "--verbose"]) == 0 + verbose = capsys.readouterr().err.splitlines() + assert verbose[2] == " m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10" + assert "... and" not in "\n".join(verbose) From 6860cd8e65207a8daa7435b716ba0676527d2a76 Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Thu, 3 Sep 2026 09:04:30 +0100 Subject: [PATCH 30/33] fix(lightdash): skip unparseable YAML and virtualenvs when walking a project A template with placeholders or a Jinja-only file crashed the walk; it is now skipped with a note naming the file. Virtualenv directories (env, venv, .venv, site-packages) are ignored like target/ and dbt_packages/. --- .../lightdash/src/ossie_lightdash/cli.py | 6 +++-- .../src/ossie_lightdash/dbt_project.py | 26 ++++++++++++++----- converters/lightdash/tests/test_cli.py | 5 +++- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/converters/lightdash/src/ossie_lightdash/cli.py b/converters/lightdash/src/ossie_lightdash/cli.py index cd5766aa..93c8c475 100644 --- a/converters/lightdash/src/ossie_lightdash/cli.py +++ b/converters/lightdash/src/ossie_lightdash/cli.py @@ -28,7 +28,7 @@ from ossie import OssieDialect, OssieDocument from ossie_lightdash.catalog import load_catalog from ossie_lightdash.converter_issues import ISSUE_EXPLANATIONS -from ossie_lightdash.dbt_project import load_schema +from ossie_lightdash.dbt_project import load_schema_with_skips from ossie_lightdash.lightdash_to_ossie import LightdashToOssieConverter from ossie_lightdash.ossie_to_lightdash import OssieToLightdashConverter @@ -255,7 +255,9 @@ def main(argv: Optional[List[str]] = None) -> int: ) summary = f"Wrote {len(result.output['models'])} model(s) to {args.output}." else: - schema_yml = load_schema(args.input) + schema_yml, skipped = load_schema_with_skips(args.input) + for file in skipped: + print(f"Skipped {file}: not valid YAML (a template or Jinja-only file?)", file=sys.stderr) result = LightdashToOssieConverter(OssieDialect[args.dialect]).convert( schema_yml, database=args.database, diff --git a/converters/lightdash/src/ossie_lightdash/dbt_project.py b/converters/lightdash/src/ossie_lightdash/dbt_project.py index f0376741..17f1fbc0 100644 --- a/converters/lightdash/src/ossie_lightdash/dbt_project.py +++ b/converters/lightdash/src/ossie_lightdash/dbt_project.py @@ -22,16 +22,23 @@ """ from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict, List, Tuple import yaml -# Directories dbt generates; nothing in them is authored schema. -_SKIPPED_DIRS = {"target", "dbt_packages", "logs", ".git", "node_modules"} +# Directories dbt or Python tooling generate; nothing in them is authored schema. +_SKIPPED_DIRS = {"target", "dbt_packages", "logs", ".git", "node_modules", "env", "venv", ".venv", "site-packages", "__pycache__"} def load_schema(path: Path) -> Dict[str, Any]: - """Return ``{"version": 2, "models": [...], "seeds": [...]}`` for ``path``. + """Return ``{"version": 2, "models": [...], "seeds": [...]}`` for ``path``.""" + schema, _ = load_schema_with_skips(path) + return schema + + +def load_schema_with_skips(path: Path) -> Tuple[Dict[str, Any], List[Path]]: + """``load_schema`` plus the files that were skipped because they are not + valid YAML (templates with placeholders, Jinja-only files, ...). A file is read as is. A directory is walked in sorted order; every ``.yml`` / ``.yaml`` file contributes its list-valued ``models:`` and @@ -39,20 +46,25 @@ def load_schema(path: Path) -> Dict[str, Any]: is skipped by that rule), and generated directories are ignored. """ if path.is_file(): - return yaml.safe_load(path.read_text(encoding="utf-8")) or {} + return yaml.safe_load(path.read_text(encoding="utf-8")) or {}, [] models: List[Dict[str, Any]] = [] seeds: List[Dict[str, Any]] = [] + skipped: List[Path] = [] for file in sorted(path.rglob("*.y*ml")): if file.suffix not in (".yml", ".yaml"): continue if _SKIPPED_DIRS & set(file.relative_to(path).parts[:-1]): continue - document = yaml.safe_load(file.read_text(encoding="utf-8")) or {} + try: + document = yaml.safe_load(file.read_text(encoding="utf-8")) or {} + except yaml.YAMLError: + skipped.append(file) + continue if not isinstance(document, dict): continue if isinstance(document.get("models"), list): models.extend(document["models"]) if isinstance(document.get("seeds"), list): seeds.extend(document["seeds"]) - return {"version": 2, "models": models, "seeds": seeds} + return {"version": 2, "models": models, "seeds": seeds}, skipped diff --git a/converters/lightdash/tests/test_cli.py b/converters/lightdash/tests/test_cli.py index 02da335d..ca0179d5 100644 --- a/converters/lightdash/tests/test_cli.py +++ b/converters/lightdash/tests/test_cli.py @@ -90,7 +90,7 @@ def test_export_dbt_meta_still_writes_one_schema_file(tmp_path): assert main(["export", str(TPCDS_PATH), str(schema_yml), "--format", "dbt-meta"]) == 0 assert yaml.safe_load(schema_yml.read_text())["version"] == 2 -def test_import_reads_a_whole_dbt_project(tmp_path): +def test_import_reads_a_whole_dbt_project(tmp_path, capsys): project = tmp_path / "dbt" (project / "models" / "marts").mkdir(parents=True) (project / "target").mkdir() @@ -103,11 +103,14 @@ def test_import_reads_a_whole_dbt_project(tmp_path): ) (project / "data.yml").write_text("seeds:\n - name: statuses\n columns:\n - name: code\n") (project / "target" / "stale.yml").write_text("models:\n - name: stale\n") + (project / "models" / "template.yml").write_text("{project_name}:\n +materialized: view\n") document_path = tmp_path / "model.yaml" assert main(["import", "--input", str(project), "--output", str(document_path), "--schema", "marts"]) == 0 document = OssieDocument.model_validate(yaml.safe_load(document_path.read_text())) assert [d.name for d in document.semantic_model[0].datasets] == ["customers", "orders", "statuses"] + err = capsys.readouterr().err + assert "Skipped" in err and "template.yml" in err assert document.semantic_model[0].metrics[0].name == "orders_total" def test_import_takes_types_from_a_dbt_catalog(tmp_path): From f1725172a58bb3060d9226f634e22be9506277d5 Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Thu, 3 Sep 2026 09:04:50 +0100 Subject: [PATCH 31/33] fix(lightdash): a missing input path is an error, not an empty document --- converters/lightdash/src/ossie_lightdash/cli.py | 2 ++ converters/lightdash/tests/test_cli.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/converters/lightdash/src/ossie_lightdash/cli.py b/converters/lightdash/src/ossie_lightdash/cli.py index 93c8c475..927f0f1c 100644 --- a/converters/lightdash/src/ossie_lightdash/cli.py +++ b/converters/lightdash/src/ossie_lightdash/cli.py @@ -122,6 +122,8 @@ def _resolve_io(parser: argparse.ArgumentParser, args: argparse.Namespace) -> No args.output = args.output_flag or args.output if args.input is None or args.output is None: parser.error("both --input and --output are required") + if not args.input.exists(): + parser.error(f"input not found: {args.input}") _SHOWN_PER_TYPE = 8 diff --git a/converters/lightdash/tests/test_cli.py b/converters/lightdash/tests/test_cli.py index ca0179d5..a3d58192 100644 --- a/converters/lightdash/tests/test_cli.py +++ b/converters/lightdash/tests/test_cli.py @@ -149,3 +149,8 @@ def test_issues_are_grouped_by_type_unless_verbose(tmp_path, capsys): verbose = capsys.readouterr().err.splitlines() assert verbose[2] == " m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10" assert "... and" not in "\n".join(verbose) + +def test_missing_input_is_an_error(tmp_path, capsys): + with pytest.raises(SystemExit): + main(["import", "-i", str(tmp_path / "nope"), "-o", str(tmp_path / "out.yaml")]) + assert "input not found" in capsys.readouterr().err From 4b58c0dbc13b34df98efe0bae3f2568a10b41086 Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Thu, 3 Sep 2026 09:21:01 +0100 Subject: [PATCH 32/33] feat(lightdash): import Lightdash's own dbt-free model files import now reads Lightdash model files (type: model, sql_from, a dimensions list) as well as dbt schema files, mixed freely in one directory. A model file is folded into the dbt model shape: dimensions become columns, its sql_from is the dataset source verbatim (so --database/--schema are not needed for it), and the rest becomes model meta. This closes the round trip for dbt-less Lightdash projects and for anything export writes. --- converters/lightdash/README.md | 23 ++++--- .../src/ossie_lightdash/dbt_project.py | 63 +++++++++++++++++-- .../src/ossie_lightdash/lightdash_to_ossie.py | 31 ++++++--- converters/lightdash/tests/test_cli.py | 18 ++++++ .../tests/test_lightdash_to_ossie.py | 43 +++++++++++++ 5 files changed, 153 insertions(+), 25 deletions(-) diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md index d38420ff..bf181f69 100644 --- a/converters/lightdash/README.md +++ b/converters/lightdash/README.md @@ -30,9 +30,10 @@ translates between that shape and Ossie. `sql_from`, typed dimensions), which `lightdash deploy` reads as they are; `--format dbt-meta` produces one dbt `schema.yml` with Lightdash `meta` blocks instead, for projects that keep their definitions in dbt. -- **Import** (`lightdash_to_ossie`): a Lightdash-flavoured dbt `schema.yml` → - an Ossie document, for teams adopting Ossie as the source of truth for - definitions they already maintain in Lightdash. +- **Import** (`lightdash_to_ossie`): a Lightdash project → an Ossie document, + for teams adopting Ossie as the source of truth for definitions they already + maintain in Lightdash. Both Lightdash shapes are read: dbt `schema.yml` + files with Lightdash `meta`, and Lightdash's own dbt-free model files. Lightdash-only attributes travel in `custom_extensions` under the registered vendor token `LIGHTDASH`, so Lightdash → Ossie → Lightdash restores the @@ -150,11 +151,15 @@ dialect with a `DIALECT_UNAVAILABLE` issue when an expression offers neither. ## Input shape -`import` reads the `models:` and `seeds:` entries of a dbt schema file (seeds -are tables to Lightdash too), or of every YAML file under a directory: point it -at the dbt project root and it walks `models/`, `seeds/` and the rest in sorted -order, ignoring `target/`, `dbt_packages/` and `dbt_project.yml`. A join whose -target is not among them is skipped +`import` reads two shapes, mixed freely: dbt schema files (`models:` and +`seeds:` entries; seeds are tables to Lightdash too) and Lightdash model files +(`type: model`, `sql_from`, a `dimensions:` list, the format `export` writes). +Point it at a file or at a project root and it walks `models/`, `seeds/`, +`lightdash/models/` and the rest in sorted order, ignoring `target/`, +`dbt_packages/`, virtualenvs and `dbt_project.yml`. A model file's `sql_from` +(or a dbt model's `meta.sql_from`) is the dataset `source` verbatim, so +`--database` / `--schema` only apply to models that do not name their own +relation. A join whose target is not among them is skipped with a `JOIN_TARGET_UNKNOWN` issue, since Ossie relationships may only reference datasets in the document. @@ -265,8 +270,6 @@ dataset is unrestricted). Encoding metric filters as `CASE WHEN` and from the document. - Custom extensions of other vendors (`FOREIGN_EXTENSION_IGNORED`); they remain untouched in the Ossie document. -- Import reads the dbt-meta flavour only; Lightdash model files are not yet - read back. - Documents are emitted at the in-repo spec version; dbt-core 1.12's native Ossie parsing accepts `0.1.0` / `0.1.1` only. diff --git a/converters/lightdash/src/ossie_lightdash/dbt_project.py b/converters/lightdash/src/ossie_lightdash/dbt_project.py index 17f1fbc0..d3d2dbb5 100644 --- a/converters/lightdash/src/ossie_lightdash/dbt_project.py +++ b/converters/lightdash/src/ossie_lightdash/dbt_project.py @@ -14,11 +14,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Read Lightdash-flavoured dbt schema definitions from a file or a project. +"""Read Lightdash definitions from a file or a project directory. -dbt lets a project spread its ``models:`` and ``seeds:`` entries over any -number of YAML files, so ``import`` accepts a directory as well as a single -schema file and merges everything it finds. +Two shapes are understood and merged: dbt schema files (``models:`` / +``seeds:`` lists with Lightdash ``meta``) and Lightdash's own dbt-free model +files (``type: model``, ``sql_from``, a ``dimensions:`` list). The latter are +folded into the dbt shape, with ``sql_from`` kept as the model's source, so +the converter has one input model. """ from pathlib import Path @@ -26,10 +28,55 @@ import yaml +_MODEL_FILE_TYPES = {"model", "model/v1beta", "model/v1"} +_DIMENSION_OWN_KEYS = {"name", "description", "metrics", "additional_dimensions"} +_MODEL_OWN_KEYS = {"type", "name", "description", "dimensions"} + # Directories dbt or Python tooling generate; nothing in them is authored schema. _SKIPPED_DIRS = {"target", "dbt_packages", "logs", ".git", "node_modules", "env", "venv", ".venv", "site-packages", "__pycache__"} +def is_model_file(document: Any) -> bool: + """True for a Lightdash dbt-free model file (``type: model`` + dimensions).""" + return ( + isinstance(document, dict) + and document.get("type") in _MODEL_FILE_TYPES + and isinstance(document.get("name"), str) + and isinstance(document.get("dimensions"), list) + ) + + +def model_file_to_dbt_model(document: Dict[str, Any]) -> Dict[str, Any]: + """Fold a Lightdash model file into the dbt model shape the converter reads. + + Dimensions become columns whose ``meta.dimension`` carries everything but + the column-level keys; ``metrics`` and ``additional_dimensions`` stay at + column level; every other top-level key (``sql_from``, ``joins``, + ``metrics``, ``primary_key``, ``sql_filter``, ...) becomes model meta. + """ + columns: List[Dict[str, Any]] = [] + for dimension in document["dimensions"]: + if not isinstance(dimension, dict) or "name" not in dimension: + continue + column: Dict[str, Any] = {"name": dimension["name"]} + if dimension.get("description"): + column["description"] = dimension["description"] + meta: Dict[str, Any] = { + "dimension": {k: v for k, v in dimension.items() if k not in _DIMENSION_OWN_KEYS} + } + for key in ("metrics", "additional_dimensions"): + if dimension.get(key): + meta[key] = dimension[key] + column["meta"] = meta + columns.append(column) + model: Dict[str, Any] = {"name": document["name"]} + if document.get("description"): + model["description"] = document["description"] + model["meta"] = {k: v for k, v in document.items() if k not in _MODEL_OWN_KEYS} + model["columns"] = columns + return model + + def load_schema(path: Path) -> Dict[str, Any]: """Return ``{"version": 2, "models": [...], "seeds": [...]}`` for ``path``.""" schema, _ = load_schema_with_skips(path) @@ -46,7 +93,10 @@ def load_schema_with_skips(path: Path) -> Tuple[Dict[str, Any], List[Path]]: is skipped by that rule), and generated directories are ignored. """ if path.is_file(): - return yaml.safe_load(path.read_text(encoding="utf-8")) or {}, [] + document = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if is_model_file(document): + return {"version": 2, "models": [model_file_to_dbt_model(document)]}, [] + return document, [] models: List[Dict[str, Any]] = [] seeds: List[Dict[str, Any]] = [] @@ -63,6 +113,9 @@ def load_schema_with_skips(path: Path) -> Tuple[Dict[str, Any], List[Path]]: continue if not isinstance(document, dict): continue + if is_model_file(document): + models.append(model_file_to_dbt_model(document)) + continue if isinstance(document.get("models"), list): models.extend(document["models"]) if isinstance(document.get("seeds"), list): diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py index 008fccd7..37cd6e66 100644 --- a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py @@ -70,7 +70,7 @@ _STRUCTURAL_DIMENSION_KEYS = {"label", "sql", "ai_hint", "hidden"} _STRUCTURAL_JOIN_KEYS = {"join", "sql_on"} # Model meta with Ossie vocabulary; everything else is stashed on the dataset. -_HANDLED_MODEL_KEYS = {"metrics", "joins", "primary_key", "ai_hint"} +_HANDLED_MODEL_KEYS = {"metrics", "joins", "primary_key", "ai_hint", "sql_from"} _HANDLED_COLUMN_KEYS = {"dimension", "metrics"} # Model meta that changes query results, not just presentation. _ROW_FILTER_KEYS = ("sql_filter", "sql_where", "required_filters") @@ -365,16 +365,22 @@ def _convert_model( catalog: Optional[Catalog] = None, ) -> Tuple[OssieDataset, List[OssieMetric], List[_Edge], List[_Edge]]: name = model["name"] - source = ".".join(part for part in [database, schema, name] if part) - if schema is None: - issues.append( - ConverterIssue( - issue_type=ConverterIssueType.SOURCE_UNQUALIFIED, - element_name=name, + model_meta = lightdash_meta(model) + # A model that names its own relation (Lightdash model files, or + # `meta.sql_from` in dbt) is the source verbatim; otherwise the source + # is assembled from the flags. + if isinstance(model_meta.get("sql_from"), str) and model_meta["sql_from"].strip(): + source = model_meta["sql_from"].strip() + else: + source = ".".join(part for part in [database, schema, name] if part) + if schema is None: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.SOURCE_UNQUALIFIED, + element_name=name, + ) ) - ) - model_meta = lightdash_meta(model) joins = model_meta.get("joins") or [] aliases = { join["alias"]: join["join"] @@ -488,7 +494,12 @@ def _convert_column( rewritten = context.rewrite(dimension_meta["sql"], column_name) if rewritten is None: return None - expression = rewritten + # `${TABLE}.col` on column `col` is the plain column. + expression = ( + column_name + if rewritten == f"{context.dataset_name}.{column_name}" + else rewritten + ) datatype = lightdash_type_to_datatype(dimension_meta.get("type")) if datatype is None: datatype = context.catalog_datatypes.get(column_name.lower()) diff --git a/converters/lightdash/tests/test_cli.py b/converters/lightdash/tests/test_cli.py index a3d58192..5691e506 100644 --- a/converters/lightdash/tests/test_cli.py +++ b/converters/lightdash/tests/test_cli.py @@ -154,3 +154,21 @@ def test_missing_input_is_an_error(tmp_path, capsys): with pytest.raises(SystemExit): main(["import", "-i", str(tmp_path / "nope"), "-o", str(tmp_path / "out.yaml")]) assert "input not found" in capsys.readouterr().err + +def test_lightdash_model_files_import_and_round_trip(tmp_path): + project = tmp_path / "project" + assert main(["export", "-i", str(TPCDS_PATH), "-o", str(project), "--dialect", "SNOWFLAKE"]) == 0 + back = tmp_path / "back.yaml" + # No --database/--schema: the model files name their own sources. + assert main(["import", "-i", str(project), "-o", str(back), "--dialect", "SNOWFLAKE"]) == 0 + original = OssieDocument.model_validate(yaml.safe_load(TPCDS_PATH.read_text())).semantic_model[0] + roundtripped = OssieDocument.model_validate(yaml.safe_load(back.read_text())).semantic_model[0] + assert {d.name: d.source for d in roundtripped.datasets} == {d.name: d.source for d in original.datasets} + assert {d.name: d.primary_key for d in roundtripped.datasets} == {d.name: d.primary_key for d in original.datasets} + assert {(d.name, f.name): f.dimension is not None for d in roundtripped.datasets for f in d.fields} == { + (d.name, f.name): f.dimension is not None for d in original.datasets for f in d.fields + } + assert {(r.from_dataset, r.to) for r in roundtripped.relationships} == { + (r.from_dataset, r.to) for r in original.relationships + } + assert len(roundtripped.metrics) == len(original.metrics) diff --git a/converters/lightdash/tests/test_lightdash_to_ossie.py b/converters/lightdash/tests/test_lightdash_to_ossie.py index 3b06b6c2..250ca10a 100644 --- a/converters/lightdash/tests/test_lightdash_to_ossie.py +++ b/converters/lightdash/tests/test_lightdash_to_ossie.py @@ -915,3 +915,46 @@ def test_catalog_types_fill_the_gaps_but_never_override_authored_types(self): for issue in result.issues if issue.issue_type is ConverterIssueType.CATALOG_MODEL_MISSING ] == ["orphan"] + + def test_lightdash_model_file_is_read_like_dbt_meta(self): + from ossie_lightdash.dbt_project import model_file_to_dbt_model + + model_file = { + "type": "model", + "name": "orders", + "label": "Orders", + "description": "One row per order", + "sql_from": "SELECT * FROM marts.orders WHERE deleted = false", + "primary_key": "order_id", + "sql_filter": "${TABLE}.season >= 2025", + "joins": [{"join": "customers", "sql_on": "${orders.customer_id} = ${customers.customer_id}"}], + "metrics": {"aov": {"type": "number", "sql": "${total_amount} / NULLIF(${order_count}, 0)"}}, + "dimensions": [ + {"name": "order_id", "type": "number", "sql": "${TABLE}.order_id", "hidden": True, + "metrics": {"order_count": {"type": "count_distinct"}}}, + {"name": "amount", "type": "number", "sql": "${TABLE}.amount", "format": "usd", + "metrics": {"total_amount": {"type": "sum"}}}, + {"name": "customer_id", "type": "string", "sql": "${TABLE}.customer_id"}, + {"name": "amount_bucket", "type": "string", "sql": "CASE WHEN ${amount} > 100 THEN 'big' ELSE 'small' END"}, + ], + } + schema_yml = {"models": [model_file_to_dbt_model(model_file), + {"name": "customers", "columns": [{"name": "customer_id"}]}]} + result = LightdashToOssieConverter().convert(schema_yml, schema="ignored") + orders = result.output.semantic_model[0].datasets[0] + assert orders.source == "SELECT * FROM marts.orders WHERE deleted = false" + assert orders.primary_key == ["order_id"] + assert _raw_lightdash_data(orders) == {"label": "Orders", "sql_filter": "${TABLE}.season >= 2025"} + fields = {f.name: f for f in orders.fields} + assert fields["order_id"].dimension is None # hidden + assert fields["amount"].datatype is OssieDataType.DECIMAL + assert fields["amount"].expression.dialects[0].expression == "amount" # ${TABLE}.amount collapses + assert _lightdash_data(fields["amount"]) == {"type": "number", "format": "usd"} + assert fields["amount_bucket"].expression.dialects[0].expression == ( + "CASE WHEN orders.amount > 100 THEN 'big' ELSE 'small' END" + ) + assert _metric(result.output, "aov").expression.dialects[0].expression == ( + "(SUM(orders.amount)) / NULLIF((COUNT(DISTINCT orders.order_id)), 0)" + ) + assert result.output.semantic_model[0].relationships[0].to == "customers" + assert not any(i.issue_type is ConverterIssueType.SOURCE_UNQUALIFIED for i in result.issues) From 9eef6f2f0b506176b22f9cebdc0641de7df8c5d3 Mon Sep 17 00:00:00 2001 From: Lukas Spiss Date: Fri, 4 Sep 2026 10:18:02 +0100 Subject: [PATCH 33/33] chore(lightdash): remove workflow trailing whitespace --- .github/workflows/converter-lightdash-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/converter-lightdash-ci.yml b/.github/workflows/converter-lightdash-ci.yml index 462d954b..abbb4c38 100644 --- a/.github/workflows/converter-lightdash-ci.yml +++ b/.github/workflows/converter-lightdash-ci.yml @@ -24,12 +24,12 @@ on: branches: [ "main" ] paths: - 'converters/lightdash/**' - - '.github/workflows/converter-lightdash-ci.yml' + - '.github/workflows/converter-lightdash-ci.yml' pull_request: branches: [ "main" ] paths: - 'converters/lightdash/**' - - '.github/workflows/converter-lightdash-ci.yml' + - '.github/workflows/converter-lightdash-ci.yml' jobs: build: