diff --git a/.github/workflows/converter-lightdash-ci.yml b/.github/workflows/converter-lightdash-ci.yml new file mode 100644 index 00000000..abbb4c38 --- /dev/null +++ b/.github/workflows/converter-lightdash-ci.yml @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Converters Lightdash CI + +on: + push: + branches: [ "main" ] + paths: + - 'converters/lightdash/**' + - '.github/workflows/converter-lightdash-ci.yml' + pull_request: + branches: [ "main" ] + paths: + - 'converters/lightdash/**' + - '.github/workflows/converter-lightdash-ci.yml' + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" + + - name: Sync dependencies + working-directory: converters/lightdash + run: | + uv sync + + - name: Unit Tests + working-directory: converters/lightdash + run: | + uv run pytest diff --git a/converters/README.md b/converters/README.md index 417d7663..4151fea5 100644 --- a/converters/README.md +++ b/converters/README.md @@ -76,6 +76,7 @@ The Ossie specification currently defines extensions for the following vendors: | `OMNI` | Omni semantic model | | `WISDOM` | WisdomAI domain | | `NVIDIA_GSF` | NVIDIA Generative Semantic Fabric standalone YAML | +| `LIGHTDASH` | Lightdash semantic layer (dbt `meta`) | Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. diff --git a/converters/lightdash/README.md b/converters/lightdash/README.md new file mode 100644 index 00000000..bf181f69 --- /dev/null +++ b/converters/lightdash/README.md @@ -0,0 +1,301 @@ + + +# Apache Ossie <> Lightdash converter + +Bidirectional converter between Ossie documents and +[Lightdash](https://github.com/lightdash/lightdash) semantic definitions. +Lightdash reads its semantic layer from dbt `schema.yml` files: dimensions and +metrics are declared per column (and per model) under `meta`. This converter +translates between that shape and Ossie. + +- **Export** (`ossie_to_lightdash`): Ossie document → a Lightdash project. + The default output is Lightdash's own dbt-free model files (`type: model`, + `sql_from`, typed dimensions), which `lightdash deploy` reads as they are; + `--format dbt-meta` produces one dbt `schema.yml` with Lightdash `meta` + blocks instead, for projects that keep their definitions in dbt. +- **Import** (`lightdash_to_ossie`): a Lightdash project → an Ossie document, + for teams adopting Ossie as the source of truth for definitions they already + maintain in Lightdash. Both Lightdash shapes are read: dbt `schema.yml` + files with Lightdash `meta`, and Lightdash's own dbt-free model files. + +Lightdash-only attributes travel in `custom_extensions` under the registered +vendor token `LIGHTDASH`, so Lightdash → Ossie → Lightdash restores the +project exactly while every other consumer works from the core vocabulary. + +## Installation + +The core `apache-ossie` package is not on PyPI yet, so install both from git +(no checkout needed; `pipx` works the same way): + +```bash +pip install "apache-ossie @ git+https://github.com/apache/ossie@main#subdirectory=python" +pip install "apache-ossie-lightdash @ git+https://github.com/apache/ossie@main#subdirectory=converters/lightdash" +``` + +From a checkout of this directory, `uv sync` (or `pip install -e .`) does the +same and picks up the in-repo core package. Once both packages are published, +`pip install apache-ossie-lightdash` is all that is needed. Python 3.11+. + +## Usage + +``` +# Ossie -> a deployable Lightdash project (no dbt needed) +ossie-lightdash export -i semantic_model.yaml -o my-project --dialect BIGQUERY +cd my-project && lightdash deploy + +# Ossie -> one dbt schema.yml with Lightdash meta, for a dbt project +ossie-lightdash export -i semantic_model.yaml -o schema.yml --format dbt-meta --dialect BIGQUERY [--meta-under-config] + +# Lightdash dbt meta -> Ossie (a schema file, or a whole dbt project directory) +ossie-lightdash import -i path/to/dbt -o semantic_model.json --database analytics_db --schema marts --dialect BIGQUERY \ + --catalog path/to/dbt/target/catalog.json +``` + +`--catalog` takes the `catalog.json` that `dbt docs generate` writes: the +warehouse's real column types fill in `datatype` for every column without an +authored `type` (authored types win), reduced to Ossie's vocabulary +(`INT64` → `Integer`, `NUMBER(12,2)` → `Decimal`, `TIMESTAMP_TZ` → +`DateTimeTz`, ...). Lightdash learns most of its types from the warehouse +rather than from YAML, so without a catalog most fields leave untyped. A +model the catalog does not know is reported (`CATALOG_MODEL_MISSING`), which +is also how a stale catalog shows. + +The default export writes `my-project/lightdash/models/.yml`, one file +per dataset, and a starter `my-project/lightdash.config.yml` whose +`warehouse.type` is derived from `--dialect` (`BIGQUERY`, `SNOWFLAKE`, +`DATABRICKS`) or given with `--warehouse`; an existing config is left alone. +Each dataset's `source` becomes the model's `sql_from` verbatim, a table +reference or a query. + +`-i/--input` and `-o/--output` follow the other converters; the two +positional forms (`export in out`) work too. Issues (anything lost or +approximated, see below) are printed to stderr as `[ISSUE_TYPE] element`. + +### Python API + +```python +from ossie import OssieDialect +from ossie_lightdash import LightdashToOssieConverter, OssieToLightdashConverter + +result = LightdashToOssieConverter(OssieDialect.BIGQUERY).convert( + schema_yml, database="analytics_db", schema="marts" +) +result.output # OssieDocument +result.issues # [ConverterIssue(issue_type, element_name), ...] + +models = OssieToLightdashConverter(OssieDialect.BIGQUERY).convert_models(document) +models.output # [{"type": "model", "name": ..., "sql_from": ..., "dimensions": [...]}, ...] + +exported = OssieToLightdashConverter(OssieDialect.BIGQUERY).convert(document) +exported.output # {"version": 2, "models": [...]} (dbt-meta flavour) +``` + +## Mapping + +The table describes the dbt-meta flavour; the model-file flavour is the same +mapping with three differences: `dataset.source` is the model's `sql_from`, +every dimension carries its own `type` and `sql` (a field without a datatype +gets `string` with a `DIMENSION_TYPE_DEFAULTED` issue), and model meta that +the dbt flavour has to stash (`sql_filter`, `group_details`, +`default_time_dimension`, ...) are ordinary top-level keys of the model file. + +| Ossie | Lightdash (dbt meta) | +| ----- | -------------------- | +| `dataset` | dbt model (`name` = table part of `source`) | +| `dataset.source` | assembled on import from `--database` / `--schema` / model name | +| `dataset.primary_key` | model `meta.primary_key` (a single key exports as a string, a composite key as a list) | +| `ai_context` on datasets, fields and metrics | `ai_hint` on models, dimensions and metrics; a multi-line instruction is a list of hints, and the synonyms / examples of the structured form are rendered as extra hints on export | +| `field` with `dimension` | a column: every dbt column is a Lightdash dimension by default, so a dimension with nothing else to say is a plain column entry with no `meta` | +| `field` without `dimension` (measure-only) | `columns[].meta.dimension.hidden: true` — a hidden column is the closest Lightdash comes to a field that is not for grouping; it keeps its `type`, `label` and `ai_hint` | +| `field.datatype` | `meta.dimension.type` (`String`→`string`, `Integer`/`Decimal`/`Float`→`number`, `Date`→`date`, `DateTime`/`DateTimeTz`→`timestamp`, `Boolean`→`boolean`, `Time`/`Opaque`→`string`); on import `number` maps back to `Decimal` | +| `field.dimension.is_time` | `time_intervals: OFF` ↔ an explicit `is_time: false` on a temporal column; otherwise not carried — a non-temporal time axis (e.g. a year stored as `Integer`) has no Lightdash equivalent | +| `field.label` / `.description` | `meta.dimension.label` / column `description` | +| `field.expression` (≠ column name) | `meta.dimension.sql` (`dataset.col` ↔ `${TABLE}.col`) | +| `metric.name` | Lightdash scopes metric names per model, Ossie per semantic model, so the Ossie name is Lightdash's own field id `_` (`orders_total_amount`); the bare name is stashed in the extension and restored on export. An Ossie metric with no stash exports under its name minus a `_` prefix, if it has one | +| `metric.datatype` | derived on import: `Integer` for counts, `Decimal` for numeric aggregates over a `number` column, the column's type for `min`/`max`, the declared type for `boolean`/`string`/`date`/`timestamp` metrics | +| `metric` that is one aggregation over a column (`SUM(ds.col)`, `COUNT(DISTINCT ds.col)`, `SUM(DISTINCT ds.col)`, `PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY ds.col)`, ...) | column-level `meta.metrics.` with a typed metric (`sum`, `count_distinct`, `sum_distinct`, `percentile` + `percentile: 90`, ...) | +| `metric` that is one aggregation over any other expression (`AVG(CASE WHEN ds.status = 'done' THEN 1 ELSE 0 END)`) | model-level `meta.metrics.` with the typed metric and the operand as `sql` | +| `metric` with any other expression | model-level `meta.metrics.` with `type: number` + `sql`, on the dataset that joins every other dataset the expression references (`${joined_model.col}`) | +| `relationship` | `meta.joins` (`sql_on` built from / parsed into column pairs, `relationship: many-to-one` unless a stashed one says otherwise); the `alias` and other join attributes (`relationship`, `type`, `fields`, ...) travel in the relationship's `lightdash` extension, and a dataset joined more than once from the same model is aliased on export | +| a join Ossie cannot reproduce — chained through another joined model (`${projects.org_id} = ${organizations.org_id}` on the `queries` explore), an expression join (`LOWER(a) = b`), extra conditions | the column pairs still become relationships (a chained pair derives the edge between the two models it names, unless that model declares it itself); the join is stashed verbatim on the dataset's `lightdash` extension and restored on export, replacing the generated join to the same target and alias. Pair order and side order (`${T.y} = ${M.x}`) are not semantic and need no stash | +| model meta without Ossie vocabulary (`label`, `hidden`, `sql_filter`, `group_details`, `default_time_dimension`, `required_filters`, `order_fields_by`, ...) and column meta outside `dimension` / `metrics` (`additional_dimensions`, ...) | stashed on the dataset's / field's `lightdash` extension (the latter under `column_meta`) and restored on export; invisible to other consumers | +| bare column names in Lightdash SQL (`SUM(budget_use)`) | qualified with the dataset when they name one of the model's columns (outside string literals, not when called as a function); the hosting model is also stashed as `model` so an expression that names no dataset can still be placed on export | +| `${TABLE}.col`, `${col}`, `${other_model.col}` in Lightdash SQL | `dataset.col` / `other_model.col`; `${alias.col}` is flattened onto the aliased model, `${metric}` is replaced by that metric's expression | +| Lightdash presentation attributes (`label`, `format`, `round`, `compact`, `group_label`, `hidden`, ...) | `custom_extensions` with `vendor_name: LIGHTDASH` (the registered vendor token; the lowercase name of earlier documents is still read); on export the extension data is overlaid onto the generated definition (structural keys — `sql`/`label` on dimensions, `sql`/`description` on metrics and `type`/`percentile` on metrics whose expression is a recognised aggregation, `join`/`sql_on` on joins — are protected and cannot be overridden) | + +## Dialects + +Lightdash SQL is written for the project's warehouse, so `import --dialect` +labels the emitted expressions with that warehouse's Ossie dialect +(`BIGQUERY`, `SNOWFLAKE`, `DATABRICKS`); warehouses without an Ossie dialect +(Postgres, Redshift, ...) keep the default `ANSI_SQL`. `export --dialect` +prefers that dialect, falls back to `ANSI_SQL`, and takes the first available +dialect with a `DIALECT_UNAVAILABLE` issue when an expression offers neither. + +## Input shape + +`import` reads two shapes, mixed freely: dbt schema files (`models:` and +`seeds:` entries; seeds are tables to Lightdash too) and Lightdash model files +(`type: model`, `sql_from`, a `dimensions:` list, the format `export` writes). +Point it at a file or at a project root and it walks `models/`, `seeds/`, +`lightdash/models/` and the rest in sorted order, ignoring `target/`, +`dbt_packages/`, virtualenvs and `dbt_project.yml`. A model file's `sql_from` +(or a dbt model's `meta.sql_from`) is the dataset `source` verbatim, so +`--database` / `--schema` only apply to models that do not name their own +relation. A join whose target is not among them is skipped +with a `JOIN_TARGET_UNKNOWN` issue, since Ossie relationships may only +reference datasets in the document. + +## Where the meta lives + +dbt 1.10+ places `meta` under `config:`; Lightdash reads both the top-level +`meta` and `config.meta` and lets `config.meta` win, and so does the import +direction. Export writes top-level `meta` by default; pass +`--meta-under-config` to emit the `config.meta` placement instead. + +## Recommended source shape for dbt-native flows + +If the Ossie documents are also consumed by dbt's native OSI parsing, prefer +importing **without** `--database` (i.e. `schema.table` sources): the database +is usually environment-dependent in dbt projects, and a database-less source +keeps one document valid across environments (see +[dbt-core#15649](https://github.com/dbt-labs/dbt-core/issues/15649)). +Omitting `--schema` as well is reported as a `SOURCE_UNQUALIFIED` issue. + +## Fidelity and unavoidable losses + +The two models disagree in five places. Each disagreement is handled the same +way: carry what has vocabulary on both sides, keep the rest on the Lightdash +side of the document, and report whatever another consumer of the document +will not see. + +**Scope of metric names.** Lightdash scopes metric names per model, Ossie per +semantic model, and every real Lightdash project has a `count` on several +models. The Ossie name is therefore Lightdash's own field id, +`_`, always rather than only on collision: a metric's name must +not change because a metric was added to a different model, and the field id +is what Lightdash users already see in its API and URLs. The bare name is +stashed and restored, so Lightdash → Ossie → Lightdash is exact; an Ossie +document not written by Lightdash sees its metric names normalised once +(`total_sales` on `store_sales` becomes `store_sales_total_sales`). + +**What a dimension is.** Every dbt column is a Lightdash dimension unless it +is hidden; an Ossie field is a dimension only when it says so. `hidden: true` +and "no `dimension`" are the two ends of one mapping. The cost is that a +hidden column cannot carry Ossie's `is_time` role, and Lightdash has no way to +say "a dimension that is not for grouping but is not a measure either"; neither +side has such a thing today. + +**Joins versus relationships.** Ossie's relationships form one graph. A +Lightdash explore is a base model plus an explicit list of joins, which may +reach a table through another one, join on an expression, or add conditions. +The graph part becomes relationships (a chained join derives the edge between +the two models it names); the explore part is stashed verbatim on the dataset +and restored, so the explore comes back exactly while other consumers see the +graph. Because Lightdash never resolves `${other.column}` transitively, a +metric spanning datasets is hosted on the model that joins every referenced +dataset directly, and dropped with an issue when no such model exists. + +**Query-time evaluation.** Lightdash evaluates project parameters, user +attributes and Liquid templating when a query runs. No downstream consumer +can, so a dimension or metric whose SQL depends on them is skipped on import +and reported rather than shipped as SQL that is not SQL. Metric-to-metric +references are inlined for the same reason: Ossie metrics cannot reference +each other. + +**Types.** Lightdash's `number` covers Ossie's `Integer`, `Decimal` and +`Float`, and its `timestamp` covers `DateTime` and `DateTimeTz`, so datatypes +authored in YAML round-trip by category rather than by exact member. A column +with no authored `type` leaves without a datatype unless `--catalog` supplies +the warehouse's, in which case the exact member is known; Lightdash itself +learns those types from the warehouse, not from the YAML. + +### Kept for Lightdash only + +Stashed in the `LIGHTDASH` extension and restored on export (as `meta` in the +dbt flavour, as top-level keys in the model-file flavour), invisible to other +consumers: presentation attributes of dimensions and metrics (`format`, +`round`, `compact`, `groups`, `urls`, `show_underlying_values`, ...); model +meta without Ossie vocabulary (`label`, `hidden`, `group_details`, +`default_time_dimension`, `order_fields_by`, ...); column meta outside +`dimension` / `metrics` (`additional_dimensions`); join attributes (`alias`, +`type`, `fields`, ...); and joins Ossie cannot reproduce. + +Two of these change query results rather than presentation and are therefore +reported: a metric's `filters` (`METRIC_FILTER_NOT_PORTABLE` — the Ossie +expression is the unfiltered aggregate) and a model's `sql_filter` / +`sql_where` / `required_filters` (`ROW_FILTER_NOT_PORTABLE` — the Ossie +dataset is unrestricted). Encoding metric filters as `CASE WHEN` and +`sql_filter` as a query `source` is the planned fix. + +### Approximated + +- A dataset joined more than once is referenced through its first join when + an expression names it (`date_dim.year` → `${date_dim.year}`, not the + aliased second join); a `${alias.column}` reference is flattened onto the + joined dataset with an `ALIAS_REFERENCE_FLATTENED` issue. +- A name that still collides after qualification (model `orders` + metric + `x_total` versus model `orders_x` + metric `total`) is suffixed with a + `METRIC_NAME_COLLISION` issue. +- `ai_context` synonyms and examples of the structured form are rendered as + extra `ai_hint` lines on export; on import `ai_hint` becomes a plain + instruction string. +- A single-column `primary_key` exports as a string, a composite one as a list. + +### Not carried + +- `unique_keys` — Lightdash has no corresponding concept. +- `ai_context` and custom extensions on relationships. +- `dataset.name` when it differs from the source table name: the dbt model is + named after the table part of `source`. +- Relationships with mismatched `from_columns` / `to_columns` lengths + (`RELATIONSHIP_COLUMNS_MISMATCHED`), and relationships to datasets missing + from the document. +- Custom extensions of other vendors (`FOREIGN_EXTENSION_IGNORED`); they + remain untouched in the Ossie document. +- Documents are emitted at the in-repo spec version; dbt-core 1.12's native + Ossie parsing accepts `0.1.0` / `0.1.1` only. + +## Issues + +Every loss or approximation is reported as a `ConverterIssue`: on import +`JOIN_STASHED` / `JOIN_SQL_UNPARSED` (a join kept for Lightdash only), +`JOIN_TARGET_UNKNOWN`, `METRIC_FILTER_NOT_PORTABLE`, `ROW_FILTER_NOT_PORTABLE`, +`EXPRESSION_NOT_PORTABLE`, `METRIC_REFERENCE_INLINED`, +`ALIAS_REFERENCE_FLATTENED`, `METRIC_NAME_COLLISION`, `SOURCE_UNQUALIFIED`, +`METRIC_SQL_MISSING`; on export `CROSS_DATASET_METRIC_DROPPED`, +`FIELD_REFERENCE_UNJOINED`, `TIME_ROLE_NOT_REPRESENTABLE`, +`DIMENSION_TYPE_DEFAULTED`, `COLUMN_META_NOT_REPRESENTABLE`, `CATALOG_MODEL_MISSING`, `DIALECT_UNAVAILABLE`, +`RELATIONSHIP_COLUMNS_MISMATCHED`, `EXTENSION_DATA_INVALID`, +`FOREIGN_EXTENSION_IGNORED`. + +## Development + +``` +uv sync +uv run pytest +``` + +The exported TPC-DS project compiles with `lightdash compile` (5 explores, 0 +errors). Beyond the unit tests and the TPC-DS round trip, the converter is +exercised against real Lightdash projects (the public jaffle-shop demo and two +production projects on BigQuery): every model's meta and every join survive +Lightdash → Ossie → Lightdash, every uniquely named metric returns with its +expression unchanged, and the documents pass `validation/validate.py`. diff --git a/converters/lightdash/pyproject.toml b/converters/lightdash/pyproject.toml new file mode 100644 index 00000000..0d7a2090 --- /dev/null +++ b/converters/lightdash/pyproject.toml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "pytest>=8.0", +] + +[project] +name = "apache-ossie-lightdash" +version = "0.1.0.dev0" +description = "Lightdash (dbt meta semantic definitions) <> Apache Ossie converter" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] +requires-python = ">=3.11" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Lightdash" +] +dependencies = [ + "apache-ossie>=0.2.0.dev0", + "PyYAML>=6.0", +] + +[project.scripts] +ossie-lightdash = "ossie_lightdash.cli:main" + +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_lightdash"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev" +] + +[tool.uv.sources] +# apache-ossie is not yet published to PyPI; resolve it from the in-repo +# package for now. Remove this block once apache-ossie published to PyPI. +apache-ossie = { path = "../../python", editable = true} diff --git a/converters/lightdash/src/ossie_lightdash/__init__.py b/converters/lightdash/src/ossie_lightdash/__init__.py new file mode 100644 index 00000000..9e0848b9 --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/__init__.py @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from ossie_lightdash.converter_issues import ( + ConverterIssue, + ConverterIssueType, + ConverterResult, +) +from ossie_lightdash.dbt_project import load_schema +from ossie_lightdash.lightdash_to_ossie import LightdashToOssieConverter +from ossie_lightdash.ossie_to_lightdash import OssieToLightdashConverter + +__all__ = [ + "ConverterIssue", + "ConverterIssueType", + "ConverterResult", + "LightdashToOssieConverter", + "OssieToLightdashConverter", + "load_schema", +] diff --git a/converters/lightdash/src/ossie_lightdash/catalog.py b/converters/lightdash/src/ossie_lightdash/catalog.py new file mode 100644 index 00000000..d9bae66a --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/catalog.py @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Column types from a dbt ``catalog.json``. + +``dbt docs generate`` records the warehouse's real type for every column of +every model. Lightdash learns most dimension types from the warehouse rather +than from YAML, so a Lightdash project usually has few authored types; the +catalog fills the gaps with the physical type reduced to Ossie's logical +vocabulary. +""" + +import json +import re +from pathlib import Path +from typing import Dict, Optional + +from ossie import OssieDataType + +# Column types keyed by model name, then by lower-cased column name. +Catalog = Dict[str, Dict[str, str]] + +_TYPE_RE = re.compile(r"^\s*([A-Za-z_][A-Za-z_0-9 ]*?)\s*(?:\(\s*(\d+)\s*(?:,\s*(\d+))?\s*\))?\s*$") + +_EXACT = { + "INT": OssieDataType.INTEGER, + "INTEGER": OssieDataType.INTEGER, + "INT64": OssieDataType.INTEGER, + "BIGINT": OssieDataType.INTEGER, + "SMALLINT": OssieDataType.INTEGER, + "TINYINT": OssieDataType.INTEGER, + "INT2": OssieDataType.INTEGER, + "INT4": OssieDataType.INTEGER, + "INT8": OssieDataType.INTEGER, + "FLOAT": OssieDataType.FLOAT, + "FLOAT4": OssieDataType.FLOAT, + "FLOAT8": OssieDataType.FLOAT, + "FLOAT64": OssieDataType.FLOAT, + "DOUBLE": OssieDataType.FLOAT, + "DOUBLE PRECISION": OssieDataType.FLOAT, + "REAL": OssieDataType.FLOAT, + "STRING": OssieDataType.STRING, + "VARCHAR": OssieDataType.STRING, + "NVARCHAR": OssieDataType.STRING, + "CHAR": OssieDataType.STRING, + "CHARACTER": OssieDataType.STRING, + "CHARACTER VARYING": OssieDataType.STRING, + "TEXT": OssieDataType.STRING, + "BOOL": OssieDataType.BOOLEAN, + "BOOLEAN": OssieDataType.BOOLEAN, + "DATE": OssieDataType.DATE, + "DATETIME": OssieDataType.DATE_TIME, + "TIMESTAMP": OssieDataType.DATE_TIME, + "TIMESTAMP_NTZ": OssieDataType.DATE_TIME, + "TIMESTAMP WITHOUT TIME ZONE": OssieDataType.DATE_TIME, + "TIMESTAMPTZ": OssieDataType.DATE_TIME_TZ, + "TIMESTAMP_TZ": OssieDataType.DATE_TIME_TZ, + "TIMESTAMP_LTZ": OssieDataType.DATE_TIME_TZ, + "TIMESTAMP WITH TIME ZONE": OssieDataType.DATE_TIME_TZ, + "TIME": OssieDataType.TIME, +} +# Exact-decimal families: integral when the scale is 0 (or, for NUMBER, when +# Snowflake's default NUMBER(38,0) is spelled without arguments). +_DECIMAL_FAMILY = {"NUMERIC", "BIGNUMERIC", "DECIMAL", "NUMBER"} + + +def warehouse_type_to_datatype(warehouse_type: str) -> Optional[OssieDataType]: + """Reduce a physical column type to an Ossie datatype, or None when the + type is outside the portable vocabulary (arrays, structs, JSON, ...).""" + match = _TYPE_RE.match(warehouse_type or "") + if match is None: + return None + base = " ".join(match.group(1).upper().split()) + scale = match.group(3) + if base in _DECIMAL_FAMILY: + if scale is not None: + return OssieDataType.INTEGER if int(scale) == 0 else OssieDataType.DECIMAL + return OssieDataType.INTEGER if base == "NUMBER" else OssieDataType.DECIMAL + return _EXACT.get(base) + + +def load_catalog(path: Path) -> Catalog: + """Read a dbt ``catalog.json`` into ``{model: {column: type}}``. + + Models and seeds are keyed by their dbt name (the last segment of the + node's unique id); column names are lower-cased because warehouses differ + in how they report case. + """ + document = json.loads(Path(path).read_text(encoding="utf-8")) + catalog: Catalog = {} + for unique_id, node in (document.get("nodes") or {}).items(): + kind = unique_id.split(".", 1)[0] + if kind not in ("model", "seed"): + continue + name = unique_id.rsplit(".", 1)[-1] + catalog[name] = { + column_name.lower(): (column.get("type") or "") + for column_name, column in (node.get("columns") or {}).items() + } + return catalog diff --git a/converters/lightdash/src/ossie_lightdash/cli.py b/converters/lightdash/src/ossie_lightdash/cli.py new file mode 100644 index 00000000..927f0f1c --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/cli.py @@ -0,0 +1,294 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Command line interface for the Ossie <> Lightdash converter.""" + +import argparse +import json +import sys +from pathlib import Path +from typing import List, Optional + +import yaml + +from ossie import OssieDialect, OssieDocument +from ossie_lightdash.catalog import load_catalog +from ossie_lightdash.converter_issues import ISSUE_EXPLANATIONS +from ossie_lightdash.dbt_project import load_schema_with_skips +from ossie_lightdash.lightdash_to_ossie import LightdashToOssieConverter +from ossie_lightdash.ossie_to_lightdash import OssieToLightdashConverter + + +def _read_document(path: Path) -> OssieDocument: + text = path.read_text(encoding="utf-8") + if path.suffix == ".json": + return OssieDocument.model_validate_json(text) + return OssieDocument.model_validate(yaml.safe_load(text)) + + +# Ossie dialects that name a Lightdash warehouse type. +_WAREHOUSE_BY_DIALECT = { + OssieDialect.BIGQUERY: "bigquery", + OssieDialect.SNOWFLAKE: "snowflake", + OssieDialect.DATABRICKS: "databricks", +} + + +_PLACEHOLDER = "CHANGE_ME" + + +def _existing_warehouse_type(config: Path) -> Optional[str]: + """warehouse.type of an existing config; None when there is no config or + no readable type in it.""" + if not config.exists(): + return None + try: + document = yaml.safe_load(config.read_text(encoding="utf-8")) or {} + except yaml.YAMLError: + return None + warehouse = document.get("warehouse") if isinstance(document, dict) else None + if isinstance(warehouse, dict) and isinstance(warehouse.get("type"), str): + return warehouse["type"] + return None + + +def _write_lightdash_project( + models, output: Path, *, name: str, warehouse: Optional[str] +) -> None: + """Write one model file per dataset plus a starter lightdash.config.yml. + + Files go to ``/lightdash/models/.yml``, the layout + ``lightdash deploy`` looks for; an existing config is left alone. + """ + models_dir = output / "lightdash" / "models" + models_dir.mkdir(parents=True, exist_ok=True) + for model in models: + (models_dir / f"{model['name']}.yml").write_text( + yaml.safe_dump(model, sort_keys=False, allow_unicode=True), encoding="utf-8" + ) + # The config is rewritten while it still carries the placeholder and kept + # once a real warehouse type is in it, whether we or the user put it there. + config = output / "lightdash.config.yml" + existing = _existing_warehouse_type(config) + if existing is not None and existing != _PLACEHOLDER: + print(f"{config}: kept (warehouse.type is {existing}).", file=sys.stderr) + return + config.write_text( + yaml.safe_dump( + { + "name": name, + "version": "1.0", + "warehouse": {"type": warehouse or _PLACEHOLDER}, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + if warehouse is None: + print( + f"{config}: warehouse.type is {_PLACEHOLDER}; pass --warehouse, or a " + "--dialect Lightdash knows (BIGQUERY, SNOWFLAKE, DATABRICKS), and " + "re-run. lightdash compile will refuse the placeholder.", + file=sys.stderr, + ) + + +def _add_io_arguments(parser: argparse.ArgumentParser, input_help: str, output_help: str) -> None: + """``-i/--input`` and ``-o/--output`` like the other converters; the two + positionals are still accepted.""" + parser.add_argument("-v", "--verbose", action="store_true", help="list every affected element, not just the first few per issue type") + parser.add_argument("-i", "--input", dest="input_flag", metavar="INPUT", type=Path, help=input_help) + parser.add_argument("-o", "--output", dest="output_flag", metavar="OUTPUT", type=Path, help=output_help) + parser.add_argument("input", nargs="?", type=Path, help=argparse.SUPPRESS) + parser.add_argument("output", nargs="?", type=Path, help=argparse.SUPPRESS) + + +def _resolve_io(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: + args.input = args.input_flag or args.input + args.output = args.output_flag or args.output + if args.input is None or args.output is None: + parser.error("both --input and --output are required") + if not args.input.exists(): + parser.error(f"input not found: {args.input}") + + +_SHOWN_PER_TYPE = 8 +_WRAP = 88 + + +def _wrap(names, indent: str = " ") -> str: + lines, line = [], indent + for name in names: + piece = name + ", " + if len(line) + len(piece) > _WRAP and line.strip(): + lines.append(line.rstrip(", ")) + line = indent + line += piece + lines.append(line.rstrip(", ")) + return "\n".join(lines) + + +def _print_issues(issues, verbose: bool = False) -> None: + """A short block per issue type: header with the count, the explanation, + then the affected elements (the first few, or all of them with + ``verbose``).""" + by_type: "dict" = {} + for issue in issues: + by_type.setdefault(issue.issue_type, []).append(issue.element_name) + for issue_type, elements in by_type.items(): + unique = list(dict.fromkeys(elements)) + noun = "element" if len(unique) == 1 else "elements" + print(f"{issue_type.value} ({len(unique)} {noun})", file=sys.stderr) + print(f" {ISSUE_EXPLANATIONS.get(issue_type, '')}", file=sys.stderr) + shown = unique if verbose else unique[:_SHOWN_PER_TYPE] + print(_wrap(shown), file=sys.stderr) + if len(unique) > len(shown): + print(f" ... and {len(unique) - len(shown)} more", file=sys.stderr) + print(file=sys.stderr) + if issues: + hint = "" if verbose else " Pass --verbose to list every element." + print(f"{len(issues)} issue(s); everything else converted cleanly.{hint}", file=sys.stderr) + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(prog="ossie-lightdash") + subparsers = parser.add_subparsers(dest="command", required=True) + + export_parser = subparsers.add_parser( + "export", + help="Ossie document (.json/.yaml) -> Lightdash model files, or a dbt schema.yml", + ) + _add_io_arguments( + export_parser, + "Ossie document (.json or .yaml)", + "project directory (lightdash-yml) or schema file (dbt-meta)", + ) + export_parser.add_argument( + "--format", + choices=["lightdash-yml", "dbt-meta"], + default="lightdash-yml", + help="lightdash-yml: Lightdash's dbt-free model files, deployable as they are " + "(default); dbt-meta: one dbt schema.yml with Lightdash meta blocks", + ) + export_parser.add_argument( + "--warehouse", + default=None, + help="warehouse.type for the generated lightdash.config.yml " + "(default: derived from --dialect when possible)", + ) + export_parser.add_argument( + "--dialect", + choices=[dialect.name for dialect in OssieDialect], + default=OssieDialect.ANSI_SQL.name, + help="preferred expression dialect (falls back to ANSI_SQL)", + ) + export_parser.add_argument( + "--meta-under-config", + action="store_true", + help="write Lightdash meta under `config:` (dbt 1.10+) instead of top-level `meta:`", + ) + + import_parser = subparsers.add_parser( + "import", + help="Lightdash dbt schema.yml, or a dbt project directory -> Ossie document (.json/.yaml)", + ) + _add_io_arguments( + import_parser, + "a dbt schema file, or a directory walked for models: and seeds:", + "Ossie document to write (.json or .yaml)", + ) + import_parser.add_argument("--database", default=None) + import_parser.add_argument("--schema", default=None) + import_parser.add_argument( + "--semantic-model-name", default="lightdash_semantic_model" + ) + import_parser.add_argument( + "--dialect", + choices=[dialect.name for dialect in OssieDialect], + default=OssieDialect.ANSI_SQL.name, + help="dialect the Lightdash SQL is written in (the project's warehouse)", + ) + import_parser.add_argument( + "--catalog", + type=Path, + default=None, + help="dbt target/catalog.json (from `dbt docs generate`): warehouse column " + "types fill in datatypes for columns without an authored type", + ) + + args = parser.parse_args(argv) + _resolve_io(export_parser if args.command == "export" else import_parser, args) + + if args.command == "export": + dialect = OssieDialect[args.dialect] + document = _read_document(args.input) + converter = OssieToLightdashConverter(dialect, meta_under_config=args.meta_under_config) + if args.format == "lightdash-yml": + result = converter.convert_models(document) + _write_lightdash_project( + result.output, + args.output, + name=document.semantic_model[0].name if document.semantic_model else "ossie", + warehouse=args.warehouse or _WAREHOUSE_BY_DIALECT.get(dialect), + ) + summary = ( + f"Wrote {len(result.output)} model file(s) to {args.output / 'lightdash' / 'models'}" + f" and {args.output / 'lightdash.config.yml'}; run `lightdash compile` there." + ) + else: + result = converter.convert(document) + args.output.write_text( + yaml.safe_dump(result.output, sort_keys=False, allow_unicode=True), + encoding="utf-8", + ) + summary = f"Wrote {len(result.output['models'])} model(s) to {args.output}." + else: + schema_yml, skipped = load_schema_with_skips(args.input) + for file in skipped: + print(f"Skipped {file}: not valid YAML (a template or Jinja-only file?)", file=sys.stderr) + result = LightdashToOssieConverter(OssieDialect[args.dialect]).convert( + schema_yml, + database=args.database, + schema=args.schema, + semantic_model_name=args.semantic_model_name, + catalog=load_catalog(args.catalog) if args.catalog else None, + ) + semantic_model = result.output.semantic_model[0] + summary = ( + f"Wrote {len(semantic_model.datasets or [])} dataset(s), " + f"{len(semantic_model.metrics or [])} metric(s), " + f"{len(semantic_model.relationships or [])} relationship(s) to {args.output}." + ) + document = result.output.model_dump(mode="json", by_alias=True, exclude_none=True) + if args.output.suffix == ".json": + args.output.write_text( + json.dumps(document, indent=2, ensure_ascii=False), encoding="utf-8" + ) + else: + args.output.write_text( + yaml.safe_dump(document, sort_keys=False, allow_unicode=True), + encoding="utf-8", + ) + + # Issues first, then what was written, all on stderr like the other converters. + _print_issues(result.issues, verbose=args.verbose) + print(summary, file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/converters/lightdash/src/ossie_lightdash/converter_issues.py b/converters/lightdash/src/ossie_lightdash/converter_issues.py new file mode 100644 index 00000000..5536efea --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/converter_issues.py @@ -0,0 +1,133 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from dataclasses import dataclass +from enum import Enum +from typing import Generic, List, TypeVar + + +class ConverterIssueType(Enum): + """Identifies the kind of information loss that occurred during conversion.""" + + # Import: the dataset source could not be qualified with a schema/database. + SOURCE_UNQUALIFIED = "SOURCE_UNQUALIFIED" + # Import: a join's sql_on yields no column pairs (expression join); it is + # stashed on the dataset for Lightdash but has no relationship. + JOIN_SQL_UNPARSED = "JOIN_SQL_UNPARSED" + # Import: a join Ossie relationships cannot reproduce exactly (chained + # through another joined model, extra conditions) is stashed verbatim on + # the dataset; any column pairs it contains still become relationships. + JOIN_STASHED = "JOIN_STASHED" + # Export: a metric references more than one dataset, which a Lightdash + # model metric cannot express. + CROSS_DATASET_METRIC_DROPPED = "CROSS_DATASET_METRIC_DROPPED" + # Export: a relationship's from_columns/to_columns differ in length, so a + # correct sql_on cannot be built. + RELATIONSHIP_COLUMNS_MISMATCHED = "RELATIONSHIP_COLUMNS_MISMATCHED" + # Export: a `lightdash` extension whose data is not valid JSON cannot be + # applied; its presentation attributes are lost. + EXTENSION_DATA_INVALID = "EXTENSION_DATA_INVALID" + # Import: a model-level metric without `sql` has no expressible Ossie + # expression and is skipped. + METRIC_SQL_MISSING = "METRIC_SQL_MISSING" + # Export: a field is marked as a time-axis role (`dimension.is_time`) that + # its datatype does not already imply; Lightdash has no equivalent marker. + TIME_ROLE_NOT_REPRESENTABLE = "TIME_ROLE_NOT_REPRESENTABLE" + # Export: a custom extension from another vendor cannot be carried into + # Lightdash meta (it remains in the Ossie document itself). + FOREIGN_EXTENSION_IGNORED = "FOREIGN_EXTENSION_IGNORED" + # Import: an expression references project parameters or user attributes + # (`${lightdash.parameters.x}`, `${ld.user.x}`), which have no Ossie form; + # the element is skipped. + EXPRESSION_NOT_PORTABLE = "EXPRESSION_NOT_PORTABLE" + # Import: a `${metric}` reference was replaced by that metric's expression, + # since Ossie metrics cannot reference each other. + METRIC_REFERENCE_INLINED = "METRIC_REFERENCE_INLINED" + # Import: a `${alias.column}` reference to an aliased join was rewritten to + # the joined dataset; Ossie has no join aliases, so the join path is lost. + ALIAS_REFERENCE_FLATTENED = "ALIAS_REFERENCE_FLATTENED" + # Export: neither the requested dialect nor ANSI_SQL is available for an + # expression; the first available dialect is used instead. + DIALECT_UNAVAILABLE = "DIALECT_UNAVAILABLE" + # Export: a field expression references a dataset the field's dataset does + # not join, so Lightdash cannot resolve the reference. + FIELD_REFERENCE_UNJOINED = "FIELD_REFERENCE_UNJOINED" + # Import: two metrics still share a name after qualification with their + # model name; the later one is suffixed. + METRIC_NAME_COLLISION = "METRIC_NAME_COLLISION" + # Import: a metric's `filters` are kept for Lightdash only; the Ossie + # expression is unfiltered, so other consumers compute a different number. + METRIC_FILTER_NOT_PORTABLE = "METRIC_FILTER_NOT_PORTABLE" + # Import: a model's `sql_filter` / `sql_where` / `required_filters` are kept + # for Lightdash only; the Ossie dataset is unrestricted for other consumers. + ROW_FILTER_NOT_PORTABLE = "ROW_FILTER_NOT_PORTABLE" + # Export (Lightdash YAML): a dimension needs a type and the field has no + # datatype, so `string` is assumed. + DIMENSION_TYPE_DEFAULTED = "DIMENSION_TYPE_DEFAULTED" + # Export (Lightdash YAML): stashed column meta other than + # `additional_dimensions` has no place on a YAML dimension. + COLUMN_META_NOT_REPRESENTABLE = "COLUMN_META_NOT_REPRESENTABLE" + # Import: --catalog was given but has no entry for the model, so its + # columns get no types from it. + CATALOG_MODEL_MISSING = "CATALOG_MODEL_MISSING" + # Import: a join targets a model that is not in the input, so the + # relationship would reference an unknown dataset and is skipped. + JOIN_TARGET_UNKNOWN = "JOIN_TARGET_UNKNOWN" + + +# One line per issue type, for people reading the CLI output. +ISSUE_EXPLANATIONS = { + ConverterIssueType.SOURCE_UNQUALIFIED: "no --schema given, so the dataset source is just the model name", + ConverterIssueType.JOIN_SQL_UNPARSED: "no column pair in sql_on; kept for Lightdash only, no relationship", + ConverterIssueType.JOIN_STASHED: "chained join, expression join or extra conditions; kept verbatim for Lightdash, relationships derived from the column pairs", + ConverterIssueType.CROSS_DATASET_METRIC_DROPPED: "no single model joins every dataset the expression references", + ConverterIssueType.RELATIONSHIP_COLUMNS_MISMATCHED: "from_columns and to_columns differ in length; skipped", + ConverterIssueType.EXTENSION_DATA_INVALID: "LIGHTDASH extension data is not valid JSON; its attributes are lost", + ConverterIssueType.METRIC_SQL_MISSING: "model-level metric without sql; skipped", + ConverterIssueType.TIME_ROLE_NOT_REPRESENTABLE: "is_time on a non-date type (e.g. an integer year); Lightdash has no such marker, the column is a plain dimension", + ConverterIssueType.FOREIGN_EXTENSION_IGNORED: "another vendor's extension; left untouched in the Ossie document", + ConverterIssueType.EXPRESSION_NOT_PORTABLE: "SQL uses parameters, user attributes or Liquid, which only Lightdash can evaluate; skipped", + ConverterIssueType.METRIC_REFERENCE_INLINED: "${metric} reference replaced by that metric's expression", + ConverterIssueType.ALIAS_REFERENCE_FLATTENED: "${alias.column} now points at the joined model; which join was meant is lost", + ConverterIssueType.DIALECT_UNAVAILABLE: "neither --dialect nor ANSI_SQL offered; first available dialect used", + ConverterIssueType.FIELD_REFERENCE_UNJOINED: "expression names a dataset this model does not join; emitted as is", + ConverterIssueType.METRIC_NAME_COLLISION: "still a duplicate after _; suffixed", + ConverterIssueType.METRIC_FILTER_NOT_PORTABLE: "filters kept for Lightdash only; other tools see the unfiltered aggregate", + ConverterIssueType.ROW_FILTER_NOT_PORTABLE: "sql_filter / required_filters kept for Lightdash only; other tools see all rows", + ConverterIssueType.DIMENSION_TYPE_DEFAULTED: "no datatype on the field; dimension type set to string", + ConverterIssueType.COLUMN_META_NOT_REPRESENTABLE: "stashed column meta other than additional_dimensions has no place on a model-file dimension; dropped", + ConverterIssueType.JOIN_TARGET_UNKNOWN: "join to a model that is not in the input; skipped", + ConverterIssueType.CATALOG_MODEL_MISSING: "not in the catalog (stale, or never built); its columns get no types from it", +} + + +@dataclass(frozen=True) +class ConverterIssue: + """Records a single instance of information loss during conversion.""" + + issue_type: ConverterIssueType + element_name: str + + +T = TypeVar("T") + + +@dataclass(frozen=True) +class ConverterResult(Generic[T]): + """Return value of a converter's convert() method, pairing the output with any conversion issues.""" + + output: T + issues: List[ConverterIssue] diff --git a/converters/lightdash/src/ossie_lightdash/datatype_utils.py b/converters/lightdash/src/ossie_lightdash/datatype_utils.py new file mode 100644 index 00000000..bd432c18 --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/datatype_utils.py @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Translation between Ossie datatypes and Lightdash dimension types. + +Ossie carries a portable logical datatype on every field; Lightdash has a +coarser set of dimension types. The mapping is therefore lossy in one +direction: several Ossie numeric and temporal types collapse onto a single +Lightdash type, so a round-trip preserves the *category* of a datatype but not +always its exact member (e.g. ``Integer`` comes back as ``Decimal``). +""" + +from typing import Optional + +from ossie import OssieDataType + +# Ossie datatype -> Lightdash dimension type. +_DATATYPE_TO_LIGHTDASH = { + OssieDataType.STRING: "string", + OssieDataType.INTEGER: "number", + OssieDataType.DECIMAL: "number", + OssieDataType.FLOAT: "number", + OssieDataType.BOOLEAN: "boolean", + OssieDataType.DATE: "date", + OssieDataType.DATE_TIME: "timestamp", + OssieDataType.DATE_TIME_TZ: "timestamp", + # Lightdash has no time-of-day dimension type; a string keeps the value + # visible rather than dropping the column. + OssieDataType.TIME: "string", + OssieDataType.OPAQUE: "string", +} + +# Lightdash dimension type -> Ossie datatype. Numeric widths are not expressed +# in Lightdash, so `number` maps to the widest exact type. +_LIGHTDASH_TO_DATATYPE = { + "string": OssieDataType.STRING, + "number": OssieDataType.DECIMAL, + "boolean": OssieDataType.BOOLEAN, + "date": OssieDataType.DATE, + "timestamp": OssieDataType.DATE_TIME, +} + +_TEMPORAL = {OssieDataType.DATE, OssieDataType.TIME, OssieDataType.DATE_TIME, OssieDataType.DATE_TIME_TZ} + + +def datatype_to_lightdash_type(datatype: Optional[OssieDataType]) -> Optional[str]: + """Return the Lightdash dimension type for an Ossie datatype, if any.""" + if datatype is None: + return None + return _DATATYPE_TO_LIGHTDASH.get(datatype) + + +def lightdash_type_to_datatype(lightdash_type: Optional[str]) -> Optional[OssieDataType]: + """Return the Ossie datatype for a Lightdash dimension type, if any.""" + if lightdash_type is None: + return None + return _LIGHTDASH_TO_DATATYPE.get(lightdash_type) + + +_COUNT_TYPES = {"count", "count_distinct"} +_NUMERIC_AGGREGATES = { + "sum", "sum_distinct", "average", "average_distinct", "median", "percentile" +} +_ORDER_AGGREGATES = {"min", "max"} +_VALUE_TYPES = { + "boolean": OssieDataType.BOOLEAN, + "string": OssieDataType.STRING, + "date": OssieDataType.DATE, + "timestamp": OssieDataType.DATE_TIME, +} + + +def metric_datatype( + lightdash_type: str, column_type: Optional[str] +) -> Optional[OssieDataType]: + """The Ossie datatype of a Lightdash metric's result, when it is implied. + + Counts are integers; numeric aggregates over a `number` column are + decimals; min/max keep the column's type; value-typed metrics (`boolean`, + `string`, `date`, `timestamp`) declare their own type. Anything else + (`number` with arbitrary SQL, aggregates over untyped columns) is left + unset rather than guessed. + """ + if lightdash_type in _COUNT_TYPES: + return OssieDataType.INTEGER + if lightdash_type in _VALUE_TYPES: + return _VALUE_TYPES[lightdash_type] + if lightdash_type in _NUMERIC_AGGREGATES and column_type == "number": + return OssieDataType.DECIMAL + if lightdash_type in _ORDER_AGGREGATES: + return lightdash_type_to_datatype(column_type) + return None + + +def is_temporal(datatype: Optional[OssieDataType]) -> bool: + """True when the datatype represents a point in time.""" + return datatype in _TEMPORAL diff --git a/converters/lightdash/src/ossie_lightdash/dbt_project.py b/converters/lightdash/src/ossie_lightdash/dbt_project.py new file mode 100644 index 00000000..d3d2dbb5 --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/dbt_project.py @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Read Lightdash definitions from a file or a project directory. + +Two shapes are understood and merged: dbt schema files (``models:`` / +``seeds:`` lists with Lightdash ``meta``) and Lightdash's own dbt-free model +files (``type: model``, ``sql_from``, a ``dimensions:`` list). The latter are +folded into the dbt shape, with ``sql_from`` kept as the model's source, so +the converter has one input model. +""" + +from pathlib import Path +from typing import Any, Dict, List, Tuple + +import yaml + +_MODEL_FILE_TYPES = {"model", "model/v1beta", "model/v1"} +_DIMENSION_OWN_KEYS = {"name", "description", "metrics", "additional_dimensions"} +_MODEL_OWN_KEYS = {"type", "name", "description", "dimensions"} + +# Directories dbt or Python tooling generate; nothing in them is authored schema. +_SKIPPED_DIRS = {"target", "dbt_packages", "logs", ".git", "node_modules", "env", "venv", ".venv", "site-packages", "__pycache__"} + + +def is_model_file(document: Any) -> bool: + """True for a Lightdash dbt-free model file (``type: model`` + dimensions).""" + return ( + isinstance(document, dict) + and document.get("type") in _MODEL_FILE_TYPES + and isinstance(document.get("name"), str) + and isinstance(document.get("dimensions"), list) + ) + + +def model_file_to_dbt_model(document: Dict[str, Any]) -> Dict[str, Any]: + """Fold a Lightdash model file into the dbt model shape the converter reads. + + Dimensions become columns whose ``meta.dimension`` carries everything but + the column-level keys; ``metrics`` and ``additional_dimensions`` stay at + column level; every other top-level key (``sql_from``, ``joins``, + ``metrics``, ``primary_key``, ``sql_filter``, ...) becomes model meta. + """ + columns: List[Dict[str, Any]] = [] + for dimension in document["dimensions"]: + if not isinstance(dimension, dict) or "name" not in dimension: + continue + column: Dict[str, Any] = {"name": dimension["name"]} + if dimension.get("description"): + column["description"] = dimension["description"] + meta: Dict[str, Any] = { + "dimension": {k: v for k, v in dimension.items() if k not in _DIMENSION_OWN_KEYS} + } + for key in ("metrics", "additional_dimensions"): + if dimension.get(key): + meta[key] = dimension[key] + column["meta"] = meta + columns.append(column) + model: Dict[str, Any] = {"name": document["name"]} + if document.get("description"): + model["description"] = document["description"] + model["meta"] = {k: v for k, v in document.items() if k not in _MODEL_OWN_KEYS} + model["columns"] = columns + return model + + +def load_schema(path: Path) -> Dict[str, Any]: + """Return ``{"version": 2, "models": [...], "seeds": [...]}`` for ``path``.""" + schema, _ = load_schema_with_skips(path) + return schema + + +def load_schema_with_skips(path: Path) -> Tuple[Dict[str, Any], List[Path]]: + """``load_schema`` plus the files that were skipped because they are not + valid YAML (templates with placeholders, Jinja-only files, ...). + + A file is read as is. A directory is walked in sorted order; every + ``.yml`` / ``.yaml`` file contributes its list-valued ``models:`` and + ``seeds:`` entries (``dbt_project.yml`` has a dict-valued ``models:`` and + is skipped by that rule), and generated directories are ignored. + """ + if path.is_file(): + document = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if is_model_file(document): + return {"version": 2, "models": [model_file_to_dbt_model(document)]}, [] + return document, [] + + models: List[Dict[str, Any]] = [] + seeds: List[Dict[str, Any]] = [] + skipped: List[Path] = [] + for file in sorted(path.rglob("*.y*ml")): + if file.suffix not in (".yml", ".yaml"): + continue + if _SKIPPED_DIRS & set(file.relative_to(path).parts[:-1]): + continue + try: + document = yaml.safe_load(file.read_text(encoding="utf-8")) or {} + except yaml.YAMLError: + skipped.append(file) + continue + if not isinstance(document, dict): + continue + if is_model_file(document): + models.append(model_file_to_dbt_model(document)) + continue + if isinstance(document.get("models"), list): + models.extend(document["models"]) + if isinstance(document.get("seeds"), list): + seeds.extend(document["seeds"]) + return {"version": 2, "models": models, "seeds": seeds}, skipped diff --git a/converters/lightdash/src/ossie_lightdash/expression_utils.py b/converters/lightdash/src/ossie_lightdash/expression_utils.py new file mode 100644 index 00000000..09692757 --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/expression_utils.py @@ -0,0 +1,292 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Expression helpers shared by both conversion directions. + +Lightdash SQL snippets reference the current model as ``${TABLE}.column``, +sibling fields as ``${column}`` or ``${metric}``, joined models as +``${other_table.column}`` and project parameters or user attributes as +``${lightdash.…}`` / ``${ld.…}``. Ossie expressions reference columns as +``dataset.column``. These helpers translate between the two spellings and +recognise the aggregation shapes that map onto Lightdash's typed metrics. +""" + +import re +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional + +_REFERENCE_RE = re.compile(r"\$\{([^}]+)\}") +_NON_PORTABLE_REFERENCE_RE = re.compile(r"\$\{(?:lightdash|ld)\.[^}]*\}") +# Liquid tags and bare `ld.parameters.x` / `ld.query.filters` / `ld.user.x` +# references inside them are evaluated by Lightdash at query time. +_NON_PORTABLE_SYNTAX_RE = re.compile( + r"\{%|\{\{|(?[A-Za-z_]+)\s*\((?P.*)\)\s*$", re.DOTALL) +_DISTINCT_RE = re.compile(r"^DISTINCT\s+(?P.+)$", re.IGNORECASE | re.DOTALL) +_PERCENTILE_RE = re.compile( + r"^\s*PERCENTILE_CONT\s*\(\s*(?P[0-9]*\.?[0-9]+)\s*\)\s*" + r"WITHIN\s+GROUP\s*\(\s*ORDER\s+BY\s+(?P.+?)\s*\)\s*$", + re.IGNORECASE | re.DOTALL, +) + + +@dataclass(frozen=True) +class ParsedAggregation: + lightdash_type: str + inner: str + percentile: Optional[float] = None + + +def _is_single_call_body(body: str) -> bool: + """True when the parentheses in ``body`` are balanced, i.e. the outer + parentheses around it belong to one function call.""" + depth = 0 + for char in body: + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth < 0: + return False + return depth == 0 + + +def parse_aggregation(expression: str) -> Optional[ParsedAggregation]: + """Recognise an expression that is exactly one aggregation over an operand. + + ``SUM(x)``, ``COUNT(DISTINCT x)``, ``SUM(DISTINCT x)`` and + ``PERCENTILE_CONT(p) WITHIN GROUP (ORDER BY x)`` map onto Lightdash's typed + metrics; the operand may be a bare column reference or any expression. + Returns None for anything else (``COUNT(*)``, arithmetic between + aggregations, unknown functions). + """ + percentile_match = _PERCENTILE_RE.match(expression) + if percentile_match: + percentile = float(percentile_match.group("fraction")) * 100 + if percentile.is_integer(): + percentile = int(percentile) + return ParsedAggregation( + "percentile", percentile_match.group("inner").strip(), percentile + ) + + match = _CALL_RE.match(expression) + if match is None or not _is_single_call_body(match.group("body")): + return None + function = match.group("func").upper() + inner = match.group("body").strip() + distinct = _DISTINCT_RE.match(inner) + if distinct: + lightdash_type = _DISTINCT_FUNCTION_TO_TYPE.get(function) + inner = distinct.group("inner").strip() + else: + lightdash_type = _FUNCTION_TO_TYPE.get(function) + if lightdash_type is None or not inner or inner == "*": + return None + return ParsedAggregation(lightdash_type, inner) + + +def build_aggregation( + lightdash_type: str, inner: str, percentile: Optional[float] = None +) -> Optional[str]: + """Build the Ossie expression for a typed Lightdash metric over ``inner``. + + Returns None for metric types that are not aggregations (``number``, + ``string``, ``boolean``, ...). + """ + if lightdash_type == "percentile": + fraction = (50 if percentile is None else percentile) / 100 + return f"PERCENTILE_CONT({fraction:g}) WITHIN GROUP (ORDER BY {inner})" + if lightdash_type in _DISTINCT_AGGREGATE_FUNCTIONS: + return f"{_DISTINCT_AGGREGATE_FUNCTIONS[lightdash_type]}(DISTINCT {inner})" + function = _AGGREGATE_FUNCTIONS.get(lightdash_type) + if function is None: + return None + return f"{function}({inner})" + + +def is_column_reference(expression: str) -> bool: + """True for a bare ``column`` or ``qualifier.column`` reference.""" + return _COLUMN_REFERENCE_RE.match(expression) is not None + + +def strip_qualifier(column_ref: str) -> str: + """Return the bare column name of a possibly ``qualifier.column`` reference.""" + return column_ref.rsplit(".", 1)[-1] + + +def qualifier_of(column_ref: str) -> Optional[str]: + """Return the qualifier of a ``qualifier.column`` reference, if present.""" + if "." in column_ref: + return column_ref.rsplit(".", 1)[0] + return None + + +def ossie_sql_to_lightdash( + expression: str, dataset: str, references: Optional[Dict[str, str]] = None +) -> str: + """Rewrite Ossie column references into Lightdash references. + + ``dataset.column`` becomes ``${TABLE}.column``; a dataset in ``references`` + (joined from ``dataset``) becomes ``${.column}``, where the + join reference is the joined model's name or its alias. + """ + names = {dataset: None, **(references or {})} + + def replace(match: "re.Match[str]") -> str: + name, column = match.group(1), match.group(2) + reference = names[name] + if reference is None: + return f"${{TABLE}}.{column}" + return f"${{{reference}.{column}}}" + + alternatives = "|".join(re.escape(name) for name in names) + return re.sub(rf"\b({alternatives})\.(\w+)", replace, expression) + + +def has_non_portable_reference(sql: str) -> bool: + """True when the SQL depends on Lightdash query-time evaluation: project + parameters, user attributes, or Liquid templating.""" + return ( + _NON_PORTABLE_REFERENCE_RE.search(sql) is not None + or _NON_PORTABLE_SYNTAX_RE.search(sql) is not None + ) + + +@dataclass +class RewriteResult: + expression: str + inlined_metrics: List[str] = field(default_factory=list) + flattened_aliases: List[str] = field(default_factory=list) + + +def lightdash_sql_to_ossie( + sql: str, + dataset: str, + *, + aliases: Optional[Dict[str, str]] = None, + resolve_metric: Optional[Callable[[str], Optional[str]]] = None, +) -> RewriteResult: + """Rewrite Lightdash references into Ossie ``dataset.column`` references. + + ``${TABLE}.column`` and bare ``${column}`` refer to the current model; + ``${other_table.column}`` refers to a joined model and becomes a + cross-dataset reference; ``${alias.column}`` is flattened onto the joined + model; ``${metric}`` is replaced by that metric's expression when + ``resolve_metric`` knows it. Parameter and user-attribute references are + left untouched: callers check ``has_non_portable_reference`` first. + """ + alias_map = aliases or {} + result = RewriteResult(expression="") + + def replace(match: "re.Match[str]") -> str: + reference = match.group(1) + if _NON_PORTABLE_REFERENCE_RE.match(match.group(0)): + return match.group(0) + if reference == "TABLE": + return dataset + if reference.startswith("TABLE."): + return f"{dataset}.{reference[len('TABLE.'):]}" + if "." in reference: + table, column = reference.split(".", 1) + if table in alias_map: + result.flattened_aliases.append(table) + return f"{alias_map[table]}.{column}" + return reference + if resolve_metric is not None: + resolved = resolve_metric(reference) + if resolved is not None: + result.inlined_metrics.append(reference) + return f"({resolved})" + return f"{dataset}.{reference}" + + result.expression = _REFERENCE_RE.sub(replace, sql) + return result + + +_STRING_LITERAL_RE = re.compile(r"('(?:[^']|'')*')") + + +def qualify_bare_columns(sql: str, dataset: str, columns) -> str: + """Prefix bare references to the model's own columns with the dataset name. + + Lightdash lets SQL name a column of the current model without ``${TABLE}`` + (``SUM(budget_use)``); Ossie consumers need ``dataset.column``. Only + identifiers that name one of ``columns`` are touched, never inside string + literals, never when already qualified, never when followed by ``(``. + """ + lookup = {column.lower(): column for column in columns} + if not lookup: + return sql + pattern = re.compile( + r"(? str: + return f"{dataset}.{match.group(1)}" + + parts = _STRING_LITERAL_RE.split(sql) + return "".join( + part if index % 2 else pattern.sub(replace, part) + for index, part in enumerate(parts) + ) + + +def referenced_datasets(expression: str, dataset_names: set) -> set: + """Return which of the given dataset names an Ossie expression references.""" + found = set() + for match in re.finditer(r"([A-Za-z_]\w*)\.\w+", expression): + if match.group(1) in dataset_names: + found.add(match.group(1)) + return found diff --git a/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py new file mode 100644 index 00000000..37cd6e66 --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/lightdash_to_ossie.py @@ -0,0 +1,690 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Convert Lightdash semantic definitions into an Ossie document. + +The input is a dbt ``schema.yml``-shaped dictionary whose ``meta`` blocks +carry Lightdash dimensions, metrics and joins. Structural information becomes +first-class Ossie vocabulary (datasets, fields, metrics, relationships); +Lightdash presentation attributes without Ossie vocabulary (``format``, +``round``, ``group_label``, ``hidden``, ...) are preserved in +``custom_extensions`` entries with ``vendor_name: LIGHTDASH`` so that the +export direction can reproduce them exactly. +""" + +import json +import re +from typing import Any, Dict, List, Optional, Set, Tuple + +from ossie import ( + OssieCustomExtension, + OssieDataset, + OssieDialect, + OssieDialectExpression, + OssieDimension, + OssieDocument, + OssieExpression, + OssieField, + OssieMetric, + OssieRelationship, + OssieSemanticModel, +) + +from ossie_lightdash.converter_issues import ( + ConverterIssue, + ConverterIssueType, + ConverterResult, +) +from ossie_lightdash.catalog import Catalog, warehouse_type_to_datatype +from ossie_lightdash.datatype_utils import ( + datatype_to_lightdash_type, + lightdash_type_to_datatype, + metric_datatype, +) +from ossie_lightdash.expression_utils import ( + AGGREGATE_TYPES, + build_aggregation, + has_non_portable_reference, + lightdash_sql_to_ossie, + qualify_bare_columns, +) + +LIGHTDASH_VENDOR_NAME = "LIGHTDASH" + +# Keys that are structurally encoded in Ossie vocabulary and therefore must NOT +# be duplicated into the extension (a stale copy would win on export). +_STRUCTURAL_METRIC_KEYS = {"sql", "description", "ai_hint", "name", "model"} +_STRUCTURAL_DIMENSION_KEYS = {"label", "sql", "ai_hint", "hidden"} +_STRUCTURAL_JOIN_KEYS = {"join", "sql_on"} +# Model meta with Ossie vocabulary; everything else is stashed on the dataset. +_HANDLED_MODEL_KEYS = {"metrics", "joins", "primary_key", "ai_hint", "sql_from"} +_HANDLED_COLUMN_KEYS = {"dimension", "metrics"} +# Model meta that changes query results, not just presentation. +_ROW_FILTER_KEYS = ("sql_filter", "sql_where", "required_filters") + +_JOIN_PAIR_RE = re.compile( + r"\$\{(\w+)\.(\w+)\}\s*=\s*\$\{(\w+)\.(\w+)\}", +) + + +class _Edge: + """A relationship derived from a join, before de-duplication.""" + + def __init__(self, from_model, to_model, from_columns, to_columns, reference, extras): + self.from_model = from_model + self.to_model = to_model + self.from_columns = from_columns + self.to_columns = to_columns + self.reference = reference + self.extras = extras + + @property + def columns_key(self): + return (self.from_model, self.to_model, tuple(self.from_columns), tuple(self.to_columns)) + + @property + def key(self): + # An aliased join is a distinct relationship even on the same columns. + return (*self.columns_key, self.reference) + + +def _expression(expression: str, dialect: OssieDialect) -> OssieExpression: + return OssieExpression( + dialects=[OssieDialectExpression(dialect=dialect, expression=expression)] + ) + + +def _lightdash_extension(data: Dict[str, Any]) -> List[OssieCustomExtension]: + if not data: + return [] + return [ + OssieCustomExtension( + vendor_name=LIGHTDASH_VENDOR_NAME, + data=json.dumps(data, ensure_ascii=False, sort_keys=True), + ) + ] + + +def _ai_context(ai_hint: Any) -> Optional[str]: + """Lightdash `ai_hint` (a string or a list of strings) as Ossie `ai_context`.""" + if isinstance(ai_hint, list): + return "\n".join(str(hint) for hint in ai_hint) or None + if isinstance(ai_hint, str): + return ai_hint or None + return None + + +def _primary_key(primary_key: Any) -> Optional[List[str]]: + if isinstance(primary_key, str): + return [primary_key] + if isinstance(primary_key, list) and primary_key: + return [str(column) for column in primary_key] + return None + + +def _merge_meta(base: Any, override: Any) -> Dict[str, Any]: + """Deep-merge two meta blocks; keys in ``override`` win.""" + merged: Dict[str, Any] = dict(base) if isinstance(base, dict) else {} + if isinstance(override, dict): + for key, value in override.items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = _merge_meta(merged[key], value) + else: + merged[key] = value + return merged + + +def lightdash_meta(node: Dict[str, Any]) -> Dict[str, Any]: + """The Lightdash meta of a dbt model or column. + + dbt 1.10+ moved ``meta`` under ``config``; Lightdash reads both and lets + ``config.meta`` win, so the converter merges them the same way. + """ + return _merge_meta(node.get("meta"), (node.get("config") or {}).get("meta")) + + +def _unique_name(base: str, used: Set[str]) -> str: + name = base + suffix = 2 + while name in used: + name = f"{base}_{suffix}" + suffix += 1 + used.add(name) + return name + + +class _ModelContext: + """Expression rewriting for one model: alias resolution, metric inlining + and non-portable reference detection, with the issues they raise.""" + + def __init__( + self, + dataset_name: str, + aliases: Dict[str, str], + definitions: Dict[str, Tuple[Dict[str, Any], Optional[str]]], + issues: List[ConverterIssue], + ) -> None: + self.dataset_name = dataset_name + self.aliases = aliases + self.definitions = definitions + self.issues = issues + self.column_types: Dict[str, str] = {} + self.column_names: List[str] = [] + # Datatypes from --catalog, consulted when a column has no authored type. + self.catalog_datatypes: Dict[str, "OssieDataType"] = {} + self._expressions: Dict[str, Optional[str]] = {} + self._resolving: Set[str] = set() + + def rewrite(self, sql: str, element_name: str) -> Optional[str]: + """Rewrite Lightdash SQL into an Ossie expression, or None (with an + issue) when it references parameters or user attributes.""" + if has_non_portable_reference(sql): + self.issues.append( + ConverterIssue( + issue_type=ConverterIssueType.EXPRESSION_NOT_PORTABLE, + element_name=element_name, + ) + ) + return None + result = lightdash_sql_to_ossie( + sql, + self.dataset_name, + aliases=self.aliases, + resolve_metric=self.metric_expression, + ) + result.expression = qualify_bare_columns( + result.expression, self.dataset_name, self.column_names + ) + for _ in result.inlined_metrics: + self.issues.append( + ConverterIssue( + issue_type=ConverterIssueType.METRIC_REFERENCE_INLINED, + element_name=element_name, + ) + ) + for _ in result.flattened_aliases: + self.issues.append( + ConverterIssue( + issue_type=ConverterIssueType.ALIAS_REFERENCE_FLATTENED, + element_name=element_name, + ) + ) + return result.expression + + def metric_expression(self, name: str) -> Optional[str]: + """The Ossie expression of one of this model's metrics, or None when the + name is not a metric, the metric is not portable, or it references + itself.""" + if name not in self.definitions: + return None + if name in self._expressions: + return self._expressions[name] + if name in self._resolving: + return None + self._resolving.add(name) + definition, column = self.definitions[name] + expression = self._build_expression(name, definition, column) + self._resolving.discard(name) + self._expressions[name] = expression + return expression + + def _build_expression( + self, name: str, definition: Dict[str, Any], column: Optional[str] + ) -> Optional[str]: + sql = definition.get("sql") + if sql: + inner = self.rewrite(sql, name) + if inner is None: + return None + elif column is not None: + inner = f"{self.dataset_name}.{column}" + else: + return None + lightdash_type = definition.get("type", "number") + return ( + build_aggregation(lightdash_type, inner, definition.get("percentile")) + or inner + ) + + +class LightdashToOssieConverter: + """Converts a Lightdash-flavoured dbt schema.yml dict into an OssieDocument. + + Lightdash SQL is written for the project's warehouse; ``dialect`` labels the + emitted expressions accordingly (``ANSI_SQL`` when the warehouse has no + Ossie dialect, e.g. Postgres or Redshift). + """ + + def __init__(self, dialect: OssieDialect = OssieDialect.ANSI_SQL) -> None: + self._dialect = dialect + + def convert( + self, + schema_yml: Dict[str, Any], + *, + database: Optional[str] = None, + schema: Optional[str] = None, + semantic_model_name: str = "lightdash_semantic_model", + catalog: Optional[Catalog] = None, + ) -> ConverterResult[OssieDocument]: + issues: List[ConverterIssue] = [] + datasets: List[OssieDataset] = [] + metrics: List[OssieMetric] = [] + relationships: List[OssieRelationship] = [] + relationship_names: Set[str] = set() + metric_names: Set[str] = set() + direct_edges: List[_Edge] = [] + derived_edges: List[_Edge] = [] + + # Seeds are tables to Lightdash just like models. + nodes = [*(schema_yml.get("models") or []), *(schema_yml.get("seeds") or [])] + for model in nodes: + dataset, model_metrics, model_direct, model_derived = self._convert_model( + model, + database=database, + schema=schema, + issues=issues, + metric_names=metric_names, + catalog=catalog, + ) + datasets.append(dataset) + metrics.extend(model_metrics) + direct_edges.extend(model_direct) + derived_edges.extend(model_derived) + + # Edges a model declares itself come first, so an edge that a chained + # join merely passes through does not shadow the declared one. + dataset_names = {dataset.name for dataset in datasets} + seen_edges: Set[tuple] = set() + seen_columns: Set[tuple] = set() + for edge in [*direct_edges, *derived_edges]: + if edge.to_model not in dataset_names or edge.from_model not in dataset_names: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.JOIN_TARGET_UNKNOWN, + element_name=f"{edge.from_model} -> {edge.to_model}", + ) + ) + continue + derived = edge in derived_edges + # A derived edge is redundant with any declared edge on the same + # columns; a declared edge is redundant only with an identical one. + if (derived and edge.columns_key in seen_columns) or edge.key in seen_edges: + continue + seen_edges.add(edge.key) + seen_columns.add(edge.columns_key) + relationships.append( + OssieRelationship.model_validate( + { + "name": _unique_name( + f"{edge.from_model}_to_{edge.reference}", relationship_names + ), + "from": edge.from_model, + "to": edge.to_model, + "from_columns": edge.from_columns, + "to_columns": edge.to_columns, + "custom_extensions": _lightdash_extension(edge.extras) or None, + } + ) + ) + + document = OssieDocument( + version="0.2.0.dev0", + semantic_model=[ + OssieSemanticModel( + name=semantic_model_name, + datasets=datasets, + metrics=metrics or None, + relationships=relationships or None, + ) + ], + ) + return ConverterResult(output=document, issues=issues) + + def _convert_model( + self, + model: Dict[str, Any], + *, + database: Optional[str], + schema: Optional[str], + issues: List[ConverterIssue], + metric_names: Set[str], + catalog: Optional[Catalog] = None, + ) -> Tuple[OssieDataset, List[OssieMetric], List[_Edge], List[_Edge]]: + name = model["name"] + model_meta = lightdash_meta(model) + # A model that names its own relation (Lightdash model files, or + # `meta.sql_from` in dbt) is the source verbatim; otherwise the source + # is assembled from the flags. + if isinstance(model_meta.get("sql_from"), str) and model_meta["sql_from"].strip(): + source = model_meta["sql_from"].strip() + else: + source = ".".join(part for part in [database, schema, name] if part) + if schema is None: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.SOURCE_UNQUALIFIED, + element_name=name, + ) + ) + + joins = model_meta.get("joins") or [] + aliases = { + join["alias"]: join["join"] + for join in joins + if join.get("alias") and join.get("join") + } + + # Metric definitions are collected before any SQL is rewritten so that + # `${metric}` references can be inlined. + definitions: Dict[str, Tuple[Dict[str, Any], Optional[str]]] = {} + for column in model.get("columns") or []: + column_meta = lightdash_meta(column) + for metric_name, definition in (column_meta.get("metrics") or {}).items(): + definitions[metric_name] = (definition, column["name"]) + for metric_name, definition in (model_meta.get("metrics") or {}).items(): + definitions[metric_name] = (definition, None) + + context = _ModelContext(name, aliases, definitions, issues) + context.column_names = [column["name"] for column in model.get("columns") or []] + if catalog is not None: + if name in catalog: + for column_name, warehouse_type in catalog[name].items(): + datatype = warehouse_type_to_datatype(warehouse_type) + if datatype is not None: + context.catalog_datatypes[column_name] = datatype + else: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.CATALOG_MODEL_MISSING, + element_name=name, + ) + ) + for column in model.get("columns") or []: + dimension_meta = lightdash_meta(column).get("dimension") or {} + # Authored types are intent and win; the catalog fills the gaps. + if dimension_meta.get("type"): + context.column_types[column["name"]] = dimension_meta["type"] + elif column["name"].lower() in context.catalog_datatypes: + lightdash_type = datatype_to_lightdash_type( + context.catalog_datatypes[column["name"].lower()] + ) + if lightdash_type is not None: + context.column_types[column["name"]] = lightdash_type + + fields: List[OssieField] = [] + for column in model.get("columns") or []: + field = self._convert_column(column, context) + if field is not None: + fields.append(field) + + metrics: List[OssieMetric] = [] + for metric_name, (definition, column_name) in definitions.items(): + metric = self._convert_metric( + metric_name, definition, column_name, context, metric_names + ) + if metric is not None: + metrics.append(metric) + + direct_edges, derived_edges, stashed_joins = self._convert_joins( + joins, from_model=name, aliases=aliases, issues=issues + ) + + # Model meta without Ossie vocabulary, and joins Ossie cannot reproduce + # exactly, travel on the dataset so export restores the explore as is. + stash = { + key: value for key, value in model_meta.items() if key not in _HANDLED_MODEL_KEYS + } + if stashed_joins: + stash["joins"] = stashed_joins + if any(model_meta.get(key) for key in _ROW_FILTER_KEYS): + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.ROW_FILTER_NOT_PORTABLE, + element_name=name, + ) + ) + + dataset = OssieDataset( + name=name, + source=source, + description=model.get("description"), + primary_key=_primary_key(model_meta.get("primary_key")), + ai_context=_ai_context(model_meta.get("ai_hint")), + fields=fields or None, + custom_extensions=_lightdash_extension(stash) or None, + ) + return dataset, metrics, direct_edges, derived_edges + + def _convert_column( + self, column: Dict[str, Any], context: _ModelContext + ) -> Optional[OssieField]: + column_name = column["name"] + column_meta = lightdash_meta(column) + dimension_meta = column_meta.get("dimension") + + expression = column_name + datatype = None + label: Optional[str] = None + ai_context: Optional[str] = None + extension_data: Dict[str, Any] = {} + # Every dbt column is a Lightdash dimension unless it is hidden; a + # hidden column is the closest Lightdash comes to a measure-only field. + hidden = bool((dimension_meta or {}).get("hidden")) + dimension: Optional[OssieDimension] = None if hidden else OssieDimension() + if dimension_meta is None: + datatype = context.catalog_datatypes.get(column_name.lower()) + if dimension_meta is not None: + label = dimension_meta.get("label") + ai_context = _ai_context(dimension_meta.get("ai_hint")) + if dimension_meta.get("sql"): + rewritten = context.rewrite(dimension_meta["sql"], column_name) + if rewritten is None: + return None + # `${TABLE}.col` on column `col` is the plain column. + expression = ( + column_name + if rewritten == f"{context.dataset_name}.{column_name}" + else rewritten + ) + datatype = lightdash_type_to_datatype(dimension_meta.get("type")) + if datatype is None: + datatype = context.catalog_datatypes.get(column_name.lower()) + # `is_time` is a role marker in Ossie, not a type. Lightdash's only + # role marker is `time_intervals: OFF`, which withdraws a temporal + # column from the time axis; otherwise `is_time` is left unset so + # the datatype decides. + excluded = set(_STRUCTURAL_DIMENSION_KEYS) + time_intervals = dimension_meta.get("time_intervals") + if not hidden and (time_intervals is False or time_intervals == "OFF"): + dimension = OssieDimension(is_time=False) + excluded.add("time_intervals") + extension_data = { + key: value + for key, value in dimension_meta.items() + if key not in excluded + } + column_extras = { + key: value for key, value in column_meta.items() if key not in _HANDLED_COLUMN_KEYS + } + if column_extras: + extension_data["column_meta"] = column_extras + + return OssieField( + name=column_name, + expression=_expression(expression, self._dialect), + dimension=dimension, + datatype=datatype, + label=label, + description=column.get("description"), + ai_context=ai_context, + custom_extensions=_lightdash_extension(extension_data) or None, + ) + + def _convert_metric( + self, + metric_name: str, + definition: Dict[str, Any], + column: Optional[str], + context: _ModelContext, + metric_names: Set[str], + ) -> Optional[OssieMetric]: + if column is None and not definition.get("sql"): + context.issues.append( + ConverterIssue( + issue_type=ConverterIssueType.METRIC_SQL_MISSING, + element_name=metric_name, + ) + ) + return None + expression = context.metric_expression(metric_name) + if expression is None: + return None + if definition.get("filters"): + context.issues.append( + ConverterIssue( + issue_type=ConverterIssueType.METRIC_FILTER_NOT_PORTABLE, + element_name=metric_name, + ) + ) + + # Typed aggregations (and their percentile) are recovered from the + # expression on export; only types an expression cannot encode + # (`boolean`, `string`, `date`, ...) travel in the extension. + lightdash_type = definition.get("type", "number") + excluded = set(_STRUCTURAL_METRIC_KEYS) + if lightdash_type == "number" or lightdash_type in AGGREGATE_TYPES: + excluded.add("type") + if lightdash_type == "percentile": + excluded.add("percentile") + extension_data = { + key: value for key, value in definition.items() if key not in excluded + } + # Lightdash scopes metric names per model; Ossie scopes them per + # semantic model. The Ossie name is Lightdash's own field id, + # `_`, and the bare name travels in the extension so + # export restores it exactly. + extension_data["name"] = metric_name + extension_data["model"] = context.dataset_name + qualified = f"{context.dataset_name}_{metric_name}" + ossie_name = _unique_name(qualified, metric_names) + if ossie_name != qualified: + context.issues.append( + ConverterIssue( + issue_type=ConverterIssueType.METRIC_NAME_COLLISION, + element_name=metric_name, + ) + ) + return OssieMetric( + name=ossie_name, + expression=_expression(expression, self._dialect), + datatype=metric_datatype( + lightdash_type, + context.column_types.get(column) if not definition.get("sql") else None, + ), + description=definition.get("description"), + ai_context=_ai_context(definition.get("ai_hint")), + custom_extensions=_lightdash_extension(extension_data) or None, + ) + + @staticmethod + def _convert_joins( + joins: List[Dict[str, Any]], + *, + from_model: str, + aliases: Dict[str, str], + issues: List[ConverterIssue], + ) -> Tuple[List[_Edge], List[_Edge], List[Dict[str, Any]]]: + """Relationships and stashed joins for one model's explore. + + A pair ``${M.x} = ${T.y}`` on model M's join to T is a direct edge + M -> T. A pair through another model already joined in the explore, + ``${A.x} = ${T.y}``, is a chained join: the edge A -> T is derived and + the join itself is stashed verbatim, since Ossie relationships cannot + say which explore includes it. A join whose ``sql_on`` the export + direction would not rebuild identically (extra conditions, expression + joins) is stashed the same way. + """ + direct: List[_Edge] = [] + derived: List[_Edge] = [] + stashed: List[Dict[str, Any]] = [] + joined = {from_model} + for join in joins: + to_model = join.get("join") + if not to_model: + stashed.append(join) + continue + reference = join.get("alias") or to_model + joined.add(reference) + pairs = _JOIN_PAIR_RE.findall(join.get("sql_on") or "") + direct_columns: Tuple[List[str], List[str]] = ([], []) + chained: Dict[str, Tuple[List[str], List[str]]] = {} + for left_table, left_column, right_table, right_column in pairs: + if left_table == reference: + other, other_column, target_column = right_table, right_column, left_column + elif right_table == reference: + other, other_column, target_column = left_table, left_column, right_column + else: + continue + if other == from_model: + direct_columns[0].append(other_column) + direct_columns[1].append(target_column) + elif other in joined: + columns = chained.setdefault(other, ([], [])) + columns[0].append(other_column) + columns[1].append(target_column) + extras = { + key: value for key, value in join.items() if key not in _STRUCTURAL_JOIN_KEYS + } + if direct_columns[0]: + direct.append( + _Edge(from_model, to_model, *direct_columns, reference, extras) + ) + for other, (other_columns, target_columns) in chained.items(): + derived.append( + _Edge(aliases.get(other, other), to_model, other_columns, target_columns, to_model, {}) + ) + # Pair order and side order are not semantic: `${T.y} = ${M.x}` + # rebuilds as `${M.x} = ${T.y}` without needing a stash. + rebuilt_pairs = { + (from_model, left, reference, right) for left, right in zip(*direct_columns) + } + original_pairs = { + (a, b, c, d) if a == from_model else (c, d, a, b) + for a, b, c, d in pairs + } + residue = _JOIN_PAIR_RE.sub("", join.get("sql_on") or "") + reproducible = ( + original_pairs == rebuilt_pairs + and not residue.replace("AND", "").replace("and", "").strip() + ) + if not direct_columns[0] and not chained: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.JOIN_SQL_UNPARSED, + element_name=f"{from_model} -> {to_model}", + ) + ) + stashed.append(join) + elif chained or not reproducible: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.JOIN_STASHED, + element_name=f"{from_model} -> {to_model}", + ) + ) + stashed.append(join) + return direct, derived, stashed diff --git a/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py new file mode 100644 index 00000000..42466dea --- /dev/null +++ b/converters/lightdash/src/ossie_lightdash/ossie_to_lightdash.py @@ -0,0 +1,613 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Convert an Ossie document into Lightdash semantic definitions. + +``convert`` produces a dbt ``schema.yml``-shaped dictionary whose ``meta`` +blocks carry Lightdash dimensions, metrics and joins, ready to be merged into +a dbt project that Lightdash reads. ``convert_models`` produces Lightdash's +own dbt-free model files (``type: model``, ``sql_from``, typed dimensions), +which ``lightdash deploy`` reads directly. Lightdash-specific presentation attributes that +have no Ossie vocabulary round-trip through ``custom_extensions`` entries with +``vendor_name: LIGHTDASH``; their keys are overlaid onto the generated +definitions and win for presentation attributes, while structural keys +(``sql``/``label`` on dimensions, ``sql``/``description`` on metrics, +``join``/``sql_on`` on joins) are protected so they can never override the +Ossie-derived definition. +""" + +import json +from typing import Any, Dict, List, Optional, Set, Tuple + +from ossie import ( + OssieDataset, + OssieDialect, + OssieDocument, + OssieExpression, + OssieMetric, + OssieSemanticModel, +) + +from ossie_lightdash.converter_issues import ( + ConverterIssue, + ConverterIssueType, + ConverterResult, +) +from ossie_lightdash.datatype_utils import datatype_to_lightdash_type, is_temporal +from ossie_lightdash.expression_utils import ( + is_column_reference, + ossie_sql_to_lightdash, + parse_aggregation, + qualifier_of, + referenced_datasets, + strip_qualifier, +) + +LIGHTDASH_VENDOR_NAME = "LIGHTDASH" + +# Structural keys are owned by Ossie vocabulary (the import direction never puts +# them into the extension); dropping them here keeps a hand-authored extension +# from overriding the Ossie-derived definition. ``type`` stays overridable on +# metrics whose expression is not a recognised aggregation: it is the channel +# for types an expression cannot express (``boolean``, ``string``, ...). +_PROTECTED_DIMENSION_KEYS = {"sql", "label", "ai_hint", "hidden", "column_meta"} +_PROTECTED_METRIC_KEYS = {"sql", "description", "ai_hint", "name", "model"} +_PROTECTED_AGGREGATION_KEYS = _PROTECTED_METRIC_KEYS | {"type", "percentile"} +_PROTECTED_JOIN_KEYS = {"join", "sql_on", "alias"} +# Dataset-level stash keys that map to Ossie vocabulary or are handled apart. +_PROTECTED_MODEL_KEYS = {"joins", "metrics", "primary_key", "ai_hint"} +# Order of the top-level keys in a Lightdash model file; anything else +# (stashed model meta) follows in source order. +_MODEL_KEY_ORDER = ("type", "name", "label", "description", "sql_from", "primary_key", "ai_hint") +_DIMENSION_KEY_ORDER = ("name", "type", "label", "description", "sql", "hidden") + +# Joins a dataset declares, keyed by the joined dataset: the name Lightdash SQL +# uses to reference it (the joined model, or its alias). The first join to a +# dataset wins when it is joined more than once. +JoinReferences = Dict[str, str] + + +def _lightdash_extension_data(element: Any, issues: List[ConverterIssue]) -> Dict[str, Any]: + """Return the ``lightdash`` vendor extension data of an Ossie element, if any.""" + data: Dict[str, Any] = {} + for extension in element.custom_extensions or []: + # The registered token is LIGHTDASH; documents written before the + # registration used the lowercase name. + if extension.vendor_name.upper() == LIGHTDASH_VENDOR_NAME: + try: + data.update(json.loads(extension.data)) + except (TypeError, ValueError): + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.EXTENSION_DATA_INVALID, + element_name=getattr(element, "name", ""), + ) + ) + else: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.FOREIGN_EXTENSION_IGNORED, + element_name=getattr(element, "name", ""), + ) + ) + return data + + +def _ai_hint(ai_context: Any) -> Any: + """Ossie `ai_context` as a Lightdash `ai_hint`: one line stays a string, a + multi-line instruction becomes a list, and synonyms and examples of the + structured form are rendered as extra hints.""" + if ai_context is None: + return None + if isinstance(ai_context, str): + hints = ai_context.split("\n") + else: + hints = (ai_context.instructions or "").split("\n") + if ai_context.synonyms: + hints.append("Also known as: " + ", ".join(ai_context.synonyms)) + if ai_context.examples: + hints.append("Example questions: " + "; ".join(ai_context.examples)) + hints = [hint for hint in hints if hint] + if not hints: + return None + return hints[0] if len(hints) == 1 else hints + + +def _ordered(entries: Dict[str, Any], order: Tuple[str, ...]) -> Dict[str, Any]: + """Return ``entries`` with the keys in ``order`` first, the rest as they came.""" + return { + **{key: entries[key] for key in order if key in entries}, + **{key: value for key, value in entries.items() if key not in order}, + } + + +def _to_lightdash_model( + dataset: OssieDataset, model: Dict[str, Any], issues: List[ConverterIssue] +) -> Dict[str, Any]: + """Reshape a dbt-flavoured model into a Lightdash model file.""" + meta = dict(model.get("meta") or {}) + out: Dict[str, Any] = {"type": "model", "name": model["name"]} + if model.get("description"): + out["description"] = model["description"] + out["sql_from"] = dataset.source + joins = meta.pop("joins", None) + model_metrics = meta.pop("metrics", None) + out.update(meta) + if joins: + out["joins"] = joins + if model_metrics: + out["metrics"] = model_metrics + + dimensions: List[Dict[str, Any]] = [] + for column in model.get("columns") or []: + column_meta = column.get("meta") or {} + dimension = dict(column_meta.get("dimension") or {}) + dimension["name"] = column["name"] + if column.get("description") and "description" not in dimension: + dimension["description"] = column["description"] + if not dimension.get("type"): + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.DIMENSION_TYPE_DEFAULTED, + element_name=f"{model['name']}.{column['name']}", + ) + ) + dimension["type"] = "string" + if not dimension.get("sql"): + dimension["sql"] = f"${{TABLE}}.{column['name']}" + if column_meta.get("metrics"): + dimension["metrics"] = column_meta["metrics"] + if column_meta.get("additional_dimensions"): + dimension["additional_dimensions"] = column_meta["additional_dimensions"] + if any(key not in ("dimension", "metrics", "additional_dimensions") for key in column_meta): + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.COLUMN_META_NOT_REPRESENTABLE, + element_name=f"{model['name']}.{column['name']}", + ) + ) + dimensions.append(_ordered(dimension, _DIMENSION_KEY_ORDER)) + out["dimensions"] = dimensions + return _ordered(out, _MODEL_KEY_ORDER) + + +def _nest_meta_under_config(node: Dict[str, Any]) -> None: + """Move a node's ``meta`` under ``config`` (the dbt 1.10+ placement).""" + meta = node.pop("meta", None) + if meta: + node["config"] = {"meta": meta} + + +def _model_name_for(dataset: OssieDataset) -> str: + """A Lightdash table is addressed by its dbt model name = the source's table part.""" + return dataset.source.rsplit(".", 1)[-1] + + +class OssieToLightdashConverter: + """Converts an OssieDocument into a Lightdash-flavoured dbt schema.yml dict. + + ``dialect`` is the expression dialect to prefer (the project's warehouse); + ``ANSI_SQL`` is the fallback, and an expression offering neither is taken + from its first dialect with a ``DIALECT_UNAVAILABLE`` issue. + """ + + def __init__( + self, + dialect: OssieDialect = OssieDialect.ANSI_SQL, + *, + meta_under_config: bool = False, + ) -> None: + self._dialect = dialect + self._meta_under_config = meta_under_config + + def convert(self, document: OssieDocument) -> ConverterResult[Dict[str, Any]]: + """Ossie document → dbt ``schema.yml`` dict with Lightdash ``meta``.""" + issues: List[ConverterIssue] = [] + models: List[Dict[str, Any]] = [] + for semantic_model in document.semantic_model: + models.extend(model for _, model in self._convert_semantic_model(semantic_model, issues)) + if self._meta_under_config: + for model in models: + _nest_meta_under_config(model) + for column in model.get("columns") or []: + _nest_meta_under_config(column) + return ConverterResult(output={"version": 2, "models": models}, issues=issues) + + def convert_models(self, document: OssieDocument) -> ConverterResult[List[Dict[str, Any]]]: + """Ossie document → Lightdash model files (one dict per dataset). + + The dbt-free format has a home for everything the dbt flavour has to + stash: ``sql_from`` is the dataset's ``source`` verbatim, model meta + becomes top-level keys, and every dimension carries its own ``type`` + and ``sql``. + """ + issues: List[ConverterIssue] = [] + models: List[Dict[str, Any]] = [] + for semantic_model in document.semantic_model: + for dataset, model in self._convert_semantic_model(semantic_model, issues): + models.append(_to_lightdash_model(dataset, model, issues)) + return ConverterResult(output=models, issues=issues) + + def _pick_expression( + self, expression: OssieExpression, element_name: str, issues: List[ConverterIssue] + ) -> str: + by_dialect = { + dialect_expression.dialect: dialect_expression.expression + for dialect_expression in expression.dialects + } + for dialect in (self._dialect, OssieDialect.ANSI_SQL): + if dialect in by_dialect: + return by_dialect[dialect] + if not expression.dialects: + return "" + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.DIALECT_UNAVAILABLE, + element_name=element_name, + ) + ) + return expression.dialects[0].expression + + def _convert_semantic_model( + self, semantic_model: OssieSemanticModel, issues: List[ConverterIssue] + ) -> List[Tuple[OssieDataset, Dict[str, Any]]]: + datasets = semantic_model.datasets or [] + dataset_names = {dataset.name for dataset in datasets} + model_name_by_dataset = { + dataset.name: _model_name_for(dataset) for dataset in datasets + } + + # Joins are planned first: field and metric expressions may reference + # other datasets only through the joins their own dataset declares. + joins_by_dataset, references_by_dataset = self._plan_joins( + semantic_model, model_name_by_dataset, issues + ) + + models_by_dataset: Dict[str, Dict[str, Any]] = {} + columns_by_dataset: Dict[str, Dict[str, Dict[str, Any]]] = {} + stash_by_dataset: Dict[str, Dict[str, Any]] = {} + for dataset in datasets: + stash = _lightdash_extension_data(dataset, issues) + stash_by_dataset[dataset.name] = stash + # A stashed join (chained, expression, extra conditions) is + # restored verbatim, replacing the generated join to the same + # target and alias, and its target becomes referenceable. + joins = joins_by_dataset.setdefault(dataset.name, []) + references = references_by_dataset.setdefault(dataset.name, {}) + for stashed_join in stash.get("joins") or []: + if not isinstance(stashed_join, dict) or not stashed_join.get("join"): + continue + key = (stashed_join.get("join"), stashed_join.get("alias")) + for index, join in enumerate(joins): + if (join.get("join"), join.get("alias")) == key: + joins[index] = stashed_join + break + else: + joins.append(stashed_join) + references.setdefault( + stashed_join["join"], stashed_join.get("alias") or stashed_join["join"] + ) + if not joins: + del joins_by_dataset[dataset.name] + + for dataset in datasets: + model, columns = self._convert_dataset( + dataset, dataset_names, references_by_dataset.get(dataset.name, {}), issues + ) + meta = model.setdefault("meta", {}) + meta.update( + { + key: value + for key, value in stash_by_dataset[dataset.name].items() + if key not in _PROTECTED_MODEL_KEYS + } + ) + if not meta: + del model["meta"] + models_by_dataset[dataset.name] = model + columns_by_dataset[dataset.name] = columns + + for metric in semantic_model.metrics or []: + self._convert_metric( + metric, + [dataset.name for dataset in datasets], + models_by_dataset, + columns_by_dataset, + references_by_dataset, + issues, + ) + + for dataset_name, joins in joins_by_dataset.items(): + models_by_dataset[dataset_name].setdefault("meta", {})["joins"] = joins + + return [(dataset, models_by_dataset[dataset.name]) for dataset in datasets] + + def _plan_joins( + self, + semantic_model: OssieSemanticModel, + model_name_by_dataset: Dict[str, str], + issues: List[ConverterIssue], + ) -> Tuple[Dict[str, List[Dict[str, Any]]], Dict[str, JoinReferences]]: + joins_by_dataset: Dict[str, List[Dict[str, Any]]] = {} + references_by_dataset: Dict[str, JoinReferences] = {} + joined_pairs: Set[Tuple[str, str]] = set() + for relationship in semantic_model.relationships or []: + from_model_name = model_name_by_dataset.get(relationship.from_dataset) + to_model_name = model_name_by_dataset.get(relationship.to) + if from_model_name is None or to_model_name is None: + continue + if len(relationship.from_columns) != len(relationship.to_columns): + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.RELATIONSHIP_COLUMNS_MISMATCHED, + element_name=relationship.name, + ) + ) + continue + extension_data = _lightdash_extension_data(relationship, issues) + # Lightdash refuses to join the same table twice without an alias; + # a stashed alias is restored, otherwise the relationship name + # aliases every repeat of a dataset pair. + pair = (relationship.from_dataset, relationship.to) + alias = extension_data.get("alias") + if alias is None and pair in joined_pairs: + alias = relationship.name + joined_pairs.add(pair) + join_reference = alias or to_model_name + sql_on = " AND ".join( + f"${{{from_model_name}.{from_column}}} = ${{{join_reference}.{to_column}}}" + for from_column, to_column in zip( + relationship.from_columns, relationship.to_columns + ) + ) + # An Ossie relationship always runs from the many side to the one + # side; a stashed Lightdash `relationship` overrides it below. + join: Dict[str, Any] = {"join": to_model_name} + if alias: + join["alias"] = alias + join["sql_on"] = sql_on + join["relationship"] = "many-to-one" + join.update( + { + key: value + for key, value in extension_data.items() + if key not in _PROTECTED_JOIN_KEYS + } + ) + joins_by_dataset.setdefault(relationship.from_dataset, []).append(join) + references_by_dataset.setdefault(relationship.from_dataset, {}).setdefault( + relationship.to, join_reference + ) + return joins_by_dataset, references_by_dataset + + def _convert_dataset( + self, + dataset: OssieDataset, + dataset_names: Set[str], + references: JoinReferences, + issues: List[ConverterIssue], + ) -> tuple: + columns_by_name: Dict[str, Dict[str, Any]] = {} + for field in dataset.fields or []: + column: Dict[str, Any] = {"name": field.name} + if field.description: + column["description"] = field.description + + dimension: Dict[str, Any] = {} + if field.dimension is None: + # Every Lightdash column is a dimension; a measure-only Ossie + # field is a hidden one. + dimension["hidden"] = True + if field.label: + dimension["label"] = field.label + ai_hint = _ai_hint(field.ai_context) + if ai_hint is not None: + dimension["ai_hint"] = ai_hint + lightdash_type = datatype_to_lightdash_type(field.datatype) + if lightdash_type is not None: + dimension["type"] = lightdash_type + elif field.dimension is not None and field.dimension.is_time: + # No datatype to translate, but the field is declared as a + # time axis: `date` is the closest Lightdash type. + dimension["type"] = "date" + if ( + field.dimension is not None + and field.dimension.is_time + and not is_temporal(field.datatype) + and field.datatype is not None + ): + # A non-temporal datatype flagged as a time axis (e.g. a year + # stored as Integer) has no Lightdash equivalent. + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.TIME_ROLE_NOT_REPRESENTABLE, + element_name=field.name, + ) + ) + if ( + field.dimension is not None + and field.dimension.is_time is False + and is_temporal(field.datatype) + ): + # Explicitly withdrawn from the time axis: Lightdash's role + # marker for that is `time_intervals: OFF`. + dimension["time_intervals"] = "OFF" + expression = self._pick_expression(field.expression, field.name, issues) + if expression and expression != field.name: + unjoined = referenced_datasets(expression, dataset_names) - { + dataset.name, + *references, + } + if unjoined: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.FIELD_REFERENCE_UNJOINED, + element_name=field.name, + ) + ) + dimension["sql"] = ossie_sql_to_lightdash( + expression, dataset.name, references + ) + field_data = _lightdash_extension_data(field, issues) + dimension.update( + { + key: value + for key, value in field_data.items() + if key not in _PROTECTED_DIMENSION_KEYS + } + ) + # A dimension with nothing to say is Lightdash's default: no meta. + if dimension: + column["meta"] = {"dimension": dimension} + column_meta = field_data.get("column_meta") + if isinstance(column_meta, dict) and column_meta: + column.setdefault("meta", {}).update( + { + key: value + for key, value in column_meta.items() + if key not in ("dimension", "metrics") + } + ) + columns_by_name[field.name] = column + + model: Dict[str, Any] = {"name": _model_name_for(dataset)} + if dataset.description: + model["description"] = dataset.description + meta: Dict[str, Any] = {} + if dataset.primary_key: + keys = list(dataset.primary_key) + meta["primary_key"] = keys[0] if len(keys) == 1 else keys + ai_hint = _ai_hint(dataset.ai_context) + if ai_hint is not None: + meta["ai_hint"] = ai_hint + if meta: + model["meta"] = meta + model["columns"] = list(columns_by_name.values()) + return model, columns_by_name + + def _convert_metric( + self, + metric: OssieMetric, + dataset_names: List[str], + models_by_dataset: Dict[str, Dict[str, Any]], + columns_by_dataset: Dict[str, Dict[str, Dict[str, Any]]], + references_by_dataset: Dict[str, JoinReferences], + issues: List[ConverterIssue], + ) -> None: + expression = self._pick_expression(metric.expression, metric.name, issues) + extension_data = _lightdash_extension_data(metric, issues) + + target_dataset = self._resolve_target_dataset( + expression, dataset_names, references_by_dataset + ) + # An expression that names no dataset at all can still be placed when + # the stash says which model it came from. + stashed_model = extension_data.get("model") + if target_dataset is None and stashed_model in dataset_names: + if not referenced_datasets(expression, set(dataset_names)): + target_dataset = stashed_model + if target_dataset is None: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.CROSS_DATASET_METRIC_DROPPED, + element_name=metric.name, + ) + ) + return + references = references_by_dataset.get(target_dataset, {}) + + definition: Dict[str, Any] = {} + if metric.description: + definition["description"] = metric.description + ai_hint = _ai_hint(metric.ai_context) + if ai_hint is not None: + definition["ai_hint"] = ai_hint + + # A single aggregation becomes a typed metric: on the column when the + # operand is one of the dataset's columns, otherwise on the model with + # the operand as `sql`. Anything else is a `number` metric with raw SQL. + target_column: Optional[str] = None + parsed = parse_aggregation(expression) + if parsed is not None: + definition["type"] = parsed.lightdash_type + inner = parsed.inner + if ( + is_column_reference(inner) + and qualifier_of(inner) in (None, target_dataset) + and strip_qualifier(inner) in columns_by_dataset[target_dataset] + ): + target_column = strip_qualifier(inner) + else: + definition["sql"] = ossie_sql_to_lightdash(inner, target_dataset, references) + if parsed.percentile is not None: + definition["percentile"] = parsed.percentile + protected = _PROTECTED_AGGREGATION_KEYS + else: + definition["type"] = extension_data.get("type", "number") + definition["sql"] = ossie_sql_to_lightdash(expression, target_dataset, references) + protected = _PROTECTED_METRIC_KEYS + + definition.update( + { + key: value + for key, value in extension_data.items() + if key not in protected + } + ) + + # The Lightdash name is the stashed one, else the Ossie name with the + # `_` field-id prefix removed, else the Ossie name as is. + lightdash_name = extension_data.get("name") + if not isinstance(lightdash_name, str) or not lightdash_name: + prefix = f"{target_dataset}_" + lightdash_name = ( + metric.name[len(prefix):] + if metric.name.startswith(prefix) and len(metric.name) > len(prefix) + else metric.name + ) + + if target_column is not None: + column = columns_by_dataset[target_dataset][target_column] + metrics = column.setdefault("meta", {}).setdefault("metrics", {}) + metrics[lightdash_name] = definition + else: + model = models_by_dataset[target_dataset] + metrics = model.setdefault("meta", {}).setdefault("metrics", {}) + metrics[lightdash_name] = definition + + @staticmethod + def _resolve_target_dataset( + expression: str, + dataset_names: List[str], + references_by_dataset: Dict[str, JoinReferences], + ) -> Optional[str]: + """The dataset whose model hosts the metric. + + A metric spanning several datasets lives on the one that joins all the + others directly: Lightdash resolves ``${other.column}`` only against + the joins the hosting model declares, never transitively. + """ + referenced = referenced_datasets(expression, set(dataset_names)) + if len(referenced) == 0: + return dataset_names[0] if len(dataset_names) == 1 else None + if len(referenced) == 1: + return next(iter(referenced)) + for name in dataset_names: + if name in referenced and referenced - {name} <= set( + references_by_dataset.get(name, {}) + ): + return name + return None diff --git a/converters/lightdash/tests/__init__.py b/converters/lightdash/tests/__init__.py new file mode 100644 index 00000000..13a83393 --- /dev/null +++ b/converters/lightdash/tests/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/converters/lightdash/tests/test_cli.py b/converters/lightdash/tests/test_cli.py new file mode 100644 index 00000000..5691e506 --- /dev/null +++ b/converters/lightdash/tests/test_cli.py @@ -0,0 +1,174 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Command line round trips through both output formats.""" + +import json +from pathlib import Path + +import pytest +import yaml + +from ossie import OssieDocument +from ossie_lightdash.cli import main + +TPCDS_PATH = Path(__file__).parent / ".." / ".." / ".." / "examples" / "tpcds_semantic_model.yaml" + + +@pytest.mark.parametrize("suffix", [".yaml", ".json"]) +def test_import_writes_a_loadable_document(tmp_path, suffix): + schema_yml = tmp_path / "schema.yml" + assert main(["export", str(TPCDS_PATH), str(schema_yml), "--format", "dbt-meta"]) == 0 + + document_path = tmp_path / f"semantic_model{suffix}" + assert main(["import", str(schema_yml), str(document_path), "--schema", "public"]) == 0 + + text = document_path.read_text(encoding="utf-8") + if suffix == ".json": + document = OssieDocument.model_validate_json(text) + else: + document = OssieDocument.model_validate(yaml.safe_load(text)) + assert {dataset.name for dataset in document.semantic_model[0].datasets} == { + "store_sales", "date_dim", "customer", "item", "store" + } + +def test_export_writes_a_lightdash_project(tmp_path, capsys): + project = tmp_path / "project" + assert main(["export", "-i", str(TPCDS_PATH), "-o", str(project), "--dialect", "BIGQUERY"]) == 0 + captured = capsys.readouterr() + # Like the other converters: stdout stays clean, stderr carries the report. + assert captured.out == "" + assert captured.err.splitlines()[-1].startswith("Wrote 5 model file(s) to ") + # The one loss is named, explained, and counted. + assert captured.err.splitlines()[:5] == [ + "TIME_ROLE_NOT_REPRESENTABLE (1 element)", + " is_time on a non-date type (e.g. an integer year); Lightdash has no such marker, " + "the column is a plain dimension", + " d_year", + "", + "1 issue(s); everything else converted cleanly. Pass --verbose to list every element.", + ] + files = sorted(p.name for p in (project / "lightdash" / "models").iterdir()) + assert files == ["customer.yml", "date_dim.yml", "item.yml", "store.yml", "store_sales.yml"] + model = yaml.safe_load((project / "lightdash" / "models" / "store_sales.yml").read_text()) + assert model["type"] == "model" + assert model["sql_from"] == "tpcds.public.store_sales" + assert all({"name", "type", "sql"} <= set(d) for d in model["dimensions"]) + config = yaml.safe_load((project / "lightdash.config.yml").read_text()) + assert config["warehouse"] == {"type": "bigquery"} + assert config["name"] == "tpcds_retail_model" + # A config that someone has given a real warehouse type is kept ... + (project / "lightdash.config.yml").write_text("name: mine\nversion: '1.0'\nwarehouse:\n type: postgres\n") + assert main(["export", str(TPCDS_PATH), str(project)]) == 0 + assert yaml.safe_load((project / "lightdash.config.yml").read_text())["name"] == "mine" + + +def test_placeholder_config_is_replaced_on_the_next_run(tmp_path): + project = tmp_path / "project" + # ... but the placeholder we wrote ourselves is not. + assert main(["export", "-i", str(TPCDS_PATH), "-o", str(project)]) == 0 + assert yaml.safe_load((project / "lightdash.config.yml").read_text())["warehouse"] == {"type": "CHANGE_ME"} + assert main(["export", "-i", str(TPCDS_PATH), "-o", str(project), "--dialect", "BIGQUERY"]) == 0 + assert yaml.safe_load((project / "lightdash.config.yml").read_text())["warehouse"] == {"type": "bigquery"} + + +def test_export_dbt_meta_still_writes_one_schema_file(tmp_path): + schema_yml = tmp_path / "schema.yml" + assert main(["export", str(TPCDS_PATH), str(schema_yml), "--format", "dbt-meta"]) == 0 + assert yaml.safe_load(schema_yml.read_text())["version"] == 2 + +def test_import_reads_a_whole_dbt_project(tmp_path, capsys): + project = tmp_path / "dbt" + (project / "models" / "marts").mkdir(parents=True) + (project / "target").mkdir() + (project / "dbt_project.yml").write_text("name: p\nmodels:\n p:\n +materialized: table\n") + (project / "models" / "orders.yml").write_text( + "models:\n - name: orders\n columns:\n - name: amount\n meta:\n metrics:\n total: {type: sum}\n" + ) + (project / "models" / "marts" / "customers.yaml").write_text( + "models:\n - name: customers\n columns:\n - name: id\n" + ) + (project / "data.yml").write_text("seeds:\n - name: statuses\n columns:\n - name: code\n") + (project / "target" / "stale.yml").write_text("models:\n - name: stale\n") + (project / "models" / "template.yml").write_text("{project_name}:\n +materialized: view\n") + + document_path = tmp_path / "model.yaml" + assert main(["import", "--input", str(project), "--output", str(document_path), "--schema", "marts"]) == 0 + document = OssieDocument.model_validate(yaml.safe_load(document_path.read_text())) + assert [d.name for d in document.semantic_model[0].datasets] == ["customers", "orders", "statuses"] + err = capsys.readouterr().err + assert "Skipped" in err and "template.yml" in err + assert document.semantic_model[0].metrics[0].name == "orders_total" + +def test_import_takes_types_from_a_dbt_catalog(tmp_path): + schema_yml = tmp_path / "schema.yml" + schema_yml.write_text("models:\n - name: races\n columns:\n - name: race_date\n - name: laps\n") + catalog = tmp_path / "catalog.json" + catalog.write_text(json.dumps({ + "nodes": { + "model.demo.races": {"columns": {"race_date": {"type": "DATE"}, "LAPS": {"type": "INT64"}}} + } + })) + out = tmp_path / "model.yaml" + assert main(["import", str(schema_yml), str(out), "--schema", "marts", "--catalog", str(catalog)]) == 0 + fields = {f["name"]: f for f in yaml.safe_load(out.read_text())["semantic_model"][0]["datasets"][0]["fields"]} + assert fields["race_date"]["datatype"] == "Date" + assert fields["laps"]["datatype"] == "Integer" + +def test_input_and_output_are_required(capsys): + with pytest.raises(SystemExit): + main(["export", "-i", str(TPCDS_PATH)]) + assert "both --input and --output are required" in capsys.readouterr().err + +def test_issues_are_grouped_by_type_unless_verbose(tmp_path, capsys): + schema_yml = tmp_path / "schema.yml" + columns = "".join( + f" - name: c{i}\n meta:\n metrics:\n m{i}: {{type: sum, filters: [{{x: y}}]}}\n" + for i in range(11) + ) + schema_yml.write_text("models:\n - name: t\n columns:\n" + columns) + assert main(["import", "-i", str(schema_yml), "-o", str(tmp_path / "a.yaml"), "--schema", "s"]) == 0 + grouped = capsys.readouterr().err.splitlines() + assert grouped[0] == "METRIC_FILTER_NOT_PORTABLE (11 elements)" + assert grouped[2] == " m0, m1, m2, m3, m4, m5, m6, m7" + assert grouped[3] == " ... and 3 more" + assert main(["import", "-i", str(schema_yml), "-o", str(tmp_path / "b.yaml"), "--schema", "s", "--verbose"]) == 0 + verbose = capsys.readouterr().err.splitlines() + assert verbose[2] == " m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10" + assert "... and" not in "\n".join(verbose) + +def test_missing_input_is_an_error(tmp_path, capsys): + with pytest.raises(SystemExit): + main(["import", "-i", str(tmp_path / "nope"), "-o", str(tmp_path / "out.yaml")]) + assert "input not found" in capsys.readouterr().err + +def test_lightdash_model_files_import_and_round_trip(tmp_path): + project = tmp_path / "project" + assert main(["export", "-i", str(TPCDS_PATH), "-o", str(project), "--dialect", "SNOWFLAKE"]) == 0 + back = tmp_path / "back.yaml" + # No --database/--schema: the model files name their own sources. + assert main(["import", "-i", str(project), "-o", str(back), "--dialect", "SNOWFLAKE"]) == 0 + original = OssieDocument.model_validate(yaml.safe_load(TPCDS_PATH.read_text())).semantic_model[0] + roundtripped = OssieDocument.model_validate(yaml.safe_load(back.read_text())).semantic_model[0] + assert {d.name: d.source for d in roundtripped.datasets} == {d.name: d.source for d in original.datasets} + assert {d.name: d.primary_key for d in roundtripped.datasets} == {d.name: d.primary_key for d in original.datasets} + assert {(d.name, f.name): f.dimension is not None for d in roundtripped.datasets for f in d.fields} == { + (d.name, f.name): f.dimension is not None for d in original.datasets for f in d.fields + } + assert {(r.from_dataset, r.to) for r in roundtripped.relationships} == { + (r.from_dataset, r.to) for r in original.relationships + } + assert len(roundtripped.metrics) == len(original.metrics) diff --git a/converters/lightdash/tests/test_lightdash_to_ossie.py b/converters/lightdash/tests/test_lightdash_to_ossie.py new file mode 100644 index 00000000..250ca10a --- /dev/null +++ b/converters/lightdash/tests/test_lightdash_to_ossie.py @@ -0,0 +1,960 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +from ossie import OssieDataType, OssieDialect + +from ossie_lightdash import ConverterIssueType, LightdashToOssieConverter + +SCHEMA_YML = { + "version": 2, + "models": [ + { + "name": "orders", + "description": "One row per order", + "meta": { + "primary_key": "order_id", + "ai_hint": ["Orders placed in the web shop.", "One row per order."], + "joins": [ + { + "join": "customers", + "sql_on": "${orders.customer_id} = ${customers.customer_id}", + } + ], + "metrics": { + "conversion_rate": { + "type": "number", + "label": "Conversion rate", + "format": "percent", + "round": 1, + "sql": "SUM(${TABLE}.completed_count) / NULLIF(SUM(${TABLE}.total_count), 0)", + } + }, + }, + "columns": [ + { + "name": "order_date", + "description": "Date the order was placed", + "meta": {"dimension": {"label": "Order date", "type": "date"}}, + }, + { + "name": "status", + "meta": { + "dimension": { + "label": "Status", + "type": "string", + "ai_hint": "Order lifecycle stage.", + } + }, + }, + { + "name": "updated_at", + "meta": { + "dimension": {"type": "timestamp", "time_intervals": "OFF"} + }, + }, + { + "name": "shipped_at", + "meta": { + "dimension": {"type": "timestamp", "time_intervals": ["DAY", "MONTH"]} + }, + }, + { + "name": "amount", + "description": "Order amount", + "meta": { + "dimension": {"type": "number"}, + "metrics": { + "total_amount": { + "type": "sum", + "label": "Total amount", + "format": "usd", + "ai_hint": "Revenue before refunds.", + }, + "latest_amount": {"type": "max"}, + "median_amount": {"type": "median"}, + "p90_amount": {"type": "percentile", "percentile": 90}, + } + }, + }, + {"name": "completed_count"}, + {"name": "total_count"}, + { + "name": "customer_id", + "meta": { + "metrics": { + "unique_customers": {"type": "count_distinct"}, + } + }, + }, + ], + }, + { + "name": "customers", + "meta": {"primary_key": ["customer_id", "region"]}, + "columns": [{"name": "customer_id"}], + }, + ], +} + + +def _raw_lightdash_data(element): + for extension in element.custom_extensions or []: + if extension.vendor_name.upper() == "LIGHTDASH": + return json.loads(extension.data) + return {} + + +def _metric(document, name): + """Look a metric up by its Lightdash name (stashed in the extension).""" + return next( + m + for m in document.semantic_model[0].metrics + if _raw_lightdash_data(m).get("name") == name + ) + + +def _lightdash_data(element): + data = _raw_lightdash_data(element) + data.pop("name", None) + data.pop("model", None) + return data + + +class TestLightdashToOssie: + def test_dataset_source_is_qualified(self): + result = LightdashToOssieConverter().convert( + SCHEMA_YML, database="analytics_db", schema="marts" + ) + dataset = result.output.semantic_model[0].datasets[0] + assert dataset.source == "analytics_db.marts.orders" + assert not any( + issue.issue_type is ConverterIssueType.SOURCE_UNQUALIFIED + for issue in result.issues + ) + + def test_missing_schema_is_reported(self): + result = LightdashToOssieConverter().convert(SCHEMA_YML) + dataset = result.output.semantic_model[0].datasets[0] + assert dataset.source == "orders" + assert any( + issue.issue_type is ConverterIssueType.SOURCE_UNQUALIFIED + for issue in result.issues + ) + + def test_time_dimension(self): + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") + field = result.output.semantic_model[0].datasets[0].fields[0] + assert field.name == "order_date" + assert field.label == "Order date" + assert field.description == "Date the order was placed" + assert field.dimension is not None + # The Lightdash type becomes a datatype; `is_time` is an Ossie role + # marker with no Lightdash source, so it stays unset. + assert field.datatype is OssieDataType.DATE + assert field.dimension.is_time is None + withdrawn = result.output.semantic_model[0].datasets[0].fields[2] + assert withdrawn.name == "updated_at" + assert withdrawn.dimension.is_time is False + assert _lightdash_data(withdrawn) == {"type": "timestamp"} + # A custom interval list is not a role marker: it stays in the extension. + custom = result.output.semantic_model[0].datasets[0].fields[3] + assert custom.dimension.is_time is None + assert _lightdash_data(custom) == {"type": "timestamp", "time_intervals": ["DAY", "MONTH"]} + + def test_dimension_types_become_datatypes(self): + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") + by_name = { + field.name: field + for field in result.output.semantic_model[0].datasets[0].fields + } + assert by_name["status"].datatype is OssieDataType.STRING + assert by_name["order_date"].datatype is OssieDataType.DATE + + def test_typed_metric_becomes_aggregation_expression(self): + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") + metric = _metric(result.output, "total_amount") + assert metric.expression.dialects[0].expression == "SUM(orders.amount)" + assert _lightdash_data(metric) == {"label": "Total amount", "format": "usd"} + assert metric.ai_context == "Revenue before refunds." + + def test_count_distinct_metric(self): + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") + metric = _metric(result.output, "unique_customers") + assert ( + metric.expression.dialects[0].expression + == "COUNT(DISTINCT orders.customer_id)" + ) + + def test_percentile_metric_becomes_percentile_cont(self): + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") + metric = _metric(result.output, "p90_amount") + assert ( + metric.expression.dialects[0].expression + == "PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY orders.amount)" + ) + assert _lightdash_data(metric) == {} + + def test_sql_metric_expression_is_rewritten(self): + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") + metric = _metric(result.output, "conversion_rate") + assert ( + metric.expression.dialects[0].expression + == "SUM(orders.completed_count) / NULLIF(SUM(orders.total_count), 0)" + ) + assert _lightdash_data(metric) == { + "label": "Conversion rate", + "format": "percent", + "round": 1, + } + + def test_join_becomes_relationship(self): + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") + relationship = result.output.semantic_model[0].relationships[0] + assert relationship.from_dataset == "orders" + assert relationship.to == "customers" + assert relationship.from_columns == ["customer_id"] + assert relationship.to_columns == ["customer_id"] + + def test_percentile_with_sql_orders_by_the_expression(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": { + "metrics": { + "p90_custom": { + "type": "percentile", + "percentile": 90, + "sql": "${TABLE}.amount - ${TABLE}.discount", + } + } + }, + "columns": [], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + metric = _metric(result.output, "p90_custom") + assert ( + metric.expression.dialects[0].expression + == "PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY orders.amount - orders.discount)" + ) + assert _lightdash_data(metric) == {} + + def test_joined_table_references_become_cross_dataset(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": { + "metrics": { + "orders_per_customer": { + "type": "number", + "sql": "COUNT(${TABLE}.order_id) / COUNT(DISTINCT ${customers.customer_id})", + } + } + }, + "columns": [], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + metric = _metric(result.output, "orders_per_customer") + assert ( + metric.expression.dialects[0].expression + == "COUNT(orders.order_id) / COUNT(DISTINCT customers.customer_id)" + ) + + def test_model_metric_without_sql_is_skipped(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": {"metrics": {"broken_metric": {"type": "number"}}}, + "columns": [], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + assert result.output.semantic_model[0].metrics is None + assert any( + issue.issue_type is ConverterIssueType.METRIC_SQL_MISSING + and issue.element_name == "broken_metric" + for issue in result.issues + ) + + def test_unparseable_join_is_reported(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": { + "joins": [{"join": "customers", "sql_on": "1 = 1"}], + }, + "columns": [], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + assert result.output.semantic_model[0].relationships is None + assert any( + issue.issue_type is ConverterIssueType.JOIN_SQL_UNPARSED + for issue in result.issues + ) + + def test_typed_metric_with_sql_aggregates_the_expression(self): + schema_yml = { + "models": [ + { + "name": "work_orders", + "columns": [ + { + "name": "status", + "meta": { + "metrics": { + "completion_rate": { + "type": "average", + "sql": "CASE WHEN ${status} = 'Completed' THEN 1 ELSE 0 END", + }, + "distinct_total": {"type": "sum_distinct"}, + } + }, + } + ], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + assert ( + _metric(result.output, "completion_rate").expression.dialects[0].expression + == "AVG(CASE WHEN work_orders.status = 'Completed' THEN 1 ELSE 0 END)" + ) + distinct_total = _metric(result.output, "distinct_total") + assert ( + distinct_total.expression.dialects[0].expression + == "SUM(DISTINCT work_orders.status)" + ) + assert _lightdash_data(distinct_total) == {} + + def test_metric_reference_is_inlined(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": { + "metrics": { + "amount_per_customer": { + "type": "number", + "sql": "${total_amount} / NULLIF(${unique_customers}, 0)", + } + } + }, + "columns": [ + {"name": "amount", "meta": {"metrics": {"total_amount": {"type": "sum"}}}}, + { + "name": "customer_id", + "meta": {"metrics": {"unique_customers": {"type": "count_distinct"}}}, + }, + ], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + assert ( + _metric(result.output, "amount_per_customer").expression.dialects[0].expression + == "(SUM(orders.amount)) / NULLIF((COUNT(DISTINCT orders.customer_id)), 0)" + ) + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.METRIC_REFERENCE_INLINED + ] == ["amount_per_customer", "amount_per_customer"] + + def test_bare_field_references_resolve_to_the_dataset(self): + schema_yml = { + "models": [ + { + "name": "customers", + "columns": [ + {"name": "first_name", "meta": {"dimension": {"type": "string"}}}, + { + "name": "full_name", + "meta": { + "dimension": { + "type": "string", + "sql": "${first_name} || ' ' || ${TABLE}.last_name", + } + }, + }, + { + "name": "order_count", + "meta": { + "dimension": { + "type": "number", + "sql": "(SELECT COUNT(*) FROM orders WHERE orders.customer_id = ${TABLE}.customer_id)", + } + }, + }, + ], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + by_name = { + field.name: field.expression.dialects[0].expression + for field in result.output.semantic_model[0].datasets[0].fields + } + assert by_name["full_name"] == "customers.first_name || ' ' || customers.last_name" + assert by_name["order_count"] == ( + "(SELECT COUNT(*) FROM orders WHERE orders.customer_id = customers.customer_id)" + ) + + def test_parameter_references_skip_the_element(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": { + "metrics": { + "my_orders": { + "type": "number", + "sql": "SUM(CASE WHEN ${TABLE}.owner = ${ld.user.email} THEN 1 END)", + } + } + }, + "columns": [ + { + "name": "is_recent", + "meta": { + "dimension": { + "type": "boolean", + "sql": "${TABLE}.order_date >= ${lightdash.parameters.start_date}", + } + }, + }, + { + "name": "status_label", + "meta": { + "dimension": { + "type": "string", + "sql": "{% if ld.query.filters contains 'orders.status' %} 'filtered' {% else %} ${TABLE}.status {% endif %}", + } + }, + }, + {"name": "order_date", "meta": {"dimension": {"type": "date"}}}, + ], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + dataset = result.output.semantic_model[0].datasets[0] + assert [field.name for field in dataset.fields] == ["order_date"] + assert result.output.semantic_model[0].metrics is None + assert sorted( + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.EXPRESSION_NOT_PORTABLE + ) == ["is_recent", "my_orders", "status_label"] + + def test_aliased_joins_become_relationships(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": { + "joins": [ + { + "join": "date_dim", + "alias": "sold_date", + "sql_on": "${orders.sold_date_id} = ${sold_date.date_id}", + "relationship": "many-to-one", + }, + { + "join": "date_dim", + "alias": "return_date", + "sql_on": "${orders.return_date_id} = ${return_date.date_id}", + }, + ] + }, + "columns": [ + { + "name": "sold_year", + "meta": {"dimension": {"type": "number", "sql": "${sold_date.year}"}}, + } + ], + }, + {"name": "date_dim", "columns": [{"name": "date_id"}, {"name": "year"}]}, + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + relationships = result.output.semantic_model[0].relationships + assert [(r.name, r.to, r.from_columns, r.to_columns) for r in relationships] == [ + ("orders_to_sold_date", "date_dim", ["sold_date_id"], ["date_id"]), + ("orders_to_return_date", "date_dim", ["return_date_id"], ["date_id"]), + ] + assert _lightdash_data(relationships[0]) == { + "alias": "sold_date", + "relationship": "many-to-one", + } + field = result.output.semantic_model[0].datasets[0].fields[0] + assert field.expression.dialects[0].expression == "date_dim.year" + assert any( + issue.issue_type is ConverterIssueType.ALIAS_REFERENCE_FLATTENED + and issue.element_name == "sold_year" + for issue in result.issues + ) + + def test_expressions_carry_the_warehouse_dialect(self): + result = LightdashToOssieConverter(OssieDialect.BIGQUERY).convert( + SCHEMA_YML, schema="marts" + ) + metric = _metric(result.output, "total_amount") + assert [d.dialect for d in metric.expression.dialects] == [OssieDialect.BIGQUERY] + field = result.output.semantic_model[0].datasets[0].fields[0] + assert field.expression.dialects[0].dialect is OssieDialect.BIGQUERY + + def test_primary_key_and_ai_hint_become_dataset_attributes(self): + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") + orders, customers = result.output.semantic_model[0].datasets + assert orders.primary_key == ["order_id"] + assert orders.ai_context == "Orders placed in the web shop.\nOne row per order." + assert customers.primary_key == ["customer_id", "region"] + status = next(field for field in orders.fields if field.name == "status") + assert status.ai_context == "Order lifecycle stage." + assert _lightdash_data(status) == {"type": "string"} + + def test_metric_datatypes_follow_the_aggregation(self): + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") + assert _metric(result.output, "unique_customers").datatype is OssieDataType.INTEGER + assert _metric(result.output, "total_amount").datatype is OssieDataType.DECIMAL + assert _metric(result.output, "latest_amount").datatype is OssieDataType.DECIMAL + assert _metric(result.output, "conversion_rate").datatype is None + + def test_config_meta_is_read_and_wins_over_meta(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": {"primary_key": "legacy_id"}, + "config": { + "meta": { + "primary_key": "order_id", + "joins": [ + { + "join": "customers", + "sql_on": "${orders.customer_id} = ${customers.customer_id}", + } + ], + } + }, + "columns": [ + { + "name": "amount", + "meta": {"dimension": {"type": "number", "label": "Amount"}}, + "config": { + "meta": { + "dimension": {"label": "Order amount"}, + "metrics": {"total_amount": {"type": "sum"}}, + } + }, + } + ], + }, + {"name": "customers", "columns": [{"name": "customer_id"}]}, + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + orders = result.output.semantic_model[0].datasets[0] + assert orders.primary_key == ["order_id"] + assert orders.fields[0].label == "Order amount" + assert orders.fields[0].datatype is OssieDataType.DECIMAL + assert _metric(result.output, "total_amount").expression.dialects[0].expression == ( + "SUM(orders.amount)" + ) + assert result.output.semantic_model[0].relationships[0].to == "customers" + + def test_metric_names_are_qualified_with_the_model(self): + result = LightdashToOssieConverter().convert(SCHEMA_YML, schema="marts") + metric = _metric(result.output, "total_amount") + assert metric.name == "orders_total_amount" + assert _raw_lightdash_data(metric)["name"] == "total_amount" + assert _raw_lightdash_data(metric)["model"] == "orders" + assert [m.name for m in result.output.semantic_model[0].metrics][:3] == [ + "orders_total_amount", + "orders_latest_amount", + "orders_median_amount", + ] + + def test_qualified_names_that_still_collide_are_suffixed(self): + schema_yml = { + "models": [ + { + "name": "orders", + "columns": [{"name": "amount", "meta": {"metrics": {"x_total": {"type": "sum"}}}}], + }, + { + "name": "orders_x", + "columns": [{"name": "amount", "meta": {"metrics": {"total": {"type": "sum"}}}}], + }, + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + assert [m.name for m in result.output.semantic_model[0].metrics] == [ + "orders_x_total", + "orders_x_total_2", + ] + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.METRIC_NAME_COLLISION + ] == ["total"] + + def test_seeds_are_datasets_and_joins_to_missing_models_are_skipped(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": { + "joins": [ + {"join": "order_statuses", "sql_on": "${orders.status_id} = ${order_statuses.id}"}, + {"join": "not_in_this_file", "sql_on": "${orders.x} = ${not_in_this_file.x}"}, + ] + }, + "columns": [{"name": "status_id"}], + } + ], + "seeds": [{"name": "order_statuses", "columns": [{"name": "id"}]}], + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + semantic_model = result.output.semantic_model[0] + assert [dataset.name for dataset in semantic_model.datasets] == ["orders", "order_statuses"] + assert [relationship.to for relationship in semantic_model.relationships] == ["order_statuses"] + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.JOIN_TARGET_UNKNOWN + ] == ["orders -> not_in_this_file"] + + def test_bare_column_names_in_sql_are_qualified(self): + schema_yml = { + "models": [ + { + "name": "budgets", + "meta": { + "metrics": { + "use_percentage": { + "type": "number", + "sql": "SUM(budget_use) / NULLIF(SUM(budget_total), 0)", + }, + "mobile_share": { + "type": "number", + "sql": "SUM(CASE WHEN device_type = 'device_type' THEN 1 END) / COUNT(*)", + }, + } + }, + "columns": [ + {"name": "budget_use"}, + {"name": "budget_total"}, + {"name": "device_type"}, + {"name": "count", "meta": {"dimension": {"sql": "COUNT(budget_use) OVER ()"}}}, + ], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + assert _metric(result.output, "use_percentage").expression.dialects[0].expression == ( + "SUM(budgets.budget_use) / NULLIF(SUM(budgets.budget_total), 0)" + ) + # The literal is untouched; the column reference is qualified. + assert _metric(result.output, "mobile_share").expression.dialects[0].expression == ( + "SUM(CASE WHEN budgets.device_type = 'device_type' THEN 1 END) / COUNT(*)" + ) + # A column named like a function is not qualified when called. + fields = {f.name: f.expression.dialects[0].expression for f in result.output.semantic_model[0].datasets[0].fields} + assert fields["count"] == "COUNT(budgets.budget_use) OVER ()" + + def test_chained_and_expression_joins_are_stashed_and_edges_derived(self): + schema_yml = { + "models": [ + { + "name": "queries", + "meta": { + "sql_filter": "${TABLE}.deleted = false", + "group_details": {"usage": {"label": "Usage"}}, + "joins": [ + {"join": "projects", "sql_on": "${queries.project_id} = ${projects.project_id}"}, + { + "join": "organizations", + "sql_on": "${projects.organization_id} = ${organizations.organization_id}", + }, + { + "join": "users", + "sql_on": "LOWER(${queries.user_email}) = ${users.email}", + }, + { + "join": "warehouses", + "sql_on": "${queries.warehouse_id} = ${warehouses.id} AND ${warehouses.active}", + }, + ], + }, + "columns": [{"name": "project_id"}], + }, + { + "name": "projects", + "meta": { + "joins": [ + { + "join": "organizations", + "sql_on": "${projects.organization_id} = ${organizations.organization_id}", + "relationship": "many-to-one", + } + ] + }, + "columns": [{"name": "project_id"}, {"name": "organization_id"}], + }, + {"name": "organizations", "columns": [{"name": "organization_id"}]}, + {"name": "users", "columns": [{"name": "email"}]}, + {"name": "warehouses", "columns": [{"name": "id"}, {"name": "active"}]}, + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + semantic_model = result.output.semantic_model[0] + edges = [(r.from_dataset, r.to, r.from_columns, r.to_columns) for r in semantic_model.relationships] + # The chained edge projects -> organizations is declared on projects + # too; the declared one wins and carries its extras. + assert edges == [ + ("queries", "projects", ["project_id"], ["project_id"]), + ("queries", "warehouses", ["warehouse_id"], ["id"]), + ("projects", "organizations", ["organization_id"], ["organization_id"]), + ] + assert _raw_lightdash_data(semantic_model.relationships[2]) == {"relationship": "many-to-one"} + stash = _raw_lightdash_data(semantic_model.datasets[0]) + assert stash["sql_filter"] == "${TABLE}.deleted = false" + assert stash["group_details"] == {"usage": {"label": "Usage"}} + assert [join["join"] for join in stash["joins"]] == ["organizations", "users", "warehouses"] + assert { + (issue.issue_type.value, issue.element_name) + for issue in result.issues + if issue.issue_type in (ConverterIssueType.JOIN_STASHED, ConverterIssueType.JOIN_SQL_UNPARSED) + } == { + ("JOIN_STASHED", "queries -> organizations"), + ("JOIN_SQL_UNPARSED", "queries -> users"), + ("JOIN_STASHED", "queries -> warehouses"), + } + + def test_chained_join_without_a_declared_edge_derives_one(self): + schema_yml = { + "models": [ + { + "name": "users", + "meta": { + "joins": [ + {"join": "roles", "sql_on": "${users.id} = ${roles.user_id}"}, + {"join": "projects", "sql_on": "${roles.project_id} = ${projects.id}"}, + ] + }, + "columns": [{"name": "id"}], + }, + {"name": "roles", "columns": [{"name": "user_id"}, {"name": "project_id"}]}, + {"name": "projects", "columns": [{"name": "id"}]}, + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + edges = [(r.from_dataset, r.to) for r in result.output.semantic_model[0].relationships] + assert edges == [("users", "roles"), ("roles", "projects")] + + def test_column_meta_outside_dimension_is_stashed(self): + schema_yml = { + "models": [ + { + "name": "orders", + "columns": [ + { + "name": "amount", + "meta": { + "dimension": {"type": "number"}, + "additional_dimensions": {"amount_bucket": {"type": "string", "sql": "CASE WHEN ${TABLE}.amount > 100 THEN 'big' ELSE 'small' END"}}, + }, + } + ], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + field = result.output.semantic_model[0].datasets[0].fields[0] + assert _lightdash_data(field) == { + "type": "number", + "column_meta": { + "additional_dimensions": { + "amount_bucket": {"type": "string", "sql": "CASE WHEN ${TABLE}.amount > 100 THEN 'big' ELSE 'small' END"} + } + }, + } + + def test_every_column_is_a_dimension_unless_hidden(self): + schema_yml = { + "models": [ + { + "name": "orders", + "columns": [ + {"name": "plain"}, + {"name": "shown", "meta": {"dimension": {"hidden": False, "type": "string"}}}, + {"name": "hidden_key", "meta": {"dimension": {"hidden": True, "type": "number", "label": "Key"}}}, + ], + } + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + fields = {f.name: f for f in result.output.semantic_model[0].datasets[0].fields} + assert fields["plain"].dimension is not None + assert fields["shown"].dimension is not None + assert _lightdash_data(fields["shown"]) == {"type": "string"} + assert fields["hidden_key"].dimension is None + assert fields["hidden_key"].datatype is OssieDataType.DECIMAL + assert fields["hidden_key"].label == "Key" + assert _lightdash_data(fields["hidden_key"]) == {"type": "number"} + + def test_filters_that_other_consumers_cannot_see_are_reported(self): + schema_yml = { + "models": [ + { + "name": "orders", + "meta": {"sql_filter": "${TABLE}.deleted = false"}, + "columns": [ + { + "name": "amount", + "meta": { + "metrics": { + "paid_amount": {"type": "sum", "filters": [{"status": "paid"}]}, + "total_amount": {"type": "sum"}, + } + }, + } + ], + }, + {"name": "customers", "meta": {"required_filters": [{"region": "EU"}]}, "columns": []}, + ] + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts") + # The values are stashed for Lightdash, and the loss for everyone else is reported. + paid = _metric(result.output, "paid_amount") + assert paid.expression.dialects[0].expression == "SUM(orders.amount)" + assert _lightdash_data(paid) == {"filters": [{"status": "paid"}]} + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.METRIC_FILTER_NOT_PORTABLE + ] == ["paid_amount"] + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.ROW_FILTER_NOT_PORTABLE + ] == ["orders", "customers"] + + def test_catalog_types_fill_the_gaps_but_never_override_authored_types(self): + from ossie_lightdash.catalog import warehouse_type_to_datatype + + assert warehouse_type_to_datatype("INT64") is OssieDataType.INTEGER + assert warehouse_type_to_datatype("NUMBER(38,0)") is OssieDataType.INTEGER + assert warehouse_type_to_datatype("NUMBER(12,2)") is OssieDataType.DECIMAL + assert warehouse_type_to_datatype("NUMERIC") is OssieDataType.DECIMAL + assert warehouse_type_to_datatype("character varying(255)") is OssieDataType.STRING + assert warehouse_type_to_datatype("TIMESTAMP_TZ(9)") is OssieDataType.DATE_TIME_TZ + assert warehouse_type_to_datatype("ARRAY") is None + + schema_yml = { + "models": [ + { + "name": "results", + "columns": [ + {"name": "points", "meta": {"dimension": {"type": "string"}}}, + {"name": "position", "meta": {"dimension": {"label": "Position"}}}, + {"name": "race_date"}, + {"name": "payload"}, + {"name": "laps", "meta": {"metrics": {"total_laps": {"type": "sum"}}}}, + ], + }, + {"name": "orphan", "columns": [{"name": "id"}]}, + ] + } + catalog = { + "results": { + "points": "FLOAT64", + "position": "INT64", + "race_date": "DATE", + "payload": "STRUCT", + "laps": "INT64", + } + } + result = LightdashToOssieConverter().convert(schema_yml, schema="marts", catalog=catalog) + fields = {f.name: f for f in result.output.semantic_model[0].datasets[0].fields} + assert fields["points"].datatype is OssieDataType.STRING # authored wins + assert fields["position"].datatype is OssieDataType.INTEGER # gap filled + assert fields["race_date"].datatype is OssieDataType.DATE # no meta at all + assert fields["payload"].datatype is None # outside the vocabulary + # A catalog type also feeds the metric datatype. + assert _metric(result.output, "total_laps").datatype is OssieDataType.DECIMAL + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.CATALOG_MODEL_MISSING + ] == ["orphan"] + + def test_lightdash_model_file_is_read_like_dbt_meta(self): + from ossie_lightdash.dbt_project import model_file_to_dbt_model + + model_file = { + "type": "model", + "name": "orders", + "label": "Orders", + "description": "One row per order", + "sql_from": "SELECT * FROM marts.orders WHERE deleted = false", + "primary_key": "order_id", + "sql_filter": "${TABLE}.season >= 2025", + "joins": [{"join": "customers", "sql_on": "${orders.customer_id} = ${customers.customer_id}"}], + "metrics": {"aov": {"type": "number", "sql": "${total_amount} / NULLIF(${order_count}, 0)"}}, + "dimensions": [ + {"name": "order_id", "type": "number", "sql": "${TABLE}.order_id", "hidden": True, + "metrics": {"order_count": {"type": "count_distinct"}}}, + {"name": "amount", "type": "number", "sql": "${TABLE}.amount", "format": "usd", + "metrics": {"total_amount": {"type": "sum"}}}, + {"name": "customer_id", "type": "string", "sql": "${TABLE}.customer_id"}, + {"name": "amount_bucket", "type": "string", "sql": "CASE WHEN ${amount} > 100 THEN 'big' ELSE 'small' END"}, + ], + } + schema_yml = {"models": [model_file_to_dbt_model(model_file), + {"name": "customers", "columns": [{"name": "customer_id"}]}]} + result = LightdashToOssieConverter().convert(schema_yml, schema="ignored") + orders = result.output.semantic_model[0].datasets[0] + assert orders.source == "SELECT * FROM marts.orders WHERE deleted = false" + assert orders.primary_key == ["order_id"] + assert _raw_lightdash_data(orders) == {"label": "Orders", "sql_filter": "${TABLE}.season >= 2025"} + fields = {f.name: f for f in orders.fields} + assert fields["order_id"].dimension is None # hidden + assert fields["amount"].datatype is OssieDataType.DECIMAL + assert fields["amount"].expression.dialects[0].expression == "amount" # ${TABLE}.amount collapses + assert _lightdash_data(fields["amount"]) == {"type": "number", "format": "usd"} + assert fields["amount_bucket"].expression.dialects[0].expression == ( + "CASE WHEN orders.amount > 100 THEN 'big' ELSE 'small' END" + ) + assert _metric(result.output, "aov").expression.dialects[0].expression == ( + "(SUM(orders.amount)) / NULLIF((COUNT(DISTINCT orders.order_id)), 0)" + ) + assert result.output.semantic_model[0].relationships[0].to == "customers" + assert not any(i.issue_type is ConverterIssueType.SOURCE_UNQUALIFIED for i in result.issues) diff --git a/converters/lightdash/tests/test_ossie_to_lightdash.py b/converters/lightdash/tests/test_ossie_to_lightdash.py new file mode 100644 index 00000000..70786423 --- /dev/null +++ b/converters/lightdash/tests/test_ossie_to_lightdash.py @@ -0,0 +1,759 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +from ossie import ( + OssieAIContextObject, + OssieCustomExtension, + OssieDataType, + OssieDataset, + OssieDialect, + OssieDialectExpression, + OssieDimension, + OssieDocument, + OssieExpression, + OssieField, + OssieMetric, + OssieRelationship, + OssieSemanticModel, +) + +from ossie_lightdash import ( + ConverterIssueType, + LightdashToOssieConverter, + OssieToLightdashConverter, +) + + +def _ansi(expression: str) -> OssieExpression: + return OssieExpression( + dialects=[ + OssieDialectExpression(dialect=OssieDialect.ANSI_SQL, expression=expression) + ] + ) + + +def _document() -> OssieDocument: + orders = OssieDataset( + name="orders", + source="analytics_db.marts.orders", + description="One row per order", + fields=[ + OssieField( + name="order_date", + expression=_ansi("order_date"), + dimension=OssieDimension(is_time=True), + label="Order date", + ), + OssieField( + name="status", + expression=_ansi("status"), + dimension=OssieDimension(is_time=False), + ), + OssieField(name="amount", expression=_ansi("amount")), + OssieField(name="customer_id", expression=_ansi("customer_id")), + ], + ) + customers = OssieDataset( + name="customers", + source="analytics_db.marts.customers", + fields=[OssieField(name="customer_id", expression=_ansi("customer_id"))], + ) + metrics = [ + OssieMetric( + name="total_amount", + expression=_ansi("SUM(orders.amount)"), + description="Sum of order amounts", + custom_extensions=[ + OssieCustomExtension( + vendor_name="LIGHTDASH", + data=json.dumps({"label": "Total amount", "format": "usd"}), + ) + ], + ), + OssieMetric( + name="conversion_rate", + expression=_ansi( + "SUM(orders.completed_count) / NULLIF(SUM(orders.total_count), 0)" + ), + custom_extensions=[ + OssieCustomExtension( + vendor_name="LIGHTDASH", + data=json.dumps({"format": "percent", "round": 1}), + ) + ], + ), + OssieMetric( + name="p90_amount", + expression=_ansi("PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY orders.amount)"), + ), + OssieMetric( + name="distinct_amount", + expression=_ansi("SUM(DISTINCT orders.amount)"), + ), + OssieMetric( + name="completed_rate", + expression=_ansi("AVG(CASE WHEN orders.status = 'completed' THEN 1 ELSE 0 END)"), + ), + OssieMetric( + name="cross_dataset", + expression=_ansi("SUM(orders.amount) / COUNT(customers.customer_id)"), + ), + OssieMetric( + name="foreign_vendor_metric", + expression=_ansi("SUM(orders.amount)"), + custom_extensions=[ + OssieCustomExtension(vendor_name="somebi", data='{"x": 1}') + ], + ), + ] + relationships = [ + OssieRelationship.model_validate( + { + "name": "orders_to_customers", + "from": "orders", + "to": "customers", + "from_columns": ["customer_id"], + "to_columns": ["customer_id"], + } + ) + ] + return OssieDocument( + version="0.2.0.dev0", + semantic_model=[ + OssieSemanticModel( + name="sales", + datasets=[orders, customers], + metrics=metrics, + relationships=relationships, + ) + ], + ) + + +def _model(output, name): + return next(m for m in output["models"] if m["name"] == name) + + +def _column(model, name): + return next(c for c in model["columns"] if c["name"] == name) + + +class TestOssieToLightdash: + def test_time_dimension_exports_date_type(self): + result = OssieToLightdashConverter().convert(_document()) + column = _column(_model(result.output, "orders"), "order_date") + assert column["meta"]["dimension"] == {"label": "Order date", "type": "date"} + + def test_dimension_with_nothing_to_say_needs_no_meta(self): + # Every Lightdash column is a dimension by default. + result = OssieToLightdashConverter().convert(_document()) + column = _column(_model(result.output, "orders"), "status") + assert "meta" not in column + + def test_measure_only_field_becomes_a_hidden_dimension(self): + result = OssieToLightdashConverter().convert(_document()) + column = _column(_model(result.output, "orders"), "amount") + assert column["meta"]["dimension"] == {"hidden": True} + + def test_simple_aggregation_becomes_column_metric(self): + result = OssieToLightdashConverter().convert(_document()) + column = _column(_model(result.output, "orders"), "amount") + metric = column["meta"]["metrics"]["total_amount"] + assert metric["type"] == "sum" + assert metric["label"] == "Total amount" + assert metric["format"] == "usd" + assert metric["description"] == "Sum of order amounts" + assert "sql" not in metric + + def test_complex_expression_becomes_model_metric(self): + result = OssieToLightdashConverter().convert(_document()) + metric = _model(result.output, "orders")["meta"]["metrics"]["conversion_rate"] + assert metric["type"] == "number" + assert ( + metric["sql"] + == "SUM(${TABLE}.completed_count) / NULLIF(SUM(${TABLE}.total_count), 0)" + ) + assert metric["format"] == "percent" + assert metric["round"] == 1 + + def test_metric_over_a_joined_dataset_lives_on_the_joining_model(self): + result = OssieToLightdashConverter().convert(_document()) + metric = _model(result.output, "orders")["meta"]["metrics"]["cross_dataset"] + assert metric == { + "type": "number", + "sql": "SUM(${TABLE}.amount) / COUNT(${customers.customer_id})", + } + assert not any( + issue.issue_type is ConverterIssueType.CROSS_DATASET_METRIC_DROPPED + for issue in result.issues + ) + + def test_metric_over_an_unjoined_dataset_is_dropped_with_issue(self): + document = _document() + tampered = document.model_copy(deep=True) + # customers does not join orders, so no model can host the metric. + tampered.semantic_model[0] = tampered.semantic_model[0].model_copy( + update={ + "relationships": [], + "metrics": [ + *tampered.semantic_model[0].metrics, + OssieMetric( + name="reversed", + expression=_ansi("COUNT(customers.customer_id) / SUM(orders.amount)"), + ), + ], + } + ) + result = OssieToLightdashConverter().convert(tampered) + assert "metrics" not in _model(result.output, "customers").get("meta", {}) + assert { + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.CROSS_DATASET_METRIC_DROPPED + } == {"cross_dataset", "reversed"} + + def test_field_references_resolve_through_declared_joins(self): + document = _document() + tampered = document.model_copy(deep=True) + orders = tampered.semantic_model[0].datasets[0] + orders.fields.append( + OssieField( + name="customer_key", + expression=_ansi("UPPER(customers.customer_id)"), + dimension=OssieDimension(), + ) + ) + customers = tampered.semantic_model[0].datasets[1] + customers.fields.append( + OssieField( + name="last_order_status", + expression=_ansi("orders.status"), + dimension=OssieDimension(), + ) + ) + result = OssieToLightdashConverter().convert(tampered) + joined = _column(_model(result.output, "orders"), "customer_key") + assert joined["meta"]["dimension"]["sql"] == "UPPER(${customers.customer_id})" + unjoined = _column(_model(result.output, "customers"), "last_order_status") + assert unjoined["meta"]["dimension"]["sql"] == "orders.status" + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.FIELD_REFERENCE_UNJOINED + ] == ["last_order_status"] + + def test_preferred_dialect_falls_back_to_ansi_then_reports(self): + document = _document() + tampered = document.model_copy(deep=True) + orders = tampered.semantic_model[0].datasets[0] + orders.fields[1] = OssieField( + name="status", + expression=OssieExpression( + dialects=[ + OssieDialectExpression(dialect=OssieDialect.ANSI_SQL, expression="status"), + OssieDialectExpression( + dialect=OssieDialect.BIGQUERY, expression="LOWER(status)" + ), + ] + ), + dimension=OssieDimension(), + ) + orders.fields.append( + OssieField( + name="snowflake_only", + expression=OssieExpression( + dialects=[ + OssieDialectExpression( + dialect=OssieDialect.SNOWFLAKE, expression="status::VARCHAR" + ) + ] + ), + dimension=OssieDimension(), + ) + ) + result = OssieToLightdashConverter(OssieDialect.BIGQUERY).convert(tampered) + model = _model(result.output, "orders") + assert _column(model, "status")["meta"]["dimension"]["sql"] == "LOWER(status)" + assert ( + _column(model, "snowflake_only")["meta"]["dimension"]["sql"] == "status::VARCHAR" + ) + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.DIALECT_UNAVAILABLE + ] == ["snowflake_only"] + + def test_foreign_extension_is_reported(self): + result = OssieToLightdashConverter().convert(_document()) + assert any( + issue.issue_type is ConverterIssueType.FOREIGN_EXTENSION_IGNORED + and issue.element_name == "foreign_vendor_metric" + for issue in result.issues + ) + + def test_extension_vendor_name_is_the_registered_token(self): + result = LightdashToOssieConverter().convert( + {"models": [{"name": "t", "columns": [{"name": "c", "meta": {"dimension": {"format": "usd"}}}]}]}, + schema="s", + ) + field = result.output.semantic_model[0].datasets[0].fields[0] + assert field.custom_extensions[0].vendor_name == "LIGHTDASH" + # Documents written before the registration used the lowercase name. + legacy = OssieToLightdashConverter().convert( + result.output.model_copy( + update={ + "semantic_model": [ + result.output.semantic_model[0].model_copy( + update={ + "datasets": [ + result.output.semantic_model[0].datasets[0].model_copy( + update={ + "fields": [ + field.model_copy( + update={ + "custom_extensions": [ + OssieCustomExtension(vendor_name="lightdash", data=field.custom_extensions[0].data) + ] + } + ) + ] + } + ) + ] + } + ) + ] + } + ) + ) + assert _column(_model(legacy.output, "t"), "c")["meta"]["dimension"]["format"] == "usd" + assert not any( + issue.issue_type is ConverterIssueType.FOREIGN_EXTENSION_IGNORED for issue in legacy.issues + ) + + def test_extension_cannot_override_structural_keys(self): + document = _document() + tampered = document.model_copy(deep=True) + metric = tampered.semantic_model[0].metrics[0].model_copy( + update={ + "custom_extensions": [ + OssieCustomExtension( + vendor_name="LIGHTDASH", + data=json.dumps( + {"label": "Total amount", "sql": "1 + 1", "description": "stale"} + ), + ) + ] + } + ) + tampered.semantic_model[0].metrics[0] = metric + result = OssieToLightdashConverter().convert(tampered) + column = _column(_model(result.output, "orders"), "amount") + exported = column["meta"]["metrics"]["total_amount"] + assert exported["label"] == "Total amount" + assert "sql" not in exported + assert exported["description"] == "Sum of order amounts" + + def test_mismatched_relationship_columns_are_skipped(self): + document = _document() + tampered = document.model_copy(deep=True) + relationship = OssieRelationship.model_validate( + { + "name": "broken", + "from": "orders", + "to": "customers", + "from_columns": ["customer_id", "order_id"], + "to_columns": ["customer_id"], + } + ) + tampered.semantic_model[0].relationships[0] = relationship + result = OssieToLightdashConverter().convert(tampered) + assert "joins" not in _model(result.output, "orders").get("meta", {}) + assert any( + issue.issue_type is ConverterIssueType.RELATIONSHIP_COLUMNS_MISMATCHED + and issue.element_name == "broken" + for issue in result.issues + ) + + def test_invalid_extension_json_is_reported(self): + document = _document() + tampered = document.model_copy(deep=True) + metric = tampered.semantic_model[0].metrics[0].model_copy( + update={ + "custom_extensions": [ + OssieCustomExtension(vendor_name="LIGHTDASH", data="{not json") + ] + } + ) + tampered.semantic_model[0].metrics[0] = metric + result = OssieToLightdashConverter().convert(tampered) + assert any( + issue.issue_type is ConverterIssueType.EXTENSION_DATA_INVALID + and issue.element_name == "total_amount" + for issue in result.issues + ) + + def test_relationship_becomes_join(self): + result = OssieToLightdashConverter().convert(_document()) + joins = _model(result.output, "orders")["meta"]["joins"] + assert joins == [ + { + "join": "customers", + "sql_on": "${orders.customer_id} = ${customers.customer_id}", + "relationship": "many-to-one", + } + ] + + def test_percentile_cont_becomes_percentile_metric(self): + result = OssieToLightdashConverter().convert(_document()) + column = _column(_model(result.output, "orders"), "amount") + assert column["meta"]["metrics"]["p90_amount"] == { + "type": "percentile", + "percentile": 90, + } + assert column["meta"]["metrics"]["distinct_amount"] == {"type": "sum_distinct"} + + def test_aggregation_over_expression_becomes_typed_model_metric(self): + result = OssieToLightdashConverter().convert(_document()) + metric = _model(result.output, "orders")["meta"]["metrics"]["completed_rate"] + assert metric == { + "type": "average", + "sql": "CASE WHEN ${TABLE}.status = 'completed' THEN 1 ELSE 0 END", + } + + def test_repeated_relationship_gets_an_alias(self): + document = _document() + tampered = document.model_copy(deep=True) + tampered.semantic_model[0].relationships.append( + OssieRelationship.model_validate( + { + "name": "orders_to_referrer", + "from": "orders", + "to": "customers", + "from_columns": ["amount"], + "to_columns": ["customer_id"], + } + ) + ) + result = OssieToLightdashConverter().convert(tampered) + assert _model(result.output, "orders")["meta"]["joins"] == [ + { + "join": "customers", + "sql_on": "${orders.customer_id} = ${customers.customer_id}", + "relationship": "many-to-one", + }, + { + "join": "customers", + "alias": "orders_to_referrer", + "sql_on": "${orders.amount} = ${orders_to_referrer.customer_id}", + "relationship": "many-to-one", + }, + ] + + def test_join_alias_and_attributes_restore_from_extension(self): + document = _document() + tampered = document.model_copy(deep=True) + tampered.semantic_model[0].relationships[0] = OssieRelationship.model_validate( + { + "name": "orders_to_customers", + "from": "orders", + "to": "customers", + "from_columns": ["customer_id"], + "to_columns": ["customer_id"], + "custom_extensions": [ + { + "vendor_name": "LIGHTDASH", + "data": json.dumps( + {"alias": "buyer", "relationship": "many-to-one", "sql_on": "1 = 1"} + ), + } + ], + } + ) + result = OssieToLightdashConverter().convert(tampered) + assert _model(result.output, "orders")["meta"]["joins"] == [ + { + "join": "customers", + "alias": "buyer", + "sql_on": "${orders.customer_id} = ${buyer.customer_id}", + "relationship": "many-to-one", + } + ] + + def test_primary_key_and_ai_context_become_model_meta(self): + document = _document() + tampered = document.model_copy(deep=True) + datasets = tampered.semantic_model[0].datasets + datasets[0] = datasets[0].model_copy( + update={ + "primary_key": ["order_id"], + "ai_context": "Orders placed in the web shop.\nOne row per order.", + } + ) + datasets[1] = datasets[1].model_copy( + update={ + "primary_key": ["customer_id", "region"], + "ai_context": OssieAIContextObject( + instructions="Customer master data.", + synonyms=("clients", "buyers"), + ), + } + ) + result = OssieToLightdashConverter().convert(tampered) + orders_meta = _model(result.output, "orders")["meta"] + assert orders_meta["primary_key"] == "order_id" + assert orders_meta["ai_hint"] == [ + "Orders placed in the web shop.", + "One row per order.", + ] + customers_meta = _model(result.output, "customers")["meta"] + assert customers_meta["primary_key"] == ["customer_id", "region"] + assert customers_meta["ai_hint"] == [ + "Customer master data.", + "Also known as: clients, buyers", + ] + + def test_field_and_metric_ai_context_become_ai_hints(self): + document = _document() + tampered = document.model_copy(deep=True) + orders = tampered.semantic_model[0].datasets[0] + orders.fields[1] = orders.fields[1].model_copy(update={"ai_context": "Order lifecycle stage."}) + orders.fields[2] = orders.fields[2].model_copy( + update={"ai_context": "Gross amount.", "datatype": OssieDataType.DECIMAL} + ) + metrics = tampered.semantic_model[0].metrics + metrics[0] = metrics[0].model_copy(update={"ai_context": "Revenue before refunds."}) + result = OssieToLightdashConverter().convert(tampered) + model = _model(result.output, "orders") + assert _column(model, "status")["meta"]["dimension"]["ai_hint"] == "Order lifecycle stage." + # A hidden dimension keeps its hint and its type. + assert _column(model, "amount")["meta"]["dimension"] == { + "hidden": True, + "ai_hint": "Gross amount.", + "type": "number", + } + assert _column(model, "amount")["meta"]["metrics"]["total_amount"]["ai_hint"] == ( + "Revenue before refunds." + ) + + def test_time_axis_withdrawn_becomes_time_intervals_off(self): + document = _document() + tampered = document.model_copy(deep=True) + orders = tampered.semantic_model[0].datasets[0] + orders.fields[0] = orders.fields[0].model_copy( + update={"datatype": OssieDataType.DATE, "dimension": OssieDimension(is_time=False)} + ) + result = OssieToLightdashConverter().convert(tampered) + column = _column(_model(result.output, "orders"), "order_date") + assert column["meta"]["dimension"] == { + "label": "Order date", + "type": "date", + "time_intervals": "OFF", + } + + def test_meta_can_be_placed_under_config(self): + result = OssieToLightdashConverter(meta_under_config=True).convert(_document()) + model = _model(result.output, "orders") + assert "meta" not in model + assert model["config"]["meta"]["joins"][0]["join"] == "customers" + column = _column(model, "order_date") + assert "meta" not in column + assert column["config"]["meta"]["dimension"]["type"] == "date" + + def test_lightdash_metric_name_comes_from_the_stash_then_the_prefix(self): + document = _document() + tampered = document.model_copy(deep=True) + tampered.semantic_model[0] = tampered.semantic_model[0].model_copy( + update={ + "metrics": [ + OssieMetric( + name="orders_total_amount", + expression=_ansi("SUM(orders.amount)"), + custom_extensions=[ + OssieCustomExtension( + vendor_name="LIGHTDASH", + data=json.dumps({"name": "total_amount", "label": "Total"}), + ) + ], + ), + OssieMetric(name="orders_max_amount", expression=_ansi("MAX(orders.amount)")), + OssieMetric(name="min_amount", expression=_ansi("MIN(orders.amount)")), + ] + } + ) + result = OssieToLightdashConverter().convert(tampered) + metrics = _column(_model(result.output, "orders"), "amount")["meta"]["metrics"] + assert metrics == { + "total_amount": {"type": "sum", "label": "Total"}, + "max_amount": {"type": "max"}, + "min_amount": {"type": "min"}, + } + + def test_unqualified_metric_is_placed_on_the_stashed_model(self): + document = _document() + tampered = document.model_copy(deep=True) + tampered.semantic_model[0] = tampered.semantic_model[0].model_copy( + update={ + "metrics": [ + OssieMetric( + name="customers_row_count", + expression=_ansi("COUNT(*)"), + custom_extensions=[ + OssieCustomExtension( + vendor_name="LIGHTDASH", + data=json.dumps({"name": "row_count", "model": "customers"}), + ) + ], + ), + OssieMetric(name="unplaceable", expression=_ansi("COUNT(*)")), + ] + } + ) + result = OssieToLightdashConverter().convert(tampered) + assert _model(result.output, "customers")["meta"]["metrics"] == { + "row_count": {"type": "number", "sql": "COUNT(*)"} + } + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.CROSS_DATASET_METRIC_DROPPED + ] == ["unplaceable"] + + def test_stashed_joins_and_meta_restore_the_explore(self): + document = _document() + tampered = document.model_copy(deep=True) + datasets = tampered.semantic_model[0].datasets + datasets[0] = datasets[0].model_copy( + update={ + "custom_extensions": [ + OssieCustomExtension( + vendor_name="LIGHTDASH", + data=json.dumps( + { + "sql_filter": "${TABLE}.deleted = false", + "joins": [ + { + "join": "customers", + "sql_on": "${orders.customer_id} = ${customers.customer_id} AND ${customers.active}", + }, + {"join": "regions", "sql_on": "${customers.region_id} = ${regions.id}"}, + ], + } + ), + ) + ] + } + ) + datasets[1] = datasets[1].model_copy( + update={ + "fields": [ + datasets[1].fields[0].model_copy( + update={ + "custom_extensions": [ + OssieCustomExtension( + vendor_name="LIGHTDASH", + data=json.dumps({"column_meta": {"additional_dimensions": {"id_prefix": {"type": "string", "sql": "LEFT(${TABLE}.customer_id, 2)"}}}}), + ) + ] + } + ) + ] + } + ) + regions = OssieDataset( + name="regions", + source="analytics_db.marts.regions", + fields=[OssieField(name="id", expression=_ansi("id"))], + ) + tampered.semantic_model[0] = tampered.semantic_model[0].model_copy( + update={ + "datasets": [*datasets, regions], + "metrics": [ + OssieMetric( + name="regional_spread", + expression=_ansi("COUNT(orders.customer_id) / COUNT(DISTINCT regions.id)"), + ), + ], + } + ) + result = OssieToLightdashConverter().convert(tampered) + orders = _model(result.output, "orders") + # The stashed join replaces the generated one to the same target and + # the chained join is appended; the model meta comes back as is. + assert orders["meta"]["joins"] == [ + { + "join": "customers", + "sql_on": "${orders.customer_id} = ${customers.customer_id} AND ${customers.active}", + }, + {"join": "regions", "sql_on": "${customers.region_id} = ${regions.id}"}, + ] + assert orders["meta"]["sql_filter"] == "${TABLE}.deleted = false" + # A metric over the chained target resolves through the stashed join. + assert orders["meta"]["metrics"]["regional_spread"] == { + "type": "number", + "sql": "COUNT(${TABLE}.customer_id) / COUNT(DISTINCT ${regions.id})", + } + customer_id = _column(_model(result.output, "customers"), "customer_id") + assert customer_id["meta"]["additional_dimensions"] == { + "id_prefix": {"type": "string", "sql": "LEFT(${TABLE}.customer_id, 2)"} + } + + def test_lightdash_model_files_are_deployable_as_they_are(self): + document = _document() + tampered = document.model_copy(deep=True) + datasets = tampered.semantic_model[0].datasets + datasets[0] = datasets[0].model_copy( + update={ + "primary_key": ["order_id"], + "custom_extensions": [ + OssieCustomExtension( + vendor_name="LIGHTDASH", + data=json.dumps({"sql_filter": "${TABLE}.deleted = false", "label": "Orders"}), + ) + ], + } + ) + result = OssieToLightdashConverter().convert_models(tampered) + orders = next(m for m in result.output if m["name"] == "orders") + assert list(orders)[:4] == ["type", "name", "label", "description"] + assert orders["type"] == "model" + assert orders["sql_from"] == "analytics_db.marts.orders" + assert orders["primary_key"] == "order_id" + assert orders["sql_filter"] == "${TABLE}.deleted = false" + assert orders["joins"][0]["join"] == "customers" + assert orders["metrics"]["conversion_rate"]["type"] == "number" + dimensions = {d["name"]: d for d in orders["dimensions"]} + # Every dimension carries its own type and sql; a measure-only field + # is a hidden one; column metrics sit under their dimension. + assert dimensions["order_date"] == { + "name": "order_date", + "type": "date", + "label": "Order date", + "sql": "${TABLE}.order_date", + } + assert dimensions["amount"]["hidden"] is True + assert dimensions["amount"]["sql"] == "${TABLE}.amount" + assert dimensions["amount"]["metrics"]["total_amount"]["type"] == "sum" + # No datatype on `status`: the type is assumed and reported. + assert dimensions["status"]["type"] == "string" + assert [ + issue.element_name + for issue in result.issues + if issue.issue_type is ConverterIssueType.DIMENSION_TYPE_DEFAULTED + ] == ["orders.status", "orders.amount", "orders.customer_id", "customers.customer_id"] diff --git a/converters/lightdash/tests/test_tpcds_roundtrip.py b/converters/lightdash/tests/test_tpcds_roundtrip.py new file mode 100644 index 00000000..22695ca7 --- /dev/null +++ b/converters/lightdash/tests/test_tpcds_roundtrip.py @@ -0,0 +1,173 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Round-trip the in-repo TPC-DS example through the Lightdash converter. + +Ossie -> Lightdash schema.yml -> Ossie must preserve the structural core: +datasets, fields (and their dimension-ness), single-dataset metrics and +relationships. Cross-dataset metrics are the documented exception: Lightdash +model metrics cannot reference other tables, so the export direction drops +them with a CROSS_DATASET_METRIC_DROPPED issue. +""" + +import json +from pathlib import Path + +import yaml + +from ossie import OssieDataType, OssieDocument + +from ossie_lightdash import ( + ConverterIssueType, + LightdashToOssieConverter, + OssieToLightdashConverter, +) + +TPCDS_PATH = Path(__file__).parent / ".." / ".." / ".." / "examples" / "tpcds_semantic_model.yaml" + + +def _lightdash_name(metric) -> str: + for extension in metric.custom_extensions or []: + if extension.vendor_name.upper() == "LIGHTDASH": + return json.loads(extension.data)["name"] + return metric.name + + +def _load_tpcds() -> OssieDocument: + return OssieDocument.model_validate(yaml.safe_load(TPCDS_PATH.read_text())) + + +def _roundtrip(): + original = _load_tpcds() + exported = OssieToLightdashConverter().convert(original) + reimported = LightdashToOssieConverter().convert( + exported.output, + database="tpcds", + schema="public", + semantic_model_name=original.semantic_model[0].name, + ) + return original, exported, reimported + + +class TestTpcdsRoundtrip: + def test_datasets_and_sources_are_preserved(self): + original, _, reimported = _roundtrip() + original_sources = { + dataset.name: dataset.source + for dataset in original.semantic_model[0].datasets + } + roundtripped_sources = { + dataset.name: dataset.source + for dataset in reimported.output.semantic_model[0].datasets + } + assert roundtripped_sources == original_sources + + def test_fields_and_dimension_markers_are_preserved(self): + original, _, reimported = _roundtrip() + for original_dataset, roundtripped_dataset in zip( + original.semantic_model[0].datasets, + reimported.output.semantic_model[0].datasets, + ): + original_fields = { + field.name: field.dimension is not None + for field in original_dataset.fields or [] + } + roundtripped_fields = { + field.name: field.dimension is not None + for field in roundtripped_dataset.fields or [] + } + assert roundtripped_fields == original_fields + + def test_datatype_categories_survive_the_round_trip(self): + """Lightdash types are coarser than Ossie datatypes, so a round-trip + preserves the category (temporal / numeric / string / boolean) rather + than the exact member (e.g. Integer comes back as Decimal).""" + categories = { + OssieDataType.STRING: "string", + OssieDataType.BOOLEAN: "boolean", + OssieDataType.INTEGER: "number", + OssieDataType.DECIMAL: "number", + OssieDataType.FLOAT: "number", + OssieDataType.DATE: "date", + OssieDataType.DATE_TIME: "timestamp", + OssieDataType.DATE_TIME_TZ: "timestamp", + } + original, _, reimported = _roundtrip() + for original_dataset, roundtripped_dataset in zip( + original.semantic_model[0].datasets, + reimported.output.semantic_model[0].datasets, + ): + roundtripped_by_name = { + field.name: field for field in roundtripped_dataset.fields or [] + } + for field in original_dataset.fields or []: + # Measure-only fields have no Lightdash dimension to carry a + # type, so their datatype is not expected to survive. + if field.datatype not in categories or field.dimension is None: + continue + roundtripped = roundtripped_by_name[field.name] + assert categories.get(roundtripped.datatype) == categories[ + field.datatype + ], field.name + + def test_time_role_marker_is_not_carried_into_lightdash(self): + """`is_time` is an Ossie role marker with no Lightdash equivalent, so the + import direction leaves it unset instead of inferring one.""" + _, _, reimported = _roundtrip() + for dataset in reimported.output.semantic_model[0].datasets: + for field in dataset.fields or []: + if field.dimension is not None: + assert field.dimension.is_time is None, field.name + + def test_single_dataset_metrics_survive_with_expressions(self): + original, exported, reimported = _roundtrip() + dropped = { + issue.element_name + for issue in exported.issues + if issue.issue_type is ConverterIssueType.CROSS_DATASET_METRIC_DROPPED + } + original_metrics = { + metric.name: metric.expression.dialects[0].expression + for metric in original.semantic_model[0].metrics or [] + if metric.name not in dropped + } + # Re-imported metrics carry Lightdash's `_` id as their + # Ossie name; the original name is the stashed Lightdash name. + roundtripped_metrics = { + _lightdash_name(metric): metric.expression.dialects[0].expression + for metric in reimported.output.semantic_model[0].metrics or [] + } + assert set(roundtripped_metrics) == set(original_metrics) + assert "store_sales_total_sales" in { + metric.name for metric in reimported.output.semantic_model[0].metrics + } + for name, expression in original_metrics.items(): + assert roundtripped_metrics[name].replace(" ", "") == expression.replace( + " ", "" + ), name + + def test_relationships_are_preserved(self): + original, _, reimported = _roundtrip() + original_edges = { + (r.from_dataset, r.to, tuple(r.from_columns), tuple(r.to_columns)) + for r in original.semantic_model[0].relationships or [] + } + roundtripped_edges = { + (r.from_dataset, r.to, tuple(r.from_columns), tuple(r.to_columns)) + for r in reimported.output.semantic_model[0].relationships or [] + } + assert roundtripped_edges == original_edges diff --git a/converters/lightdash/uv.lock b/converters/lightdash/uv.lock new file mode 100644 index 00000000..682505de --- /dev/null +++ b/converters/lightdash/uv.lock @@ -0,0 +1,304 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "apache-ossie" +version = "0.2.0.dev0" +source = { editable = "../../python" } +dependencies = [ + { name = "pydantic" }, + { name = "pyyaml" }, +] + +[package.metadata] +requires-dist = [ + { name = "pydantic", specifier = ">=2.0" }, + { name = "pyyaml", specifier = ">=6.0" }, +] + +[[package]] +name = "apache-ossie-lightdash" +version = "0.1.0.dev0" +source = { editable = "." } +dependencies = [ + { name = "apache-ossie" }, + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "apache-ossie", editable = "../../python" }, + { name = "pyyaml", specifier = ">=6.0" }, +] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8.0" }] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index c87af723..a1efdb43 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -28,7 +28,7 @@ }, "Vendor": { "type": "string", - "examples": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM"], + "examples": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM", "LIGHTDASH"], "description": "Vendor name for custom extensions. Any string value is accepted." }, "AIContext": { diff --git a/core-spec/spec.md b/core-spec/spec.md index 2850edfe..8b33452a 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -447,6 +447,7 @@ The following are well-known examples: | `GOODDATA` | GoodData-specific attributes | | `HONEYDEW` | Honeydew-specific attributes | | `WISDOM` | WisdomAI-specific attributes | +| `LIGHTDASH` | Lightdash-specific attributes | ### Examples diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 4678b004..93109ad6 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -55,7 +55,7 @@ datatypes: - "Opaque" # Known type outside the portable vocabulary # Vendor name for custom extensions (free-form string) -# Examples: "COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM" +# Examples: "COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM", "LIGHTDASH" vendor_name: string diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index e782275c..d3188bdb 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -71,6 +71,7 @@ class OssieVendor(str, Enum): GOODDATA = "GOODDATA" SEMANTIDO = "SEMANTIDO" WISDOM = "WISDOM" + LIGHTDASH = "LIGHTDASH" class OssieAIContextObject(BaseModel):