From 4b3bb709c6bf46cb4bfdb00471e33c42a769dd13 Mon Sep 17 00:00:00 2001 From: Zack Martin Date: Mon, 17 Aug 2026 12:04:08 -0700 Subject: [PATCH] [OSSIE][SOLID] Add bidirectional Solid semantic model converter Adds converters/solid, a bidirectional offline converter between an Apache Ossie semantic model and a Solid semantic model YAML export, following the hub-and-spoke pattern described in converters/README.md. Import (Solid -> Ossie) preserves every Solid-only construct -- example queries, benchmark questions, quality rank, indexes, sample values, and each column's raw warehouse type -- in custom_extensions[SOLID], so the round trip is lossless. Export (Ossie -> Solid) emits the key order Solid's own exporter uses, and warns rather than silently dropping the Ossie constructs Solid's format cannot hold. Three areas needed design decisions, documented in the converter README: * Dialect resolution. Solid's export does not record its source warehouse, but Ossie requires a dialect on every expression. The converter infers one from the raw column type vocabulary (NUMBER/TEXT means Snowflake, LONG/MAP means Databricks, INT64/FLOAT64 means BigQuery), overridable with --dialect and falling back to ANSI_SQL with a warning. * Metric expressions. Solid stores formulas against bare column names with the owning table recorded separately; Ossie expects them qualified. The rewrite is a surgical splice at tokenizer offsets rather than a re-render, because round-tripping a parsed tree through sqlglot's generator canonicalizes SQL the converter was only asked to qualify. The parser cross-checks the token scan, and any disagreement leaves the expression exactly as written. * Cardinality. Ossie encodes it by direction; a Solid relationship is an undirected pair of column lists. The one side is recovered from the primary keys, with the original orientation preserved for export. Tests cover both directions against three fixtures -- the repository's TPC-DS model expressed as a Solid export, plus a Databricks and a BigQuery model -- and validate every converted document against core-spec/osi-schema.json. Reviewed-by: Eden Litvin --- .github/workflows/converter-solid-ci.yml | 63 ++ converters/README.md | 1 + converters/solid/README.md | 357 +++++++++ converters/solid/pyproject.toml | 63 ++ converters/solid/src/ossie_solid/__init__.py | 36 + converters/solid/src/ossie_solid/_common.py | 391 ++++++++++ converters/solid/src/ossie_solid/cli.py | 97 +++ converters/solid/src/ossie_solid/datatypes.py | 285 +++++++ .../solid/src/ossie_solid/expressions.py | 286 +++++++ .../solid/src/ossie_solid/ossie_to_solid.py | 465 +++++++++++ .../solid/src/ossie_solid/solid_to_ossie.py | 484 ++++++++++++ converters/solid/tests/conftest.py | 144 ++++ .../solid/tests/fixtures/bigquery_solid.yaml | 78 ++ .../tests/fixtures/databricks_solid.yaml | 106 +++ .../solid/tests/fixtures/foreign_ossie.yaml | 149 ++++ .../solid/tests/fixtures/tpcds_ossie.yaml | 723 ++++++++++++++++++ .../solid/tests/fixtures/tpcds_solid.yaml | 424 ++++++++++ converters/solid/tests/test_cli.py | 111 +++ converters/solid/tests/test_cross_vendor.py | 246 ++++++ converters/solid/tests/test_datatypes.py | 212 +++++ converters/solid/tests/test_expressions.py | 223 ++++++ converters/solid/tests/test_ossie_to_solid.py | 441 +++++++++++ converters/solid/tests/test_roundtrip.py | 215 ++++++ converters/solid/tests/test_solid_to_ossie.py | 347 +++++++++ converters/solid/uv.lock | 337 ++++++++ 25 files changed, 6284 insertions(+) create mode 100644 .github/workflows/converter-solid-ci.yml create mode 100644 converters/solid/README.md create mode 100644 converters/solid/pyproject.toml create mode 100644 converters/solid/src/ossie_solid/__init__.py create mode 100644 converters/solid/src/ossie_solid/_common.py create mode 100644 converters/solid/src/ossie_solid/cli.py create mode 100644 converters/solid/src/ossie_solid/datatypes.py create mode 100644 converters/solid/src/ossie_solid/expressions.py create mode 100644 converters/solid/src/ossie_solid/ossie_to_solid.py create mode 100644 converters/solid/src/ossie_solid/solid_to_ossie.py create mode 100644 converters/solid/tests/conftest.py create mode 100644 converters/solid/tests/fixtures/bigquery_solid.yaml create mode 100644 converters/solid/tests/fixtures/databricks_solid.yaml create mode 100644 converters/solid/tests/fixtures/foreign_ossie.yaml create mode 100644 converters/solid/tests/fixtures/tpcds_ossie.yaml create mode 100644 converters/solid/tests/fixtures/tpcds_solid.yaml create mode 100644 converters/solid/tests/test_cli.py create mode 100644 converters/solid/tests/test_cross_vendor.py create mode 100644 converters/solid/tests/test_datatypes.py create mode 100644 converters/solid/tests/test_expressions.py create mode 100644 converters/solid/tests/test_ossie_to_solid.py create mode 100644 converters/solid/tests/test_roundtrip.py create mode 100644 converters/solid/tests/test_solid_to_ossie.py create mode 100644 converters/solid/uv.lock diff --git a/.github/workflows/converter-solid-ci.yml b/.github/workflows/converter-solid-ci.yml new file mode 100644 index 00000000..674f737c --- /dev/null +++ b/.github/workflows/converter-solid-ci.yml @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Converters Solid CI + +on: + push: + branches: [ "main" ] + paths: + - 'converters/solid/**' + - '.github/workflows/converter-solid-ci.yml' + pull_request: + branches: [ "main" ] + paths: + - 'converters/solid/**' + - '.github/workflows/converter-solid-ci.yml' + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" + + - name: Sync dependencies + working-directory: converters/solid + run: | + uv sync + + - name: Unit Tests + working-directory: converters/solid + run: | + uv run pytest diff --git a/converters/README.md b/converters/README.md index 417d7663..080c8dba 100644 --- a/converters/README.md +++ b/converters/README.md @@ -76,6 +76,7 @@ The Ossie specification currently defines extensions for the following vendors: | `OMNI` | Omni semantic model | | `WISDOM` | WisdomAI domain | | `NVIDIA_GSF` | NVIDIA Generative Semantic Fabric standalone YAML | +| `SOLID` | Solid semantic model | Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. diff --git a/converters/solid/README.md b/converters/solid/README.md new file mode 100644 index 00000000..b9b3956c --- /dev/null +++ b/converters/solid/README.md @@ -0,0 +1,357 @@ + + +# Apache Ossie Solid Converter + +Bidirectional, offline conversion between an [Apache Ossie](https://github.com/apache/ossie) +semantic model and a [Solid](https://www.getsolid.ai/) semantic model YAML export. No +Solid connection required. + +- **Import** (`ossie-solid import`): Solid → Apache Ossie. Solid features Apache Ossie + has no native field for are preserved in `custom_extensions[SOLID]`, so + `Solid → Apache Ossie → Solid` is lossless. +- **Export** (`ossie-solid export`): Apache Ossie → Solid, in the key order + Solid's own exporter uses. + +Solid is an AI analytics platform whose semantic layer is authored against a warehouse +catalog and consumed by its text-to-SQL agents. Its export is the YAML you get from +**Download Solid Semantic Model YAML**, and the same document Solid stores as a semantic +layer version. + +On **export** (Apache Ossie → Solid), Apache Ossie constructs with no Solid slot — +`unique_keys`, field `label`, computed dimensions, metric `datatype`, relationship +`ai_context`, foreign-vendor `custom_extensions`, `ai_context.examples` below the model +level — are **dropped with a warning**. On +**import** (Solid → Apache Ossie), Solid-only features are instead **preserved** in +`custom_extensions[SOLID]`. Any input that breaks a [requirement](#requirements) +**raises a `ConversionError`** — the converter never silently drops a field or produces +an invalid result. + +## Installation + +```bash +pip install apache-ossie-solid # once published to PyPI +``` + +Or, from a checkout of this directory: + +```bash +pip install -e . +``` + +Runtime dependencies are `PyYAML` and `sqlglot`. Python 3.11+. + +## Usage + +### Command line + +```bash +ossie-solid import -i solid_model.yaml -o model.yaml [--dialect SNOWFLAKE] +``` + +```bash +ossie-solid export -i model.yaml -o solid_model.yaml [--dialect SNOWFLAKE] +``` + +With no `-o`, output goes to stdout; warnings always go to stderr. `--name` overrides the +model name in either direction. `--dialect` is described under +[Dialect resolution](#dialect-resolution). + +### Python API + +```python +from ossie_solid import convert_solid_to_ossie, convert_ossie_to_solid + +ossie_yaml = convert_solid_to_ossie(solid_yaml_str) # or dialect="SNOWFLAKE" +solid_yaml = convert_ossie_to_solid(ossie_yaml_str, model_name="sales") +``` + +## Dialect resolution + +**Solid's YAML does not record which warehouse a model came from.** The source system is +held on the asset row in Solid's own database and is dropped when the YAML is rendered, +but Apache Ossie requires a dialect on every expression. The converter resolves one in +this order: + +1. **`--dialect`**, when given. Always wins. +2. **Inference from the column type vocabulary.** Solid copies each column's `type` out + of the catalog verbatim, so the names identify the warehouse: `NUMBER`/`TEXT`/ + `VARIANT` mean Snowflake, `LONG`/`MAP`/`BIGINT` mean Databricks, `INT64`/`FLOAT64`/ + `BOOL` mean BigQuery. Only names unique to one warehouse vote; shared ones + (`STRING`, `BOOLEAN`, `DATE`, `TIMESTAMP_NTZ`) are ignored. +3. **`ANSI_SQL`**, with a warning, when nothing votes or two warehouses tie. + +The resolved dialect is recorded in `custom_extensions[SOLID]`, so an export reads +expressions back in the dialect they were written in without being told again. + +On export, `--dialect` selects which expression dialect to *read*; without it the +converter uses the dialect recorded at import, then the single non-`ANSI_SQL` dialect the +model's expressions use. A field with neither the chosen dialect nor `ANSI_SQL` raises. + +**Only SQL dialects are read.** Apache Ossie's dialect enum also covers expression +languages that are not SQL — `MDX`, `TABLEAU`, `MAQL` — which this converter cannot +parse, qualify or unqualify. One of those never becomes the resolved dialect: the model +is reported, and each expression's `ANSI_SQL` form is read instead. An expression +offering only a non-SQL dialect raises rather than passing a formula Solid cannot +execute into a SQL field. (This is what lets GoodData's `MAQL`-and-`ANSI_SQL` models +convert on their SQL side.) + +## Mapping + +Each row maps in both directions; the **Notes** flag where a behavior is specific to +**import** (Solid → Apache Ossie) or **export** (Apache Ossie → Solid). + +### Model + +| Apache Ossie | Solid | Notes | +|---|---|---| +| `semantic_model[0].name` | `semantic_model.name` | Solid holds one model per file; export warns if the document has more. | +| `description` | `model_llm_description` | | +| `ai_context.instructions` | `business_context.custom_instructions` | Import resolves Solid's `@` markup to the display names it carries and keeps the tagged original in the stash, so an export restores the live catalog references. | +| `ai_context.examples` | `business_context.business_questions` | | +| `custom_extensions[SOLID]` | `business_context.model_description`, `example_queries`, `benchmark_questions` | No Apache Ossie core equivalent. | + +### Datasets + +| Apache Ossie | Solid | Notes | +|---|---|---| +| `dataset.source` | `tables[].name` | Solid names a table by its `catalog.schema.table`. | +| `dataset.name` | — | Derived on import from the last part of the FQN, widening to `schema_table` and then the whole FQN if a shorter form is taken. Apache Ossie dataset names double as expression qualifiers, so a dotted name is not usable. | +| `description` | `description` | Solid's AI-written description. | +| `ai_context.instructions` | `manual_description` | Solid's human-written annotation. | +| `ai_context.synonyms` | `synonyms` | | +| `primary_key` (array) | `primary_key` (scalar) | Solid joins a composite key with `", "` into one scalar; import splits it, export rejoins it. | +| `custom_extensions[SOLID]` | `quality_rank`, `indexes` | No Apache Ossie core equivalent. Export emits `quality_rank` even when unknown, as Solid does. | + +### Fields + +| Apache Ossie | Solid | Notes | +|---|---|---| +| `fields[]` | `dimensions[]` + `facts[]` | Solid splits columns two ways; Apache Ossie has one list. Import records the split as the presence of the `dimension` block, and export reads it back. An Apache Ossie model that carries no `dimension` block anywhere is split the way Solid itself does it — by data type. | +| `dimension.is_time` | — | Set explicitly on import from the resolved `datatype`, rather than left to the spec's default, so a consumer that does not implement the default still reads the same role. | +| `expression.dialects[]` | the column's own name | A Solid column's expression is a bare reference to itself; a name needing quoting is quoted for the resolved dialect. | +| `datatype` | `type` | See [Data types](#data-types). | +| `description` | `description` | | +| `ai_context.instructions` | `manual_description` | | +| `ai_context.synonyms` | `synonyms` | | +| `custom_extensions[SOLID]` | `type` (raw), `sample_values` | The raw warehouse type is always kept, since the portable `datatype` is lossy. | + +A Solid **fact may be a table-scoped metric** rather than a catalog column — it carries an +`expression` and no `type`. Import keeps the expression and marks the field +`role: metric` in the stash; export re-emits it as an expression-only fact. + +### Relationships + +| Apache Ossie | Solid | Notes | +|---|---|---| +| `from` / `to` | `left_table` / `right_table` | See [Cardinality](#cardinality). | +| `from_columns` / `to_columns` | `join_keys.left` / `join_keys.right` | Positional; import rejects a length mismatch. | +| `name` | — | Solid relationships are unnamed. Import generates `{from}_to_{to}`, suffixed if that collides. | +| `ai_context` | — | A Solid relationship is a table pair and its join keys, with no free text. Export warns — except for the one-to-one note import itself writes, which is this converter's own marker and means nothing to Solid. | +| `custom_extensions[SOLID]` | — | Records the original left/right orientation and position, so an export reproduces Solid's ordering. | + +### Metrics + +| Apache Ossie | Solid | Notes | +|---|---|---| +| `metrics[]` | `metrics[]` | Solid metrics are already model-level. | +| `expression.dialects[]` | `expression` + `tables[]` | See [Metric expressions](#metric-expressions). | +| `description` | `description` | | +| `datatype` | — | Solid types a metric by evaluating its formula against the warehouse, so there is no slot for a declared result type. Export warns. | +| `ai_context.synonyms` | `synonyms` | | +| `custom_extensions[SOLID]` | `tables[]` | Stashed only when the qualified expression does not already name the same datasets — for a cross-table metric, or one referencing no column at all (`COUNT(*)`). | + +## Cardinality + +Apache Ossie encodes cardinality through direction — `from` is the many side, `to` the one +side — but a Solid relationship is an undirected pair of column lists. Import recovers the +direction from the primary keys: + +| Condition | Result | +|---|---| +| The right table's primary key is exactly its join columns | `from` = left, `to` = right | +| The left table's primary key is exactly its join columns | `from` = right, `to` = left (recorded as `flipped`) | +| Both | One-to-one: direction is arbitrary, and a note is added to the relationship's `ai_context` | +| Neither | `from` = left, with a warning — Solid records no cardinality and its `unique_keys` are not exported | + +## Metric expressions + +Solid stores a metric formula against **bare** column names and records the owning table +separately; `is_valid_metric_formula` in solid-server actively strips alias prefixes +before saving. Apache Ossie expects a metric to be self-contained, qualifying each column +with its dataset. So: + +- **Import** prefixes each bare column with its dataset when `tables[]` names exactly one + table (`SUM(ss_ext_sales_price)` → `SUM(store_sales.ss_ext_sales_price)`). +- **Export** strips those qualifiers back out. + +Both edits are a **surgical splice**, not a re-render. Each column is located by +sqlglot's tokenizer, which reports byte offsets and keeps string literals and quoted +identifiers in token types of their own, and the qualifier is inserted or removed at those +offsets. Everything else is left byte-for-byte intact — round-tripping a parsed tree +through `Expression.sql()` would canonicalize as it generates, turning +`CAST(x AS FLOAT)` into `CAST(x AS DOUBLE)` and `EXTRACT(year FROM d)` into +`DATE_PART(YEAR, d)`, and a converter has no business rewriting SQL it was only asked to +qualify. + +The parser is still used as a cross-check: it decides which names are genuinely bare +column references, and a token scan that disagrees with it means some occurrence is +something else. On any disagreement — an unparseable expression, or a column also named +`year` in `EXTRACT(year FROM …)` — the expression is left **exactly as written** and a +warning is emitted. An unqualified metric is a far smaller problem than a corrupted one. + +**A cross-table metric cannot be qualified.** Solid's `tables[]` may hold more than one +table, and the alias-to-table binding lives in `metric.column_ids`, which its YAML export +does not carry. Those metrics keep their bare column names, keep `tables[]` in the stash, +and emit a warning. + +## Data types + +Solid stores the **raw** warehouse type string verbatim (`NUMBER(38,0)`, `TEXT`, `INT64`, +`MAP`), so mapping onto Apache Ossie's portable `datatype` is lossy — +`NUMBER(38,2)`, `NUMERIC` and `DECIMAL` all collapse to `Decimal`. The raw string is +therefore always kept in the stash and is what an export re-emits; the derived table below +is only used for an Apache Ossie model that has no stash to read. + +| Warehouse type | Apache Ossie `datatype` | +|---|---| +| `STRING`, `TEXT`, `VARCHAR`, `CHAR` | `String` | +| `INT`, `BIGINT`, `LONG`, `INT64`, `TINYINT` | `Integer` | +| `NUMBER(p,0)`, `NUMERIC(p,0)`, `DECIMAL(p,0)` | `Integer` | +| `NUMBER(p,s>0)`, `NUMERIC`, `DECIMAL`, `BIGNUMERIC` | `Decimal` | +| `FLOAT`, `FLOAT64`, `DOUBLE`, `REAL` | `Float` | +| `BOOLEAN`, `BOOL` | `Boolean` | +| `DATE` / `TIME` | `Date` / `Time` | +| `TIMESTAMP_NTZ`, `DATETIME` | `DateTime` | +| `TIMESTAMP_TZ`, `TIMESTAMP_LTZ` | `DateTimeTz` | +| `TIMESTAMP` | `DateTime` on Snowflake, `DateTimeTz` on Databricks and BigQuery | +| `VARIANT`, `OBJECT`, `ARRAY`, `MAP`, `STRUCT`, `BINARY`, `GEOGRAPHY` | `Opaque` | +| anything else | omitted, with a warning | + +A declared scale of `0` is read as an integer, which is what keeps Snowflake ids and +counts — stored as `NUMBER(38,0)` — from being typed `Decimal`. + +## Requirements + +Conversion raises a `ConversionError` (rather than guessing or emitting something invalid) +when an input breaks one of these: + +- the Apache Ossie `version` is neither `0.2.0.dev0` nor `0.1.1` (export) — see + [Spec versions](#spec-versions); +- the input YAML is malformed, or its root is not a mapping; +- a Solid model has no `tables`, or an Apache Ossie model has no `datasets` — the spec + requires at least one dataset; +- a relationship's `from_columns`/`to_columns` differ in length, or name a dataset that + is not declared; +- `--dialect` names a dialect outside `ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`, `BIGQUERY`; +- a field or metric has neither the selected dialect nor `ANSI_SQL` (export); +- the recorded `custom_extensions[SOLID]` dialect is not one of the four above — which + only a hand-edited stash can produce (export). + +## Spec versions + +Output is always written as **`0.2.0.dev0`**, the current draft. On export, a document +declaring either `0.2.0.dev0` or **`0.1.1`** is read; anything else raises. + +`0.1.1` is accepted because it is the only *released* spec version, so it is what models +in the wild — and several of this repository's own converter fixtures — declare. As a +document, a `0.1.1` model is a `0.2.0.dev0` model minus three purely additive changes: +`datatype` on `Field` and `Metric`, `BIGQUERY` in the dialect enum, and a free-form +rather than enumerated `vendor_name`. Nothing in a `0.1.1` document is therefore invalid +under `0.2.0.dev0`, and no separate read path is needed. Only the absence of `datatype` +is visible to this converter, and a `0.2.0.dev0` model that simply omits it — as most do +— is already handled the same way. Reading a `0.1.1` document warns, naming that gap. + +## Known differences + +- **An empty description is normalized away.** solid-server renders a whitespace-only + `manual_description` as an empty block scalar, which parses back as `''`. That carries + no information, so the converter drops it rather than round-tripping the emptiness. +- **`unique_keys` cannot survive an export.** Solid's YAML has no unique-key field; a + single unique key identical to the primary key is dropped silently, any other is + dropped with a warning. +- **A Solid model carries no `unique_keys` to import**, so a re-imported model relies on + `primary_key` alone to determine relationship cardinality. + +## Development + +```bash +uv sync && uv run pytest +``` + +The suite covers both directions against three Solid fixtures — the repository's +[TPC-DS model](../../examples/tpcds_semantic_model.yaml) expressed as a Solid export, plus +a Databricks and a BigQuery model — and validates every converted document against the +[Apache Ossie JSON Schema](../../core-spec/ossie-schema.json). + +`tests/test_cross_vendor.py` covers the case the round-trip tests cannot: an Apache Ossie +model produced by *another vendor's* converter, which carries no `custom_extensions[SOLID]` +stash, no `dimension` blocks, usually no `datatype`, and metrics written against bare +column names. It sweeps every Ossie fixture the other converters in this repository ship, +asserting that each converts to a well-formed Solid model that re-imports schema-valid. +That sweep deliberately pins **no** warning counts, because those fixtures belong to other +converters and each converter's CI is path-filtered to its own directory — a pin here would +break on a main-branch push from a PR that never ran this suite. The exact interop gaps are +pinned instead against +[`tests/fixtures/foreign_ossie.yaml`](tests/fixtures/foreign_ossie.yaml), a fixture this +converter owns that reproduces the same constructs. + +To regenerate the expected-output fixture after an intentional change: + +```bash +uv run ossie-solid import -i tests/fixtures/tpcds_solid.yaml -o tests/fixtures/tpcds_ossie.yaml +``` + +then re-add the license header at the top of the file. + +## Future effort + +Both the Apache Ossie specification and Solid's semantic model are still evolving, and +this converter will be updated to track them. Several gaps are worth naming. + +Three of these are **open interop decisions**, pinned as such in +`tests/test_cross_vendor.py` rather than resolved. Each is a judgement about what Solid +should do with a foreign model, not a defect in the transform: + +- **A column with no `datatype` gets an empty Solid `type`.** `datatype` is optional in + the spec and most converters omit it, so this is the largest single gap when importing + a foreign model — see the counts in `test_the_documented_interop_gaps_are_exactly_these`. + It cannot be closed offline: Solid types its columns from the warehouse catalog, and + the honest fix is to reconcile against the live catalog at import time rather than to + guess here. What the converter owes that flow is a machine-readable list of what needs + reconciling, which today it only reports as warnings. +- **A field renamed relative to its column loses the column it reads.** Given + `name: ticket_number` over `expression: ss_ticket_number`, the alias survives and the + real column does not, so the Solid model names a column the warehouse does not have. + Emitting the underlying name and keeping the alias as a synonym would be truer, but it + changes which identifier downstream Solid consumers see and so needs a product call. +- **A metric written against bare column names reaches Solid with `tables: []`.** Only a + dataset-qualified reference identifies its owner. Resolving bare names against each + dataset's declared fields would fix most cases; it is inference, so it is deliberately + not done silently. + +And two are **format-level**: + +- **Verified queries have no home in the core spec.** Solid's `example_queries` and + `benchmark_questions` are currently stashed, but the same construct appears across the + ecosystem — Snowflake Cortex `verified_queries`, WisdomAI reviewed queries — and looks + like a candidate for a core field rather than N vendor extensions. +- **Solid's export does not record its source warehouse**, which is why the dialect has to + be inferred. Adding it to the export would make the format self-describing; the + converter would read it first and keep inference only as a fallback for existing files. diff --git a/converters/solid/pyproject.toml b/converters/solid/pyproject.toml new file mode 100644 index 00000000..f9622a33 --- /dev/null +++ b/converters/solid/pyproject.toml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "pytest>=8.0", + "jsonschema>=4.26.0", +] + +[project] +name = "apache-ossie-solid" +version = "0.2.0.dev0" +description = "Solid semantic model <> Apache Ossie converter" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] +requires-python = ">=3.11" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Solid" +] +dependencies = [ + "PyYAML>=6.0", + "sqlglot>=25.0", +] + +[project.scripts] +ossie-solid = "ossie_solid.cli:main" + +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_solid"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev" +] diff --git a/converters/solid/src/ossie_solid/__init__.py b/converters/solid/src/ossie_solid/__init__.py new file mode 100644 index 00000000..ef699631 --- /dev/null +++ b/converters/solid/src/ossie_solid/__init__.py @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Bidirectional converter between Apache Ossie and Solid semantic models.""" + +from ._common import ( + OSSIE_VERSION, + SUPPORTED_DIALECTS, + VENDOR, + ConversionError, +) +from .ossie_to_solid import convert_ossie_to_solid +from .solid_to_ossie import convert_solid_to_ossie + +__all__ = [ + "OSSIE_VERSION", + "SUPPORTED_DIALECTS", + "VENDOR", + "ConversionError", + "convert_ossie_to_solid", + "convert_solid_to_ossie", +] diff --git a/converters/solid/src/ossie_solid/_common.py b/converters/solid/src/ossie_solid/_common.py new file mode 100644 index 00000000..7cd454c1 --- /dev/null +++ b/converters/solid/src/ossie_solid/_common.py @@ -0,0 +1,391 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared helpers for the Apache Ossie <-> Solid semantic model converters. + +Both directions are pure offline YAML transforms. The cross-cutting concerns live +here: version constants, the `custom_extensions` stash protocol, YAML I/O, the +Solid asset-link markup, and small identifier helpers. +""" + +import json +import re +import warnings + +import yaml + +# Apache Ossie semantic model spec version this converter writes (see core-spec). +# +# NOTE: this spoke has no `apache-ossie` runtime dependency, so nothing updates the +# constant automatically -- it MUST be bumped in lockstep with the `version` in +# `core-spec/` whenever the spec version moves. +OSSIE_VERSION = "0.2.0.dev0" + +# Spec versions an Apache Ossie document may declare and still be read (see +# convert_ossie_to_solid). Output is always written as OSSIE_VERSION. +# +# 0.1.1 is the only *released* spec version, so it is what models in the wild and +# several of the repository's own converter fixtures declare. As a document, a 0.1.1 +# model is a 0.2.0.dev0 model minus three additions: `datatype` on Field and Metric, +# `BIGQUERY` in the dialect enum, and a free-form (rather than enumerated) +# `vendor_name`. Every one of those is additive, so nothing in a 0.1.1 document is +# invalid under 0.2.0.dev0 and no separate read path is needed -- only the absence of +# `datatype` is visible to this converter, and that is already handled the same way a +# 0.2.0.dev0 model that simply omits it is handled. +READABLE_OSSIE_VERSIONS = ("0.2.0.dev0", "0.1.1") + +# Read-only spec versions whose omissions are worth naming when one is encountered. +OSSIE_VERSION_NOTES = { + "0.1.1": "it predates the `datatype` field, so a column with no stashed raw type " + "gets an empty Solid 'type'", +} + +# Vendor id used for the `custom_extensions` stash. +VENDOR = "SOLID" + +# Note import writes to a one-to-one relationship's `ai_context.instructions`, where +# Apache Ossie's directional from/to carries no meaning (see solid_to_ossie._orient). +# Export recognizes it as this converter's own marker rather than a user annotation, so +# it is dropped silently on the way back while real annotations are reported. +ONE_TO_ONE_NOTE = ("One-to-one relationship: both sides are unique on the join " + "columns, so the from/to direction is arbitrary.") + +# Bump when the shape of a stashed `data` blob changes. +STASH_VERSION = 1 + +# Apache Ossie dialects this converter can resolve a Solid model to. Solid itself +# records the warehouse on the asset row, not in the exported YAML, so the dialect is +# either supplied explicitly or inferred from the column type vocabulary +# (see datatypes.infer_dialect). +DIALECT_ANSI = "ANSI_SQL" +DIALECT_SNOWFLAKE = "SNOWFLAKE" +DIALECT_DATABRICKS = "DATABRICKS" +DIALECT_BIGQUERY = "BIGQUERY" + +SUPPORTED_DIALECTS = ( + DIALECT_ANSI, + DIALECT_SNOWFLAKE, + DIALECT_DATABRICKS, + DIALECT_BIGQUERY, +) + +# Apache Ossie dialect -> sqlglot dialect name. ANSI_SQL maps to sqlglot's default +# ("" / None) parser rather than a vendor grammar. +SQLGLOT_DIALECTS = { + DIALECT_ANSI: None, + DIALECT_SNOWFLAKE: "snowflake", + DIALECT_DATABRICKS: "databricks", + DIALECT_BIGQUERY: "bigquery", +} + +# Solid renders a composite primary key as a single comma-joined scalar +# (`primary_key: 'ORDER_ID, LINE_NO'`); see semantic_layer_utils.py in solid-server. +# Apache Ossie models it as a column array, so the two are split/joined on this. +PK_SEPARATOR = ", " + +# Solid embeds references to its own catalog objects inside `custom_instructions` as +# self-closing markup carrying an internal UUID: +# +# @ +# +# There is no closing tag and attribute order is not significant. Solid resolves these +# to their `name` before the text reaches an LLM; this converter does the same for the +# Apache Ossie `ai_context.instructions` a downstream tool will read, and keeps the raw +# tagged text in the stash so an export reproduces the original byte-for-byte. +_ASSET_LINK_RE = re.compile(r"@]+)>", re.IGNORECASE) +_ASSET_LINK_ATTR_RE = re.compile(r"(\w+)\s*=\s*[\"']([^\"']*)[\"']") + + +class ConversionError(Exception): + """Raised when an input cannot be converted.""" + + +def warn(scope, message): + """Emit a conversion warning. The CLI surfaces these on stderr.""" + warnings.warn(f"[{scope}] {message}", stacklevel=2) + + +def require(obj, key, what): + """Return `obj[key]`, or raise a clean ConversionError if it is missing/empty. + + Presence is tested by key rather than truthiness, so a legitimately falsy value such + as `0` or `False` is returned; a missing key, a null, or an empty/whitespace-only + string is rejected. + """ + if not isinstance(obj, dict) or key not in obj or obj[key] is None: + raise ConversionError(f"{what} is missing required '{key}'") + value = obj[key] + if isinstance(value, str) and not value.strip(): + raise ConversionError(f"{what} has an empty '{key}'") + return value + + +def require_str(obj, key, what): + """Like require(), but also enforce that the value is a string -- so a non-string + scalar (e.g. a YAML number used as a name) raises a clean ConversionError instead of + failing later inside a string operation.""" + value = require(obj, key, what) + if not isinstance(value, str): + raise ConversionError( + f"{what}: '{key}' must be a string, got {type(value).__name__}") + return value + + +def load_yaml(text): + """Parse YAML, surfacing a syntax error as a ConversionError so callers (and the + CLI) get a clean message rather than a raw traceback.""" + try: + return yaml.safe_load(text) + except yaml.YAMLError as e: + raise ConversionError(f"Invalid YAML: {e}") from e + + +def dump_yaml(obj): + """Serialize to YAML, preserving key insertion order and allowing Unicode through + unescaped (Solid descriptions routinely contain non-ASCII punctuation).""" + return yaml.safe_dump( + obj, sort_keys=False, default_flow_style=False, allow_unicode=True, width=100 + ) + + +def read_stash(obj): + """Return the SOLID stash dict on an Apache Ossie object, or {} if absent. + + The `_v` version marker is stripped from the returned dict. + """ + for ext in (obj or {}).get("custom_extensions") or []: + if ext.get("vendor_name") == VENDOR: + try: + data = json.loads(ext.get("data") or "{}") + except json.JSONDecodeError as e: + raise ConversionError( + f"SOLID custom_extensions data is not valid JSON: {e}") from e + if not isinstance(data, dict): + raise ConversionError( + "SOLID custom_extensions data must be a JSON object") + data.pop("_v", None) + return data + return {} + + +def write_stash(obj, data): + """Attach a SOLID `custom_extensions` entry holding `data` (a dict). + + No-op when `data` is empty, so an object with nothing Solid-specific to carry stays + clean. Merges into an existing SOLID entry if one is already present. + """ + if not data: + return + payload = {"_v": STASH_VERSION} + payload.update(data) + blob = json.dumps(payload, ensure_ascii=False) + exts = obj.setdefault("custom_extensions", []) + for ext in exts: + if ext.get("vendor_name") == VENDOR: + ext["data"] = blob + return + exts.append({"vendor_name": VENDOR, "data": blob}) + + +def foreign_vendor_extensions(obj): + """Return the non-SOLID custom_extensions on an object. + + Solid's YAML has no slot for vendor metadata, so these are dropped on export; the + caller warns rather than discarding them silently. + """ + return [ + ext + for ext in (obj or {}).get("custom_extensions") or [] + if ext.get("vendor_name") != VENDOR + ] + + +def pick_expression(ossie_expression, dialect): + """Choose the SQL string for an Apache Ossie expression: `dialect`, else ANSI_SQL. + + Returns (expression, matched) where `matched` is False when the value came from the + ANSI_SQL fallback rather than the requested dialect, so the caller can warn. Returns + (None, False) when neither dialect is present. + """ + dialects = { + d.get("dialect"): d.get("expression") + for d in (ossie_expression or {}).get("dialects") or [] + } + for candidate, matched in ((dialect, True), (DIALECT_ANSI, False)): + expr = dialects.get(candidate) + if expr is None: + continue + if not isinstance(expr, str): + raise ConversionError( + f"expression must be a string, got {type(expr).__name__}") + return expr, matched + return None, False + + +def readable_dialects(dialect): + """Name the dialects `pick_expression` will accept, for an error message. + + Reads as "SNOWFLAKE or ANSI_SQL", and as plain "ANSI_SQL" when the selected dialect + *is* ANSI_SQL rather than the tautological "ANSI_SQL or ANSI_SQL". + """ + if dialect == DIALECT_ANSI: + return DIALECT_ANSI + return f"{dialect} or {DIALECT_ANSI}" + + +def resolve_asset_links(text): + """Replace Solid `@` markup with the tag's `name` attribute. + + Mirrors solid-server's `replace_asset_links_with_display_names`, which is applied + before instructions are handed to an LLM. A tag with no usable `name` is left + verbatim rather than deleted, so no text is silently lost. + """ + if not isinstance(text, str) or "@ Solid converter. + + ossie-solid import -i solid_model.yaml [-o model.yaml] [--dialect SNOWFLAKE] + ossie-solid export -i model.yaml [-o solid_model.yaml] [--dialect SNOWFLAKE] + +`import` converts a Solid semantic model export into an Apache Ossie semantic model; +`export` does the reverse. With no `-o` the result goes to stdout. Conversions that drop +or approximate information emit warnings to stderr. +""" + +import argparse +import sys +import warnings + +from ._common import SUPPORTED_DIALECTS, ConversionError +from .ossie_to_solid import convert_ossie_to_solid +from .solid_to_ossie import convert_solid_to_ossie + + +def _build_parser(): + parser = argparse.ArgumentParser( + prog="ossie-solid", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = parser.add_subparsers(dest="command") + sub.required = True + + imp = sub.add_parser( + "import", help="Solid semantic model YAML -> Apache Ossie semantic model") + imp.add_argument("-i", "--input", required=True, help="Solid YAML file") + imp.add_argument("-o", "--output", help="output Apache Ossie YAML (default: stdout)") + imp.add_argument( + "-d", "--dialect", choices=SUPPORTED_DIALECTS, + help="expression dialect to label expressions with (default: inferred from the " + "column type vocabulary, falling back to ANSI_SQL)") + imp.add_argument( + "--name", help="Apache Ossie model name (default: the Solid model's name)") + + exp = sub.add_parser( + "export", help="Apache Ossie semantic model -> Solid semantic model YAML") + exp.add_argument("-i", "--input", required=True, help="Apache Ossie YAML file") + exp.add_argument("-o", "--output", help="output Solid YAML (default: stdout)") + exp.add_argument( + "-d", "--dialect", choices=SUPPORTED_DIALECTS, + help="expression dialect to read (default: the dialect recorded at import, " + "else the one the model's expressions use)") + exp.add_argument("--name", help="Solid model name (default: the Apache Ossie " + "model's name)") + return parser + + +def _show_warning(message, category, filename, lineno, file=None, line=None): + """Render conversion warnings as plain stderr lines rather than Python tracebacks.""" + print(f"Warning: {message}", file=sys.stderr) + + +def main(argv=None): + args = _build_parser().parse_args(argv) + convert = convert_solid_to_ossie if args.command == "import" else convert_ossie_to_solid + try: + with open(args.input) as handle: + source = handle.read() + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.showwarning = _show_warning + output = convert(source, dialect=args.dialect, model_name=args.name) + if args.output: + with open(args.output, "w") as handle: + handle.write(output) + else: + sys.stdout.write(output) + except (ConversionError, OSError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/converters/solid/src/ossie_solid/datatypes.py b/converters/solid/src/ossie_solid/datatypes.py new file mode 100644 index 00000000..6fe44487 --- /dev/null +++ b/converters/solid/src/ossie_solid/datatypes.py @@ -0,0 +1,285 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Warehouse type vocabulary handling for the Solid converter. + +Solid stores a column's `type` as the **raw** warehouse type string, verbatim from the +catalog -- `NUMBER(38,0)`, `TEXT`, `INT64`, `LONG` -- with no normalization. Two things +follow from that: + +1. The vocabulary identifies the warehouse, which is how `infer_dialect` recovers a + dialect that the Solid YAML itself never records (see README, "Dialect resolution"). +2. Mapping it onto Apache Ossie's portable `datatype` enum is lossy (`NUMBER(38,2)`, + `NUMERIC` and `DECIMAL` all collapse to `Decimal`), so the raw string is always kept + in the SOLID stash and is what an export re-emits. +""" + +import re + +from ._common import ( + DIALECT_ANSI, + DIALECT_BIGQUERY, + DIALECT_DATABRICKS, + DIALECT_SNOWFLAKE, + SUPPORTED_DIALECTS, + ConversionError, + warn, +) + +# Apache Ossie portable logical data types (core-spec/ossie-schema.json $defs.DataType). +STRING = "String" +INTEGER = "Integer" +DECIMAL = "Decimal" +FLOAT = "Float" +BOOLEAN = "Boolean" +DATE = "Date" +TIME = "Time" +DATE_TIME = "DateTime" +DATE_TIME_TZ = "DateTimeTz" +OPAQUE = "Opaque" + +TEMPORAL_DATATYPES = frozenset({DATE, TIME, DATE_TIME, DATE_TIME_TZ}) + +# Type names that appear in exactly one of the three warehouses Solid exports from, and +# so identify it. Names shared across warehouses (STRING is both Databricks and +# BigQuery; STRUCT likewise, since `parse_type` strips the angle-bracket params that +# would tell them apart; TIMESTAMP_NTZ is both Snowflake and Databricks; BOOLEAN, DATE, +# FLOAT and DECIMAL are near-universal) are deliberately excluded -- no signal. +_DIALECT_MARKERS = { + DIALECT_SNOWFLAKE: frozenset( + {"NUMBER", "TEXT", "VARIANT", "OBJECT", "TIMESTAMP_LTZ", "TIMESTAMP_TZ", + "GEOGRAPHY", "GEOMETRY"} + ), + DIALECT_DATABRICKS: frozenset( + {"LONG", "MAP", "BIGINT", "TINYINT", "SMALLINT", "BYTE", "SHORT", "VOID"} + ), + DIALECT_BIGQUERY: frozenset( + {"INT64", "FLOAT64", "BOOL", "BIGNUMERIC", "BYTES", "RECORD"} + ), +} + +# Warehouse type -> Apache Ossie datatype, for names that mean the same thing +# everywhere. Dialect-sensitive names (the TIMESTAMP family, NUMBER/NUMERIC scale) are +# resolved in `to_ossie_datatype` instead. +_COMMON_TYPES = { + # character + "STRING": STRING, "TEXT": STRING, "VARCHAR": STRING, "CHAR": STRING, + "CHARACTER": STRING, "NVARCHAR": STRING, "NCHAR": STRING, "STRING_TYPE": STRING, + # exact integral + "INT": INTEGER, "INTEGER": INTEGER, "BIGINT": INTEGER, "SMALLINT": INTEGER, + "TINYINT": INTEGER, "BYTEINT": INTEGER, "LONG": INTEGER, "SHORT": INTEGER, + "BYTE": INTEGER, "INT64": INTEGER, "INT2": INTEGER, "INT4": INTEGER, + "INT8": INTEGER, "SERIAL": INTEGER, + # approximate + "FLOAT": FLOAT, "FLOAT4": FLOAT, "FLOAT8": FLOAT, "FLOAT64": FLOAT, + "DOUBLE": FLOAT, "DOUBLE PRECISION": FLOAT, "REAL": FLOAT, + # boolean + "BOOLEAN": BOOLEAN, "BOOL": BOOLEAN, + # temporal (dialect-independent) + "DATE": DATE, "TIME": TIME, + # not representable in the portable vocabulary + "VARIANT": OPAQUE, "OBJECT": OPAQUE, "ARRAY": OPAQUE, "MAP": OPAQUE, + "STRUCT": OPAQUE, "RECORD": OPAQUE, "JSON": OPAQUE, "XML": OPAQUE, + "BINARY": OPAQUE, "VARBINARY": OPAQUE, "BYTES": OPAQUE, "GEOGRAPHY": OPAQUE, + "GEOMETRY": OPAQUE, "INTERVAL": OPAQUE, "VOID": OPAQUE, "NULL": OPAQUE, +} + +# Exact-scale decimal families. Whether one is Integer or Decimal depends on the scale, +# which is parsed off the type string when present. +_DECIMAL_TYPES = frozenset({"NUMBER", "NUMERIC", "DECIMAL", "BIGNUMERIC", "DEC"}) + +# TIMESTAMP means different things per warehouse: BigQuery's and Databricks' are +# instants (offset-aware), Snowflake's bare TIMESTAMP is an alias for TIMESTAMP_NTZ. +_TIMESTAMP_BY_DIALECT = { + DIALECT_SNOWFLAKE: DATE_TIME, + DIALECT_DATABRICKS: DATE_TIME_TZ, + DIALECT_BIGQUERY: DATE_TIME_TZ, + DIALECT_ANSI: DATE_TIME, +} + +_TIMESTAMP_VARIANTS = { + "TIMESTAMP_NTZ": DATE_TIME, + "TIMESTAMPNTZ": DATE_TIME, + "DATETIME": DATE_TIME, + "TIMESTAMP_TZ": DATE_TIME_TZ, + "TIMESTAMPTZ": DATE_TIME_TZ, + "TIMESTAMP_LTZ": DATE_TIME_TZ, + "TIMESTAMPLTZ": DATE_TIME_TZ, + "TIMESTAMP WITH TIME ZONE": DATE_TIME_TZ, + "TIMESTAMP WITHOUT TIME ZONE": DATE_TIME, + "TIMESTAMP_UNSPECIFIED": DATE_TIME, +} + +# Apache Ossie datatype -> the type name an export writes back when the original raw +# string is unavailable (a hand-authored Apache Ossie model with no SOLID stash). Solid +# never interprets these beyond its fact/dimension split, so a representative name per +# dialect is enough. +_DATATYPE_TO_RAW = { + DIALECT_SNOWFLAKE: { + STRING: "TEXT", INTEGER: "NUMBER", DECIMAL: "NUMBER", FLOAT: "FLOAT", + BOOLEAN: "BOOLEAN", DATE: "DATE", TIME: "TIME", DATE_TIME: "TIMESTAMP_NTZ", + DATE_TIME_TZ: "TIMESTAMP_TZ", OPAQUE: "VARIANT", + }, + DIALECT_DATABRICKS: { + STRING: "STRING", INTEGER: "LONG", DECIMAL: "DECIMAL", FLOAT: "DOUBLE", + BOOLEAN: "BOOLEAN", DATE: "DATE", TIME: "STRING", DATE_TIME: "TIMESTAMP_NTZ", + DATE_TIME_TZ: "TIMESTAMP", OPAQUE: "STRING", + }, + DIALECT_BIGQUERY: { + STRING: "STRING", INTEGER: "INT64", DECIMAL: "NUMERIC", FLOAT: "FLOAT64", + BOOLEAN: "BOOL", DATE: "DATE", TIME: "TIME", DATE_TIME: "DATETIME", + DATE_TIME_TZ: "TIMESTAMP", OPAQUE: "STRING", + }, + DIALECT_ANSI: { + STRING: "VARCHAR", INTEGER: "INTEGER", DECIMAL: "DECIMAL", FLOAT: "DOUBLE", + BOOLEAN: "BOOLEAN", DATE: "DATE", TIME: "TIME", DATE_TIME: "TIMESTAMP", + DATE_TIME_TZ: "TIMESTAMP WITH TIME ZONE", OPAQUE: "VARCHAR", + }, +} + +# Leading type name, with any parameter list (`NUMBER(38,2)`, `VARCHAR(16777216)`) and +# any element type (`ARRAY`, `MAP`) captured separately. +_TYPE_RE = re.compile(r"^\s*([A-Za-z_][A-Za-z0-9_ ]*?)\s*(?:\(([^)]*)\)|<.*>)?\s*$") + + +def parse_type(raw): + """Split a raw warehouse type into (BASE_NAME, params) with BASE_NAME upper-cased. + + `NUMBER(38,2)` -> ("NUMBER", ["38", "2"]); `ARRAY` -> ("ARRAY", []); + an unparseable or empty value -> (None, []). + """ + if not isinstance(raw, str) or not raw.strip(): + return None, [] + match = _TYPE_RE.match(raw) + if not match: + return raw.strip().upper(), [] + base = " ".join(match.group(1).split()).upper() + params = [p.strip() for p in (match.group(2) or "").split(",") if p.strip()] + return base or None, params + + +def infer_dialect(raw_types): + """Infer the Apache Ossie dialect from a model's column type vocabulary. + + Solid's YAML export does not record the source warehouse -- it lives on the asset + row in Solid's own database and is dropped at render time -- but the raw type names + it copies out of the catalog do identify it. Each distinctive name votes for its + warehouse; the warehouse with the most votes wins. + + Returns (dialect, confident). `confident` is False when no marker was seen or two + warehouses tie, in which case the dialect is ANSI_SQL and the caller should warn. + """ + votes = dict.fromkeys(_DIALECT_MARKERS, 0) + for raw in raw_types: + base, _ = parse_type(raw) + if not base: + continue + for dialect, markers in _DIALECT_MARKERS.items(): + if base in markers: + votes[dialect] += 1 + ranked = sorted(votes.items(), key=lambda kv: (-kv[1], kv[0])) + best, best_votes = ranked[0] + if best_votes == 0: + return DIALECT_ANSI, False + if len(ranked) > 1 and ranked[1][1] == best_votes: + return DIALECT_ANSI, False + return best, True + + +def to_ossie_datatype(raw, dialect): + """Map a raw warehouse type onto an Apache Ossie `datatype`. + + Returns None when the type is absent or unrecognized -- the spec says to omit + `datatype` when it is unknown rather than guess. A type that is known but outside + the portable vocabulary (VARIANT, MAP, STRUCT, ...) maps to `Opaque`; the raw name + is preserved in the SOLID stash either way. + """ + base, params = parse_type(raw) + if not base: + return None + if base in _DECIMAL_TYPES: + # A declared scale of 0 is an integer -- Snowflake stores every integral column + # as NUMBER(38,0), so honouring the scale is what keeps counts and ids from + # being typed as Decimal. An undeclared scale defaults to 0 in Snowflake and + # BigQuery NUMERIC, but Databricks DECIMAL defaults to (10,0) -- also scale 0. + if len(params) >= 2: + return INTEGER if params[1] == "0" else DECIMAL + return INTEGER + if base == "TIMESTAMP": + return _TIMESTAMP_BY_DIALECT.get(dialect, DATE_TIME) + if base in _TIMESTAMP_VARIANTS: + return _TIMESTAMP_VARIANTS[base] + return _COMMON_TYPES.get(base) + + +def to_raw_type(datatype, dialect): + """Map an Apache Ossie `datatype` back to a representative warehouse type name. + + Only used when exporting a field that carries no SOLID stash (a hand-authored + Apache Ossie model). A field that came from Solid re-emits its original raw string. + """ + if not datatype: + return None + table = _DATATYPE_TO_RAW.get(dialect) or _DATATYPE_TO_RAW[DIALECT_ANSI] + return table.get(datatype) + + +# Solid splits a table's columns into `facts` and `dimensions` by data type, using a +# lowercase prefix match against this set (see semantic_layer_data_types.py in +# solid-server). Mirrored here so an export of a hand-authored Apache Ossie model -- +# where no field carries the `dimension` block that records the original split -- lands +# columns in the same bucket Solid would have chosen. +_SOLID_FACT_TYPE_PREFIXES = ( + "decimal", "double", "int", "float", "number", "numeric", "real", "tiny", "long", +) + + +def is_solid_fact_type(raw): + """True if Solid would classify a column of this raw type as a fact.""" + if not isinstance(raw, str): + return False + lowered = raw.strip().lower() + return any(lowered.startswith(p) for p in _SOLID_FACT_TYPE_PREFIXES) + + +def normalize_dialect(dialect): + """Validate and upper-case an Apache Ossie dialect name.""" + normalized = str(dialect).strip().upper() + if normalized not in SUPPORTED_DIALECTS: + raise ConversionError( + f"Unsupported dialect '{dialect}'. Choose one of: " + f"{', '.join(SUPPORTED_DIALECTS)}" + ) + return normalized + + +def resolve_dialect(explicit, raw_types, scope="model"): + """Pick the dialect for a conversion: an explicit choice wins, else inference. + + Warns when inference had to fall back to ANSI_SQL, since a fallback means every + expression is being labelled with a dialect it was not written in. + """ + if explicit: + return normalize_dialect(explicit) + dialect, confident = infer_dialect(raw_types) + if not confident: + warn( + scope, + "could not infer the source warehouse from the column type vocabulary; " + f"defaulting to {DIALECT_ANSI}. Pass --dialect to set it explicitly.", + ) + return dialect diff --git a/converters/solid/src/ossie_solid/expressions.py b/converters/solid/src/ossie_solid/expressions.py new file mode 100644 index 00000000..036a3f96 --- /dev/null +++ b/converters/solid/src/ossie_solid/expressions.py @@ -0,0 +1,286 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Metric expression rewriting between Solid and Apache Ossie conventions. + +Solid stores a metric formula against **bare** column names and records the owning table +separately (`tables: [shop.sales.orders]`); solid-server's `is_valid_metric_formula` +actively strips alias prefixes before saving. Apache Ossie instead expects a metric to be +self-contained, qualifying each column with its dataset +(`SUM(store_sales.ss_ext_sales_price)`), so the two directions here add and remove that +qualifier. + +The rewrite is a **surgical splice**, not a re-render. Every column occurrence is located +by sqlglot's tokenizer, which reports character positions and keeps string literals and +quoted identifiers in token types of their own, and the qualifier is inserted or removed +at those offsets. Everything else in the expression is left character-for-character +intact. + +That matters because round-tripping a parsed tree through `Expression.sql()` does not +return the author's SQL -- sqlglot canonicalizes as it generates, so +`CAST(x AS FLOAT)` comes back as `CAST(x AS DOUBLE)` and +`EXTRACT(year FROM d)` as `DATE_PART(YEAR, d)`. Those are semantically equal and +textually different, and a converter has no business rewriting SQL it was only asked to +qualify. + +The parser is still used, as a cross-check: it decides which names are genuinely bare +column references, and a token scan that disagrees with it means some occurrence is +something else (`EXTRACT(year FROM ...)` where a column is also called `year`). On any +disagreement the expression is left exactly as written -- an unqualified metric is a far +smaller problem than a corrupted one. +""" + +from collections import Counter + +import sqlglot +from sqlglot import exp +from sqlglot.errors import SqlglotError +from sqlglot.tokens import TokenType + +from ._common import SQLGLOT_DIALECTS + +# Token types that can carry an identifier: an unquoted name, or a quoted one whose +# offsets span the quote characters as well. +_NAME_TOKENS = (TokenType.VAR, TokenType.IDENTIFIER) + +# Conversion outcomes, returned alongside the expression. +QUALIFIED = "qualified" +UNQUALIFIED = "unqualified" +UNCHANGED = "unchanged" +UNPARSED = "unparsed" +AMBIGUOUS = "ambiguous" + + +def _read(dialect): + """Apache Ossie dialect name -> the sqlglot dialect to parse/tokenize with.""" + return SQLGLOT_DIALECTS.get(dialect) + + +def _parse(expression, dialect): + """Parse a scalar/aggregate expression, or return None if it does not parse. + + Metric expressions are fragments rather than statements, so `parse_one` is used + directly; a fragment sqlglot has no grammar for yields None and the caller leaves the + text alone. + """ + try: + return sqlglot.parse_one(expression, read=_read(dialect)) or None + except (SqlglotError, ValueError, RecursionError): + return None + + +def _tokenize(expression, dialect): + """Tokenize an expression, or return None if it cannot be tokenized.""" + try: + return sqlglot.tokenize(expression, read=_read(dialect)) + except (SqlglotError, ValueError, RecursionError): + return None + + +def _unquote(text): + """Strip the quoting a tokenizer may have left on an identifier's text.""" + return text.strip().strip('"`[]') + + +def column_reference(name, dialect): + """Render a column name as an Apache Ossie field expression. + + A Solid field's expression is just its own column, so the usual result is the bare + name. A name that is not a plain identifier -- one with spaces, or one that collides + with a reserved word -- is quoted with the target dialect's rules so the expression + stays valid SQL. + """ + if not isinstance(name, str) or not name.strip(): + return name + bare = exp.to_identifier(name, quoted=False) + if bare is not None: + rendered = bare.sql(dialect=_read(dialect)) + parsed = _parse(rendered, dialect) + # The bare form is usable only if it reads back as the same single column; a + # reserved word parses as some other node type, or not at all. + if isinstance(parsed, exp.Column) and parsed.name == name and not parsed.table: + return rendered + quoted = exp.to_identifier(name, quoted=True) + return quoted.sql(dialect=_read(dialect)) if quoted is not None else name + + +def _bare_columns(tree, known): + """Count, per name, the bare column references the AST reports. + + Only names in `known` are counted, since only those are candidates for qualification. + """ + counts = Counter() + for column in tree.find_all(exp.Column): + if column.table: + continue + name = column.name.lower() + if name in known: + counts[name] += 1 + return counts + + +def _qualified_columns(tree, known): + """Count, per (qualifier, name), the column references already qualified by a name in + `known`. A three-part reference (`db.table.column`) is excluded: its qualifier is a + schema, not a dataset.""" + counts = Counter() + for column in tree.find_all(exp.Column): + table = column.args.get("table") + if table is None or column.args.get("db"): + continue + if column.table.lower() in known: + counts[(column.table.lower(), column.name.lower())] += 1 + return counts + + +def _splice(expression, edits): + """Apply (start, end, replacement) edits to `expression`, right to left. + + `end` is exclusive. Applying in reverse order keeps earlier offsets valid. + """ + result = expression + for start, end, replacement in sorted(edits, key=lambda e: e[0], reverse=True): + result = result[:start] + replacement + result[end:] + return result + + +def qualify_metric(expression, dialect, dataset, columns): + """Prefix every bare column in `expression` with `dataset`. + + `columns` is the set of column names the dataset owns, compared case-insensitively + (Solid models routinely mix `ACCOUNT_ID` in the column list with `account_id` in the + formula). A column that already carries a qualifier is left as the author wrote it. + + Returns (expression, status): + "qualified" -- at least one column was prefixed, by splicing into the original text + "unchanged" -- nothing needed prefixing; the text is returned verbatim + "unparsed" -- sqlglot could not read it; the text is returned verbatim + "ambiguous" -- the token scan and the parse disagree on which names are columns, + so the text is returned verbatim rather than risk a bad edit + """ + if not expression or not expression.strip(): + return expression, UNCHANGED + known = {c.lower() for c in columns} + if not known: + return expression, UNCHANGED + + tree = _parse(expression, dialect) + tokens = _tokenize(expression, dialect) + if tree is None or tokens is None: + return expression, UNPARSED + + expected = _bare_columns(tree, known) + if not expected: + return expression, UNCHANGED + + targets = [] + for index, token in enumerate(tokens): + if token.token_type not in _NAME_TOKENS: + continue + if _unquote(token.text).lower() not in known: + continue + previous = tokens[index - 1] if index else None + following = tokens[index + 1] if index + 1 < len(tokens) else None + # Already qualified (`t.col`), itself a qualifier (`col.x`), or a function call + # (`col(...)`) -- none of these is a bare column reference. + if previous is not None and previous.token_type is TokenType.DOT: + continue + if following is not None and following.token_type in ( + TokenType.DOT, TokenType.L_PAREN): + continue + targets.append(token) + + if Counter(_unquote(t.text).lower() for t in targets) != expected: + return expression, AMBIGUOUS + + prefix = f"{column_reference(dataset, dialect)}." + return _splice( + expression, + [(t.start, t.start, prefix) for t in targets], + ), QUALIFIED + + +def unqualify_metric(expression, dialect, datasets): + """Strip a dataset qualifier off every column in `expression`. + + The inverse of `qualify_metric`, used when exporting back to Solid, whose formulas are + stored bare. `datasets` is the set of dataset names that may legitimately appear as a + qualifier; any other qualifier -- a genuine table alias an author wrote by hand -- is + preserved. + + Returns (expression, status) with the same vocabulary as `qualify_metric`, except that + a successful edit reports "unqualified". + """ + if not expression or not expression.strip(): + return expression, UNCHANGED + known = {d.lower() for d in datasets} + if not known: + return expression, UNCHANGED + + tree = _parse(expression, dialect) + tokens = _tokenize(expression, dialect) + if tree is None or tokens is None: + return expression, UNPARSED + + expected = _qualified_columns(tree, known) + if not expected: + return expression, UNCHANGED + + edits, found = [], Counter() + for index, token in enumerate(tokens): + if token.token_type not in _NAME_TOKENS: + continue + if _unquote(token.text).lower() not in known: + continue + dot = tokens[index + 1] if index + 1 < len(tokens) else None + name = tokens[index + 2] if index + 2 < len(tokens) else None + if dot is None or dot.token_type is not TokenType.DOT: + continue + if name is None or name.token_type not in _NAME_TOKENS: + continue + previous = tokens[index - 1] if index else None + # `db.table.column`: the leading name is a schema, so the middle one is not a + # dataset qualifier this converter added. + if previous is not None and previous.token_type is TokenType.DOT: + continue + after = tokens[index + 3] if index + 3 < len(tokens) else None + if after is not None and after.token_type is TokenType.DOT: + continue + found[(_unquote(token.text).lower(), _unquote(name.text).lower())] += 1 + # Delete the qualifier and its dot; `end` is inclusive on a sqlglot token. + edits.append((token.start, dot.end + 1, "")) + + if found != expected: + return expression, AMBIGUOUS + return _splice(expression, edits), UNQUALIFIED + + +def referenced_datasets(expression, dialect, datasets): + """Return the dataset names a metric expression qualifies columns with. + + Used on export to reconstruct Solid's `metrics[].tables` when an Apache Ossie model + carries no SOLID stash. Order follows `datasets`, so the result is deterministic. + """ + tree = _parse(expression, dialect) + if tree is None: + return [] + seen = set() + for column in tree.find_all(exp.Column): + if column.table and not column.args.get("db"): + seen.add(column.table.lower()) + return [d for d in datasets if d.lower() in seen] + diff --git a/converters/solid/src/ossie_solid/ossie_to_solid.py b/converters/solid/src/ossie_solid/ossie_to_solid.py new file mode 100644 index 00000000..360a4641 --- /dev/null +++ b/converters/solid/src/ossie_solid/ossie_to_solid.py @@ -0,0 +1,465 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert an Apache Ossie semantic model to a Solid semantic model YAML export. + +Pure offline conversion. A model that came from `convert_solid_to_ossie` restores its +Solid-only fields from `custom_extensions[SOLID]`; a hand-authored Apache Ossie model is +converted on its core fields alone, with anything Solid's format cannot hold reported as +a warning rather than dropped silently. + +Key order in the output follows solid-server's own export template, so a converted model +diffs cleanly against one Solid produced itself. + +Usage (CLI): + ossie-solid export -i model.yaml [-o solid_model.yaml] [--dialect SNOWFLAKE] +""" + +from . import datatypes +from ._common import ( + DIALECT_ANSI, + ONE_TO_ONE_NOTE, + OSSIE_VERSION, + OSSIE_VERSION_NOTES, + PK_SEPARATOR, + READABLE_OSSIE_VERSIONS, + SUPPORTED_DIALECTS, + ConversionError, + ai_context_parts, + clean_text, + dump_yaml, + foreign_vendor_extensions, + load_yaml, + pick_expression, + read_stash, + readable_dialects, + require_str, + string_list, + warn, +) +from .expressions import AMBIGUOUS, UNPARSED, referenced_datasets, unqualify_metric + + +def convert_ossie_to_solid(ossie_yaml_str, dialect=None, model_name=None): + """Parse an Apache Ossie semantic model and return Solid YAML (a string). + + `dialect` chooses which expression dialect to read; when omitted it comes from the + SOLID stash, else from whichever non-ANSI dialect the model's expressions use. + `model_name` overrides the Solid model name. + """ + document = load_yaml(ossie_yaml_str) + if not isinstance(document, dict): + raise ConversionError( + "Invalid Apache Ossie model: expected a mapping at the root") + + version = str(document.get("version", "")) + if version not in READABLE_OSSIE_VERSIONS: + raise ConversionError( + f"Unsupported Apache Ossie version '{version}'. This converter reads " + f"{', '.join('v' + v for v in READABLE_OSSIE_VERSIONS)}." + ) + if version != OSSIE_VERSION: + note = OSSIE_VERSION_NOTES.get(version) + warn("model", f"the document declares Apache Ossie v{version}, not the current " + f"v{OSSIE_VERSION}" + + (f"; {note}" if note else "")) + + models = document.get("semantic_model") + if not isinstance(models, list) or not models: + raise ConversionError( + "Apache Ossie document is missing a non-empty 'semantic_model' list") + if len(models) > 1: + warn("model", f"the document holds {len(models)} semantic models but Solid's " + f"format holds one; only '{models[0].get('name')}' was converted") + + return dump_yaml({"semantic_model": _convert_model(models[0], dialect, model_name)}) + + +def _convert_model(ossie, explicit_dialect, model_name): + if not isinstance(ossie, dict): + raise ConversionError("Each entry of 'semantic_model' must be a mapping") + datasets = ossie.get("datasets") + if not isinstance(datasets, list) or not datasets: + raise ConversionError("Apache Ossie model has no 'datasets'") + + stash = read_stash(ossie) + resolved_dialect = _resolve_dialect(explicit_dialect, stash, ossie) + _warn_foreign_extensions(ossie, "model", ossie.get("name")) + + by_name = {} + for dataset in datasets: + if not isinstance(dataset, dict): + raise ConversionError("Each entry of 'datasets' must be a mapping") + by_name[require_str(dataset, "name", "dataset")] = dataset + # Whether the model records Solid's fact/dimension split at all. If no field + # anywhere carries a `dimension` block the model was not produced by this converter, + # so the split is re-derived from the data types the way Solid itself does it. + has_dimension_metadata = any( + isinstance(field, dict) and field.get("dimension") is not None + for dataset in datasets + for field in dataset.get("fields") or [] + ) + + instructions, _, examples = ai_context_parts(ossie.get("ai_context")) + business_context = {} + if examples: + business_context["business_questions"] = examples + model_description = clean_text(stash.get("model_description")) + if model_description: + business_context["model_description"] = model_description + # The stash holds the asset-link markup Solid wrote; `ai_context.instructions` holds + # the same text with links resolved to display names. Prefer the markup so a + # re-import into Solid keeps its catalog references live. + custom_instructions = clean_text(stash.get("custom_instructions")) or instructions + if custom_instructions: + business_context["custom_instructions"] = custom_instructions + + solid = {"name": model_name or require_str(ossie, "name", "semantic_model")} + solid["business_context"] = business_context + llm_description = clean_text(stash.get("model_llm_description")) + if llm_description is None and "model_description" not in stash: + # No stash to read: an Apache Ossie `description` is the model's primary + # description, which is the role Solid's `model_llm_description` plays. + llm_description = clean_text(ossie.get("description")) + if llm_description: + solid["model_llm_description"] = llm_description + + solid["tables"] = [ + _convert_dataset(d, resolved_dialect, has_dimension_metadata) for d in datasets + ] + + metrics = _convert_metrics( + ossie.get("metrics") or [], by_name, resolved_dialect) + if metrics: + solid["metrics"] = metrics + relationships = _convert_relationships(ossie.get("relationships") or [], by_name) + if relationships: + solid["relationships"] = relationships + for key in ("example_queries", "benchmark_questions"): + if stash.get(key): + solid[key] = stash[key] + return solid + + +def _resolve_dialect(explicit, stash, ossie): + """Pick which Apache Ossie expression dialect to read expressions from. + + An explicit choice wins, then the dialect the SOLID stash recorded at import time, + then the single non-ANSI dialect the model's expressions use. A model written purely + in ANSI_SQL resolves to ANSI_SQL. + """ + if explicit: + return datatypes.normalize_dialect(explicit) + stashed = clean_text(stash.get("dialect")) + if stashed: + # Written by this converter's own import, so it is always one of the supported + # names; validating it means a hand-edited stash fails loudly rather than + # feeding an unknown dialect to the expression rewriter. + return datatypes.normalize_dialect(stashed) + + # Apache Ossie's dialect enum includes expression languages that are not SQL (MDX, + # TABLEAU, MAQL). Those cannot be parsed, qualified or unqualified by this + # converter, so they never become the resolved dialect -- naming one here would + # hand a non-SQL formula to sqlglot and to Solid as though it were SQL. + found, unreadable = set(), set() + for expression in _all_expressions(ossie): + for entry in (expression or {}).get("dialects") or []: + dialect = entry.get("dialect") + if not dialect or dialect == DIALECT_ANSI: + continue + (found if dialect in SUPPORTED_DIALECTS else unreadable).add(dialect) + if unreadable: + warn("model", f"the model carries {', '.join(sorted(unreadable))} expressions, " + f"which are not SQL this converter can read; only the " + f"{DIALECT_ANSI} form of each expression will be used") + if len(found) == 1: + return found.pop() + if len(found) > 1: + warn("model", f"expressions mix the {', '.join(sorted(found))} dialects; " + f"reading {DIALECT_ANSI}. Pass --dialect to choose one.") + return DIALECT_ANSI + + +def _all_expressions(ossie): + for dataset in ossie.get("datasets") or []: + for field in dataset.get("fields") or []: + if isinstance(field, dict): + yield field.get("expression") + for metric in ossie.get("metrics") or []: + if isinstance(metric, dict): + yield metric.get("expression") + + +def _warn_foreign_extensions(obj, scope, name): + """Warn about custom_extensions belonging to other vendors. + + Solid's format has no slot for vendor metadata, so these cannot be carried across. + """ + foreign = foreign_vendor_extensions(obj) + if foreign: + vendors = ", ".join(sorted({e.get("vendor_name", "?") for e in foreign})) + warn(scope, f"'{name}': custom_extensions for {vendors} have no Solid " + f"equivalent and were dropped") + + +def _convert_dataset(dataset, dialect, has_dimension_metadata): + name = require_str(dataset, "name", "dataset") + stash = read_stash(dataset) + _warn_foreign_extensions(dataset, "dataset", name) + + # Solid identifies a table by its fully-qualified name, which is the Apache Ossie + # `source`; the Apache Ossie dataset name is a local alias with no Solid slot. + table = {"name": require_str(dataset, "source", f"dataset '{name}'")} + + description = clean_text(dataset.get("description")) + if description: + table["description"] = description + instructions, synonyms, examples = ai_context_parts(dataset.get("ai_context")) + if instructions: + table["manual_description"] = instructions + if synonyms: + table["synonyms"] = synonyms + if examples: + warn("dataset", f"'{name}': ai_context.examples has no Solid equivalent at the " + f"table level and was dropped") + + primary_key = string_list(dataset.get("primary_key")) + if primary_key: + # Solid stores a composite key as one comma-joined scalar. + table["primary_key"] = PK_SEPARATOR.join(primary_key) + unique_keys = dataset.get("unique_keys") + if unique_keys and unique_keys != [primary_key]: + warn("dataset", f"'{name}': unique_keys have no Solid equivalent and were " + f"dropped") + # Solid emits `quality_rank` on every table, empty when the rank is unset. + table["quality_rank"] = stash.get("quality_rank", "") + if stash.get("indexes"): + table["indexes"] = list(stash["indexes"]) + + dimensions, facts = [], [] + for field in dataset.get("fields") or []: + if not isinstance(field, dict): + raise ConversionError(f"Dataset '{name}': each field must be a mapping") + column, is_dimension = _convert_field( + field, dialect, name, has_dimension_metadata) + (dimensions if is_dimension else facts).append(column) + if dimensions: + table["dimensions"] = dimensions + # Unlike `dimensions`, Solid always emits `facts` -- as `[]` when a table has none. + table["facts"] = facts + return table + + +def _convert_field(field, dialect, dataset_name, has_dimension_metadata): + name = require_str(field, "name", f"field in '{dataset_name}'") + stash = read_stash(field) + _warn_foreign_extensions(field, "field", f"{dataset_name}.{name}") + + expression, matched = pick_expression(field.get("expression"), dialect) + if expression is None: + raise ConversionError( + f"Field '{dataset_name}.{name}' has no " + f"{readable_dialects(dialect)} expression") + if not matched and dialect != DIALECT_ANSI: + warn("field", f"'{dataset_name}.{name}' has no {dialect} expression; the " + f"{DIALECT_ANSI} one was used") + + raw_type = clean_text(stash.get("type")) or datatypes.to_raw_type( + field.get("datatype"), dialect) + is_dimension = _is_dimension(field, stash, raw_type, has_dimension_metadata) + + column = {"name": name} + # A Solid fact may be a table-scoped metric instead of a catalog column, in which + # case it carries an expression and no type. Everything else is a column, whose + # expression is just its own name. + is_metric_fact = ( + not is_dimension + and (stash.get("role") == "metric" or (raw_type is None and _is_computed(expression, name))) + ) + if is_metric_fact: + raw_type = None + elif raw_type: + column["type"] = raw_type + else: + warn("field", f"'{dataset_name}.{name}' has no datatype; Solid's 'type' was " + f"left empty") + column["type"] = "" + + description = clean_text(field.get("description")) + if description: + column["description"] = description + instructions, synonyms, examples = ai_context_parts(field.get("ai_context")) + if instructions: + column["manual_description"] = instructions + if is_metric_fact: + column["expression"] = expression + elif _is_computed(expression, name): + warn("field", f"'{dataset_name}.{name}' is a computed field " + f"(`{expression}`); Solid columns map to catalog columns, so the " + f"expression was dropped") + if synonyms: + column["synonyms"] = synonyms + if examples: + warn("field", f"'{dataset_name}.{name}': ai_context.examples has no Solid " + f"equivalent and was dropped") + if field.get("label"): + warn("field", f"'{dataset_name}.{name}': label '{field['label']}' has no Solid " + f"equivalent and was dropped") + if stash.get("sample_values"): + column["sample_values"] = list(stash["sample_values"]) + return column, is_dimension + + +def _is_dimension(field, stash, raw_type, has_dimension_metadata): + """Decide whether a field belongs in Solid's `dimensions` or its `facts`. + + A model this converter produced records the original split as the presence of the + `dimension` block. A hand-authored Apache Ossie model has no such marker, so the + split is re-derived from the data type the way solid-server does it. + """ + if has_dimension_metadata: + return field.get("dimension") is not None + if stash.get("role") == "metric": + return False + return not datatypes.is_solid_fact_type(raw_type) + + +def _is_computed(expression, name): + """True if a field's expression is something other than a plain reference to itself.""" + return expression.strip().strip('"`[]').lower() != name.strip().lower() + + +def _convert_relationships(relationships, by_name): + converted = [] + for relationship in relationships: + if not isinstance(relationship, dict): + raise ConversionError("Each entry of 'relationships' must be a mapping") + name = relationship.get("name", "") + source = require_str(relationship, "from", f"relationship '{name}'") + target = require_str(relationship, "to", f"relationship '{name}'") + for end in (source, target): + if end not in by_name: + raise ConversionError( + f"Relationship '{name}' references dataset '{end}', which is not " + f"declared in 'datasets'") + source_columns = string_list(relationship.get("from_columns")) + target_columns = string_list(relationship.get("to_columns")) + if not source_columns or not target_columns: + raise ConversionError( + f"Relationship '{name}' is missing from_columns/to_columns") + if len(source_columns) != len(target_columns): + raise ConversionError( + f"Relationship '{name}' has {len(source_columns)} from_columns but " + f"{len(target_columns)} to_columns; they must correspond positionally") + + stash = read_stash(relationship) + _warn_foreign_extensions(relationship, "relationship", name) + # A Solid relationship is just a table pair and its join keys -- it carries no + # free text -- so any annotation on the Apache Ossie side is dropped. The + # exception is the note import writes for a one-to-one, which is this + # converter's own marker and means nothing to Solid. + instructions, synonyms, examples = ai_context_parts( + relationship.get("ai_context")) + if instructions == ONE_TO_ONE_NOTE: + instructions = None + dropped = [ + label + for label, value in (("instructions", instructions), + ("synonyms", synonyms), + ("examples", examples)) + if value + ] + if dropped: + warn("relationship", + f"'{name}': ai_context.{', ai_context.'.join(dropped)} " + f"{'have' if len(dropped) > 1 else 'has'} no Solid equivalent -- a " + f"Solid relationship carries only its tables and join keys -- and " + f"{'were' if len(dropped) > 1 else 'was'} dropped") + if stash.get("flipped"): + # Import flipped this pair to put the many side on `from`; restore the + # left/right order Solid wrote. + left, right = target, source + left_columns, right_columns = target_columns, source_columns + else: + left, right = source, target + left_columns, right_columns = source_columns, target_columns + + converted.append({ + "left_table": by_name[left]["source"], + "right_table": by_name[right]["source"], + "join_keys": {"left": left_columns, "right": right_columns}, + }) + return converted + + +def _convert_metrics(metrics, by_name, dialect): + dataset_names = list(by_name) + converted = [] + for metric in metrics: + if not isinstance(metric, dict): + raise ConversionError("Each entry of 'metrics' must be a mapping") + name = require_str(metric, "name", "metric") + stash = read_stash(metric) + _warn_foreign_extensions(metric, "metric", name) + # Solid types a metric by evaluating its formula against the warehouse, so its + # YAML has no slot for a declared result type. + if metric.get("datatype"): + warn("metric", f"'{name}': datatype '{metric['datatype']}' has no Solid " + f"equivalent and was dropped") + + expression, matched = pick_expression(metric.get("expression"), dialect) + if expression is None: + raise ConversionError( + f"Metric '{name}' has no {readable_dialects(dialect)} expression") + if not matched and dialect != DIALECT_ANSI: + warn("metric", f"'{name}' has no {dialect} expression; the {DIALECT_ANSI} " + f"one was used") + + # Solid stores formulas against bare columns and records the owning tables + # separately, so the dataset qualifier Apache Ossie carries is stripped back out. + tables = stash.get("tables") + if tables is None: + tables = [ + by_name[d]["source"] + for d in referenced_datasets(expression, dialect, dataset_names) + ] + bare, status = unqualify_metric(expression, dialect, dataset_names) + if status in (UNPARSED, AMBIGUOUS): + warn("metric", f"'{name}': the dataset qualifiers could not be removed " + f"safely ({status}), so the expression was left as written; " + f"Solid stores formulas with bare column names") + if not tables: + warn("metric", f"'{name}' names no table; Solid needs at least one to " + f"resolve its columns") + + solid_metric = {"name": name} + description = clean_text(metric.get("description")) + if description: + solid_metric["description"] = description + solid_metric["expression"] = bare + _, synonyms, examples = ai_context_parts(metric.get("ai_context")) + if synonyms: + solid_metric["synonyms"] = synonyms + if examples: + warn("metric", f"'{name}': ai_context.examples has no Solid equivalent and " + f"was dropped") + solid_metric["tables"] = list(tables) + converted.append(solid_metric) + return converted + + +__all__ = ["convert_ossie_to_solid"] diff --git a/converters/solid/src/ossie_solid/solid_to_ossie.py b/converters/solid/src/ossie_solid/solid_to_ossie.py new file mode 100644 index 00000000..e73fb636 --- /dev/null +++ b/converters/solid/src/ossie_solid/solid_to_ossie.py @@ -0,0 +1,484 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert a Solid semantic model YAML export to an Apache Ossie semantic model. + +Pure offline conversion. Solid features Apache Ossie has no native field for -- +`example_queries`, `benchmark_questions`, `quality_rank`, `indexes`, `sample_values`, +and each column's raw warehouse type -- are preserved in `custom_extensions[SOLID]`, so +converting back reproduces the original export. + +Usage (CLI): + ossie-solid import -i solid_model.yaml [-o model.yaml] [--dialect SNOWFLAKE] +""" + +from . import datatypes +from ._common import ( + ONE_TO_ONE_NOTE, + OSSIE_VERSION, + ConversionError, + build_ai_context, + clean_text, + dataset_name_for, + dump_yaml, + has_asset_links, + load_yaml, + require_str, + resolve_asset_links, + string_list, + unique_name, + warn, + write_stash, +) +from .expressions import ( + AMBIGUOUS, + UNCHANGED, + UNPARSED, + column_reference, + qualify_metric, + referenced_datasets, +) + + +def convert_solid_to_ossie(solid_yaml_str, dialect=None, model_name=None): + """Parse a Solid semantic model export and return Apache Ossie YAML (a string). + + `dialect` forces the Apache Ossie expression dialect; when omitted it is inferred + from the column type vocabulary (see datatypes.infer_dialect). `model_name` + overrides the Apache Ossie model name. + """ + document = load_yaml(solid_yaml_str) + if not isinstance(document, dict): + raise ConversionError( + "Invalid Solid semantic model: expected a mapping at the root") + solid = document.get("semantic_model") + if solid is None: + raise ConversionError("Solid semantic model is missing the top-level " + "'semantic_model' key") + if isinstance(solid, list): + # An Apache Ossie document also has a `semantic_model` key, but holding a list. + # Naming the confusion is more useful than a generic type error. + raise ConversionError( + "'semantic_model' is a list; that is the Apache Ossie layout, not Solid's. " + "Did you mean `ossie-solid export`?" + ) + if not isinstance(solid, dict): + raise ConversionError("'semantic_model' must be a mapping") + + model = _convert_model(solid, dialect, model_name) + return dump_yaml({"version": OSSIE_VERSION, "semantic_model": [model]}) + + +def _convert_model(solid, explicit_dialect, model_name): + tables = solid.get("tables") or [] + if not isinstance(tables, list) or not tables: + # Apache Ossie requires at least one dataset (schema: datasets.minItems = 1). + raise ConversionError("Solid semantic model has no 'tables'; an Apache Ossie " + "model requires at least one dataset") + + resolved_dialect = datatypes.resolve_dialect( + explicit_dialect, _all_raw_types(tables), scope="model") + + # Solid names a table by its fully-qualified `catalog.schema.table`, which becomes + # the Apache Ossie `source`; the dataset name is derived from it and is what + # relationships and metric expressions refer to. + datasets, by_source = [], {} + taken_names = set() + for table in tables: + dataset = _convert_table(table, resolved_dialect, taken_names) + datasets.append(dataset) + by_source[dataset["source"].lower()] = dataset + + relationships = _convert_relationships( + solid.get("relationships") or [], by_source) + metrics = _convert_metrics( + solid.get("metrics") or [], by_source, datasets, resolved_dialect) + + business = solid.get("business_context") + business = business if isinstance(business, dict) else {} + llm_description = clean_text(solid.get("model_llm_description")) + customer_description = clean_text(business.get("model_description")) + raw_instructions = clean_text(business.get("custom_instructions")) + + model = {"name": model_name or require_str(solid, "name", "semantic_model")} + description = llm_description or customer_description + if description: + model["description"] = description + ai_context = build_ai_context( + # Solid resolves its asset-link markup to plain display names before the text + # reaches an LLM; do the same for the Apache Ossie consumer, keeping the tagged + # original in the stash. + instructions=resolve_asset_links(raw_instructions) if raw_instructions else None, + examples=string_list(business.get("business_questions")), + ) + if ai_context: + model["ai_context"] = ai_context + + model["datasets"] = datasets + if relationships: + model["relationships"] = relationships + if metrics: + model["metrics"] = metrics + + stash = {"dialect": resolved_dialect} + # The model carries three free-text fields but Apache Ossie offers two slots, so + # both raw descriptions are recorded; export reads them back rather than guessing + # which one the single `description` came from. + if llm_description: + stash["model_llm_description"] = llm_description + if customer_description: + stash["model_description"] = customer_description + if raw_instructions and has_asset_links(raw_instructions): + stash["custom_instructions"] = raw_instructions + for key in ("example_queries", "benchmark_questions"): + value = solid.get(key) + if value: + stash[key] = value + write_stash(model, stash) + return model + + +def _all_raw_types(tables): + """Every raw column type in the model, for dialect inference.""" + types = [] + for table in tables: + if not isinstance(table, dict): + continue + for column in _columns_of(table): + if isinstance(column, dict) and column.get("type"): + types.append(column["type"]) + return types + + +def _columns_of(table): + """Dimensions followed by facts, skipping anything that is not a mapping.""" + columns = [] + for key in ("dimensions", "facts"): + for column in table.get(key) or []: + if isinstance(column, dict): + columns.append(column) + return columns + + +def _convert_table(table, dialect, taken_names): + if not isinstance(table, dict): + raise ConversionError("Each entry of 'tables' must be a mapping") + source = require_str(table, "name", "table") + name = dataset_name_for(source, taken_names) + + dataset = {"name": name, "source": source} + + primary_key = _split_primary_key(table.get("primary_key")) + if primary_key: + dataset["primary_key"] = primary_key + + description = clean_text(table.get("description")) + if description: + dataset["description"] = description + # Solid carries an AI-written `description` and a human-written + # `manual_description`. Apache Ossie has one description field, so the AI one -- the + # consistently populated, consumer-facing text -- maps to `description` and the + # human annotation maps to `ai_context.instructions`. The split is 1:1 in both + # directions, so neither needs stashing. + ai_context = build_ai_context( + instructions=clean_text(table.get("manual_description")), + synonyms=string_list(table.get("synonyms")), + ) + if ai_context: + dataset["ai_context"] = ai_context + + fields, field_names = [], set() + for column in table.get("dimensions") or []: + fields.append(_convert_column(column, dialect, name, field_names, True)) + for column in table.get("facts") or []: + fields.append(_convert_column(column, dialect, name, field_names, False)) + if fields: + dataset["fields"] = fields + + stash = {} + quality_rank = clean_text(table.get("quality_rank")) + if quality_rank: + stash["quality_rank"] = quality_rank + indexes = string_list(table.get("indexes")) + if indexes: + stash["indexes"] = indexes + write_stash(dataset, stash) + return dataset + + +def _split_primary_key(value): + """Split Solid's comma-joined primary key scalar into Apache Ossie's column array. + + solid-server joins a composite key's column names with `", "` into a single scalar + (`primary_key: 'ORDER_ID, LINE_NO'`), so splitting on the comma is what recovers the + composite. A non-string value is tolerated: a list is taken as-is, since a + hand-edited export may already carry one. + """ + if isinstance(value, list): + return string_list(value) + text = clean_text(value) + if not text: + return [] + return [part.strip() for part in text.split(",") if part.strip()] + + +def _convert_column(column, dialect, dataset_name, taken, is_dimension): + if not isinstance(column, dict): + raise ConversionError( + f"Dataset '{dataset_name}': each column entry must be a mapping") + raw_name = require_str(column, "name", f"column in '{dataset_name}'") + name = unique_name(raw_name, taken) + if name != raw_name: + warn("field", f"'{dataset_name}' declares '{raw_name}' more than once; the " + f"duplicate was renamed to '{name}'") + + raw_type = clean_text(column.get("type")) + # Solid renders two kinds of fact: a catalog column (which has a `type`) and a + # table-scoped metric (which has an `expression` and no `type`). The former's + # expression is just its own name. + solid_expression = clean_text(column.get("expression")) + expression = solid_expression or column_reference(raw_name, dialect) + + field = { + "name": name, + "expression": {"dialects": [{"dialect": dialect, "expression": expression}]}, + } + description = clean_text(column.get("description")) + if description: + field["description"] = description + + datatype = datatypes.to_ossie_datatype(raw_type, dialect) + if datatype: + field["datatype"] = datatype + elif raw_type: + warn("field", f"'{dataset_name}.{name}': warehouse type '{raw_type}' has no " + f"Apache Ossie datatype; 'datatype' was omitted") + + if is_dimension: + # The presence of the `dimension` block is what records that Solid classified + # this column as a dimension rather than a fact, so it is always emitted for a + # dimension -- with `is_time` stated explicitly rather than left to the spec's + # datatype-derived default, so a consumer that does not implement that default + # still reads the same role. + field["dimension"] = {"is_time": datatype in datatypes.TEMPORAL_DATATYPES} + + ai_context = build_ai_context( + instructions=clean_text(column.get("manual_description")), + synonyms=string_list(column.get("synonyms")), + ) + if ai_context: + field["ai_context"] = ai_context + + stash = {} + if raw_type: + # Apache Ossie's portable datatype is lossy (NUMBER(38,2), NUMERIC and DECIMAL + # all collapse to Decimal), so the catalog's own type name is kept verbatim. + stash["type"] = raw_type + elif solid_expression: + stash["role"] = "metric" + sample_values = column.get("sample_values") + if sample_values: + stash["sample_values"] = sample_values + write_stash(field, stash) + return field + + +def _convert_relationships(relationships, by_source): + converted, taken = [], set() + for relationship in relationships: + if not isinstance(relationship, dict): + raise ConversionError("Each entry of 'relationships' must be a mapping") + left_source = require_str(relationship, "left_table", "relationship") + right_source = require_str(relationship, "right_table", "relationship") + left = by_source.get(left_source.lower()) + right = by_source.get(right_source.lower()) + if left is None or right is None: + missing = left_source if left is None else right_source + warn("relationship", + f"'{left_source}' -> '{right_source}' references table '{missing}', " + f"which is not in 'tables'; the relationship was dropped") + continue + + join_keys = relationship.get("join_keys") + join_keys = join_keys if isinstance(join_keys, dict) else {} + left_columns = string_list(join_keys.get("left")) + right_columns = string_list(join_keys.get("right")) + if not left_columns or not right_columns: + warn("relationship", + f"'{left['name']}' -> '{right['name']}' has no join keys; the " + f"relationship was dropped (an Apache Ossie relationship requires at " + f"least one column on each side)") + continue + if len(left_columns) != len(right_columns): + raise ConversionError( + f"Relationship '{left['name']}' -> '{right['name']}' has " + f"{len(left_columns)} left join key(s) but {len(right_columns)} right; " + f"the columns must correspond positionally" + ) + + converted.append(_orient(left, right, left_columns, right_columns, taken)) + return converted + + +def _orient(left, right, left_columns, right_columns, taken): + """Turn a Solid join into a directed Apache Ossie relationship. + + Apache Ossie encodes cardinality through direction -- `from` is the many side, `to` + the one side -- but a Solid relationship is an undirected pair of column lists. The + one side is therefore recovered from the primary keys: whichever end's primary key + is exactly its join columns is unique on those columns, and so is the one side. + """ + left_is_key = _covers_primary_key(left, left_columns) + right_is_key = _covers_primary_key(right, right_columns) + + note = None + if right_is_key and not left_is_key: + flipped = False + elif left_is_key and not right_is_key: + flipped = True + elif left_is_key and right_is_key: + # Both ends are unique on their join columns: a one-to-one. Apache Ossie has no + # dedicated form, so the direction is arbitrary and recorded in `ai_context`. + flipped = False + note = ONE_TO_ONE_NOTE + else: + flipped = False + warn("relationship", + f"'{left['name']}' -> '{right['name']}': neither side's primary key " + f"matches its join columns, so the many/one direction could not be " + f"determined; assumed '{left['name']}' is the many side. Solid does not " + f"record cardinality.") + + if flipped: + source, target = right, left + source_columns, target_columns = right_columns, left_columns + else: + source, target = left, right + source_columns, target_columns = left_columns, right_columns + + name = unique_name(f"{source['name']}_to_{target['name']}", taken) + relationship = { + "name": name, + "from": source["name"], + "to": target["name"], + "from_columns": source_columns, + "to_columns": target_columns, + } + if note: + relationship["ai_context"] = {"instructions": note} + if flipped: + # Only the orientation needs recording: relationships keep their list position in + # both directions, so the ordering restores itself. Export re-emits + # `left_table`/`right_table` the way Solid wrote them. + write_stash(relationship, {"flipped": True}) + return relationship + + +def _covers_primary_key(dataset, columns): + """True if `columns` is exactly the dataset's primary key (order-insensitive).""" + primary_key = dataset.get("primary_key") + if not primary_key: + return False + return {c.lower() for c in primary_key} == {c.lower() for c in columns} + + +def _convert_metrics(metrics, by_source, datasets, dialect): + dataset_names = [d["name"] for d in datasets] + converted, taken = [], set() + for metric in metrics: + if not isinstance(metric, dict): + raise ConversionError("Each entry of 'metrics' must be a mapping") + raw_name = require_str(metric, "name", "metric") + name = unique_name(raw_name, taken) + if name != raw_name: + warn("metric", f"'{raw_name}' is declared more than once; the duplicate " + f"was renamed to '{name}'") + + expression = clean_text(metric.get("expression")) + if not expression: + warn("metric", f"'{name}' has no expression and was dropped (an Apache " + f"Ossie metric requires one)") + continue + + owners, unknown = [], [] + for source in string_list(metric.get("tables")): + dataset = by_source.get(source.lower()) + (owners if dataset is not None else unknown).append(dataset or source) + for source in unknown: + warn("metric", f"'{name}' references table '{source}', which is not in " + f"'tables'") + + expression, status = _qualify(name, expression, owners, dialect) + + converted_metric = { + "name": name, + "expression": {"dialects": [{"dialect": dialect, "expression": expression}]}, + } + description = clean_text(metric.get("description")) + if description: + converted_metric["description"] = description + synonyms = string_list(metric.get("synonyms")) + if synonyms: + converted_metric["ai_context"] = {"synonyms": synonyms} + + # Solid's `tables` list only needs stashing when the qualified expression does + # not already name the same datasets -- which is the case for an unqualifiable + # expression, and for one that references no column at all (`COUNT(*)`). + expected = [d["name"] for d in owners] + if (unknown + or status in (UNPARSED, AMBIGUOUS) + or referenced_datasets(expression, dialect, dataset_names) != expected): + write_stash(converted_metric, + {"tables": string_list(metric.get("tables"))}) + converted.append(converted_metric) + return converted + + +def _qualify(name, expression, owners, dialect): + """Qualify a Solid metric's bare column references with their dataset name. + + Solid stores formulas against bare columns and records the owning table separately, + so the dataset is unambiguous only when exactly one table owns the metric. A + multi-table metric is left verbatim: solid-server strips alias prefixes when the + formula is saved and keeps the alias-to-table binding in `metric.column_ids`, which + its YAML export does not carry, so the binding cannot be recovered from the file. + """ + if len(owners) != 1: + if len(owners) > 1: + warn("metric", + f"'{name}' spans {len(owners)} tables " + f"({', '.join(d['name'] for d in owners)}); its column references " + f"cannot be attributed to a dataset and were left unqualified") + else: + warn("metric", f"'{name}' names no owning table; its column references " + f"were left unqualified") + return expression, UNCHANGED + + dataset = owners[0] + columns = [f["name"] for f in dataset.get("fields") or []] + qualified, status = qualify_metric(expression, dialect, dataset["name"], columns) + if status == UNPARSED: + warn("metric", f"'{name}': expression could not be parsed as {dialect} SQL and " + f"was left unqualified") + elif status == AMBIGUOUS: + warn("metric", f"'{name}': a column name in the expression also appears in a " + f"non-column position, so the reference could not be qualified " + f"safely; the expression was left as written") + return qualified, status + + +__all__ = ["convert_solid_to_ossie"] diff --git a/converters/solid/tests/conftest.py b/converters/solid/tests/conftest.py new file mode 100644 index 00000000..ffb615b4 --- /dev/null +++ b/converters/solid/tests/conftest.py @@ -0,0 +1,144 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared test helpers: fixture loading, warning capture, and schema validation.""" + +import json +import sys +import warnings +from pathlib import Path + +import pytest +import yaml + +FIXTURES = Path(__file__).parent / "fixtures" + +# The converter is checked against the specification's own JSON Schema, which lives in +# the repository rather than in this package. +SCHEMA_PATH = Path(__file__).parents[3] / "core-spec" / "ossie-schema.json" +EXAMPLES = Path(__file__).parents[3] / "examples" + +# The other converters' own test fixtures, used read-only by the cross-vendor +# interop sweep (see test_cross_vendor.py). +CONVERTERS = Path(__file__).parents[2] + +if str(Path(__file__).parents[1] / "src") not in sys.path: + sys.path.insert(0, str(Path(__file__).parents[1] / "src")) + + +def fixture(name): + """Read a fixture file as text.""" + return (FIXTURES / name).read_text() + + +def example(name): + """Read a model from the repository's `examples/` directory.""" + return (EXAMPLES / name).read_text() + + +def convert_quietly(func, *args, **kwargs): + """Run a converter, returning (output, [warning messages]).""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + output = func(*args, **kwargs) + return output, [str(w.message) for w in caught] + + +def model_of(ossie_yaml): + """The first semantic model in an Apache Ossie document.""" + return yaml.safe_load(ossie_yaml)["semantic_model"][0] + + +def solid_model_of(solid_yaml): + """The semantic model in a Solid document.""" + return yaml.safe_load(solid_yaml)["semantic_model"] + + +def by_name(items): + """Index a list of named mappings by `name`.""" + return {item["name"]: item for item in items or []} + + +def normalized_yaml(text): + """Parse YAML and normalize block-scalar whitespace. + + Solid writes descriptions as block scalars, so the same text can differ between two + renderings only by trailing newlines and line-end whitespace. That is not a + conversion difference, so it is normalized away before comparing. + """ + + def walk(node): + if isinstance(node, dict): + return {k: walk(v) for k, v in node.items()} + if isinstance(node, list): + return [walk(v) for v in node] + if isinstance(node, str): + return "\n".join(line.rstrip() for line in node.strip().splitlines()) + return node + + return walk(yaml.safe_load(text)) + + +def dataset_of(model, name): + return by_name(model["datasets"])[name] + + +def field_of(model, dataset, field): + return by_name(dataset_of(model, dataset).get("fields"))[field] + + +def stash_of(obj): + """The SOLID custom_extensions payload on an object, or {} when absent.""" + for ext in obj.get("custom_extensions") or []: + if ext["vendor_name"] == "SOLID": + data = json.loads(ext["data"]) + data.pop("_v", None) + return data + return {} + + +def expression_of(obj): + """The single dialect expression on a field or metric.""" + dialects = obj["expression"]["dialects"] + assert len(dialects) == 1, "converter emits exactly one dialect per expression" + return dialects[0]["expression"] + + +@pytest.fixture(scope="session") +def ossie_validator(): + """A Draft 2020-12 validator for the Apache Ossie core schema. + + Skips the whole test if `jsonschema` is not installed, so the suite still runs + without the dev extra. + """ + jsonschema = pytest.importorskip("jsonschema") + schema = json.loads(SCHEMA_PATH.read_text()) + return jsonschema.Draft202012Validator(schema) + + +@pytest.fixture +def assert_valid_ossie(ossie_validator): + """Assert that a converted document satisfies the Apache Ossie JSON Schema.""" + + def _assert(ossie_yaml): + document = yaml.safe_load(ossie_yaml) + errors = sorted(ossie_validator.iter_errors(document), key=lambda e: list(e.path)) + assert not errors, "\n".join( + f"{'/'.join(str(p) for p in e.path)}: {e.message}" for e in errors + ) + + return _assert diff --git a/converters/solid/tests/fixtures/bigquery_solid.yaml b/converters/solid/tests/fixtures/bigquery_solid.yaml new file mode 100644 index 00000000..06e2e5fa --- /dev/null +++ b/converters/solid/tests/fixtures/bigquery_solid.yaml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# A small Solid export whose type vocabulary is BigQuery's: INT64, FLOAT64, BOOL. +# The project id carries a hyphen, which a dataset name must not, and the metric uses +# COUNTIF -- a BigQuery-only function that has to survive qualification untouched. +semantic_model: + name: 'web_sessions' + + business_context: + model_description: | + Session-level web analytics. + + model_llm_description: | + Web session analytics: traffic, engagement, and signup conversion. + + tables: + - name: 'analytics-prod.web.sessions' + description: | + One row per web session. + synonyms: + - 'visits' + primary_key: 'session_id' + quality_rank: 'high' + dimensions: + - name: 'session_id' + type: 'STRING' + description: Session identifier. + - name: 'channel' + type: 'STRING' + description: Acquisition channel. + sample_values: + - 'organic' + - 'paid' + - name: 'is_logged_in' + type: 'BOOL' + description: Whether the visitor was authenticated. + - name: 'started_on' + type: 'DATE' + description: Session start date. + - name: 'started_at' + type: 'DATETIME' + description: Session start, with no timezone. + facts: + - name: 'page_views' + type: 'INT64' + description: Pages viewed during the session. + - name: 'engagement_seconds' + type: 'FLOAT64' + description: Seconds of active engagement. + + metrics: + - name: 'LOGGED_IN_SESSIONS' + description: | + Sessions where the visitor was authenticated. + expression: 'COUNTIF(is_logged_in = TRUE)' + tables: + - 'analytics-prod.web.sessions' + - name: 'AVG_ENGAGEMENT' + description: | + Mean active engagement per session. + expression: 'AVG(engagement_seconds)' + tables: + - 'analytics-prod.web.sessions' diff --git a/converters/solid/tests/fixtures/databricks_solid.yaml b/converters/solid/tests/fixtures/databricks_solid.yaml new file mode 100644 index 00000000..4d71b5c5 --- /dev/null +++ b/converters/solid/tests/fixtures/databricks_solid.yaml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# A small Solid export whose type vocabulary is Databricks': LONG, STRING, DOUBLE, MAP. +# Used to check that dialect inference resolves DATABRICKS from types alone, and that +# TIMESTAMP is read as an instant here (unlike Snowflake, where it is not). +semantic_model: + name: 'orders_analytics' + + business_context: + business_questions: + - 'How many orders shipped late last month?' + + model_llm_description: | + Order fulfilment analytics over the warehouse order stream. + + tables: + - name: 'main.sales.orders' + description: | + One row per order. + synonyms: + - 'order stream' + primary_key: 'order_id' + quality_rank: 'high' + dimensions: + - name: 'order_id' + type: 'LONG' + description: Order identifier. + - name: 'customer_id' + type: 'LONG' + description: Customer who placed the order. + - name: 'status' + type: 'STRING' + description: Fulfilment status. + sample_values: + - 'shipped' + - 'pending' + - name: 'placed_at' + type: 'TIMESTAMP' + description: Instant the order was placed. + - name: 'ingested_at' + type: 'TIMESTAMP_NTZ' + description: Pipeline load time, with no timezone. + - name: 'attributes' + type: 'MAP' + description: Free-form order attributes. + facts: + - name: 'order_total' + type: 'DOUBLE' + description: Order total in USD. + - name: 'line_count' + type: 'INT' + description: Number of lines on the order. + + - name: 'main.sales.customers' + description: | + Customer master. + primary_key: 'customer_id' + quality_rank: 'mediocre' + dimensions: + - name: 'customer_id' + type: 'LONG' + description: Customer identifier. + - name: 'region' + type: 'STRING' + description: Sales region. + facts: [] + + metrics: + - name: 'LATE_ORDER_COUNT' + description: | + Orders whose status is still pending. + expression: "COUNT(CASE WHEN status = 'pending' THEN order_id END)" + tables: + - 'main.sales.orders' + - name: 'GROSS_REVENUE' + description: | + Sum of order totals. + expression: 'SUM(order_total)' + synonyms: + - 'revenue' + tables: + - 'main.sales.orders' + + relationships: + - left_table: 'main.sales.orders' + right_table: 'main.sales.customers' + join_keys: + left: + - 'customer_id' + right: + - 'customer_id' diff --git a/converters/solid/tests/fixtures/foreign_ossie.yaml b/converters/solid/tests/fixtures/foreign_ossie.yaml new file mode 100644 index 00000000..314896d7 --- /dev/null +++ b/converters/solid/tests/fixtures/foreign_ossie.yaml @@ -0,0 +1,149 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# A stand-in for an Apache Ossie model produced by SOME OTHER vendor's converter. +# +# Every construct here was observed in the Ossie fixtures the other converters in this +# repository ship (dbt, Databricks, GoodData, Omni, NVIDIA GSF, Orion Belt). This file +# reproduces them under `converters/solid/` so the interop gaps can be pinned exactly +# without this converter's CI depending on another converter's fixtures -- the +# workflows are path-filtered per converter, so a pin against a foreign fixture would +# break on a main-branch push from a PR that never ran this suite. +# +# What makes it "foreign", and what each construct exercises: +# +# - no `custom_extensions[SOLID]` stash anywhere, so every fallback path is used +# - no `dimension` block anywhere, so the fact/dimension split falls back to +# Solid's own data-type rule +# - `datatype` omitted on most fields (it is optional in the spec, and the other +# converters largely do omit it) -> an empty Solid `type` +# - `store_sales.ticket_number`: a field RENAMED relative to its column +# - `date_dim.d_year`: a `label`, which Solid has no slot for +# - `store_sales`: `unique_keys`, which Solid has no slot for +# - `revenue`: a metric written against BARE column names, as a vendor that keeps +# the owning table elsewhere would write it +# - `order_count`: `COUNT(*)`, which references no column at all +# - `revenue`: a metric `datatype` +# - `store_sales_to_date`: a relationship annotation +# - foreign `custom_extensions` at the model, dataset, field and metric levels +version: "0.2.0.dev0" +semantic_model: + - name: foreign_retail_model + description: Retail sales, as another vendor's converter would emit it. + ai_context: + instructions: Prefer net sales over gross for revenue questions. + examples: + - What were total sales last quarter? + custom_extensions: + - vendor_name: DBT + data: '{"package": "jaffle_shop"}' + datasets: + - name: store_sales + source: tpcds.public.store_sales + description: One row per line item on a sales ticket. + primary_key: + - ss_item_sk + - ss_ticket_number + unique_keys: + - - ss_ticket_number + custom_extensions: + - vendor_name: DATABRICKS + data: '{"table_type": "MANAGED"}' + fields: + - name: ss_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_item_sk + - name: ss_ticket_number + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ticket_number + - name: ss_sold_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sold_date_sk + # Renamed: the Solid column that comes out of this is named for the alias, + # not for the column that actually exists in the warehouse. + - name: ticket_number + description: Ticket number, exposed under a friendlier name. + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ticket_number + custom_extensions: + - vendor_name: DATABRICKS + data: '{"comment": "renamed for readability"}' + - name: ss_net_paid + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_net_paid + - name: date_dim + source: tpcds.public.date_dim + primary_key: + - d_date_sk + fields: + - name: d_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: d_date_sk + - name: d_year + label: Year + datatype: Integer + expression: + dialects: + - dialect: ANSI_SQL + expression: d_year + ai_context: + synonyms: + - year + relationships: + - name: store_sales_to_date + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - d_date_sk + ai_context: + instructions: Sales are dated by the sold date, not the ship date. + synonyms: + - sold on + metrics: + # Bare column names: the owning table is not recoverable from a qualifier. + - name: revenue + description: Net sales revenue. + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(ss_net_paid) + custom_extensions: + - vendor_name: DBT + data: '{"agg": "sum"}' + # References no column at all. + - name: order_count + description: Number of tickets. + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(*) diff --git a/converters/solid/tests/fixtures/tpcds_ossie.yaml b/converters/solid/tests/fixtures/tpcds_ossie.yaml new file mode 100644 index 00000000..13978acb --- /dev/null +++ b/converters/solid/tests/fixtures/tpcds_ossie.yaml @@ -0,0 +1,723 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Expected output of `ossie-solid import -i tpcds_solid.yaml`. Regenerate with: +# ossie-solid import -i tests/fixtures/tpcds_solid.yaml -o tests/fixtures/tpcds_ossie.yaml +# then re-add this header. +version: 0.2.0.dev0 +version: 0.2.0.dev0 +semantic_model: +- name: tpcds_retail_model + description: 'TPC-DS retail semantic model for sales and customer analytics. Supports time-based + + analysis, customer segmentation, product performance, and store operations metrics.' + ai_context: + instructions: '- Always join through public.store_sales rather than aggregating a dimension on its + own. + + - Revenue means store_sales.ss_ext_sales_price, not ss_sales_price, which is a per-unit figure. + + - Exclude rows where ss_sold_date_sk is null; those are unshipped orders.' + examples: + - What were total sales and net profit by month for the last four quarters? + - Which brands drove the most revenue in the Electronics category? + - How does sales per employee compare across stores in each state? + datasets: + - name: store_sales + source: tpcds.public.store_sales + primary_key: + - ss_item_sk + - ss_ticket_number + description: Fact table containing all store sales transactions, one row per line item. + ai_context: + instructions: Grain is (ss_item_sk, ss_ticket_number). Loaded nightly from the POS extract. + synonyms: + - sales transactions + - store purchases + - retail sales + - POS data + fields: + - name: ss_sold_date_sk + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_sold_date_sk + description: Foreign key to the date dimension. + datatype: Integer + dimension: + is_time: false + ai_context: + synonyms: + - sale date + - transaction date + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: ss_item_sk + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_item_sk + description: Foreign key to the item dimension. + datatype: Integer + dimension: + is_time: false + ai_context: + synonyms: + - product + - item + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: ss_ticket_number + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_ticket_number + description: Ticket number identifying the sale the line item belongs to. + datatype: Integer + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: ss_customer_sk + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_customer_sk + description: Foreign key to the customer dimension. + datatype: Integer + dimension: + is_time: false + ai_context: + synonyms: + - customer + - buyer + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: ss_store_sk + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_store_sk + description: Foreign key to the store dimension. + datatype: Integer + dimension: + is_time: false + ai_context: + synonyms: + - store + - location + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: ss_sold_at + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_sold_at + description: Wall-clock time the sale was rung up, in the store's local time. + datatype: DateTime + dimension: + is_time: true + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TIMESTAMP_NTZ"}' + - name: ss_quantity + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_quantity + description: Quantity of items sold. + datatype: Integer + ai_context: + synonyms: + - units sold + - quantity + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: ss_sales_price + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_sales_price + description: Sales price per unit. + datatype: Decimal + ai_context: + instructions: Per-unit price. Use ss_ext_sales_price for line revenue. + synonyms: + - unit price + - price + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(7,2)"}' + - name: ss_ext_sales_price + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_ext_sales_price + description: Extended sales price (quantity * price). + datatype: Decimal + ai_context: + synonyms: + - total price + - line total + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(7,2)"}' + - name: ss_net_profit + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_net_profit + description: Net profit from the sale. + datatype: Decimal + ai_context: + synonyms: + - profit + - margin + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(7,2)"}' + - name: ss_discount_pct + expression: + dialects: + - dialect: SNOWFLAKE + expression: 1 - (ss_sales_price / NULLIF(ss_list_price, 0)) + description: Discount applied to the line, as a fraction of list price. + ai_context: + synonyms: + - discount rate + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "role": "metric"}' + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "quality_rank": "high", "indexes": ["ss_sold_date_sk", "ss_customer_sk"]}' + - name: date_dim + source: tpcds.public.date_dim + primary_key: + - d_date_sk + description: Date dimension with calendar attributes. + ai_context: + synonyms: + - calendar + - dates + - time periods + fields: + - name: d_date_sk + expression: + dialects: + - dialect: SNOWFLAKE + expression: d_date_sk + description: Surrogate key for date. + datatype: Integer + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: d_date + expression: + dialects: + - dialect: SNOWFLAKE + expression: d_date + description: Calendar date. + datatype: Date + dimension: + is_time: true + ai_context: + synonyms: + - date + - day + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "DATE"}' + - name: d_year + expression: + dialects: + - dialect: SNOWFLAKE + expression: d_year + description: Calendar year. + datatype: Integer + dimension: + is_time: false + ai_context: + synonyms: + - year + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)", "sample_values": ["2022", "2023", "2024"]}' + - name: d_quarter_name + expression: + dialects: + - dialect: SNOWFLAKE + expression: d_quarter_name + description: Quarter name, for example 2024Q1. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - quarter + - fiscal quarter + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT", "sample_values": ["2024Q1", "2024Q2"]}' + - name: d_month_name + expression: + dialects: + - dialect: SNOWFLAKE + expression: d_month_name + description: Month name. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - month + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "quality_rank": "high"}' + - name: customer + source: tpcds.public.customer + primary_key: + - c_customer_sk + description: Customer dimension with demographic information. + ai_context: + synonyms: + - customers + - shoppers + - buyers + fields: + - name: c_customer_sk + expression: + dialects: + - dialect: SNOWFLAKE + expression: c_customer_sk + description: Surrogate key for customer. + datatype: Integer + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: c_customer_id + expression: + dialects: + - dialect: SNOWFLAKE + expression: c_customer_id + description: Business key for customer. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - customer ID + - customer number + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: c_first_name + expression: + dialects: + - dialect: SNOWFLAKE + expression: c_first_name + description: Customer first name. + datatype: String + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: c_last_name + expression: + dialects: + - dialect: SNOWFLAKE + expression: c_last_name + description: Customer last name. + datatype: String + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: c_email_address + expression: + dialects: + - dialect: SNOWFLAKE + expression: c_email_address + description: Customer email address. + datatype: String + dimension: + is_time: false + ai_context: + instructions: Nullable. Roughly 12% of rows have no email on file. + synonyms: + - email + - contact + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: c_preferences + expression: + dialects: + - dialect: SNOWFLAKE + expression: c_preferences + description: Semi-structured marketing preference document. + datatype: Opaque + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "VARIANT"}' + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "quality_rank": "mediocre"}' + - name: item + source: tpcds.public.item + primary_key: + - i_item_sk + description: Item dimension with product attributes. + ai_context: + synonyms: + - products + - items + - merchandise + fields: + - name: i_item_sk + expression: + dialects: + - dialect: SNOWFLAKE + expression: i_item_sk + description: Surrogate key for item. + datatype: Integer + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: i_item_id + expression: + dialects: + - dialect: SNOWFLAKE + expression: i_item_id + description: Business key for item. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - item ID + - product ID + - SKU + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: i_item_desc + expression: + dialects: + - dialect: SNOWFLAKE + expression: i_item_desc + description: Item description. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - product description + - item name + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: i_brand + expression: + dialects: + - dialect: SNOWFLAKE + expression: i_brand + description: Brand name. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - brand + - manufacturer + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: i_category + expression: + dialects: + - dialect: SNOWFLAKE + expression: i_category + description: Item category. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - product category + - department + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT", "sample_values": ["Electronics", "Home", "Sports"]}' + - name: i_current_price + expression: + dialects: + - dialect: SNOWFLAKE + expression: i_current_price + description: Current price of the item. + datatype: Decimal + ai_context: + synonyms: + - price + - list price + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(7,2)"}' + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "quality_rank": "high"}' + - name: store + source: tpcds.public.store + primary_key: + - s_store_sk + description: Store dimension with location and store attributes. + ai_context: + synonyms: + - stores + - retail locations + - branches + fields: + - name: s_store_sk + expression: + dialects: + - dialect: SNOWFLAKE + expression: s_store_sk + description: Surrogate key for store. + datatype: Integer + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: s_store_id + expression: + dialects: + - dialect: SNOWFLAKE + expression: s_store_id + description: Business key for store. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - store ID + - store number + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: s_store_name + expression: + dialects: + - dialect: SNOWFLAKE + expression: s_store_name + description: Store name. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - store name + - location name + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: s_city + expression: + dialects: + - dialect: SNOWFLAKE + expression: s_city + description: City where the store is located. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - city + - location + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: s_state + expression: + dialects: + - dialect: SNOWFLAKE + expression: s_state + description: State where the store is located. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - state + - region + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT", "sample_values": ["CA", "NY", "TX"]}' + - name: s_is_franchise + expression: + dialects: + - dialect: SNOWFLAKE + expression: s_is_franchise + description: Whether the location is franchise-operated. + datatype: Boolean + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "BOOLEAN"}' + - name: s_number_employees + expression: + dialects: + - dialect: SNOWFLAKE + expression: s_number_employees + description: Number of employees at the store. + datatype: Integer + ai_context: + synonyms: + - employee count + - staff size + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "quality_rank": "high"}' + relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - d_date_sk + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "flipped": true}' + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + - name: store_sales_to_item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + - name: store_sales_to_store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk + metrics: + - name: TOTAL_SALES + expression: + dialects: + - dialect: SNOWFLAKE + expression: SUM(store_sales.ss_ext_sales_price) + description: Total sales revenue across all transactions. + ai_context: + synonyms: + - total revenue + - gross sales + - sales amount + - name: TOTAL_PROFIT + expression: + dialects: + - dialect: SNOWFLAKE + expression: SUM(store_sales.ss_net_profit) + description: Total net profit from store sales. + ai_context: + synonyms: + - net profit + - total earnings + - name: TRANSACTION_COUNT + expression: + dialects: + - dialect: SNOWFLAKE + expression: COUNT(DISTINCT store_sales.ss_ticket_number) + description: Number of distinct sales tickets. + - name: ROW_COUNT + expression: + dialects: + - dialect: SNOWFLAKE + expression: COUNT(*) + description: Row count, used as a denominator in coverage checks. + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "tables": ["tpcds.public.store_sales"]}' + - name: CUSTOMER_LIFETIME_VALUE + expression: + dialects: + - dialect: SNOWFLAKE + expression: SUM(ss_ext_sales_price) / COUNT(DISTINCT c_customer_sk) + description: 'Average lifetime sales value per customer. Spans two tables, so Solid stores the + + formula with bare column names and resolves them at query time.' + ai_context: + synonyms: + - CLV + - LTV + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "tables": ["tpcds.public.store_sales", "tpcds.public.customer"]}' + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "dialect": "SNOWFLAKE", "model_llm_description": "TPC-DS retail semantic model for + sales and customer analytics. Supports time-based\nanalysis, customer segmentation, product performance, + and store operations metrics.", "model_description": "Retail analytics for the TPC-DS benchmark + schema: store sales transactions joined\nto their date, customer, item, and store dimensions.", + "custom_instructions": "- Always join through @ rather than aggregating a dimension on its own.\n- Revenue + means @, + not ss_sales_price, which is a per-unit figure.\n- Exclude rows where ss_sold_date_sk is null; those + are unshipped orders.", "example_queries": [{"name": "Monthly revenue and profit", "description": + "Revenue and profit by calendar month for the trailing twelve months.\n", "sql": "SELECT d.d_year,\n d.d_month_name,\n SUM(ss.ss_ext_sales_price) + AS revenue,\n SUM(ss.ss_net_profit) AS profit\nFROM tpcds.public.store_sales ss\nJOIN + tpcds.public.date_dim d ON d.d_date_sk = ss.ss_sold_date_sk\nWHERE d.d_date >= DATEADD(month, -12, + CURRENT_DATE())\nGROUP BY 1, 2\nORDER BY 1, 2\n"}, {"name": "Sales per employee by state", "description": + "Store productivity, ranked within each state.\n", "sql": "SELECT s.s_state,\n s.s_store_name,\n SUM(ss.ss_ext_sales_price) + / NULLIF(MAX(s.s_number_employees), 0) AS sales_per_employee\nFROM tpcds.public.store_sales ss\nJOIN + tpcds.public.store s ON s.s_store_sk = ss.ss_store_sk\nGROUP BY 1, 2\nORDER BY 1, 3 DESC\n"}], "benchmark_questions": + [{"question": "What was total revenue in 2024Q1?", "is_enabled": true, "expected_sql": "SELECT SUM(ss.ss_ext_sales_price) + AS revenue\nFROM tpcds.public.store_sales ss\nJOIN tpcds.public.date_dim d ON d.d_date_sk = ss.ss_sold_date_sk\nWHERE + d.d_quarter_name = ''2024Q1''\n"}, {"question": "Which brand had the highest revenue in the Electronics + category?", "is_enabled": false}]}' diff --git a/converters/solid/tests/fixtures/tpcds_solid.yaml b/converters/solid/tests/fixtures/tpcds_solid.yaml new file mode 100644 index 00000000..ea8168c7 --- /dev/null +++ b/converters/solid/tests/fixtures/tpcds_solid.yaml @@ -0,0 +1,424 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The TPC-DS retail model from ../../../examples/tpcds_semantic_model.yaml, expressed +# as a Solid semantic model export. Types use Snowflake's vocabulary, which is also what +# the converter's dialect inference reads to recover SNOWFLAKE from this file. +# +# Deliberately exercises the Solid constructs that have no Apache Ossie core equivalent +# (quality_rank, indexes, sample_values, example_queries, benchmark_questions, +# manual_description, asset-link markup) and the ones that need reshaping (a composite +# primary key written as one comma-joined scalar, a relationship whose primary key sits +# on the left, an expression-only fact, and both single- and multi-table metrics). +semantic_model: + name: 'tpcds_retail_model' + + business_context: + business_questions: + - 'What were total sales and net profit by month for the last four quarters?' + - 'Which brands drove the most revenue in the Electronics category?' + - 'How does sales per employee compare across stores in each state?' + model_description: | + Retail analytics for the TPC-DS benchmark schema: store sales transactions joined + to their date, customer, item, and store dimensions. + custom_instructions: | + - Always join through @ rather than aggregating a dimension on its own. + - Revenue means @, not ss_sales_price, which is a per-unit figure. + - Exclude rows where ss_sold_date_sk is null; those are unshipped orders. + + model_llm_description: | + TPC-DS retail semantic model for sales and customer analytics. Supports time-based + analysis, customer segmentation, product performance, and store operations metrics. + + tables: + - name: 'tpcds.public.store_sales' + description: | + Fact table containing all store sales transactions, one row per line item. + manual_description: | + Grain is (ss_item_sk, ss_ticket_number). Loaded nightly from the POS extract. + synonyms: + - 'sales transactions' + - 'store purchases' + - 'retail sales' + - 'POS data' + primary_key: 'ss_item_sk, ss_ticket_number' + quality_rank: 'high' + indexes: + - 'ss_sold_date_sk' + - 'ss_customer_sk' + dimensions: + - name: 'ss_sold_date_sk' + type: 'NUMBER(38,0)' + description: | + Foreign key to the date dimension. + synonyms: + - 'sale date' + - 'transaction date' + - name: 'ss_item_sk' + type: 'NUMBER(38,0)' + description: | + Foreign key to the item dimension. + synonyms: + - 'product' + - 'item' + - name: 'ss_ticket_number' + type: 'NUMBER(38,0)' + description: | + Ticket number identifying the sale the line item belongs to. + - name: 'ss_customer_sk' + type: 'NUMBER(38,0)' + description: | + Foreign key to the customer dimension. + synonyms: + - 'customer' + - 'buyer' + - name: 'ss_store_sk' + type: 'NUMBER(38,0)' + description: | + Foreign key to the store dimension. + synonyms: + - 'store' + - 'location' + - name: 'ss_sold_at' + type: 'TIMESTAMP_NTZ' + description: | + Wall-clock time the sale was rung up, in the store's local time. + facts: + - name: 'ss_quantity' + type: 'NUMBER(38,0)' + description: Quantity of items sold. + synonyms: + - 'units sold' + - 'quantity' + - name: 'ss_sales_price' + type: 'NUMBER(7,2)' + description: Sales price per unit. + manual_description: Per-unit price. Use ss_ext_sales_price for line revenue. + synonyms: + - 'unit price' + - 'price' + - name: 'ss_ext_sales_price' + type: 'NUMBER(7,2)' + description: Extended sales price (quantity * price). + synonyms: + - 'total price' + - 'line total' + - name: 'ss_net_profit' + type: 'NUMBER(7,2)' + description: Net profit from the sale. + synonyms: + - 'profit' + - 'margin' + - name: 'ss_discount_pct' + description: Discount applied to the line, as a fraction of list price. + expression: '1 - (ss_sales_price / NULLIF(ss_list_price, 0))' + synonyms: + - 'discount rate' + + - name: 'tpcds.public.date_dim' + description: | + Date dimension with calendar attributes. + synonyms: + - 'calendar' + - 'dates' + - 'time periods' + primary_key: 'd_date_sk' + quality_rank: 'high' + dimensions: + - name: 'd_date_sk' + type: 'NUMBER(38,0)' + description: Surrogate key for date. + - name: 'd_date' + type: 'DATE' + description: Calendar date. + synonyms: + - 'date' + - 'day' + - name: 'd_year' + type: 'NUMBER(38,0)' + description: Calendar year. + synonyms: + - 'year' + sample_values: + - '2022' + - '2023' + - '2024' + - name: 'd_quarter_name' + type: 'TEXT' + description: Quarter name, for example 2024Q1. + synonyms: + - 'quarter' + - 'fiscal quarter' + sample_values: + - '2024Q1' + - '2024Q2' + - name: 'd_month_name' + type: 'TEXT' + description: Month name. + synonyms: + - 'month' + facts: [] + + - name: 'tpcds.public.customer' + description: | + Customer dimension with demographic information. + synonyms: + - 'customers' + - 'shoppers' + - 'buyers' + primary_key: 'c_customer_sk' + quality_rank: 'mediocre' + dimensions: + - name: 'c_customer_sk' + type: 'NUMBER(38,0)' + description: Surrogate key for customer. + - name: 'c_customer_id' + type: 'TEXT' + description: Business key for customer. + synonyms: + - 'customer ID' + - 'customer number' + - name: 'c_first_name' + type: 'TEXT' + description: Customer first name. + - name: 'c_last_name' + type: 'TEXT' + description: Customer last name. + - name: 'c_email_address' + type: 'TEXT' + description: Customer email address. + manual_description: Nullable. Roughly 12% of rows have no email on file. + synonyms: + - 'email' + - 'contact' + - name: 'c_preferences' + type: 'VARIANT' + description: Semi-structured marketing preference document. + facts: [] + + - name: 'tpcds.public.item' + description: | + Item dimension with product attributes. + synonyms: + - 'products' + - 'items' + - 'merchandise' + primary_key: 'i_item_sk' + quality_rank: 'high' + dimensions: + - name: 'i_item_sk' + type: 'NUMBER(38,0)' + description: Surrogate key for item. + - name: 'i_item_id' + type: 'TEXT' + description: Business key for item. + synonyms: + - 'item ID' + - 'product ID' + - 'SKU' + - name: 'i_item_desc' + type: 'TEXT' + description: Item description. + synonyms: + - 'product description' + - 'item name' + - name: 'i_brand' + type: 'TEXT' + description: Brand name. + synonyms: + - 'brand' + - 'manufacturer' + - name: 'i_category' + type: 'TEXT' + description: Item category. + synonyms: + - 'product category' + - 'department' + sample_values: + - 'Electronics' + - 'Home' + - 'Sports' + facts: + - name: 'i_current_price' + type: 'NUMBER(7,2)' + description: Current price of the item. + synonyms: + - 'price' + - 'list price' + + - name: 'tpcds.public.store' + description: | + Store dimension with location and store attributes. + synonyms: + - 'stores' + - 'retail locations' + - 'branches' + primary_key: 's_store_sk' + quality_rank: 'high' + dimensions: + - name: 's_store_sk' + type: 'NUMBER(38,0)' + description: Surrogate key for store. + - name: 's_store_id' + type: 'TEXT' + description: Business key for store. + synonyms: + - 'store ID' + - 'store number' + - name: 's_store_name' + type: 'TEXT' + description: Store name. + synonyms: + - 'store name' + - 'location name' + - name: 's_city' + type: 'TEXT' + description: City where the store is located. + synonyms: + - 'city' + - 'location' + - name: 's_state' + type: 'TEXT' + description: State where the store is located. + synonyms: + - 'state' + - 'region' + sample_values: + - 'CA' + - 'NY' + - 'TX' + - name: 's_is_franchise' + type: 'BOOLEAN' + description: Whether the location is franchise-operated. + facts: + - name: 's_number_employees' + type: 'NUMBER(38,0)' + description: Number of employees at the store. + synonyms: + - 'employee count' + - 'staff size' + + metrics: + - name: 'TOTAL_SALES' + description: | + Total sales revenue across all transactions. + expression: 'SUM(ss_ext_sales_price)' + synonyms: + - 'total revenue' + - 'gross sales' + - 'sales amount' + tables: + - 'tpcds.public.store_sales' + - name: 'TOTAL_PROFIT' + description: | + Total net profit from store sales. + expression: 'SUM(ss_net_profit)' + synonyms: + - 'net profit' + - 'total earnings' + tables: + - 'tpcds.public.store_sales' + - name: 'TRANSACTION_COUNT' + description: | + Number of distinct sales tickets. + expression: 'COUNT(DISTINCT ss_ticket_number)' + tables: + - 'tpcds.public.store_sales' + - name: 'ROW_COUNT' + description: | + Row count, used as a denominator in coverage checks. + expression: 'COUNT(*)' + tables: + - 'tpcds.public.store_sales' + - name: 'CUSTOMER_LIFETIME_VALUE' + description: | + Average lifetime sales value per customer. Spans two tables, so Solid stores the + formula with bare column names and resolves them at query time. + expression: 'SUM(ss_ext_sales_price) / COUNT(DISTINCT c_customer_sk)' + synonyms: + - 'CLV' + - 'LTV' + tables: + - 'tpcds.public.store_sales' + - 'tpcds.public.customer' + + relationships: + - left_table: 'tpcds.public.date_dim' + right_table: 'tpcds.public.store_sales' + join_keys: + left: + - 'd_date_sk' + right: + - 'ss_sold_date_sk' + - left_table: 'tpcds.public.store_sales' + right_table: 'tpcds.public.customer' + join_keys: + left: + - 'ss_customer_sk' + right: + - 'c_customer_sk' + - left_table: 'tpcds.public.store_sales' + right_table: 'tpcds.public.item' + join_keys: + left: + - 'ss_item_sk' + right: + - 'i_item_sk' + - left_table: 'tpcds.public.store_sales' + right_table: 'tpcds.public.store' + join_keys: + left: + - 'ss_store_sk' + right: + - 's_store_sk' + + example_queries: + - name: 'Monthly revenue and profit' + description: | + Revenue and profit by calendar month for the trailing twelve months. + sql: | + SELECT d.d_year, + d.d_month_name, + SUM(ss.ss_ext_sales_price) AS revenue, + SUM(ss.ss_net_profit) AS profit + FROM tpcds.public.store_sales ss + JOIN tpcds.public.date_dim d ON d.d_date_sk = ss.ss_sold_date_sk + WHERE d.d_date >= DATEADD(month, -12, CURRENT_DATE()) + GROUP BY 1, 2 + ORDER BY 1, 2 + - name: 'Sales per employee by state' + description: | + Store productivity, ranked within each state. + sql: | + SELECT s.s_state, + s.s_store_name, + SUM(ss.ss_ext_sales_price) / NULLIF(MAX(s.s_number_employees), 0) AS sales_per_employee + FROM tpcds.public.store_sales ss + JOIN tpcds.public.store s ON s.s_store_sk = ss.ss_store_sk + GROUP BY 1, 2 + ORDER BY 1, 3 DESC + + benchmark_questions: + - question: 'What was total revenue in 2024Q1?' + is_enabled: true + expected_sql: | + SELECT SUM(ss.ss_ext_sales_price) AS revenue + FROM tpcds.public.store_sales ss + JOIN tpcds.public.date_dim d ON d.d_date_sk = ss.ss_sold_date_sk + WHERE d.d_quarter_name = '2024Q1' + - question: 'Which brand had the highest revenue in the Electronics category?' + is_enabled: false diff --git a/converters/solid/tests/test_cli.py b/converters/solid/tests/test_cli.py new file mode 100644 index 00000000..53a25c1a --- /dev/null +++ b/converters/solid/tests/test_cli.py @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The `ossie-solid` command line interface.""" + +import pytest +import yaml +from conftest import FIXTURES, fixture, normalized_yaml + +from ossie_solid.cli import main + + +def test_import_writes_an_ossie_model_to_a_file(tmp_path): + out = tmp_path / "model.yaml" + assert main(["import", "-i", str(FIXTURES / "tpcds_solid.yaml"), "-o", str(out)]) == 0 + assert yaml.safe_load(out.read_text())["version"] == "0.2.0.dev0" + + +def test_import_writes_to_stdout_without_an_output_path(capsys): + assert main(["import", "-i", str(FIXTURES / "databricks_solid.yaml")]) == 0 + assert yaml.safe_load(capsys.readouterr().out)["semantic_model"][0]["name"] == ( + "orders_analytics" + ) + + +def test_export_writes_a_solid_model_to_a_file(tmp_path): + out = tmp_path / "solid.yaml" + assert main(["export", "-i", str(FIXTURES / "tpcds_ossie.yaml"), "-o", str(out)]) == 0 + assert yaml.safe_load(out.read_text())["semantic_model"]["name"] == ( + "tpcds_retail_model" + ) + + +def test_a_round_trip_through_the_cli_matches_the_input(tmp_path, capsys): + ossie = tmp_path / "model.yaml" + solid = tmp_path / "solid.yaml" + main(["import", "-i", str(FIXTURES / "databricks_solid.yaml"), "-o", str(ossie)]) + main(["export", "-i", str(ossie), "-o", str(solid)]) + assert normalized_yaml(solid.read_text()) == normalized_yaml( + fixture("databricks_solid.yaml") + ) + + +def test_warnings_go_to_stderr_as_plain_lines(capsys): + main(["import", "-i", str(FIXTURES / "tpcds_solid.yaml")]) + err = capsys.readouterr().err + assert err.startswith("Warning: [metric]") + assert "Traceback" not in err + + +def test_a_conversion_error_exits_non_zero_with_a_message(capsys, tmp_path): + bad = tmp_path / "bad.yaml" + bad.write_text("semantic_model: [unclosed\n") + assert main(["import", "-i", str(bad)]) == 1 + assert capsys.readouterr().err.startswith("Error: Invalid YAML") + + +def test_a_missing_input_file_exits_non_zero_with_a_message(capsys): + assert main(["import", "-i", "/nonexistent/model.yaml"]) == 1 + assert "Error:" in capsys.readouterr().err + + +def test_the_dialect_flag_is_honoured(tmp_path): + out = tmp_path / "model.yaml" + main([ + "import", + "-i", str(FIXTURES / "databricks_solid.yaml"), + "-o", str(out), + "--dialect", "ANSI_SQL", + ]) + model = yaml.safe_load(out.read_text())["semantic_model"][0] + assert model["datasets"][0]["fields"][0]["expression"]["dialects"][0]["dialect"] == ( + "ANSI_SQL" + ) + + +def test_an_unknown_dialect_is_rejected_by_the_parser(): + with pytest.raises(SystemExit): + main(["import", "-i", str(FIXTURES / "tpcds_solid.yaml"), "--dialect", "ORACLE"]) + + +def test_the_name_flag_renames_the_model(tmp_path): + out = tmp_path / "model.yaml" + main([ + "import", + "-i", str(FIXTURES / "tpcds_solid.yaml"), + "-o", str(out), + "--name", "renamed_model", + ]) + assert yaml.safe_load(out.read_text())["semantic_model"][0]["name"] == ( + "renamed_model" + ) + + +def test_a_missing_subcommand_is_rejected(): + with pytest.raises(SystemExit): + main([]) diff --git a/converters/solid/tests/test_cross_vendor.py b/converters/solid/tests/test_cross_vendor.py new file mode 100644 index 00000000..d5704b87 --- /dev/null +++ b/converters/solid/tests/test_cross_vendor.py @@ -0,0 +1,246 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Interop with Apache Ossie models this converter did not produce. + +Import (Solid -> Apache Ossie) is covered by round-trip equality, because both ends of +that trip are Solid's own format. Export (Apache Ossie -> Solid) has no such anchor: the +models it will really be handed come from *other vendors' converters*, and those look +nothing like the ones this converter emits -- no SOLID stash, no `dimension` blocks, +usually no `datatype`, metrics written against bare column names. + +Two complementary passes cover that: + +1. A **sweep** over the Ossie fixtures the other converters in this repository ship. + It asserts only the robust contract -- every model converts, and the result is a + well-formed Solid model that re-imports schema-valid. It deliberately pins no + warning counts: those fixtures belong to other converters, and each converter's CI + is path-filtered to its own directory, so a pin here would break on a main-branch + push from a PR that never ran this suite. + +2. Exact pins against `foreign_ossie.yaml`, a fixture *this* converter owns that + reproduces the same constructs. That is where the interop gaps are recorded + precisely, so a change in any of them shows up as a diff in this file. +""" + +import json + +import pytest +import yaml +from conftest import ( + CONVERTERS, + convert_quietly, + fixture, + model_of, + solid_model_of, +) + +from ossie_solid import convert_ossie_to_solid, convert_solid_to_ossie + + +def _foreign_ossie_fixtures(): + """Every Apache Ossie document among the other converters' fixtures. + + Discovered rather than listed, so a fixture that is renamed or removed upstream + drops out of the sweep instead of failing it. + """ + found = [] + for path in sorted(CONVERTERS.glob("*/tests/fixtures/*.yaml")): + if path.parts[-4] == "solid": + continue + try: + document = yaml.safe_load(path.read_text()) + except yaml.YAMLError: + continue + # An Apache Ossie document keys `semantic_model` to a list; Solid's and every + # other vendor's native format does not. + if isinstance(document, dict) and isinstance( + document.get("semantic_model"), list + ): + found.append(path) + return found + + +FOREIGN = _foreign_ossie_fixtures() + + +def _fixture_id(path): + return f"{path.parts[-4]}/{path.name}" + + +@pytest.mark.skipif(not FOREIGN, reason="no other converters ship Ossie fixtures") +@pytest.mark.parametrize("path", FOREIGN, ids=_fixture_id) +def test_another_vendors_model_converts_to_a_well_formed_solid_model(path): + """The bar is a Solid model Solid could actually read, not a warning-free one. + + Losses are expected here and are reported as warnings; what must not happen is a + hard failure or a structurally invalid Solid document. + """ + solid, _ = convert_quietly(convert_ossie_to_solid, path.read_text()) + model = solid_model_of(solid) + + assert model["name"] + assert model["tables"], "a Solid model needs at least one table" + for table in model["tables"]: + assert table["name"], "every Solid table needs its fully-qualified name" + # Solid always emits `facts`, as [] when a table has none. + assert "facts" in table + for column in (table.get("dimensions") or []) + table["facts"]: + assert column["name"] + for metric in model.get("metrics") or []: + assert metric["name"] and metric["expression"] + for relationship in model.get("relationships") or []: + assert relationship["left_table"] and relationship["right_table"] + keys = relationship["join_keys"] + assert len(keys["left"]) == len(keys["right"]) + + +@pytest.mark.skipif(not FOREIGN, reason="no other converters ship Ossie fixtures") +@pytest.mark.parametrize("path", FOREIGN, ids=_fixture_id) +def test_reimporting_another_vendors_model_is_schema_valid(path, assert_valid_ossie): + """The full hop: their Apache Ossie -> Solid -> Apache Ossie, still valid.""" + solid, _ = convert_quietly(convert_ossie_to_solid, path.read_text()) + ossie, _ = convert_quietly(convert_solid_to_ossie, solid, dialect="ANSI_SQL") + assert_valid_ossie(ossie) + + +@pytest.mark.skipif(not FOREIGN, reason="no other converters ship Ossie fixtures") +def test_the_sweep_actually_found_the_other_converters(): + """Guard against the discovery silently matching nothing and the sweep passing. + + A lower bound, not an exact count, so a converter added or removed upstream does + not fail this suite. + """ + vendors = {path.parts[-4] for path in FOREIGN} + assert len(vendors) >= 3, f"only found fixtures for {vendors}" + + +# --- exact pins, against a fixture this converter owns ----------------------------- + + +@pytest.fixture(scope="module") +def foreign(): + solid, warnings_ = convert_quietly( + convert_ossie_to_solid, fixture("foreign_ossie.yaml") + ) + return solid_model_of(solid), warnings_ + + +def test_a_foreign_model_converts_to_the_expected_solid_shape(foreign): + model, _ = foreign + assert model["name"] == "foreign_retail_model" + assert [t["name"] for t in model["tables"]] == [ + "tpcds.public.store_sales", + "tpcds.public.date_dim", + ] + # No `dimension` block anywhere, so the split falls back to Solid's type rule: + # a numeric type is a fact, everything else a dimension. With no datatype at all, + # a column cannot be shown to be numeric and so lands in `dimensions`. + store_sales = model["tables"][0] + assert [c["name"] for c in store_sales["facts"]] == ["ss_net_paid"] + assert [c["name"] for c in store_sales["dimensions"]] == [ + "ss_item_sk", + "ss_ticket_number", + "ss_sold_date_sk", + "ticket_number", + ] + assert store_sales["primary_key"] == "ss_item_sk, ss_ticket_number" + + +def test_the_documented_interop_gaps_are_exactly_these(foreign): + """The record of what is lost bringing another vendor's model into Solid. + + Each line is a known gap, not a defect to be fixed here. Widening or narrowing any + of them should be a visible diff in this test. + """ + _, warnings_ = foreign + counts = { + # No `datatype` -> no Solid `type`. The single largest gap: Solid's columns are + # typed from the warehouse catalog, and an offline converter has no catalog. + "has no datatype": 5, + # A field renamed relative to its column keeps the alias and loses the column + # it actually reads, because a Solid column IS a catalog column. + "is a computed field": 1, + # Solid has no slot for any of these. + "label 'Year' has no Solid equivalent": 1, + "unique_keys have no Solid equivalent": 1, + "datatype 'Decimal' has no Solid equivalent": 1, + "ai_context.instructions, ai_context.synonyms have no Solid equivalent": 1, + "custom_extensions for": 4, + # Solid resolves a metric's columns through `tables`, and a metric written + # against bare column names carries nothing to resolve them from. + "names no table": 2, + } + for text, expected in counts.items(): + assert sum(text in w for w in warnings_) == expected, text + assert len(warnings_) == sum(counts.values()) == 16 + + +def test_a_renamed_field_loses_the_column_it_reads(foreign): + """Pinned because it is the sharpest gap: the output names a column that is not + in the warehouse. `ticket_number` reads `ss_ticket_number`, but Solid columns map + to catalog columns, so only the alias survives.""" + model, warnings_ = foreign + renamed = {c["name"] for c in model["tables"][0]["dimensions"]} + assert "ticket_number" in renamed + assert any( + "ticket_number' is a computed field (`ss_ticket_number`)" in w + for w in warnings_ + ) + + +def test_a_bare_column_metric_reaches_solid_with_no_tables(foreign): + """Also pinned as a gap: Solid needs `tables` to resolve a formula's columns. + + Only a dataset-qualified reference identifies its owner, and another vendor has no + reason to qualify -- Apache Ossie's own TPC-DS example does, which is why this does + not show up against it. + """ + model, _ = foreign + assert {m["name"]: m["tables"] for m in model["metrics"]} == { + "revenue": [], + "order_count": [], + } + + +def test_a_foreign_model_survives_the_round_trip_back_to_ossie(assert_valid_ossie): + solid, _ = convert_quietly(convert_ossie_to_solid, fixture("foreign_ossie.yaml")) + ossie, _ = convert_quietly(convert_solid_to_ossie, solid, dialect="ANSI_SQL") + assert_valid_ossie(ossie) + + original = model_of(fixture("foreign_ossie.yaml")) + restored = model_of(ossie) + assert restored["name"] == original["name"] + assert [d["source"] for d in restored["datasets"]] == [ + d["source"] for d in original["datasets"] + ] + # The relationship survives with its direction intact: `store_sales` joins + # `date_dim` on the latter's primary key, so it is the many side. + edge = restored["relationships"][0] + assert (edge["from"], edge["to"]) == ("store_sales", "date_dim") + + +def test_the_dialect_recorded_on_a_reimported_foreign_model_is_explicit(): + """A foreign model has no stash, so the re-import records what it resolved to.""" + solid, _ = convert_quietly(convert_ossie_to_solid, fixture("foreign_ossie.yaml")) + ossie, _ = convert_quietly(convert_solid_to_ossie, solid, dialect="ANSI_SQL") + stash = next( + json.loads(ext["data"]) + for ext in model_of(ossie)["custom_extensions"] + if ext["vendor_name"] == "SOLID" + ) + assert stash["dialect"] == "ANSI_SQL" diff --git a/converters/solid/tests/test_datatypes.py b/converters/solid/tests/test_datatypes.py new file mode 100644 index 00000000..07a04ee6 --- /dev/null +++ b/converters/solid/tests/test_datatypes.py @@ -0,0 +1,212 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Warehouse type vocabulary: dialect inference and datatype mapping.""" + +import pytest +import yaml +from conftest import convert_quietly, field_of, fixture, model_of, stash_of + +from ossie_solid import ConversionError, convert_solid_to_ossie +from ossie_solid.datatypes import ( + infer_dialect, + is_solid_fact_type, + normalize_dialect, + parse_type, + to_ossie_datatype, + to_raw_type, +) + + +@pytest.mark.parametrize( + ("raw", "base", "params"), + [ + ("NUMBER(38,0)", "NUMBER", ["38", "0"]), + ("number(7, 2)", "NUMBER", ["7", "2"]), + ("TEXT", "TEXT", []), + ("VARCHAR(16777216)", "VARCHAR", ["16777216"]), + ("ARRAY", "ARRAY", []), + ("MAP", "MAP", []), + ("TIMESTAMP WITH TIME ZONE", "TIMESTAMP WITH TIME ZONE", []), + ("", None, []), + (None, None, []), + ], +) +def test_a_raw_type_splits_into_a_base_name_and_parameters(raw, base, params): + assert parse_type(raw) == (base, params) + + +@pytest.mark.parametrize( + ("types", "expected"), + [ + (["NUMBER(38,0)", "TEXT", "TIMESTAMP_NTZ"], "SNOWFLAKE"), + (["LONG", "STRING", "MAP"], "DATABRICKS"), + (["INT64", "STRING", "BOOL", "FLOAT64"], "BIGQUERY"), + # A STRUCT column decides nothing on its own, but must not stop the + # surrounding vocabulary from deciding. + (["STRUCT", "LONG", "STRING"], "DATABRICKS"), + (["STRUCT", "RECORD", "INT64"], "BIGQUERY"), + # Names every warehouse shares carry no signal. + (["STRING", "DATE", "BOOLEAN", "DECIMAL"], "ANSI_SQL"), + ([], "ANSI_SQL"), + # A tie between two vocabularies is not a decision. + (["NUMBER", "INT64"], "ANSI_SQL"), + ], +) +def test_the_warehouse_is_inferred_from_its_distinctive_type_names(types, expected): + dialect, confident = infer_dialect(types) + assert dialect == expected + assert confident is (expected != "ANSI_SQL") + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("tpcds_solid.yaml", "SNOWFLAKE"), + ("databricks_solid.yaml", "DATABRICKS"), + ("bigquery_solid.yaml", "BIGQUERY"), + ], +) +def test_each_fixture_resolves_to_its_own_warehouse(name, expected): + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture(name)) + assert stash_of(model_of(ossie))["dialect"] == expected + + +@pytest.mark.parametrize("struct", ["STRUCT", "STRUCT"]) +def test_a_struct_column_alone_does_not_identify_a_warehouse(struct): + """STRUCT is Databricks and BigQuery both, and the angle-bracket params that + would tell them apart are stripped by `parse_type` before the vote is counted -- + so it belongs to neither marker set. Regression: it was once listed as + `STRUCT<>` under Databricks, a string `parse_type` can never return, which left + BigQuery holding the only STRUCT marker and a Databricks model claiming BIGQUERY. + """ + assert infer_dialect([struct, "STRING", "DATE"]) == ("ANSI_SQL", False) + + +def test_an_unrecognizable_vocabulary_falls_back_to_ansi_with_a_warning(): + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + for table in solid["semantic_model"]["tables"]: + for column in table["dimensions"] + table["facts"]: + column["type"] = "STRING" + ossie, warnings_ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + assert any("could not infer the source warehouse" in w for w in warnings_) + assert stash_of(model_of(ossie))["dialect"] == "ANSI_SQL" + + +def test_an_explicit_dialect_suppresses_inference(): + ossie, warnings_ = convert_quietly( + convert_solid_to_ossie, fixture("databricks_solid.yaml"), dialect="ANSI_SQL" + ) + assert stash_of(model_of(ossie))["dialect"] == "ANSI_SQL" + assert not any("could not infer" in w for w in warnings_) + + +def test_an_unsupported_dialect_is_rejected(): + with pytest.raises(ConversionError, match="Unsupported dialect"): + convert_solid_to_ossie(fixture("databricks_solid.yaml"), dialect="POSTGRES") + + +def test_dialect_names_are_normalized_case_insensitively(): + assert normalize_dialect("snowflake") == "SNOWFLAKE" + + +@pytest.mark.parametrize( + ("raw", "dialect", "expected"), + [ + # A declared scale of zero is an integer, which is how Snowflake stores ids. + ("NUMBER(38,0)", "SNOWFLAKE", "Integer"), + ("NUMBER(7,2)", "SNOWFLAKE", "Decimal"), + ("NUMBER", "SNOWFLAKE", "Integer"), + ("NUMERIC(10,4)", "BIGQUERY", "Decimal"), + ("TEXT", "SNOWFLAKE", "String"), + ("STRING", "DATABRICKS", "String"), + ("LONG", "DATABRICKS", "Integer"), + ("INT64", "BIGQUERY", "Integer"), + ("FLOAT64", "BIGQUERY", "Float"), + ("DOUBLE", "DATABRICKS", "Float"), + ("BOOL", "BIGQUERY", "Boolean"), + ("BOOLEAN", "SNOWFLAKE", "Boolean"), + ("DATE", "SNOWFLAKE", "Date"), + # TIMESTAMP means different things per warehouse. + ("TIMESTAMP", "SNOWFLAKE", "DateTime"), + ("TIMESTAMP", "DATABRICKS", "DateTimeTz"), + ("TIMESTAMP", "BIGQUERY", "DateTimeTz"), + ("TIMESTAMP_NTZ", "SNOWFLAKE", "DateTime"), + ("TIMESTAMP_NTZ", "DATABRICKS", "DateTime"), + ("TIMESTAMP_TZ", "SNOWFLAKE", "DateTimeTz"), + ("DATETIME", "BIGQUERY", "DateTime"), + # Known, but outside the portable vocabulary. + ("VARIANT", "SNOWFLAKE", "Opaque"), + ("MAP", "DATABRICKS", "Opaque"), + ("ARRAY", "SNOWFLAKE", "Opaque"), + # Unknown: the spec says to omit datatype rather than guess. + ("SOMETHING_ELSE", "SNOWFLAKE", None), + ("", "SNOWFLAKE", None), + (None, "SNOWFLAKE", None), + ], +) +def test_warehouse_types_map_onto_the_portable_datatypes(raw, dialect, expected): + assert to_ossie_datatype(raw, dialect) == expected + + +def test_an_unmappable_type_omits_the_datatype_and_warns(): + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + solid["semantic_model"]["tables"][0]["dimensions"][2]["type"] = "SOMETHING_ELSE" + ossie, warnings_ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + assert any("SOMETHING_ELSE" in w for w in warnings_) + field = field_of(model_of(ossie), "orders", "status") + assert "datatype" not in field + # The raw name is still kept, so the export is unaffected. + assert stash_of(field)["type"] == "SOMETHING_ELSE" + + +@pytest.mark.parametrize( + ("datatype", "dialect", "expected"), + [ + ("String", "SNOWFLAKE", "TEXT"), + ("String", "DATABRICKS", "STRING"), + ("String", "BIGQUERY", "STRING"), + ("String", "ANSI_SQL", "VARCHAR"), + ("Integer", "BIGQUERY", "INT64"), + ("Float", "DATABRICKS", "DOUBLE"), + ("DateTimeTz", "SNOWFLAKE", "TIMESTAMP_TZ"), + (None, "SNOWFLAKE", None), + ], +) +def test_portable_datatypes_map_back_to_a_representative_warehouse_type( + datatype, dialect, expected +): + assert to_raw_type(datatype, dialect) == expected + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("NUMBER(38,0)", True), + ("DECIMAL(10,2)", True), + ("DOUBLE", True), + ("INT64", True), + ("LONG", True), + ("TEXT", False), + ("STRING", False), + ("DATE", False), + ("BOOLEAN", False), + (None, False), + ], +) +def test_solids_own_fact_type_rule_is_mirrored(raw, expected): + assert is_solid_fact_type(raw) is expected diff --git a/converters/solid/tests/test_expressions.py b/converters/solid/tests/test_expressions.py new file mode 100644 index 00000000..f05b3c51 --- /dev/null +++ b/converters/solid/tests/test_expressions.py @@ -0,0 +1,223 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Metric expression qualification. + +The property that matters most here is that the converter never rewrites SQL it was only +asked to qualify: the splice must leave every byte outside the inserted qualifier alone. +""" + +import pytest + +from ossie_solid.expressions import ( + AMBIGUOUS, + QUALIFIED, + UNCHANGED, + UNPARSED, + UNQUALIFIED, + column_reference, + qualify_metric, + referenced_datasets, + unqualify_metric, +) + +COLUMNS = ["amount", "status", "order_id", "region", "ss_sales_price", "ss_list_price"] + + +@pytest.mark.parametrize( + ("expression", "expected"), + [ + ("SUM(amount)", "SUM(orders.amount)"), + ("SUM(amount) / 12.0", "SUM(orders.amount) / 12.0"), + ("COUNT(DISTINCT order_id)", "COUNT(DISTINCT orders.order_id)"), + ("SUM(SUM(amount)) OVER ()", "SUM(SUM(orders.amount)) OVER ()"), + ( + "SUM(CASE WHEN status = 'shipped' THEN amount ELSE 0 END)", + "SUM(CASE WHEN orders.status = 'shipped' THEN orders.amount ELSE 0 END)", + ), + ( + "1 - (ss_sales_price / NULLIF(ss_list_price, 0))", + "1 - (orders.ss_sales_price / NULLIF(orders.ss_list_price, 0))", + ), + ], +) +def test_bare_columns_are_prefixed_with_the_owning_dataset(expression, expected): + assert qualify_metric(expression, "SNOWFLAKE", "orders", COLUMNS) == ( + expected, + QUALIFIED, + ) + + +def test_a_column_name_inside_a_string_literal_is_not_touched(): + result, status = qualify_metric( + "COUNT(CASE WHEN status = 'amount' THEN 1 END)", "SNOWFLAKE", "orders", COLUMNS + ) + assert result == "COUNT(CASE WHEN orders.status = 'amount' THEN 1 END)" + assert status == QUALIFIED + + +def test_an_already_qualified_column_is_left_alone(): + assert qualify_metric("SUM(o.amount)", "SNOWFLAKE", "orders", COLUMNS) == ( + "SUM(o.amount)", + UNCHANGED, + ) + + +def test_a_function_whose_name_matches_a_column_is_not_qualified(): + assert qualify_metric("status(amount)", "SNOWFLAKE", "orders", COLUMNS)[0] == ( + "status(orders.amount)" + ) + + +def test_a_column_the_dataset_does_not_own_is_left_bare(): + assert qualify_metric("SUM(other_col)", "SNOWFLAKE", "orders", COLUMNS) == ( + "SUM(other_col)", + UNCHANGED, + ) + + +def test_column_matching_is_case_insensitive(): + result, status = qualify_metric("SUM(AMOUNT)", "SNOWFLAKE", "orders", COLUMNS) + assert result == "SUM(orders.AMOUNT)" + assert status == QUALIFIED + + +def test_an_expression_with_no_columns_is_returned_verbatim(): + assert qualify_metric("COUNT(*)", "SNOWFLAKE", "orders", COLUMNS) == ( + "COUNT(*)", + UNCHANGED, + ) + + +def test_a_name_used_in_a_non_column_position_blocks_the_edit(): + # `year` is a column here *and* the datepart keyword inside EXTRACT, so the token + # scan and the parse disagree and the expression is left exactly as written. + result, status = qualify_metric( + "SUM(EXTRACT(year FROM closed_at))", "SNOWFLAKE", "orders", ["year", "closed_at"] + ) + assert result == "SUM(EXTRACT(year FROM closed_at))" + assert status == AMBIGUOUS + + +@pytest.mark.parametrize( + ("expression", "dialect"), + [ + # sqlglot canonicalizes both of these when it re-renders a parsed tree; a splice + # must not. + ("ROUND(CAST(amount AS FLOAT), 2)", "SNOWFLAKE"), + ("SUM(amount) + 1", "SNOWFLAKE"), + ("COUNTIF(status = 'active')", "BIGQUERY"), + ("SUM(amount) /* trailing comment */", "SNOWFLAKE"), + ], +) +def test_everything_outside_the_inserted_qualifier_is_preserved_byte_for_byte( + expression, dialect +): + result, status = qualify_metric(expression, dialect, "orders", COLUMNS) + assert status == QUALIFIED, "otherwise this asserts nothing" + assert result.replace("orders.", "") == expression + + +def test_an_unparseable_expression_is_returned_verbatim(): + assert qualify_metric("SUM(amount", "SNOWFLAKE", "orders", COLUMNS) == ( + "SUM(amount", + UNPARSED, + ) + + +def test_a_quoted_identifier_is_qualified_with_its_quotes_intact(): + result, status = qualify_metric( + 'SUM("odd name")', "SNOWFLAKE", "orders", ["odd name"] + ) + assert result == 'SUM(orders."odd name")' + assert status == QUALIFIED + + +# --- the inverse ------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("expression", "expected"), + [ + ("SUM(orders.amount)", "SUM(amount)"), + ( + "SUM(CASE WHEN orders.status = 'x' THEN orders.amount END)", + "SUM(CASE WHEN status = 'x' THEN amount END)", + ), + ], +) +def test_a_dataset_qualifier_is_stripped_back_out(expression, expected): + assert unqualify_metric(expression, "SNOWFLAKE", ["orders"]) == ( + expected, + UNQUALIFIED, + ) + + +def test_a_qualifier_that_is_not_a_dataset_is_preserved(): + assert unqualify_metric("SUM(p.amount)", "SNOWFLAKE", ["orders"]) == ( + "SUM(p.amount)", + UNCHANGED, + ) + + +def test_a_schema_qualified_reference_is_left_alone(): + assert unqualify_metric("SUM(db.orders.amount)", "SNOWFLAKE", ["orders"]) == ( + "SUM(db.orders.amount)", + UNCHANGED, + ) + + +@pytest.mark.parametrize( + ("expression", "dialect"), + [ + ("SUM(orders.amount) / NULLIF(COUNT(orders.order_id), 0)", "SNOWFLAKE"), + ("ROUND(CAST(orders.amount AS FLOAT), 2)", "SNOWFLAKE"), + ("COUNTIF(orders.status = 'active')", "BIGQUERY"), + ], +) +def test_qualifying_and_unqualifying_returns_the_original_text(expression, dialect): + bare, _ = unqualify_metric(expression, dialect, ["orders"]) + again, _ = qualify_metric(bare, dialect, "orders", COLUMNS) + assert again == expression + + +# --- helpers ----------------------------------------------------------------------- + + +def test_the_datasets_a_metric_references_are_reported_in_declaration_order(): + assert referenced_datasets( + "SUM(orders.amount) / COUNT(customers.id)", + "SNOWFLAKE", + ["customers", "orders", "items"], + ) == ["customers", "orders"] + + +def test_no_datasets_are_reported_for_a_bare_expression(): + assert referenced_datasets("SUM(amount)", "SNOWFLAKE", ["orders"]) == [] + + +@pytest.mark.parametrize( + ("name", "dialect", "expected"), + [ + ("amount", "SNOWFLAKE", "amount"), + ("odd name", "SNOWFLAKE", '"odd name"'), + ("odd name", "DATABRICKS", "`odd name`"), + ("select", "SNOWFLAKE", '"select"'), + ], +) +def test_a_column_name_renders_as_a_valid_reference(name, dialect, expected): + assert column_reference(name, dialect) == expected diff --git a/converters/solid/tests/test_ossie_to_solid.py b/converters/solid/tests/test_ossie_to_solid.py new file mode 100644 index 00000000..93c31620 --- /dev/null +++ b/converters/solid/tests/test_ossie_to_solid.py @@ -0,0 +1,441 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Apache Ossie -> Solid semantic model.""" + +import warnings + +import pytest +import yaml +from conftest import ( + by_name, + convert_quietly, + example, + fixture, + solid_model_of, +) + +from ossie_solid import ConversionError, convert_ossie_to_solid, convert_solid_to_ossie + + +@pytest.fixture(scope="module") +def from_fixture(): + """The TPC-DS Apache Ossie fixture exported back to Solid.""" + solid, _ = convert_quietly(convert_ossie_to_solid, fixture("tpcds_ossie.yaml")) + return solid_model_of(solid) + + +@pytest.fixture(scope="module") +def from_example(): + """The repository's hand-authored TPC-DS model exported to Solid. + + It carries no SOLID stash, so this exercises every fallback path. + """ + solid, _ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + return solid_model_of(solid) + + +def test_key_order_matches_solids_own_export_template(from_fixture): + assert list(from_fixture) == [ + "name", + "business_context", + "model_llm_description", + "tables", + "metrics", + "relationships", + "example_queries", + "benchmark_questions", + ] + + +def test_table_key_order_matches_solids_own_export_template(from_fixture): + assert list(by_name(from_fixture["tables"])["tpcds.public.store_sales"]) == [ + "name", + "description", + "manual_description", + "synonyms", + "primary_key", + "quality_rank", + "indexes", + "dimensions", + "facts", + ] + + +# --- datasets --------------------------------------------------------------------- + + +def test_the_dataset_source_becomes_the_table_name(from_fixture): + assert "tpcds.public.store_sales" in by_name(from_fixture["tables"]) + + +def test_a_composite_primary_key_is_rejoined_into_one_scalar(from_fixture): + table = by_name(from_fixture["tables"])["tpcds.public.store_sales"] + assert table["primary_key"] == "ss_item_sk, ss_ticket_number" + + +def test_quality_rank_is_always_emitted_even_when_unknown(from_example): + # The repository example has no SOLID stash, so no rank is known. + assert all(t["quality_rank"] == "" for t in from_example["tables"]) + + +def test_facts_is_always_emitted_even_when_empty(from_fixture): + table = by_name(from_fixture["tables"])["tpcds.public.date_dim"] + assert table["facts"] == [] + + +def test_unique_keys_are_dropped_with_a_warning(): + _, warnings_ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + assert any("unique_keys have no Solid equivalent" in w for w in warnings_) + + +def test_foreign_vendor_extensions_are_dropped_with_a_warning(): + _, warnings_ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + assert any("DBT, SALESFORCE" in w for w in warnings_) + + +# --- fields ----------------------------------------------------------------------- + + +def test_the_dimension_block_decides_the_fact_dimension_split(from_fixture): + table = by_name(from_fixture["tables"])["tpcds.public.store_sales"] + assert "ss_customer_sk" in by_name(table["dimensions"]) + assert "ss_net_profit" in by_name(table["facts"]) + + +def test_without_dimension_metadata_the_split_falls_back_to_solids_type_rule(): + # An Apache Ossie model whose fields carry no `dimension` block: numeric fields + # become facts and the rest dimensions, which is how Solid itself splits them. + ossie = yaml.safe_load(example("tpcds_semantic_model.yaml")) + for dataset in ossie["semantic_model"][0]["datasets"]: + for field in dataset["fields"]: + field.pop("dimension", None) + solid, _ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(ossie)) + table = by_name(solid_model_of(solid)["tables"])["tpcds.public.store_sales"] + assert "ss_quantity" in by_name(table["facts"]) + assert "ss_customer_sk" in by_name(table["facts"]) # Integer -> fact + table = by_name(solid_model_of(solid)["tables"])["tpcds.public.customer"] + assert "c_email_address" in by_name(table["dimensions"]) # String -> dimension + + +def test_the_stashed_raw_type_wins_over_the_portable_datatype(from_fixture): + table = by_name(from_fixture["tables"])["tpcds.public.store_sales"] + assert by_name(table["facts"])["ss_sales_price"]["type"] == "NUMBER(7,2)" + + +def test_without_a_stash_the_type_is_derived_from_the_datatype(from_example): + table = by_name(from_example["tables"])["tpcds.public.store_sales"] + # The example is ANSI_SQL, so the ANSI type names are used. + assert by_name(table["facts"])["ss_sales_price"]["type"] == "DECIMAL" + + +def test_a_field_with_no_datatype_gets_an_empty_type_and_a_warning(): + solid, warnings_ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + table = by_name(solid_model_of(solid)["tables"])["tpcds.public.date_dim"] + assert by_name(table["dimensions"])["d_quarter_name"]["type"] == "" + assert any("d_quarter_name" in w and "no datatype" in w for w in warnings_) + + +def test_an_expression_only_fact_round_trips_as_an_expression(from_fixture): + table = by_name(from_fixture["tables"])["tpcds.public.store_sales"] + fact = by_name(table["facts"])["ss_discount_pct"] + assert fact["expression"] == "1 - (ss_sales_price / NULLIF(ss_list_price, 0))" + assert "type" not in fact + + +def test_a_computed_dimension_drops_its_expression_with_a_warning(): + _, warnings_ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + assert any("customer_full_name" in w and "computed field" in w for w in warnings_) + + +def test_a_field_label_is_dropped_with_a_warning(): + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + ossie["semantic_model"][0]["datasets"][0]["fields"][0]["label"] = "filter" + _, warnings_ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(ossie)) + assert any("label 'filter'" in w for w in warnings_) + + +# --- metrics ---------------------------------------------------------------------- + + +def test_a_qualified_metric_is_written_back_with_bare_columns(from_fixture): + metric = by_name(from_fixture["metrics"])["TOTAL_SALES"] + assert metric["expression"] == "SUM(ss_ext_sales_price)" + assert metric["tables"] == ["tpcds.public.store_sales"] + + +def test_the_owning_tables_are_recovered_from_the_expression_without_a_stash( + from_example, +): + metric = by_name(from_example["metrics"])["customer_lifetime_value"] + assert metric["tables"] == ["tpcds.public.store_sales", "tpcds.public.customer"] + + +def test_metric_key_order_matches_solids_own_export_template(from_fixture): + assert list(by_name(from_fixture["metrics"])["TOTAL_SALES"]) == [ + "name", + "description", + "expression", + "synonyms", + "tables", + ] + + +# --- relationships ---------------------------------------------------------------- + + +def test_a_flipped_relationship_is_restored_to_solids_left_right_order(from_fixture): + relationship = from_fixture["relationships"][0] + assert relationship["left_table"] == "tpcds.public.date_dim" + assert relationship["right_table"] == "tpcds.public.store_sales" + + +def test_relationship_order_is_preserved(from_fixture): + assert [r["right_table"] for r in from_fixture["relationships"]] == [ + "tpcds.public.store_sales", + "tpcds.public.customer", + "tpcds.public.item", + "tpcds.public.store", + ] + + +def test_a_relationship_naming_an_undeclared_dataset_is_rejected(): + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + ossie["semantic_model"][0]["relationships"][0]["to"] = "nope" + with pytest.raises(ConversionError, match="not declared in 'datasets'"): + convert_ossie_to_solid(yaml.safe_dump(ossie)) + + +# --- dialect selection ------------------------------------------------------------ + + +def test_the_dialect_recorded_at_import_is_reused(from_fixture): + # NUMBER(7,2) is a Snowflake type name; reading it back proves the SNOWFLAKE + # dialect recorded in the stash was used. + table = by_name(from_fixture["tables"])["tpcds.public.store_sales"] + assert by_name(table["facts"])["ss_sales_price"]["type"] == "NUMBER(7,2)" + + +def test_an_explicit_dialect_overrides_the_stash(): + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture("databricks_solid.yaml")) + solid, _ = convert_quietly(convert_ossie_to_solid, ossie, dialect="DATABRICKS") + assert solid_model_of(solid)["name"] == "orders_analytics" + + +def test_forcing_a_dialect_the_model_does_not_carry_is_an_error(): + # The TPC-DS fixture is SNOWFLAKE-only, with no ANSI_SQL fallback to read. + with pytest.raises(ConversionError, match="no DATABRICKS or ANSI_SQL expression"): + convert_ossie_to_solid(fixture("tpcds_ossie.yaml"), dialect="DATABRICKS") + + +def test_an_unsupported_dialect_is_rejected(): + with pytest.raises(ConversionError, match="Unsupported dialect"): + convert_ossie_to_solid(fixture("tpcds_ossie.yaml"), dialect="ORACLE") + + +def test_a_missing_dialect_expression_falls_back_to_ansi_with_a_warning(): + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + field = ossie["semantic_model"][0]["datasets"][0]["fields"][0] + field["expression"]["dialects"] = [ + {"dialect": "ANSI_SQL", "expression": "ss_sold_date_sk"} + ] + _, warnings_ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(ossie)) + assert any("no SNOWFLAKE expression" in w for w in warnings_) + + +# --- input validation -------------------------------------------------------------- + + +def test_a_wrong_spec_version_is_rejected(): + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + ossie["version"] = "0.1.0" + with pytest.raises(ConversionError, match="Unsupported Apache Ossie version"): + convert_ossie_to_solid(yaml.safe_dump(ossie)) + + +def test_a_solid_document_is_rejected(): + with pytest.raises(ConversionError, match="Unsupported Apache Ossie version"): + convert_ossie_to_solid(fixture("tpcds_solid.yaml")) + + +def test_extra_semantic_models_are_dropped_with_a_warning(): + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + second = dict(ossie["semantic_model"][0]) + second["name"] = "second_model" + ossie["semantic_model"].append(second) + solid, warnings_ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(ossie)) + assert any("holds 2 semantic models" in w for w in warnings_) + assert solid_model_of(solid)["name"] == "tpcds_retail_model" + + +# --- constructs Solid's format cannot hold ----------------------------------------- + + +def test_a_metric_datatype_is_dropped_with_a_warning(): + """Solid types a metric by evaluating its formula, so a declared type has no slot.""" + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + ossie["semantic_model"][0]["metrics"][0]["datatype"] = "Decimal" + solid, warnings_ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(ossie)) + assert any("datatype 'Decimal' has no Solid equivalent" in w for w in warnings_) + assert "datatype" not in solid_model_of(solid)["metrics"][0] + + +def test_a_relationship_annotation_is_dropped_with_a_warning(): + """A Solid relationship carries only its tables and join keys.""" + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + ossie["semantic_model"][0]["relationships"][0]["ai_context"] = { + "instructions": "join only on settled sales", + "synonyms": ["sold on"], + } + _, warnings_ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(ossie)) + dropped = [w for w in warnings_ if "no Solid equivalent" in w and "relationship" in w] + assert len(dropped) == 1 + assert "ai_context.instructions" in dropped[0] + assert "ai_context.synonyms" in dropped[0] + + +def test_the_one_to_one_note_import_writes_is_not_reported_as_a_loss(): + """That note is this converter's own marker, not a user annotation. + + Import adds it where Apache Ossie's from/to direction is arbitrary; dropping it on + the way back loses nothing, so it must not be reported as a dropped annotation. + """ + solid = yaml.safe_load(fixture("tpcds_solid.yaml")) + # Make both ends of the first join unique on their join columns -> a one-to-one. + join = solid["semantic_model"]["relationships"][0] + tables = {t["name"]: t for t in solid["semantic_model"]["tables"]} + tables[join["left_table"]]["primary_key"] = ", ".join(join["join_keys"]["left"]) + tables[join["right_table"]]["primary_key"] = ", ".join(join["join_keys"]["right"]) + + ossie, _ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + annotated = [ + r for r in yaml.safe_load(ossie)["semantic_model"][0]["relationships"] + if (r.get("ai_context") or {}).get("instructions") + ] + assert annotated, "expected import to annotate the one-to-one" + + _, warnings_ = convert_quietly(convert_ossie_to_solid, ossie) + assert not [w for w in warnings_ if "ai_context.instructions" in w], warnings_ + + +# --- non-SQL dialects -------------------------------------------------------------- + + +def test_a_non_sql_dialect_never_becomes_the_resolved_dialect(): + """MDX/TABLEAU/MAQL are in Apache Ossie's enum but are not SQL. + + Resolving to one would hand a non-SQL formula to the expression rewriter, and to + Solid, as though it were SQL. The ANSI_SQL form is read instead. + """ + ossie = { + "version": "0.2.0.dev0", + "semantic_model": [{ + "name": "m", + "datasets": [{ + "name": "a", + "source": "db.s.a", + "fields": [{ + "name": "amt", + "datatype": "Decimal", + "expression": {"dialects": [ + {"dialect": "TABLEAU", "expression": "[Amount]"}, + {"dialect": "ANSI_SQL", "expression": "amt"}, + ]}, + }], + }], + }], + } + solid, warnings_ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(ossie)) + assert any("not SQL this converter can read" in w for w in warnings_) + column = solid_model_of(solid)["tables"][0]["facts"][0] + # The ANSI_SQL form was read, so the column is a plain reference to itself and no + # Tableau formula leaked into the Solid model. + assert "expression" not in column + assert column["name"] == "amt" + + +def test_a_non_sql_dialect_with_no_ansi_form_is_an_error(): + """Better a clear failure than a Tableau formula in a SQL field.""" + ossie = { + "version": "0.2.0.dev0", + "semantic_model": [{ + "name": "m", + "datasets": [{ + "name": "a", + "source": "db.s.a", + "fields": [{ + "name": "amt", + "expression": {"dialects": [ + {"dialect": "MDX", "expression": "[Measures].[Amt]"}, + ]}, + }], + }], + }], + } + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with pytest.raises(ConversionError, match="no ANSI_SQL expression"): + convert_ossie_to_solid(yaml.safe_dump(ossie)) + + +def test_a_hand_edited_stash_dialect_is_rejected(): + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + model = ossie["semantic_model"][0] + for ext in model["custom_extensions"]: + if ext["vendor_name"] == "SOLID": + ext["data"] = ext["data"].replace("SNOWFLAKE", "MAQL") + with pytest.raises(ConversionError, match="Unsupported dialect"): + convert_ossie_to_solid(yaml.safe_dump(ossie)) + + +# --- spec versions ----------------------------------------------------------------- + + +def test_the_released_0_1_1_spec_is_read_with_a_warning(): + """0.1.1 is the only released spec version, so models in the wild declare it. + + A 0.1.1 document is a 0.2.0.dev0 document minus additive fields, so it is read + rather than rejected -- but the version gap is reported, since the missing + `datatype` is what leaves a column's Solid `type` empty. + """ + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + ossie["version"] = "0.1.1" + solid, warnings_ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(ossie)) + assert any("declares Apache Ossie v0.1.1" in w for w in warnings_) + assert any("predates the `datatype` field" in w for w in warnings_) + # The model still converts in full: the stashed raw types carry the column types. + assert solid_model_of(solid)["tables"][0]["facts"][0]["type"] + + +def test_reading_an_older_spec_still_writes_the_current_one(): + solid = yaml.safe_load(fixture("tpcds_solid.yaml")) + ossie, _ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + aged = yaml.safe_load(ossie) + aged["version"] = "0.1.1" + back, _ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(aged)) + again, _ = convert_quietly(convert_solid_to_ossie, back) + assert yaml.safe_load(again)["version"] == "0.2.0.dev0" diff --git a/converters/solid/tests/test_roundtrip.py b/converters/solid/tests/test_roundtrip.py new file mode 100644 index 00000000..a15d2ad7 --- /dev/null +++ b/converters/solid/tests/test_roundtrip.py @@ -0,0 +1,215 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Round-trip fidelity in both directions. + +`Solid -> Apache Ossie -> Solid` is exact: everything Solid's format can express either +maps onto an Apache Ossie core field or is preserved in `custom_extensions[SOLID]`. + +`Apache Ossie -> Solid -> Apache Ossie` is lossy by construction, because Solid's format +has no slot for several Apache Ossie constructs (unique keys, foreign-vendor extensions, +computed dimensions, multi-dialect expressions). Those losses are asserted explicitly +here, so a regression that widens them fails the suite. +""" + +import pytest +import yaml +from conftest import ( + by_name, + convert_quietly, + example, + fixture, + model_of, + normalized_yaml, + solid_model_of, +) + +from ossie_solid import convert_ossie_to_solid, convert_solid_to_ossie + +SOLID_FIXTURES = ["tpcds_solid.yaml", "databricks_solid.yaml", "bigquery_solid.yaml"] + + + +@pytest.mark.parametrize("name", SOLID_FIXTURES) +def test_solid_to_ossie_to_solid_is_exact(name): + original = fixture(name) + ossie, _ = convert_quietly(convert_solid_to_ossie, original) + restored, _ = convert_quietly(convert_ossie_to_solid, ossie) + assert normalized_yaml(restored) == normalized_yaml(original) + + +@pytest.mark.parametrize("name", SOLID_FIXTURES) +def test_solid_round_trip_is_stable_on_a_second_pass(name): + """A second round trip changes nothing a first one did not.""" + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture(name)) + once, _ = convert_quietly(convert_ossie_to_solid, ossie) + twice_ossie, _ = convert_quietly(convert_solid_to_ossie, once) + twice, _ = convert_quietly(convert_ossie_to_solid, twice_ossie) + assert normalized_yaml(twice) == normalized_yaml(once) + assert yaml.safe_load(twice_ossie) == yaml.safe_load(ossie) + + +@pytest.mark.parametrize("name", SOLID_FIXTURES) +def test_the_intermediate_ossie_model_is_schema_valid(name, assert_valid_ossie): + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture(name)) + assert_valid_ossie(ossie) + + +def test_ossie_to_solid_to_ossie_stays_schema_valid(assert_valid_ossie): + solid, _ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + ossie, _ = convert_quietly(convert_solid_to_ossie, solid, dialect="ANSI_SQL") + assert_valid_ossie(ossie) + + +def test_ossie_to_solid_to_ossie_preserves_the_model_shape(): + original = model_of(example("tpcds_semantic_model.yaml")) + solid, _ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + restored, _ = convert_quietly(convert_solid_to_ossie, solid, dialect="ANSI_SQL") + restored = model_of(restored) + + assert restored["name"] == original["name"] + assert [d["source"] for d in restored["datasets"]] == [ + d["source"] for d in original["datasets"] + ] + assert [d["name"] for d in restored["datasets"]] == [ + d["name"] for d in original["datasets"] + ] + assert sorted(by_name(restored["metrics"])) == sorted(by_name(original["metrics"])) + for dataset in original["datasets"]: + assert by_name(restored["datasets"])[dataset["name"]].get("primary_key") == ( + dataset.get("primary_key") + ) + + +def test_ossie_to_solid_to_ossie_preserves_relationship_semantics(): + original = model_of(example("tpcds_semantic_model.yaml")) + solid, _ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + restored = model_of( + convert_quietly(convert_solid_to_ossie, solid, dialect="ANSI_SQL")[0] + ) + + def edges(model): + return { + (r["from"], r["to"], tuple(r["from_columns"]), tuple(r["to_columns"])) + for r in model["relationships"] + } + + assert edges(restored) == edges(original) + + +def test_ossie_to_solid_to_ossie_preserves_metric_expressions(): + original = model_of(example("tpcds_semantic_model.yaml")) + solid, _ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + restored = model_of( + convert_quietly(convert_solid_to_ossie, solid, dialect="ANSI_SQL")[0] + ) + + def expressions(model): + return { + m["name"]: m["expression"]["dialects"][0]["expression"] + for m in model["metrics"] + } + + before, after = expressions(original), expressions(restored) + # Single-table metrics survive verbatim. Cross-table ones lose their qualifiers, + # because Solid's format cannot record which table each column belongs to. + assert after["total_sales"] == before["total_sales"] + assert after["total_profit"] == before["total_profit"] + assert after["customer_lifetime_value"] == ( + "SUM(ss_ext_sales_price) / COUNT(DISTINCT c_customer_sk)" + ) + + +def test_the_documented_losses_of_an_ossie_export_are_exactly_these(): + """Pin the known one-way losses, so a regression that adds another one fails.""" + _, going = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + assert {w.split("]")[0].lstrip("[") for w in going} == { + "model", "dataset", "field", "metric", "relationship" + } + assert sum("custom_extensions for" in w for w in going) == 1 + assert sum("unique_keys have no Solid equivalent" in w for w in going) == 1 + assert sum("computed field" in w for w in going) == 1 + assert sum("no datatype" in w for w in going) == 2 + # Solid types a metric by evaluating its formula, and its relationships carry no + # free text, so a declared metric datatype and a relationship annotation both have + # nowhere to land. + assert sum("datatype 'Decimal' has no Solid equivalent" in w for w in going) == 5 + assert sum("ai_context.synonyms has no Solid equivalent" in w for w in going) == 4 + assert len(going) == 14 + + +def test_reimporting_that_export_only_loses_the_cross_table_qualifiers(): + solid, _ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + _, coming = convert_quietly(convert_solid_to_ossie, solid, dialect="ANSI_SQL") + assert all("spans 2 tables" in w for w in coming), coming + assert len(coming) == 2 + + +def test_a_solid_round_trip_warns_only_about_unqualifiable_metrics(): + _, going = convert_quietly(convert_solid_to_ossie, fixture("tpcds_solid.yaml")) + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture("tpcds_solid.yaml")) + _, coming = convert_quietly(convert_ossie_to_solid, ossie) + assert all("spans 2 tables" in w for w in going), going + assert coming == [], coming + + +@pytest.mark.parametrize( + ("name", "dialect"), + [ + ("tpcds_solid.yaml", "SNOWFLAKE"), + ("databricks_solid.yaml", "DATABRICKS"), + ("bigquery_solid.yaml", "BIGQUERY"), + ], +) +def test_the_dialect_survives_a_round_trip(name, dialect): + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture(name)) + model = model_of(ossie) + for field in model["datasets"][0]["fields"]: + assert field["expression"]["dialects"][0]["dialect"] == dialect + restored, _ = convert_quietly(convert_ossie_to_solid, ossie) + again, _ = convert_quietly(convert_solid_to_ossie, restored) + assert model_of(again)["datasets"][0]["fields"][0]["expression"]["dialects"][0][ + "dialect" + ] == dialect + + +def test_an_empty_description_is_normalized_away(): + """A documented, deliberate difference. + + solid-server renders `manual_description` from a whitespace-only value as an empty + block scalar, which parses back as `''`. That carries no information, so the + converter drops it rather than round-tripping the emptiness. + """ + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + solid["semantic_model"]["tables"][0]["dimensions"][0]["manual_description"] = "" + ossie, _ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + assert "ai_context" not in model_of(ossie)["datasets"][0]["fields"][0] + restored, _ = convert_quietly(convert_ossie_to_solid, ossie) + column = solid_model_of(restored)["tables"][0]["dimensions"][0] + assert "manual_description" not in column diff --git a/converters/solid/tests/test_solid_to_ossie.py b/converters/solid/tests/test_solid_to_ossie.py new file mode 100644 index 00000000..5625203f --- /dev/null +++ b/converters/solid/tests/test_solid_to_ossie.py @@ -0,0 +1,347 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Solid semantic model -> Apache Ossie.""" + +import pytest +import yaml +from conftest import ( + by_name, + convert_quietly, + dataset_of, + expression_of, + field_of, + fixture, + model_of, + stash_of, +) + +from ossie_solid import ConversionError, convert_solid_to_ossie + + +@pytest.fixture(scope="module") +def tpcds(): + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture("tpcds_solid.yaml")) + return model_of(ossie) + + +def test_output_matches_the_committed_fixture(): + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture("tpcds_solid.yaml")) + assert yaml.safe_load(ossie) == yaml.safe_load(fixture("tpcds_ossie.yaml")) + + +def test_output_validates_against_the_ossie_schema(assert_valid_ossie): + for name in ("tpcds_solid.yaml", "databricks_solid.yaml", "bigquery_solid.yaml"): + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture(name)) + assert_valid_ossie(ossie) + + +def test_document_declares_the_spec_version(): + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture("tpcds_solid.yaml")) + document = yaml.safe_load(ossie) + assert document["version"] == "0.2.0.dev0" + assert len(document["semantic_model"]) == 1 + + +# --- datasets --------------------------------------------------------------------- + + +def test_table_fqn_becomes_the_source_and_its_last_part_the_name(tpcds): + dataset = dataset_of(tpcds, "store_sales") + assert dataset["source"] == "tpcds.public.store_sales" + + +def test_dataset_names_are_disambiguated_when_the_last_part_collides(): + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + # Two tables named `orders` in different schemas: the first keeps the short name, + # the second widens to schema_table rather than colliding. + solid["semantic_model"]["tables"][1]["name"] = "main.returns.orders" + solid["semantic_model"]["relationships"][0]["right_table"] = "main.returns.orders" + solid["semantic_model"]["tables"][1]["primary_key"] = "customer_id" + ossie, _ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + names = [d["name"] for d in model_of(ossie)["datasets"]] + assert names == ["orders", "returns_orders"] + + +def test_composite_primary_key_scalar_is_split_into_columns(tpcds): + assert dataset_of(tpcds, "store_sales")["primary_key"] == [ + "ss_item_sk", + "ss_ticket_number", + ] + + +def test_single_primary_key_becomes_a_one_element_list(tpcds): + assert dataset_of(tpcds, "customer")["primary_key"] == ["c_customer_sk"] + + +def test_table_descriptions_split_across_description_and_ai_context(tpcds): + dataset = dataset_of(tpcds, "store_sales") + assert dataset["description"].startswith("Fact table containing all store sales") + assert dataset["ai_context"]["instructions"].startswith("Grain is (ss_item_sk") + + +def test_table_synonyms_become_ai_context_synonyms(tpcds): + assert "POS data" in dataset_of(tpcds, "store_sales")["ai_context"]["synonyms"] + + +def test_quality_rank_and_indexes_are_stashed(tpcds): + stash = stash_of(dataset_of(tpcds, "store_sales")) + assert stash["quality_rank"] == "high" + assert stash["indexes"] == ["ss_sold_date_sk", "ss_customer_sk"] + + +def test_a_table_without_indexes_gets_no_index_stash(tpcds): + assert "indexes" not in stash_of(dataset_of(tpcds, "customer")) + + +# --- fields ----------------------------------------------------------------------- + + +def test_dimensions_and_facts_merge_into_one_field_list(tpcds): + fields = by_name(dataset_of(tpcds, "store_sales")["fields"]) + assert "ss_customer_sk" in fields # a dimension + assert "ss_net_profit" in fields # a fact + + +def test_a_dimension_carries_the_dimension_block_and_a_fact_does_not(tpcds): + assert field_of(tpcds, "store_sales", "ss_customer_sk")["dimension"] == { + "is_time": False + } + assert "dimension" not in field_of(tpcds, "store_sales", "ss_net_profit") + + +def test_a_temporal_dimension_is_marked_is_time(tpcds): + assert field_of(tpcds, "store_sales", "ss_sold_at")["dimension"] == {"is_time": True} + assert field_of(tpcds, "date_dim", "d_date")["dimension"] == {"is_time": True} + + +def test_a_column_expression_is_its_own_name(tpcds): + assert expression_of(field_of(tpcds, "customer", "c_email_address")) == ( + "c_email_address" + ) + + +def test_the_raw_warehouse_type_is_stashed_alongside_the_portable_datatype(tpcds): + field = field_of(tpcds, "store_sales", "ss_sales_price") + assert field["datatype"] == "Decimal" + assert stash_of(field)["type"] == "NUMBER(7,2)" + + +def test_sample_values_are_stashed(tpcds): + assert stash_of(field_of(tpcds, "item", "i_category"))["sample_values"] == [ + "Electronics", + "Home", + "Sports", + ] + + +def test_an_expression_only_fact_keeps_its_expression_and_is_marked_a_metric(tpcds): + field = field_of(tpcds, "store_sales", "ss_discount_pct") + assert expression_of(field) == "1 - (ss_sales_price / NULLIF(ss_list_price, 0))" + assert stash_of(field) == {"role": "metric"} + assert "datatype" not in field + + +def test_a_column_description_pair_maps_to_description_and_ai_context(tpcds): + field = field_of(tpcds, "store_sales", "ss_sales_price") + assert field["description"] == "Sales price per unit." + assert field["ai_context"]["instructions"].startswith("Per-unit price.") + + +# --- relationships ---------------------------------------------------------------- + + +def test_direction_follows_the_primary_key_not_the_left_right_order(tpcds): + # date_dim is the left table in the fixture, but its primary key covers the join, + # so it is the one side and store_sales must be the many side. + relationship = by_name(tpcds["relationships"])["store_sales_to_date_dim"] + assert relationship["from"] == "store_sales" + assert relationship["to"] == "date_dim" + assert relationship["from_columns"] == ["ss_sold_date_sk"] + assert relationship["to_columns"] == ["d_date_sk"] + assert stash_of(relationship)["flipped"] is True + + +def test_a_relationship_already_in_many_to_one_order_is_not_flipped(tpcds): + relationship = by_name(tpcds["relationships"])["store_sales_to_customer"] + assert relationship["from"] == "store_sales" + assert "flipped" not in stash_of(relationship) + + +def test_relationships_keep_their_declaration_order(tpcds): + assert [r["name"] for r in tpcds["relationships"]] == [ + "store_sales_to_date_dim", + "store_sales_to_customer", + "store_sales_to_item", + "store_sales_to_store", + ] + + +def test_only_a_flipped_relationship_carries_a_stash(tpcds): + stashes = [stash_of(r) for r in tpcds["relationships"]] + assert stashes == [{"flipped": True}, {}, {}, {}] + + +def test_undetermined_cardinality_warns_and_keeps_solid_order(): + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + del solid["semantic_model"]["tables"][1]["primary_key"] + ossie, warnings_ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + assert any("could not be determined" in w for w in warnings_) + relationship = model_of(ossie)["relationships"][0] + assert (relationship["from"], relationship["to"]) == ("orders", "customers") + + +def test_a_one_to_one_relationship_is_annotated(): + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + # Make the join unique on both ends. + solid["semantic_model"]["tables"][0]["primary_key"] = "customer_id" + ossie, _ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + relationship = model_of(ossie)["relationships"][0] + assert "One-to-one" in relationship["ai_context"]["instructions"] + + +def test_a_relationship_naming_an_unknown_table_is_dropped_with_a_warning(): + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + solid["semantic_model"]["relationships"][0]["right_table"] = "main.sales.missing" + ossie, warnings_ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + assert any("main.sales.missing" in w for w in warnings_) + assert "relationships" not in model_of(ossie) + + +def test_mismatched_join_key_counts_are_rejected(): + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + solid["semantic_model"]["relationships"][0]["join_keys"]["right"] = [ + "customer_id", + "region", + ] + with pytest.raises(ConversionError, match="correspond positionally"): + convert_solid_to_ossie(yaml.safe_dump(solid)) + + +# --- metrics ---------------------------------------------------------------------- + + +def test_a_single_table_metric_is_qualified_with_its_dataset(tpcds): + metrics = by_name(tpcds["metrics"]) + assert expression_of(metrics["TOTAL_SALES"]) == ( + "SUM(store_sales.ss_ext_sales_price)" + ) + + +def test_a_qualified_metric_needs_no_table_stash(tpcds): + assert stash_of(by_name(tpcds["metrics"])["TOTAL_SALES"]) == {} + + +def test_a_metric_referencing_no_column_keeps_its_tables_in_the_stash(tpcds): + metric = by_name(tpcds["metrics"])["ROW_COUNT"] + assert expression_of(metric) == "COUNT(*)" + assert stash_of(metric)["tables"] == ["tpcds.public.store_sales"] + + +def test_a_cross_table_metric_is_left_unqualified_and_warns(): + _, warnings_ = convert_quietly(convert_solid_to_ossie, fixture("tpcds_solid.yaml")) + assert any( + "CUSTOMER_LIFETIME_VALUE" in w and "spans 2 tables" in w for w in warnings_ + ) + + +def test_a_cross_table_metric_keeps_its_tables_in_the_stash(tpcds): + metric = by_name(tpcds["metrics"])["CUSTOMER_LIFETIME_VALUE"] + assert expression_of(metric) == ( + "SUM(ss_ext_sales_price) / COUNT(DISTINCT c_customer_sk)" + ) + assert stash_of(metric)["tables"] == [ + "tpcds.public.store_sales", + "tpcds.public.customer", + ] + + +def test_metric_synonyms_become_ai_context_synonyms(tpcds): + assert by_name(tpcds["metrics"])["TOTAL_SALES"]["ai_context"]["synonyms"] == [ + "total revenue", + "gross sales", + "sales amount", + ] + + +def test_a_metric_without_an_expression_is_dropped_with_a_warning(): + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + del solid["semantic_model"]["metrics"][0]["expression"] + ossie, warnings_ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + assert any("LATE_ORDER_COUNT" in w and "no expression" in w for w in warnings_) + assert [m["name"] for m in model_of(ossie)["metrics"]] == ["GROSS_REVENUE"] + + +# --- model level ------------------------------------------------------------------ + + +def test_asset_link_markup_is_resolved_in_ai_context_and_kept_raw_in_the_stash(tpcds): + instructions = tpcds["ai_context"]["instructions"] + assert "public.store_sales" in instructions + assert "assetlink" not in instructions + assert "@