diff --git a/.github/workflows/converter-solid-ci.yml b/.github/workflows/converter-solid-ci.yml new file mode 100644 index 00000000..674f737c --- /dev/null +++ b/.github/workflows/converter-solid-ci.yml @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Converters Solid CI + +on: + push: + branches: [ "main" ] + paths: + - 'converters/solid/**' + - '.github/workflows/converter-solid-ci.yml' + pull_request: + branches: [ "main" ] + paths: + - 'converters/solid/**' + - '.github/workflows/converter-solid-ci.yml' + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" + + - name: Sync dependencies + working-directory: converters/solid + run: | + uv sync + + - name: Unit Tests + working-directory: converters/solid + run: | + uv run pytest diff --git a/converters/README.md b/converters/README.md index 417d7663..080c8dba 100644 --- a/converters/README.md +++ b/converters/README.md @@ -76,6 +76,7 @@ The Ossie specification currently defines extensions for the following vendors: | `OMNI` | Omni semantic model | | `WISDOM` | WisdomAI domain | | `NVIDIA_GSF` | NVIDIA Generative Semantic Fabric standalone YAML | +| `SOLID` | Solid semantic model | Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. diff --git a/converters/solid/README.md b/converters/solid/README.md new file mode 100644 index 00000000..b9b3956c --- /dev/null +++ b/converters/solid/README.md @@ -0,0 +1,357 @@ + + +# Apache Ossie Solid Converter + +Bidirectional, offline conversion between an [Apache Ossie](https://github.com/apache/ossie) +semantic model and a [Solid](https://www.getsolid.ai/) semantic model YAML export. No +Solid connection required. + +- **Import** (`ossie-solid import`): Solid → Apache Ossie. Solid features Apache Ossie + has no native field for are preserved in `custom_extensions[SOLID]`, so + `Solid → Apache Ossie → Solid` is lossless. +- **Export** (`ossie-solid export`): Apache Ossie → Solid, in the key order + Solid's own exporter uses. + +Solid is an AI analytics platform whose semantic layer is authored against a warehouse +catalog and consumed by its text-to-SQL agents. Its export is the YAML you get from +**Download Solid Semantic Model YAML**, and the same document Solid stores as a semantic +layer version. + +On **export** (Apache Ossie → Solid), Apache Ossie constructs with no Solid slot — +`unique_keys`, field `label`, computed dimensions, metric `datatype`, relationship +`ai_context`, foreign-vendor `custom_extensions`, `ai_context.examples` below the model +level — are **dropped with a warning**. On +**import** (Solid → Apache Ossie), Solid-only features are instead **preserved** in +`custom_extensions[SOLID]`. Any input that breaks a [requirement](#requirements) +**raises a `ConversionError`** — the converter never silently drops a field or produces +an invalid result. + +## Installation + +```bash +pip install apache-ossie-solid # once published to PyPI +``` + +Or, from a checkout of this directory: + +```bash +pip install -e . +``` + +Runtime dependencies are `PyYAML` and `sqlglot`. Python 3.11+. + +## Usage + +### Command line + +```bash +ossie-solid import -i solid_model.yaml -o model.yaml [--dialect SNOWFLAKE] +``` + +```bash +ossie-solid export -i model.yaml -o solid_model.yaml [--dialect SNOWFLAKE] +``` + +With no `-o`, output goes to stdout; warnings always go to stderr. `--name` overrides the +model name in either direction. `--dialect` is described under +[Dialect resolution](#dialect-resolution). + +### Python API + +```python +from ossie_solid import convert_solid_to_ossie, convert_ossie_to_solid + +ossie_yaml = convert_solid_to_ossie(solid_yaml_str) # or dialect="SNOWFLAKE" +solid_yaml = convert_ossie_to_solid(ossie_yaml_str, model_name="sales") +``` + +## Dialect resolution + +**Solid's YAML does not record which warehouse a model came from.** The source system is +held on the asset row in Solid's own database and is dropped when the YAML is rendered, +but Apache Ossie requires a dialect on every expression. The converter resolves one in +this order: + +1. **`--dialect`**, when given. Always wins. +2. **Inference from the column type vocabulary.** Solid copies each column's `type` out + of the catalog verbatim, so the names identify the warehouse: `NUMBER`/`TEXT`/ + `VARIANT` mean Snowflake, `LONG`/`MAP`/`BIGINT` mean Databricks, `INT64`/`FLOAT64`/ + `BOOL` mean BigQuery. Only names unique to one warehouse vote; shared ones + (`STRING`, `BOOLEAN`, `DATE`, `TIMESTAMP_NTZ`) are ignored. +3. **`ANSI_SQL`**, with a warning, when nothing votes or two warehouses tie. + +The resolved dialect is recorded in `custom_extensions[SOLID]`, so an export reads +expressions back in the dialect they were written in without being told again. + +On export, `--dialect` selects which expression dialect to *read*; without it the +converter uses the dialect recorded at import, then the single non-`ANSI_SQL` dialect the +model's expressions use. A field with neither the chosen dialect nor `ANSI_SQL` raises. + +**Only SQL dialects are read.** Apache Ossie's dialect enum also covers expression +languages that are not SQL — `MDX`, `TABLEAU`, `MAQL` — which this converter cannot +parse, qualify or unqualify. One of those never becomes the resolved dialect: the model +is reported, and each expression's `ANSI_SQL` form is read instead. An expression +offering only a non-SQL dialect raises rather than passing a formula Solid cannot +execute into a SQL field. (This is what lets GoodData's `MAQL`-and-`ANSI_SQL` models +convert on their SQL side.) + +## Mapping + +Each row maps in both directions; the **Notes** flag where a behavior is specific to +**import** (Solid → Apache Ossie) or **export** (Apache Ossie → Solid). + +### Model + +| Apache Ossie | Solid | Notes | +|---|---|---| +| `semantic_model[0].name` | `semantic_model.name` | Solid holds one model per file; export warns if the document has more. | +| `description` | `model_llm_description` | | +| `ai_context.instructions` | `business_context.custom_instructions` | Import resolves Solid's `@` markup to the display names it carries and keeps the tagged original in the stash, so an export restores the live catalog references. | +| `ai_context.examples` | `business_context.business_questions` | | +| `custom_extensions[SOLID]` | `business_context.model_description`, `example_queries`, `benchmark_questions` | No Apache Ossie core equivalent. | + +### Datasets + +| Apache Ossie | Solid | Notes | +|---|---|---| +| `dataset.source` | `tables[].name` | Solid names a table by its `catalog.schema.table`. | +| `dataset.name` | — | Derived on import from the last part of the FQN, widening to `schema_table` and then the whole FQN if a shorter form is taken. Apache Ossie dataset names double as expression qualifiers, so a dotted name is not usable. | +| `description` | `description` | Solid's AI-written description. | +| `ai_context.instructions` | `manual_description` | Solid's human-written annotation. | +| `ai_context.synonyms` | `synonyms` | | +| `primary_key` (array) | `primary_key` (scalar) | Solid joins a composite key with `", "` into one scalar; import splits it, export rejoins it. | +| `custom_extensions[SOLID]` | `quality_rank`, `indexes` | No Apache Ossie core equivalent. Export emits `quality_rank` even when unknown, as Solid does. | + +### Fields + +| Apache Ossie | Solid | Notes | +|---|---|---| +| `fields[]` | `dimensions[]` + `facts[]` | Solid splits columns two ways; Apache Ossie has one list. Import records the split as the presence of the `dimension` block, and export reads it back. An Apache Ossie model that carries no `dimension` block anywhere is split the way Solid itself does it — by data type. | +| `dimension.is_time` | — | Set explicitly on import from the resolved `datatype`, rather than left to the spec's default, so a consumer that does not implement the default still reads the same role. | +| `expression.dialects[]` | the column's own name | A Solid column's expression is a bare reference to itself; a name needing quoting is quoted for the resolved dialect. | +| `datatype` | `type` | See [Data types](#data-types). | +| `description` | `description` | | +| `ai_context.instructions` | `manual_description` | | +| `ai_context.synonyms` | `synonyms` | | +| `custom_extensions[SOLID]` | `type` (raw), `sample_values` | The raw warehouse type is always kept, since the portable `datatype` is lossy. | + +A Solid **fact may be a table-scoped metric** rather than a catalog column — it carries an +`expression` and no `type`. Import keeps the expression and marks the field +`role: metric` in the stash; export re-emits it as an expression-only fact. + +### Relationships + +| Apache Ossie | Solid | Notes | +|---|---|---| +| `from` / `to` | `left_table` / `right_table` | See [Cardinality](#cardinality). | +| `from_columns` / `to_columns` | `join_keys.left` / `join_keys.right` | Positional; import rejects a length mismatch. | +| `name` | — | Solid relationships are unnamed. Import generates `{from}_to_{to}`, suffixed if that collides. | +| `ai_context` | — | A Solid relationship is a table pair and its join keys, with no free text. Export warns — except for the one-to-one note import itself writes, which is this converter's own marker and means nothing to Solid. | +| `custom_extensions[SOLID]` | — | Records the original left/right orientation and position, so an export reproduces Solid's ordering. | + +### Metrics + +| Apache Ossie | Solid | Notes | +|---|---|---| +| `metrics[]` | `metrics[]` | Solid metrics are already model-level. | +| `expression.dialects[]` | `expression` + `tables[]` | See [Metric expressions](#metric-expressions). | +| `description` | `description` | | +| `datatype` | — | Solid types a metric by evaluating its formula against the warehouse, so there is no slot for a declared result type. Export warns. | +| `ai_context.synonyms` | `synonyms` | | +| `custom_extensions[SOLID]` | `tables[]` | Stashed only when the qualified expression does not already name the same datasets — for a cross-table metric, or one referencing no column at all (`COUNT(*)`). | + +## Cardinality + +Apache Ossie encodes cardinality through direction — `from` is the many side, `to` the one +side — but a Solid relationship is an undirected pair of column lists. Import recovers the +direction from the primary keys: + +| Condition | Result | +|---|---| +| The right table's primary key is exactly its join columns | `from` = left, `to` = right | +| The left table's primary key is exactly its join columns | `from` = right, `to` = left (recorded as `flipped`) | +| Both | One-to-one: direction is arbitrary, and a note is added to the relationship's `ai_context` | +| Neither | `from` = left, with a warning — Solid records no cardinality and its `unique_keys` are not exported | + +## Metric expressions + +Solid stores a metric formula against **bare** column names and records the owning table +separately; `is_valid_metric_formula` in solid-server actively strips alias prefixes +before saving. Apache Ossie expects a metric to be self-contained, qualifying each column +with its dataset. So: + +- **Import** prefixes each bare column with its dataset when `tables[]` names exactly one + table (`SUM(ss_ext_sales_price)` → `SUM(store_sales.ss_ext_sales_price)`). +- **Export** strips those qualifiers back out. + +Both edits are a **surgical splice**, not a re-render. Each column is located by +sqlglot's tokenizer, which reports byte offsets and keeps string literals and quoted +identifiers in token types of their own, and the qualifier is inserted or removed at those +offsets. Everything else is left byte-for-byte intact — round-tripping a parsed tree +through `Expression.sql()` would canonicalize as it generates, turning +`CAST(x AS FLOAT)` into `CAST(x AS DOUBLE)` and `EXTRACT(year FROM d)` into +`DATE_PART(YEAR, d)`, and a converter has no business rewriting SQL it was only asked to +qualify. + +The parser is still used as a cross-check: it decides which names are genuinely bare +column references, and a token scan that disagrees with it means some occurrence is +something else. On any disagreement — an unparseable expression, or a column also named +`year` in `EXTRACT(year FROM …)` — the expression is left **exactly as written** and a +warning is emitted. An unqualified metric is a far smaller problem than a corrupted one. + +**A cross-table metric cannot be qualified.** Solid's `tables[]` may hold more than one +table, and the alias-to-table binding lives in `metric.column_ids`, which its YAML export +does not carry. Those metrics keep their bare column names, keep `tables[]` in the stash, +and emit a warning. + +## Data types + +Solid stores the **raw** warehouse type string verbatim (`NUMBER(38,0)`, `TEXT`, `INT64`, +`MAP`), so mapping onto Apache Ossie's portable `datatype` is lossy — +`NUMBER(38,2)`, `NUMERIC` and `DECIMAL` all collapse to `Decimal`. The raw string is +therefore always kept in the stash and is what an export re-emits; the derived table below +is only used for an Apache Ossie model that has no stash to read. + +| Warehouse type | Apache Ossie `datatype` | +|---|---| +| `STRING`, `TEXT`, `VARCHAR`, `CHAR` | `String` | +| `INT`, `BIGINT`, `LONG`, `INT64`, `TINYINT` | `Integer` | +| `NUMBER(p,0)`, `NUMERIC(p,0)`, `DECIMAL(p,0)` | `Integer` | +| `NUMBER(p,s>0)`, `NUMERIC`, `DECIMAL`, `BIGNUMERIC` | `Decimal` | +| `FLOAT`, `FLOAT64`, `DOUBLE`, `REAL` | `Float` | +| `BOOLEAN`, `BOOL` | `Boolean` | +| `DATE` / `TIME` | `Date` / `Time` | +| `TIMESTAMP_NTZ`, `DATETIME` | `DateTime` | +| `TIMESTAMP_TZ`, `TIMESTAMP_LTZ` | `DateTimeTz` | +| `TIMESTAMP` | `DateTime` on Snowflake, `DateTimeTz` on Databricks and BigQuery | +| `VARIANT`, `OBJECT`, `ARRAY`, `MAP`, `STRUCT`, `BINARY`, `GEOGRAPHY` | `Opaque` | +| anything else | omitted, with a warning | + +A declared scale of `0` is read as an integer, which is what keeps Snowflake ids and +counts — stored as `NUMBER(38,0)` — from being typed `Decimal`. + +## Requirements + +Conversion raises a `ConversionError` (rather than guessing or emitting something invalid) +when an input breaks one of these: + +- the Apache Ossie `version` is neither `0.2.0.dev0` nor `0.1.1` (export) — see + [Spec versions](#spec-versions); +- the input YAML is malformed, or its root is not a mapping; +- a Solid model has no `tables`, or an Apache Ossie model has no `datasets` — the spec + requires at least one dataset; +- a relationship's `from_columns`/`to_columns` differ in length, or name a dataset that + is not declared; +- `--dialect` names a dialect outside `ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`, `BIGQUERY`; +- a field or metric has neither the selected dialect nor `ANSI_SQL` (export); +- the recorded `custom_extensions[SOLID]` dialect is not one of the four above — which + only a hand-edited stash can produce (export). + +## Spec versions + +Output is always written as **`0.2.0.dev0`**, the current draft. On export, a document +declaring either `0.2.0.dev0` or **`0.1.1`** is read; anything else raises. + +`0.1.1` is accepted because it is the only *released* spec version, so it is what models +in the wild — and several of this repository's own converter fixtures — declare. As a +document, a `0.1.1` model is a `0.2.0.dev0` model minus three purely additive changes: +`datatype` on `Field` and `Metric`, `BIGQUERY` in the dialect enum, and a free-form +rather than enumerated `vendor_name`. Nothing in a `0.1.1` document is therefore invalid +under `0.2.0.dev0`, and no separate read path is needed. Only the absence of `datatype` +is visible to this converter, and a `0.2.0.dev0` model that simply omits it — as most do +— is already handled the same way. Reading a `0.1.1` document warns, naming that gap. + +## Known differences + +- **An empty description is normalized away.** solid-server renders a whitespace-only + `manual_description` as an empty block scalar, which parses back as `''`. That carries + no information, so the converter drops it rather than round-tripping the emptiness. +- **`unique_keys` cannot survive an export.** Solid's YAML has no unique-key field; a + single unique key identical to the primary key is dropped silently, any other is + dropped with a warning. +- **A Solid model carries no `unique_keys` to import**, so a re-imported model relies on + `primary_key` alone to determine relationship cardinality. + +## Development + +```bash +uv sync && uv run pytest +``` + +The suite covers both directions against three Solid fixtures — the repository's +[TPC-DS model](../../examples/tpcds_semantic_model.yaml) expressed as a Solid export, plus +a Databricks and a BigQuery model — and validates every converted document against the +[Apache Ossie JSON Schema](../../core-spec/ossie-schema.json). + +`tests/test_cross_vendor.py` covers the case the round-trip tests cannot: an Apache Ossie +model produced by *another vendor's* converter, which carries no `custom_extensions[SOLID]` +stash, no `dimension` blocks, usually no `datatype`, and metrics written against bare +column names. It sweeps every Ossie fixture the other converters in this repository ship, +asserting that each converts to a well-formed Solid model that re-imports schema-valid. +That sweep deliberately pins **no** warning counts, because those fixtures belong to other +converters and each converter's CI is path-filtered to its own directory — a pin here would +break on a main-branch push from a PR that never ran this suite. The exact interop gaps are +pinned instead against +[`tests/fixtures/foreign_ossie.yaml`](tests/fixtures/foreign_ossie.yaml), a fixture this +converter owns that reproduces the same constructs. + +To regenerate the expected-output fixture after an intentional change: + +```bash +uv run ossie-solid import -i tests/fixtures/tpcds_solid.yaml -o tests/fixtures/tpcds_ossie.yaml +``` + +then re-add the license header at the top of the file. + +## Future effort + +Both the Apache Ossie specification and Solid's semantic model are still evolving, and +this converter will be updated to track them. Several gaps are worth naming. + +Three of these are **open interop decisions**, pinned as such in +`tests/test_cross_vendor.py` rather than resolved. Each is a judgement about what Solid +should do with a foreign model, not a defect in the transform: + +- **A column with no `datatype` gets an empty Solid `type`.** `datatype` is optional in + the spec and most converters omit it, so this is the largest single gap when importing + a foreign model — see the counts in `test_the_documented_interop_gaps_are_exactly_these`. + It cannot be closed offline: Solid types its columns from the warehouse catalog, and + the honest fix is to reconcile against the live catalog at import time rather than to + guess here. What the converter owes that flow is a machine-readable list of what needs + reconciling, which today it only reports as warnings. +- **A field renamed relative to its column loses the column it reads.** Given + `name: ticket_number` over `expression: ss_ticket_number`, the alias survives and the + real column does not, so the Solid model names a column the warehouse does not have. + Emitting the underlying name and keeping the alias as a synonym would be truer, but it + changes which identifier downstream Solid consumers see and so needs a product call. +- **A metric written against bare column names reaches Solid with `tables: []`.** Only a + dataset-qualified reference identifies its owner. Resolving bare names against each + dataset's declared fields would fix most cases; it is inference, so it is deliberately + not done silently. + +And two are **format-level**: + +- **Verified queries have no home in the core spec.** Solid's `example_queries` and + `benchmark_questions` are currently stashed, but the same construct appears across the + ecosystem — Snowflake Cortex `verified_queries`, WisdomAI reviewed queries — and looks + like a candidate for a core field rather than N vendor extensions. +- **Solid's export does not record its source warehouse**, which is why the dialect has to + be inferred. Adding it to the export would make the format self-describing; the + converter would read it first and keep inference only as a fallback for existing files. diff --git a/converters/solid/pyproject.toml b/converters/solid/pyproject.toml new file mode 100644 index 00000000..f9622a33 --- /dev/null +++ b/converters/solid/pyproject.toml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "pytest>=8.0", + "jsonschema>=4.26.0", +] + +[project] +name = "apache-ossie-solid" +version = "0.2.0.dev0" +description = "Solid semantic model <> Apache Ossie converter" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] +requires-python = ">=3.11" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Solid" +] +dependencies = [ + "PyYAML>=6.0", + "sqlglot>=25.0", +] + +[project.scripts] +ossie-solid = "ossie_solid.cli:main" + +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_solid"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev" +] diff --git a/converters/solid/src/ossie_solid/__init__.py b/converters/solid/src/ossie_solid/__init__.py new file mode 100644 index 00000000..ef699631 --- /dev/null +++ b/converters/solid/src/ossie_solid/__init__.py @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Bidirectional converter between Apache Ossie and Solid semantic models.""" + +from ._common import ( + OSSIE_VERSION, + SUPPORTED_DIALECTS, + VENDOR, + ConversionError, +) +from .ossie_to_solid import convert_ossie_to_solid +from .solid_to_ossie import convert_solid_to_ossie + +__all__ = [ + "OSSIE_VERSION", + "SUPPORTED_DIALECTS", + "VENDOR", + "ConversionError", + "convert_ossie_to_solid", + "convert_solid_to_ossie", +] diff --git a/converters/solid/src/ossie_solid/_common.py b/converters/solid/src/ossie_solid/_common.py new file mode 100644 index 00000000..7cd454c1 --- /dev/null +++ b/converters/solid/src/ossie_solid/_common.py @@ -0,0 +1,391 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared helpers for the Apache Ossie <-> Solid semantic model converters. + +Both directions are pure offline YAML transforms. The cross-cutting concerns live +here: version constants, the `custom_extensions` stash protocol, YAML I/O, the +Solid asset-link markup, and small identifier helpers. +""" + +import json +import re +import warnings + +import yaml + +# Apache Ossie semantic model spec version this converter writes (see core-spec). +# +# NOTE: this spoke has no `apache-ossie` runtime dependency, so nothing updates the +# constant automatically -- it MUST be bumped in lockstep with the `version` in +# `core-spec/` whenever the spec version moves. +OSSIE_VERSION = "0.2.0.dev0" + +# Spec versions an Apache Ossie document may declare and still be read (see +# convert_ossie_to_solid). Output is always written as OSSIE_VERSION. +# +# 0.1.1 is the only *released* spec version, so it is what models in the wild and +# several of the repository's own converter fixtures declare. As a document, a 0.1.1 +# model is a 0.2.0.dev0 model minus three additions: `datatype` on Field and Metric, +# `BIGQUERY` in the dialect enum, and a free-form (rather than enumerated) +# `vendor_name`. Every one of those is additive, so nothing in a 0.1.1 document is +# invalid under 0.2.0.dev0 and no separate read path is needed -- only the absence of +# `datatype` is visible to this converter, and that is already handled the same way a +# 0.2.0.dev0 model that simply omits it is handled. +READABLE_OSSIE_VERSIONS = ("0.2.0.dev0", "0.1.1") + +# Read-only spec versions whose omissions are worth naming when one is encountered. +OSSIE_VERSION_NOTES = { + "0.1.1": "it predates the `datatype` field, so a column with no stashed raw type " + "gets an empty Solid 'type'", +} + +# Vendor id used for the `custom_extensions` stash. +VENDOR = "SOLID" + +# Note import writes to a one-to-one relationship's `ai_context.instructions`, where +# Apache Ossie's directional from/to carries no meaning (see solid_to_ossie._orient). +# Export recognizes it as this converter's own marker rather than a user annotation, so +# it is dropped silently on the way back while real annotations are reported. +ONE_TO_ONE_NOTE = ("One-to-one relationship: both sides are unique on the join " + "columns, so the from/to direction is arbitrary.") + +# Bump when the shape of a stashed `data` blob changes. +STASH_VERSION = 1 + +# Apache Ossie dialects this converter can resolve a Solid model to. Solid itself +# records the warehouse on the asset row, not in the exported YAML, so the dialect is +# either supplied explicitly or inferred from the column type vocabulary +# (see datatypes.infer_dialect). +DIALECT_ANSI = "ANSI_SQL" +DIALECT_SNOWFLAKE = "SNOWFLAKE" +DIALECT_DATABRICKS = "DATABRICKS" +DIALECT_BIGQUERY = "BIGQUERY" + +SUPPORTED_DIALECTS = ( + DIALECT_ANSI, + DIALECT_SNOWFLAKE, + DIALECT_DATABRICKS, + DIALECT_BIGQUERY, +) + +# Apache Ossie dialect -> sqlglot dialect name. ANSI_SQL maps to sqlglot's default +# ("" / None) parser rather than a vendor grammar. +SQLGLOT_DIALECTS = { + DIALECT_ANSI: None, + DIALECT_SNOWFLAKE: "snowflake", + DIALECT_DATABRICKS: "databricks", + DIALECT_BIGQUERY: "bigquery", +} + +# Solid renders a composite primary key as a single comma-joined scalar +# (`primary_key: 'ORDER_ID, LINE_NO'`); see semantic_layer_utils.py in solid-server. +# Apache Ossie models it as a column array, so the two are split/joined on this. +PK_SEPARATOR = ", " + +# Solid embeds references to its own catalog objects inside `custom_instructions` as +# self-closing markup carrying an internal UUID: +# +# @ +# +# There is no closing tag and attribute order is not significant. Solid resolves these +# to their `name` before the text reaches an LLM; this converter does the same for the +# Apache Ossie `ai_context.instructions` a downstream tool will read, and keeps the raw +# tagged text in the stash so an export reproduces the original byte-for-byte. +_ASSET_LINK_RE = re.compile(r"@]+)>", re.IGNORECASE) +_ASSET_LINK_ATTR_RE = re.compile(r"(\w+)\s*=\s*[\"']([^\"']*)[\"']") + + +class ConversionError(Exception): + """Raised when an input cannot be converted.""" + + +def warn(scope, message): + """Emit a conversion warning. The CLI surfaces these on stderr.""" + warnings.warn(f"[{scope}] {message}", stacklevel=2) + + +def require(obj, key, what): + """Return `obj[key]`, or raise a clean ConversionError if it is missing/empty. + + Presence is tested by key rather than truthiness, so a legitimately falsy value such + as `0` or `False` is returned; a missing key, a null, or an empty/whitespace-only + string is rejected. + """ + if not isinstance(obj, dict) or key not in obj or obj[key] is None: + raise ConversionError(f"{what} is missing required '{key}'") + value = obj[key] + if isinstance(value, str) and not value.strip(): + raise ConversionError(f"{what} has an empty '{key}'") + return value + + +def require_str(obj, key, what): + """Like require(), but also enforce that the value is a string -- so a non-string + scalar (e.g. a YAML number used as a name) raises a clean ConversionError instead of + failing later inside a string operation.""" + value = require(obj, key, what) + if not isinstance(value, str): + raise ConversionError( + f"{what}: '{key}' must be a string, got {type(value).__name__}") + return value + + +def load_yaml(text): + """Parse YAML, surfacing a syntax error as a ConversionError so callers (and the + CLI) get a clean message rather than a raw traceback.""" + try: + return yaml.safe_load(text) + except yaml.YAMLError as e: + raise ConversionError(f"Invalid YAML: {e}") from e + + +def dump_yaml(obj): + """Serialize to YAML, preserving key insertion order and allowing Unicode through + unescaped (Solid descriptions routinely contain non-ASCII punctuation).""" + return yaml.safe_dump( + obj, sort_keys=False, default_flow_style=False, allow_unicode=True, width=100 + ) + + +def read_stash(obj): + """Return the SOLID stash dict on an Apache Ossie object, or {} if absent. + + The `_v` version marker is stripped from the returned dict. + """ + for ext in (obj or {}).get("custom_extensions") or []: + if ext.get("vendor_name") == VENDOR: + try: + data = json.loads(ext.get("data") or "{}") + except json.JSONDecodeError as e: + raise ConversionError( + f"SOLID custom_extensions data is not valid JSON: {e}") from e + if not isinstance(data, dict): + raise ConversionError( + "SOLID custom_extensions data must be a JSON object") + data.pop("_v", None) + return data + return {} + + +def write_stash(obj, data): + """Attach a SOLID `custom_extensions` entry holding `data` (a dict). + + No-op when `data` is empty, so an object with nothing Solid-specific to carry stays + clean. Merges into an existing SOLID entry if one is already present. + """ + if not data: + return + payload = {"_v": STASH_VERSION} + payload.update(data) + blob = json.dumps(payload, ensure_ascii=False) + exts = obj.setdefault("custom_extensions", []) + for ext in exts: + if ext.get("vendor_name") == VENDOR: + ext["data"] = blob + return + exts.append({"vendor_name": VENDOR, "data": blob}) + + +def foreign_vendor_extensions(obj): + """Return the non-SOLID custom_extensions on an object. + + Solid's YAML has no slot for vendor metadata, so these are dropped on export; the + caller warns rather than discarding them silently. + """ + return [ + ext + for ext in (obj or {}).get("custom_extensions") or [] + if ext.get("vendor_name") != VENDOR + ] + + +def pick_expression(ossie_expression, dialect): + """Choose the SQL string for an Apache Ossie expression: `dialect`, else ANSI_SQL. + + Returns (expression, matched) where `matched` is False when the value came from the + ANSI_SQL fallback rather than the requested dialect, so the caller can warn. Returns + (None, False) when neither dialect is present. + """ + dialects = { + d.get("dialect"): d.get("expression") + for d in (ossie_expression or {}).get("dialects") or [] + } + for candidate, matched in ((dialect, True), (DIALECT_ANSI, False)): + expr = dialects.get(candidate) + if expr is None: + continue + if not isinstance(expr, str): + raise ConversionError( + f"expression must be a string, got {type(expr).__name__}") + return expr, matched + return None, False + + +def readable_dialects(dialect): + """Name the dialects `pick_expression` will accept, for an error message. + + Reads as "SNOWFLAKE or ANSI_SQL", and as plain "ANSI_SQL" when the selected dialect + *is* ANSI_SQL rather than the tautological "ANSI_SQL or ANSI_SQL". + """ + if dialect == DIALECT_ANSI: + return DIALECT_ANSI + return f"{dialect} or {DIALECT_ANSI}" + + +def resolve_asset_links(text): + """Replace Solid `@` markup with the tag's `name` attribute. + + Mirrors solid-server's `replace_asset_links_with_display_names`, which is applied + before instructions are handed to an LLM. A tag with no usable `name` is left + verbatim rather than deleted, so no text is silently lost. + """ + if not isinstance(text, str) or "@ Solid converter. + + ossie-solid import -i solid_model.yaml [-o model.yaml] [--dialect SNOWFLAKE] + ossie-solid export -i model.yaml [-o solid_model.yaml] [--dialect SNOWFLAKE] + +`import` converts a Solid semantic model export into an Apache Ossie semantic model; +`export` does the reverse. With no `-o` the result goes to stdout. Conversions that drop +or approximate information emit warnings to stderr. +""" + +import argparse +import sys +import warnings + +from ._common import SUPPORTED_DIALECTS, ConversionError +from .ossie_to_solid import convert_ossie_to_solid +from .solid_to_ossie import convert_solid_to_ossie + + +def _build_parser(): + parser = argparse.ArgumentParser( + prog="ossie-solid", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = parser.add_subparsers(dest="command") + sub.required = True + + imp = sub.add_parser( + "import", help="Solid semantic model YAML -> Apache Ossie semantic model") + imp.add_argument("-i", "--input", required=True, help="Solid YAML file") + imp.add_argument("-o", "--output", help="output Apache Ossie YAML (default: stdout)") + imp.add_argument( + "-d", "--dialect", choices=SUPPORTED_DIALECTS, + help="expression dialect to label expressions with (default: inferred from the " + "column type vocabulary, falling back to ANSI_SQL)") + imp.add_argument( + "--name", help="Apache Ossie model name (default: the Solid model's name)") + + exp = sub.add_parser( + "export", help="Apache Ossie semantic model -> Solid semantic model YAML") + exp.add_argument("-i", "--input", required=True, help="Apache Ossie YAML file") + exp.add_argument("-o", "--output", help="output Solid YAML (default: stdout)") + exp.add_argument( + "-d", "--dialect", choices=SUPPORTED_DIALECTS, + help="expression dialect to read (default: the dialect recorded at import, " + "else the one the model's expressions use)") + exp.add_argument("--name", help="Solid model name (default: the Apache Ossie " + "model's name)") + return parser + + +def _show_warning(message, category, filename, lineno, file=None, line=None): + """Render conversion warnings as plain stderr lines rather than Python tracebacks.""" + print(f"Warning: {message}", file=sys.stderr) + + +def main(argv=None): + args = _build_parser().parse_args(argv) + convert = convert_solid_to_ossie if args.command == "import" else convert_ossie_to_solid + try: + with open(args.input) as handle: + source = handle.read() + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.showwarning = _show_warning + output = convert(source, dialect=args.dialect, model_name=args.name) + if args.output: + with open(args.output, "w") as handle: + handle.write(output) + else: + sys.stdout.write(output) + except (ConversionError, OSError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/converters/solid/src/ossie_solid/datatypes.py b/converters/solid/src/ossie_solid/datatypes.py new file mode 100644 index 00000000..6fe44487 --- /dev/null +++ b/converters/solid/src/ossie_solid/datatypes.py @@ -0,0 +1,285 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Warehouse type vocabulary handling for the Solid converter. + +Solid stores a column's `type` as the **raw** warehouse type string, verbatim from the +catalog -- `NUMBER(38,0)`, `TEXT`, `INT64`, `LONG` -- with no normalization. Two things +follow from that: + +1. The vocabulary identifies the warehouse, which is how `infer_dialect` recovers a + dialect that the Solid YAML itself never records (see README, "Dialect resolution"). +2. Mapping it onto Apache Ossie's portable `datatype` enum is lossy (`NUMBER(38,2)`, + `NUMERIC` and `DECIMAL` all collapse to `Decimal`), so the raw string is always kept + in the SOLID stash and is what an export re-emits. +""" + +import re + +from ._common import ( + DIALECT_ANSI, + DIALECT_BIGQUERY, + DIALECT_DATABRICKS, + DIALECT_SNOWFLAKE, + SUPPORTED_DIALECTS, + ConversionError, + warn, +) + +# Apache Ossie portable logical data types (core-spec/ossie-schema.json $defs.DataType). +STRING = "String" +INTEGER = "Integer" +DECIMAL = "Decimal" +FLOAT = "Float" +BOOLEAN = "Boolean" +DATE = "Date" +TIME = "Time" +DATE_TIME = "DateTime" +DATE_TIME_TZ = "DateTimeTz" +OPAQUE = "Opaque" + +TEMPORAL_DATATYPES = frozenset({DATE, TIME, DATE_TIME, DATE_TIME_TZ}) + +# Type names that appear in exactly one of the three warehouses Solid exports from, and +# so identify it. Names shared across warehouses (STRING is both Databricks and +# BigQuery; STRUCT likewise, since `parse_type` strips the angle-bracket params that +# would tell them apart; TIMESTAMP_NTZ is both Snowflake and Databricks; BOOLEAN, DATE, +# FLOAT and DECIMAL are near-universal) are deliberately excluded -- no signal. +_DIALECT_MARKERS = { + DIALECT_SNOWFLAKE: frozenset( + {"NUMBER", "TEXT", "VARIANT", "OBJECT", "TIMESTAMP_LTZ", "TIMESTAMP_TZ", + "GEOGRAPHY", "GEOMETRY"} + ), + DIALECT_DATABRICKS: frozenset( + {"LONG", "MAP", "BIGINT", "TINYINT", "SMALLINT", "BYTE", "SHORT", "VOID"} + ), + DIALECT_BIGQUERY: frozenset( + {"INT64", "FLOAT64", "BOOL", "BIGNUMERIC", "BYTES", "RECORD"} + ), +} + +# Warehouse type -> Apache Ossie datatype, for names that mean the same thing +# everywhere. Dialect-sensitive names (the TIMESTAMP family, NUMBER/NUMERIC scale) are +# resolved in `to_ossie_datatype` instead. +_COMMON_TYPES = { + # character + "STRING": STRING, "TEXT": STRING, "VARCHAR": STRING, "CHAR": STRING, + "CHARACTER": STRING, "NVARCHAR": STRING, "NCHAR": STRING, "STRING_TYPE": STRING, + # exact integral + "INT": INTEGER, "INTEGER": INTEGER, "BIGINT": INTEGER, "SMALLINT": INTEGER, + "TINYINT": INTEGER, "BYTEINT": INTEGER, "LONG": INTEGER, "SHORT": INTEGER, + "BYTE": INTEGER, "INT64": INTEGER, "INT2": INTEGER, "INT4": INTEGER, + "INT8": INTEGER, "SERIAL": INTEGER, + # approximate + "FLOAT": FLOAT, "FLOAT4": FLOAT, "FLOAT8": FLOAT, "FLOAT64": FLOAT, + "DOUBLE": FLOAT, "DOUBLE PRECISION": FLOAT, "REAL": FLOAT, + # boolean + "BOOLEAN": BOOLEAN, "BOOL": BOOLEAN, + # temporal (dialect-independent) + "DATE": DATE, "TIME": TIME, + # not representable in the portable vocabulary + "VARIANT": OPAQUE, "OBJECT": OPAQUE, "ARRAY": OPAQUE, "MAP": OPAQUE, + "STRUCT": OPAQUE, "RECORD": OPAQUE, "JSON": OPAQUE, "XML": OPAQUE, + "BINARY": OPAQUE, "VARBINARY": OPAQUE, "BYTES": OPAQUE, "GEOGRAPHY": OPAQUE, + "GEOMETRY": OPAQUE, "INTERVAL": OPAQUE, "VOID": OPAQUE, "NULL": OPAQUE, +} + +# Exact-scale decimal families. Whether one is Integer or Decimal depends on the scale, +# which is parsed off the type string when present. +_DECIMAL_TYPES = frozenset({"NUMBER", "NUMERIC", "DECIMAL", "BIGNUMERIC", "DEC"}) + +# TIMESTAMP means different things per warehouse: BigQuery's and Databricks' are +# instants (offset-aware), Snowflake's bare TIMESTAMP is an alias for TIMESTAMP_NTZ. +_TIMESTAMP_BY_DIALECT = { + DIALECT_SNOWFLAKE: DATE_TIME, + DIALECT_DATABRICKS: DATE_TIME_TZ, + DIALECT_BIGQUERY: DATE_TIME_TZ, + DIALECT_ANSI: DATE_TIME, +} + +_TIMESTAMP_VARIANTS = { + "TIMESTAMP_NTZ": DATE_TIME, + "TIMESTAMPNTZ": DATE_TIME, + "DATETIME": DATE_TIME, + "TIMESTAMP_TZ": DATE_TIME_TZ, + "TIMESTAMPTZ": DATE_TIME_TZ, + "TIMESTAMP_LTZ": DATE_TIME_TZ, + "TIMESTAMPLTZ": DATE_TIME_TZ, + "TIMESTAMP WITH TIME ZONE": DATE_TIME_TZ, + "TIMESTAMP WITHOUT TIME ZONE": DATE_TIME, + "TIMESTAMP_UNSPECIFIED": DATE_TIME, +} + +# Apache Ossie datatype -> the type name an export writes back when the original raw +# string is unavailable (a hand-authored Apache Ossie model with no SOLID stash). Solid +# never interprets these beyond its fact/dimension split, so a representative name per +# dialect is enough. +_DATATYPE_TO_RAW = { + DIALECT_SNOWFLAKE: { + STRING: "TEXT", INTEGER: "NUMBER", DECIMAL: "NUMBER", FLOAT: "FLOAT", + BOOLEAN: "BOOLEAN", DATE: "DATE", TIME: "TIME", DATE_TIME: "TIMESTAMP_NTZ", + DATE_TIME_TZ: "TIMESTAMP_TZ", OPAQUE: "VARIANT", + }, + DIALECT_DATABRICKS: { + STRING: "STRING", INTEGER: "LONG", DECIMAL: "DECIMAL", FLOAT: "DOUBLE", + BOOLEAN: "BOOLEAN", DATE: "DATE", TIME: "STRING", DATE_TIME: "TIMESTAMP_NTZ", + DATE_TIME_TZ: "TIMESTAMP", OPAQUE: "STRING", + }, + DIALECT_BIGQUERY: { + STRING: "STRING", INTEGER: "INT64", DECIMAL: "NUMERIC", FLOAT: "FLOAT64", + BOOLEAN: "BOOL", DATE: "DATE", TIME: "TIME", DATE_TIME: "DATETIME", + DATE_TIME_TZ: "TIMESTAMP", OPAQUE: "STRING", + }, + DIALECT_ANSI: { + STRING: "VARCHAR", INTEGER: "INTEGER", DECIMAL: "DECIMAL", FLOAT: "DOUBLE", + BOOLEAN: "BOOLEAN", DATE: "DATE", TIME: "TIME", DATE_TIME: "TIMESTAMP", + DATE_TIME_TZ: "TIMESTAMP WITH TIME ZONE", OPAQUE: "VARCHAR", + }, +} + +# Leading type name, with any parameter list (`NUMBER(38,2)`, `VARCHAR(16777216)`) and +# any element type (`ARRAY`, `MAP`) captured separately. +_TYPE_RE = re.compile(r"^\s*([A-Za-z_][A-Za-z0-9_ ]*?)\s*(?:\(([^)]*)\)|<.*>)?\s*$") + + +def parse_type(raw): + """Split a raw warehouse type into (BASE_NAME, params) with BASE_NAME upper-cased. + + `NUMBER(38,2)` -> ("NUMBER", ["38", "2"]); `ARRAY` -> ("ARRAY", []); + an unparseable or empty value -> (None, []). + """ + if not isinstance(raw, str) or not raw.strip(): + return None, [] + match = _TYPE_RE.match(raw) + if not match: + return raw.strip().upper(), [] + base = " ".join(match.group(1).split()).upper() + params = [p.strip() for p in (match.group(2) or "").split(",") if p.strip()] + return base or None, params + + +def infer_dialect(raw_types): + """Infer the Apache Ossie dialect from a model's column type vocabulary. + + Solid's YAML export does not record the source warehouse -- it lives on the asset + row in Solid's own database and is dropped at render time -- but the raw type names + it copies out of the catalog do identify it. Each distinctive name votes for its + warehouse; the warehouse with the most votes wins. + + Returns (dialect, confident). `confident` is False when no marker was seen or two + warehouses tie, in which case the dialect is ANSI_SQL and the caller should warn. + """ + votes = dict.fromkeys(_DIALECT_MARKERS, 0) + for raw in raw_types: + base, _ = parse_type(raw) + if not base: + continue + for dialect, markers in _DIALECT_MARKERS.items(): + if base in markers: + votes[dialect] += 1 + ranked = sorted(votes.items(), key=lambda kv: (-kv[1], kv[0])) + best, best_votes = ranked[0] + if best_votes == 0: + return DIALECT_ANSI, False + if len(ranked) > 1 and ranked[1][1] == best_votes: + return DIALECT_ANSI, False + return best, True + + +def to_ossie_datatype(raw, dialect): + """Map a raw warehouse type onto an Apache Ossie `datatype`. + + Returns None when the type is absent or unrecognized -- the spec says to omit + `datatype` when it is unknown rather than guess. A type that is known but outside + the portable vocabulary (VARIANT, MAP, STRUCT, ...) maps to `Opaque`; the raw name + is preserved in the SOLID stash either way. + """ + base, params = parse_type(raw) + if not base: + return None + if base in _DECIMAL_TYPES: + # A declared scale of 0 is an integer -- Snowflake stores every integral column + # as NUMBER(38,0), so honouring the scale is what keeps counts and ids from + # being typed as Decimal. An undeclared scale defaults to 0 in Snowflake and + # BigQuery NUMERIC, but Databricks DECIMAL defaults to (10,0) -- also scale 0. + if len(params) >= 2: + return INTEGER if params[1] == "0" else DECIMAL + return INTEGER + if base == "TIMESTAMP": + return _TIMESTAMP_BY_DIALECT.get(dialect, DATE_TIME) + if base in _TIMESTAMP_VARIANTS: + return _TIMESTAMP_VARIANTS[base] + return _COMMON_TYPES.get(base) + + +def to_raw_type(datatype, dialect): + """Map an Apache Ossie `datatype` back to a representative warehouse type name. + + Only used when exporting a field that carries no SOLID stash (a hand-authored + Apache Ossie model). A field that came from Solid re-emits its original raw string. + """ + if not datatype: + return None + table = _DATATYPE_TO_RAW.get(dialect) or _DATATYPE_TO_RAW[DIALECT_ANSI] + return table.get(datatype) + + +# Solid splits a table's columns into `facts` and `dimensions` by data type, using a +# lowercase prefix match against this set (see semantic_layer_data_types.py in +# solid-server). Mirrored here so an export of a hand-authored Apache Ossie model -- +# where no field carries the `dimension` block that records the original split -- lands +# columns in the same bucket Solid would have chosen. +_SOLID_FACT_TYPE_PREFIXES = ( + "decimal", "double", "int", "float", "number", "numeric", "real", "tiny", "long", +) + + +def is_solid_fact_type(raw): + """True if Solid would classify a column of this raw type as a fact.""" + if not isinstance(raw, str): + return False + lowered = raw.strip().lower() + return any(lowered.startswith(p) for p in _SOLID_FACT_TYPE_PREFIXES) + + +def normalize_dialect(dialect): + """Validate and upper-case an Apache Ossie dialect name.""" + normalized = str(dialect).strip().upper() + if normalized not in SUPPORTED_DIALECTS: + raise ConversionError( + f"Unsupported dialect '{dialect}'. Choose one of: " + f"{', '.join(SUPPORTED_DIALECTS)}" + ) + return normalized + + +def resolve_dialect(explicit, raw_types, scope="model"): + """Pick the dialect for a conversion: an explicit choice wins, else inference. + + Warns when inference had to fall back to ANSI_SQL, since a fallback means every + expression is being labelled with a dialect it was not written in. + """ + if explicit: + return normalize_dialect(explicit) + dialect, confident = infer_dialect(raw_types) + if not confident: + warn( + scope, + "could not infer the source warehouse from the column type vocabulary; " + f"defaulting to {DIALECT_ANSI}. Pass --dialect to set it explicitly.", + ) + return dialect diff --git a/converters/solid/src/ossie_solid/expressions.py b/converters/solid/src/ossie_solid/expressions.py new file mode 100644 index 00000000..036a3f96 --- /dev/null +++ b/converters/solid/src/ossie_solid/expressions.py @@ -0,0 +1,286 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Metric expression rewriting between Solid and Apache Ossie conventions. + +Solid stores a metric formula against **bare** column names and records the owning table +separately (`tables: [shop.sales.orders]`); solid-server's `is_valid_metric_formula` +actively strips alias prefixes before saving. Apache Ossie instead expects a metric to be +self-contained, qualifying each column with its dataset +(`SUM(store_sales.ss_ext_sales_price)`), so the two directions here add and remove that +qualifier. + +The rewrite is a **surgical splice**, not a re-render. Every column occurrence is located +by sqlglot's tokenizer, which reports character positions and keeps string literals and +quoted identifiers in token types of their own, and the qualifier is inserted or removed +at those offsets. Everything else in the expression is left character-for-character +intact. + +That matters because round-tripping a parsed tree through `Expression.sql()` does not +return the author's SQL -- sqlglot canonicalizes as it generates, so +`CAST(x AS FLOAT)` comes back as `CAST(x AS DOUBLE)` and +`EXTRACT(year FROM d)` as `DATE_PART(YEAR, d)`. Those are semantically equal and +textually different, and a converter has no business rewriting SQL it was only asked to +qualify. + +The parser is still used, as a cross-check: it decides which names are genuinely bare +column references, and a token scan that disagrees with it means some occurrence is +something else (`EXTRACT(year FROM ...)` where a column is also called `year`). On any +disagreement the expression is left exactly as written -- an unqualified metric is a far +smaller problem than a corrupted one. +""" + +from collections import Counter + +import sqlglot +from sqlglot import exp +from sqlglot.errors import SqlglotError +from sqlglot.tokens import TokenType + +from ._common import SQLGLOT_DIALECTS + +# Token types that can carry an identifier: an unquoted name, or a quoted one whose +# offsets span the quote characters as well. +_NAME_TOKENS = (TokenType.VAR, TokenType.IDENTIFIER) + +# Conversion outcomes, returned alongside the expression. +QUALIFIED = "qualified" +UNQUALIFIED = "unqualified" +UNCHANGED = "unchanged" +UNPARSED = "unparsed" +AMBIGUOUS = "ambiguous" + + +def _read(dialect): + """Apache Ossie dialect name -> the sqlglot dialect to parse/tokenize with.""" + return SQLGLOT_DIALECTS.get(dialect) + + +def _parse(expression, dialect): + """Parse a scalar/aggregate expression, or return None if it does not parse. + + Metric expressions are fragments rather than statements, so `parse_one` is used + directly; a fragment sqlglot has no grammar for yields None and the caller leaves the + text alone. + """ + try: + return sqlglot.parse_one(expression, read=_read(dialect)) or None + except (SqlglotError, ValueError, RecursionError): + return None + + +def _tokenize(expression, dialect): + """Tokenize an expression, or return None if it cannot be tokenized.""" + try: + return sqlglot.tokenize(expression, read=_read(dialect)) + except (SqlglotError, ValueError, RecursionError): + return None + + +def _unquote(text): + """Strip the quoting a tokenizer may have left on an identifier's text.""" + return text.strip().strip('"`[]') + + +def column_reference(name, dialect): + """Render a column name as an Apache Ossie field expression. + + A Solid field's expression is just its own column, so the usual result is the bare + name. A name that is not a plain identifier -- one with spaces, or one that collides + with a reserved word -- is quoted with the target dialect's rules so the expression + stays valid SQL. + """ + if not isinstance(name, str) or not name.strip(): + return name + bare = exp.to_identifier(name, quoted=False) + if bare is not None: + rendered = bare.sql(dialect=_read(dialect)) + parsed = _parse(rendered, dialect) + # The bare form is usable only if it reads back as the same single column; a + # reserved word parses as some other node type, or not at all. + if isinstance(parsed, exp.Column) and parsed.name == name and not parsed.table: + return rendered + quoted = exp.to_identifier(name, quoted=True) + return quoted.sql(dialect=_read(dialect)) if quoted is not None else name + + +def _bare_columns(tree, known): + """Count, per name, the bare column references the AST reports. + + Only names in `known` are counted, since only those are candidates for qualification. + """ + counts = Counter() + for column in tree.find_all(exp.Column): + if column.table: + continue + name = column.name.lower() + if name in known: + counts[name] += 1 + return counts + + +def _qualified_columns(tree, known): + """Count, per (qualifier, name), the column references already qualified by a name in + `known`. A three-part reference (`db.table.column`) is excluded: its qualifier is a + schema, not a dataset.""" + counts = Counter() + for column in tree.find_all(exp.Column): + table = column.args.get("table") + if table is None or column.args.get("db"): + continue + if column.table.lower() in known: + counts[(column.table.lower(), column.name.lower())] += 1 + return counts + + +def _splice(expression, edits): + """Apply (start, end, replacement) edits to `expression`, right to left. + + `end` is exclusive. Applying in reverse order keeps earlier offsets valid. + """ + result = expression + for start, end, replacement in sorted(edits, key=lambda e: e[0], reverse=True): + result = result[:start] + replacement + result[end:] + return result + + +def qualify_metric(expression, dialect, dataset, columns): + """Prefix every bare column in `expression` with `dataset`. + + `columns` is the set of column names the dataset owns, compared case-insensitively + (Solid models routinely mix `ACCOUNT_ID` in the column list with `account_id` in the + formula). A column that already carries a qualifier is left as the author wrote it. + + Returns (expression, status): + "qualified" -- at least one column was prefixed, by splicing into the original text + "unchanged" -- nothing needed prefixing; the text is returned verbatim + "unparsed" -- sqlglot could not read it; the text is returned verbatim + "ambiguous" -- the token scan and the parse disagree on which names are columns, + so the text is returned verbatim rather than risk a bad edit + """ + if not expression or not expression.strip(): + return expression, UNCHANGED + known = {c.lower() for c in columns} + if not known: + return expression, UNCHANGED + + tree = _parse(expression, dialect) + tokens = _tokenize(expression, dialect) + if tree is None or tokens is None: + return expression, UNPARSED + + expected = _bare_columns(tree, known) + if not expected: + return expression, UNCHANGED + + targets = [] + for index, token in enumerate(tokens): + if token.token_type not in _NAME_TOKENS: + continue + if _unquote(token.text).lower() not in known: + continue + previous = tokens[index - 1] if index else None + following = tokens[index + 1] if index + 1 < len(tokens) else None + # Already qualified (`t.col`), itself a qualifier (`col.x`), or a function call + # (`col(...)`) -- none of these is a bare column reference. + if previous is not None and previous.token_type is TokenType.DOT: + continue + if following is not None and following.token_type in ( + TokenType.DOT, TokenType.L_PAREN): + continue + targets.append(token) + + if Counter(_unquote(t.text).lower() for t in targets) != expected: + return expression, AMBIGUOUS + + prefix = f"{column_reference(dataset, dialect)}." + return _splice( + expression, + [(t.start, t.start, prefix) for t in targets], + ), QUALIFIED + + +def unqualify_metric(expression, dialect, datasets): + """Strip a dataset qualifier off every column in `expression`. + + The inverse of `qualify_metric`, used when exporting back to Solid, whose formulas are + stored bare. `datasets` is the set of dataset names that may legitimately appear as a + qualifier; any other qualifier -- a genuine table alias an author wrote by hand -- is + preserved. + + Returns (expression, status) with the same vocabulary as `qualify_metric`, except that + a successful edit reports "unqualified". + """ + if not expression or not expression.strip(): + return expression, UNCHANGED + known = {d.lower() for d in datasets} + if not known: + return expression, UNCHANGED + + tree = _parse(expression, dialect) + tokens = _tokenize(expression, dialect) + if tree is None or tokens is None: + return expression, UNPARSED + + expected = _qualified_columns(tree, known) + if not expected: + return expression, UNCHANGED + + edits, found = [], Counter() + for index, token in enumerate(tokens): + if token.token_type not in _NAME_TOKENS: + continue + if _unquote(token.text).lower() not in known: + continue + dot = tokens[index + 1] if index + 1 < len(tokens) else None + name = tokens[index + 2] if index + 2 < len(tokens) else None + if dot is None or dot.token_type is not TokenType.DOT: + continue + if name is None or name.token_type not in _NAME_TOKENS: + continue + previous = tokens[index - 1] if index else None + # `db.table.column`: the leading name is a schema, so the middle one is not a + # dataset qualifier this converter added. + if previous is not None and previous.token_type is TokenType.DOT: + continue + after = tokens[index + 3] if index + 3 < len(tokens) else None + if after is not None and after.token_type is TokenType.DOT: + continue + found[(_unquote(token.text).lower(), _unquote(name.text).lower())] += 1 + # Delete the qualifier and its dot; `end` is inclusive on a sqlglot token. + edits.append((token.start, dot.end + 1, "")) + + if found != expected: + return expression, AMBIGUOUS + return _splice(expression, edits), UNQUALIFIED + + +def referenced_datasets(expression, dialect, datasets): + """Return the dataset names a metric expression qualifies columns with. + + Used on export to reconstruct Solid's `metrics[].tables` when an Apache Ossie model + carries no SOLID stash. Order follows `datasets`, so the result is deterministic. + """ + tree = _parse(expression, dialect) + if tree is None: + return [] + seen = set() + for column in tree.find_all(exp.Column): + if column.table and not column.args.get("db"): + seen.add(column.table.lower()) + return [d for d in datasets if d.lower() in seen] + diff --git a/converters/solid/src/ossie_solid/ossie_to_solid.py b/converters/solid/src/ossie_solid/ossie_to_solid.py new file mode 100644 index 00000000..360a4641 --- /dev/null +++ b/converters/solid/src/ossie_solid/ossie_to_solid.py @@ -0,0 +1,465 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert an Apache Ossie semantic model to a Solid semantic model YAML export. + +Pure offline conversion. A model that came from `convert_solid_to_ossie` restores its +Solid-only fields from `custom_extensions[SOLID]`; a hand-authored Apache Ossie model is +converted on its core fields alone, with anything Solid's format cannot hold reported as +a warning rather than dropped silently. + +Key order in the output follows solid-server's own export template, so a converted model +diffs cleanly against one Solid produced itself. + +Usage (CLI): + ossie-solid export -i model.yaml [-o solid_model.yaml] [--dialect SNOWFLAKE] +""" + +from . import datatypes +from ._common import ( + DIALECT_ANSI, + ONE_TO_ONE_NOTE, + OSSIE_VERSION, + OSSIE_VERSION_NOTES, + PK_SEPARATOR, + READABLE_OSSIE_VERSIONS, + SUPPORTED_DIALECTS, + ConversionError, + ai_context_parts, + clean_text, + dump_yaml, + foreign_vendor_extensions, + load_yaml, + pick_expression, + read_stash, + readable_dialects, + require_str, + string_list, + warn, +) +from .expressions import AMBIGUOUS, UNPARSED, referenced_datasets, unqualify_metric + + +def convert_ossie_to_solid(ossie_yaml_str, dialect=None, model_name=None): + """Parse an Apache Ossie semantic model and return Solid YAML (a string). + + `dialect` chooses which expression dialect to read; when omitted it comes from the + SOLID stash, else from whichever non-ANSI dialect the model's expressions use. + `model_name` overrides the Solid model name. + """ + document = load_yaml(ossie_yaml_str) + if not isinstance(document, dict): + raise ConversionError( + "Invalid Apache Ossie model: expected a mapping at the root") + + version = str(document.get("version", "")) + if version not in READABLE_OSSIE_VERSIONS: + raise ConversionError( + f"Unsupported Apache Ossie version '{version}'. This converter reads " + f"{', '.join('v' + v for v in READABLE_OSSIE_VERSIONS)}." + ) + if version != OSSIE_VERSION: + note = OSSIE_VERSION_NOTES.get(version) + warn("model", f"the document declares Apache Ossie v{version}, not the current " + f"v{OSSIE_VERSION}" + + (f"; {note}" if note else "")) + + models = document.get("semantic_model") + if not isinstance(models, list) or not models: + raise ConversionError( + "Apache Ossie document is missing a non-empty 'semantic_model' list") + if len(models) > 1: + warn("model", f"the document holds {len(models)} semantic models but Solid's " + f"format holds one; only '{models[0].get('name')}' was converted") + + return dump_yaml({"semantic_model": _convert_model(models[0], dialect, model_name)}) + + +def _convert_model(ossie, explicit_dialect, model_name): + if not isinstance(ossie, dict): + raise ConversionError("Each entry of 'semantic_model' must be a mapping") + datasets = ossie.get("datasets") + if not isinstance(datasets, list) or not datasets: + raise ConversionError("Apache Ossie model has no 'datasets'") + + stash = read_stash(ossie) + resolved_dialect = _resolve_dialect(explicit_dialect, stash, ossie) + _warn_foreign_extensions(ossie, "model", ossie.get("name")) + + by_name = {} + for dataset in datasets: + if not isinstance(dataset, dict): + raise ConversionError("Each entry of 'datasets' must be a mapping") + by_name[require_str(dataset, "name", "dataset")] = dataset + # Whether the model records Solid's fact/dimension split at all. If no field + # anywhere carries a `dimension` block the model was not produced by this converter, + # so the split is re-derived from the data types the way Solid itself does it. + has_dimension_metadata = any( + isinstance(field, dict) and field.get("dimension") is not None + for dataset in datasets + for field in dataset.get("fields") or [] + ) + + instructions, _, examples = ai_context_parts(ossie.get("ai_context")) + business_context = {} + if examples: + business_context["business_questions"] = examples + model_description = clean_text(stash.get("model_description")) + if model_description: + business_context["model_description"] = model_description + # The stash holds the asset-link markup Solid wrote; `ai_context.instructions` holds + # the same text with links resolved to display names. Prefer the markup so a + # re-import into Solid keeps its catalog references live. + custom_instructions = clean_text(stash.get("custom_instructions")) or instructions + if custom_instructions: + business_context["custom_instructions"] = custom_instructions + + solid = {"name": model_name or require_str(ossie, "name", "semantic_model")} + solid["business_context"] = business_context + llm_description = clean_text(stash.get("model_llm_description")) + if llm_description is None and "model_description" not in stash: + # No stash to read: an Apache Ossie `description` is the model's primary + # description, which is the role Solid's `model_llm_description` plays. + llm_description = clean_text(ossie.get("description")) + if llm_description: + solid["model_llm_description"] = llm_description + + solid["tables"] = [ + _convert_dataset(d, resolved_dialect, has_dimension_metadata) for d in datasets + ] + + metrics = _convert_metrics( + ossie.get("metrics") or [], by_name, resolved_dialect) + if metrics: + solid["metrics"] = metrics + relationships = _convert_relationships(ossie.get("relationships") or [], by_name) + if relationships: + solid["relationships"] = relationships + for key in ("example_queries", "benchmark_questions"): + if stash.get(key): + solid[key] = stash[key] + return solid + + +def _resolve_dialect(explicit, stash, ossie): + """Pick which Apache Ossie expression dialect to read expressions from. + + An explicit choice wins, then the dialect the SOLID stash recorded at import time, + then the single non-ANSI dialect the model's expressions use. A model written purely + in ANSI_SQL resolves to ANSI_SQL. + """ + if explicit: + return datatypes.normalize_dialect(explicit) + stashed = clean_text(stash.get("dialect")) + if stashed: + # Written by this converter's own import, so it is always one of the supported + # names; validating it means a hand-edited stash fails loudly rather than + # feeding an unknown dialect to the expression rewriter. + return datatypes.normalize_dialect(stashed) + + # Apache Ossie's dialect enum includes expression languages that are not SQL (MDX, + # TABLEAU, MAQL). Those cannot be parsed, qualified or unqualified by this + # converter, so they never become the resolved dialect -- naming one here would + # hand a non-SQL formula to sqlglot and to Solid as though it were SQL. + found, unreadable = set(), set() + for expression in _all_expressions(ossie): + for entry in (expression or {}).get("dialects") or []: + dialect = entry.get("dialect") + if not dialect or dialect == DIALECT_ANSI: + continue + (found if dialect in SUPPORTED_DIALECTS else unreadable).add(dialect) + if unreadable: + warn("model", f"the model carries {', '.join(sorted(unreadable))} expressions, " + f"which are not SQL this converter can read; only the " + f"{DIALECT_ANSI} form of each expression will be used") + if len(found) == 1: + return found.pop() + if len(found) > 1: + warn("model", f"expressions mix the {', '.join(sorted(found))} dialects; " + f"reading {DIALECT_ANSI}. Pass --dialect to choose one.") + return DIALECT_ANSI + + +def _all_expressions(ossie): + for dataset in ossie.get("datasets") or []: + for field in dataset.get("fields") or []: + if isinstance(field, dict): + yield field.get("expression") + for metric in ossie.get("metrics") or []: + if isinstance(metric, dict): + yield metric.get("expression") + + +def _warn_foreign_extensions(obj, scope, name): + """Warn about custom_extensions belonging to other vendors. + + Solid's format has no slot for vendor metadata, so these cannot be carried across. + """ + foreign = foreign_vendor_extensions(obj) + if foreign: + vendors = ", ".join(sorted({e.get("vendor_name", "?") for e in foreign})) + warn(scope, f"'{name}': custom_extensions for {vendors} have no Solid " + f"equivalent and were dropped") + + +def _convert_dataset(dataset, dialect, has_dimension_metadata): + name = require_str(dataset, "name", "dataset") + stash = read_stash(dataset) + _warn_foreign_extensions(dataset, "dataset", name) + + # Solid identifies a table by its fully-qualified name, which is the Apache Ossie + # `source`; the Apache Ossie dataset name is a local alias with no Solid slot. + table = {"name": require_str(dataset, "source", f"dataset '{name}'")} + + description = clean_text(dataset.get("description")) + if description: + table["description"] = description + instructions, synonyms, examples = ai_context_parts(dataset.get("ai_context")) + if instructions: + table["manual_description"] = instructions + if synonyms: + table["synonyms"] = synonyms + if examples: + warn("dataset", f"'{name}': ai_context.examples has no Solid equivalent at the " + f"table level and was dropped") + + primary_key = string_list(dataset.get("primary_key")) + if primary_key: + # Solid stores a composite key as one comma-joined scalar. + table["primary_key"] = PK_SEPARATOR.join(primary_key) + unique_keys = dataset.get("unique_keys") + if unique_keys and unique_keys != [primary_key]: + warn("dataset", f"'{name}': unique_keys have no Solid equivalent and were " + f"dropped") + # Solid emits `quality_rank` on every table, empty when the rank is unset. + table["quality_rank"] = stash.get("quality_rank", "") + if stash.get("indexes"): + table["indexes"] = list(stash["indexes"]) + + dimensions, facts = [], [] + for field in dataset.get("fields") or []: + if not isinstance(field, dict): + raise ConversionError(f"Dataset '{name}': each field must be a mapping") + column, is_dimension = _convert_field( + field, dialect, name, has_dimension_metadata) + (dimensions if is_dimension else facts).append(column) + if dimensions: + table["dimensions"] = dimensions + # Unlike `dimensions`, Solid always emits `facts` -- as `[]` when a table has none. + table["facts"] = facts + return table + + +def _convert_field(field, dialect, dataset_name, has_dimension_metadata): + name = require_str(field, "name", f"field in '{dataset_name}'") + stash = read_stash(field) + _warn_foreign_extensions(field, "field", f"{dataset_name}.{name}") + + expression, matched = pick_expression(field.get("expression"), dialect) + if expression is None: + raise ConversionError( + f"Field '{dataset_name}.{name}' has no " + f"{readable_dialects(dialect)} expression") + if not matched and dialect != DIALECT_ANSI: + warn("field", f"'{dataset_name}.{name}' has no {dialect} expression; the " + f"{DIALECT_ANSI} one was used") + + raw_type = clean_text(stash.get("type")) or datatypes.to_raw_type( + field.get("datatype"), dialect) + is_dimension = _is_dimension(field, stash, raw_type, has_dimension_metadata) + + column = {"name": name} + # A Solid fact may be a table-scoped metric instead of a catalog column, in which + # case it carries an expression and no type. Everything else is a column, whose + # expression is just its own name. + is_metric_fact = ( + not is_dimension + and (stash.get("role") == "metric" or (raw_type is None and _is_computed(expression, name))) + ) + if is_metric_fact: + raw_type = None + elif raw_type: + column["type"] = raw_type + else: + warn("field", f"'{dataset_name}.{name}' has no datatype; Solid's 'type' was " + f"left empty") + column["type"] = "" + + description = clean_text(field.get("description")) + if description: + column["description"] = description + instructions, synonyms, examples = ai_context_parts(field.get("ai_context")) + if instructions: + column["manual_description"] = instructions + if is_metric_fact: + column["expression"] = expression + elif _is_computed(expression, name): + warn("field", f"'{dataset_name}.{name}' is a computed field " + f"(`{expression}`); Solid columns map to catalog columns, so the " + f"expression was dropped") + if synonyms: + column["synonyms"] = synonyms + if examples: + warn("field", f"'{dataset_name}.{name}': ai_context.examples has no Solid " + f"equivalent and was dropped") + if field.get("label"): + warn("field", f"'{dataset_name}.{name}': label '{field['label']}' has no Solid " + f"equivalent and was dropped") + if stash.get("sample_values"): + column["sample_values"] = list(stash["sample_values"]) + return column, is_dimension + + +def _is_dimension(field, stash, raw_type, has_dimension_metadata): + """Decide whether a field belongs in Solid's `dimensions` or its `facts`. + + A model this converter produced records the original split as the presence of the + `dimension` block. A hand-authored Apache Ossie model has no such marker, so the + split is re-derived from the data type the way solid-server does it. + """ + if has_dimension_metadata: + return field.get("dimension") is not None + if stash.get("role") == "metric": + return False + return not datatypes.is_solid_fact_type(raw_type) + + +def _is_computed(expression, name): + """True if a field's expression is something other than a plain reference to itself.""" + return expression.strip().strip('"`[]').lower() != name.strip().lower() + + +def _convert_relationships(relationships, by_name): + converted = [] + for relationship in relationships: + if not isinstance(relationship, dict): + raise ConversionError("Each entry of 'relationships' must be a mapping") + name = relationship.get("name", "") + source = require_str(relationship, "from", f"relationship '{name}'") + target = require_str(relationship, "to", f"relationship '{name}'") + for end in (source, target): + if end not in by_name: + raise ConversionError( + f"Relationship '{name}' references dataset '{end}', which is not " + f"declared in 'datasets'") + source_columns = string_list(relationship.get("from_columns")) + target_columns = string_list(relationship.get("to_columns")) + if not source_columns or not target_columns: + raise ConversionError( + f"Relationship '{name}' is missing from_columns/to_columns") + if len(source_columns) != len(target_columns): + raise ConversionError( + f"Relationship '{name}' has {len(source_columns)} from_columns but " + f"{len(target_columns)} to_columns; they must correspond positionally") + + stash = read_stash(relationship) + _warn_foreign_extensions(relationship, "relationship", name) + # A Solid relationship is just a table pair and its join keys -- it carries no + # free text -- so any annotation on the Apache Ossie side is dropped. The + # exception is the note import writes for a one-to-one, which is this + # converter's own marker and means nothing to Solid. + instructions, synonyms, examples = ai_context_parts( + relationship.get("ai_context")) + if instructions == ONE_TO_ONE_NOTE: + instructions = None + dropped = [ + label + for label, value in (("instructions", instructions), + ("synonyms", synonyms), + ("examples", examples)) + if value + ] + if dropped: + warn("relationship", + f"'{name}': ai_context.{', ai_context.'.join(dropped)} " + f"{'have' if len(dropped) > 1 else 'has'} no Solid equivalent -- a " + f"Solid relationship carries only its tables and join keys -- and " + f"{'were' if len(dropped) > 1 else 'was'} dropped") + if stash.get("flipped"): + # Import flipped this pair to put the many side on `from`; restore the + # left/right order Solid wrote. + left, right = target, source + left_columns, right_columns = target_columns, source_columns + else: + left, right = source, target + left_columns, right_columns = source_columns, target_columns + + converted.append({ + "left_table": by_name[left]["source"], + "right_table": by_name[right]["source"], + "join_keys": {"left": left_columns, "right": right_columns}, + }) + return converted + + +def _convert_metrics(metrics, by_name, dialect): + dataset_names = list(by_name) + converted = [] + for metric in metrics: + if not isinstance(metric, dict): + raise ConversionError("Each entry of 'metrics' must be a mapping") + name = require_str(metric, "name", "metric") + stash = read_stash(metric) + _warn_foreign_extensions(metric, "metric", name) + # Solid types a metric by evaluating its formula against the warehouse, so its + # YAML has no slot for a declared result type. + if metric.get("datatype"): + warn("metric", f"'{name}': datatype '{metric['datatype']}' has no Solid " + f"equivalent and was dropped") + + expression, matched = pick_expression(metric.get("expression"), dialect) + if expression is None: + raise ConversionError( + f"Metric '{name}' has no {readable_dialects(dialect)} expression") + if not matched and dialect != DIALECT_ANSI: + warn("metric", f"'{name}' has no {dialect} expression; the {DIALECT_ANSI} " + f"one was used") + + # Solid stores formulas against bare columns and records the owning tables + # separately, so the dataset qualifier Apache Ossie carries is stripped back out. + tables = stash.get("tables") + if tables is None: + tables = [ + by_name[d]["source"] + for d in referenced_datasets(expression, dialect, dataset_names) + ] + bare, status = unqualify_metric(expression, dialect, dataset_names) + if status in (UNPARSED, AMBIGUOUS): + warn("metric", f"'{name}': the dataset qualifiers could not be removed " + f"safely ({status}), so the expression was left as written; " + f"Solid stores formulas with bare column names") + if not tables: + warn("metric", f"'{name}' names no table; Solid needs at least one to " + f"resolve its columns") + + solid_metric = {"name": name} + description = clean_text(metric.get("description")) + if description: + solid_metric["description"] = description + solid_metric["expression"] = bare + _, synonyms, examples = ai_context_parts(metric.get("ai_context")) + if synonyms: + solid_metric["synonyms"] = synonyms + if examples: + warn("metric", f"'{name}': ai_context.examples has no Solid equivalent and " + f"was dropped") + solid_metric["tables"] = list(tables) + converted.append(solid_metric) + return converted + + +__all__ = ["convert_ossie_to_solid"] diff --git a/converters/solid/src/ossie_solid/solid_to_ossie.py b/converters/solid/src/ossie_solid/solid_to_ossie.py new file mode 100644 index 00000000..e73fb636 --- /dev/null +++ b/converters/solid/src/ossie_solid/solid_to_ossie.py @@ -0,0 +1,484 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert a Solid semantic model YAML export to an Apache Ossie semantic model. + +Pure offline conversion. Solid features Apache Ossie has no native field for -- +`example_queries`, `benchmark_questions`, `quality_rank`, `indexes`, `sample_values`, +and each column's raw warehouse type -- are preserved in `custom_extensions[SOLID]`, so +converting back reproduces the original export. + +Usage (CLI): + ossie-solid import -i solid_model.yaml [-o model.yaml] [--dialect SNOWFLAKE] +""" + +from . import datatypes +from ._common import ( + ONE_TO_ONE_NOTE, + OSSIE_VERSION, + ConversionError, + build_ai_context, + clean_text, + dataset_name_for, + dump_yaml, + has_asset_links, + load_yaml, + require_str, + resolve_asset_links, + string_list, + unique_name, + warn, + write_stash, +) +from .expressions import ( + AMBIGUOUS, + UNCHANGED, + UNPARSED, + column_reference, + qualify_metric, + referenced_datasets, +) + + +def convert_solid_to_ossie(solid_yaml_str, dialect=None, model_name=None): + """Parse a Solid semantic model export and return Apache Ossie YAML (a string). + + `dialect` forces the Apache Ossie expression dialect; when omitted it is inferred + from the column type vocabulary (see datatypes.infer_dialect). `model_name` + overrides the Apache Ossie model name. + """ + document = load_yaml(solid_yaml_str) + if not isinstance(document, dict): + raise ConversionError( + "Invalid Solid semantic model: expected a mapping at the root") + solid = document.get("semantic_model") + if solid is None: + raise ConversionError("Solid semantic model is missing the top-level " + "'semantic_model' key") + if isinstance(solid, list): + # An Apache Ossie document also has a `semantic_model` key, but holding a list. + # Naming the confusion is more useful than a generic type error. + raise ConversionError( + "'semantic_model' is a list; that is the Apache Ossie layout, not Solid's. " + "Did you mean `ossie-solid export`?" + ) + if not isinstance(solid, dict): + raise ConversionError("'semantic_model' must be a mapping") + + model = _convert_model(solid, dialect, model_name) + return dump_yaml({"version": OSSIE_VERSION, "semantic_model": [model]}) + + +def _convert_model(solid, explicit_dialect, model_name): + tables = solid.get("tables") or [] + if not isinstance(tables, list) or not tables: + # Apache Ossie requires at least one dataset (schema: datasets.minItems = 1). + raise ConversionError("Solid semantic model has no 'tables'; an Apache Ossie " + "model requires at least one dataset") + + resolved_dialect = datatypes.resolve_dialect( + explicit_dialect, _all_raw_types(tables), scope="model") + + # Solid names a table by its fully-qualified `catalog.schema.table`, which becomes + # the Apache Ossie `source`; the dataset name is derived from it and is what + # relationships and metric expressions refer to. + datasets, by_source = [], {} + taken_names = set() + for table in tables: + dataset = _convert_table(table, resolved_dialect, taken_names) + datasets.append(dataset) + by_source[dataset["source"].lower()] = dataset + + relationships = _convert_relationships( + solid.get("relationships") or [], by_source) + metrics = _convert_metrics( + solid.get("metrics") or [], by_source, datasets, resolved_dialect) + + business = solid.get("business_context") + business = business if isinstance(business, dict) else {} + llm_description = clean_text(solid.get("model_llm_description")) + customer_description = clean_text(business.get("model_description")) + raw_instructions = clean_text(business.get("custom_instructions")) + + model = {"name": model_name or require_str(solid, "name", "semantic_model")} + description = llm_description or customer_description + if description: + model["description"] = description + ai_context = build_ai_context( + # Solid resolves its asset-link markup to plain display names before the text + # reaches an LLM; do the same for the Apache Ossie consumer, keeping the tagged + # original in the stash. + instructions=resolve_asset_links(raw_instructions) if raw_instructions else None, + examples=string_list(business.get("business_questions")), + ) + if ai_context: + model["ai_context"] = ai_context + + model["datasets"] = datasets + if relationships: + model["relationships"] = relationships + if metrics: + model["metrics"] = metrics + + stash = {"dialect": resolved_dialect} + # The model carries three free-text fields but Apache Ossie offers two slots, so + # both raw descriptions are recorded; export reads them back rather than guessing + # which one the single `description` came from. + if llm_description: + stash["model_llm_description"] = llm_description + if customer_description: + stash["model_description"] = customer_description + if raw_instructions and has_asset_links(raw_instructions): + stash["custom_instructions"] = raw_instructions + for key in ("example_queries", "benchmark_questions"): + value = solid.get(key) + if value: + stash[key] = value + write_stash(model, stash) + return model + + +def _all_raw_types(tables): + """Every raw column type in the model, for dialect inference.""" + types = [] + for table in tables: + if not isinstance(table, dict): + continue + for column in _columns_of(table): + if isinstance(column, dict) and column.get("type"): + types.append(column["type"]) + return types + + +def _columns_of(table): + """Dimensions followed by facts, skipping anything that is not a mapping.""" + columns = [] + for key in ("dimensions", "facts"): + for column in table.get(key) or []: + if isinstance(column, dict): + columns.append(column) + return columns + + +def _convert_table(table, dialect, taken_names): + if not isinstance(table, dict): + raise ConversionError("Each entry of 'tables' must be a mapping") + source = require_str(table, "name", "table") + name = dataset_name_for(source, taken_names) + + dataset = {"name": name, "source": source} + + primary_key = _split_primary_key(table.get("primary_key")) + if primary_key: + dataset["primary_key"] = primary_key + + description = clean_text(table.get("description")) + if description: + dataset["description"] = description + # Solid carries an AI-written `description` and a human-written + # `manual_description`. Apache Ossie has one description field, so the AI one -- the + # consistently populated, consumer-facing text -- maps to `description` and the + # human annotation maps to `ai_context.instructions`. The split is 1:1 in both + # directions, so neither needs stashing. + ai_context = build_ai_context( + instructions=clean_text(table.get("manual_description")), + synonyms=string_list(table.get("synonyms")), + ) + if ai_context: + dataset["ai_context"] = ai_context + + fields, field_names = [], set() + for column in table.get("dimensions") or []: + fields.append(_convert_column(column, dialect, name, field_names, True)) + for column in table.get("facts") or []: + fields.append(_convert_column(column, dialect, name, field_names, False)) + if fields: + dataset["fields"] = fields + + stash = {} + quality_rank = clean_text(table.get("quality_rank")) + if quality_rank: + stash["quality_rank"] = quality_rank + indexes = string_list(table.get("indexes")) + if indexes: + stash["indexes"] = indexes + write_stash(dataset, stash) + return dataset + + +def _split_primary_key(value): + """Split Solid's comma-joined primary key scalar into Apache Ossie's column array. + + solid-server joins a composite key's column names with `", "` into a single scalar + (`primary_key: 'ORDER_ID, LINE_NO'`), so splitting on the comma is what recovers the + composite. A non-string value is tolerated: a list is taken as-is, since a + hand-edited export may already carry one. + """ + if isinstance(value, list): + return string_list(value) + text = clean_text(value) + if not text: + return [] + return [part.strip() for part in text.split(",") if part.strip()] + + +def _convert_column(column, dialect, dataset_name, taken, is_dimension): + if not isinstance(column, dict): + raise ConversionError( + f"Dataset '{dataset_name}': each column entry must be a mapping") + raw_name = require_str(column, "name", f"column in '{dataset_name}'") + name = unique_name(raw_name, taken) + if name != raw_name: + warn("field", f"'{dataset_name}' declares '{raw_name}' more than once; the " + f"duplicate was renamed to '{name}'") + + raw_type = clean_text(column.get("type")) + # Solid renders two kinds of fact: a catalog column (which has a `type`) and a + # table-scoped metric (which has an `expression` and no `type`). The former's + # expression is just its own name. + solid_expression = clean_text(column.get("expression")) + expression = solid_expression or column_reference(raw_name, dialect) + + field = { + "name": name, + "expression": {"dialects": [{"dialect": dialect, "expression": expression}]}, + } + description = clean_text(column.get("description")) + if description: + field["description"] = description + + datatype = datatypes.to_ossie_datatype(raw_type, dialect) + if datatype: + field["datatype"] = datatype + elif raw_type: + warn("field", f"'{dataset_name}.{name}': warehouse type '{raw_type}' has no " + f"Apache Ossie datatype; 'datatype' was omitted") + + if is_dimension: + # The presence of the `dimension` block is what records that Solid classified + # this column as a dimension rather than a fact, so it is always emitted for a + # dimension -- with `is_time` stated explicitly rather than left to the spec's + # datatype-derived default, so a consumer that does not implement that default + # still reads the same role. + field["dimension"] = {"is_time": datatype in datatypes.TEMPORAL_DATATYPES} + + ai_context = build_ai_context( + instructions=clean_text(column.get("manual_description")), + synonyms=string_list(column.get("synonyms")), + ) + if ai_context: + field["ai_context"] = ai_context + + stash = {} + if raw_type: + # Apache Ossie's portable datatype is lossy (NUMBER(38,2), NUMERIC and DECIMAL + # all collapse to Decimal), so the catalog's own type name is kept verbatim. + stash["type"] = raw_type + elif solid_expression: + stash["role"] = "metric" + sample_values = column.get("sample_values") + if sample_values: + stash["sample_values"] = sample_values + write_stash(field, stash) + return field + + +def _convert_relationships(relationships, by_source): + converted, taken = [], set() + for relationship in relationships: + if not isinstance(relationship, dict): + raise ConversionError("Each entry of 'relationships' must be a mapping") + left_source = require_str(relationship, "left_table", "relationship") + right_source = require_str(relationship, "right_table", "relationship") + left = by_source.get(left_source.lower()) + right = by_source.get(right_source.lower()) + if left is None or right is None: + missing = left_source if left is None else right_source + warn("relationship", + f"'{left_source}' -> '{right_source}' references table '{missing}', " + f"which is not in 'tables'; the relationship was dropped") + continue + + join_keys = relationship.get("join_keys") + join_keys = join_keys if isinstance(join_keys, dict) else {} + left_columns = string_list(join_keys.get("left")) + right_columns = string_list(join_keys.get("right")) + if not left_columns or not right_columns: + warn("relationship", + f"'{left['name']}' -> '{right['name']}' has no join keys; the " + f"relationship was dropped (an Apache Ossie relationship requires at " + f"least one column on each side)") + continue + if len(left_columns) != len(right_columns): + raise ConversionError( + f"Relationship '{left['name']}' -> '{right['name']}' has " + f"{len(left_columns)} left join key(s) but {len(right_columns)} right; " + f"the columns must correspond positionally" + ) + + converted.append(_orient(left, right, left_columns, right_columns, taken)) + return converted + + +def _orient(left, right, left_columns, right_columns, taken): + """Turn a Solid join into a directed Apache Ossie relationship. + + Apache Ossie encodes cardinality through direction -- `from` is the many side, `to` + the one side -- but a Solid relationship is an undirected pair of column lists. The + one side is therefore recovered from the primary keys: whichever end's primary key + is exactly its join columns is unique on those columns, and so is the one side. + """ + left_is_key = _covers_primary_key(left, left_columns) + right_is_key = _covers_primary_key(right, right_columns) + + note = None + if right_is_key and not left_is_key: + flipped = False + elif left_is_key and not right_is_key: + flipped = True + elif left_is_key and right_is_key: + # Both ends are unique on their join columns: a one-to-one. Apache Ossie has no + # dedicated form, so the direction is arbitrary and recorded in `ai_context`. + flipped = False + note = ONE_TO_ONE_NOTE + else: + flipped = False + warn("relationship", + f"'{left['name']}' -> '{right['name']}': neither side's primary key " + f"matches its join columns, so the many/one direction could not be " + f"determined; assumed '{left['name']}' is the many side. Solid does not " + f"record cardinality.") + + if flipped: + source, target = right, left + source_columns, target_columns = right_columns, left_columns + else: + source, target = left, right + source_columns, target_columns = left_columns, right_columns + + name = unique_name(f"{source['name']}_to_{target['name']}", taken) + relationship = { + "name": name, + "from": source["name"], + "to": target["name"], + "from_columns": source_columns, + "to_columns": target_columns, + } + if note: + relationship["ai_context"] = {"instructions": note} + if flipped: + # Only the orientation needs recording: relationships keep their list position in + # both directions, so the ordering restores itself. Export re-emits + # `left_table`/`right_table` the way Solid wrote them. + write_stash(relationship, {"flipped": True}) + return relationship + + +def _covers_primary_key(dataset, columns): + """True if `columns` is exactly the dataset's primary key (order-insensitive).""" + primary_key = dataset.get("primary_key") + if not primary_key: + return False + return {c.lower() for c in primary_key} == {c.lower() for c in columns} + + +def _convert_metrics(metrics, by_source, datasets, dialect): + dataset_names = [d["name"] for d in datasets] + converted, taken = [], set() + for metric in metrics: + if not isinstance(metric, dict): + raise ConversionError("Each entry of 'metrics' must be a mapping") + raw_name = require_str(metric, "name", "metric") + name = unique_name(raw_name, taken) + if name != raw_name: + warn("metric", f"'{raw_name}' is declared more than once; the duplicate " + f"was renamed to '{name}'") + + expression = clean_text(metric.get("expression")) + if not expression: + warn("metric", f"'{name}' has no expression and was dropped (an Apache " + f"Ossie metric requires one)") + continue + + owners, unknown = [], [] + for source in string_list(metric.get("tables")): + dataset = by_source.get(source.lower()) + (owners if dataset is not None else unknown).append(dataset or source) + for source in unknown: + warn("metric", f"'{name}' references table '{source}', which is not in " + f"'tables'") + + expression, status = _qualify(name, expression, owners, dialect) + + converted_metric = { + "name": name, + "expression": {"dialects": [{"dialect": dialect, "expression": expression}]}, + } + description = clean_text(metric.get("description")) + if description: + converted_metric["description"] = description + synonyms = string_list(metric.get("synonyms")) + if synonyms: + converted_metric["ai_context"] = {"synonyms": synonyms} + + # Solid's `tables` list only needs stashing when the qualified expression does + # not already name the same datasets -- which is the case for an unqualifiable + # expression, and for one that references no column at all (`COUNT(*)`). + expected = [d["name"] for d in owners] + if (unknown + or status in (UNPARSED, AMBIGUOUS) + or referenced_datasets(expression, dialect, dataset_names) != expected): + write_stash(converted_metric, + {"tables": string_list(metric.get("tables"))}) + converted.append(converted_metric) + return converted + + +def _qualify(name, expression, owners, dialect): + """Qualify a Solid metric's bare column references with their dataset name. + + Solid stores formulas against bare columns and records the owning table separately, + so the dataset is unambiguous only when exactly one table owns the metric. A + multi-table metric is left verbatim: solid-server strips alias prefixes when the + formula is saved and keeps the alias-to-table binding in `metric.column_ids`, which + its YAML export does not carry, so the binding cannot be recovered from the file. + """ + if len(owners) != 1: + if len(owners) > 1: + warn("metric", + f"'{name}' spans {len(owners)} tables " + f"({', '.join(d['name'] for d in owners)}); its column references " + f"cannot be attributed to a dataset and were left unqualified") + else: + warn("metric", f"'{name}' names no owning table; its column references " + f"were left unqualified") + return expression, UNCHANGED + + dataset = owners[0] + columns = [f["name"] for f in dataset.get("fields") or []] + qualified, status = qualify_metric(expression, dialect, dataset["name"], columns) + if status == UNPARSED: + warn("metric", f"'{name}': expression could not be parsed as {dialect} SQL and " + f"was left unqualified") + elif status == AMBIGUOUS: + warn("metric", f"'{name}': a column name in the expression also appears in a " + f"non-column position, so the reference could not be qualified " + f"safely; the expression was left as written") + return qualified, status + + +__all__ = ["convert_solid_to_ossie"] diff --git a/converters/solid/tests/conftest.py b/converters/solid/tests/conftest.py new file mode 100644 index 00000000..ffb615b4 --- /dev/null +++ b/converters/solid/tests/conftest.py @@ -0,0 +1,144 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared test helpers: fixture loading, warning capture, and schema validation.""" + +import json +import sys +import warnings +from pathlib import Path + +import pytest +import yaml + +FIXTURES = Path(__file__).parent / "fixtures" + +# The converter is checked against the specification's own JSON Schema, which lives in +# the repository rather than in this package. +SCHEMA_PATH = Path(__file__).parents[3] / "core-spec" / "ossie-schema.json" +EXAMPLES = Path(__file__).parents[3] / "examples" + +# The other converters' own test fixtures, used read-only by the cross-vendor +# interop sweep (see test_cross_vendor.py). +CONVERTERS = Path(__file__).parents[2] + +if str(Path(__file__).parents[1] / "src") not in sys.path: + sys.path.insert(0, str(Path(__file__).parents[1] / "src")) + + +def fixture(name): + """Read a fixture file as text.""" + return (FIXTURES / name).read_text() + + +def example(name): + """Read a model from the repository's `examples/` directory.""" + return (EXAMPLES / name).read_text() + + +def convert_quietly(func, *args, **kwargs): + """Run a converter, returning (output, [warning messages]).""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + output = func(*args, **kwargs) + return output, [str(w.message) for w in caught] + + +def model_of(ossie_yaml): + """The first semantic model in an Apache Ossie document.""" + return yaml.safe_load(ossie_yaml)["semantic_model"][0] + + +def solid_model_of(solid_yaml): + """The semantic model in a Solid document.""" + return yaml.safe_load(solid_yaml)["semantic_model"] + + +def by_name(items): + """Index a list of named mappings by `name`.""" + return {item["name"]: item for item in items or []} + + +def normalized_yaml(text): + """Parse YAML and normalize block-scalar whitespace. + + Solid writes descriptions as block scalars, so the same text can differ between two + renderings only by trailing newlines and line-end whitespace. That is not a + conversion difference, so it is normalized away before comparing. + """ + + def walk(node): + if isinstance(node, dict): + return {k: walk(v) for k, v in node.items()} + if isinstance(node, list): + return [walk(v) for v in node] + if isinstance(node, str): + return "\n".join(line.rstrip() for line in node.strip().splitlines()) + return node + + return walk(yaml.safe_load(text)) + + +def dataset_of(model, name): + return by_name(model["datasets"])[name] + + +def field_of(model, dataset, field): + return by_name(dataset_of(model, dataset).get("fields"))[field] + + +def stash_of(obj): + """The SOLID custom_extensions payload on an object, or {} when absent.""" + for ext in obj.get("custom_extensions") or []: + if ext["vendor_name"] == "SOLID": + data = json.loads(ext["data"]) + data.pop("_v", None) + return data + return {} + + +def expression_of(obj): + """The single dialect expression on a field or metric.""" + dialects = obj["expression"]["dialects"] + assert len(dialects) == 1, "converter emits exactly one dialect per expression" + return dialects[0]["expression"] + + +@pytest.fixture(scope="session") +def ossie_validator(): + """A Draft 2020-12 validator for the Apache Ossie core schema. + + Skips the whole test if `jsonschema` is not installed, so the suite still runs + without the dev extra. + """ + jsonschema = pytest.importorskip("jsonschema") + schema = json.loads(SCHEMA_PATH.read_text()) + return jsonschema.Draft202012Validator(schema) + + +@pytest.fixture +def assert_valid_ossie(ossie_validator): + """Assert that a converted document satisfies the Apache Ossie JSON Schema.""" + + def _assert(ossie_yaml): + document = yaml.safe_load(ossie_yaml) + errors = sorted(ossie_validator.iter_errors(document), key=lambda e: list(e.path)) + assert not errors, "\n".join( + f"{'/'.join(str(p) for p in e.path)}: {e.message}" for e in errors + ) + + return _assert diff --git a/converters/solid/tests/fixtures/bigquery_solid.yaml b/converters/solid/tests/fixtures/bigquery_solid.yaml new file mode 100644 index 00000000..06e2e5fa --- /dev/null +++ b/converters/solid/tests/fixtures/bigquery_solid.yaml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# A small Solid export whose type vocabulary is BigQuery's: INT64, FLOAT64, BOOL. +# The project id carries a hyphen, which a dataset name must not, and the metric uses +# COUNTIF -- a BigQuery-only function that has to survive qualification untouched. +semantic_model: + name: 'web_sessions' + + business_context: + model_description: | + Session-level web analytics. + + model_llm_description: | + Web session analytics: traffic, engagement, and signup conversion. + + tables: + - name: 'analytics-prod.web.sessions' + description: | + One row per web session. + synonyms: + - 'visits' + primary_key: 'session_id' + quality_rank: 'high' + dimensions: + - name: 'session_id' + type: 'STRING' + description: Session identifier. + - name: 'channel' + type: 'STRING' + description: Acquisition channel. + sample_values: + - 'organic' + - 'paid' + - name: 'is_logged_in' + type: 'BOOL' + description: Whether the visitor was authenticated. + - name: 'started_on' + type: 'DATE' + description: Session start date. + - name: 'started_at' + type: 'DATETIME' + description: Session start, with no timezone. + facts: + - name: 'page_views' + type: 'INT64' + description: Pages viewed during the session. + - name: 'engagement_seconds' + type: 'FLOAT64' + description: Seconds of active engagement. + + metrics: + - name: 'LOGGED_IN_SESSIONS' + description: | + Sessions where the visitor was authenticated. + expression: 'COUNTIF(is_logged_in = TRUE)' + tables: + - 'analytics-prod.web.sessions' + - name: 'AVG_ENGAGEMENT' + description: | + Mean active engagement per session. + expression: 'AVG(engagement_seconds)' + tables: + - 'analytics-prod.web.sessions' diff --git a/converters/solid/tests/fixtures/databricks_solid.yaml b/converters/solid/tests/fixtures/databricks_solid.yaml new file mode 100644 index 00000000..4d71b5c5 --- /dev/null +++ b/converters/solid/tests/fixtures/databricks_solid.yaml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# A small Solid export whose type vocabulary is Databricks': LONG, STRING, DOUBLE, MAP. +# Used to check that dialect inference resolves DATABRICKS from types alone, and that +# TIMESTAMP is read as an instant here (unlike Snowflake, where it is not). +semantic_model: + name: 'orders_analytics' + + business_context: + business_questions: + - 'How many orders shipped late last month?' + + model_llm_description: | + Order fulfilment analytics over the warehouse order stream. + + tables: + - name: 'main.sales.orders' + description: | + One row per order. + synonyms: + - 'order stream' + primary_key: 'order_id' + quality_rank: 'high' + dimensions: + - name: 'order_id' + type: 'LONG' + description: Order identifier. + - name: 'customer_id' + type: 'LONG' + description: Customer who placed the order. + - name: 'status' + type: 'STRING' + description: Fulfilment status. + sample_values: + - 'shipped' + - 'pending' + - name: 'placed_at' + type: 'TIMESTAMP' + description: Instant the order was placed. + - name: 'ingested_at' + type: 'TIMESTAMP_NTZ' + description: Pipeline load time, with no timezone. + - name: 'attributes' + type: 'MAP' + description: Free-form order attributes. + facts: + - name: 'order_total' + type: 'DOUBLE' + description: Order total in USD. + - name: 'line_count' + type: 'INT' + description: Number of lines on the order. + + - name: 'main.sales.customers' + description: | + Customer master. + primary_key: 'customer_id' + quality_rank: 'mediocre' + dimensions: + - name: 'customer_id' + type: 'LONG' + description: Customer identifier. + - name: 'region' + type: 'STRING' + description: Sales region. + facts: [] + + metrics: + - name: 'LATE_ORDER_COUNT' + description: | + Orders whose status is still pending. + expression: "COUNT(CASE WHEN status = 'pending' THEN order_id END)" + tables: + - 'main.sales.orders' + - name: 'GROSS_REVENUE' + description: | + Sum of order totals. + expression: 'SUM(order_total)' + synonyms: + - 'revenue' + tables: + - 'main.sales.orders' + + relationships: + - left_table: 'main.sales.orders' + right_table: 'main.sales.customers' + join_keys: + left: + - 'customer_id' + right: + - 'customer_id' diff --git a/converters/solid/tests/fixtures/foreign_ossie.yaml b/converters/solid/tests/fixtures/foreign_ossie.yaml new file mode 100644 index 00000000..314896d7 --- /dev/null +++ b/converters/solid/tests/fixtures/foreign_ossie.yaml @@ -0,0 +1,149 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# A stand-in for an Apache Ossie model produced by SOME OTHER vendor's converter. +# +# Every construct here was observed in the Ossie fixtures the other converters in this +# repository ship (dbt, Databricks, GoodData, Omni, NVIDIA GSF, Orion Belt). This file +# reproduces them under `converters/solid/` so the interop gaps can be pinned exactly +# without this converter's CI depending on another converter's fixtures -- the +# workflows are path-filtered per converter, so a pin against a foreign fixture would +# break on a main-branch push from a PR that never ran this suite. +# +# What makes it "foreign", and what each construct exercises: +# +# - no `custom_extensions[SOLID]` stash anywhere, so every fallback path is used +# - no `dimension` block anywhere, so the fact/dimension split falls back to +# Solid's own data-type rule +# - `datatype` omitted on most fields (it is optional in the spec, and the other +# converters largely do omit it) -> an empty Solid `type` +# - `store_sales.ticket_number`: a field RENAMED relative to its column +# - `date_dim.d_year`: a `label`, which Solid has no slot for +# - `store_sales`: `unique_keys`, which Solid has no slot for +# - `revenue`: a metric written against BARE column names, as a vendor that keeps +# the owning table elsewhere would write it +# - `order_count`: `COUNT(*)`, which references no column at all +# - `revenue`: a metric `datatype` +# - `store_sales_to_date`: a relationship annotation +# - foreign `custom_extensions` at the model, dataset, field and metric levels +version: "0.2.0.dev0" +semantic_model: + - name: foreign_retail_model + description: Retail sales, as another vendor's converter would emit it. + ai_context: + instructions: Prefer net sales over gross for revenue questions. + examples: + - What were total sales last quarter? + custom_extensions: + - vendor_name: DBT + data: '{"package": "jaffle_shop"}' + datasets: + - name: store_sales + source: tpcds.public.store_sales + description: One row per line item on a sales ticket. + primary_key: + - ss_item_sk + - ss_ticket_number + unique_keys: + - - ss_ticket_number + custom_extensions: + - vendor_name: DATABRICKS + data: '{"table_type": "MANAGED"}' + fields: + - name: ss_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_item_sk + - name: ss_ticket_number + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ticket_number + - name: ss_sold_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sold_date_sk + # Renamed: the Solid column that comes out of this is named for the alias, + # not for the column that actually exists in the warehouse. + - name: ticket_number + description: Ticket number, exposed under a friendlier name. + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ticket_number + custom_extensions: + - vendor_name: DATABRICKS + data: '{"comment": "renamed for readability"}' + - name: ss_net_paid + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_net_paid + - name: date_dim + source: tpcds.public.date_dim + primary_key: + - d_date_sk + fields: + - name: d_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: d_date_sk + - name: d_year + label: Year + datatype: Integer + expression: + dialects: + - dialect: ANSI_SQL + expression: d_year + ai_context: + synonyms: + - year + relationships: + - name: store_sales_to_date + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - d_date_sk + ai_context: + instructions: Sales are dated by the sold date, not the ship date. + synonyms: + - sold on + metrics: + # Bare column names: the owning table is not recoverable from a qualifier. + - name: revenue + description: Net sales revenue. + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(ss_net_paid) + custom_extensions: + - vendor_name: DBT + data: '{"agg": "sum"}' + # References no column at all. + - name: order_count + description: Number of tickets. + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(*) diff --git a/converters/solid/tests/fixtures/tpcds_ossie.yaml b/converters/solid/tests/fixtures/tpcds_ossie.yaml new file mode 100644 index 00000000..13978acb --- /dev/null +++ b/converters/solid/tests/fixtures/tpcds_ossie.yaml @@ -0,0 +1,723 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Expected output of `ossie-solid import -i tpcds_solid.yaml`. Regenerate with: +# ossie-solid import -i tests/fixtures/tpcds_solid.yaml -o tests/fixtures/tpcds_ossie.yaml +# then re-add this header. +version: 0.2.0.dev0 +version: 0.2.0.dev0 +semantic_model: +- name: tpcds_retail_model + description: 'TPC-DS retail semantic model for sales and customer analytics. Supports time-based + + analysis, customer segmentation, product performance, and store operations metrics.' + ai_context: + instructions: '- Always join through public.store_sales rather than aggregating a dimension on its + own. + + - Revenue means store_sales.ss_ext_sales_price, not ss_sales_price, which is a per-unit figure. + + - Exclude rows where ss_sold_date_sk is null; those are unshipped orders.' + examples: + - What were total sales and net profit by month for the last four quarters? + - Which brands drove the most revenue in the Electronics category? + - How does sales per employee compare across stores in each state? + datasets: + - name: store_sales + source: tpcds.public.store_sales + primary_key: + - ss_item_sk + - ss_ticket_number + description: Fact table containing all store sales transactions, one row per line item. + ai_context: + instructions: Grain is (ss_item_sk, ss_ticket_number). Loaded nightly from the POS extract. + synonyms: + - sales transactions + - store purchases + - retail sales + - POS data + fields: + - name: ss_sold_date_sk + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_sold_date_sk + description: Foreign key to the date dimension. + datatype: Integer + dimension: + is_time: false + ai_context: + synonyms: + - sale date + - transaction date + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: ss_item_sk + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_item_sk + description: Foreign key to the item dimension. + datatype: Integer + dimension: + is_time: false + ai_context: + synonyms: + - product + - item + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: ss_ticket_number + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_ticket_number + description: Ticket number identifying the sale the line item belongs to. + datatype: Integer + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: ss_customer_sk + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_customer_sk + description: Foreign key to the customer dimension. + datatype: Integer + dimension: + is_time: false + ai_context: + synonyms: + - customer + - buyer + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: ss_store_sk + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_store_sk + description: Foreign key to the store dimension. + datatype: Integer + dimension: + is_time: false + ai_context: + synonyms: + - store + - location + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: ss_sold_at + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_sold_at + description: Wall-clock time the sale was rung up, in the store's local time. + datatype: DateTime + dimension: + is_time: true + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TIMESTAMP_NTZ"}' + - name: ss_quantity + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_quantity + description: Quantity of items sold. + datatype: Integer + ai_context: + synonyms: + - units sold + - quantity + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: ss_sales_price + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_sales_price + description: Sales price per unit. + datatype: Decimal + ai_context: + instructions: Per-unit price. Use ss_ext_sales_price for line revenue. + synonyms: + - unit price + - price + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(7,2)"}' + - name: ss_ext_sales_price + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_ext_sales_price + description: Extended sales price (quantity * price). + datatype: Decimal + ai_context: + synonyms: + - total price + - line total + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(7,2)"}' + - name: ss_net_profit + expression: + dialects: + - dialect: SNOWFLAKE + expression: ss_net_profit + description: Net profit from the sale. + datatype: Decimal + ai_context: + synonyms: + - profit + - margin + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(7,2)"}' + - name: ss_discount_pct + expression: + dialects: + - dialect: SNOWFLAKE + expression: 1 - (ss_sales_price / NULLIF(ss_list_price, 0)) + description: Discount applied to the line, as a fraction of list price. + ai_context: + synonyms: + - discount rate + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "role": "metric"}' + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "quality_rank": "high", "indexes": ["ss_sold_date_sk", "ss_customer_sk"]}' + - name: date_dim + source: tpcds.public.date_dim + primary_key: + - d_date_sk + description: Date dimension with calendar attributes. + ai_context: + synonyms: + - calendar + - dates + - time periods + fields: + - name: d_date_sk + expression: + dialects: + - dialect: SNOWFLAKE + expression: d_date_sk + description: Surrogate key for date. + datatype: Integer + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: d_date + expression: + dialects: + - dialect: SNOWFLAKE + expression: d_date + description: Calendar date. + datatype: Date + dimension: + is_time: true + ai_context: + synonyms: + - date + - day + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "DATE"}' + - name: d_year + expression: + dialects: + - dialect: SNOWFLAKE + expression: d_year + description: Calendar year. + datatype: Integer + dimension: + is_time: false + ai_context: + synonyms: + - year + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)", "sample_values": ["2022", "2023", "2024"]}' + - name: d_quarter_name + expression: + dialects: + - dialect: SNOWFLAKE + expression: d_quarter_name + description: Quarter name, for example 2024Q1. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - quarter + - fiscal quarter + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT", "sample_values": ["2024Q1", "2024Q2"]}' + - name: d_month_name + expression: + dialects: + - dialect: SNOWFLAKE + expression: d_month_name + description: Month name. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - month + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "quality_rank": "high"}' + - name: customer + source: tpcds.public.customer + primary_key: + - c_customer_sk + description: Customer dimension with demographic information. + ai_context: + synonyms: + - customers + - shoppers + - buyers + fields: + - name: c_customer_sk + expression: + dialects: + - dialect: SNOWFLAKE + expression: c_customer_sk + description: Surrogate key for customer. + datatype: Integer + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: c_customer_id + expression: + dialects: + - dialect: SNOWFLAKE + expression: c_customer_id + description: Business key for customer. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - customer ID + - customer number + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: c_first_name + expression: + dialects: + - dialect: SNOWFLAKE + expression: c_first_name + description: Customer first name. + datatype: String + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: c_last_name + expression: + dialects: + - dialect: SNOWFLAKE + expression: c_last_name + description: Customer last name. + datatype: String + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: c_email_address + expression: + dialects: + - dialect: SNOWFLAKE + expression: c_email_address + description: Customer email address. + datatype: String + dimension: + is_time: false + ai_context: + instructions: Nullable. Roughly 12% of rows have no email on file. + synonyms: + - email + - contact + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: c_preferences + expression: + dialects: + - dialect: SNOWFLAKE + expression: c_preferences + description: Semi-structured marketing preference document. + datatype: Opaque + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "VARIANT"}' + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "quality_rank": "mediocre"}' + - name: item + source: tpcds.public.item + primary_key: + - i_item_sk + description: Item dimension with product attributes. + ai_context: + synonyms: + - products + - items + - merchandise + fields: + - name: i_item_sk + expression: + dialects: + - dialect: SNOWFLAKE + expression: i_item_sk + description: Surrogate key for item. + datatype: Integer + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: i_item_id + expression: + dialects: + - dialect: SNOWFLAKE + expression: i_item_id + description: Business key for item. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - item ID + - product ID + - SKU + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: i_item_desc + expression: + dialects: + - dialect: SNOWFLAKE + expression: i_item_desc + description: Item description. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - product description + - item name + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: i_brand + expression: + dialects: + - dialect: SNOWFLAKE + expression: i_brand + description: Brand name. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - brand + - manufacturer + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: i_category + expression: + dialects: + - dialect: SNOWFLAKE + expression: i_category + description: Item category. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - product category + - department + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT", "sample_values": ["Electronics", "Home", "Sports"]}' + - name: i_current_price + expression: + dialects: + - dialect: SNOWFLAKE + expression: i_current_price + description: Current price of the item. + datatype: Decimal + ai_context: + synonyms: + - price + - list price + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(7,2)"}' + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "quality_rank": "high"}' + - name: store + source: tpcds.public.store + primary_key: + - s_store_sk + description: Store dimension with location and store attributes. + ai_context: + synonyms: + - stores + - retail locations + - branches + fields: + - name: s_store_sk + expression: + dialects: + - dialect: SNOWFLAKE + expression: s_store_sk + description: Surrogate key for store. + datatype: Integer + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + - name: s_store_id + expression: + dialects: + - dialect: SNOWFLAKE + expression: s_store_id + description: Business key for store. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - store ID + - store number + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: s_store_name + expression: + dialects: + - dialect: SNOWFLAKE + expression: s_store_name + description: Store name. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - store name + - location name + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: s_city + expression: + dialects: + - dialect: SNOWFLAKE + expression: s_city + description: City where the store is located. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - city + - location + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT"}' + - name: s_state + expression: + dialects: + - dialect: SNOWFLAKE + expression: s_state + description: State where the store is located. + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - state + - region + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "TEXT", "sample_values": ["CA", "NY", "TX"]}' + - name: s_is_franchise + expression: + dialects: + - dialect: SNOWFLAKE + expression: s_is_franchise + description: Whether the location is franchise-operated. + datatype: Boolean + dimension: + is_time: false + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "BOOLEAN"}' + - name: s_number_employees + expression: + dialects: + - dialect: SNOWFLAKE + expression: s_number_employees + description: Number of employees at the store. + datatype: Integer + ai_context: + synonyms: + - employee count + - staff size + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "type": "NUMBER(38,0)"}' + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "quality_rank": "high"}' + relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - d_date_sk + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "flipped": true}' + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + - name: store_sales_to_item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + - name: store_sales_to_store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk + metrics: + - name: TOTAL_SALES + expression: + dialects: + - dialect: SNOWFLAKE + expression: SUM(store_sales.ss_ext_sales_price) + description: Total sales revenue across all transactions. + ai_context: + synonyms: + - total revenue + - gross sales + - sales amount + - name: TOTAL_PROFIT + expression: + dialects: + - dialect: SNOWFLAKE + expression: SUM(store_sales.ss_net_profit) + description: Total net profit from store sales. + ai_context: + synonyms: + - net profit + - total earnings + - name: TRANSACTION_COUNT + expression: + dialects: + - dialect: SNOWFLAKE + expression: COUNT(DISTINCT store_sales.ss_ticket_number) + description: Number of distinct sales tickets. + - name: ROW_COUNT + expression: + dialects: + - dialect: SNOWFLAKE + expression: COUNT(*) + description: Row count, used as a denominator in coverage checks. + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "tables": ["tpcds.public.store_sales"]}' + - name: CUSTOMER_LIFETIME_VALUE + expression: + dialects: + - dialect: SNOWFLAKE + expression: SUM(ss_ext_sales_price) / COUNT(DISTINCT c_customer_sk) + description: 'Average lifetime sales value per customer. Spans two tables, so Solid stores the + + formula with bare column names and resolves them at query time.' + ai_context: + synonyms: + - CLV + - LTV + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "tables": ["tpcds.public.store_sales", "tpcds.public.customer"]}' + custom_extensions: + - vendor_name: SOLID + data: '{"_v": 1, "dialect": "SNOWFLAKE", "model_llm_description": "TPC-DS retail semantic model for + sales and customer analytics. Supports time-based\nanalysis, customer segmentation, product performance, + and store operations metrics.", "model_description": "Retail analytics for the TPC-DS benchmark + schema: store sales transactions joined\nto their date, customer, item, and store dimensions.", + "custom_instructions": "- Always join through @ rather than aggregating a dimension on its own.\n- Revenue + means @, + not ss_sales_price, which is a per-unit figure.\n- Exclude rows where ss_sold_date_sk is null; those + are unshipped orders.", "example_queries": [{"name": "Monthly revenue and profit", "description": + "Revenue and profit by calendar month for the trailing twelve months.\n", "sql": "SELECT d.d_year,\n d.d_month_name,\n SUM(ss.ss_ext_sales_price) + AS revenue,\n SUM(ss.ss_net_profit) AS profit\nFROM tpcds.public.store_sales ss\nJOIN + tpcds.public.date_dim d ON d.d_date_sk = ss.ss_sold_date_sk\nWHERE d.d_date >= DATEADD(month, -12, + CURRENT_DATE())\nGROUP BY 1, 2\nORDER BY 1, 2\n"}, {"name": "Sales per employee by state", "description": + "Store productivity, ranked within each state.\n", "sql": "SELECT s.s_state,\n s.s_store_name,\n SUM(ss.ss_ext_sales_price) + / NULLIF(MAX(s.s_number_employees), 0) AS sales_per_employee\nFROM tpcds.public.store_sales ss\nJOIN + tpcds.public.store s ON s.s_store_sk = ss.ss_store_sk\nGROUP BY 1, 2\nORDER BY 1, 3 DESC\n"}], "benchmark_questions": + [{"question": "What was total revenue in 2024Q1?", "is_enabled": true, "expected_sql": "SELECT SUM(ss.ss_ext_sales_price) + AS revenue\nFROM tpcds.public.store_sales ss\nJOIN tpcds.public.date_dim d ON d.d_date_sk = ss.ss_sold_date_sk\nWHERE + d.d_quarter_name = ''2024Q1''\n"}, {"question": "Which brand had the highest revenue in the Electronics + category?", "is_enabled": false}]}' diff --git a/converters/solid/tests/fixtures/tpcds_solid.yaml b/converters/solid/tests/fixtures/tpcds_solid.yaml new file mode 100644 index 00000000..ea8168c7 --- /dev/null +++ b/converters/solid/tests/fixtures/tpcds_solid.yaml @@ -0,0 +1,424 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The TPC-DS retail model from ../../../examples/tpcds_semantic_model.yaml, expressed +# as a Solid semantic model export. Types use Snowflake's vocabulary, which is also what +# the converter's dialect inference reads to recover SNOWFLAKE from this file. +# +# Deliberately exercises the Solid constructs that have no Apache Ossie core equivalent +# (quality_rank, indexes, sample_values, example_queries, benchmark_questions, +# manual_description, asset-link markup) and the ones that need reshaping (a composite +# primary key written as one comma-joined scalar, a relationship whose primary key sits +# on the left, an expression-only fact, and both single- and multi-table metrics). +semantic_model: + name: 'tpcds_retail_model' + + business_context: + business_questions: + - 'What were total sales and net profit by month for the last four quarters?' + - 'Which brands drove the most revenue in the Electronics category?' + - 'How does sales per employee compare across stores in each state?' + model_description: | + Retail analytics for the TPC-DS benchmark schema: store sales transactions joined + to their date, customer, item, and store dimensions. + custom_instructions: | + - Always join through @ rather than aggregating a dimension on its own. + - Revenue means @, not ss_sales_price, which is a per-unit figure. + - Exclude rows where ss_sold_date_sk is null; those are unshipped orders. + + model_llm_description: | + TPC-DS retail semantic model for sales and customer analytics. Supports time-based + analysis, customer segmentation, product performance, and store operations metrics. + + tables: + - name: 'tpcds.public.store_sales' + description: | + Fact table containing all store sales transactions, one row per line item. + manual_description: | + Grain is (ss_item_sk, ss_ticket_number). Loaded nightly from the POS extract. + synonyms: + - 'sales transactions' + - 'store purchases' + - 'retail sales' + - 'POS data' + primary_key: 'ss_item_sk, ss_ticket_number' + quality_rank: 'high' + indexes: + - 'ss_sold_date_sk' + - 'ss_customer_sk' + dimensions: + - name: 'ss_sold_date_sk' + type: 'NUMBER(38,0)' + description: | + Foreign key to the date dimension. + synonyms: + - 'sale date' + - 'transaction date' + - name: 'ss_item_sk' + type: 'NUMBER(38,0)' + description: | + Foreign key to the item dimension. + synonyms: + - 'product' + - 'item' + - name: 'ss_ticket_number' + type: 'NUMBER(38,0)' + description: | + Ticket number identifying the sale the line item belongs to. + - name: 'ss_customer_sk' + type: 'NUMBER(38,0)' + description: | + Foreign key to the customer dimension. + synonyms: + - 'customer' + - 'buyer' + - name: 'ss_store_sk' + type: 'NUMBER(38,0)' + description: | + Foreign key to the store dimension. + synonyms: + - 'store' + - 'location' + - name: 'ss_sold_at' + type: 'TIMESTAMP_NTZ' + description: | + Wall-clock time the sale was rung up, in the store's local time. + facts: + - name: 'ss_quantity' + type: 'NUMBER(38,0)' + description: Quantity of items sold. + synonyms: + - 'units sold' + - 'quantity' + - name: 'ss_sales_price' + type: 'NUMBER(7,2)' + description: Sales price per unit. + manual_description: Per-unit price. Use ss_ext_sales_price for line revenue. + synonyms: + - 'unit price' + - 'price' + - name: 'ss_ext_sales_price' + type: 'NUMBER(7,2)' + description: Extended sales price (quantity * price). + synonyms: + - 'total price' + - 'line total' + - name: 'ss_net_profit' + type: 'NUMBER(7,2)' + description: Net profit from the sale. + synonyms: + - 'profit' + - 'margin' + - name: 'ss_discount_pct' + description: Discount applied to the line, as a fraction of list price. + expression: '1 - (ss_sales_price / NULLIF(ss_list_price, 0))' + synonyms: + - 'discount rate' + + - name: 'tpcds.public.date_dim' + description: | + Date dimension with calendar attributes. + synonyms: + - 'calendar' + - 'dates' + - 'time periods' + primary_key: 'd_date_sk' + quality_rank: 'high' + dimensions: + - name: 'd_date_sk' + type: 'NUMBER(38,0)' + description: Surrogate key for date. + - name: 'd_date' + type: 'DATE' + description: Calendar date. + synonyms: + - 'date' + - 'day' + - name: 'd_year' + type: 'NUMBER(38,0)' + description: Calendar year. + synonyms: + - 'year' + sample_values: + - '2022' + - '2023' + - '2024' + - name: 'd_quarter_name' + type: 'TEXT' + description: Quarter name, for example 2024Q1. + synonyms: + - 'quarter' + - 'fiscal quarter' + sample_values: + - '2024Q1' + - '2024Q2' + - name: 'd_month_name' + type: 'TEXT' + description: Month name. + synonyms: + - 'month' + facts: [] + + - name: 'tpcds.public.customer' + description: | + Customer dimension with demographic information. + synonyms: + - 'customers' + - 'shoppers' + - 'buyers' + primary_key: 'c_customer_sk' + quality_rank: 'mediocre' + dimensions: + - name: 'c_customer_sk' + type: 'NUMBER(38,0)' + description: Surrogate key for customer. + - name: 'c_customer_id' + type: 'TEXT' + description: Business key for customer. + synonyms: + - 'customer ID' + - 'customer number' + - name: 'c_first_name' + type: 'TEXT' + description: Customer first name. + - name: 'c_last_name' + type: 'TEXT' + description: Customer last name. + - name: 'c_email_address' + type: 'TEXT' + description: Customer email address. + manual_description: Nullable. Roughly 12% of rows have no email on file. + synonyms: + - 'email' + - 'contact' + - name: 'c_preferences' + type: 'VARIANT' + description: Semi-structured marketing preference document. + facts: [] + + - name: 'tpcds.public.item' + description: | + Item dimension with product attributes. + synonyms: + - 'products' + - 'items' + - 'merchandise' + primary_key: 'i_item_sk' + quality_rank: 'high' + dimensions: + - name: 'i_item_sk' + type: 'NUMBER(38,0)' + description: Surrogate key for item. + - name: 'i_item_id' + type: 'TEXT' + description: Business key for item. + synonyms: + - 'item ID' + - 'product ID' + - 'SKU' + - name: 'i_item_desc' + type: 'TEXT' + description: Item description. + synonyms: + - 'product description' + - 'item name' + - name: 'i_brand' + type: 'TEXT' + description: Brand name. + synonyms: + - 'brand' + - 'manufacturer' + - name: 'i_category' + type: 'TEXT' + description: Item category. + synonyms: + - 'product category' + - 'department' + sample_values: + - 'Electronics' + - 'Home' + - 'Sports' + facts: + - name: 'i_current_price' + type: 'NUMBER(7,2)' + description: Current price of the item. + synonyms: + - 'price' + - 'list price' + + - name: 'tpcds.public.store' + description: | + Store dimension with location and store attributes. + synonyms: + - 'stores' + - 'retail locations' + - 'branches' + primary_key: 's_store_sk' + quality_rank: 'high' + dimensions: + - name: 's_store_sk' + type: 'NUMBER(38,0)' + description: Surrogate key for store. + - name: 's_store_id' + type: 'TEXT' + description: Business key for store. + synonyms: + - 'store ID' + - 'store number' + - name: 's_store_name' + type: 'TEXT' + description: Store name. + synonyms: + - 'store name' + - 'location name' + - name: 's_city' + type: 'TEXT' + description: City where the store is located. + synonyms: + - 'city' + - 'location' + - name: 's_state' + type: 'TEXT' + description: State where the store is located. + synonyms: + - 'state' + - 'region' + sample_values: + - 'CA' + - 'NY' + - 'TX' + - name: 's_is_franchise' + type: 'BOOLEAN' + description: Whether the location is franchise-operated. + facts: + - name: 's_number_employees' + type: 'NUMBER(38,0)' + description: Number of employees at the store. + synonyms: + - 'employee count' + - 'staff size' + + metrics: + - name: 'TOTAL_SALES' + description: | + Total sales revenue across all transactions. + expression: 'SUM(ss_ext_sales_price)' + synonyms: + - 'total revenue' + - 'gross sales' + - 'sales amount' + tables: + - 'tpcds.public.store_sales' + - name: 'TOTAL_PROFIT' + description: | + Total net profit from store sales. + expression: 'SUM(ss_net_profit)' + synonyms: + - 'net profit' + - 'total earnings' + tables: + - 'tpcds.public.store_sales' + - name: 'TRANSACTION_COUNT' + description: | + Number of distinct sales tickets. + expression: 'COUNT(DISTINCT ss_ticket_number)' + tables: + - 'tpcds.public.store_sales' + - name: 'ROW_COUNT' + description: | + Row count, used as a denominator in coverage checks. + expression: 'COUNT(*)' + tables: + - 'tpcds.public.store_sales' + - name: 'CUSTOMER_LIFETIME_VALUE' + description: | + Average lifetime sales value per customer. Spans two tables, so Solid stores the + formula with bare column names and resolves them at query time. + expression: 'SUM(ss_ext_sales_price) / COUNT(DISTINCT c_customer_sk)' + synonyms: + - 'CLV' + - 'LTV' + tables: + - 'tpcds.public.store_sales' + - 'tpcds.public.customer' + + relationships: + - left_table: 'tpcds.public.date_dim' + right_table: 'tpcds.public.store_sales' + join_keys: + left: + - 'd_date_sk' + right: + - 'ss_sold_date_sk' + - left_table: 'tpcds.public.store_sales' + right_table: 'tpcds.public.customer' + join_keys: + left: + - 'ss_customer_sk' + right: + - 'c_customer_sk' + - left_table: 'tpcds.public.store_sales' + right_table: 'tpcds.public.item' + join_keys: + left: + - 'ss_item_sk' + right: + - 'i_item_sk' + - left_table: 'tpcds.public.store_sales' + right_table: 'tpcds.public.store' + join_keys: + left: + - 'ss_store_sk' + right: + - 's_store_sk' + + example_queries: + - name: 'Monthly revenue and profit' + description: | + Revenue and profit by calendar month for the trailing twelve months. + sql: | + SELECT d.d_year, + d.d_month_name, + SUM(ss.ss_ext_sales_price) AS revenue, + SUM(ss.ss_net_profit) AS profit + FROM tpcds.public.store_sales ss + JOIN tpcds.public.date_dim d ON d.d_date_sk = ss.ss_sold_date_sk + WHERE d.d_date >= DATEADD(month, -12, CURRENT_DATE()) + GROUP BY 1, 2 + ORDER BY 1, 2 + - name: 'Sales per employee by state' + description: | + Store productivity, ranked within each state. + sql: | + SELECT s.s_state, + s.s_store_name, + SUM(ss.ss_ext_sales_price) / NULLIF(MAX(s.s_number_employees), 0) AS sales_per_employee + FROM tpcds.public.store_sales ss + JOIN tpcds.public.store s ON s.s_store_sk = ss.ss_store_sk + GROUP BY 1, 2 + ORDER BY 1, 3 DESC + + benchmark_questions: + - question: 'What was total revenue in 2024Q1?' + is_enabled: true + expected_sql: | + SELECT SUM(ss.ss_ext_sales_price) AS revenue + FROM tpcds.public.store_sales ss + JOIN tpcds.public.date_dim d ON d.d_date_sk = ss.ss_sold_date_sk + WHERE d.d_quarter_name = '2024Q1' + - question: 'Which brand had the highest revenue in the Electronics category?' + is_enabled: false diff --git a/converters/solid/tests/test_cli.py b/converters/solid/tests/test_cli.py new file mode 100644 index 00000000..53a25c1a --- /dev/null +++ b/converters/solid/tests/test_cli.py @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The `ossie-solid` command line interface.""" + +import pytest +import yaml +from conftest import FIXTURES, fixture, normalized_yaml + +from ossie_solid.cli import main + + +def test_import_writes_an_ossie_model_to_a_file(tmp_path): + out = tmp_path / "model.yaml" + assert main(["import", "-i", str(FIXTURES / "tpcds_solid.yaml"), "-o", str(out)]) == 0 + assert yaml.safe_load(out.read_text())["version"] == "0.2.0.dev0" + + +def test_import_writes_to_stdout_without_an_output_path(capsys): + assert main(["import", "-i", str(FIXTURES / "databricks_solid.yaml")]) == 0 + assert yaml.safe_load(capsys.readouterr().out)["semantic_model"][0]["name"] == ( + "orders_analytics" + ) + + +def test_export_writes_a_solid_model_to_a_file(tmp_path): + out = tmp_path / "solid.yaml" + assert main(["export", "-i", str(FIXTURES / "tpcds_ossie.yaml"), "-o", str(out)]) == 0 + assert yaml.safe_load(out.read_text())["semantic_model"]["name"] == ( + "tpcds_retail_model" + ) + + +def test_a_round_trip_through_the_cli_matches_the_input(tmp_path, capsys): + ossie = tmp_path / "model.yaml" + solid = tmp_path / "solid.yaml" + main(["import", "-i", str(FIXTURES / "databricks_solid.yaml"), "-o", str(ossie)]) + main(["export", "-i", str(ossie), "-o", str(solid)]) + assert normalized_yaml(solid.read_text()) == normalized_yaml( + fixture("databricks_solid.yaml") + ) + + +def test_warnings_go_to_stderr_as_plain_lines(capsys): + main(["import", "-i", str(FIXTURES / "tpcds_solid.yaml")]) + err = capsys.readouterr().err + assert err.startswith("Warning: [metric]") + assert "Traceback" not in err + + +def test_a_conversion_error_exits_non_zero_with_a_message(capsys, tmp_path): + bad = tmp_path / "bad.yaml" + bad.write_text("semantic_model: [unclosed\n") + assert main(["import", "-i", str(bad)]) == 1 + assert capsys.readouterr().err.startswith("Error: Invalid YAML") + + +def test_a_missing_input_file_exits_non_zero_with_a_message(capsys): + assert main(["import", "-i", "/nonexistent/model.yaml"]) == 1 + assert "Error:" in capsys.readouterr().err + + +def test_the_dialect_flag_is_honoured(tmp_path): + out = tmp_path / "model.yaml" + main([ + "import", + "-i", str(FIXTURES / "databricks_solid.yaml"), + "-o", str(out), + "--dialect", "ANSI_SQL", + ]) + model = yaml.safe_load(out.read_text())["semantic_model"][0] + assert model["datasets"][0]["fields"][0]["expression"]["dialects"][0]["dialect"] == ( + "ANSI_SQL" + ) + + +def test_an_unknown_dialect_is_rejected_by_the_parser(): + with pytest.raises(SystemExit): + main(["import", "-i", str(FIXTURES / "tpcds_solid.yaml"), "--dialect", "ORACLE"]) + + +def test_the_name_flag_renames_the_model(tmp_path): + out = tmp_path / "model.yaml" + main([ + "import", + "-i", str(FIXTURES / "tpcds_solid.yaml"), + "-o", str(out), + "--name", "renamed_model", + ]) + assert yaml.safe_load(out.read_text())["semantic_model"][0]["name"] == ( + "renamed_model" + ) + + +def test_a_missing_subcommand_is_rejected(): + with pytest.raises(SystemExit): + main([]) diff --git a/converters/solid/tests/test_cross_vendor.py b/converters/solid/tests/test_cross_vendor.py new file mode 100644 index 00000000..d5704b87 --- /dev/null +++ b/converters/solid/tests/test_cross_vendor.py @@ -0,0 +1,246 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Interop with Apache Ossie models this converter did not produce. + +Import (Solid -> Apache Ossie) is covered by round-trip equality, because both ends of +that trip are Solid's own format. Export (Apache Ossie -> Solid) has no such anchor: the +models it will really be handed come from *other vendors' converters*, and those look +nothing like the ones this converter emits -- no SOLID stash, no `dimension` blocks, +usually no `datatype`, metrics written against bare column names. + +Two complementary passes cover that: + +1. A **sweep** over the Ossie fixtures the other converters in this repository ship. + It asserts only the robust contract -- every model converts, and the result is a + well-formed Solid model that re-imports schema-valid. It deliberately pins no + warning counts: those fixtures belong to other converters, and each converter's CI + is path-filtered to its own directory, so a pin here would break on a main-branch + push from a PR that never ran this suite. + +2. Exact pins against `foreign_ossie.yaml`, a fixture *this* converter owns that + reproduces the same constructs. That is where the interop gaps are recorded + precisely, so a change in any of them shows up as a diff in this file. +""" + +import json + +import pytest +import yaml +from conftest import ( + CONVERTERS, + convert_quietly, + fixture, + model_of, + solid_model_of, +) + +from ossie_solid import convert_ossie_to_solid, convert_solid_to_ossie + + +def _foreign_ossie_fixtures(): + """Every Apache Ossie document among the other converters' fixtures. + + Discovered rather than listed, so a fixture that is renamed or removed upstream + drops out of the sweep instead of failing it. + """ + found = [] + for path in sorted(CONVERTERS.glob("*/tests/fixtures/*.yaml")): + if path.parts[-4] == "solid": + continue + try: + document = yaml.safe_load(path.read_text()) + except yaml.YAMLError: + continue + # An Apache Ossie document keys `semantic_model` to a list; Solid's and every + # other vendor's native format does not. + if isinstance(document, dict) and isinstance( + document.get("semantic_model"), list + ): + found.append(path) + return found + + +FOREIGN = _foreign_ossie_fixtures() + + +def _fixture_id(path): + return f"{path.parts[-4]}/{path.name}" + + +@pytest.mark.skipif(not FOREIGN, reason="no other converters ship Ossie fixtures") +@pytest.mark.parametrize("path", FOREIGN, ids=_fixture_id) +def test_another_vendors_model_converts_to_a_well_formed_solid_model(path): + """The bar is a Solid model Solid could actually read, not a warning-free one. + + Losses are expected here and are reported as warnings; what must not happen is a + hard failure or a structurally invalid Solid document. + """ + solid, _ = convert_quietly(convert_ossie_to_solid, path.read_text()) + model = solid_model_of(solid) + + assert model["name"] + assert model["tables"], "a Solid model needs at least one table" + for table in model["tables"]: + assert table["name"], "every Solid table needs its fully-qualified name" + # Solid always emits `facts`, as [] when a table has none. + assert "facts" in table + for column in (table.get("dimensions") or []) + table["facts"]: + assert column["name"] + for metric in model.get("metrics") or []: + assert metric["name"] and metric["expression"] + for relationship in model.get("relationships") or []: + assert relationship["left_table"] and relationship["right_table"] + keys = relationship["join_keys"] + assert len(keys["left"]) == len(keys["right"]) + + +@pytest.mark.skipif(not FOREIGN, reason="no other converters ship Ossie fixtures") +@pytest.mark.parametrize("path", FOREIGN, ids=_fixture_id) +def test_reimporting_another_vendors_model_is_schema_valid(path, assert_valid_ossie): + """The full hop: their Apache Ossie -> Solid -> Apache Ossie, still valid.""" + solid, _ = convert_quietly(convert_ossie_to_solid, path.read_text()) + ossie, _ = convert_quietly(convert_solid_to_ossie, solid, dialect="ANSI_SQL") + assert_valid_ossie(ossie) + + +@pytest.mark.skipif(not FOREIGN, reason="no other converters ship Ossie fixtures") +def test_the_sweep_actually_found_the_other_converters(): + """Guard against the discovery silently matching nothing and the sweep passing. + + A lower bound, not an exact count, so a converter added or removed upstream does + not fail this suite. + """ + vendors = {path.parts[-4] for path in FOREIGN} + assert len(vendors) >= 3, f"only found fixtures for {vendors}" + + +# --- exact pins, against a fixture this converter owns ----------------------------- + + +@pytest.fixture(scope="module") +def foreign(): + solid, warnings_ = convert_quietly( + convert_ossie_to_solid, fixture("foreign_ossie.yaml") + ) + return solid_model_of(solid), warnings_ + + +def test_a_foreign_model_converts_to_the_expected_solid_shape(foreign): + model, _ = foreign + assert model["name"] == "foreign_retail_model" + assert [t["name"] for t in model["tables"]] == [ + "tpcds.public.store_sales", + "tpcds.public.date_dim", + ] + # No `dimension` block anywhere, so the split falls back to Solid's type rule: + # a numeric type is a fact, everything else a dimension. With no datatype at all, + # a column cannot be shown to be numeric and so lands in `dimensions`. + store_sales = model["tables"][0] + assert [c["name"] for c in store_sales["facts"]] == ["ss_net_paid"] + assert [c["name"] for c in store_sales["dimensions"]] == [ + "ss_item_sk", + "ss_ticket_number", + "ss_sold_date_sk", + "ticket_number", + ] + assert store_sales["primary_key"] == "ss_item_sk, ss_ticket_number" + + +def test_the_documented_interop_gaps_are_exactly_these(foreign): + """The record of what is lost bringing another vendor's model into Solid. + + Each line is a known gap, not a defect to be fixed here. Widening or narrowing any + of them should be a visible diff in this test. + """ + _, warnings_ = foreign + counts = { + # No `datatype` -> no Solid `type`. The single largest gap: Solid's columns are + # typed from the warehouse catalog, and an offline converter has no catalog. + "has no datatype": 5, + # A field renamed relative to its column keeps the alias and loses the column + # it actually reads, because a Solid column IS a catalog column. + "is a computed field": 1, + # Solid has no slot for any of these. + "label 'Year' has no Solid equivalent": 1, + "unique_keys have no Solid equivalent": 1, + "datatype 'Decimal' has no Solid equivalent": 1, + "ai_context.instructions, ai_context.synonyms have no Solid equivalent": 1, + "custom_extensions for": 4, + # Solid resolves a metric's columns through `tables`, and a metric written + # against bare column names carries nothing to resolve them from. + "names no table": 2, + } + for text, expected in counts.items(): + assert sum(text in w for w in warnings_) == expected, text + assert len(warnings_) == sum(counts.values()) == 16 + + +def test_a_renamed_field_loses_the_column_it_reads(foreign): + """Pinned because it is the sharpest gap: the output names a column that is not + in the warehouse. `ticket_number` reads `ss_ticket_number`, but Solid columns map + to catalog columns, so only the alias survives.""" + model, warnings_ = foreign + renamed = {c["name"] for c in model["tables"][0]["dimensions"]} + assert "ticket_number" in renamed + assert any( + "ticket_number' is a computed field (`ss_ticket_number`)" in w + for w in warnings_ + ) + + +def test_a_bare_column_metric_reaches_solid_with_no_tables(foreign): + """Also pinned as a gap: Solid needs `tables` to resolve a formula's columns. + + Only a dataset-qualified reference identifies its owner, and another vendor has no + reason to qualify -- Apache Ossie's own TPC-DS example does, which is why this does + not show up against it. + """ + model, _ = foreign + assert {m["name"]: m["tables"] for m in model["metrics"]} == { + "revenue": [], + "order_count": [], + } + + +def test_a_foreign_model_survives_the_round_trip_back_to_ossie(assert_valid_ossie): + solid, _ = convert_quietly(convert_ossie_to_solid, fixture("foreign_ossie.yaml")) + ossie, _ = convert_quietly(convert_solid_to_ossie, solid, dialect="ANSI_SQL") + assert_valid_ossie(ossie) + + original = model_of(fixture("foreign_ossie.yaml")) + restored = model_of(ossie) + assert restored["name"] == original["name"] + assert [d["source"] for d in restored["datasets"]] == [ + d["source"] for d in original["datasets"] + ] + # The relationship survives with its direction intact: `store_sales` joins + # `date_dim` on the latter's primary key, so it is the many side. + edge = restored["relationships"][0] + assert (edge["from"], edge["to"]) == ("store_sales", "date_dim") + + +def test_the_dialect_recorded_on_a_reimported_foreign_model_is_explicit(): + """A foreign model has no stash, so the re-import records what it resolved to.""" + solid, _ = convert_quietly(convert_ossie_to_solid, fixture("foreign_ossie.yaml")) + ossie, _ = convert_quietly(convert_solid_to_ossie, solid, dialect="ANSI_SQL") + stash = next( + json.loads(ext["data"]) + for ext in model_of(ossie)["custom_extensions"] + if ext["vendor_name"] == "SOLID" + ) + assert stash["dialect"] == "ANSI_SQL" diff --git a/converters/solid/tests/test_datatypes.py b/converters/solid/tests/test_datatypes.py new file mode 100644 index 00000000..07a04ee6 --- /dev/null +++ b/converters/solid/tests/test_datatypes.py @@ -0,0 +1,212 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Warehouse type vocabulary: dialect inference and datatype mapping.""" + +import pytest +import yaml +from conftest import convert_quietly, field_of, fixture, model_of, stash_of + +from ossie_solid import ConversionError, convert_solid_to_ossie +from ossie_solid.datatypes import ( + infer_dialect, + is_solid_fact_type, + normalize_dialect, + parse_type, + to_ossie_datatype, + to_raw_type, +) + + +@pytest.mark.parametrize( + ("raw", "base", "params"), + [ + ("NUMBER(38,0)", "NUMBER", ["38", "0"]), + ("number(7, 2)", "NUMBER", ["7", "2"]), + ("TEXT", "TEXT", []), + ("VARCHAR(16777216)", "VARCHAR", ["16777216"]), + ("ARRAY", "ARRAY", []), + ("MAP", "MAP", []), + ("TIMESTAMP WITH TIME ZONE", "TIMESTAMP WITH TIME ZONE", []), + ("", None, []), + (None, None, []), + ], +) +def test_a_raw_type_splits_into_a_base_name_and_parameters(raw, base, params): + assert parse_type(raw) == (base, params) + + +@pytest.mark.parametrize( + ("types", "expected"), + [ + (["NUMBER(38,0)", "TEXT", "TIMESTAMP_NTZ"], "SNOWFLAKE"), + (["LONG", "STRING", "MAP"], "DATABRICKS"), + (["INT64", "STRING", "BOOL", "FLOAT64"], "BIGQUERY"), + # A STRUCT column decides nothing on its own, but must not stop the + # surrounding vocabulary from deciding. + (["STRUCT", "LONG", "STRING"], "DATABRICKS"), + (["STRUCT", "RECORD", "INT64"], "BIGQUERY"), + # Names every warehouse shares carry no signal. + (["STRING", "DATE", "BOOLEAN", "DECIMAL"], "ANSI_SQL"), + ([], "ANSI_SQL"), + # A tie between two vocabularies is not a decision. + (["NUMBER", "INT64"], "ANSI_SQL"), + ], +) +def test_the_warehouse_is_inferred_from_its_distinctive_type_names(types, expected): + dialect, confident = infer_dialect(types) + assert dialect == expected + assert confident is (expected != "ANSI_SQL") + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("tpcds_solid.yaml", "SNOWFLAKE"), + ("databricks_solid.yaml", "DATABRICKS"), + ("bigquery_solid.yaml", "BIGQUERY"), + ], +) +def test_each_fixture_resolves_to_its_own_warehouse(name, expected): + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture(name)) + assert stash_of(model_of(ossie))["dialect"] == expected + + +@pytest.mark.parametrize("struct", ["STRUCT", "STRUCT"]) +def test_a_struct_column_alone_does_not_identify_a_warehouse(struct): + """STRUCT is Databricks and BigQuery both, and the angle-bracket params that + would tell them apart are stripped by `parse_type` before the vote is counted -- + so it belongs to neither marker set. Regression: it was once listed as + `STRUCT<>` under Databricks, a string `parse_type` can never return, which left + BigQuery holding the only STRUCT marker and a Databricks model claiming BIGQUERY. + """ + assert infer_dialect([struct, "STRING", "DATE"]) == ("ANSI_SQL", False) + + +def test_an_unrecognizable_vocabulary_falls_back_to_ansi_with_a_warning(): + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + for table in solid["semantic_model"]["tables"]: + for column in table["dimensions"] + table["facts"]: + column["type"] = "STRING" + ossie, warnings_ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + assert any("could not infer the source warehouse" in w for w in warnings_) + assert stash_of(model_of(ossie))["dialect"] == "ANSI_SQL" + + +def test_an_explicit_dialect_suppresses_inference(): + ossie, warnings_ = convert_quietly( + convert_solid_to_ossie, fixture("databricks_solid.yaml"), dialect="ANSI_SQL" + ) + assert stash_of(model_of(ossie))["dialect"] == "ANSI_SQL" + assert not any("could not infer" in w for w in warnings_) + + +def test_an_unsupported_dialect_is_rejected(): + with pytest.raises(ConversionError, match="Unsupported dialect"): + convert_solid_to_ossie(fixture("databricks_solid.yaml"), dialect="POSTGRES") + + +def test_dialect_names_are_normalized_case_insensitively(): + assert normalize_dialect("snowflake") == "SNOWFLAKE" + + +@pytest.mark.parametrize( + ("raw", "dialect", "expected"), + [ + # A declared scale of zero is an integer, which is how Snowflake stores ids. + ("NUMBER(38,0)", "SNOWFLAKE", "Integer"), + ("NUMBER(7,2)", "SNOWFLAKE", "Decimal"), + ("NUMBER", "SNOWFLAKE", "Integer"), + ("NUMERIC(10,4)", "BIGQUERY", "Decimal"), + ("TEXT", "SNOWFLAKE", "String"), + ("STRING", "DATABRICKS", "String"), + ("LONG", "DATABRICKS", "Integer"), + ("INT64", "BIGQUERY", "Integer"), + ("FLOAT64", "BIGQUERY", "Float"), + ("DOUBLE", "DATABRICKS", "Float"), + ("BOOL", "BIGQUERY", "Boolean"), + ("BOOLEAN", "SNOWFLAKE", "Boolean"), + ("DATE", "SNOWFLAKE", "Date"), + # TIMESTAMP means different things per warehouse. + ("TIMESTAMP", "SNOWFLAKE", "DateTime"), + ("TIMESTAMP", "DATABRICKS", "DateTimeTz"), + ("TIMESTAMP", "BIGQUERY", "DateTimeTz"), + ("TIMESTAMP_NTZ", "SNOWFLAKE", "DateTime"), + ("TIMESTAMP_NTZ", "DATABRICKS", "DateTime"), + ("TIMESTAMP_TZ", "SNOWFLAKE", "DateTimeTz"), + ("DATETIME", "BIGQUERY", "DateTime"), + # Known, but outside the portable vocabulary. + ("VARIANT", "SNOWFLAKE", "Opaque"), + ("MAP", "DATABRICKS", "Opaque"), + ("ARRAY", "SNOWFLAKE", "Opaque"), + # Unknown: the spec says to omit datatype rather than guess. + ("SOMETHING_ELSE", "SNOWFLAKE", None), + ("", "SNOWFLAKE", None), + (None, "SNOWFLAKE", None), + ], +) +def test_warehouse_types_map_onto_the_portable_datatypes(raw, dialect, expected): + assert to_ossie_datatype(raw, dialect) == expected + + +def test_an_unmappable_type_omits_the_datatype_and_warns(): + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + solid["semantic_model"]["tables"][0]["dimensions"][2]["type"] = "SOMETHING_ELSE" + ossie, warnings_ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + assert any("SOMETHING_ELSE" in w for w in warnings_) + field = field_of(model_of(ossie), "orders", "status") + assert "datatype" not in field + # The raw name is still kept, so the export is unaffected. + assert stash_of(field)["type"] == "SOMETHING_ELSE" + + +@pytest.mark.parametrize( + ("datatype", "dialect", "expected"), + [ + ("String", "SNOWFLAKE", "TEXT"), + ("String", "DATABRICKS", "STRING"), + ("String", "BIGQUERY", "STRING"), + ("String", "ANSI_SQL", "VARCHAR"), + ("Integer", "BIGQUERY", "INT64"), + ("Float", "DATABRICKS", "DOUBLE"), + ("DateTimeTz", "SNOWFLAKE", "TIMESTAMP_TZ"), + (None, "SNOWFLAKE", None), + ], +) +def test_portable_datatypes_map_back_to_a_representative_warehouse_type( + datatype, dialect, expected +): + assert to_raw_type(datatype, dialect) == expected + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("NUMBER(38,0)", True), + ("DECIMAL(10,2)", True), + ("DOUBLE", True), + ("INT64", True), + ("LONG", True), + ("TEXT", False), + ("STRING", False), + ("DATE", False), + ("BOOLEAN", False), + (None, False), + ], +) +def test_solids_own_fact_type_rule_is_mirrored(raw, expected): + assert is_solid_fact_type(raw) is expected diff --git a/converters/solid/tests/test_expressions.py b/converters/solid/tests/test_expressions.py new file mode 100644 index 00000000..f05b3c51 --- /dev/null +++ b/converters/solid/tests/test_expressions.py @@ -0,0 +1,223 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Metric expression qualification. + +The property that matters most here is that the converter never rewrites SQL it was only +asked to qualify: the splice must leave every byte outside the inserted qualifier alone. +""" + +import pytest + +from ossie_solid.expressions import ( + AMBIGUOUS, + QUALIFIED, + UNCHANGED, + UNPARSED, + UNQUALIFIED, + column_reference, + qualify_metric, + referenced_datasets, + unqualify_metric, +) + +COLUMNS = ["amount", "status", "order_id", "region", "ss_sales_price", "ss_list_price"] + + +@pytest.mark.parametrize( + ("expression", "expected"), + [ + ("SUM(amount)", "SUM(orders.amount)"), + ("SUM(amount) / 12.0", "SUM(orders.amount) / 12.0"), + ("COUNT(DISTINCT order_id)", "COUNT(DISTINCT orders.order_id)"), + ("SUM(SUM(amount)) OVER ()", "SUM(SUM(orders.amount)) OVER ()"), + ( + "SUM(CASE WHEN status = 'shipped' THEN amount ELSE 0 END)", + "SUM(CASE WHEN orders.status = 'shipped' THEN orders.amount ELSE 0 END)", + ), + ( + "1 - (ss_sales_price / NULLIF(ss_list_price, 0))", + "1 - (orders.ss_sales_price / NULLIF(orders.ss_list_price, 0))", + ), + ], +) +def test_bare_columns_are_prefixed_with_the_owning_dataset(expression, expected): + assert qualify_metric(expression, "SNOWFLAKE", "orders", COLUMNS) == ( + expected, + QUALIFIED, + ) + + +def test_a_column_name_inside_a_string_literal_is_not_touched(): + result, status = qualify_metric( + "COUNT(CASE WHEN status = 'amount' THEN 1 END)", "SNOWFLAKE", "orders", COLUMNS + ) + assert result == "COUNT(CASE WHEN orders.status = 'amount' THEN 1 END)" + assert status == QUALIFIED + + +def test_an_already_qualified_column_is_left_alone(): + assert qualify_metric("SUM(o.amount)", "SNOWFLAKE", "orders", COLUMNS) == ( + "SUM(o.amount)", + UNCHANGED, + ) + + +def test_a_function_whose_name_matches_a_column_is_not_qualified(): + assert qualify_metric("status(amount)", "SNOWFLAKE", "orders", COLUMNS)[0] == ( + "status(orders.amount)" + ) + + +def test_a_column_the_dataset_does_not_own_is_left_bare(): + assert qualify_metric("SUM(other_col)", "SNOWFLAKE", "orders", COLUMNS) == ( + "SUM(other_col)", + UNCHANGED, + ) + + +def test_column_matching_is_case_insensitive(): + result, status = qualify_metric("SUM(AMOUNT)", "SNOWFLAKE", "orders", COLUMNS) + assert result == "SUM(orders.AMOUNT)" + assert status == QUALIFIED + + +def test_an_expression_with_no_columns_is_returned_verbatim(): + assert qualify_metric("COUNT(*)", "SNOWFLAKE", "orders", COLUMNS) == ( + "COUNT(*)", + UNCHANGED, + ) + + +def test_a_name_used_in_a_non_column_position_blocks_the_edit(): + # `year` is a column here *and* the datepart keyword inside EXTRACT, so the token + # scan and the parse disagree and the expression is left exactly as written. + result, status = qualify_metric( + "SUM(EXTRACT(year FROM closed_at))", "SNOWFLAKE", "orders", ["year", "closed_at"] + ) + assert result == "SUM(EXTRACT(year FROM closed_at))" + assert status == AMBIGUOUS + + +@pytest.mark.parametrize( + ("expression", "dialect"), + [ + # sqlglot canonicalizes both of these when it re-renders a parsed tree; a splice + # must not. + ("ROUND(CAST(amount AS FLOAT), 2)", "SNOWFLAKE"), + ("SUM(amount) + 1", "SNOWFLAKE"), + ("COUNTIF(status = 'active')", "BIGQUERY"), + ("SUM(amount) /* trailing comment */", "SNOWFLAKE"), + ], +) +def test_everything_outside_the_inserted_qualifier_is_preserved_byte_for_byte( + expression, dialect +): + result, status = qualify_metric(expression, dialect, "orders", COLUMNS) + assert status == QUALIFIED, "otherwise this asserts nothing" + assert result.replace("orders.", "") == expression + + +def test_an_unparseable_expression_is_returned_verbatim(): + assert qualify_metric("SUM(amount", "SNOWFLAKE", "orders", COLUMNS) == ( + "SUM(amount", + UNPARSED, + ) + + +def test_a_quoted_identifier_is_qualified_with_its_quotes_intact(): + result, status = qualify_metric( + 'SUM("odd name")', "SNOWFLAKE", "orders", ["odd name"] + ) + assert result == 'SUM(orders."odd name")' + assert status == QUALIFIED + + +# --- the inverse ------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("expression", "expected"), + [ + ("SUM(orders.amount)", "SUM(amount)"), + ( + "SUM(CASE WHEN orders.status = 'x' THEN orders.amount END)", + "SUM(CASE WHEN status = 'x' THEN amount END)", + ), + ], +) +def test_a_dataset_qualifier_is_stripped_back_out(expression, expected): + assert unqualify_metric(expression, "SNOWFLAKE", ["orders"]) == ( + expected, + UNQUALIFIED, + ) + + +def test_a_qualifier_that_is_not_a_dataset_is_preserved(): + assert unqualify_metric("SUM(p.amount)", "SNOWFLAKE", ["orders"]) == ( + "SUM(p.amount)", + UNCHANGED, + ) + + +def test_a_schema_qualified_reference_is_left_alone(): + assert unqualify_metric("SUM(db.orders.amount)", "SNOWFLAKE", ["orders"]) == ( + "SUM(db.orders.amount)", + UNCHANGED, + ) + + +@pytest.mark.parametrize( + ("expression", "dialect"), + [ + ("SUM(orders.amount) / NULLIF(COUNT(orders.order_id), 0)", "SNOWFLAKE"), + ("ROUND(CAST(orders.amount AS FLOAT), 2)", "SNOWFLAKE"), + ("COUNTIF(orders.status = 'active')", "BIGQUERY"), + ], +) +def test_qualifying_and_unqualifying_returns_the_original_text(expression, dialect): + bare, _ = unqualify_metric(expression, dialect, ["orders"]) + again, _ = qualify_metric(bare, dialect, "orders", COLUMNS) + assert again == expression + + +# --- helpers ----------------------------------------------------------------------- + + +def test_the_datasets_a_metric_references_are_reported_in_declaration_order(): + assert referenced_datasets( + "SUM(orders.amount) / COUNT(customers.id)", + "SNOWFLAKE", + ["customers", "orders", "items"], + ) == ["customers", "orders"] + + +def test_no_datasets_are_reported_for_a_bare_expression(): + assert referenced_datasets("SUM(amount)", "SNOWFLAKE", ["orders"]) == [] + + +@pytest.mark.parametrize( + ("name", "dialect", "expected"), + [ + ("amount", "SNOWFLAKE", "amount"), + ("odd name", "SNOWFLAKE", '"odd name"'), + ("odd name", "DATABRICKS", "`odd name`"), + ("select", "SNOWFLAKE", '"select"'), + ], +) +def test_a_column_name_renders_as_a_valid_reference(name, dialect, expected): + assert column_reference(name, dialect) == expected diff --git a/converters/solid/tests/test_ossie_to_solid.py b/converters/solid/tests/test_ossie_to_solid.py new file mode 100644 index 00000000..93c31620 --- /dev/null +++ b/converters/solid/tests/test_ossie_to_solid.py @@ -0,0 +1,441 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Apache Ossie -> Solid semantic model.""" + +import warnings + +import pytest +import yaml +from conftest import ( + by_name, + convert_quietly, + example, + fixture, + solid_model_of, +) + +from ossie_solid import ConversionError, convert_ossie_to_solid, convert_solid_to_ossie + + +@pytest.fixture(scope="module") +def from_fixture(): + """The TPC-DS Apache Ossie fixture exported back to Solid.""" + solid, _ = convert_quietly(convert_ossie_to_solid, fixture("tpcds_ossie.yaml")) + return solid_model_of(solid) + + +@pytest.fixture(scope="module") +def from_example(): + """The repository's hand-authored TPC-DS model exported to Solid. + + It carries no SOLID stash, so this exercises every fallback path. + """ + solid, _ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + return solid_model_of(solid) + + +def test_key_order_matches_solids_own_export_template(from_fixture): + assert list(from_fixture) == [ + "name", + "business_context", + "model_llm_description", + "tables", + "metrics", + "relationships", + "example_queries", + "benchmark_questions", + ] + + +def test_table_key_order_matches_solids_own_export_template(from_fixture): + assert list(by_name(from_fixture["tables"])["tpcds.public.store_sales"]) == [ + "name", + "description", + "manual_description", + "synonyms", + "primary_key", + "quality_rank", + "indexes", + "dimensions", + "facts", + ] + + +# --- datasets --------------------------------------------------------------------- + + +def test_the_dataset_source_becomes_the_table_name(from_fixture): + assert "tpcds.public.store_sales" in by_name(from_fixture["tables"]) + + +def test_a_composite_primary_key_is_rejoined_into_one_scalar(from_fixture): + table = by_name(from_fixture["tables"])["tpcds.public.store_sales"] + assert table["primary_key"] == "ss_item_sk, ss_ticket_number" + + +def test_quality_rank_is_always_emitted_even_when_unknown(from_example): + # The repository example has no SOLID stash, so no rank is known. + assert all(t["quality_rank"] == "" for t in from_example["tables"]) + + +def test_facts_is_always_emitted_even_when_empty(from_fixture): + table = by_name(from_fixture["tables"])["tpcds.public.date_dim"] + assert table["facts"] == [] + + +def test_unique_keys_are_dropped_with_a_warning(): + _, warnings_ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + assert any("unique_keys have no Solid equivalent" in w for w in warnings_) + + +def test_foreign_vendor_extensions_are_dropped_with_a_warning(): + _, warnings_ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + assert any("DBT, SALESFORCE" in w for w in warnings_) + + +# --- fields ----------------------------------------------------------------------- + + +def test_the_dimension_block_decides_the_fact_dimension_split(from_fixture): + table = by_name(from_fixture["tables"])["tpcds.public.store_sales"] + assert "ss_customer_sk" in by_name(table["dimensions"]) + assert "ss_net_profit" in by_name(table["facts"]) + + +def test_without_dimension_metadata_the_split_falls_back_to_solids_type_rule(): + # An Apache Ossie model whose fields carry no `dimension` block: numeric fields + # become facts and the rest dimensions, which is how Solid itself splits them. + ossie = yaml.safe_load(example("tpcds_semantic_model.yaml")) + for dataset in ossie["semantic_model"][0]["datasets"]: + for field in dataset["fields"]: + field.pop("dimension", None) + solid, _ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(ossie)) + table = by_name(solid_model_of(solid)["tables"])["tpcds.public.store_sales"] + assert "ss_quantity" in by_name(table["facts"]) + assert "ss_customer_sk" in by_name(table["facts"]) # Integer -> fact + table = by_name(solid_model_of(solid)["tables"])["tpcds.public.customer"] + assert "c_email_address" in by_name(table["dimensions"]) # String -> dimension + + +def test_the_stashed_raw_type_wins_over_the_portable_datatype(from_fixture): + table = by_name(from_fixture["tables"])["tpcds.public.store_sales"] + assert by_name(table["facts"])["ss_sales_price"]["type"] == "NUMBER(7,2)" + + +def test_without_a_stash_the_type_is_derived_from_the_datatype(from_example): + table = by_name(from_example["tables"])["tpcds.public.store_sales"] + # The example is ANSI_SQL, so the ANSI type names are used. + assert by_name(table["facts"])["ss_sales_price"]["type"] == "DECIMAL" + + +def test_a_field_with_no_datatype_gets_an_empty_type_and_a_warning(): + solid, warnings_ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + table = by_name(solid_model_of(solid)["tables"])["tpcds.public.date_dim"] + assert by_name(table["dimensions"])["d_quarter_name"]["type"] == "" + assert any("d_quarter_name" in w and "no datatype" in w for w in warnings_) + + +def test_an_expression_only_fact_round_trips_as_an_expression(from_fixture): + table = by_name(from_fixture["tables"])["tpcds.public.store_sales"] + fact = by_name(table["facts"])["ss_discount_pct"] + assert fact["expression"] == "1 - (ss_sales_price / NULLIF(ss_list_price, 0))" + assert "type" not in fact + + +def test_a_computed_dimension_drops_its_expression_with_a_warning(): + _, warnings_ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + assert any("customer_full_name" in w and "computed field" in w for w in warnings_) + + +def test_a_field_label_is_dropped_with_a_warning(): + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + ossie["semantic_model"][0]["datasets"][0]["fields"][0]["label"] = "filter" + _, warnings_ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(ossie)) + assert any("label 'filter'" in w for w in warnings_) + + +# --- metrics ---------------------------------------------------------------------- + + +def test_a_qualified_metric_is_written_back_with_bare_columns(from_fixture): + metric = by_name(from_fixture["metrics"])["TOTAL_SALES"] + assert metric["expression"] == "SUM(ss_ext_sales_price)" + assert metric["tables"] == ["tpcds.public.store_sales"] + + +def test_the_owning_tables_are_recovered_from_the_expression_without_a_stash( + from_example, +): + metric = by_name(from_example["metrics"])["customer_lifetime_value"] + assert metric["tables"] == ["tpcds.public.store_sales", "tpcds.public.customer"] + + +def test_metric_key_order_matches_solids_own_export_template(from_fixture): + assert list(by_name(from_fixture["metrics"])["TOTAL_SALES"]) == [ + "name", + "description", + "expression", + "synonyms", + "tables", + ] + + +# --- relationships ---------------------------------------------------------------- + + +def test_a_flipped_relationship_is_restored_to_solids_left_right_order(from_fixture): + relationship = from_fixture["relationships"][0] + assert relationship["left_table"] == "tpcds.public.date_dim" + assert relationship["right_table"] == "tpcds.public.store_sales" + + +def test_relationship_order_is_preserved(from_fixture): + assert [r["right_table"] for r in from_fixture["relationships"]] == [ + "tpcds.public.store_sales", + "tpcds.public.customer", + "tpcds.public.item", + "tpcds.public.store", + ] + + +def test_a_relationship_naming_an_undeclared_dataset_is_rejected(): + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + ossie["semantic_model"][0]["relationships"][0]["to"] = "nope" + with pytest.raises(ConversionError, match="not declared in 'datasets'"): + convert_ossie_to_solid(yaml.safe_dump(ossie)) + + +# --- dialect selection ------------------------------------------------------------ + + +def test_the_dialect_recorded_at_import_is_reused(from_fixture): + # NUMBER(7,2) is a Snowflake type name; reading it back proves the SNOWFLAKE + # dialect recorded in the stash was used. + table = by_name(from_fixture["tables"])["tpcds.public.store_sales"] + assert by_name(table["facts"])["ss_sales_price"]["type"] == "NUMBER(7,2)" + + +def test_an_explicit_dialect_overrides_the_stash(): + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture("databricks_solid.yaml")) + solid, _ = convert_quietly(convert_ossie_to_solid, ossie, dialect="DATABRICKS") + assert solid_model_of(solid)["name"] == "orders_analytics" + + +def test_forcing_a_dialect_the_model_does_not_carry_is_an_error(): + # The TPC-DS fixture is SNOWFLAKE-only, with no ANSI_SQL fallback to read. + with pytest.raises(ConversionError, match="no DATABRICKS or ANSI_SQL expression"): + convert_ossie_to_solid(fixture("tpcds_ossie.yaml"), dialect="DATABRICKS") + + +def test_an_unsupported_dialect_is_rejected(): + with pytest.raises(ConversionError, match="Unsupported dialect"): + convert_ossie_to_solid(fixture("tpcds_ossie.yaml"), dialect="ORACLE") + + +def test_a_missing_dialect_expression_falls_back_to_ansi_with_a_warning(): + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + field = ossie["semantic_model"][0]["datasets"][0]["fields"][0] + field["expression"]["dialects"] = [ + {"dialect": "ANSI_SQL", "expression": "ss_sold_date_sk"} + ] + _, warnings_ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(ossie)) + assert any("no SNOWFLAKE expression" in w for w in warnings_) + + +# --- input validation -------------------------------------------------------------- + + +def test_a_wrong_spec_version_is_rejected(): + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + ossie["version"] = "0.1.0" + with pytest.raises(ConversionError, match="Unsupported Apache Ossie version"): + convert_ossie_to_solid(yaml.safe_dump(ossie)) + + +def test_a_solid_document_is_rejected(): + with pytest.raises(ConversionError, match="Unsupported Apache Ossie version"): + convert_ossie_to_solid(fixture("tpcds_solid.yaml")) + + +def test_extra_semantic_models_are_dropped_with_a_warning(): + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + second = dict(ossie["semantic_model"][0]) + second["name"] = "second_model" + ossie["semantic_model"].append(second) + solid, warnings_ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(ossie)) + assert any("holds 2 semantic models" in w for w in warnings_) + assert solid_model_of(solid)["name"] == "tpcds_retail_model" + + +# --- constructs Solid's format cannot hold ----------------------------------------- + + +def test_a_metric_datatype_is_dropped_with_a_warning(): + """Solid types a metric by evaluating its formula, so a declared type has no slot.""" + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + ossie["semantic_model"][0]["metrics"][0]["datatype"] = "Decimal" + solid, warnings_ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(ossie)) + assert any("datatype 'Decimal' has no Solid equivalent" in w for w in warnings_) + assert "datatype" not in solid_model_of(solid)["metrics"][0] + + +def test_a_relationship_annotation_is_dropped_with_a_warning(): + """A Solid relationship carries only its tables and join keys.""" + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + ossie["semantic_model"][0]["relationships"][0]["ai_context"] = { + "instructions": "join only on settled sales", + "synonyms": ["sold on"], + } + _, warnings_ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(ossie)) + dropped = [w for w in warnings_ if "no Solid equivalent" in w and "relationship" in w] + assert len(dropped) == 1 + assert "ai_context.instructions" in dropped[0] + assert "ai_context.synonyms" in dropped[0] + + +def test_the_one_to_one_note_import_writes_is_not_reported_as_a_loss(): + """That note is this converter's own marker, not a user annotation. + + Import adds it where Apache Ossie's from/to direction is arbitrary; dropping it on + the way back loses nothing, so it must not be reported as a dropped annotation. + """ + solid = yaml.safe_load(fixture("tpcds_solid.yaml")) + # Make both ends of the first join unique on their join columns -> a one-to-one. + join = solid["semantic_model"]["relationships"][0] + tables = {t["name"]: t for t in solid["semantic_model"]["tables"]} + tables[join["left_table"]]["primary_key"] = ", ".join(join["join_keys"]["left"]) + tables[join["right_table"]]["primary_key"] = ", ".join(join["join_keys"]["right"]) + + ossie, _ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + annotated = [ + r for r in yaml.safe_load(ossie)["semantic_model"][0]["relationships"] + if (r.get("ai_context") or {}).get("instructions") + ] + assert annotated, "expected import to annotate the one-to-one" + + _, warnings_ = convert_quietly(convert_ossie_to_solid, ossie) + assert not [w for w in warnings_ if "ai_context.instructions" in w], warnings_ + + +# --- non-SQL dialects -------------------------------------------------------------- + + +def test_a_non_sql_dialect_never_becomes_the_resolved_dialect(): + """MDX/TABLEAU/MAQL are in Apache Ossie's enum but are not SQL. + + Resolving to one would hand a non-SQL formula to the expression rewriter, and to + Solid, as though it were SQL. The ANSI_SQL form is read instead. + """ + ossie = { + "version": "0.2.0.dev0", + "semantic_model": [{ + "name": "m", + "datasets": [{ + "name": "a", + "source": "db.s.a", + "fields": [{ + "name": "amt", + "datatype": "Decimal", + "expression": {"dialects": [ + {"dialect": "TABLEAU", "expression": "[Amount]"}, + {"dialect": "ANSI_SQL", "expression": "amt"}, + ]}, + }], + }], + }], + } + solid, warnings_ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(ossie)) + assert any("not SQL this converter can read" in w for w in warnings_) + column = solid_model_of(solid)["tables"][0]["facts"][0] + # The ANSI_SQL form was read, so the column is a plain reference to itself and no + # Tableau formula leaked into the Solid model. + assert "expression" not in column + assert column["name"] == "amt" + + +def test_a_non_sql_dialect_with_no_ansi_form_is_an_error(): + """Better a clear failure than a Tableau formula in a SQL field.""" + ossie = { + "version": "0.2.0.dev0", + "semantic_model": [{ + "name": "m", + "datasets": [{ + "name": "a", + "source": "db.s.a", + "fields": [{ + "name": "amt", + "expression": {"dialects": [ + {"dialect": "MDX", "expression": "[Measures].[Amt]"}, + ]}, + }], + }], + }], + } + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with pytest.raises(ConversionError, match="no ANSI_SQL expression"): + convert_ossie_to_solid(yaml.safe_dump(ossie)) + + +def test_a_hand_edited_stash_dialect_is_rejected(): + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + model = ossie["semantic_model"][0] + for ext in model["custom_extensions"]: + if ext["vendor_name"] == "SOLID": + ext["data"] = ext["data"].replace("SNOWFLAKE", "MAQL") + with pytest.raises(ConversionError, match="Unsupported dialect"): + convert_ossie_to_solid(yaml.safe_dump(ossie)) + + +# --- spec versions ----------------------------------------------------------------- + + +def test_the_released_0_1_1_spec_is_read_with_a_warning(): + """0.1.1 is the only released spec version, so models in the wild declare it. + + A 0.1.1 document is a 0.2.0.dev0 document minus additive fields, so it is read + rather than rejected -- but the version gap is reported, since the missing + `datatype` is what leaves a column's Solid `type` empty. + """ + ossie = yaml.safe_load(fixture("tpcds_ossie.yaml")) + ossie["version"] = "0.1.1" + solid, warnings_ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(ossie)) + assert any("declares Apache Ossie v0.1.1" in w for w in warnings_) + assert any("predates the `datatype` field" in w for w in warnings_) + # The model still converts in full: the stashed raw types carry the column types. + assert solid_model_of(solid)["tables"][0]["facts"][0]["type"] + + +def test_reading_an_older_spec_still_writes_the_current_one(): + solid = yaml.safe_load(fixture("tpcds_solid.yaml")) + ossie, _ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + aged = yaml.safe_load(ossie) + aged["version"] = "0.1.1" + back, _ = convert_quietly(convert_ossie_to_solid, yaml.safe_dump(aged)) + again, _ = convert_quietly(convert_solid_to_ossie, back) + assert yaml.safe_load(again)["version"] == "0.2.0.dev0" diff --git a/converters/solid/tests/test_roundtrip.py b/converters/solid/tests/test_roundtrip.py new file mode 100644 index 00000000..a15d2ad7 --- /dev/null +++ b/converters/solid/tests/test_roundtrip.py @@ -0,0 +1,215 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Round-trip fidelity in both directions. + +`Solid -> Apache Ossie -> Solid` is exact: everything Solid's format can express either +maps onto an Apache Ossie core field or is preserved in `custom_extensions[SOLID]`. + +`Apache Ossie -> Solid -> Apache Ossie` is lossy by construction, because Solid's format +has no slot for several Apache Ossie constructs (unique keys, foreign-vendor extensions, +computed dimensions, multi-dialect expressions). Those losses are asserted explicitly +here, so a regression that widens them fails the suite. +""" + +import pytest +import yaml +from conftest import ( + by_name, + convert_quietly, + example, + fixture, + model_of, + normalized_yaml, + solid_model_of, +) + +from ossie_solid import convert_ossie_to_solid, convert_solid_to_ossie + +SOLID_FIXTURES = ["tpcds_solid.yaml", "databricks_solid.yaml", "bigquery_solid.yaml"] + + + +@pytest.mark.parametrize("name", SOLID_FIXTURES) +def test_solid_to_ossie_to_solid_is_exact(name): + original = fixture(name) + ossie, _ = convert_quietly(convert_solid_to_ossie, original) + restored, _ = convert_quietly(convert_ossie_to_solid, ossie) + assert normalized_yaml(restored) == normalized_yaml(original) + + +@pytest.mark.parametrize("name", SOLID_FIXTURES) +def test_solid_round_trip_is_stable_on_a_second_pass(name): + """A second round trip changes nothing a first one did not.""" + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture(name)) + once, _ = convert_quietly(convert_ossie_to_solid, ossie) + twice_ossie, _ = convert_quietly(convert_solid_to_ossie, once) + twice, _ = convert_quietly(convert_ossie_to_solid, twice_ossie) + assert normalized_yaml(twice) == normalized_yaml(once) + assert yaml.safe_load(twice_ossie) == yaml.safe_load(ossie) + + +@pytest.mark.parametrize("name", SOLID_FIXTURES) +def test_the_intermediate_ossie_model_is_schema_valid(name, assert_valid_ossie): + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture(name)) + assert_valid_ossie(ossie) + + +def test_ossie_to_solid_to_ossie_stays_schema_valid(assert_valid_ossie): + solid, _ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + ossie, _ = convert_quietly(convert_solid_to_ossie, solid, dialect="ANSI_SQL") + assert_valid_ossie(ossie) + + +def test_ossie_to_solid_to_ossie_preserves_the_model_shape(): + original = model_of(example("tpcds_semantic_model.yaml")) + solid, _ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + restored, _ = convert_quietly(convert_solid_to_ossie, solid, dialect="ANSI_SQL") + restored = model_of(restored) + + assert restored["name"] == original["name"] + assert [d["source"] for d in restored["datasets"]] == [ + d["source"] for d in original["datasets"] + ] + assert [d["name"] for d in restored["datasets"]] == [ + d["name"] for d in original["datasets"] + ] + assert sorted(by_name(restored["metrics"])) == sorted(by_name(original["metrics"])) + for dataset in original["datasets"]: + assert by_name(restored["datasets"])[dataset["name"]].get("primary_key") == ( + dataset.get("primary_key") + ) + + +def test_ossie_to_solid_to_ossie_preserves_relationship_semantics(): + original = model_of(example("tpcds_semantic_model.yaml")) + solid, _ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + restored = model_of( + convert_quietly(convert_solid_to_ossie, solid, dialect="ANSI_SQL")[0] + ) + + def edges(model): + return { + (r["from"], r["to"], tuple(r["from_columns"]), tuple(r["to_columns"])) + for r in model["relationships"] + } + + assert edges(restored) == edges(original) + + +def test_ossie_to_solid_to_ossie_preserves_metric_expressions(): + original = model_of(example("tpcds_semantic_model.yaml")) + solid, _ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + restored = model_of( + convert_quietly(convert_solid_to_ossie, solid, dialect="ANSI_SQL")[0] + ) + + def expressions(model): + return { + m["name"]: m["expression"]["dialects"][0]["expression"] + for m in model["metrics"] + } + + before, after = expressions(original), expressions(restored) + # Single-table metrics survive verbatim. Cross-table ones lose their qualifiers, + # because Solid's format cannot record which table each column belongs to. + assert after["total_sales"] == before["total_sales"] + assert after["total_profit"] == before["total_profit"] + assert after["customer_lifetime_value"] == ( + "SUM(ss_ext_sales_price) / COUNT(DISTINCT c_customer_sk)" + ) + + +def test_the_documented_losses_of_an_ossie_export_are_exactly_these(): + """Pin the known one-way losses, so a regression that adds another one fails.""" + _, going = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + assert {w.split("]")[0].lstrip("[") for w in going} == { + "model", "dataset", "field", "metric", "relationship" + } + assert sum("custom_extensions for" in w for w in going) == 1 + assert sum("unique_keys have no Solid equivalent" in w for w in going) == 1 + assert sum("computed field" in w for w in going) == 1 + assert sum("no datatype" in w for w in going) == 2 + # Solid types a metric by evaluating its formula, and its relationships carry no + # free text, so a declared metric datatype and a relationship annotation both have + # nowhere to land. + assert sum("datatype 'Decimal' has no Solid equivalent" in w for w in going) == 5 + assert sum("ai_context.synonyms has no Solid equivalent" in w for w in going) == 4 + assert len(going) == 14 + + +def test_reimporting_that_export_only_loses_the_cross_table_qualifiers(): + solid, _ = convert_quietly( + convert_ossie_to_solid, example("tpcds_semantic_model.yaml") + ) + _, coming = convert_quietly(convert_solid_to_ossie, solid, dialect="ANSI_SQL") + assert all("spans 2 tables" in w for w in coming), coming + assert len(coming) == 2 + + +def test_a_solid_round_trip_warns_only_about_unqualifiable_metrics(): + _, going = convert_quietly(convert_solid_to_ossie, fixture("tpcds_solid.yaml")) + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture("tpcds_solid.yaml")) + _, coming = convert_quietly(convert_ossie_to_solid, ossie) + assert all("spans 2 tables" in w for w in going), going + assert coming == [], coming + + +@pytest.mark.parametrize( + ("name", "dialect"), + [ + ("tpcds_solid.yaml", "SNOWFLAKE"), + ("databricks_solid.yaml", "DATABRICKS"), + ("bigquery_solid.yaml", "BIGQUERY"), + ], +) +def test_the_dialect_survives_a_round_trip(name, dialect): + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture(name)) + model = model_of(ossie) + for field in model["datasets"][0]["fields"]: + assert field["expression"]["dialects"][0]["dialect"] == dialect + restored, _ = convert_quietly(convert_ossie_to_solid, ossie) + again, _ = convert_quietly(convert_solid_to_ossie, restored) + assert model_of(again)["datasets"][0]["fields"][0]["expression"]["dialects"][0][ + "dialect" + ] == dialect + + +def test_an_empty_description_is_normalized_away(): + """A documented, deliberate difference. + + solid-server renders `manual_description` from a whitespace-only value as an empty + block scalar, which parses back as `''`. That carries no information, so the + converter drops it rather than round-tripping the emptiness. + """ + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + solid["semantic_model"]["tables"][0]["dimensions"][0]["manual_description"] = "" + ossie, _ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + assert "ai_context" not in model_of(ossie)["datasets"][0]["fields"][0] + restored, _ = convert_quietly(convert_ossie_to_solid, ossie) + column = solid_model_of(restored)["tables"][0]["dimensions"][0] + assert "manual_description" not in column diff --git a/converters/solid/tests/test_solid_to_ossie.py b/converters/solid/tests/test_solid_to_ossie.py new file mode 100644 index 00000000..5625203f --- /dev/null +++ b/converters/solid/tests/test_solid_to_ossie.py @@ -0,0 +1,347 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Solid semantic model -> Apache Ossie.""" + +import pytest +import yaml +from conftest import ( + by_name, + convert_quietly, + dataset_of, + expression_of, + field_of, + fixture, + model_of, + stash_of, +) + +from ossie_solid import ConversionError, convert_solid_to_ossie + + +@pytest.fixture(scope="module") +def tpcds(): + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture("tpcds_solid.yaml")) + return model_of(ossie) + + +def test_output_matches_the_committed_fixture(): + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture("tpcds_solid.yaml")) + assert yaml.safe_load(ossie) == yaml.safe_load(fixture("tpcds_ossie.yaml")) + + +def test_output_validates_against_the_ossie_schema(assert_valid_ossie): + for name in ("tpcds_solid.yaml", "databricks_solid.yaml", "bigquery_solid.yaml"): + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture(name)) + assert_valid_ossie(ossie) + + +def test_document_declares_the_spec_version(): + ossie, _ = convert_quietly(convert_solid_to_ossie, fixture("tpcds_solid.yaml")) + document = yaml.safe_load(ossie) + assert document["version"] == "0.2.0.dev0" + assert len(document["semantic_model"]) == 1 + + +# --- datasets --------------------------------------------------------------------- + + +def test_table_fqn_becomes_the_source_and_its_last_part_the_name(tpcds): + dataset = dataset_of(tpcds, "store_sales") + assert dataset["source"] == "tpcds.public.store_sales" + + +def test_dataset_names_are_disambiguated_when_the_last_part_collides(): + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + # Two tables named `orders` in different schemas: the first keeps the short name, + # the second widens to schema_table rather than colliding. + solid["semantic_model"]["tables"][1]["name"] = "main.returns.orders" + solid["semantic_model"]["relationships"][0]["right_table"] = "main.returns.orders" + solid["semantic_model"]["tables"][1]["primary_key"] = "customer_id" + ossie, _ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + names = [d["name"] for d in model_of(ossie)["datasets"]] + assert names == ["orders", "returns_orders"] + + +def test_composite_primary_key_scalar_is_split_into_columns(tpcds): + assert dataset_of(tpcds, "store_sales")["primary_key"] == [ + "ss_item_sk", + "ss_ticket_number", + ] + + +def test_single_primary_key_becomes_a_one_element_list(tpcds): + assert dataset_of(tpcds, "customer")["primary_key"] == ["c_customer_sk"] + + +def test_table_descriptions_split_across_description_and_ai_context(tpcds): + dataset = dataset_of(tpcds, "store_sales") + assert dataset["description"].startswith("Fact table containing all store sales") + assert dataset["ai_context"]["instructions"].startswith("Grain is (ss_item_sk") + + +def test_table_synonyms_become_ai_context_synonyms(tpcds): + assert "POS data" in dataset_of(tpcds, "store_sales")["ai_context"]["synonyms"] + + +def test_quality_rank_and_indexes_are_stashed(tpcds): + stash = stash_of(dataset_of(tpcds, "store_sales")) + assert stash["quality_rank"] == "high" + assert stash["indexes"] == ["ss_sold_date_sk", "ss_customer_sk"] + + +def test_a_table_without_indexes_gets_no_index_stash(tpcds): + assert "indexes" not in stash_of(dataset_of(tpcds, "customer")) + + +# --- fields ----------------------------------------------------------------------- + + +def test_dimensions_and_facts_merge_into_one_field_list(tpcds): + fields = by_name(dataset_of(tpcds, "store_sales")["fields"]) + assert "ss_customer_sk" in fields # a dimension + assert "ss_net_profit" in fields # a fact + + +def test_a_dimension_carries_the_dimension_block_and_a_fact_does_not(tpcds): + assert field_of(tpcds, "store_sales", "ss_customer_sk")["dimension"] == { + "is_time": False + } + assert "dimension" not in field_of(tpcds, "store_sales", "ss_net_profit") + + +def test_a_temporal_dimension_is_marked_is_time(tpcds): + assert field_of(tpcds, "store_sales", "ss_sold_at")["dimension"] == {"is_time": True} + assert field_of(tpcds, "date_dim", "d_date")["dimension"] == {"is_time": True} + + +def test_a_column_expression_is_its_own_name(tpcds): + assert expression_of(field_of(tpcds, "customer", "c_email_address")) == ( + "c_email_address" + ) + + +def test_the_raw_warehouse_type_is_stashed_alongside_the_portable_datatype(tpcds): + field = field_of(tpcds, "store_sales", "ss_sales_price") + assert field["datatype"] == "Decimal" + assert stash_of(field)["type"] == "NUMBER(7,2)" + + +def test_sample_values_are_stashed(tpcds): + assert stash_of(field_of(tpcds, "item", "i_category"))["sample_values"] == [ + "Electronics", + "Home", + "Sports", + ] + + +def test_an_expression_only_fact_keeps_its_expression_and_is_marked_a_metric(tpcds): + field = field_of(tpcds, "store_sales", "ss_discount_pct") + assert expression_of(field) == "1 - (ss_sales_price / NULLIF(ss_list_price, 0))" + assert stash_of(field) == {"role": "metric"} + assert "datatype" not in field + + +def test_a_column_description_pair_maps_to_description_and_ai_context(tpcds): + field = field_of(tpcds, "store_sales", "ss_sales_price") + assert field["description"] == "Sales price per unit." + assert field["ai_context"]["instructions"].startswith("Per-unit price.") + + +# --- relationships ---------------------------------------------------------------- + + +def test_direction_follows_the_primary_key_not_the_left_right_order(tpcds): + # date_dim is the left table in the fixture, but its primary key covers the join, + # so it is the one side and store_sales must be the many side. + relationship = by_name(tpcds["relationships"])["store_sales_to_date_dim"] + assert relationship["from"] == "store_sales" + assert relationship["to"] == "date_dim" + assert relationship["from_columns"] == ["ss_sold_date_sk"] + assert relationship["to_columns"] == ["d_date_sk"] + assert stash_of(relationship)["flipped"] is True + + +def test_a_relationship_already_in_many_to_one_order_is_not_flipped(tpcds): + relationship = by_name(tpcds["relationships"])["store_sales_to_customer"] + assert relationship["from"] == "store_sales" + assert "flipped" not in stash_of(relationship) + + +def test_relationships_keep_their_declaration_order(tpcds): + assert [r["name"] for r in tpcds["relationships"]] == [ + "store_sales_to_date_dim", + "store_sales_to_customer", + "store_sales_to_item", + "store_sales_to_store", + ] + + +def test_only_a_flipped_relationship_carries_a_stash(tpcds): + stashes = [stash_of(r) for r in tpcds["relationships"]] + assert stashes == [{"flipped": True}, {}, {}, {}] + + +def test_undetermined_cardinality_warns_and_keeps_solid_order(): + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + del solid["semantic_model"]["tables"][1]["primary_key"] + ossie, warnings_ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + assert any("could not be determined" in w for w in warnings_) + relationship = model_of(ossie)["relationships"][0] + assert (relationship["from"], relationship["to"]) == ("orders", "customers") + + +def test_a_one_to_one_relationship_is_annotated(): + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + # Make the join unique on both ends. + solid["semantic_model"]["tables"][0]["primary_key"] = "customer_id" + ossie, _ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + relationship = model_of(ossie)["relationships"][0] + assert "One-to-one" in relationship["ai_context"]["instructions"] + + +def test_a_relationship_naming_an_unknown_table_is_dropped_with_a_warning(): + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + solid["semantic_model"]["relationships"][0]["right_table"] = "main.sales.missing" + ossie, warnings_ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + assert any("main.sales.missing" in w for w in warnings_) + assert "relationships" not in model_of(ossie) + + +def test_mismatched_join_key_counts_are_rejected(): + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + solid["semantic_model"]["relationships"][0]["join_keys"]["right"] = [ + "customer_id", + "region", + ] + with pytest.raises(ConversionError, match="correspond positionally"): + convert_solid_to_ossie(yaml.safe_dump(solid)) + + +# --- metrics ---------------------------------------------------------------------- + + +def test_a_single_table_metric_is_qualified_with_its_dataset(tpcds): + metrics = by_name(tpcds["metrics"]) + assert expression_of(metrics["TOTAL_SALES"]) == ( + "SUM(store_sales.ss_ext_sales_price)" + ) + + +def test_a_qualified_metric_needs_no_table_stash(tpcds): + assert stash_of(by_name(tpcds["metrics"])["TOTAL_SALES"]) == {} + + +def test_a_metric_referencing_no_column_keeps_its_tables_in_the_stash(tpcds): + metric = by_name(tpcds["metrics"])["ROW_COUNT"] + assert expression_of(metric) == "COUNT(*)" + assert stash_of(metric)["tables"] == ["tpcds.public.store_sales"] + + +def test_a_cross_table_metric_is_left_unqualified_and_warns(): + _, warnings_ = convert_quietly(convert_solid_to_ossie, fixture("tpcds_solid.yaml")) + assert any( + "CUSTOMER_LIFETIME_VALUE" in w and "spans 2 tables" in w for w in warnings_ + ) + + +def test_a_cross_table_metric_keeps_its_tables_in_the_stash(tpcds): + metric = by_name(tpcds["metrics"])["CUSTOMER_LIFETIME_VALUE"] + assert expression_of(metric) == ( + "SUM(ss_ext_sales_price) / COUNT(DISTINCT c_customer_sk)" + ) + assert stash_of(metric)["tables"] == [ + "tpcds.public.store_sales", + "tpcds.public.customer", + ] + + +def test_metric_synonyms_become_ai_context_synonyms(tpcds): + assert by_name(tpcds["metrics"])["TOTAL_SALES"]["ai_context"]["synonyms"] == [ + "total revenue", + "gross sales", + "sales amount", + ] + + +def test_a_metric_without_an_expression_is_dropped_with_a_warning(): + solid = yaml.safe_load(fixture("databricks_solid.yaml")) + del solid["semantic_model"]["metrics"][0]["expression"] + ossie, warnings_ = convert_quietly(convert_solid_to_ossie, yaml.safe_dump(solid)) + assert any("LATE_ORDER_COUNT" in w and "no expression" in w for w in warnings_) + assert [m["name"] for m in model_of(ossie)["metrics"]] == ["GROSS_REVENUE"] + + +# --- model level ------------------------------------------------------------------ + + +def test_asset_link_markup_is_resolved_in_ai_context_and_kept_raw_in_the_stash(tpcds): + instructions = tpcds["ai_context"]["instructions"] + assert "public.store_sales" in instructions + assert "assetlink" not in instructions + assert "@