diff --git a/.github/workflows/converter-thoughtspot-ci.yml b/.github/workflows/converter-thoughtspot-ci.yml new file mode 100644 index 00000000..8f3fe1ee --- /dev/null +++ b/.github/workflows/converter-thoughtspot-ci.yml @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Converters ThoughtSpot CI +on: + push: + branches: [ "main" ] + paths: + - 'converters/thoughtspot/**' + - '.github/workflows/converter-thoughtspot-ci.yml' + # catalog.py's spec_construct_names() parses this file directly (it is the + # upstream-spec coverage oracle, not vendored into converters/thoughtspot/), + # so a change here must trigger this workflow even though it touches no + # converter file. No sibling workflow parses the spec, hence this addition + # is not mirrored anywhere else. + - 'core-spec/expression_language.md' + pull_request: + branches: [ "main" ] + paths: + - 'converters/thoughtspot/**' + - '.github/workflows/converter-thoughtspot-ci.yml' + - 'core-spec/expression_language.md' + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" + + - name: Sync dependencies + working-directory: converters/thoughtspot + run: | + uv sync + + - name: Unit Tests + working-directory: converters/thoughtspot + run: | + uv run pytest diff --git a/converters/README.md b/converters/README.md index 014a7350..84326218 100644 --- a/converters/README.md +++ b/converters/README.md @@ -77,6 +77,7 @@ The Ossie specification currently defines extensions for the following vendors: | `WISDOM` | WisdomAI domain | | `NVIDIA_GSF` | NVIDIA Generative Semantic Fabric standalone YAML | | `SIGMA` | Sigma Computing data model | +| `THOUGHTSPOT` | ThoughtSpot TML (Model + Table/SQL View) | Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. diff --git a/converters/thoughtspot/.gitignore b/converters/thoughtspot/.gitignore new file mode 100644 index 00000000..d6ae8114 --- /dev/null +++ b/converters/thoughtspot/.gitignore @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +__pycache__/ +*.py[cod] +.pytest_cache/ +*.egg-info/ +dist/ +build/ +.venv/ +venv/ diff --git a/converters/thoughtspot/README.md b/converters/thoughtspot/README.md new file mode 100644 index 00000000..04b285b9 --- /dev/null +++ b/converters/thoughtspot/README.md @@ -0,0 +1,311 @@ + + +# Apache Ossie ThoughtSpot Converter + +Bidirectional, offline conversion between an [Apache Ossie](https://github.com/apache/ossie) +semantic model and ThoughtSpot TML. No ThoughtSpot connection required. + +- **ThoughtSpot TML → Ossie** (`to-ossie`): reads a Model TML document plus the Table and + SQL View documents it references, and emits one Ossie semantic model. +- **Ossie → ThoughtSpot TML** (`to-tml`): reads one Ossie semantic model and emits the + corresponding set of TML documents. + +A single Ossie semantic model corresponds to **1 + N TML documents**, not one file: one +`model:` document plus one `table:` or `sql_view:` document per dataset. The converter +reads and writes the set. File-to-file only — nothing here calls a ThoughtSpot API. + +The `THOUGHTSPOT` dialect is registered upstream — apache/ossie#351 merged 2026-09-01. +Both conversion directions are implemented and tested: example-based unit tests, an +exact-document comparison against a shared TPC-DS fixture set (the same retail schema +every sibling converter round-trips), a round-trip suite asserting preservation and +translation separately, and a Hypothesis property-based suite over adversarial +identifiers. + +## Installation + +```bash +pip install apache-ossie-thoughtspot # once published to PyPI +# or, from a checkout of this directory: +pip install -e . +``` + +The only runtime dependency is `PyYAML`. Python 3.10+. + +## Usage + +### Command line + +```bash +ossie-thoughtspot to-ossie ... -o [--issues ] [--force] +ossie-thoughtspot to-tml -o [--issues ] [--force] +``` + +`to-ossie` takes the Model TML document plus every Table/SQL View document it references +and writes one Ossie YAML file. `to-tml` takes one Ossie YAML document and writes the +corresponding TML document set — one file per document, tables before the model — into +an output directory it creates if needed. + +`-o`/`--output` is required in both directions: `to-tml` writes a set of files that has +no single-file stdout representation, so unlike some sibling converters there is no +"default: stdout" fallback. Neither subcommand overwrites an existing output file unless +`--force` is given. + +Every declared loss or degradation the conversion records is written as a JSON array of +issues — to `--issues` when given, to stderr otherwise — never mixed into the document +output. The process exits `1` when that issue log contains an ERROR-severity issue, `0` +otherwise: a conversion that only warned or informed about a declared loss is still a +successful conversion. + +### Python API + +```python +from ossie_thoughtspot import tml, tml_to_ossie, ossie_to_thoughtspot, _yaml + +# ThoughtSpot TML -> Ossie +texts = [(path, open(path).read()) for path in ("model.model.tml", "orders.table.tml")] +result = tml_to_ossie.convert(tml.load_document_set(texts)) +ossie_yaml = _yaml.dump(result.model) # result.issues: IssueLog + +# Ossie -> ThoughtSpot TML +ossie_document = _yaml.load(open("model.yaml").read()) +result = ossie_to_thoughtspot.convert(ossie_document) +for filename, text in tml.dump_document_set(result.documents): + ... # result.issues: IssueLog +``` + +`result.issues` is an `IssueLog`: `has_errors()`, `count_by_severity()`, `as_dicts()`. +Every declared loss raises an issue here — see [Coverage matrix](#coverage-matrix) and +[Expression translation](#expression-translation-what-is-not-translated-and-why) below +for what gets declared and why. + +## Mapping + +| Ossie | ThoughtSpot TML | Notes | +|---|---|---| +| `semantic_model` (one entry) | one Model document + the Table/SQL View documents it references | Exactly one `semantic_model` entry per document; more than one is a hard failure | +| `dataset` | `table:`/`sql_view:` document, surfaced via the Model's `model_tables[]` entry | One dataset per participating `model_tables[]` entry, not per physical table — a self-join or a table used twice gets two datasets sharing one `source` | +| `dataset.source` | `db`.`schema`.`db_table`, or `sql_query` for a SQL View | A dotted part is stashed individually (`source_parts`) when the joined form would be ambiguous | +| `dataset.fields` | Table `columns[]` (physical) or Model `formulas[]` + surfacing `columns[]` entry (computed) | A computed field is attributed to the one dataset every column reference in its expression resolves to; ambiguous or cross-dataset references raise an issue instead of guessing | +| `relationship` | `model_tables[].joins[]` (inline) or Table `joins_with[]` (referencing) | `from_columns`/`to_columns` are the join's equality pairs; a join with a non-equality residual (range/ASOF) narrows the same pairs, with the verbatim condition stashed — see [the payload section](#the-custom_extensionsthoughtspot-payload) below | +| `dataset.primary_key` / `unique_keys` | *(not native to TML)* | TML declares no keys; Ossie's are derived from to-one relationships targeting the dataset | +| `metric` | Model `formulas[]` + surfacing `columns[]` entry with `column_type: MEASURE` | Three TML shapes compose into one metric: a bare aggregate formula, a scalar formula plus the surfacing column's `aggregation`, or a physical column plus `aggregation` | +| `field`/`metric` `expression.dialects` | `formulas[].expr` or a physical `db_column_name` | See [Expression translation](#expression-translation-what-is-not-translated-and-why) | +| `custom_extensions[THOUGHTSPOT]` | TML fields with no Ossie equivalent | See [The `custom_extensions[THOUGHTSPOT]` payload](#the-custom_extensionsthoughtspot-payload) below | + +## The `custom_extensions[THOUGHTSPOT]` payload + +TML carries properties Ossie's core specification has no field for — a Connection name, +search-indexing settings, a display column's warehouse name when it differs from its +label, a join's exact type, and more. `TML → Ossie` stashes each one under a single +`custom_extensions` entry (`vendor_name: THOUGHTSPOT`) attached to the Ossie object it +came from; `Ossie → TML` reads the same entry back to reconstruct the original TML +property, so `TML → Ossie → TML` is lossless for everything TML itself can express. + +```yaml +custom_extensions: +- vendor_name: THOUGHTSPOT + data: '{"_v": 1, "connection_name": "My Snowflake", "tml_object": "table"}' +``` + +`data` is always a single JSON-encoded string (never a nested object — the core +specification requires this), carrying: + +- `_v`, a shape version bumped only when the payload's *shape* changes, never for a + value change — an unrecognised version is a hard failure rather than a silent + misread of a shape this converter has never seen. +- Never a `guid`, `obj_id`, or `fqn` at any depth — object identity is instance-local + and is refused outright rather than carried, at write time. +- Foreign-vendor `custom_extensions` entries on the same object pass through untouched + in both directions. +- An object with nothing to stash gets no `custom_extensions[THOUGHTSPOT]` entry at + all, so a converted document stays as small as its content requires. + +`Ossie → TML` restores a stashed value only when it is still current: several keys +(a physical column's warehouse name, a relationship's verbatim join condition, a +dataset's TML object kind) are recorded alongside a witness copy of the live value they +were stashed next to, and the stash is used only when that witness still matches the +document's current value — an edit to the Ossie document since the stash was written +(retargeting a relationship, renaming a field) makes the stash stale for that key, and +the value is re-derived instead of silently reapplied to the wrong thing. + +The full key vocabulary is documented in `src/ossie_thoughtspot/constants.py`, grouped +by which Ossie object each key's entry attaches to (model, dataset, relationship, +field/metric). + +## Expression translation: what is not translated, and why + +**A ThoughtSpot formula is captured verbatim under a `THOUGHTSPOT` dialect entry. A +portable `ANSI_SQL` sibling is added only when the whole expression is a single bare +column reference — the one shape where portability is certain. Every other shape — a +function call, an operator expression, a runtime parameter, a formula cross-reference — +is recorded THOUGHTSPOT-only, with an issue explaining why no portable sibling was +produced.** No SQL dialect is ever re-rendered into another dialect in this converter, +in either direction. + +This is a deliberate design position, not an oversight, and it is worth stating plainly +rather than leaving it to be inferred from reading `tml_to_ossie.py`: + +1. **The specification's own default is pass-through.** `core-spec/spec.md` defines + `dialects[]` as a list of `{dialect, expression}` pairs precisely so a value a + converter cannot translate can still travel, tagged with the dialect it is valid in. + Emitting an untranslated ThoughtSpot expression under `THOUGHTSPOT` and stopping + there is using the mechanism the specification provides for exactly this case, not + working around a gap in it. +2. **No converter in this repository re-renders an expression from one SQL dialect into + another.** Every sibling converter that meets a dialect it does not natively speak + either passes the expression through under its own vendor dialect or falls back to + `ANSI_SQL` when the source already provides one — none parses a foreign dialect's + grammar and re-emits it in a different one. A hand-rolled reimplementation of that + translation, done once per converter, is exactly the kind of duplicated, easy-to-get- + subtly-wrong logic a shared SQL parser would exist to prevent — and no such shared + parser exists in this project today. Building one is out of scope for a single + converter to take on unilaterally. +3. **The reference converter (`converters/databricks`) tags its own vendor dialect and + reads that first**, falling back to `ANSI_SQL` only when the source document already + provides it — it does not translate a foreign dialect into its own either. This + converter follows the same shape: prefer `THOUGHTSPOT`, add `ANSI_SQL` only when it + can be produced with certainty, never invent a translation. + +**What "certain" means in practice.** A bare column reference (`[TABLE::Column]`) is the +one shape this converter resolves without ambiguity: the reference names a physical or +computed field this document already knows how to place, so the `ANSI_SQL` sibling is +just that field's own dataset-qualified name — no expression semantics are being +translated at all, only a reference being resolved. Everything past that — even a +composition ThoughtSpot's own documentation says is exactly equivalent to a portable +form — is left THOUGHTSPOT-only. `src/ossie_thoughtspot/expressions/reverse.py` records, +for testing and future use, which of ThoughtSpot's native functions compose into a +portable Ossie expression and which do not (`ReverseDisposition`: compose fully, +compose partially, resolve only to a dialect entry, or have no Ossie form at all) — but +this inventory is not yet called from the shipped `TML → Ossie` conversion path itself. +The current release is more conservative than what that inventory shows is possible: it +never guesses, so it never translates something it has not resolved to a certainty. + +**The complementary direction.** `Ossie → TML` faces the reverse problem: rendering an +Ossie specification construct as a ThoughtSpot formula. There, `expressions/catalog.py` +maps all 146 constructs the Ossie expression language defines to a ThoughtSpot rendering +— 108 with a native equivalent, 37 as a `sql_*_op` pass-through (opaque, +warehouse-dialect-specific SQL ThoughtSpot cannot introspect, logged at WARNING every +time), and 1 (`EXISTS_IN()`) with no representation ThoughtSpot has a slot for at all +(logged at ERROR, never silently dropped). This direction *can* translate constructs to +their ThoughtSpot equivalents because the Ossie expression language — unlike an +arbitrary ThoughtSpot formula — is the one grammar this converter fully parses; nothing +here reads or re-renders raw ThoughtSpot formula syntax, or any other vendor's SQL. + +Read together with the [coverage matrix](#coverage-matrix) below, this is the +converter's whole answer to "what does not survive a round trip and why": expressions +pass through by declared design; every other construct's loss is declared per row. + +## Coverage matrix + +Every construct this converter does not carry, with its consequence. Each row raises a +structured `ConverterIssue` at conversion time — nothing is dropped silently. + +| # | Construct | Limitation | Consequence | +|---|---|---|---| +| L1 | Object identity (`guid`, `obj_id`, `fqn`) | Not carried — instance-local by construction | A round-tripped document imports as a new object | +| L2 | Row-level security (`rls_rules`) | Not carried — rule expressions name instance-local groups. **ERROR severity**: a single issue is raised, its message naming every affected table | RLS is unrepresentable in Ossie core and is security-bearing; rules must be re-applied in the target for each table named in the error | +| L3 | Presentation artifacts (Answers, Liveboards, charts) | Out of scope — Ossie models semantics, not visualisations | No loss to the semantic model | +| L4 | Spotter coaching objects | Separate object types; `ai_context.examples` is not interchangeable | Coaching must be re-created in the target | +| L5 | Aggregate-model associations (`aggregated_models`) | Entries are GUIDs of other Models — instance-local | Query routing is silently disabled; the issue is the only signal | +| L6 | Worksheets, Views, Sets, Alerts, Model Aliases | Predecessors or layers, not models | Convert the Model the alias points at instead | + +## Known limitations + +Separate from the coverage matrix above — this covers identifier derivation +correctness, not TML constructs. + +`identifiers.py`'s `normalise()` folds diacritics via Unicode NFKD decomposition before +lowercasing and substituting, so accented Latin normalises correctly: `"Café"` -> +`"cafe"`, `"Ürün"` -> `"urun"`, `"Zürich"` -> `"zurich"`. A character with **no ASCII +decomposition** (Cyrillic, CJK, and similarly non-Latin scripts) is still dropped, not +transliterated, and a name with no ASCII alphanumerics surviving still raises +`ValueError`. There is also an open question NFKD does not settle: some accented Latin +folds to a *conventional* ASCII expansion rather than the bare decomposed letter — +German `"Müller"` decomposes to `"Muller"` here, not the conventional `"Mueller"` — and +choosing between them is a product decision left to a later change. + +## Rules + +Earlier revisions of this converter's comments and docstrings cited short, letter-plus- +number rule identifiers drawn from an internal construct/expression mapping reference +that is not part of this repository and is not publicly readable — a citation of that +shape in the shipped source was therefore unresolvable from inside this repository +alone. That has been resolved: every such citation has been rewritten to state the +substance it stood for directly, in place, so nothing shipped here depends on material +outside this repository. `tests/test_shipped_references.py` enforces this going +forward — it fails the suite if a citation of that shape reappears in any shipped +file. + +**Before declaring any expression untranslatable, consult the function mapping.** Many +window and LOD constructs have exact native equivalents; declaring one untranslatable +without checking is an error. + +## Generated reference documentation + +`docs/` holds four Markdown reference documents, generated from this converter's own +code rather than hand-authored — the code is the single source of truth for the +mapping each one describes, so a document maintained separately would only be able to +drift from it: + +- [`docs/expression-mapping.md`](docs/expression-mapping.md) — every Ossie + specification construct, its classification and its ThoughtSpot rendering, generated + from `expressions/catalog.py`'s `CATALOG`. +- [`docs/reverse-inventory.md`](docs/reverse-inventory.md) — every ThoughtSpot-only + function with no specification counterpart, and how it composes (or does not) back + into a portable Ossie expression, generated from `expressions/reverse.py`'s `REVERSE`. +- [`docs/datatype-map.md`](docs/datatype-map.md) — the bidirectional datatype map and + which types are declared lossy, generated from `datatypes.py`. +- [`docs/vendor-payload.md`](docs/vendor-payload.md) — every + `custom_extensions[THOUGHTSPOT]` key, its scope and how it is treated on the return + trip, generated from `constants.py`'s `STASH_KEY_CLASSIFICATION`. + +`tools/generate_reference_docs.py` produces all four; it is dev/tooling only — not a +runtime dependency, not part of the wheel, not a `[project.scripts]` entry point. +Regenerate with: + +```bash +uv run --python 3.13 python tools/generate_reference_docs.py +``` + +`tests/test_reference_docs_current.py` regenerates on every test run and compares the +result against the committed files byte-for-byte, so a `docs/*.md` file that has +drifted from the code it describes fails the suite rather than going unnoticed. + +## Development + +```bash +uv run --python 3.13 pytest tests/ -v +``` + +`uv run` syncs the `dev` dependency group (declared via PEP 735 `[dependency-groups]`, +not an extra — `pytest`, plus `jsonschema` and `hypothesis` for the schema-validation and +property-based suites) and runs the tests in one step — see +`.github/workflows/converter-thoughtspot-ci.yml` for the CI invocation this mirrors, +run across every Python version this package declares support for. + +## Future effort + +The Apache Ossie specification is still evolving. As it adds or changes fields, this +converter will be updated to track them — extending the mapping and coverage in both +directions to keep the conversion current and to support as much as the format allows +over time. `expressions/reverse.py`'s classified inventory of ThoughtSpot-only functions +is a candidate foundation for a future, more ambitious `TML → Ossie` composition +strategy, once that expansion is deliberately taken on rather than folded into this +release. diff --git a/converters/thoughtspot/docs/datatype-map.md b/converters/thoughtspot/docs/datatype-map.md new file mode 100644 index 00000000..4cef32a3 --- /dev/null +++ b/converters/thoughtspot/docs/datatype-map.md @@ -0,0 +1,76 @@ + + + + +# Ossie <-> ThoughtSpot Datatype Map + +The bidirectional Ossie <-> ThoughtSpot TML datatype map. The map is **not injective** — several Ossie types collapse onto one TML spelling and cannot be told apart on the way back; see "Not injective" below. + +## The closed Ossie datatype enum + +`Boolean`, `Date`, `DateTime`, `DateTimeTz`, `Decimal`, `Float`, `Integer`, `Opaque`, `String`, `Time` + +## Ossie -> TML + +| Ossie datatype | TML `data_type` (default) | Notes | +|---|---|---| +| `Boolean` | `BOOLEAN` | connection-dependent spelling — `BOOLEAN` by default, `BOOL` when the connection's own TML spells it that way | +| `Date` | `DATE` | exact, single spelling | +| `DateTime` | `DATE_TIME` | exact, single spelling | +| `DateTimeTz` | `DATE_TIME` | ThoughtSpot has no offset-aware column type; the value becomes DATE_TIME and returns as DateTime. | +| `Decimal` | `DOUBLE` | exact, single spelling | +| `Float` | `DOUBLE` | connection-dependent spelling — `DOUBLE` by default, `FLOAT` when the connection's own TML spells it that way; ThoughtSpot has one approximate numeric type, so Float and Decimal both become DOUBLE and return as Decimal. | +| `Integer` | `INT64` | exact, single spelling | +| `Opaque` | `VARCHAR` | Opaque is Ossie's marker for a type outside the portable vocabulary; it becomes VARCHAR and returns as String. | +| `String` | `VARCHAR` | exact, single spelling | +| `Time` | `VARCHAR` | ThoughtSpot has no time-of-day column type; the value becomes VARCHAR. | + +A column with no declared `datatype` at all infers `INT64` rather than raising — `datatype` is optional in Ossie, but TML rejects a column with no `db_column_properties` block at all. + +## TML -> Ossie + +| TML `data_type` | Ossie datatype | +|---|---| +| `BOOL` | `Boolean` | +| `BOOLEAN` | `Boolean` | +| `DATE` | `Date` | +| `DATE_TIME` | `DateTime` | +| `DOUBLE` | `Decimal` | +| `FLOAT` | `Float` | +| `INT64` | `Integer` | +| `VARCHAR` | `String` | + +A TML `data_type` outside this map returns no Ossie datatype at all — `datatype` is optional in Ossie, so omitting it is preferred over inventing one. + +## Not injective — declared losses + +| Ossie datatype | Why the round trip is lossy | +|---|---| +| `DateTimeTz` | ThoughtSpot has no offset-aware column type; the value becomes DATE_TIME and returns as DateTime. | +| `Float` | ThoughtSpot has one approximate numeric type, so Float and Decimal both become DOUBLE and return as Decimal. | +| `Opaque` | Opaque is Ossie's marker for a type outside the portable vocabulary; it becomes VARCHAR and returns as String. | +| `Time` | ThoughtSpot has no time-of-day column type; the value becomes VARCHAR. | + diff --git a/converters/thoughtspot/docs/expression-mapping.md b/converters/thoughtspot/docs/expression-mapping.md new file mode 100644 index 00000000..1c6c5873 --- /dev/null +++ b/converters/thoughtspot/docs/expression-mapping.md @@ -0,0 +1,208 @@ + + + + +# Ossie -> ThoughtSpot Expression Mapping + +Every construct the Ossie expression language specification defines, mapped to its ThoughtSpot rendering (`Ossie -> ThoughtSpot`, the direction `expressions/catalog.py` drives). Rows follow the source's own definition order, which groups related constructs together (aggregates, then type conversion, date/time, string, math/conditional, operators, window functions) — that grouping exists only as source comments, not as data the code carries, so it is not reproduced as separate sections here. + +## Coverage + +| Classification | Count | Share | +|---|---|---| +| direct | 108 | 74% | +| passthrough | 37 | 25% | +| unmappable | 1 | 1% | +| **Total** | **146** | **100%** | + +## Constructs with no discrete specification table row + +9 `CATALOG` rows are real, intended constructs that `spec_construct_names()` cannot key on directly, because the upstream specification describes them in prose or a code fence rather than a table row with a `Syntax` column. Each is keyed via `CONVENTION_DIVERGENCES` instead, with the reason recorded per construct. + +| Construct | Why it has no discrete spec table row | +|---|---| +| `-x / +x (unary)` | unary +/- is named only in the 'Operator Precedence' list (core-spec/expression_language.md:142), never a table row | +| `CASE expr WHEN v1 THEN r1 ... END (simple)` | simple CASE is described only in the CASE Expression code fence (core-spec/expression_language.md:508-513) alongside searched CASE; the top-level summary table's single bare 'CASE WHEN' token covers the searched form and does not extend to this one | +| `Parentheses — expression grouping` | its Supported SQL Constructs row (core-spec/expression_language.md:120) carries no backtick token in either cell, the only marker the top-table extraction keys on | +| `DISTINCT aggregate modifier` | the DISTINCT modifier is described only in the Conditional Aggregations prose/code block (core-spec/expression_language.md:219-230), never a table row | +| `Column / metric reference — field, dataset.field` | its Supported SQL Constructs row (core-spec/expression_language.md:108) carries no backtick token in either cell, same reason as Parentheses | +| `EXISTS_IN()` | named only in the Reason column of the excluded 'Not Supported in Expressions' table (core-spec/expression_language.md:131), never in a table of its own | +| `OVER (PARTITION BY ... ORDER BY ...) clause` | the generic OVER syntax template (core-spec/expression_language.md:548-560) is a fenced code block, not a table | +| `Frame clause — ROWS BETWEEN ... / RANGE BETWEEN ...` | frame options are a bullet list under the OVER syntax section (core-spec/expression_language.md:556-560), not a table | +| `Window aggregation — AGG(expr) OVER (...)` | the Window Aggregations section (core-spec/expression_language.md:583-599) is prose and code examples, not a table | + +## Every construct + +| Ossie construct | Classification | ThoughtSpot rendering | Notes | +|---|---|---|---| +| `SUM(expr)` | direct | `sum ( {0} )` | — | +| `COUNT(expr)` | direct | `count ( {0} )` | Counts non-null values on both sides. | +| `COUNT(*)` | direct | `count ( {0} )` | ThoughtSpot has no count(*); the row count is count() over a column known to be non-null. The converter uses the dataset's primary_key when the model declares one, and raises an issue rather than guessing a column when it does not. | +| `COUNT(DISTINCT expr)` | direct | `unique count ( {0} )` | A space, not an underscore. count_distinct(...) is rejected by the formula parser. | +| `AVG(expr)` | direct | `average ( {0} )` | — | +| `MIN(expr)` | direct | `min ( {0} )` | ThoughtSpot min is aggregate-only — it never compares two columns row-wise. Scalar two-argument minima are LEAST, a separate row. | +| `MAX(expr)` | direct | `max ( {0} )` | Aggregate-only, as MIN. | +| `STDDEV(expr)` | direct | `stddev ( {0} )` | Sample standard deviation on both sides. | +| `STDDEV_POP(expr)` | passthrough | `STDDEV_POP({0})` — pass-through via `sql_number_aggregate_op` | ThoughtSpot stddev is sample-only; there is no population form, and substituting it would change the divisor from n-1 to n. | +| `STDDEV_SAMP(expr)` | direct | `stddev ( {0} )` | Specification alias for STDDEV (:171). | +| `VARIANCE(expr)` | direct | `variance ( {0} )` | Sample variance on both sides. | +| `VAR_POP(expr)` | passthrough | `VAR_POP({0})` — pass-through via `sql_number_aggregate_op` | Same divisor reason as STDDEV_POP. | +| `VAR_SAMP(expr)` | direct | `variance ( {0} )` | Specification alias for VARIANCE (:174). | +| `MEDIAN(expr)` | direct | `median ( {0} )` | — | +| `PERCENTILE_CONT(p) WITHIN GROUP (ORDER BY expr)` | passthrough | `PERCENTILE_CONT(0.75) WITHIN GROUP (ORDER BY {0})` — pass-through via `sql_number_aggregate_op` | No native percentile function. p is a literal in the specification's syntax, so it is baked into the template rather than passed as a placeholder. p = 0.5 is the one case with a native equivalent — median ( [x] ) — and the converter should prefer it. | +| `PERCENTILE_DISC(p) WITHIN GROUP (ORDER BY expr)` | passthrough | `PERCENTILE_DISC(0.75) WITHIN GROUP (ORDER BY {0})` — pass-through via `sql_number_aggregate_op` | As PERCENTILE_CONT; the discrete/interpolated distinction is preserved only because the template is emitted verbatim. | +| `APPROX_COUNT_DISTINCT(expr)` | passthrough | `APPROX_COUNT_DISTINCT({0})` — pass-through via `sql_int_aggregate_op` | ThoughtSpot's unique count ( [x] ) is the exact-semantics alternative: same answer to within the sketch's ~2% error, at exact-count cost. The converter emits the pass-through by default — the specification chose approximate deliberately — and offers the exact form as a documented downgrade. | +| `APPROX_PERCENTILE(expr, p)` | passthrough | `APPROX_PERCENTILE({0}, 0.5)` — pass-through via `sql_number_aggregate_op` | p baked into the template as for the exact percentiles. | +| `CAST` | direct | `per-type — see the target-type table below` | 5 of the 8 specified target types are direct; the other three — BOOLEAN, TIMESTAMP and TIME — fall back to a pass-through. | +| `TRY_CAST` | direct | `the same functions as CAST` | ThoughtSpot's to_integer / to_double / to_string already return NULL on failure, which is exactly TRY_CAST semantics — so the two rows share a mapping and it is CAST, not TRY_CAST, that is the imprecise one. A strict CAST that must error rather than null is not expressible; the converter records that in the issue log when the source distinguishes them. | +| `CURRENT_DATE or CURRENT_DATE()` | direct | `today ( )` | Both specification spellings map to the same function. | +| `CURRENT_TIMESTAMP or CURRENT_TIMESTAMP()` | direct | `now ( )` | — | +| `CURRENT_TIME or CURRENT_TIME()` | direct | `time ( now ( ) )` | ThoughtSpot has no current-time function, but time ( ) extracts the time part of a datetime, so the composition is exact. | +| `YEAR(date_expr)` | direct | `year ( {0} )` | — | +| `QUARTER(date_expr)` | direct | `quarter_number ( {0} )` | The function is quarter_number, not quarter. | +| `MONTH(date_expr)` | direct | `month_number ( {0} )` | Not month ( ) — ThoughtSpot's month returns the month NAME ('January'); month_number returns 1-12, which is what the specification means. Mapping to month would silently change the column's type from integer to string. | +| `DAY(date_expr)` | direct | `day ( {0} )` | Day of month, 1-31 on both sides. | +| `DAYOFYEAR(date_expr)` | direct | `day_number_of_year ( {0} )` | The function is day_number_of_year, not day_of_year. | +| `HOUR(timestamp_expr)` | direct | `hour_of_day ( {0} )` | The function is hour_of_day, not hour. | +| `MINUTE(timestamp_expr)` | passthrough | `MINUTE({0})` — pass-through via `sql_int_op` | No native minute-of-hour extractor; add_minutes and diff_minutes exist but neither extracts. | +| `SECOND(timestamp_expr)` | passthrough | `SECOND({0})` — pass-through via `sql_int_op` | As MINUTE. | +| `EXTRACT` | direct | `per-part — see the date-part table below` | Rewritten to the part's own ThoughtSpot function; there is no generic extractor. 8 of the 11 specified parts are direct (YEAR->year, QUARTER->quarter_number, MONTH->month_number, WEEK->week_number_of_year, DAY->day, DAYOFWEEK->day_number_of_week, DAYOFYEAR->day_number_of_year, HOUR->hour_of_day); MINUTE, SECOND and MILLISECOND fall back to sql_int_op. | +| `DATE_PART` | direct | `per-part — see the date-part table below` | Identical treatment to EXTRACT; the two spellings collapse onto one rewrite (:276-279). | +| `DATE_TRUNC(part, date_expr)` | direct | `per-precision — see the truncation table below` | ThoughtSpot has no date_trunc. The start_of_* family covers 7 of the 8 specified precisions ('year'->start_of_year, 'quarter'->start_of_quarter, 'month'->start_of_month, 'week'->start_of_week, 'day'->date ( ), 'hour'->start_of_hour, 'minute'->start_of_min — the function is start_of_min, not start_of_minute); 'second' falls back to sql_date_time_op. The specification says week truncation is Monday-start; ThoughtSpot's week start is an instance setting, so the converter verifies alignment and raises an issue when it cannot. | +| `DATEADD(part, amount, date_expr)` | direct | `per-part add_* — see the arithmetic table below` | Argument order differs: ThoughtSpot is add_days ( [d] , n ), the specification is DATEADD(day, n, d). Every specified part is reachable: day->add_days, week->add_weeks, month->add_months, year->add_years, minute->add_minutes, second->add_seconds, plus two by arithmetic on a coarser unit since there is no native add_quarters or add_hours: quarter->add_months ( [d] , 3 * n ), hour->add_minutes ( [d] , 60 * n ). | +| `DATEDIFF(part, start_date, end_date)` | direct | `per-part diff_* — see the arithmetic table below` | Argument order is reversed: ThoughtSpot is diff_days ( [end] , [start] ) — end first. Getting this wrong silently negates every duration in the model. day->diff_days, week->diff_weeks, month->diff_months, quarter->diff_quarters, year->diff_years, hour->diff_hours, minute->diff_minutes, second->diff_time (returns seconds). | +| `DATE '2024-01-15'` | direct | `to_date ( '{0}' , 'yyyy-MM-dd' )` | A bare '2024-01-15' in a ThoughtSpot formula is parsed as arithmetic (2024 - 1 - 15), so the typed literal must always be wrapped. to_date takes exactly two arguments, so the converter supplies the ISO format model; {0} is the literal date string. | +| `TIMESTAMP_NTZ '2024-01-15 10:30:00'` | passthrough | `CAST('2024-01-15 10:30:00' AS TIMESTAMP)` — pass-through via `sql_date_time_op` | to_date returns a DATE and drops the time part, so there is no native way to construct a wall-clock timestamp. A zero-placeholder template is a documented form of the pass-through — the document's own worked example is recorded verbatim here; a real occurrence's literal value is substituted per-occurrence when the template is built, out of this catalog's scope (same as CAST's per-type dispatch). | +| `TIME '10:30:00'` | passthrough | `CAST('10:30:00' AS TIME)` — pass-through via `sql_date_time_op` | ThoughtSpot has no TIME column type — time ( ) extracts a time FROM a datetime, it does not construct one — so the pass-through returns DATETIME and the date part is whatever the warehouse defaults to. Flagged with an issue for that reason, not only for the dialect. | +| `TO_DATE(string)` | direct | `to_date ( {0} , 'yyyy-MM-dd' )` | The single-argument ISO form. ThoughtSpot's to_date is strictly two-argument, so the converter supplies 'yyyy-MM-dd'. | +| `TO_TIMESTAMP(string)` | passthrough | `TO_TIMESTAMP({0})` — pass-through via `sql_date_time_op` | to_date is date-only; parsing to a timestamp would drop the time silently. | +| `TO_DATE(string, format)` | direct | `to_date ( {0} , )` | EXPERIMENTAL. Format tokens are translated, not passed through — see the format-token table. ThoughtSpot accepts Java/LDML tokens (yyyy-MM-dd) and strptime %-codes, which between them cover the specification's entire portable core. | +| `TO_TIMESTAMP(string, format)` | passthrough | `TO_TIMESTAMP({0}, 'YYYY-MM-DD HH24:MI:SS')` — pass-through via `sql_date_time_op` | EXPERIMENTAL. Date-only to_date again. The format model inside the template is the warehouse's, not Ossie's, so the token translation table does not apply — this is the sharpest case of the pass-through caveat. | +| `TO_CHAR(date_expr, format)` | passthrough | `TO_CHAR({0}, 'YYYY-MM')` — pass-through via `sql_string_op` | EXPERIMENTAL. ThoughtSpot has no general date formatter. Single-token formats do have native equivalents and the converter prefers them: 'YYYY' -> year_name ( [d] ), 'MONTH' -> month ( [d] ), 'DAY' -> day_of_week ( [d] ). Those three return locale-dependent text on both sides. | +| `CONCAT(str1, str2, ...)` | direct | `concat ( {0} , {1} , ... )` | N-ary on both sides. + does not concatenate in ThoughtSpot — it is numeric-only and the parser rejects string operands, so both \|\| and CONCAT land here. | +| `LENGTH(str)` | direct | `strlen ( {0} )` | Characters, not bytes, on both sides. | +| `LOWER(str)` | passthrough | `LOWER({0})` — pass-through via `sql_string_op` | There is no native lower in ThoughtSpot. | +| `UPPER(str)` | passthrough | `UPPER({0})` — pass-through via `sql_string_op` | There is no native upper in ThoughtSpot. LOWER/UPPER are the most-used functions in the whole passthrough set, and their absence is also what forces ILIKE and case-insensitive comparison into pass-throughs. | +| `TRIM(str)` | passthrough | `TRIM({0})` — pass-through via `sql_string_op` | There is no native trim in ThoughtSpot — live-verified 2026-07-29, rejected with 'Search did not find "trim ("'. The whole trim family is a pass-through, not just the one-sided forms. | +| `LTRIM(str)` | passthrough | `LTRIM({0})` — pass-through via `sql_string_op` | No native ltrim — live-verified 2026-07-29. | +| `RTRIM(str)` | passthrough | `RTRIM({0})` — pass-through via `sql_string_op` | As LTRIM. | +| `LEFT(str, n)` | direct | `left ( {0} , {1} )` | — | +| `RIGHT(str, n)` | direct | `right ( {0} , {1} )` | — | +| `SUBSTRING(str, start, length)` | direct | `substr ( {0} , {1} - 1 , {2} )` | Index base differs. ANSI SUBSTRING is 1-based; ThoughtSpot's substr is 0-based. The -1 is mandatory and is the single most likely off-by-one in the whole mapping. When start is an expression rather than a literal, the arithmetic is emitted rather than folded. | +| `REPLACE(str, from, to)` | passthrough | `REPLACE({0}, {1}, {2})` — pass-through via `sql_string_op` | There is no native replace in ThoughtSpot — live-verified 2026-07-29, rejected with 'Search did not find "replace ("'. | +| `SPLIT_PART(str, delimiter, part)` | passthrough | `SPLIT_PART({0}, {1}, {2})` — pass-through via `sql_string_op` | ThoughtSpot has no tokenising function at all — not split, split_part or an nth-occurrence search — so there is no composition to fall back on. | +| `POSITION(substr IN str)` | direct | `strpos ( {1} , {0} )` | Operand order is reversed (haystack first in ThoughtSpot) and the specification's infix IN form becomes a comma. 1-based, returning 0 when absent, on both sides. | +| `CHARINDEX(substr, str)` | direct | `strpos ( {1} , {0} )` | Specification alias for POSITION (:419) with the operands already in prefix order; the reversal is the same. | +| `CONTAINS(str, substr)` | direct | `contains ( {0} , {1} )` | Returns boolean on both sides. | +| `STARTSWITH(str, prefix)` | direct | `strpos ( {0} , {1} ) = 1` | There is no native starts_with — live-verified 2026-07-29. Still direct because the composition is exact and uses only native functions: strpos is 1-based, so a true prefix sits at position 1. | +| `ENDSWITH(str, suffix)` | direct | `substr ( {0} , strlen ( {0} ) - strlen ( {1} ) , strlen ( {1} ) ) = {1}` | There is no native ends_with — live-verified 2026-07-29. Direct by composition, as STARTSWITH. | +| `REGEXP_LIKE(str, pattern)` | passthrough | `REGEXP_LIKE({0}, {1})` — pass-through via `sql_bool_op` | Boolean return, so not sql_string_op. ThoughtSpot has no regular-expression support of any kind. | +| `REGEXP_EXTRACT(str, pattern)` | passthrough | `REGEXP_SUBSTR({0}, {1})` — pass-through via `sql_string_op` | The function name inside the template is dialect-specific — Snowflake spells it REGEXP_SUBSTR, others REGEXP_EXTRACT — so the converter selects it from the connection's dialect and raises an issue when the dialect is unknown. | +| `REGEXP_REPLACE(str, pattern, replacement)` | passthrough | `REGEXP_REPLACE({0},{1},{2})` — pass-through via `sql_string_op` | Name is portable; the pattern dialect (POSIX vs PCRE, backreference syntax) is not. | +| `REGEXP_COUNT(str, pattern)` | passthrough | `REGEXP_COUNT({0}, {1})` — pass-through via `sql_int_op` | Integer return. | +| `ABS(x)` | direct | `abs ( {0} )` | — | +| `ROUND(x, d)` | direct | `round ( {0} , {1} )` | — | +| `FLOOR(x)` | direct | `floor ( {0} )` | — | +| `CEIL(x)` | direct | `ceil ( {0} )` | Specification alias pair CEIL(x) / CEILING(x); both spellings map to ceil. | +| `TRUNC(x, d)` | passthrough | `TRUNC({0}, {1})` — pass-through via `sql_double_op` | Specification alias pair TRUNC(x, d) / TRUNCATE(x, d). ThoughtSpot has no truncation function. floor agrees with TRUNC only for x >= 0 and d = 0, and round disagrees at every half-value, so neither is a safe substitute. | +| `MOD(x, y)` | direct | `mod ( {0} , {1} )` | Sign-of-result for negative operands follows the warehouse on both sides. | +| `SIGN(x)` | direct | `if ( {0} > 0 ) then 1 else if ( {0} < 0 ) then -1 else 0` | No native sign, but the three-way result is exactly expressible as an if chain. The else 0 is required — ThoughtSpot rejects an if chain with no else. | +| `POWER(x, y)` | direct | `pow ( {0} , {1} )` | The function is pow. power is rejected by the parser. | +| `SQRT(x)` | direct | `sqrt ( {0} )` | — | +| `EXP(x)` | direct | `exp ( {0} )` | — | +| `LN(x)` | direct | `ln ( {0} )` | — | +| `LOG(base, x)` | direct | `safe_divide ( ln ( {1} ) , ln ( {0} ) )` | ThoughtSpot has fixed-base log2 and log10 only; base is a runtime argument here, not a literal known at catalog time, so the general change-of-base composition is the one template that is exact for every base. safe_divide rather than / guards base = 1. | +| `LOG10(x)` | direct | `log10 ( {0} )` | — | +| `SIN(x)` | direct | `sin ( {0} * 180 / 3.14159265358979 )` | ThoughtSpot trigonometry is in degrees; the specification is in radians. The conversion is mandatory — a bare sin ( {0} ) returns the sine of x degrees and is wrong for every non-zero input. | +| `COS(x)` | direct | `cos ( {0} * 180 / 3.14159265358979 )` | Degrees, as SIN. | +| `TAN(x)` | direct | `tan ( {0} * 180 / 3.14159265358979 )` | Degrees, as SIN. | +| `ASIN(x)` | direct | `( asin ( {0} ) * 3.14159265358979 / 180 )` | Inverse functions convert the other way: ThoughtSpot returns degrees, the specification expects radians. | +| `ACOS(x)` | direct | `( acos ( {0} ) * 3.14159265358979 / 180 )` | Degrees -> radians, as ASIN. | +| `ATAN(x)` | direct | `( atan ( {0} ) * 3.14159265358979 / 180 )` | Degrees -> radians, as ASIN. | +| `ATAN2(y, x)` | passthrough | `ATAN2({0}, {1})` — pass-through via `sql_double_op` | atan2 is not a two-argument atan — it is quadrant-aware and defined where x = 0. Composing it from atan plus sign tests is possible but the branch table is easy to get wrong at the axes, so the pass-through is the honest mapping. | +| `RADIANS(degrees)` | direct | `{0} * 3.14159265358979 / 180` | No native radians; the arithmetic is exact and dialect-free. | +| `DEGREES(radians)` | direct | `{0} * 180 / 3.14159265358979` | No native degrees; as RADIANS. | +| `PI()` | direct | `3.14159265358979` | No native pi. The literal is emitted at the precision ThoughtSpot's own documented composites use; sql_double_op ( "pi()" ) is available where full warehouse precision matters. | +| `GREATEST(x, y, ...)` | direct | `greatest ( {0} , {1} , ... )` | Not max. ThoughtSpot's max is an aggregate; greatest is the row-wise N-ary function. Mapping GREATEST to max would collapse the column to one value and also flip it from attribute to measure. | +| `LEAST(x, y, ...)` | direct | `least ( {0} , {1} , ... )` | Not min, for the same reason as GREATEST. | +| `IF(condition, true_result, false_result)` | direct | `if ( {0} ) then {1} else {2}` | The parentheses around the condition are mandatory for TML import — without them the parser reports "Expecting keyword '('". Applies to every condition shape, including a bare BOOL column reference. | +| `IFF(condition, true_result, false_result)` | direct | `if ( {0} ) then {1} else {2}` | Specification alias for IF. | +| `NULLIF(expr1, expr2)` | direct | `nullif ( {0} , {1} )` | — | +| `COALESCE(expr1, expr2, ...)` | direct | `ifnull ( {0} , ifnull ( {1} , {2} ) )` | ThoughtSpot's ifnull is strictly two-argument, so an N-ary COALESCE becomes a right-nested chain. Two arguments is the common case and needs no nesting. | +| `IFNULL(expr, default)` | direct | `ifnull ( {0} , {1} )` | — | +| `NVL(expr, default)` | direct | `ifnull ( {0} , {1} )` | Specification alias for two-argument COALESCE. | +| `NVL2(expr, not_null_result, null_result)` | direct | `if ( isnotnull ( {0} ) ) then {1} else {2}` | No native three-way null function; the composition is exact. | +| `ZEROIFNULL(expr)` | direct | `ifnull ( {0} , 0 )` | — | +| `NULLIFZERO(expr)` | direct | `nullif ( {0} , 0 )` | — | +| `+` | direct | `{0} + {1}` | Numeric only. ThoughtSpot's + rejects string operands, so a + that concatenates on the source side must become concat ( ). The specification does not overload +, so this only bites when translating a dialect expression. | +| `-` | direct | `{0} - {1}` | — | +| `*` | direct | `{0} * {1}` | — | +| `/` | direct | `{0} / {1}` | Both yield NULL (or a warehouse error) on divide-by-zero. ThoughtSpot's safe_divide returns 0, not NULL, so it is not a faithful substitute and is used only where the source itself guards the denominator. | +| `%` | direct | `mod ( {0} , {1} )` | ThoughtSpot has no % operator — the modulo is the function. | +| `-x / +x (unary)` | direct | `per-spelling — see note` | CONVENTION_DIVERGENCE: unary +/- is named only in the 'Operator Precedence' list, never a table row. This row merges two Ossie spellings that need DIFFERENT output — unary minus is -[x] (negation), unary plus is the identity ([x] unchanged) — so a single {0}-substitutable template would be wrong for whichever spelling didn't produce it: an earlier draft used template="-{0}", which is correct for -x but silently negates a parsed +x node (right arg count, wrong semantics, no exception — the arg-count guard cannot catch it). Forced external dispatch instead, the same treatment as TRUE, FALSE below and CAST's per-type table: the caller must choose -{0} or {0} unchanged based on which spelling it parsed, rather than getting a plausible-looking wrong answer from this row. Unary minus is where the bare-date-literal trap originates: '2024-05-01' unquoted is parsed as 2024 - 5 - 1. Date literals are always wrapped in to_date ( ). | +| `=` | direct | `{0} = {1}` | — | +| `<>` | direct | `{0} <> {1}` | — | +| `!=` | direct | `{0} != {1}` | ThoughtSpot accepts both inequality spellings, so the two rows are independent and both direct. | +| `<` | direct | `{0} < {1}` | — | +| `>` | direct | `{0} > {1}` | — | +| `<=` | direct | `{0} <= {1}` | — | +| `>=` | direct | `{0} >= {1}` | — | +| `expr1 AND expr2` | direct | `{0} and {1}` | Lower-case, infix. | +| `expr1 OR expr2` | direct | `{0} or {1}` | Lower-case, infix. | +| `NOT expr` | direct | `not ( {0} )` | Function form with parentheses, not a prefix operator — not [x] does not parse. | +| `BETWEEN` | direct | `{0} between {1} and {2}` | Inclusive on both sides. | +| `IN` | direct | `{0} in {{ {1} , {2} , ... }}` | Literal lists only on both sides — no subqueries. The curly-brace delimiter is confirmed, live-verified 2026-07-29: the round-parenthesis form is rejected with 'Expecting one of the valid keywords, such as, "ts_var", "{"'. It forces >- block-scalar YAML. The braces are doubled ({{ }}) in the template because emit_direct renders via str.format, which reads a single literal brace as the start of a field name (see test_emit.py's catalog-wide sweep). | +| `NOT IN` | direct | `not ( {0} in {{ {1} , {2} , ... }} )` | Emitted as a negated in rather than a not in keyword — the bare keyword form is not reliably accepted. Braces doubled for str.format, as IN above. | +| `str LIKE pattern` | direct | `per-pattern-shape — see note` | Prefix ('foo%') -> strpos ( {0} , 'foo' ) = 1; suffix ('%foo') -> substr ( {0} , strlen ( {0} ) - strlen ( 'foo' ) , strlen ( 'foo' ) ) = 'foo'; contains ('%foo%') -> contains ( {0} , 'foo' ). Only contains is a native function — starts_with and ends_with do not exist (live-verified 2026-07-29), so the first two shapes are compositions of native functions, same as the STARTSWITH/ENDSWITH rows. These three shapes are the overwhelming majority of LIKE use. Interior wildcards and any _ single-character wildcard have no native form and fall back to sql_bool_op ( "{0} LIKE {1}" , [s] , [pattern] ). The per-pattern-shape dispatch is out of this catalog's scope, same treatment as CAST's per-type dispatch — the actual pattern literal is a runtime value, not known at catalog-construction time. | +| `str ILIKE pattern` | passthrough | `{0} ILIKE {1}` — pass-through via `sql_bool_op` | Case-insensitive matching has no native form, and the usual workaround — fold both sides with lower — is itself a pass-through, so there is nothing to compose from. | +| `IS NULL` | direct | `isnull ( {0} )` | — | +| `IS NOT NULL` | direct | `isnotnull ( {0} )` | Native, so not composed as not ( isnull ( ) ). | +| `IS DISTINCT FROM` | direct | `if ( isnull ( {0} ) and isnull ( {1} ) ) then false else if ( isnull ( {0} ) or isnull ( {1} ) ) then true else {0} != {1}` | No native null-safe comparison, but the three-case truth table is exactly expressible. The nesting order matters: both-null must be tested before either-null. | +| `IS NOT DISTINCT FROM` | direct | `if ( isnull ( {0} ) and isnull ( {1} ) ) then true else if ( isnull ( {0} ) or isnull ( {1} ) ) then false else {0} = {1}` | The negation of the row above, written directly rather than wrapped in not ( ) — one fewer nesting level for the parser. | +| `CASE WHEN` | direct | `if ( c1 ) then r1 else if ( c2 ) then r2 else d` | The searched CASE WHEN c1 THEN r1 ... ELSE d END form. No native CASE; the chain is else if, two words. The final else is mandatory and must be type-matched — else 0 for a measure, else '' for an attribute. Omitting it raises 'Unknown data type', and a CASE with no ELSE (legal in the specification, yielding NULL) therefore needs one synthesised. The branch count is unbounded, so the template uses symbolic c1/r1/c2/r2/d names rather than being forced into a fixed {0}/{1} scheme — the same out-of-scope-dispatch treatment as CAST's per-type table. | +| `CASE expr WHEN v1 THEN r1 ... END (simple)` | direct | `if ( [expr] = v1 ) then r1 else if ( [expr] = v2 ) then r2 else d` | CONVENTION_DIVERGENCE: the simple CASE form is described only in the CASE Expression code fence, never a table row. Expanded to the searched form with an explicit equality per branch. expr is repeated per branch, so a converter should hoist an expensive expr into its own formula first. Symbolic template, as CASE WHEN above, for the same unbounded-branch-count reason. | +| str1 \|\| str2 | direct | `concat ( {0} , {1} )` | ThoughtSpot has no concatenation operator at all — + is numeric-only — so \|\| and CONCAT share one target. | +| `Parentheses — expression grouping` | direct | `( {0} )` | CONVENTION_DIVERGENCE: its Supported SQL Constructs row carries no backtick token in either cell, the only marker the top-table extraction keys on. Precedence is the standard SQL ordering on the Ossie side. The converter emits explicit parentheses around every rewritten sub-expression rather than relying on the two languages agreeing about precedence — cheap, and it removes a whole class of silent arithmetic errors. | +| `TRUE, FALSE` | direct | `true / false` | The Boolean Functions table's Syntax cell merges TRUE and FALSE into one comma-joined entry, matching what spec_construct_names() extracts. Which of the two lower-case literals is emitted depends on which the source wrote — TRUE -> true, FALSE -> false — resolved per-occurrence, out of this catalog's scope (same as CAST's per-type dispatch). A bare BOOL column reference used as a condition still needs its parentheses: if ( [T::flag] ) then ... parses, if [T::flag] then ... does not. | +| `DISTINCT aggregate modifier` | passthrough | `SUM(DISTINCT {0})` — pass-through via `sql_number_aggregate_op` | CONVENTION_DIVERGENCE: described only in the Conditional Aggregations prose/code block, never a table row. The specification allows DISTINCT on SUM as well as COUNT. ThoughtSpot has exactly one distinct-aware aggregate — unique count — which is COUNT(DISTINCT) and already has its own row. Every other DISTINCT aggregate is a pass-through. | +| `Column / metric reference — field, dataset.field` | direct | `[TABLE::Column], or [Formula Name] for a metric` | CONVENTION_DIVERGENCE: its Supported SQL Constructs row carries no backtick token in either cell, same reason as Parentheses. Always rewritten from resolved metadata, never passed through textually — the rewrite, the case-sensitivity rules and the display-name-versus-identifier problem are out of this catalog's scope. | +| `EXISTS_IN()` | unmappable | — | CONVENTION_DIVERGENCE: named only in the Reason column of the excluded 'Not Supported in Expressions' table, never in a table of its own. The single unmappable row in the whole 146-row catalog: named at :131 as the sanctioned way to filter on a subquery, but defined nowhere in the specification — no signature, no argument order, no semantics, absent from every function table. Even given a signature, ThoughtSpot's nearest capability is a sql_bool_op subquery template that requires a fully-qualified warehouse table name, which is not derivable from an Ossie expression. | +| `ROW_NUMBER() OVER (...)` | passthrough | `ROW_NUMBER() OVER (PARTITION BY {0} ORDER BY {1})` — pass-through via `sql_int_aggregate_op` | ThoughtSpot's rank is competition rank, not a row number, so it is not a substitute. Wrap in group_aggregate so the partition column reaches the GROUP BY even when the user's search omits it. | +| `RANK() OVER (...)` | direct | `rank ( sum ( [m] ) , 'desc' )` | direct for one shape only, and the boundary is proven rather than asserted: the global, ORDER BY-only form over an aggregate. Live-confirmed 2026-07-30: rank ( sum ( [m] ) , 'desc' ) and 'asc' both validate, and the arity is enforced at exactly two — a third argument in any shape (bare attribute, { [attr] }, or query_groups ( )) is rejected with 'Function rank expects only 2 arguments', so an explicit PARTITION BY is provably not expressible. Two further live-proven restrictions: the first argument must be aggregated (rank ( [m] , 'desc' ) -> 'Function rank expects 1st argument to be aggregated'), so an Ossie ORDER BY has no native target either; and it may not be a group_aggregate ( ... ), so the partition cannot be smuggled in through the measure. Every non-covered shape falls back to sql_int_aggregate_op ( "RANK() OVER (PARTITION BY {0} ORDER BY SUM({1}) DESC)" , ... ), wrapped in group_aggregate. Query-context caveat: rank carries no dynamic partition but it is evaluated over the query's result rows, so the covered shape is faithful to RANK() OVER (ORDER BY ...) only when the search returns the grain the expression assumed — a query-time semantic no import probe can observe, taken from ThoughtSpot's formula documentation rather than this run. The direction string is not validated at import ('descending' was accepted), so acceptance proves the call shape, never the ordering. | +| `DENSE_RANK() OVER (...)` | passthrough | `dense_rank() over (order by sum({0}) desc)` — pass-through via `sql_int_aggregate_op` | ThoughtSpot's rank skips ranks after a tie; dense ranking has no native form — live-confirmed 2026-07-30, dense_rank ( ... ) rejected with 'Search did not find "dense_rank ( sum ("'. Passthrough is correct: no native ThoughtSpot construct produces dense-rank semantics. | +| `NTILE(n) OVER (...)` | passthrough | `NTILE(4) OVER (ORDER BY SUM({0}))` — pass-through via `sql_int_aggregate_op` | n is a literal, baked into the template, as the aggregate percentiles are. | +| `PERCENT_RANK() OVER (...)` | direct | `1 - rank_percentile ( sum ( [m] ) , 'asc' ) / 100` | ThoughtSpot's rank_percentile is documented as (1.0 - PERCENT_RANK() OVER (ORDER BY ...)) * 100, so the inverse is exact. Two adjustments are both required: the scale (ThoughtSpot 0-100, specification 0-1) and the inversion. Dropping either produces a plausible-looking column that is wrong everywhere. Same shape restriction as RANK, and the same live-proven boundary — rank_percentile is also fixed at exactly two arguments ('Function rank_percentile expects only 2 arguments', live-verified 2026-07-30), so it too is global-only and an explicit PARTITION BY falls back to sql_number_aggregate_op ( "PERCENT_RANK() OVER (PARTITION BY {0} ORDER BY SUM({1}))" , ... ). Same evidence-class caveat as RANK: the arity is probe-proven, the global-window semantic is documentation-derived. CUME_DIST is deliberately NOT given this same composition — see that row. | +| `CUME_DIST() OVER (...)` | passthrough | `CUME_DIST() OVER (ORDER BY SUM({0}))` — pass-through via `sql_number_aggregate_op` | rank_percentile is NOT a substitute, despite PERCENT_RANK's row looking equivalent: PERCENT_RANK divides by n - 1 and starts at 0; CUME_DIST divides by n and ends at 1. They agree on no row of a tie-free window except the last, so there is no native fallback at all for this row. | +| `LAG(expr, offset, default) OVER (...)` | passthrough | `LAG({0}, 1) OVER (PARTITION BY {1} ORDER BY {2})` — pass-through via `sql_number_aggregate_op` | Reclassified direct -> passthrough 2026-07-30. The native idiom moving_sum ( [m] , n , -n , [ord] ) is real and validates (a frame of n PRECEDING to n PRECEDING) but is not equivalent to any OVER shape: moving_sum has no partition slot, and ThoughtSpot completes the partition from the query's own dimensions instead. So an Ossie LAG with a PARTITION BY cannot be expressed, and one without a PARTITION BY still cannot, because ThoughtSpot's partition is not empty. The converter emits the pass-through by default and offers the native moving_sum idiom as a documented downgrade the user must accept: correct exactly when the search's dimensions are the intended partition. The default argument has no equivalent in the native idiom — ThoughtSpot yields null outside the frame — a second reason the native form is a downgrade (the pass-through carries default fine). Subject to the same aggregation and physical-ORDER-BY-column constraints as the rest of this family. Variant recorded here is the documented default (sql_number_aggregate_op); the typed sibling applies for a non-numeric expr — LAG returns its argument's own type, not an aggregate, so a string-typed expr (LAG(order_status, 1) OVER (...)) needs the typed sibling, not this default, or it imports cleanly and aggregates wrongly. | +| `LEAD(expr, offset, default) OVER (...)` | passthrough | `LEAD({0}, 1) OVER (PARTITION BY {1} ORDER BY {2})` — pass-through via `sql_number_aggregate_op` | Mirror of LAG, reclassified for the same reason and on the same date. The native downgrade is moving_sum ( [m] , -n , n , [ord] ) — ThoughtSpot's start/end arguments use opposite sign conventions, so a forward offset is a negative start (both live-confirmed 2026-07-30). Same default limitation as LAG. Variant recorded here is the documented default (sql_number_aggregate_op); the typed sibling applies for a non-numeric expr, same reason as LAG's note — LEAD returns its argument's own type, not an aggregate. | +| `FIRST_VALUE(expr) OVER (...)` | direct | `first_value ( sum ( [m] ) , query_groups ( ) , {{ [T::date] }} )` | The section's exception, and the only window row whose direct verdict survived the 2026-07-30 rework — first_value takes a genuine explicit partition argument and a genuine explicit order axis, so the formula does define its own window. Live-confirmed 2026-07-30: query_groups ( ), a fixed single-column { [attr] }, a multi-column { [a] , [b] }, the grand-total { } and the dynamic query_groups ( ) - { [attr] } all validate in the partition slot, so a static Ossie PARTITION BY list maps straight onto it. The axis slot is typed and enforced — a bare column reference is rejected with 'Function last_value expects 3rd argument to be List', so the { } braces are mandatory (and force >- block-scalar YAML on the document side; doubled here as {{ }} because emit_direct renders via str.format, the same fix the IN/NOT IN rows above need for the same reason — verified by calling emit_direct and checking the rendered output has single braces again). Two boundaries remain: ThoughtSpot's first_value is a semi-additive function over a date axis rather than a general window function, so an OVER shape with a row frame other than the whole partition falls back to sql_number_aggregate_op ( "FIRST_VALUE({0}) OVER (...)" , ... ); and the axis column's type is not validated at import (a VARCHAR axis was accepted), so acceptance proves the call shape, not that the axis is temporal. | +| `LAST_VALUE(expr) OVER (...)` | direct | `last_value ( sum ( [m] ) , query_groups ( ) , {{ [T::date] }} )` | Same conditions, same live evidence and same fallback as FIRST_VALUE. last_value_in_period and first_value_in_period also validate in the identical three-argument shape and are the period-completeness variants (see the reverse-direction table) — out of this row's scope. Braces doubled on the axis argument for the same str.format reason as FIRST_VALUE. | +| `NTH_VALUE(expr, n) OVER (...)` | passthrough | `NTH_VALUE({0}, 2) OVER (ORDER BY {1})` — pass-through via `sql_number_aggregate_op` | ThoughtSpot's semi-additive functions reach only the first and last values of the axis — live-confirmed 2026-07-30, nth_value ( ... ) rejected with 'Search did not find "nth_value ( sum ("'. n is a literal, baked into the template, as NTILE's. Variant recorded here is the documented default (sql_number_aggregate_op); the typed sibling applies for a non-numeric expr, same reason as LAG's note — NTH_VALUE returns its argument's own type, not an aggregate. | +| `OVER (PARTITION BY ... ORDER BY ...) clause` | passthrough | `per-clause-shape — see note` — pass-through via `sql_number_aggregate_op` | CONVENTION_DIVERGENCE: the generic OVER syntax template is a fenced code block, not a table. Reclassified direct -> passthrough 2026-07-30. The previous verdict claimed a clean structural rewrite — 'PARTITION BY attrs becomes the group_aggregate grouping argument; ORDER BY becomes the window function's trailing attribute arguments' — but that holds for PARTITION BY alone and breaks the moment an ORDER BY is present, which is most window use. There are two disjoint targets and only one accepts a partition: an OVER clause with a PARTITION BY and no ORDER BY/frame is group_aggregate ( agg ( [m] ) , { [T::a] , [T::b] } , query_filters ( ) ) and is lossless; an OVER clause with an ORDER BY must target moving_*/cumulative_*, which have no partition slot at all. Live-confirmed accepted: a fixed single-column grouping { [T::pk] } inside group_aggregate (as a moving_* and a cumulative_* argument), and query_groups ( ) - { [attr] } / query_groups ( ) + { [attr] } inside group_aggregate. Live-confirmed rejected: moving_sum ( ... , [ord] , { [attr] } ) and moving_sum ( ... , [ord] , query_groups ( ) ), plus cumulative_sum ( ... , [ord] , { [attr] } ). Not probed: a bare { } or a bare query_groups ( ) as the group_aggregate grouping argument, and the query_groups ( ) form of the cumulative_sum rejection — those three cells rest on the formula reference, not this run. A partitioned, ordered window therefore has no native home and the whole clause is out of catalog scope for the general case — template records the dispatch rather than one substitutable body, same treatment as CAST's per-type table. Variant recorded here is the documented default (sql_number_aggregate_op); the typed sibling applies for a non-numeric aggregate. The reverse direction is lossy for the mirror-image reason — ThoughtSpot's ordered window functions add the query's own dimensions to the partition dynamically, which the specification cannot express. | +| `Frame clause — ROWS BETWEEN ... / RANGE BETWEEN ...` | direct | `per-frame-shape — see note` | CONVENTION_DIVERGENCE: frame options are a bullet list under the OVER syntax section, not a table. direct for the frame boundaries only — deliberately scoped, so the partition loss is counted once, on the OVER clause row, and not twice. ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -> cumulative_*. Bounded ROWS frames -> moving_* with n PRECEDING -> positive n, CURRENT ROW -> 0, n FOLLOWING -> negative -n. All four boundary shapes were live-confirmed, 2026-07-30 (moving_sum ( [m] , 2 , 0 , [ord] ), ( ... , 1 , -1 , ... ), ( ... , -1 , 1 , ... ), cumulative_sum ( [m] , [ord] )), and the positional signature is enforced — moving_sum ( [m] , [ord] ) is rejected with 'Function moving_sum expects 2nd argument to be Numeric'. RANGE frames fall back to sql_number_aggregate_op (the same variant the window-aggregation row below falls back to): ThoughtSpot's frames are row-positional, not value-ranged — live-verified on gapped dates, moving_* counts surviving rows regardless of the calendar distance between them — so a RANGE frame over a gapped sort column would silently return different numbers. A frame reaches ThoughtSpot natively only when the accompanying OVER clause declares no PARTITION BY; otherwise it is emitted verbatim inside the pass-through template the OVER row selects. Per-shape dispatch out of catalog scope, same treatment as CAST's per-type table. | +| `Window aggregation — AGG(expr) OVER (...)` | passthrough | `SUM({0}) OVER (PARTITION BY {1} ORDER BY {2} ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)` — pass-through via `sql_number_aggregate_op` | CONVENTION_DIVERGENCE: the Window Aggregations section is prose and code examples, not a table. Reclassified direct -> passthrough 2026-07-30, inheriting the OVER row's problem: the specification allows every aggregate as a window function, but every ordered ThoughtSpot target (cumulative_*, moving_*) completes its partition from the query. The unordered case remains lossless and is the group_aggregate path on the OVER row. The native family is also narrower than the specification's: cumulative_*/moving_* cover SUM, AVG, MIN and MAX only — live-confirmed 2026-07-30 that moving_count, moving_stddev and cumulative_count do not exist ('Search did not find "moving_count ("' and siblings) — so a windowed COUNT, MEDIAN, STDDEV or VARIANCE has a partitioned form via group_count/group_stddev/group_variance and no ordered or framed form of any kind. The frame is an exemplar, the same convention as NTILE's literal 4 (see the Construct.template docstring): ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW is the cumulative-aggregate boundary the Frame clause row above maps cumulative_* to, and is one concrete, valid frame among the ones a real occurrence could carry — a caller rebuilds the frame per occurrence, same as any other exemplar row. The mapping document's own cell for this row writes the frame as a literal ellipsis ('ROWS BETWEEN …'), which is prose shorthand for 'a frame clause goes here', not renderable SQL — transcribing it verbatim rendered warehouse syntax errors at query time, so this template supplies a concrete, valid frame instead. Variant recorded here is the documented default (sql_number_aggregate_op); the typed sibling applies for a non-numeric aggregate. The argument must still be an aggregate — a raw column reference cannot be nested inside window aggregation. | + diff --git a/converters/thoughtspot/docs/reverse-inventory.md b/converters/thoughtspot/docs/reverse-inventory.md new file mode 100644 index 00000000..8e55de67 --- /dev/null +++ b/converters/thoughtspot/docs/reverse-inventory.md @@ -0,0 +1,132 @@ + + + + +# ThoughtSpot -> Ossie Reverse Inventory + +ThoughtSpot's own native functions with no counterpart in the Ossie specification (`ThoughtSpot -> Ossie`, the reverse of the expression mapping above), and how each reaches — or does not reach — a portable Ossie expression. This inventory is not yet called from the shipped `TML -> Ossie` conversion path; see `converters/thoughtspot/README.md`'s "Expression translation" section for the converter's current, more conservative default. + +## Coverage + +| Disposition | Count | Share | Meaning | +|---|---|---|---| +| compose | 49 | 62% | A full, portable Ossie expression is produced. | +| partial | 8 | 10% | A real Ossie expression is produced, but it is provably incomplete. | +| dialect | 10 | 13% | Resolves to the Ossie `dialects[]` mechanism, not a portable expression. | +| stash | 12 | 15% | No Ossie expression exists at all; preserved verbatim for round-trip only. | +| **Total** | **79** | **100%** | | + +## Cross-cutting dispatch, not name-keyed + +Two checks apply before an ordinary lookup by name into `REVERSE`, so they are not rows of the table below: + +- **Fiscal-calendar argument.** Any call whose last argument is `'fiscal'`, `fiscal` stashes unconditionally, for any function name at all, before the name is looked up. +- **Hyperlink markup.** A `concat` call whose string arguments contain `{caption}` or `{/caption}` is redirected to the `concat (hyperlink markup)` row below; plain `concat` has a specification counterpart already covered by `CATALOG` and is not this module's concern. + +## Every entry + +| ThoughtSpot construct | Disposition | Composes to | Issue | Notes | +|---|---|---|---|---| +| `sum_if` | compose | `SUM(CASE WHEN {0} THEN {1} END)` | — | sum_if ( cond , x ) -> SUM(CASE WHEN cond THEN x END). | +| `count_if` | compose | `COUNT(CASE WHEN {0} THEN {1} END)` | — | count_if ( cond , x ) -> COUNT(CASE WHEN cond THEN x END). | +| `average_if` | compose | `AVG(CASE WHEN {0} THEN {1} END)` | — | average_if ( cond , x ) -> AVG(CASE WHEN cond THEN x END). | +| `min_if` | compose | `MIN(CASE WHEN {0} THEN {1} END)` | — | min_if ( cond , x ) -> MIN(CASE WHEN cond THEN x END). | +| `max_if` | compose | `MAX(CASE WHEN {0} THEN {1} END)` | — | max_if ( cond , x ) -> MAX(CASE WHEN cond THEN x END). | +| `stddev_if` | compose | `STDDEV(CASE WHEN {0} THEN {1} END)` | — | stddev_if ( cond , x ) -> STDDEV(CASE WHEN cond THEN x END). | +| `variance_if` | compose | `VARIANCE(CASE WHEN {0} THEN {1} END)` | — | variance_if ( cond , x ) -> VARIANCE(CASE WHEN cond THEN x END). | +| `unique_count_if` | compose | `COUNT(DISTINCT CASE WHEN {0} THEN {1} END)` | — | unique_count_if ( cond , x ) -> COUNT(DISTINCT CASE WHEN cond THEN x END). | +| `unique count` | compose | `COUNT(DISTINCT {0})` | — | ThoughtSpot's own spelling has a space, not an underscore. | +| `safe_divide` | compose | `COALESCE({0} / NULLIF({1}, 0), 0)` | — | The zero-not-null result is preserved by the explicit COALESCE. | +| `pow` | compose | `POWER({0}, {1})` | — | — | +| `log2` | compose | `LOG(2, {0})` | — | — | +| `strlen` | compose | `LENGTH({0})` | — | — | +| `strpos` | compose | `POSITION({1} IN {0})` | — | ThoughtSpot strpos(s, sub) -> Ossie POSITION(sub IN s); operand order reverses. | +| `substr` | compose | `SUBSTRING({0}, {1} + 1, {2})` | — | ThoughtSpot's substr is 0-based; the +1 is mandatory going this way. | +| `left` | compose | `LEFT({0}, {1})` | — | — | +| `right` | compose | `RIGHT({0}, {1})` | — | — | +| `sin` | compose | `SIN(RADIANS({0}))` | — | ThoughtSpot trigonometry is in degrees; the conversion reverses. | +| `cos` | compose | `COS(RADIANS({0}))` | — | ThoughtSpot trigonometry is in degrees; the conversion reverses. | +| `tan` | compose | `TAN(RADIANS({0}))` | — | ThoughtSpot trigonometry is in degrees; the conversion reverses. | +| `asin` | compose | `DEGREES(ASIN({0}))` | — | ThoughtSpot's inverse trig functions return degrees. | +| `acos` | compose | `DEGREES(ACOS({0}))` | — | ThoughtSpot's inverse trig functions return degrees. | +| `atan` | compose | `DEGREES(ATAN({0}))` | — | ThoughtSpot's inverse trig functions return degrees. | +| `to_integer` | compose | `CAST({0} AS INTEGER)` | — | — | +| `to_double` | compose | `CAST({0} AS DOUBLE)` | — | — | +| `to_string` | compose | `CAST({0} AS VARCHAR)` | — | — | +| `to_date` | compose | `TO_DATE({0}, {1})` | `TS-EXPR-FORMAT-TOKENS-PASSTHROUGH` · INFO | Judgment call: format-token reversal is deferred until an expression parser exists to do the translation. | +| `if` | compose | `CASE WHEN {0} THEN {1} ELSE {2} END` | — | if ( c ) then a else b -> CASE WHEN c THEN a ELSE b END, or IF(c, a, b). | +| `rank` | compose | dynamic — see `_compose_rank` in `reverse.py` | — | Global, ORDER-BY-only shape only — rank's arity is fixed at exactly two (live-confirmed), so there is never a partition to lose in this direction. | +| `rank_percentile` | compose | dynamic — see `_compose_rank_percentile` in `reverse.py` | — | Scale (0-100 -> 0-1) and inversion both reverse. | +| `moving_sum` | partial | dynamic — see `_compose_moving.._compose` in `reverse.py` | `TS-EXPR-PARTIAL-PARTITION` · WARNING | Frame and order translate exactly; the partition does not — a ThoughtSpot window formula cannot declare its own PARTITION BY, and the specification has no way to express that limitation. | +| `cumulative_sum` | partial | dynamic — see `_compose_cumulative.._compose` in `reverse.py` | `TS-EXPR-PARTIAL-PARTITION` · WARNING | Frame and order translate exactly; the partition does not — a ThoughtSpot window formula cannot declare its own PARTITION BY, and the specification has no way to express that limitation. | +| `moving_average` | partial | dynamic — see `_compose_moving.._compose` in `reverse.py` | `TS-EXPR-PARTIAL-PARTITION` · WARNING | Frame and order translate exactly; the partition does not — a ThoughtSpot window formula cannot declare its own PARTITION BY, and the specification has no way to express that limitation. | +| `cumulative_average` | partial | dynamic — see `_compose_cumulative.._compose` in `reverse.py` | `TS-EXPR-PARTIAL-PARTITION` · WARNING | Frame and order translate exactly; the partition does not — a ThoughtSpot window formula cannot declare its own PARTITION BY, and the specification has no way to express that limitation. | +| `moving_max` | partial | dynamic — see `_compose_moving.._compose` in `reverse.py` | `TS-EXPR-PARTIAL-PARTITION` · WARNING | Frame and order translate exactly; the partition does not — a ThoughtSpot window formula cannot declare its own PARTITION BY, and the specification has no way to express that limitation. | +| `cumulative_max` | partial | dynamic — see `_compose_cumulative.._compose` in `reverse.py` | `TS-EXPR-PARTIAL-PARTITION` · WARNING | Frame and order translate exactly; the partition does not — a ThoughtSpot window formula cannot declare its own PARTITION BY, and the specification has no way to express that limitation. | +| `moving_min` | partial | dynamic — see `_compose_moving.._compose` in `reverse.py` | `TS-EXPR-PARTIAL-PARTITION` · WARNING | Frame and order translate exactly; the partition does not — a ThoughtSpot window formula cannot declare its own PARTITION BY, and the specification has no way to express that limitation. | +| `cumulative_min` | partial | dynamic — see `_compose_cumulative.._compose` in `reverse.py` | `TS-EXPR-PARTIAL-PARTITION` · WARNING | Frame and order translate exactly; the partition does not — a ThoughtSpot window formula cannot declare its own PARTITION BY, and the specification has no way to express that limitation. | +| `group_aggregate` | compose | dynamic — see `_dispatch_group_aggregate` in `reverse.py` | — | Shape dispatch on the grouping/filter arguments — see _compose_grouped. | +| `group_sum` | compose | dynamic — see `_make_group_shorthand_dispatch.._dispatch` in `reverse.py` | — | Shorthand for group_aggregate(sum(m), ...) — same shape dispatch. Judgment call: the (m, grouping, filter) 3-argument shape is assumed by analogy with group_aggregate's live-confirmed form; the shorthand family's own arity was not independently live-tested. | +| `group_count` | compose | dynamic — see `_make_group_shorthand_dispatch.._dispatch` in `reverse.py` | — | Shorthand for group_aggregate(count(m), ...) — same shape dispatch. Judgment call: the (m, grouping, filter) 3-argument shape is assumed by analogy with group_aggregate's live-confirmed form; the shorthand family's own arity was not independently live-tested. | +| `group_stddev` | compose | dynamic — see `_make_group_shorthand_dispatch.._dispatch` in `reverse.py` | — | Shorthand for group_aggregate(stddev(m), ...) — same shape dispatch. Judgment call: the (m, grouping, filter) 3-argument shape is assumed by analogy with group_aggregate's live-confirmed form; the shorthand family's own arity was not independently live-tested. | +| `group_variance` | compose | dynamic — see `_make_group_shorthand_dispatch.._dispatch` in `reverse.py` | — | Shorthand for group_aggregate(variance(m), ...) — same shape dispatch. Judgment call: the (m, grouping, filter) 3-argument shape is assumed by analogy with group_aggregate's live-confirmed form; the shorthand family's own arity was not independently live-tested. | +| `last_value` | stash | — | `TS-EXPR-SEMI-ADDITIVE` · ERROR | The window clause itself round-trips; only the roll-up declaration is lost. | +| `first_value` | stash | — | `TS-EXPR-SEMI-ADDITIVE` · ERROR | The window clause itself round-trips; only the roll-up declaration is lost. | +| `last_value_in_period` | stash | — | `TS-EXPR-SEMI-ADDITIVE` · ERROR | The window clause itself round-trips; only the roll-up declaration is lost. | +| `first_value_in_period` | stash | — | `TS-EXPR-SEMI-ADDITIVE` · ERROR | The window clause itself round-trips; only the roll-up declaration is lost. | +| `sql_string_op` | dialect | dynamic — see `_dispatch_sql_op` in `reverse.py` | — | Resolves to the Ossie dialects[] mechanism for the connection's own dialect, not a portable expression — the right home for raw warehouse SQL. | +| `sql_int_op` | dialect | dynamic — see `_dispatch_sql_op` in `reverse.py` | — | Resolves to the Ossie dialects[] mechanism for the connection's own dialect, not a portable expression — the right home for raw warehouse SQL. | +| `sql_double_op` | dialect | dynamic — see `_dispatch_sql_op` in `reverse.py` | — | Resolves to the Ossie dialects[] mechanism for the connection's own dialect, not a portable expression — the right home for raw warehouse SQL. | +| `sql_bool_op` | dialect | dynamic — see `_dispatch_sql_op` in `reverse.py` | — | Resolves to the Ossie dialects[] mechanism for the connection's own dialect, not a portable expression — the right home for raw warehouse SQL. | +| `sql_date_op` | dialect | dynamic — see `_dispatch_sql_op` in `reverse.py` | — | Resolves to the Ossie dialects[] mechanism for the connection's own dialect, not a portable expression — the right home for raw warehouse SQL. | +| `sql_date_time_op` | dialect | dynamic — see `_dispatch_sql_op` in `reverse.py` | — | Resolves to the Ossie dialects[] mechanism for the connection's own dialect, not a portable expression — the right home for raw warehouse SQL. | +| `sql_string_aggregate_op` | dialect | dynamic — see `_dispatch_sql_op` in `reverse.py` | — | Resolves to the Ossie dialects[] mechanism for the connection's own dialect, not a portable expression — the right home for raw warehouse SQL. | +| `sql_int_aggregate_op` | dialect | dynamic — see `_dispatch_sql_op` in `reverse.py` | — | Resolves to the Ossie dialects[] mechanism for the connection's own dialect, not a portable expression — the right home for raw warehouse SQL. | +| `sql_number_aggregate_op` | dialect | dynamic — see `_dispatch_sql_op` in `reverse.py` | — | Resolves to the Ossie dialects[] mechanism for the connection's own dialect, not a portable expression — the right home for raw warehouse SQL. | +| `sql_date_time_aggregate_op` | dialect | dynamic — see `_dispatch_sql_op` in `reverse.py` | — | Resolves to the Ossie dialects[] mechanism for the connection's own dialect, not a portable expression — the right home for raw warehouse SQL. | +| `` | stash | — | `TS-EXPR-RUNTIME-PARAMETER` · ERROR | Synthetic key — not a callable name. See stash_runtime_parameter(). | +| `ts_username` | stash | — | `TS-EXPR-RUNTIME-IDENTITY` · ERROR | `ts_username` resolves signed-in-user identity at query time; an interchange document that carried it would describe an access-control decision, not semantics. Preserved verbatim for roundtrip. | +| `ts_groups` | stash | — | `TS-EXPR-RUNTIME-IDENTITY` · ERROR | `ts_groups` resolves signed-in-user identity at query time; an interchange document that carried it would describe an access-control decision, not semantics. Preserved verbatim for roundtrip. | +| `ts_groups_int` | stash | — | `TS-EXPR-RUNTIME-IDENTITY` · ERROR | `ts_groups_int` resolves signed-in-user identity at query time; an interchange document that carried it would describe an access-control decision, not semantics. Preserved verbatim for roundtrip. | +| `ts_org` | stash | — | `TS-EXPR-RUNTIME-IDENTITY` · ERROR | `ts_org` resolves signed-in-user identity at query time; an interchange document that carried it would describe an access-control decision, not semantics. Preserved verbatim for roundtrip. | +| `ts_email_domain` | stash | — | `TS-EXPR-RUNTIME-IDENTITY` · ERROR | `ts_email_domain` resolves signed-in-user identity at query time; an interchange document that carried it would describe an access-control decision, not semantics. Preserved verbatim for roundtrip. | +| `ts_var` | stash | — | `TS-EXPR-RUNTIME-IDENTITY` · ERROR | `ts_var` resolves signed-in-user identity at query time; an interchange document that carried it would describe an access-control decision, not semantics. Preserved verbatim for roundtrip. | +| `concat (hyperlink markup)` | stash | — | `TS-EXPR-HYPERLINK-MARKUP` · ERROR | Synthetic key, reached only via the content-pattern check in translate_thoughtspot -- plain concat (no markup) is out of this module's scope entirely. | +| `month` | compose | `TO_CHAR({0}, 'MONTH')` | `TS-EXPR-LOCALE-DEPENDENT` · WARNING | Name-returning form, distinct from month_number/year/day_number_of_week. | +| `year_name` | compose | `TO_CHAR({0}, 'YYYY')` | `TS-EXPR-LOCALE-DEPENDENT` · WARNING | Name-returning form, distinct from month_number/year/day_number_of_week. | +| `day_of_week` | compose | `TO_CHAR({0}, 'DAY')` | `TS-EXPR-LOCALE-DEPENDENT` · WARNING | Name-returning form, distinct from month_number/year/day_number_of_week. | +| `month_number_of_quarter` | compose | `MOD(MONTH({0}) - 1, 3) + 1` | — | — | +| `day_number_of_quarter` | compose | `DATEDIFF(day, DATE_TRUNC('quarter', {0}), {0}) + 1` | — | — | +| `week_number_of_month` | compose | `DATEDIFF(week, DATE_TRUNC('month', {0}), {0}) + 1` | `TS-EXPR-WEEK-START-ASSUMED` · WARNING | `week_number_of_month` is correct only if the target engine's week start agrees with the specification's fixed Monday start; ThoughtSpot's week start is an instance setting. Verify alignment before relying on this column. | +| `week_number_of_quarter` | compose | `DATEDIFF(week, DATE_TRUNC('quarter', {0}), {0}) + 1` | `TS-EXPR-WEEK-START-ASSUMED` · WARNING | `week_number_of_quarter` is correct only if the target engine's week start agrees with the specification's fixed Monday start; ThoughtSpot's week start is an instance setting. Verify alignment before relying on this column. | +| `is_weekend` | compose | `DATE_PART('dayofweek', {0}) IN (6, 7)` | `TS-EXPR-DAYOFWEEK-BASE` · WARNING | `is_weekend`'s member list (6, 7) uses ThoughtSpot's own DAYOFWEEK base (1 = Monday); the specification does not fix a base and engines disagree — confirm the target engine's base agrees before relying on this column. | +| `start_of_hour` | compose | `DATE_TRUNC('hour', {0})` | — | — | +| `start_of_min` | compose | `DATE_TRUNC('minute', {0})` | — | — | +| `date` | compose | `DATE_TRUNC('day', {0})` | — | — | +| `time` | compose | `CAST({0} AS TIME)` | — | — | +| `greatest` | compose | dynamic — see `_compose_variadic.._compose` in `reverse.py` | — | Never MAX — that would turn a row-wise attribute into an aggregate measure. | +| `least` | compose | dynamic — see `_compose_variadic.._compose` in `reverse.py` | — | Never MIN, for the same reason. | + diff --git a/converters/thoughtspot/docs/vendor-payload.md b/converters/thoughtspot/docs/vendor-payload.md new file mode 100644 index 00000000..e544c8bb --- /dev/null +++ b/converters/thoughtspot/docs/vendor-payload.md @@ -0,0 +1,108 @@ + + + + +# The `custom_extensions[THOUGHTSPOT]` Payload + +TML carries properties Ossie's core specification has no field for. `TML -> Ossie` stashes each one under a single `custom_extensions` entry attached to the Ossie object it came from; `Ossie -> TML` reads the same entry back. This page is generated from `constants.py`'s own key vocabulary and `STASH_KEY_CLASSIFICATION` — the table `test_stash_key_classification.py` enforces every key read on the `Ossie -> TML` direction must appear in. + +## Envelope + +Every `custom_extensions` entry this converter writes uses `vendor_name` `THOUGHTSPOT`. `data` is a single JSON-encoded string (never a nested object) whose own top-level `_v` field is the shape version (`1` today) — bumped only when the payload's shape changes, never for a value change; an unrecognised version is a hard failure rather than a silent misread. + +## Payload keys + +| Key | Scope | Classification | Treatment on the return trip | +|---|---|---|---| +| `tml_name` | Shared | shadows_derivable | Restored only if reconstructing it from the live document still agrees with the stashed value (self-verifying — no separate witness key); disagreement re-derives instead. | +| `db_column_name` | Field | shadows_derivable | Restored only if its witness companion key still matches the live document's current value; a mismatch means the document changed since the stash was written, so the value is re-derived instead. | +| `data_type` | Field | shadows_derivable | Restored only if its witness companion key still matches the live document's current value; a mismatch means the document changed since the stash was written, so the value is re-derived instead. | +| `column_properties` | Field | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `shape` | Metric | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `tml_object` | Dataset | shadows_derivable | Restored only if its witness companion key still matches the live document's current value; a mismatch means the document changed since the stash was written, so the value is re-derived instead. | +| `source_parts` | Dataset | shadows_derivable | Restored only if reconstructing it from the live document still agrees with the stashed value (self-verifying — no separate witness key); disagreement re-derives instead. | +| `connection_name` | Dataset | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `table_name` | Dataset | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `alias` | Dataset | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `table_properties` | Dataset | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `unsurfaced_columns` | Dataset | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. Each list entry is additionally checked against live coverage before being restored: an entry now covered by a live field is dropped rather than duplicated. | +| `sql_output_columns` | Dataset | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `on_expression` | Relationship | shadows_derivable | Restored only if its witness companion key still matches the live document's current value; a mismatch means the document changed since the stash was written, so the value is re-derived instead. | +| `type` | Relationship | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `cardinality` | Relationship | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `endpoints_swapped` | Relationship | shadows_derivable | Restored only if its witness companion key still matches the live document's current value; a mismatch means the document changed since the stash was written, so the value is re-derived instead. | +| `referencing_join` | Relationship | shadows_derivable | Restored only if reconstructing it from the live document still agrees with the stashed value (self-verifying — no separate witness key); disagreement re-derives instead. | +| `join_shape` | Relationship | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `unattributed_formulas` | Model | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `unrepresentable_joins` | Model | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `model_properties` | Model | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `parameters` | Model | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `filters` | Model | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `column_groups` | Model | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `lesson_plans` | Model | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `action_object_associations` | Model | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `constraints` | Model | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | +| `model_joins_with` | Model | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. | + +## Witness companion keys + +A `SHADOWS_DERIVABLE` key's stashed value is checked for currency before being restored; these are the witness copies that check does it against (see `stash.restore`). + +| Witness key | Checks currency for | +|---|---| +| `tml_object_source_witness` | `tml_object` | +| `data_type_ossie_datatype_witness` | `data_type` | +| `db_column_name_display_name_witness` | `db_column_name` | +| `endpoints_swapped_witness` | `endpoints_swapped` | +| `on_expression_equality_witness` | `on_expression` | + +## Nested keys under `source_parts` + +| Sub-key | Full path | +|---|---| +| `db` | `source_parts.db` | +| `db_table` | `source_parts.db_table` | +| `schema` | `source_parts.schema` | + +## `METRIC_STASH_SHAPE` value vocabulary + +| Constant | Value | +|---|---| +| `METRIC_SHAPE_COLUMN_AGGREGATION` | `column_aggregation` | +| `METRIC_SHAPE_FORMULA` | `formula` | +| `METRIC_SHAPE_SCALAR_FORMULA_PLUS_AGGREGATION` | `scalar_formula_plus_aggregation` | + +## Reclassified on a stash-only carrier + +The same key name, reclassified when it is read off a stash-only carrier (an `unrepresentable_joins[]` or `unattributed_formulas[]` entry) that has no independent Relationship/Metric/Field object of its own to diverge from. + +| Key | Classification (primary carrier) | Classification (stash-only carrier) | +|---|---|---| +| `on_expression` | shadows_derivable | information_only | +| `type` | information_only | information_only | +| `cardinality` | information_only | information_only | +| `column_properties` | information_only | information_only | + diff --git a/converters/thoughtspot/pyproject.toml b/converters/thoughtspot/pyproject.toml new file mode 100644 index 00000000..fe09593a --- /dev/null +++ b/converters/thoughtspot/pyproject.toml @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "pytest>=8.0", + # Dev/test-only: validates convert()'s output against core-spec/ossie-schema.json + # in tests/test_tml_to_ossie.py. Guarded there by pytest.importorskip("jsonschema") + # so its absence never fails the suite -- the package's only *runtime* dependency + # stays PyYAML. Same dev-group placement as converters/nvidia's pyproject.toml. + "jsonschema>=4.26.0", + # Dev/test-only: drives tests/test_roundtrip_properties.py. Guarded there by + # pytest.importorskip("hypothesis"), same discipline as jsonschema above -- + # the package's only runtime dependency stays PyYAML. Same dev-group + # placement as converters/databricks' own pyproject.toml. + "hypothesis>=6.0", +] + +[project] +name = "apache-ossie-thoughtspot" +version = "0.1.0" +description = "Convert between ThoughtSpot TML and the Apache Ossie semantic model" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] +requires-python = ">=3.10" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "ThoughtSpot" +] +dependencies = [ + "PyYAML>=6.0", +] + +[project.scripts] +ossie-thoughtspot = "ossie_thoughtspot.cli:main" + +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_thoughtspot"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev" +] diff --git a/converters/thoughtspot/src/ossie_thoughtspot/__init__.py b/converters/thoughtspot/src/ossie_thoughtspot/__init__.py new file mode 100644 index 00000000..0c7aba10 --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/__init__.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Bidirectional converter between ThoughtSpot TML and the Apache Ossie semantic model.""" + +__version__ = "0.1.0" diff --git a/converters/thoughtspot/src/ossie_thoughtspot/_yaml.py b/converters/thoughtspot/src/ossie_thoughtspot/_yaml.py new file mode 100644 index 00000000..72c2aaa4 --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/_yaml.py @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""YAML codec with the boolean resolver narrowed to YAML 1.2. + +PyYAML implements YAML 1.1, in which `on`, `off`, `yes`, `no`, `y` and `n` +resolve to booleans. TML uses such tokens as ordinary strings, so a bare +`yaml.safe_load` corrupts them silently. Only the boolean resolver is +narrowed here — no other YAML 1.1/1.2 divergence (e.g. octal/sexagesimal +number parsing) is addressed. + +Both directions matter. The loader stops 1.1 bool tokens becoming booleans; the +dumper quotes them on the way out so the next reader — which may be a 1.1 +implementation — cannot re-resolve them. +""" +import re + +import yaml + +from .errors import ConversionError + +#: YAML 1.2 core schema: only these spellings are booleans. +_YAML12_BOOL = re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$") + +#: Bare scalars YAML 1.1 resolves as booleans and YAML 1.2 does not. +_YAML11_ONLY_BOOLS = frozenset( + {"y", "Y", "yes", "Yes", "YES", "n", "N", "no", "No", "NO", + "on", "On", "ON", "off", "Off", "OFF"} +) + + +class Yaml12Loader(yaml.SafeLoader): + """SafeLoader with the YAML 1.1 boolean resolver narrowed to the 1.2 set.""" + + +# Drop the inherited bool resolver outright, then reinstate the 1.2-only one. +# Mutating in place would affect SafeLoader itself, so rebuild the mapping. +Yaml12Loader.yaml_implicit_resolvers = { + key: [(tag, regexp) for tag, regexp in resolvers if tag != "tag:yaml.org,2002:bool"] + for key, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items() +} +Yaml12Loader.add_implicit_resolver("tag:yaml.org,2002:bool", _YAML12_BOOL, list("tTfF")) + + +class Yaml12Dumper(yaml.SafeDumper): + """SafeDumper that quotes strings a YAML 1.1 reader would take for booleans.""" + + +def _represent_str(dumper: yaml.SafeDumper, data: str) -> yaml.ScalarNode: + style = "'" if data in _YAML11_ONLY_BOOLS else None + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style) + + +Yaml12Dumper.add_representer(str, _represent_str) + + +def load(text: str) -> object: + """Parse YAML text under YAML 1.2 boolean rules. + + A malformed document raises `ConversionError` naming the failure, never a + bare `yaml.YAMLError` traceback — the same never-a-bare-traceback contract + `stash.py` holds for malformed `custom_extensions` JSON. + """ + try: + return yaml.load(text, Loader=Yaml12Loader) + except yaml.YAMLError as exc: + raise ConversionError(f"malformed YAML: {exc}") from exc + + +def dump(data: object) -> str: + """Serialise to YAML, preserving insertion order and quoting 1.1 bool tokens. + + `allow_unicode=True` so a non-ASCII value (e.g. a display label) emits as + a literal character rather than a `\\xXX`/`\\uXXXX` escape — Ossie + documents are human-read YAML. + """ + return yaml.dump( + data, + Dumper=Yaml12Dumper, + sort_keys=False, + default_flow_style=False, + allow_unicode=True, + ) diff --git a/converters/thoughtspot/src/ossie_thoughtspot/cli.py b/converters/thoughtspot/src/ossie_thoughtspot/cli.py new file mode 100644 index 00000000..7d7d005e --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/cli.py @@ -0,0 +1,238 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Command-line interface for the Apache Ossie <-> ThoughtSpot TML converter. + + ossie-thoughtspot to-ossie ... -o [--issues ] + ossie-thoughtspot to-tml -o [--issues ] + +`to-ossie` reads a Model TML document plus the Table/SQL View documents it references +and writes one Apache Ossie semantic model. `to-tml` reads one Apache Ossie semantic +model and writes the corresponding TML document set -- one file per document, tables +before the model -- into an output directory. + +`-o`/`--output` is required in both directions: `to-ossie` writes exactly one file, and +`to-tml` writes a set of files that has no single-file stdout representation, so unlike +some sibling converters there is no "default: stdout" fallback here. + +Every declared loss or degradation the conversion records is written as a JSON array of +issues -- to `--issues` when given, to stderr otherwise -- and never mixed into the +document output. The process exits 1 when that issue log contains an ERROR-severity +issue, 0 otherwise: a conversion that only warned or informed about a declared loss is +still a successful conversion, and failing the exit code on a warning would just teach +scripts to ignore it. + +Neither subcommand overwrites an existing output file unless `--force` is given; without +it, a target that already exists refuses the run before anything is written. +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from . import _yaml, tml, tml_to_ossie, ossie_to_thoughtspot +from .errors import ConversionError +from .issues import IssueLog + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="ossie-thoughtspot", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = parser.add_subparsers(dest="command") + sub.required = True # set as attribute (the add_subparsers kwarg is 3.7+) + + to_ossie = sub.add_parser( + "to-ossie", help="ThoughtSpot TML documents -> one Apache Ossie semantic model" + ) + to_ossie.add_argument( + "tml_files", + nargs="+", + metavar="TML_FILE", + help="the Model TML document plus every Table/SQL View document it references", + ) + to_ossie.add_argument( + "-o", "--output", required=True, metavar="FILE", help="output Apache Ossie YAML file" + ) + to_ossie.add_argument( + "--issues", + metavar="FILE", + help="write the issue log as JSON to this path (default: stderr)", + ) + to_ossie.add_argument( + "--force", + action="store_true", + help="overwrite the output file (and --issues file) if either already exists; " + "without this flag, an existing target refuses the run before writing anything", + ) + + to_tml = sub.add_parser( + "to-tml", help="one Apache Ossie semantic model -> ThoughtSpot TML documents" + ) + to_tml.add_argument("ossie_file", metavar="OSSIE_FILE", help="Apache Ossie YAML document") + to_tml.add_argument( + "-o", + "--output", + required=True, + metavar="DIR", + help="output directory; one TML file per document is written here (tables before " + "the model), created if it does not already exist", + ) + to_tml.add_argument( + "--issues", + metavar="FILE", + help="write the issue log as JSON to this path (default: stderr)", + ) + to_tml.add_argument( + "--force", + action="store_true", + help="overwrite output files (and --issues file) if any already exist; without " + "this flag, an existing target refuses the run before writing anything", + ) + return parser + + +def _safe_target_path(directory: Path, filename: str) -> Path: + """`directory / filename`, refusing to resolve outside `directory`. + + `tml.dump_document_set` already sanitises `filename` (a document's `name` is + user-controlled TML content, and a hostile one -- `../../etc/passwd` -- previously + produced a path that escaped its output directory), so this should never trigger in + practice. It exists anyway because this module is the first caller that actually + writes these documents to disk, and a defence that lives only in the dumper is not + one this module can verify it still has. Both paths are resolved before comparing -- + a naive string-prefix check is wrong on a filesystem with symlinks in play, e.g. + macOS where `/tmp` is a symlink to `/private/tmp`. + """ + resolved_directory = directory.resolve() + target = (resolved_directory / filename).resolve() + if target != resolved_directory and resolved_directory not in target.parents: + raise ConversionError( + f"refusing to write {filename!r}: it resolves outside the output directory " + f"{resolved_directory}" + ) + return target + + +def _existing(paths: list[Path]) -> list[Path]: + return [p for p in paths if p.exists()] + + +def _refuse_overwrite(paths: list[Path]) -> str: + names = ", ".join(str(p) for p in paths) + return f"refusing to overwrite existing file(s): {names} (pass --force to overwrite)" + + +def _write_issues(issues: IssueLog, issues_path: Path | None) -> None: + """Issues as a JSON array -- to `issues_path` when given, to stderr otherwise. + + Always written, even when empty: a caller scripting against this output should be + able to rely on the shape (a JSON array) rather than on whether anything was said. + """ + text = json.dumps(issues.as_dicts(), indent=2) + if issues_path is not None: + issues_path.parent.mkdir(parents=True, exist_ok=True) + issues_path.write_text(text + "\n", encoding="utf-8") + else: + print(text, file=sys.stderr) + + +def _cmd_to_ossie(args: argparse.Namespace) -> int: + output_path = Path(args.output) + issues_path = Path(args.issues) if args.issues else None + + try: + texts = [(path, Path(path).read_text(encoding="utf-8")) for path in args.tml_files] + document_set = tml.load_document_set(texts) + result = tml_to_ossie.convert(document_set) + except (ConversionError, OSError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + targets = [output_path] + ([issues_path] if issues_path else []) + if not args.force: + existing = _existing(targets) + if existing: + print(f"Error: {_refuse_overwrite(existing)}", file=sys.stderr) + return 1 + + try: + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(_yaml.dump(result.model), encoding="utf-8") + _write_issues(result.issues, issues_path) + except OSError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + return 1 if result.issues.has_errors() else 0 + + +def _cmd_to_tml(args: argparse.Namespace) -> int: + output_dir = Path(args.output) + issues_path = Path(args.issues) if args.issues else None + + if output_dir.exists() and not output_dir.is_dir(): + print(f"Error: output path {output_dir} exists and is not a directory", file=sys.stderr) + return 1 + + try: + text = Path(args.ossie_file).read_text(encoding="utf-8") + ossie_document = _yaml.load(text) + if not isinstance(ossie_document, dict): + raise ConversionError(f"{args.ossie_file} is not an Ossie document: expected a mapping") + result = ossie_to_thoughtspot.convert(ossie_document) + files = tml.dump_document_set(result.documents) + # Path safety belongs here, before any write: a document name is user-controlled + # TML/Ossie content, and `dump_document_set` sanitises filenames for exactly this + # reason (see `_safe_target_path`). + targets = [(_safe_target_path(output_dir, name), text_) for name, text_ in files] + except (ConversionError, OSError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + all_targets = [path for path, _ in targets] + ([issues_path] if issues_path else []) + if not args.force: + existing = _existing(all_targets) + if existing: + print(f"Error: {_refuse_overwrite(existing)}", file=sys.stderr) + return 1 + + try: + output_dir.mkdir(parents=True, exist_ok=True) + for path, text_ in targets: + path.write_text(text_, encoding="utf-8") + _write_issues(result.issues, issues_path) + except OSError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + return 1 if result.issues.has_errors() else 0 + + +def main(argv: list[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + if args.command == "to-ossie": + return _cmd_to_ossie(args) + return _cmd_to_tml(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/converters/thoughtspot/src/ossie_thoughtspot/constants.py b/converters/thoughtspot/src/ossie_thoughtspot/constants.py new file mode 100644 index 00000000..5595c52e --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/constants.py @@ -0,0 +1,521 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Vocabulary constants. + +VENDOR_KEY and DIALECT hold the same string today and are deliberately separate +names. They are governed differently upstream: the vendor key needs no spec +change because `Vendor` is an `examples` list that accepts any string, while +the dialect is a closed enum — it was a pending apache/ossie#351 change, now +merged (see DIALECT_IS_REGISTERED). +""" + +from enum import Enum + +#: `custom_extensions[].vendor_name` value for ThoughtSpot-owned entries. +VENDOR_KEY = "THOUGHTSPOT" + +#: Expression-language dialect label. A registered member of the Ossie Dialect enum. +DIALECT = "THOUGHTSPOT" + +#: apache/ossie#351 merged 2026-09-01: THOUGHTSPOT is now in the closed `Dialect` +#: enum (core-spec/ossie-schema.json) and in validation/validate.py's +#: SKIP_SQL_VALIDATION set, so emitting DIALECT no longer fails schema validation. +DIALECT_IS_REGISTERED = True + +#: Dialect emitted alongside DIALECT (not instead of it) for portable expressions, so a +#: consumer that does not implement our dialect still gets something it can execute. +PORTABLE_DIALECT = "ANSI_SQL" + +#: Ossie spec series this converter targets, matched on major.minor. Not an exact +#: version: upstream's first release is proposed as 0.3.0, not 0.2.0. +SPEC_SERIES = "0.2" + +#: The exact `version` this converter writes at the root of every document it +#: emits (`{"version": DOCUMENT_VERSION, "semantic_model": [...]}`). Unlike +#: SPEC_SERIES (major.minor, used to check an *incoming* document's rough +#: compatibility), ossie-schema.json pins `version` to this exact string as a +#: `const` (`ossie-schema.json:9-13`), so a document that emits anything else +#: fails schema validation outright. Bump in lockstep with core-spec/'s own +#: `version` if it ever moves -- the same discipline converters/databricks' +#: `OSSIE_VERSION` constant documents. +DOCUMENT_VERSION = "0.2.0.dev0" + +#: Shape version of the custom_extensions payload. Bump when the +#: payload's shape changes, never for a value change. +STASH_VERSION = 1 + +#: Field/metric-level custom_extensions[THOUGHTSPOT] key holding the +#: warehouse column's own name, stashed by the TML -> Ossie direction only +#: when it differs from the column's display name (Table-backed columns +#: only -- a SQL View's own sql_output_columns dataset-level key already +#: covers the same fact). Not yet in the pinned payload schema. Shared here, +#: rather than written as a literal in each direction separately, because +#: tml_to_ossie.py (the writer) and ossie_to_thoughtspot.py (the reader) +#: must agree on the exact spelling and nothing else enforces that. +FIELD_STASH_DB_COLUMN_NAME = "db_column_name" + +#: The witness copy for FIELD_STASH_DB_COLUMN_NAME: the physical column's +#: own display name (the bracket's column part, e.g. "Amount") as it stood +#: the moment db_column_name was stashed. Ossie -> TML compares this against +#: the CURRENT bracket reference's column part: agreement means nobody +#: retargeted the field to a different physical column since, so the +#: stashed warehouse name is still trustworthy; disagreement means the +#: field now names a different column and the stashed warehouse name +#: describes the wrong one -- it is dropped rather than misapplied to the +#: new column. +FIELD_STASH_DB_COLUMN_NAME_WITNESS = "db_column_name_display_name_witness" + +# --------------------------------------------------------------------------- +# The rest of the custom_extensions[THOUGHTSPOT] payload vocabulary. +# +# Every name below is a key of the JSON object stash.write_stash serialises +# and stash.read_stash parses back -- the same channel FIELD_STASH_DB_COLUMN_NAME +# above already covers for one key. tml_to_ossie.py (the writer) and +# ossie_to_thoughtspot.py (the reader) must agree on each spelling exactly, and +# nothing but this module enforces that; several of these are, today, written +# by only one side (tml_to_ossie.py has no Model/Metric-reading counterpart yet +# in ossie_to_thoughtspot.py) -- they are named here anyway so the reader that +# is eventually written consumes the same literal, not a freshly retyped guess. +# +# Grouped by which Ossie object each key's custom_extensions entry attaches to +# -- Model, Dataset, Relationship, or Field/Metric -- because that grouping is +# itself part of the payload's schema (see the design doc's SemanticModelLevel / +# DatasetLevel / RelationshipLevel / FieldLevel / MetricLevel $defs). +# --------------------------------------------------------------------------- + +#: Exact ThoughtSpot display name, stashed whenever identifier normalisation produced +#: a different Ossie identifier. Shared across every scope that can suffer +#: this divergence: Model (`semantic_model.name`) and Metric (a Metric has no +#: `label` field to carry the display name the way a Field does). Also +#: defensively checked at Dataset scope by `_table_name` in +#: ossie_to_thoughtspot.py -- but a Dataset's own `name` is the verbatim +#: model_tables[] alias-or-name and is never run through normalisation, so +#: nothing writes this key there today; that check is symmetry with the other +#: two scopes, not a reachable path. +STASH_TML_NAME = "tml_name" + +# --- Model scope (attached to a `semantic_model` entry) -------------------- + +#: Formula-backed ATTRIBUTE columns whose references span two or more +#: datasets, so no single Ossie dataset can own the field. Preserved verbatim +#: (each entry carries at least `name` and `expr`) alongside an issue. +MODEL_STASH_UNATTRIBUTED_FORMULAS = "unattributed_formulas" + +#: Joins with no equality pair at all (a pure range or pure constant +#: condition), which cannot become a Relationship because Ossie's +#: `from_columns`/`to_columns` are required and non-empty. Preserved +#: verbatim, alongside an issue. Each entry reuses the RELATIONSHIP_STASH_* +#: keys below for the facts a real Relationship's own custom_extensions +#: entry would have carried, since it describes the same kind of TML join. +MODEL_STASH_UNREPRESENTABLE_JOINS = "unrepresentable_joins" + +#: ThoughtSpot Model-only properties with no Ossie equivalent +#: (`is_bypass_rls`, `join_progressive`, `spotter_config.is_spotter_enabled`), +#: copied verbatim under their own TML property names. +MODEL_STASH_MODEL_PROPERTIES = "model_properties" + +#: Verbatim `model.parameters[]`. No Ossie equivalent; formulas referencing +#: them are not portable. +MODEL_STASH_PARAMETERS = "parameters" + +#: Verbatim `model.filters[]`. +MODEL_STASH_FILTERS = "filters" + +#: Verbatim `model.column_groups[]` -- the search-bar data-panel folder structure. +MODEL_STASH_COLUMN_GROUPS = "column_groups" + +#: Verbatim `model.lesson_plans[]` -- the in-product guided-lesson strings +#: attached to a Model. No Ossie equivalent. +MODEL_STASH_LESSON_PLANS = "lesson_plans" + +#: Verbatim `model.action_object_associations[]` -- custom actions bound to +#: the Model by display name only. +MODEL_STASH_ACTION_OBJECT_ASSOCIATIONS = "action_object_associations" + +#: Verbatim `model.constraints` block (rolling date-window conditions per table). +MODEL_STASH_CONSTRAINTS = "constraints" + +#: Verbatim Model-level `joins_with[]` data-augmentation joins. Named +#: "model_" rather than reusing the bare TML key `joins_with` because a +#: *Table* document has its own, differently-scoped `joins_with[]` +#: (referencing-join definitions) -- the two must not collide under one +#: stash key. +MODEL_STASH_MODEL_JOINS_WITH = "model_joins_with" + +# --- Dataset scope (attached to a `datasets[]` entry) ----------------------- + +#: Whether the source TML document was a `table:` or a `sql_view:` -- +#: authoritative over `_derive_kind`'s own whitespace/dotted-identifier +#: heuristic whenever present, since it also determines which shape +#: `unsurfaced_columns` was captured in. +DATASET_STASH_TML_OBJECT = "tml_object" + +#: The witness copy for DATASET_STASH_TML_OBJECT: the dataset's own +#: `source` string as it stood the moment `tml_object` was stashed. +#: Ossie -> TML compares this against the CURRENT `source`: agreement means +#: nobody edited it since (a query rewritten as a table reference, or vice +#: versa), so the stashed kind is still trustworthy; disagreement means the +#: stash is stale and `_derive_kind` re-guesses from the current `source` +#: instead of trusting a kind that used to describe a different value. +DATASET_STASH_TML_OBJECT_WITNESS = "tml_object_source_witness" + +#: `model_tables[].alias`, when one physical table participates more than once. +DATASET_STASH_ALIAS = "alias" + +#: The underlying table object name that `alias` (above) aliases. +DATASET_STASH_TABLE_NAME = "table_name" + +#: ThoughtSpot Connection display name (case-sensitive, never a GUID). +#: Required to emit a Table document; when absent it must be supplied by the +#: caller as `build_table`'s own `connection_name` argument. +DATASET_STASH_CONNECTION_NAME = "connection_name" + +#: `db`/`schema`/`db_table` recorded individually when the dotted `source` +#: form would be ambiguous. A nested object; see the three keys below for its +#: own contents. +DATASET_STASH_SOURCE_PARTS = "source_parts" + +#: `source_parts.db`. +DATASET_STASH_SOURCE_PARTS_DB = "db" + +#: `source_parts.schema`. +DATASET_STASH_SOURCE_PARTS_SCHEMA = "schema" + +#: `source_parts.db_table`. +DATASET_STASH_SOURCE_PARTS_DB_TABLE = "db_table" + +#: Verbatim Table/SQL-View physical-column entries the Model does not +#: surface. Not semantic model content, but required to regenerate the +#: source document exactly. +DATASET_STASH_UNSURFACED_COLUMNS = "unsurfaced_columns" + +#: `{Ossie field name: sql_output_column alias}`, for every surfaced SQL +#: View column -- there is no safe way to re-derive a query output alias +#: from an Ossie field's own identifier the way a Table's db_column_name +#: might be guessed at. +DATASET_STASH_SQL_OUTPUT_COLUMNS = "sql_output_columns" + +#: ThoughtSpot Table-only properties with no Ossie equivalent (mirrors +#: MODEL_STASH_MODEL_PROPERTIES's `spotter_config` shape, at Dataset scope). +DATASET_STASH_TABLE_PROPERTIES = "table_properties" + +# --- Relationship scope (attached to a `relationships[]` entry) ------------ +# +# Also reused, unchanged, inside MODEL_STASH_UNREPRESENTABLE_JOINS entries -- +# a join that could not become a Relationship at all still needs the same +# facts recorded, under the same names, because it is the same kind of TML +# join fact either way. + +#: ThoughtSpot's join-type vocabulary (`INNER`, `LEFT_OUTER`, `RIGHT_OUTER`, +#: `OUTER`), identical in a Model inline join and a Table `joins_with[]` entry. +RELATIONSHIP_STASH_TYPE = "type" + +#: ThoughtSpot's join cardinality (`MANY_TO_ONE`, `ONE_TO_ONE`, `ONE_TO_MANY`, +#: `MANY_TO_MANY`). Always the exact TML value, verbatim, regardless of +#: whether `RELATIONSHIP_STASH_ENDPOINTS_SWAPPED` (below) also fired for this +#: relationship -- the two facts are independent: this one is never stale +#: (TML's cardinality has no Ossie-side counterpart to disagree with), while +#: whether the endpoint swap it triggered is still trustworthy is a separate, +#: witnessed question. +RELATIONSHIP_STASH_CARDINALITY = "cardinality" + +#: Whether this relationship's `from`/`to`/`from_columns`/`to_columns` were +#: swapped relative to TML's own declared join direction. core-spec/spec.yaml +#: requires a Relationship's `from` to name the many side and `to` the one +#: side, but TML's `from`/`to` (the model_tables[] entry a join is declared +#: under, and its `with`/`destination` target) do not themselves encode which +#: side is which -- `cardinality` does. Only a `ONE_TO_MANY` join has TML's +#: `from` naming the one side and `to` naming the many side -- the wrong way +#: around for Ossie's spec -- so only that cardinality ever sets this `True` +#: and swaps the emitted relationship's endpoints to compensate. `MANY_TO_ONE` +#: and `ONE_TO_ONE` are already oriented correctly and never set it. +RELATIONSHIP_STASH_ENDPOINTS_SWAPPED = "endpoints_swapped" + +#: The witness copy for RELATIONSHIP_STASH_ENDPOINTS_SWAPPED: `[from, to, +#: from_columns, to_columns]` exactly as emitted -- i.e. already swapped -- +#: at the moment the marker was stashed. `Ossie -> TML` compares this against +#: the relationship's CURRENT `from`/`to`/`from_columns`/`to_columns`: +#: agreement means nobody retargeted the relationship since, so it is safe to +#: undo the swap and recover the TML join's original `from`/`to`/columns +#: (and, with them, which dataset's `model_tables[]` entry the join is +#: nested under); disagreement means the relationship was edited since the +#: stash was written, so the swap is not undone -- the live shape is trusted +#: instead, exactly as a hand-authored relationship with no stash at all +#: would be -- and an issue records it. +RELATIONSHIP_STASH_ENDPOINTS_SWAPPED_WITNESS = "endpoints_swapped_witness" + +#: Which TML join shape produced this relationship -- `"referencing"` (a +#: named Table `joins_with[]` entry the Model points at), `"inline"` (defined +#: directly in `model_tables[].joins[]`), or `"referencing_with_inline_attrs"` +#: (both: a `referencing_join` plus a `type`/`cardinality` override). +RELATIONSHIP_STASH_JOIN_SHAPE = "join_shape" + +#: The Table `joins_with[]` entry name, when `join_shape` is `"referencing"` +#: (or the hybrid). +RELATIONSHIP_STASH_REFERENCING_JOIN = "referencing_join" + +#: The verbatim join condition. Required whenever the condition is not a +#: pure equality (range / ASOF / constant joins), because `from_columns`/ +#: `to_columns` then carry only part of it. +RELATIONSHIP_STASH_ON_EXPRESSION = "on_expression" + +#: The witness copy for `RELATIONSHIP_STASH_ON_EXPRESSION`: `[from_columns, +#: to_columns]` exactly as they stood the moment `on_expression` was +#: stashed (only ever written alongside it, i.e. only when residual +#: predicates exist). `Ossie -> TML` compares this against the relationship's +#: CURRENT `from_columns`/`to_columns` -- agreement means nobody retargeted +#: the relationship since the stash was written, so the verbatim +#: `on_expression` (and the residual narrowing it carries) is still current +#: and is restored; disagreement means the stash is stale, so both are +#: dropped and the plain equality condition is re-derived from the live +#: from_columns/to_columns instead, with an issue recording it. This is one +#: of the two places (the other is FIELD_STASH_DATA_TYPE_WITNESS below) this +#: converter uses a witness copy: "a relationship's verbatim on_expression". +RELATIONSHIP_STASH_ON_EXPRESSION_WITNESS = "on_expression_equality_witness" + +# --- Field/metric scope (attached to a `fields[]` or `metrics[]` entry) ----- +# +# FIELD_STASH_DB_COLUMN_NAME above is the original of this group; the rest +# follow its naming even though, like it, they are written for a Metric just +# as often as for a Field -- `_physical_column_stash` and +# `_unconsumed_properties` in tml_to_ossie.py build both keys identically +# regardless of which of the two the caller is converting. + +#: The exact ThoughtSpot `db_column_properties.data_type` spelling, recorded +#: only when it is not the canonical spelling `datatypes.to_tml` would emit +#: by default for the Ossie datatype (`BOOLEAN` vs `BOOL`, `FLOAT` vs +#: `DOUBLE`), so the return trip re-emits the same one. +FIELD_STASH_DATA_TYPE = "data_type" + +#: The witness copy for FIELD_STASH_DATA_TYPE: the Ossie `datatype` value +#: (`"Boolean"` or `"Float"` -- the only two `_CANONICAL_TML_SPELLING` ever +#: stashes a non-canonical spelling for) as it stood the moment the spelling +#: was recorded. `Ossie -> TML` compares this against the field's CURRENT +#: `datatype`: agreement means nobody edited the field's declared type since, +#: so the exact warehouse spelling is still trustworthy and is restored; +#: disagreement -- the field now declares a different datatype -- means the +#: spelling is stale (it names a warehouse type for the *old* datatype, not +#: this one) and is dropped, falling back to the canonical spelling +#: `datatypes.to_tml` derives for the current value instead. +FIELD_STASH_DATA_TYPE_WITNESS = "data_type_ossie_datatype_witness" + +#: ThoughtSpot column properties this converter did not read and consume +#: elsewhere -- the fail-closed complement `_unconsumed_properties` builds, +#: so a future ThoughtSpot-only property this module has never heard of is +#: preserved rather than silently dropped. Also reused, unchanged, for the +#: `column_properties` an unattributed formula's own source column carried +#: (see MODEL_STASH_UNATTRIBUTED_FORMULAS) -- the same "properties this +#: converter did not otherwise account for" concept, just attached to a +#: preserved formula instead of a built Field or Metric. +FIELD_STASH_COLUMN_PROPERTIES = "column_properties" + +# --- Metric-only scope ------------------------------------------------------- + +#: Which of the three TML shapes (`column_aggregation`, +#: `scalar_formula_plus_aggregation`, `formula`) produced this metric, so a +#: return trip can reproduce the source shape instead of collapsing all three +#: into one. `formula` is the default a document with no stash at all +#: reconstructs as, so it is the one value never written. +METRIC_STASH_SHAPE = "shape" + +#: METRIC_STASH_SHAPE's own value vocabulary -- shared here, not written as a +#: literal by tml_to_ossie.py (the writer) and re-typed as a literal by +#: ossie_to_thoughtspot.py (the reader), for the same reason every other name +#: in this file is centralised: the two must agree on the exact spelling and +#: nothing else enforces that. `METRIC_SHAPE_FORMULA` is also what a document +#: with no `shape` stash at all defaults to on the way back -- see +#: METRIC_STASH_SHAPE above for which shape is emitted by default. +METRIC_SHAPE_COLUMN_AGGREGATION = "column_aggregation" +METRIC_SHAPE_SCALAR_FORMULA_PLUS_AGGREGATION = "scalar_formula_plus_aggregation" +METRIC_SHAPE_FORMULA = "formula" + + +# --------------------------------------------------------------------------- +# Stash-freshness classification. +# +# Every custom_extensions[THOUGHTSPOT] key above answers one question before +# ossie_to_thoughtspot.py is allowed to read it: does the stashed value +# shadow something this converter could otherwise derive from the live Ossie +# document -- a field's own bracket reference, a metric's own name, a +# dataset's own source -- or is it information that exists nowhere else in +# the Ossie document at all? +# +# The first kind can go stale: a user edits the Ossie document (retargets a +# field, renames a metric, rewrites a relationship, rewrites a dataset's +# source) and the stash still describes the document as it was. A key in +# that category needs a witness and a currency check -- reused only +# when the two still agree, dropped and re-derived otherwise -- never plain +# stash-if-present. The second kind cannot go stale, because there is +# nothing on the Ossie side for it to disagree with; plain stash-if-present +# is correct there and a witness would have nothing to compare against. +# +# STASH_KEY_CLASSIFICATION is the enforcement point. Three different stash +# keys were found, independently, reading the unsafe way before this table +# existed (FIELD_STASH_DB_COLUMN_NAME, FIELD_STASH_DATA_TYPE, and a +# relationship's on_expression) — each found by generalising from the +# instance before it, not by a rule anyone consulted. This table is that +# rule, made structural: a key read anywhere in ossie_to_thoughtspot.py that +# is missing here fails test_stash_key_classification.py, so the next key +# has to declare an answer rather than default to the unsafe one. It does +# not, by itself, prove the *code* honours a SHADOWS_DERIVABLE +# classification with an actual witness -- that is still a review +# discipline -- but it makes "someone forgot" a build failure instead of a +# silent gap for a key already known to need one. +# --------------------------------------------------------------------------- + + +class StashKeyClass(Enum): + #: The stashed value could disagree with something the live Ossie + #: document itself says. Reading it MUST check currency: via + #: `stash.restore`'s witness/witness_key (FIELD_STASH_DB_COLUMN_NAME, + #: FIELD_STASH_DATA_TYPE, RELATIONSHIP_STASH_ON_EXPRESSION on a real + #: Relationship, DATASET_STASH_TML_OBJECT), or a self-verifying + #: reconstruction when the stashed value's own shape lets it check + #: itself against the live document with nothing extra stored + #: (DATASET_STASH_SOURCE_PARTS re-joins to compare against `source`; + #: STASH_TML_NAME re-normalises to compare against the live + #: identifier). Either way, a mismatch drops the stash, re-derives, and + #: logs why -- never keeps the stale value. + SHADOWS_DERIVABLE = "shadows_derivable" + #: The stashed value has no Ossie-native counterpart at all -- nothing + #: on the Ossie side represents it independently, so nothing there + #: could have diverged from it. Plain stash-if-present is correct. + INFORMATION_ONLY = "information_only" + + +#: One entry per stash key read anywhere in ossie_to_thoughtspot.py. +#: RELATIONSHIP_STASH_ON_EXPRESSION appears once, classified for its +#: primary carrier (a real Relationship, where it shadows +#: from_columns/to_columns) -- the same key read off a +#: MODEL_STASH_UNREPRESENTABLE_JOINS entry has no independent Relationship +#: object to diverge from and would be INFORMATION_ONLY in that context; +#: see the read site's own docstring, not a second table entry, since the +#: dict is keyed by string and cannot hold two classifications for one key. +STASH_KEY_CLASSIFICATION: dict[str, "StashKeyClass"] = { + # -- Shared -- + STASH_TML_NAME: StashKeyClass.SHADOWS_DERIVABLE, + + # -- Field/metric scope -- + FIELD_STASH_DB_COLUMN_NAME: StashKeyClass.SHADOWS_DERIVABLE, + FIELD_STASH_DATA_TYPE: StashKeyClass.SHADOWS_DERIVABLE, + FIELD_STASH_COLUMN_PROPERTIES: StashKeyClass.INFORMATION_ONLY, + METRIC_STASH_SHAPE: StashKeyClass.INFORMATION_ONLY, + + # -- Dataset scope -- + DATASET_STASH_TML_OBJECT: StashKeyClass.SHADOWS_DERIVABLE, + DATASET_STASH_SOURCE_PARTS: StashKeyClass.SHADOWS_DERIVABLE, + DATASET_STASH_CONNECTION_NAME: StashKeyClass.INFORMATION_ONLY, + DATASET_STASH_TABLE_NAME: StashKeyClass.INFORMATION_ONLY, + DATASET_STASH_ALIAS: StashKeyClass.INFORMATION_ONLY, + DATASET_STASH_TABLE_PROPERTIES: StashKeyClass.INFORMATION_ONLY, + DATASET_STASH_UNSURFACED_COLUMNS: StashKeyClass.INFORMATION_ONLY, # value only -- see STASH_KEYS_WITH_DERIVABLE_MEMBERSHIP below for its membership axis + DATASET_STASH_SQL_OUTPUT_COLUMNS: StashKeyClass.INFORMATION_ONLY, # per-field dict lookup, never appended -- checked, does not share unsurfaced_columns' hybrid + + # -- Relationship scope -- + RELATIONSHIP_STASH_ON_EXPRESSION: StashKeyClass.SHADOWS_DERIVABLE, + RELATIONSHIP_STASH_TYPE: StashKeyClass.INFORMATION_ONLY, + RELATIONSHIP_STASH_CARDINALITY: StashKeyClass.INFORMATION_ONLY, + # Witnessed against [from, to, from_columns, to_columns]: a ONE_TO_MANY + # join's endpoint swap is only undone while nothing has retargeted the + # relationship since it was stashed. + RELATIONSHIP_STASH_ENDPOINTS_SWAPPED: StashKeyClass.SHADOWS_DERIVABLE, + # Self-verifying (STASH_TML_NAME's own pattern, nothing extra stored): + # written equal to the relationship's own `name` at stash time, so + # agreement on read means nobody renamed the relationship since and the + # stash is still trustworthy; a mismatch means it was renamed, so the + # stashed Table joins_with[] reference is dropped rather than restored + # under the wrong, stale name. + RELATIONSHIP_STASH_REFERENCING_JOIN: StashKeyClass.SHADOWS_DERIVABLE, + # Which TML shape produced this relationship -- purely descriptive, no + # live Ossie counterpart to disagree with (mirrors METRIC_STASH_SHAPE). + RELATIONSHIP_STASH_JOIN_SHAPE: StashKeyClass.INFORMATION_ONLY, + + # -- Model scope -- + MODEL_STASH_UNATTRIBUTED_FORMULAS: StashKeyClass.INFORMATION_ONLY, + MODEL_STASH_UNREPRESENTABLE_JOINS: StashKeyClass.INFORMATION_ONLY, + MODEL_STASH_MODEL_PROPERTIES: StashKeyClass.INFORMATION_ONLY, + MODEL_STASH_PARAMETERS: StashKeyClass.INFORMATION_ONLY, + MODEL_STASH_FILTERS: StashKeyClass.INFORMATION_ONLY, + MODEL_STASH_COLUMN_GROUPS: StashKeyClass.INFORMATION_ONLY, + MODEL_STASH_LESSON_PLANS: StashKeyClass.INFORMATION_ONLY, + MODEL_STASH_ACTION_OBJECT_ASSOCIATIONS: StashKeyClass.INFORMATION_ONLY, + MODEL_STASH_CONSTRAINTS: StashKeyClass.INFORMATION_ONLY, + MODEL_STASH_MODEL_JOINS_WITH: StashKeyClass.INFORMATION_ONLY, +} + +#: Keys read only when attached to a stash-only carrier that has no +#: independent Ossie object of its own (an unrepresentable_joins[] entry, +#: an unattributed_formulas[] entry) -- the same key name as a +#: SHADOWS_DERIVABLE entry above, but INFORMATION_ONLY in this context, +#: because there is no live Relationship/Metric/Field for it to diverge +#: from. Recorded separately rather than overwriting the primary +#: classification above, so both contexts stay documented. +STASH_ONLY_CARRIER_KEY_CLASSIFICATION: dict[str, "StashKeyClass"] = { + RELATIONSHIP_STASH_ON_EXPRESSION: StashKeyClass.INFORMATION_ONLY, + RELATIONSHIP_STASH_TYPE: StashKeyClass.INFORMATION_ONLY, + RELATIONSHIP_STASH_CARDINALITY: StashKeyClass.INFORMATION_ONLY, + FIELD_STASH_COLUMN_PROPERTIES: StashKeyClass.INFORMATION_ONLY, +} + +# --------------------------------------------------------------------------- +# A second, orthogonal axis StashKeyClass alone cannot express. +# +# StashKeyClass answers one question: can this key's stashed VALUE disagree +# with something the live Ossie document says? DATASET_STASH_UNSURFACED_ +# COLUMNS answers that "no" correctly -- a physical column's own +# db_column_name/data_type has no Ossie-side counterpart to check it +# against, so INFORMATION_ONLY is the right answer for its *content*. But a +# key that holds a LIST of entries has a second question INFORMATION_ONLY +# does not cover at all: does each entry still BELONG in the list? For +# unsurfaced_columns specifically, an entry belongs only while no live field +# now covers the same physical column -- and that membership fact changes +# the moment a field is added, or retargeted, onto a column that used to be +# unsurfaced. Restoring a membership-stale entry verbatim (the value itself +# is still perfectly accurate) alongside the live field's own build of the +# same column duplicates it -- a duplicate Table/SQL-View column name, which +# does not import. This was found live: a field retargeted onto a +# previously-unsurfaced column produced exactly that duplicate, undetected +# by the value-only classification above because the value itself was never +# wrong. +# +# So "information-only in value" and "derivable in membership" are +# independent facts about one key, and a single INFORMATION_ONLY / +# SHADOWS_DERIVABLE answer cannot record both. STASH_KEYS_WITH_DERIVABLE_ +# MEMBERSHIP is the second axis: a key here is a *list*-shaped stash whose +# entries can be superseded by something the live document now covers, and +# whose read site MUST filter entries against that live coverage before +# appending them -- silently, same as an ordinary derive-instead-of-stash +# fallback, because a filtered-out entry was not lost, just no longer +# needed. Every other list-shaped INFORMATION_ONLY key was checked against +# this question directly, not assumed innocent: DATASET_STASH_SQL_OUTPUT_ +# COLUMNS is consulted only as a per-field dict lookup keyed by the live +# field's own name (never appended as a block), so a field no longer +# present just means the lookup is never made -- no duplication is +# possible, and it does not belong here. MODEL_STASH_UNATTRIBUTED_FORMULAS +# and MODEL_STASH_UNREPRESENTABLE_JOINS are appended into collections +# (formulas[]/columns[], and a table's inline joins[]) that already run +# every entry through the shared display-name allocator or accept multiple +# joins between the same pair without an import-breaking collision, so a +# name clash there is caught (and now logged -- see +# TS-MODEL-DISPLAY-NAME-COLLISION) rather than silently duplicated. Every +# scalar-valued INFORMATION_ONLY key (a single string, dict, or bool +# assigned once, never merged with anything else the live document also +# populates) has no membership question to ask at all. +STASH_KEYS_WITH_DERIVABLE_MEMBERSHIP: frozenset[str] = frozenset({ + DATASET_STASH_UNSURFACED_COLUMNS, +}) diff --git a/converters/thoughtspot/src/ossie_thoughtspot/datatypes.py b/converters/thoughtspot/src/ossie_thoughtspot/datatypes.py new file mode 100644 index 00000000..616cb861 --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/datatypes.py @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The ThoughtSpot <-> Ossie datatype map, covering every data type each side supports. + +Three properties of the map shape this module's surface. It is **not injective** — Decimal +and Float both become DOUBLE, and Time, DateTimeTz and Opaque collapse into types that +cannot carry them — so `declared_loss` names the types whose round trip is lossy in one +place rather than leaving each caller to rediscover them. Two types have a +**connection-dependent spelling** (BOOLEAN/BOOL, DOUBLE/FLOAT), which the forward direction +records so the return trip re-emits the same one. And `datatype` is **optional in Ossie but +compulsory in TML** — ThoughtSpot rejects a table whose column has no +`db_column_properties`, so `to_tml(None)` infers rather than raising. +""" +from __future__ import annotations + +#: The closed Ossie datatype enum (core specification). +OSSIE_DATATYPES = frozenset({ + "String", "Integer", "Decimal", "Float", "Boolean", + "Date", "Time", "DateTime", "DateTimeTz", "Opaque", +}) + +#: Ossie datatype -> the TML `data_type` written for it. +_TO_TML = { + "String": "VARCHAR", + "Integer": "INT64", + "Decimal": "DOUBLE", + "Float": "DOUBLE", + "Boolean": "BOOLEAN", + "Date": "DATE", + "Time": "VARCHAR", + "DateTime": "DATE_TIME", + "DateTimeTz": "DATE_TIME", + "Opaque": "VARCHAR", +} + +#: TML `data_type` -> the Ossie datatype emitted for it. Deliberately not the inverse of +#: `_TO_TML`: DOUBLE comes back as Decimal and VARCHAR as String, which is what makes the +#: types in `_DECLARED_LOSS` lossy. +_TO_OSSIE = { + "VARCHAR": "String", + "INT64": "Integer", + "DOUBLE": "Decimal", + "FLOAT": "Float", + "BOOL": "Boolean", + "BOOLEAN": "Boolean", + "DATE": "Date", + "DATE_TIME": "DateTime", +} + +#: Types whose `Ossie -> TML -> Ossie` trip cannot return the original, and why. None of +#: these can be rescued by a stash: the stash is written from a TML document, and TML +#: never held the distinction in the first place. +_DECLARED_LOSS = { + "Float": "ThoughtSpot has one approximate numeric type, so Float and Decimal both " + "become DOUBLE and return as Decimal.", + "Time": "ThoughtSpot has no time-of-day column type; the value becomes VARCHAR.", + "DateTimeTz": "ThoughtSpot has no offset-aware column type; the value becomes " + "DATE_TIME and returns as DateTime.", + "Opaque": "Opaque is Ossie's marker for a type outside the portable vocabulary; it " + "becomes VARCHAR and returns as String.", +} + +#: What a column with no declared datatype becomes. The Table TML reference advises +#: preferring INT64 and letting ThoughtSpot report a mismatch, over omitting the block. +_INFERRED = "INT64" + + +def to_tml(datatype: str | None, *, boolean_spelling: str = "BOOLEAN", + float_spelling: str = "DOUBLE") -> str: + """The TML `data_type` for an Ossie datatype. `None` infers rather than raising.""" + if datatype is None: + return _INFERRED + if datatype not in _TO_TML: + raise ValueError(f"{datatype!r} is not an Ossie datatype") + if datatype == "Boolean": + return boolean_spelling + if datatype == "Float": + return float_spelling + return _TO_TML[datatype] + + +def to_ossie(tml_type: str) -> str | None: + """The Ossie datatype for a TML `data_type`, or `None` when there is no mapping. + + `None` is a legitimate answer, not a failure: `datatype` is optional in Ossie, so + omitting it is strictly better than inventing one for a type outside the map. + """ + return _TO_OSSIE.get(tml_type) + + +def declared_loss(datatype: str) -> str | None: + """Why this datatype's round trip is lossy, or `None` when it is exact.""" + return _DECLARED_LOSS.get(datatype) diff --git a/converters/thoughtspot/src/ossie_thoughtspot/errors.py b/converters/thoughtspot/src/ossie_thoughtspot/errors.py new file mode 100644 index 00000000..229e6251 --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/errors.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Hard failures. Distinct from ConverterIssue, which records a survivable loss.""" + + +class ConversionError(Exception): + """Raised when the converter cannot proceed — e.g. a malformed stash.""" diff --git a/converters/thoughtspot/src/ossie_thoughtspot/expressions/__init__.py b/converters/thoughtspot/src/ossie_thoughtspot/expressions/__init__.py new file mode 100644 index 00000000..fbd4501a --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/expressions/__init__.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Expression translation: the Ossie expression language <-> ThoughtSpot formulas. + +Public surface: + - `Classification`, `Variant`, `Construct` — the shared vocabulary the forward + catalog, the emitters and the reverse inventory all use. + - `CATALOG` — every specification construct mapped to a ThoughtSpot rendering. + - `spec_construct_names()` — the upstream-spec coverage oracle: reads + core-spec/expression_language.md directly so a construct added upstream fails + this package's build instead of silently going unsupported. + - `CONVENTION_DIVERGENCES` — constructs the mapping document counts one row per + construct that have no discrete row in the upstream spec. + - `emit_direct`, `emit_passthrough`, `emit_unmappable` — render a `Construct` + into an actual ThoughtSpot formula, one function per `Classification`. + - `REVERSE`, `ReverseConstruct`, `ReverseDisposition`, `translate_thoughtspot`, + `stash_runtime_parameter` — the reverse-direction inventory (ThoughtSpot + functions with no counterpart in the Ossie specification) and its translator. + - `thoughtspot_dialect_entry`, `portable_dialect_entry`, + `custom_extensions_fragment` — helpers a caller combines with + `translate_thoughtspot`'s result to satisfy roundtrip at the object level. +""" +from .catalog import CATALOG, CONVENTION_DIVERGENCES, spec_construct_names +from .emit import emit_direct, emit_passthrough, emit_unmappable +from .reverse import ( + REVERSE, + ReverseConstruct, + ReverseDisposition, + custom_extensions_fragment, + portable_dialect_entry, + stash_runtime_parameter, + thoughtspot_dialect_entry, + translate_thoughtspot, +) +from ._types import Classification, Construct, Variant + +__all__ = [ + "CATALOG", + "CONVENTION_DIVERGENCES", + "Classification", + "Construct", + "REVERSE", + "ReverseConstruct", + "ReverseDisposition", + "Variant", + "custom_extensions_fragment", + "emit_direct", + "emit_passthrough", + "emit_unmappable", + "portable_dialect_entry", + "spec_construct_names", + "stash_runtime_parameter", + "thoughtspot_dialect_entry", + "translate_thoughtspot", +] diff --git a/converters/thoughtspot/src/ossie_thoughtspot/expressions/_types.py b/converters/thoughtspot/src/ossie_thoughtspot/expressions/_types.py new file mode 100644 index 00000000..a1859ff2 --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/expressions/_types.py @@ -0,0 +1,130 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The small shared vocabulary the catalog, emitters and reverse map all use.""" +from dataclasses import dataclass +from enum import Enum + + +class Classification(str, Enum): + """How a specification construct reaches ThoughtSpot. + + DIRECT a native ThoughtSpot equivalent exists, possibly as a documented + composition of native functions. + PASSTHROUGH requires a sql_*_op pass-through: warehouse-dialect-specific, and + opaque to ThoughtSpot's query planner. + UNMAPPABLE no representation; the converter raises an issue and preserves the + construct in custom_extensions. Never a silent drop. + """ + + DIRECT = "direct" + PASSTHROUGH = "passthrough" + UNMAPPABLE = "unmappable" + + +class Variant(str, Enum): + """The sql_*_op family. The variant fixes the emitted column's type AND + its measure/attribute role. The scalar variants produce attributes; the + *_aggregate_op variants produce measures. Emitting sql_int_op where + sql_int_aggregate_op was needed yields a column that imports cleanly and then + aggregates wrongly — worse than a rejected import. + """ + + BOOL = "sql_bool_op" + DATE_TIME = "sql_date_time_op" + DOUBLE = "sql_double_op" + INT = "sql_int_op" + NUMBER = "sql_number_op" + STRING = "sql_string_op" + INT_AGGREGATE = "sql_int_aggregate_op" + NUMBER_AGGREGATE = "sql_number_aggregate_op" + + +@dataclass(frozen=True) +class Construct: + """One row of the function-mapping document. + + `spec_name` the construct as the specification writes it, e.g. "SUM(expr)". + `template` for DIRECT, the ThoughtSpot formula with {0}, {1}... placeholders; + for PASSTHROUGH, the SQL body passed to the variant; None if UNMAPPABLE. + + Not every DIRECT/PASSTHROUGH template is a complete, positionally + substitutable one — two shapes diverge from that default, and both fail + loud (a raised ValueError, or rejection at TML import) rather than + silently producing a wrong answer: + + - A "dispatch" template — literal text such as "per-type — see note" or + "per-pattern-shape — see note" — for a row whose actual ThoughtSpot + rendering depends on a runtime value not known at catalog-construction + time (CAST's per-type table, the EXTRACT/DATE_PART/DATE_TRUNC/DATEADD + family, TRUE/FALSE, both CASE forms, the column/metric reference, the + unary +/- row, and several window rows). A caller building a uniform + `.format()` dispatcher off this field alone will hit these ~20 rows and + must special-case them; each row's `note` says so and describes the + real dispatch. + - An "exemplar" PASSTHROUGH template — a complete, renderable body that + bakes ONE caller-supplied value in as a literal while still declaring a + satisfiable arity (`PERCENTILE_CONT`/`DISC`'s `0.75`, `NTILE`'s `4`, + `LAG`/`LEAD`'s offset `1`, and others — see `emit_passthrough`'s + docstring for the full convention). This kind renders without error, so + the arg-count guard alone does not distinguish it from a genuinely + complete template: treating the baked-in literal as universal instead of + rebuilding the template per real occurrence is silently wrong, not + loud — `PERCENTILE_CONT(0.9)` would render as a P75 measure that imports + and runs. Each such row's `note` names the baked-in value. + `variant` required for PASSTHROUGH, forbidden otherwise. + `note` the row's caveat, verbatim enough to be traceable to the document. + """ + + spec_name: str + classification: Classification + template: str | None = None + variant: Variant | None = None + note: str = "" + + def __post_init__(self) -> None: + if self.classification is Classification.PASSTHROUGH and self.variant is None: + raise ValueError(f"{self.spec_name}: a passthrough row must name its variant") + if self.classification is not Classification.PASSTHROUGH and self.variant is not None: + raise ValueError(f"{self.spec_name}: only a passthrough row may name a variant") + if self.classification is Classification.UNMAPPABLE and self.template is not None: + raise ValueError(f"{self.spec_name}: an unmappable row has no template") + if self.classification is not Classification.UNMAPPABLE and not self.template: + raise ValueError( + f"{self.spec_name}: a {self.classification.value} row must have a template" + ) + # A PASSTHROUGH template holds only the bare inner SQL body (e.g. + # "LOWER({0})") — emit_passthrough builds the `variant.value ( "..." , args )` + # wrapper itself. A template that already contains its own variant call + # (e.g. 'sql_string_op ( "LOWER({0})" , {0} )', copied verbatim from the + # mapping document's ThoughtSpot-column cell) double-wraps at emission time: + # `sql_string_op ( "sql_string_op ( ""LOWER({0})"" , {0} )" , {0} )`. That + # reads as fine in the catalog file and is wrong the moment it runs — a real + # transcription mistake this check exists to catch. Matching on + # "{variant} (" (the space and paren) rather than a bare substring guards + # against a coincidental token inside a legitimate body; a `sql_*_op` name + # is a ThoughtSpot-side synthetic formula-function name, so it cannot + # legitimately appear inside raw warehouse SQL either. + if self.classification is Classification.PASSTHROUGH: + marker = f"{self.variant.value} (" + if marker in self.template: + raise ValueError( + f"{self.spec_name}: passthrough template already contains " + f"'{marker}' — the template must hold only the bare inner SQL " + "body; emit_passthrough builds the variant(...) wrapper itself, " + "so this template would double-wrap at emission time" + ) diff --git a/converters/thoughtspot/src/ossie_thoughtspot/expressions/catalog.py b/converters/thoughtspot/src/ossie_thoughtspot/expressions/catalog.py new file mode 100644 index 00000000..f548a26f --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/expressions/catalog.py @@ -0,0 +1,1931 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The catalog: every specification construct mapped to a ThoughtSpot rendering. + +`CATALOG` is organised into families below — Aggregate functions, Type conversion, +Date/time functions, String functions, Mathematical + Conditional functions, Operators +and constructs, and Window functions — one block per family, together covering the full +specification. `spec_construct_names()` is an oracle read from the **upstream** +`core-spec/expression_language.md`: it independently reports every construct the +specification defines, so a construct added upstream in the future fails this package's +build instead of silently going unsupported (see `test_catalog_covers_the_spec.py`). + +Extraction approach +-------------------- +The spec document mixes three kinds of content that must be told apart: + +1. Genuine constructs — a named function, operator or literal form the + specification defines, almost always as one row of a markdown table whose + row carries a `Syntax` column (e.g. `SUM(expr)`), or, for a handful of + operators/keywords with no table of their own, a row of the top-level + "Supported SQL Constructs" table. +2. Argument vocabularies — the `EXTRACT`/`DATE_PART` parts, the `DATE_TRUNC` + precisions, the `TO_DATE`/`TO_CHAR` format tokens and the `CAST` target + types. These describe values an argument may take, not constructs in + their own right, and must be excluded. +3. Informative tables — the per-engine "Common Dialect Variations" table and + the "Cross-Reference: Tool Mappings" section describe *other products'* + spellings (Tableau, Looker Studio, DAX, and per-engine SQL). Names that + appear only there are not Ossie constructs. + +The exclusions are keyed off the document's own structure — a table's own header naming +and section heading text ("Not Supported in Expressions", "Common Dialect Variations", +"Cross-Reference") — rather than a hardcoded list of names to drop, so a renamed or added +upstream construct cannot go silently unnoticed. `_extract_tables()` only looks at lines +starting with "|", so the argument-vocabulary bullet lists are simply invisible to it. + +Spelling: `CATALOG` keys must match `spec_construct_names()` exactly +------------------------------------------------------------------------------ +`spec_construct_names()` is the oracle for a `CATALOG` key's exact spelling, not any +mapping document's prose. Several rows write their Ossie-side syntax differently than +this parser extracts it; a `CATALOG` entry keyed on a mapping document's own wording +instead of this function's output will read as an "invented" construct even though it is +a real, intended row. When adding or editing a row, run `spec_construct_names()` and +match a member of it exactly, rather than transcribing another document's column text. +Grouped by which extractor produces the divergent spelling: + +- **`_extract_tables()`** (an ordinary table with a `Syntax` column): + - Alias pairs the spec merges into ONE table row keep this parser's single + extracted spelling: `CEIL(x)` (not `CEIL(x) / CEILING(x)`), `TRUNC(x, d)` + (not `.../ TRUNCATE(x, d)`). + - Two-alternative-syntax rows keep the spec's own joining word, "or": + `"CURRENT_DATE or CURRENT_DATE()"`, `"CURRENT_TIMESTAMP or + CURRENT_TIMESTAMP()"`, `"CURRENT_TIME or CURRENT_TIME()"`. + - The merged boolean-literal row (`BOOLEAN`'s `Syntax` cell) is one entry, + comma-joined: `"TRUE, FALSE"`. + - The Boolean Functions table's `AND`/`OR` rows keep the spec's own + `expr1`/`expr2` placeholder names: `"expr1 AND expr2"`, `"expr1 OR expr2"`. +- **`_extract_summary_rows()`** (the top-level "Supported SQL Constructs" + table, bare backtick token): `BETWEEN`, `IN`, `NOT IN`, `IS NULL`, `IS NOT NULL`, + `CASE WHEN`, and the raw symbols `+ - * / % = <> != < > <= >=`. +- **`_extract_null_safe_comparison_operators()`** (the "Null-Safe Comparison" + code fence, not the summary table): `IS DISTINCT FROM`, `IS NOT DISTINCT FROM`. +- **`_extract_extraction_syntax_functions()`** (the "Alternative Extraction + Syntax" code fence, bare token, no argument list): `EXTRACT`, `DATE_PART`. +- **`_extract_single_construct_headings()`** (a standalone heading with no + table, bare token): `CAST`, `TRY_CAST` (not `CAST(expression AS target_type)`). + +See `CONVENTION_DIVERGENCES` below for the (much shorter) list of constructs that have +no entry in `spec_construct_names()` at all and are exempted instead. +""" +import re +from pathlib import Path + +from ._types import Classification, Construct, Variant + +# -------------------------------------------------------------------------- +# CATALOG: organised into families, one block per family below. +# -------------------------------------------------------------------------- +CATALOG: dict[str, Construct] = {} + +# -------------------------------------------------------------------------- +# Aggregate functions — 18 rows: 12 direct / 6 passthrough / 0 unmappable. +# Source: docs/ossie/ts-ossie-function-mapping.md, "Aggregate functions" section +# (thoughtspot-agent-skills repo — not vendored here; prose above/below the table +# read in full). +# -------------------------------------------------------------------------- +CATALOG.update( + { + "SUM(expr)": Construct( + "SUM(expr)", Classification.DIRECT, template="sum ( {0} )", + ), + "COUNT(expr)": Construct( + "COUNT(expr)", Classification.DIRECT, template="count ( {0} )", + note="Counts non-null values on both sides.", + ), + "COUNT(*)": Construct( + "COUNT(*)", Classification.DIRECT, template="count ( {0} )", + note=( + "ThoughtSpot has no count(*); the row count is count() over a column " + "known to be non-null. The converter uses the dataset's primary_key " + "when the model declares one, and raises an issue rather than " + "guessing a column when it does not." + ), + ), + "COUNT(DISTINCT expr)": Construct( + "COUNT(DISTINCT expr)", Classification.DIRECT, template="unique count ( {0} )", + note=( + "A space, not an underscore. count_distinct(...) is rejected by the " + "formula parser." + ), + ), + "AVG(expr)": Construct( + "AVG(expr)", Classification.DIRECT, template="average ( {0} )", + ), + "MIN(expr)": Construct( + "MIN(expr)", Classification.DIRECT, template="min ( {0} )", + note=( + "ThoughtSpot min is aggregate-only — it never compares two columns " + "row-wise. Scalar two-argument minima are LEAST, a separate row." + ), + ), + "MAX(expr)": Construct( + "MAX(expr)", Classification.DIRECT, template="max ( {0} )", + note="Aggregate-only, as MIN.", + ), + "STDDEV(expr)": Construct( + "STDDEV(expr)", Classification.DIRECT, template="stddev ( {0} )", + note="Sample standard deviation on both sides.", + ), + "STDDEV_POP(expr)": Construct( + "STDDEV_POP(expr)", Classification.PASSTHROUGH, + template="STDDEV_POP({0})", variant=Variant.NUMBER_AGGREGATE, + note=( + "ThoughtSpot stddev is sample-only; there is no population form, and " + "substituting it would change the divisor from n-1 to n." + ), + ), + "STDDEV_SAMP(expr)": Construct( + "STDDEV_SAMP(expr)", Classification.DIRECT, template="stddev ( {0} )", + note="Specification alias for STDDEV (:171).", + ), + "VARIANCE(expr)": Construct( + "VARIANCE(expr)", Classification.DIRECT, template="variance ( {0} )", + note="Sample variance on both sides.", + ), + "VAR_POP(expr)": Construct( + "VAR_POP(expr)", Classification.PASSTHROUGH, + template="VAR_POP({0})", variant=Variant.NUMBER_AGGREGATE, + note="Same divisor reason as STDDEV_POP.", + ), + "VAR_SAMP(expr)": Construct( + "VAR_SAMP(expr)", Classification.DIRECT, template="variance ( {0} )", + note="Specification alias for VARIANCE (:174).", + ), + "MEDIAN(expr)": Construct( + "MEDIAN(expr)", Classification.DIRECT, template="median ( {0} )", + ), + "PERCENTILE_CONT(p) WITHIN GROUP (ORDER BY expr)": Construct( + "PERCENTILE_CONT(p) WITHIN GROUP (ORDER BY expr)", Classification.PASSTHROUGH, + template="PERCENTILE_CONT(0.75) WITHIN GROUP (ORDER BY {0})", + variant=Variant.NUMBER_AGGREGATE, + note=( + "No native percentile function. p is a literal in the specification's " + "syntax, so it is baked into the template rather than passed as a " + "placeholder. p = 0.5 is the one case with a native equivalent — " + "median ( [x] ) — and the converter should prefer it." + ), + ), + "PERCENTILE_DISC(p) WITHIN GROUP (ORDER BY expr)": Construct( + "PERCENTILE_DISC(p) WITHIN GROUP (ORDER BY expr)", Classification.PASSTHROUGH, + template="PERCENTILE_DISC(0.75) WITHIN GROUP (ORDER BY {0})", + variant=Variant.NUMBER_AGGREGATE, + note=( + "As PERCENTILE_CONT; the discrete/interpolated distinction is " + "preserved only because the template is emitted verbatim." + ), + ), + "APPROX_COUNT_DISTINCT(expr)": Construct( + "APPROX_COUNT_DISTINCT(expr)", Classification.PASSTHROUGH, + template="APPROX_COUNT_DISTINCT({0})", variant=Variant.INT_AGGREGATE, + note=( + "ThoughtSpot's unique count ( [x] ) is the exact-semantics " + "alternative: same answer to within the sketch's ~2% error, at " + "exact-count cost. The converter emits the pass-through by default " + "— the specification chose approximate deliberately — and offers " + "the exact form as a documented downgrade." + ), + ), + "APPROX_PERCENTILE(expr, p)": Construct( + "APPROX_PERCENTILE(expr, p)", Classification.PASSTHROUGH, + template="APPROX_PERCENTILE({0}, 0.5)", variant=Variant.NUMBER_AGGREGATE, + note="p baked into the template as for the exact percentiles.", + ), + } +) + +# -------------------------------------------------------------------------- +# Type conversion — 2 rows: 2 direct / 0 passthrough / 0 unmappable. +# Source: docs/ossie/ts-ossie-function-mapping.md, "Type conversion" section. +# +# CAST/TRY_CAST are direct rows whose target-type argument vocabulary is only +# partly covered (5 of 8 types direct, 3 fall back to a pass-through) — the +# per-type dispatch is not counted as its own construct (the target-type +# table is an argument vocabulary, marked "not counted" in the mapping +# document) and is not resolved here. Resolving a +# `CAST` occurrence to an actual formula from its target type would need an +# expression parser, which is out of scope, and whether to take on a sqlglot +# dependency for that is still unresolved; `template` records the document's +# own ThoughtSpot-column text for traceability rather than a +# directly-substitutable formula. +# -------------------------------------------------------------------------- +CATALOG.update( + { + "CAST": Construct( + "CAST", Classification.DIRECT, + template="per-type — see the target-type table below", + note=( + "5 of the 8 specified target types are direct; the other three — " + "BOOLEAN, TIMESTAMP and TIME — fall back to a pass-through." + ), + ), + "TRY_CAST": Construct( + "TRY_CAST", Classification.DIRECT, + template="the same functions as CAST", + note=( + "ThoughtSpot's to_integer / to_double / to_string already return " + "NULL on failure, which is exactly TRY_CAST semantics — so the two " + "rows share a mapping and it is CAST, not TRY_CAST, that is the " + "imprecise one. A strict CAST that must error rather than null is " + "not expressible; the converter records that in the issue log when " + "the source distinguishes them." + ), + ), + } +) + +# -------------------------------------------------------------------------- +# Date/time functions — 24 rows: 17 direct / 7 passthrough / 0 unmappable. +# Source: docs/ossie/ts-ossie-function-mapping.md, "Date/time functions" section +# (thoughtspot-agent-skills repo — not vendored here; prose above/below the table +# read in full). +# +# EXTRACT/DATE_PART date-parts, DATE_TRUNC precisions, DATEADD/DATEDIFF parts and +# TO_DATE/TO_CHAR format tokens are argument vocabularies and get no +# entry of their own (see the mapping document's "(not counted — arguments)" +# sub-tables). EXTRACT, DATE_PART, DATE_TRUNC(part, date_expr), +# DATEADD(part, amount, date_expr) and DATEDIFF(part, start_date, end_date) are +# themselves still DIRECT rows in the 24 — the per-argument dispatch happens for +# each, but the dispatch table itself is out of catalog scope (same pattern as +# CAST/TRY_CAST in Type conversion above): `template` records the mapping document's own +# ThoughtSpot-column text for traceability, and the real per-argument content +# (which native function each part/precision rewrites to, and the argument-order +# caveats) is recorded in `note`. +# -------------------------------------------------------------------------- +CATALOG.update( + { + "CURRENT_DATE or CURRENT_DATE()": Construct( + "CURRENT_DATE or CURRENT_DATE()", Classification.DIRECT, template="today ( )", + note="Both specification spellings map to the same function.", + ), + "CURRENT_TIMESTAMP or CURRENT_TIMESTAMP()": Construct( + "CURRENT_TIMESTAMP or CURRENT_TIMESTAMP()", Classification.DIRECT, template="now ( )", + ), + "CURRENT_TIME or CURRENT_TIME()": Construct( + "CURRENT_TIME or CURRENT_TIME()", Classification.DIRECT, + template="time ( now ( ) )", + note=( + "ThoughtSpot has no current-time function, but time ( ) extracts " + "the time part of a datetime, so the composition is exact." + ), + ), + "YEAR(date_expr)": Construct( + "YEAR(date_expr)", Classification.DIRECT, template="year ( {0} )", + ), + "QUARTER(date_expr)": Construct( + "QUARTER(date_expr)", Classification.DIRECT, template="quarter_number ( {0} )", + note="The function is quarter_number, not quarter.", + ), + "MONTH(date_expr)": Construct( + "MONTH(date_expr)", Classification.DIRECT, template="month_number ( {0} )", + note=( + "Not month ( ) — ThoughtSpot's month returns the month NAME " + "('January'); month_number returns 1-12, which is what the " + "specification means. Mapping to month would silently change " + "the column's type from integer to string." + ), + ), + "DAY(date_expr)": Construct( + "DAY(date_expr)", Classification.DIRECT, template="day ( {0} )", + note="Day of month, 1-31 on both sides.", + ), + "DAYOFYEAR(date_expr)": Construct( + "DAYOFYEAR(date_expr)", Classification.DIRECT, + template="day_number_of_year ( {0} )", + note="The function is day_number_of_year, not day_of_year.", + ), + "HOUR(timestamp_expr)": Construct( + "HOUR(timestamp_expr)", Classification.DIRECT, template="hour_of_day ( {0} )", + note="The function is hour_of_day, not hour.", + ), + "MINUTE(timestamp_expr)": Construct( + "MINUTE(timestamp_expr)", Classification.PASSTHROUGH, + template="MINUTE({0})", variant=Variant.INT, + note=( + "No native minute-of-hour extractor; add_minutes and " + "diff_minutes exist but neither extracts." + ), + ), + "SECOND(timestamp_expr)": Construct( + "SECOND(timestamp_expr)", Classification.PASSTHROUGH, + template="SECOND({0})", variant=Variant.INT, + note="As MINUTE.", + ), + "EXTRACT": Construct( + "EXTRACT", Classification.DIRECT, + template="per-part — see the date-part table below", + note=( + "Rewritten to the part's own ThoughtSpot function; there is no " + "generic extractor. 8 of the 11 specified parts are direct " + "(YEAR->year, QUARTER->quarter_number, MONTH->month_number, " + "WEEK->week_number_of_year, DAY->day, " + "DAYOFWEEK->day_number_of_week, DAYOFYEAR->day_number_of_year, " + "HOUR->hour_of_day); MINUTE, SECOND and MILLISECOND fall back " + "to sql_int_op." + ), + ), + "DATE_PART": Construct( + "DATE_PART", Classification.DIRECT, + template="per-part — see the date-part table below", + note="Identical treatment to EXTRACT; the two spellings collapse onto one rewrite (:276-279).", + ), + "DATE_TRUNC(part, date_expr)": Construct( + "DATE_TRUNC(part, date_expr)", Classification.DIRECT, + template="per-precision — see the truncation table below", + note=( + "ThoughtSpot has no date_trunc. The start_of_* family covers 7 " + "of the 8 specified precisions ('year'->start_of_year, " + "'quarter'->start_of_quarter, 'month'->start_of_month, " + "'week'->start_of_week, 'day'->date ( ), 'hour'->start_of_hour, " + "'minute'->start_of_min — the function is start_of_min, not " + "start_of_minute); 'second' falls back to sql_date_time_op. " + "The specification says week truncation is Monday-start; " + "ThoughtSpot's week start is an instance setting, so the " + "converter verifies alignment and raises an issue when it cannot." + ), + ), + "DATEADD(part, amount, date_expr)": Construct( + "DATEADD(part, amount, date_expr)", Classification.DIRECT, + template="per-part add_* — see the arithmetic table below", + note=( + "Argument order differs: ThoughtSpot is add_days ( [d] , n ), " + "the specification is DATEADD(day, n, d). Every specified part " + "is reachable: day->add_days, week->add_weeks, " + "month->add_months, year->add_years, minute->add_minutes, " + "second->add_seconds, plus two by arithmetic on a coarser unit " + "since there is no native add_quarters or add_hours: " + "quarter->add_months ( [d] , 3 * n ), " + "hour->add_minutes ( [d] , 60 * n )." + ), + ), + "DATEDIFF(part, start_date, end_date)": Construct( + "DATEDIFF(part, start_date, end_date)", Classification.DIRECT, + template="per-part diff_* — see the arithmetic table below", + note=( + "Argument order is reversed: ThoughtSpot is " + "diff_days ( [end] , [start] ) — end first. Getting this wrong " + "silently negates every duration in the model. day->diff_days, " + "week->diff_weeks, month->diff_months, quarter->diff_quarters, " + "year->diff_years, hour->diff_hours, minute->diff_minutes, " + "second->diff_time (returns seconds)." + ), + ), + "DATE '2024-01-15'": Construct( + "DATE '2024-01-15'", Classification.DIRECT, + template="to_date ( '{0}' , 'yyyy-MM-dd' )", + note=( + "A bare '2024-01-15' in a ThoughtSpot formula is parsed as " + "arithmetic (2024 - 1 - 15), so the typed literal must always " + "be wrapped. to_date takes exactly two arguments, so the " + "converter supplies the ISO format model; {0} is the literal " + "date string." + ), + ), + "TIMESTAMP_NTZ '2024-01-15 10:30:00'": Construct( + "TIMESTAMP_NTZ '2024-01-15 10:30:00'", Classification.PASSTHROUGH, + template="CAST('2024-01-15 10:30:00' AS TIMESTAMP)", + variant=Variant.DATE_TIME, + note=( + "to_date returns a DATE and drops the time part, so there is " + "no native way to construct a wall-clock timestamp. A " + "zero-placeholder template is a documented form of the " + "pass-through — the document's own worked example is recorded " + "verbatim here; a real occurrence's literal value is " + "substituted per-occurrence when the template is built, out of " + "this catalog's scope (same as CAST's per-type dispatch)." + ), + ), + "TIME '10:30:00'": Construct( + "TIME '10:30:00'", Classification.PASSTHROUGH, + template="CAST('10:30:00' AS TIME)", + variant=Variant.DATE_TIME, + note=( + "ThoughtSpot has no TIME column type — time ( ) extracts a " + "time FROM a datetime, it does not construct one — so the " + "pass-through returns DATETIME and the date part is whatever " + "the warehouse defaults to. Flagged with an issue for that " + "reason, not only for the dialect." + ), + ), + "TO_DATE(string)": Construct( + "TO_DATE(string)", Classification.DIRECT, + template="to_date ( {0} , 'yyyy-MM-dd' )", + note=( + "The single-argument ISO form. ThoughtSpot's to_date is " + "strictly two-argument, so the converter supplies 'yyyy-MM-dd'." + ), + ), + "TO_TIMESTAMP(string)": Construct( + "TO_TIMESTAMP(string)", Classification.PASSTHROUGH, + template="TO_TIMESTAMP({0})", variant=Variant.DATE_TIME, + note="to_date is date-only; parsing to a timestamp would drop the time silently.", + ), + "TO_DATE(string, format)": Construct( + "TO_DATE(string, format)", Classification.DIRECT, + template="to_date ( {0} , )", + note=( + "EXPERIMENTAL. Format tokens are translated, not passed " + "through — see the format-token table. ThoughtSpot accepts " + "Java/LDML tokens (yyyy-MM-dd) and strptime %-codes, which " + "between them cover the specification's entire portable core." + ), + ), + "TO_TIMESTAMP(string, format)": Construct( + "TO_TIMESTAMP(string, format)", Classification.PASSTHROUGH, + template="TO_TIMESTAMP({0}, 'YYYY-MM-DD HH24:MI:SS')", + variant=Variant.DATE_TIME, + note=( + "EXPERIMENTAL. Date-only to_date again. The format model " + "inside the template is the warehouse's, not Ossie's, so the " + "token translation table does not apply — this is the " + "sharpest case of the pass-through caveat." + ), + ), + "TO_CHAR(date_expr, format)": Construct( + "TO_CHAR(date_expr, format)", Classification.PASSTHROUGH, + template="TO_CHAR({0}, 'YYYY-MM')", variant=Variant.STRING, + note=( + "EXPERIMENTAL. ThoughtSpot has no general date formatter. " + "Single-token formats do have native equivalents and the " + "converter prefers them: 'YYYY' -> year_name ( [d] ), " + "'MONTH' -> month ( [d] ), 'DAY' -> day_of_week ( [d] ). Those " + "three return locale-dependent text on both sides." + ), + ), + } +) + +# -------------------------------------------------------------------------- +# String functions — 21 rows: 10 direct / 11 passthrough / 0 unmappable. +# Source: docs/ossie/ts-ossie-function-mapping.md, "String functions" section +# (thoughtspot-agent-skills repo — not vendored here; prose above/below the table +# read in full). +# +# This family is over half passthrough, and the reasons run against intuition +# rather than with it: LOWER/UPPER/TRIM/LTRIM/RTRIM/REPLACE are passthrough not +# because they behave differently in ThoughtSpot but because ThoughtSpot has no +# native equivalent at all (live-verified 2026-07-29: TRIM and REPLACE were +# rejected with "Search did not find ..."). STARTSWITH/ENDSWITH run the other +# way: also no native function, but their compositions use only native +# functions (strpos/substr/strlen), so they stay direct. There is no +# regular-expression support of any kind, so every REGEXP_* row is passthrough +# with no native fallback. +# -------------------------------------------------------------------------- +CATALOG.update( + { + "CONCAT(str1, str2, ...)": Construct( + "CONCAT(str1, str2, ...)", Classification.DIRECT, + template="concat ( {0} , {1} , ... )", + note=( + "N-ary on both sides. + does not concatenate in ThoughtSpot — " + "it is numeric-only and the parser rejects string operands, so " + "both || and CONCAT land here." + ), + ), + "LENGTH(str)": Construct( + "LENGTH(str)", Classification.DIRECT, template="strlen ( {0} )", + note="Characters, not bytes, on both sides.", + ), + "LOWER(str)": Construct( + "LOWER(str)", Classification.PASSTHROUGH, + template="LOWER({0})", variant=Variant.STRING, + note="There is no native lower in ThoughtSpot.", + ), + "UPPER(str)": Construct( + "UPPER(str)", Classification.PASSTHROUGH, + template="UPPER({0})", variant=Variant.STRING, + note=( + "There is no native upper in ThoughtSpot. LOWER/UPPER are the " + "most-used functions in the whole passthrough set, and their " + "absence is also what forces ILIKE and case-insensitive " + "comparison into pass-throughs." + ), + ), + "TRIM(str)": Construct( + "TRIM(str)", Classification.PASSTHROUGH, + template="TRIM({0})", variant=Variant.STRING, + note=( + "There is no native trim in ThoughtSpot — live-verified " + "2026-07-29, rejected with 'Search did not find \"trim (\"'. " + "The whole trim family is a pass-through, not just the " + "one-sided forms." + ), + ), + "LTRIM(str)": Construct( + "LTRIM(str)", Classification.PASSTHROUGH, + template="LTRIM({0})", variant=Variant.STRING, + note="No native ltrim — live-verified 2026-07-29.", + ), + "RTRIM(str)": Construct( + "RTRIM(str)", Classification.PASSTHROUGH, + template="RTRIM({0})", variant=Variant.STRING, + note="As LTRIM.", + ), + "LEFT(str, n)": Construct( + "LEFT(str, n)", Classification.DIRECT, template="left ( {0} , {1} )", + ), + "RIGHT(str, n)": Construct( + "RIGHT(str, n)", Classification.DIRECT, template="right ( {0} , {1} )", + ), + "SUBSTRING(str, start, length)": Construct( + "SUBSTRING(str, start, length)", Classification.DIRECT, + template="substr ( {0} , {1} - 1 , {2} )", + note=( + "Index base differs. ANSI SUBSTRING is 1-based; ThoughtSpot's " + "substr is 0-based. The -1 is mandatory and is the single most " + "likely off-by-one in the whole mapping. When start is an " + "expression rather than a literal, the arithmetic is emitted " + "rather than folded." + ), + ), + "REPLACE(str, from, to)": Construct( + "REPLACE(str, from, to)", Classification.PASSTHROUGH, + template="REPLACE({0}, {1}, {2})", + variant=Variant.STRING, + note=( + "There is no native replace in ThoughtSpot — live-verified " + "2026-07-29, rejected with 'Search did not find \"replace (\"'." + ), + ), + "SPLIT_PART(str, delimiter, part)": Construct( + "SPLIT_PART(str, delimiter, part)", Classification.PASSTHROUGH, + template="SPLIT_PART({0}, {1}, {2})", + variant=Variant.STRING, + note=( + "ThoughtSpot has no tokenising function at all — not split, " + "split_part or an nth-occurrence search — so there is no " + "composition to fall back on." + ), + ), + "POSITION(substr IN str)": Construct( + "POSITION(substr IN str)", Classification.DIRECT, + template="strpos ( {1} , {0} )", + note=( + "Operand order is reversed (haystack first in ThoughtSpot) and " + "the specification's infix IN form becomes a comma. 1-based, " + "returning 0 when absent, on both sides." + ), + ), + "CHARINDEX(substr, str)": Construct( + "CHARINDEX(substr, str)", Classification.DIRECT, + template="strpos ( {1} , {0} )", + note=( + "Specification alias for POSITION (:419) with the operands " + "already in prefix order; the reversal is the same." + ), + ), + "CONTAINS(str, substr)": Construct( + "CONTAINS(str, substr)", Classification.DIRECT, + template="contains ( {0} , {1} )", + note="Returns boolean on both sides.", + ), + "STARTSWITH(str, prefix)": Construct( + "STARTSWITH(str, prefix)", Classification.DIRECT, + template="strpos ( {0} , {1} ) = 1", + note=( + "There is no native starts_with — live-verified 2026-07-29. " + "Still direct because the composition is exact and uses only " + "native functions: strpos is 1-based, so a true prefix sits " + "at position 1." + ), + ), + "ENDSWITH(str, suffix)": Construct( + "ENDSWITH(str, suffix)", Classification.DIRECT, + template="substr ( {0} , strlen ( {0} ) - strlen ( {1} ) , strlen ( {1} ) ) = {1}", + note=( + "There is no native ends_with — live-verified 2026-07-29. " + "Direct by composition, as STARTSWITH." + ), + ), + "REGEXP_LIKE(str, pattern)": Construct( + "REGEXP_LIKE(str, pattern)", Classification.PASSTHROUGH, + template="REGEXP_LIKE({0}, {1})", + variant=Variant.BOOL, + note=( + "Boolean return, so not sql_string_op. ThoughtSpot has no " + "regular-expression support of any kind." + ), + ), + "REGEXP_EXTRACT(str, pattern)": Construct( + "REGEXP_EXTRACT(str, pattern)", Classification.PASSTHROUGH, + template="REGEXP_SUBSTR({0}, {1})", + variant=Variant.STRING, + note=( + "The function name inside the template is dialect-specific — " + "Snowflake spells it REGEXP_SUBSTR, others REGEXP_EXTRACT — so " + "the converter selects it from the connection's dialect and " + "raises an issue when the dialect is unknown." + ), + ), + "REGEXP_REPLACE(str, pattern, replacement)": Construct( + "REGEXP_REPLACE(str, pattern, replacement)", Classification.PASSTHROUGH, + template="REGEXP_REPLACE({0},{1},{2})", + variant=Variant.STRING, + note=( + "Name is portable; the pattern dialect (POSIX vs PCRE, " + "backreference syntax) is not." + ), + ), + "REGEXP_COUNT(str, pattern)": Construct( + "REGEXP_COUNT(str, pattern)", Classification.PASSTHROUGH, + template="REGEXP_COUNT({0}, {1})", + variant=Variant.INT, + note="Integer return.", + ), + } +) + +# -------------------------------------------------------------------------- +# Mathematical + Conditional functions — 34 rows: 32 direct / +# 2 passthrough / 0 unmappable. +# Source: docs/ossie/ts-ossie-function-mapping.md, "Mathematical functions" and +# "Conditional functions" sections (thoughtspot-agent-skills repo — not +# vendored here; prose above/below the tables read in full). +# +# Nearly every row here is direct, several by composition: SIGN has +# no native function but composes exactly as a three-way `if` chain — the +# trailing `else 0` is mandatory, ThoughtSpot rejects an `if` with no `else`. +# RADIANS/DEGREES are bare dialect-free arithmetic, not passthroughs. PI is a +# literal at the precision ThoughtSpot's own documented trig composites use. +# ThoughtSpot's trigonometry is degrees-native while the specification is +# radians-native, so SIN/COS/TAN convert degrees->radians on the way in +# (`* 180 / pi`) and ASIN/ACOS/ATAN convert radians->degrees on the way out +# (`* pi / 180`) — opposite directions, easy to transpose by mistake. +# GREATEST/LEAST are deliberately NOT mapped to max/min: ThoughtSpot's max/min +# are aggregate-only, so that mapping would both collapse the row-wise N-ary +# result to one value and flip it from attribute to measure. +# +# Only two rows are passthrough: TRUNC/TRUNCATE (no native truncation — floor +# only agrees with it for x >= 0, d = 0, and round disagrees at every +# half-value) and ATAN2 (quadrant-aware and defined where x = 0, so it is not +# a two-argument ATAN composition, unlike every other inverse trig function +# in this family). +# -------------------------------------------------------------------------- +CATALOG.update( + { + "ABS(x)": Construct( + "ABS(x)", Classification.DIRECT, template="abs ( {0} )", + ), + "ROUND(x, d)": Construct( + "ROUND(x, d)", Classification.DIRECT, template="round ( {0} , {1} )", + ), + "FLOOR(x)": Construct( + "FLOOR(x)", Classification.DIRECT, template="floor ( {0} )", + ), + "CEIL(x)": Construct( + "CEIL(x)", Classification.DIRECT, template="ceil ( {0} )", + note="Specification alias pair CEIL(x) / CEILING(x); both spellings map to ceil.", + ), + "TRUNC(x, d)": Construct( + "TRUNC(x, d)", Classification.PASSTHROUGH, + template="TRUNC({0}, {1})", variant=Variant.DOUBLE, + note=( + "Specification alias pair TRUNC(x, d) / TRUNCATE(x, d). " + "ThoughtSpot has no truncation function. floor agrees with " + "TRUNC only for x >= 0 and d = 0, and round disagrees at every " + "half-value, so neither is a safe substitute." + ), + ), + "MOD(x, y)": Construct( + "MOD(x, y)", Classification.DIRECT, template="mod ( {0} , {1} )", + note="Sign-of-result for negative operands follows the warehouse on both sides.", + ), + "SIGN(x)": Construct( + "SIGN(x)", Classification.DIRECT, + template="if ( {0} > 0 ) then 1 else if ( {0} < 0 ) then -1 else 0", + note=( + "No native sign, but the three-way result is exactly " + "expressible as an if chain. The else 0 is required — " + "ThoughtSpot rejects an if chain with no else." + ), + ), + "POWER(x, y)": Construct( + "POWER(x, y)", Classification.DIRECT, template="pow ( {0} , {1} )", + note="The function is pow. power is rejected by the parser.", + ), + "SQRT(x)": Construct( + "SQRT(x)", Classification.DIRECT, template="sqrt ( {0} )", + ), + "EXP(x)": Construct( + "EXP(x)", Classification.DIRECT, template="exp ( {0} )", + ), + "LN(x)": Construct( + "LN(x)", Classification.DIRECT, template="ln ( {0} )", + ), + "LOG(base, x)": Construct( + "LOG(base, x)", Classification.DIRECT, + template="safe_divide ( ln ( {1} ) , ln ( {0} ) )", + note=( + "ThoughtSpot has fixed-base log2 and log10 only; base is a " + "runtime argument here, not a literal known at catalog time, " + "so the general change-of-base composition is the one " + "template that is exact for every base. safe_divide rather " + "than / guards base = 1." + ), + ), + "LOG10(x)": Construct( + "LOG10(x)", Classification.DIRECT, template="log10 ( {0} )", + ), + "SIN(x)": Construct( + "SIN(x)", Classification.DIRECT, + template="sin ( {0} * 180 / 3.14159265358979 )", + note=( + "ThoughtSpot trigonometry is in degrees; the specification is " + "in radians. The conversion is mandatory — a bare sin ( {0} ) " + "returns the sine of x degrees and is wrong for every " + "non-zero input." + ), + ), + "COS(x)": Construct( + "COS(x)", Classification.DIRECT, + template="cos ( {0} * 180 / 3.14159265358979 )", + note="Degrees, as SIN.", + ), + "TAN(x)": Construct( + "TAN(x)", Classification.DIRECT, + template="tan ( {0} * 180 / 3.14159265358979 )", + note="Degrees, as SIN.", + ), + "ASIN(x)": Construct( + "ASIN(x)", Classification.DIRECT, + template="( asin ( {0} ) * 3.14159265358979 / 180 )", + note=( + "Inverse functions convert the other way: ThoughtSpot returns " + "degrees, the specification expects radians." + ), + ), + "ACOS(x)": Construct( + "ACOS(x)", Classification.DIRECT, + template="( acos ( {0} ) * 3.14159265358979 / 180 )", + note="Degrees -> radians, as ASIN.", + ), + "ATAN(x)": Construct( + "ATAN(x)", Classification.DIRECT, + template="( atan ( {0} ) * 3.14159265358979 / 180 )", + note="Degrees -> radians, as ASIN.", + ), + "ATAN2(y, x)": Construct( + "ATAN2(y, x)", Classification.PASSTHROUGH, + template="ATAN2({0}, {1})", variant=Variant.DOUBLE, + note=( + "atan2 is not a two-argument atan — it is quadrant-aware and " + "defined where x = 0. Composing it from atan plus sign tests " + "is possible but the branch table is easy to get wrong at the " + "axes, so the pass-through is the honest mapping." + ), + ), + "RADIANS(degrees)": Construct( + "RADIANS(degrees)", Classification.DIRECT, + template="{0} * 3.14159265358979 / 180", + note="No native radians; the arithmetic is exact and dialect-free.", + ), + "DEGREES(radians)": Construct( + "DEGREES(radians)", Classification.DIRECT, + template="{0} * 180 / 3.14159265358979", + note="No native degrees; as RADIANS.", + ), + "PI()": Construct( + "PI()", Classification.DIRECT, template="3.14159265358979", + note=( + "No native pi. The literal is emitted at the precision " + "ThoughtSpot's own documented composites use; " + 'sql_double_op ( "pi()" ) is available where full warehouse ' + "precision matters." + ), + ), + "GREATEST(x, y, ...)": Construct( + "GREATEST(x, y, ...)", Classification.DIRECT, + template="greatest ( {0} , {1} , ... )", + note=( + "Not max. ThoughtSpot's max is an aggregate; greatest is the " + "row-wise N-ary function. Mapping GREATEST to max would " + "collapse the column to one value and also flip it from " + "attribute to measure." + ), + ), + "LEAST(x, y, ...)": Construct( + "LEAST(x, y, ...)", Classification.DIRECT, + template="least ( {0} , {1} , ... )", + note="Not min, for the same reason as GREATEST.", + ), + "IF(condition, true_result, false_result)": Construct( + "IF(condition, true_result, false_result)", Classification.DIRECT, + template="if ( {0} ) then {1} else {2}", + note=( + "The parentheses around the condition are mandatory for TML " + "import — without them the parser reports \"Expecting keyword " + "'('\". Applies to every condition shape, including a bare " + "BOOL column reference." + ), + ), + "IFF(condition, true_result, false_result)": Construct( + "IFF(condition, true_result, false_result)", Classification.DIRECT, + template="if ( {0} ) then {1} else {2}", + note="Specification alias for IF.", + ), + "NULLIF(expr1, expr2)": Construct( + "NULLIF(expr1, expr2)", Classification.DIRECT, + template="nullif ( {0} , {1} )", + ), + "COALESCE(expr1, expr2, ...)": Construct( + "COALESCE(expr1, expr2, ...)", Classification.DIRECT, + template="ifnull ( {0} , ifnull ( {1} , {2} ) )", + note=( + "ThoughtSpot's ifnull is strictly two-argument, so an N-ary " + "COALESCE becomes a right-nested chain. Two arguments is the " + "common case and needs no nesting." + ), + ), + "IFNULL(expr, default)": Construct( + "IFNULL(expr, default)", Classification.DIRECT, + template="ifnull ( {0} , {1} )", + ), + "NVL(expr, default)": Construct( + "NVL(expr, default)", Classification.DIRECT, + template="ifnull ( {0} , {1} )", + note="Specification alias for two-argument COALESCE.", + ), + "NVL2(expr, not_null_result, null_result)": Construct( + "NVL2(expr, not_null_result, null_result)", Classification.DIRECT, + template="if ( isnotnull ( {0} ) ) then {1} else {2}", + note="No native three-way null function; the composition is exact.", + ), + "ZEROIFNULL(expr)": Construct( + "ZEROIFNULL(expr)", Classification.DIRECT, + template="ifnull ( {0} , 0 )", + ), + "NULLIFZERO(expr)": Construct( + "NULLIFZERO(expr)", Classification.DIRECT, + template="nullif ( {0} , 0 )", + ), + } +) + +# -------------------------------------------------------------------------- +# Operators and constructs — 33 rows: 30 direct / 2 passthrough / +# 1 unmappable. +# Source: docs/ossie/ts-ossie-function-mapping.md, "Operators and constructs" +# section (thoughtspot-agent-skills repo — not vendored here; prose above/below +# the table read in full). +# +# The document's own section header states that CASE (both forms) and the +# boolean literals/operators are rowed HERE, not under Conditional functions — +# confirmed by the arithmetic: 25 Math + 9 Conditional (the Mathematical + +# Conditional functions family above) + 33 here would double-count CASE otherwise. +# +# spec_construct_names() extracts the BARE operator/keyword token for most of +# this family, not the document's own "a + b"-style worked-example row header — +# confirmed live before writing this block (see test_catalog_operators.py's +# docstring for the full list). Six rows have no discrete spec table row at +# all and are keyed via CONVENTION_DIVERGENCES instead: unary -x/+x, the +# simple CASE form, Parentheses, the DISTINCT modifier, the column/metric +# reference, and EXISTS_IN() itself. +# +# This family holds the single UNMAPPABLE row in the whole 146-row catalog: +# EXISTS_IN() is named at :131 as the sanctioned way to filter on a subquery, +# but the specification defines it nowhere — no signature, no argument order, +# no semantics, absent from every function table. Construct.__post_init__ +# forbids a template or variant on an UNMAPPABLE row, so this is the one entry +# in the whole file with neither. +# +# LIKE is direct despite ThoughtSpot having no native starts_with/ends_with: +# the prefix/suffix/contains compositions it needs use only native functions, +# the same reasoning as the String functions family's STARTSWITH/ +# ENDSWITH rows above. ILIKE is passthrough for the opposite reason — +# case-insensitive matching has no native form, and the usual lower()-fold +# workaround is itself a passthrough, so there is nothing to compose from. The +# DISTINCT aggregate modifier is passthrough for every aggregate except COUNT, +# which already has its own native unique count row (COUNT(DISTINCT expr), +# in Aggregate functions above). +# -------------------------------------------------------------------------- +CATALOG.update( + { + "+": Construct( + "+", Classification.DIRECT, template="{0} + {1}", + note=( + "Numeric only. ThoughtSpot's + rejects string operands, so a + " + "that concatenates on the source side must become concat ( ). " + "The specification does not overload +, so this only bites " + "when translating a dialect expression." + ), + ), + "-": Construct( + "-", Classification.DIRECT, template="{0} - {1}", + ), + "*": Construct( + "*", Classification.DIRECT, template="{0} * {1}", + ), + "/": Construct( + "/", Classification.DIRECT, template="{0} / {1}", + note=( + "Both yield NULL (or a warehouse error) on divide-by-zero. " + "ThoughtSpot's safe_divide returns 0, not NULL, so it is not " + "a faithful substitute and is used only where the source " + "itself guards the denominator." + ), + ), + "%": Construct( + "%", Classification.DIRECT, template="mod ( {0} , {1} )", + note="ThoughtSpot has no % operator — the modulo is the function.", + ), + "-x / +x (unary)": Construct( + "-x / +x (unary)", Classification.DIRECT, + template="per-spelling — see note", + note=( + "CONVENTION_DIVERGENCE: unary +/- is named only in the " + "'Operator Precedence' list, never a table row. This row " + "merges two Ossie spellings that need DIFFERENT output — " + "unary minus is -[x] (negation), unary plus is the identity " + "([x] unchanged) — so a single {0}-substitutable template " + "would be wrong for whichever spelling didn't produce it: " + "an earlier draft used template=\"-{0}\", which is correct " + "for -x but silently negates a parsed +x node (right arg " + "count, wrong semantics, no exception — the arg-count guard " + "cannot catch it). Forced external dispatch instead, the " + "same treatment as TRUE, FALSE below and CAST's per-type " + "table: the caller must choose -{0} or {0} " + "unchanged based on which spelling it parsed, rather than " + "getting a plausible-looking wrong answer from this row. " + "Unary minus is where the bare-date-literal trap " + "originates: '2024-05-01' unquoted is parsed as " + "2024 - 5 - 1. Date literals are always wrapped in " + "to_date ( )." + ), + ), + "=": Construct( + "=", Classification.DIRECT, template="{0} = {1}", + ), + "<>": Construct( + "<>", Classification.DIRECT, template="{0} <> {1}", + ), + "!=": Construct( + "!=", Classification.DIRECT, template="{0} != {1}", + note="ThoughtSpot accepts both inequality spellings, so the two rows are independent and both direct.", + ), + "<": Construct( + "<", Classification.DIRECT, template="{0} < {1}", + ), + ">": Construct( + ">", Classification.DIRECT, template="{0} > {1}", + ), + "<=": Construct( + "<=", Classification.DIRECT, template="{0} <= {1}", + ), + ">=": Construct( + ">=", Classification.DIRECT, template="{0} >= {1}", + ), + "expr1 AND expr2": Construct( + "expr1 AND expr2", Classification.DIRECT, template="{0} and {1}", + note="Lower-case, infix.", + ), + "expr1 OR expr2": Construct( + "expr1 OR expr2", Classification.DIRECT, template="{0} or {1}", + note="Lower-case, infix.", + ), + "NOT expr": Construct( + "NOT expr", Classification.DIRECT, template="not ( {0} )", + note=( + "Function form with parentheses, not a prefix operator — " + "not [x] does not parse." + ), + ), + "BETWEEN": Construct( + "BETWEEN", Classification.DIRECT, + template="{0} between {1} and {2}", + note="Inclusive on both sides.", + ), + "IN": Construct( + "IN", Classification.DIRECT, + template="{0} in {{ {1} , {2} , ... }}", + note=( + "Literal lists only on both sides — no subqueries. The " + "curly-brace delimiter is confirmed, live-verified " + "2026-07-29: the round-parenthesis form is rejected with " + "'Expecting one of the valid keywords, such as, \"ts_var\", " + "\"{\"'. It forces >- block-scalar YAML. The braces are " + "doubled ({{ }}) in the template because emit_direct renders " + "via str.format, which reads a single literal brace as the " + "start of a field name (see test_emit.py's catalog-wide " + "sweep)." + ), + ), + "NOT IN": Construct( + "NOT IN", Classification.DIRECT, + template="not ( {0} in {{ {1} , {2} , ... }} )", + note=( + "Emitted as a negated in rather than a not in keyword — the " + "bare keyword form is not reliably accepted. Braces doubled " + "for str.format, as IN above." + ), + ), + "str LIKE pattern": Construct( + "str LIKE pattern", Classification.DIRECT, + template="per-pattern-shape — see note", + note=( + "Prefix ('foo%') -> strpos ( {0} , 'foo' ) = 1; suffix " + "('%foo') -> substr ( {0} , strlen ( {0} ) - strlen ( 'foo' " + ") , strlen ( 'foo' ) ) = 'foo'; contains ('%foo%') -> " + "contains ( {0} , 'foo' ). Only contains is a native " + "function — starts_with and ends_with do not exist " + "(live-verified 2026-07-29), so the " + "first two shapes are compositions of native functions, " + "same as the STARTSWITH/ENDSWITH rows. These " + "three shapes are the overwhelming majority of LIKE use. " + "Interior wildcards and any _ single-character wildcard have " + "no native form and fall back to " + 'sql_bool_op ( "{0} LIKE {1}" , [s] , [pattern] ). ' + "The per-pattern-shape dispatch is out of this catalog's " + "scope, same treatment as CAST's per-type dispatch " + "— the actual pattern literal is a runtime value, not known " + "at catalog-construction time." + ), + ), + "str ILIKE pattern": Construct( + "str ILIKE pattern", Classification.PASSTHROUGH, + template="{0} ILIKE {1}", variant=Variant.BOOL, + note=( + "Case-insensitive matching has no native form, and the " + "usual workaround — fold both sides with lower — is itself " + "a pass-through, so there is nothing to compose from." + ), + ), + "IS NULL": Construct( + "IS NULL", Classification.DIRECT, template="isnull ( {0} )", + ), + "IS NOT NULL": Construct( + "IS NOT NULL", Classification.DIRECT, template="isnotnull ( {0} )", + note="Native, so not composed as not ( isnull ( ) ).", + ), + "IS DISTINCT FROM": Construct( + "IS DISTINCT FROM", Classification.DIRECT, + template=( + "if ( isnull ( {0} ) and isnull ( {1} ) ) then false else " + "if ( isnull ( {0} ) or isnull ( {1} ) ) then true else " + "{0} != {1}" + ), + note=( + "No native null-safe comparison, but the three-case truth " + "table is exactly expressible. The nesting order matters: " + "both-null must be tested before either-null." + ), + ), + "IS NOT DISTINCT FROM": Construct( + "IS NOT DISTINCT FROM", Classification.DIRECT, + template=( + "if ( isnull ( {0} ) and isnull ( {1} ) ) then true else " + "if ( isnull ( {0} ) or isnull ( {1} ) ) then false else " + "{0} = {1}" + ), + note=( + "The negation of the row above, written directly rather " + "than wrapped in not ( ) — one fewer nesting level for the " + "parser." + ), + ), + "CASE WHEN": Construct( + "CASE WHEN", Classification.DIRECT, + template="if ( c1 ) then r1 else if ( c2 ) then r2 else d", + note=( + "The searched CASE WHEN c1 THEN r1 ... ELSE d END form. No " + "native CASE; the chain is else if, two words. The final " + "else is mandatory and must be type-matched — else 0 for a " + "measure, else '' for an attribute. Omitting it raises " + "'Unknown data type', and a CASE with no ELSE (legal in the " + "specification, yielding NULL) therefore needs one " + "synthesised. The branch count is unbounded, so the " + "template uses symbolic c1/r1/c2/r2/d names rather than " + "being forced into a fixed {0}/{1} scheme — the same " + "out-of-scope-dispatch treatment " + "as CAST's per-type table." + ), + ), + "CASE expr WHEN v1 THEN r1 ... END (simple)": Construct( + "CASE expr WHEN v1 THEN r1 ... END (simple)", Classification.DIRECT, + template="if ( [expr] = v1 ) then r1 else if ( [expr] = v2 ) then r2 else d", + note=( + "CONVENTION_DIVERGENCE: the simple CASE form is described " + "only in the CASE Expression code fence, never a table row. " + "Expanded to the searched form with an explicit equality " + "per branch. expr is repeated per branch, so a converter " + "should hoist an expensive expr into its own formula first. " + "Symbolic template, as CASE WHEN above, for the same " + "unbounded-branch-count reason." + ), + ), + "str1 || str2": Construct( + "str1 || str2", Classification.DIRECT, template="concat ( {0} , {1} )", + note=( + "ThoughtSpot has no concatenation operator at all — + is " + "numeric-only — so || and CONCAT share one target." + ), + ), + "Parentheses — expression grouping": Construct( + "Parentheses — expression grouping", Classification.DIRECT, + template="( {0} )", + note=( + "CONVENTION_DIVERGENCE: its Supported SQL Constructs row " + "carries no backtick token in either cell, the only marker " + "the top-table extraction keys on. Precedence is the " + "standard SQL ordering on the Ossie side. The converter " + "emits explicit parentheses around every rewritten " + "sub-expression rather than relying on the two languages " + "agreeing about precedence — cheap, and it removes a whole " + "class of silent arithmetic errors." + ), + ), + "TRUE, FALSE": Construct( + "TRUE, FALSE", Classification.DIRECT, template="true / false", + note=( + "The Boolean Functions table's Syntax cell merges TRUE and " + "FALSE into one comma-joined entry, matching what " + "spec_construct_names() extracts. Which of the two " + "lower-case literals is emitted depends on which the source " + "wrote — TRUE -> true, FALSE -> false — resolved " + "per-occurrence, out of this catalog's scope (same as " + "CAST's per-type dispatch). A bare BOOL column reference " + "used as a condition still needs its parentheses: " + "if ( [T::flag] ) then ... parses, if [T::flag] then ... " + "does not." + ), + ), + "DISTINCT aggregate modifier": Construct( + "DISTINCT aggregate modifier", Classification.PASSTHROUGH, + template="SUM(DISTINCT {0})", variant=Variant.NUMBER_AGGREGATE, + note=( + "CONVENTION_DIVERGENCE: described only in the Conditional " + "Aggregations prose/code block, never a table row. The " + "specification allows DISTINCT on SUM as well as COUNT. " + "ThoughtSpot has exactly one distinct-aware aggregate — " + "unique count — which is COUNT(DISTINCT) and already has " + "its own row. Every other DISTINCT aggregate is a " + "pass-through." + ), + ), + "Column / metric reference — field, dataset.field": Construct( + "Column / metric reference — field, dataset.field", + Classification.DIRECT, + template="[TABLE::Column], or [Formula Name] for a metric", + note=( + "CONVENTION_DIVERGENCE: its Supported SQL Constructs row " + "carries no backtick token in either cell, same reason as " + "Parentheses. Always rewritten from resolved metadata, " + "never passed through textually — the rewrite, the " + "case-sensitivity rules and the display-name-versus-" + "identifier problem are out of this catalog's scope." + ), + ), + "EXISTS_IN()": Construct( + "EXISTS_IN()", Classification.UNMAPPABLE, + note=( + "CONVENTION_DIVERGENCE: named only in the Reason column of " + "the excluded 'Not Supported in Expressions' table, never " + "in a table of its own. The single unmappable row in the " + "whole 146-row catalog: named at :131 as the sanctioned way " + "to filter on a subquery, but defined nowhere in the " + "specification — no signature, no argument order, no " + "semantics, absent from every function table. Even given a " + "signature, ThoughtSpot's nearest capability is a " + "sql_bool_op subquery template that requires a " + "fully-qualified warehouse table name, which is not " + "derivable from an Ossie expression." + ), + ), + } +) + +# -------------------------------------------------------------------------- +# Window functions — 14 rows: 5 direct / 9 passthrough / 0 unmappable. +# Source: docs/ossie/ts-ossie-function-mapping.md, "Window functions" section, plus +# "Window rows live-confirmed — 2026-07-30" (thoughtspot-agent-skills repo — not +# vendored here; prose above/below the table read in full). +# +# This is the hardest family, and the last one — it completes the 146-row catalog. +# Three constraints govern it: +# +# - A raw aggregate cannot be nested inside a ThoughtSpot window function. The +# argument must be a column reference or a group_aggregate ( ... ). Live-confirmed +# both directions: the raw-aggregate form is rejected, the group_aggregate form +# validates, for moving_* and cumulative_* alike. +# - ThoughtSpot's ORDER BY column must be a physical column reference, not a +# formula. A formula column in the sort position fails to resolve. +# - A ThoughtSpot window formula cannot declare its own PARTITION BY; the +# window shape is completed from the search context. There is no argument slot +# for a partition and none can be added — live-confirmed by rejection, +# 2026-07-30 (a fifth { [attr] } or query_groups ( ) argument to moving_sum, +# and a third to cumulative_sum, are both rejected at the parser). rank / +# rank_percentile are the stricter case: arity fixed at exactly two, enforced +# ("Function rank expects only 2 arguments"), so they are always global. This +# is why LAG, LEAD, the OVER clause and window aggregation moved +# direct -> passthrough in the 2026-07-30 rework (52 live probes, 31 accepted / +# 21 rejected) — a native idiom (moving_sum as the LAG/LEAD idiom) exists but +# is NOT equivalent to any OVER shape, because it has no partition slot and +# ThoughtSpot's partition is never empty. +# +# FIRST_VALUE/LAST_VALUE are the section's one exception: they take a genuine, +# explicit partition argument and a genuine, explicit order axis — both +# live-confirmed accepted, including a multi-column fixed partition — so the +# formula does define its own window, and they stay direct. RANK/PERCENT_RANK and +# the frame-clause boundaries also stay direct, with their native reach now proven +# by rejection rather than asserted. +# +# PERCENT_RANK is direct via rank_percentile (both the 0-100 scale and the +# inversion are required); CUME_DIST is NOT a rank_percentile substitute — +# PERCENT_RANK divides by n-1 and starts at 0, CUME_DIST divides by n and ends at +# 1 — so CUME_DIST stays passthrough with no native fallback at all. +# +# RANK, PERCENT_RANK, FIRST_VALUE and LAST_VALUE record the document's own worked +# example verbatim — symbolic bracket names ([m], [dim], [ord], [attr], [T::date]), +# not numbered {0}/{1} substitution slots — because resolving which actual column +# fills each slot needs model metadata not known at catalog-construction time; same +# out-of-scope-dispatch treatment as CASE WHEN's c1/r1 names and CAST's per-type +# table. FIRST_VALUE/LAST_VALUE's worked example carries ThoughtSpot's literal +# `{ [T::date] }` list syntax for the axis argument; since these are DIRECT rows +# rendered via emit_direct's str.format, the literal braces are doubled ({{ }}) +# the same fix the IN/NOT IN rows above need for the same reason — verified here by +# actually calling emit_direct and checking the rendered output has single braces +# again (see test_catalog_window.py). +# +# Three of the 14 rows have no discrete row of their own in the upstream spec — +# the OVER clause, the frame clause and window aggregation are keyed via the +# pre-existing CONVENTION_DIVERGENCES entries (copied verbatim, not retyped, to +# avoid an em-dash/ellipsis mismatch) rather than spec_construct_names(). The +# other 11 key on spec_construct_names()'s own extraction from the "Ranking +# Functions" and "Offset Functions" tables' Syntax column — confirmed live before +# writing this block. +# -------------------------------------------------------------------------- +CATALOG.update( + { + "ROW_NUMBER() OVER (...)": Construct( + "ROW_NUMBER() OVER (...)", Classification.PASSTHROUGH, + template="ROW_NUMBER() OVER (PARTITION BY {0} ORDER BY {1})", + variant=Variant.INT_AGGREGATE, + note=( + "ThoughtSpot's rank is competition rank, not a row number, so it " + "is not a substitute. Wrap in group_aggregate so the " + "partition column reaches the GROUP BY even when the user's " + "search omits it." + ), + ), + "RANK() OVER (...)": Construct( + "RANK() OVER (...)", Classification.DIRECT, + template="rank ( sum ( [m] ) , 'desc' )", + note=( + "direct for one shape only, and the boundary is proven rather " + "than asserted: the global, ORDER BY-only form over an " + "aggregate. Live-confirmed 2026-07-30: rank ( sum ( [m] ) , " + "'desc' ) and 'asc' both validate, and the arity is enforced at " + "exactly two — a third argument in any shape (bare attribute, " + "{ [attr] }, or query_groups ( )) is rejected with 'Function " + "rank expects only 2 arguments', so an explicit PARTITION BY is " + "provably not expressible. Two further live-proven " + "restrictions: the first argument must be aggregated (rank " + "( [m] , 'desc' ) -> 'Function rank expects 1st argument to be " + "aggregated'), so an Ossie ORDER BY has " + "no native target either; and it may not be a " + "group_aggregate ( ... ), so the partition cannot be smuggled " + "in through the measure. Every non-covered shape falls back to " + "sql_int_aggregate_op ( \"RANK() OVER (PARTITION BY {0} ORDER " + "BY SUM({1}) DESC)\" , ... ), wrapped in group_aggregate. Query-context " + "caveat: rank carries no dynamic partition but it is " + "evaluated over the query's result rows, so the covered shape " + "is faithful to RANK() OVER (ORDER BY ...) only when the search " + "returns the grain the expression assumed — a query-time " + "semantic no import probe can observe, taken from ThoughtSpot's " + "formula documentation rather than this run. The direction " + "string is not validated at import ('descending' was " + "accepted), so acceptance proves the call shape, never the " + "ordering." + ), + ), + "DENSE_RANK() OVER (...)": Construct( + "DENSE_RANK() OVER (...)", Classification.PASSTHROUGH, + template="dense_rank() over (order by sum({0}) desc)", + variant=Variant.INT_AGGREGATE, + note=( + "ThoughtSpot's rank skips ranks after a tie; dense ranking has " + "no native form — live-confirmed 2026-07-30, dense_rank ( ... ) " + "rejected with 'Search did not find \"dense_rank ( sum (\"'. " + "Passthrough is correct: no native ThoughtSpot construct produces " + "dense-rank semantics." + ), + ), + "NTILE(n) OVER (...)": Construct( + "NTILE(n) OVER (...)", Classification.PASSTHROUGH, + template="NTILE(4) OVER (ORDER BY SUM({0}))", + variant=Variant.INT_AGGREGATE, + note="n is a literal, baked into the template, as the aggregate percentiles are.", + ), + "PERCENT_RANK() OVER (...)": Construct( + "PERCENT_RANK() OVER (...)", Classification.DIRECT, + template="1 - rank_percentile ( sum ( [m] ) , 'asc' ) / 100", + note=( + "ThoughtSpot's rank_percentile is documented as " + "(1.0 - PERCENT_RANK() OVER (ORDER BY ...)) * 100, so the " + "inverse is exact. Two adjustments are both required: the " + "scale (ThoughtSpot 0-100, specification 0-1) and the " + "inversion. Dropping either produces a plausible-looking " + "column that is wrong everywhere. Same shape restriction as " + "RANK, and the same live-proven boundary — rank_percentile is " + "also fixed at exactly two arguments ('Function rank_percentile " + "expects only 2 arguments', live-verified 2026-07-30), so it " + "too is global-only and an explicit PARTITION BY falls back to " + "sql_number_aggregate_op ( \"PERCENT_RANK() OVER (PARTITION BY " + "{0} ORDER BY SUM({1}))\" , ... ). Same evidence-class " + "caveat as RANK: the arity is probe-proven, the global-window " + "semantic is documentation-derived. CUME_DIST is deliberately " + "NOT given this same composition — see that row." + ), + ), + "CUME_DIST() OVER (...)": Construct( + "CUME_DIST() OVER (...)", Classification.PASSTHROUGH, + template="CUME_DIST() OVER (ORDER BY SUM({0}))", + variant=Variant.NUMBER_AGGREGATE, + note=( + "rank_percentile is NOT a substitute, despite PERCENT_RANK's " + "row looking equivalent: PERCENT_RANK divides by n - 1 and " + "starts at 0; CUME_DIST divides by n and ends at 1. They agree " + "on no row of a tie-free window except the last, so there is no " + "native fallback at all for this row." + ), + ), + "LAG(expr, offset, default) OVER (...)": Construct( + "LAG(expr, offset, default) OVER (...)", Classification.PASSTHROUGH, + template="LAG({0}, 1) OVER (PARTITION BY {1} ORDER BY {2})", + variant=Variant.NUMBER_AGGREGATE, + note=( + "Reclassified direct -> passthrough 2026-07-30. The " + "native idiom moving_sum ( [m] , n , -n , [ord] ) is real and " + "validates (a frame of n PRECEDING to n PRECEDING) but is not " + "equivalent to any OVER shape: moving_sum has no partition " + "slot, and ThoughtSpot completes the partition from the " + "query's own dimensions instead. So an Ossie LAG with a " + "PARTITION BY cannot be expressed, and one without a " + "PARTITION BY still cannot, because ThoughtSpot's partition is " + "not empty. The converter emits the pass-through by default " + "and offers the native moving_sum idiom as a documented " + "downgrade the user must accept: correct exactly when the " + "search's dimensions are the intended partition. The default " + "argument has no equivalent in the native idiom — ThoughtSpot " + "yields null outside the frame — a second reason the native " + "form is a downgrade (the pass-through carries default fine). " + "Subject to the same aggregation and physical-ORDER-BY-column " + "constraints as the rest of this family. Variant recorded here is the documented " + "default (sql_number_aggregate_op); the typed sibling applies " + "for a non-numeric expr — LAG returns its argument's own type, " + "not an aggregate, so a string-typed expr (LAG(order_status, " + "1) OVER (...)) needs the typed sibling, not this default, or " + "it imports cleanly and aggregates wrongly." + ), + ), + "LEAD(expr, offset, default) OVER (...)": Construct( + "LEAD(expr, offset, default) OVER (...)", Classification.PASSTHROUGH, + template="LEAD({0}, 1) OVER (PARTITION BY {1} ORDER BY {2})", + variant=Variant.NUMBER_AGGREGATE, + note=( + "Mirror of LAG, reclassified for the same reason and on the " + "same date. The native downgrade is " + "moving_sum ( [m] , -n , n , [ord] ) — ThoughtSpot's start/end " + "arguments use opposite sign conventions, so a forward offset " + "is a negative start (both live-confirmed 2026-07-30). Same " + "default limitation as LAG. Variant recorded here is the " + "documented default (sql_number_aggregate_op); the typed " + "sibling applies for a non-numeric expr, same reason as LAG's " + "note — LEAD returns its argument's own type, not an aggregate." + ), + ), + "FIRST_VALUE(expr) OVER (...)": Construct( + "FIRST_VALUE(expr) OVER (...)", Classification.DIRECT, + template="first_value ( sum ( [m] ) , query_groups ( ) , {{ [T::date] }} )", + note=( + "The section's exception, and the only window row whose direct " + "verdict survived the 2026-07-30 rework — first_value takes a " + "genuine explicit partition argument and a genuine explicit " + "order axis, so the formula does define its own window. " + "Live-confirmed 2026-07-30: query_groups ( ), " + "a fixed single-column { [attr] }, a multi-column " + "{ [a] , [b] }, the grand-total { } and the dynamic " + "query_groups ( ) - { [attr] } all validate in the partition " + "slot, so a static Ossie PARTITION BY list maps straight onto " + "it. The axis slot is typed and enforced — a bare column " + "reference is rejected with 'Function last_value expects 3rd " + "argument to be List', so the { } braces are mandatory (and " + "force >- block-scalar YAML on the document side; doubled here " + "as {{ }} because emit_direct renders via str.format, the same " + "fix the IN/NOT IN rows above need for the same reason — " + "verified by calling emit_direct and checking the rendered " + "output has single braces again). " + "Two boundaries remain: ThoughtSpot's first_value is a " + "semi-additive function over a date axis rather than a general " + "window function, so an OVER shape with a row frame other than " + "the whole partition falls back to " + "sql_number_aggregate_op ( \"FIRST_VALUE({0}) OVER (...)\" , " + "... ); and the axis column's type is not validated at " + "import (a VARCHAR axis was accepted), so acceptance proves " + "the call shape, not that the axis is temporal." + ), + ), + "LAST_VALUE(expr) OVER (...)": Construct( + "LAST_VALUE(expr) OVER (...)", Classification.DIRECT, + template="last_value ( sum ( [m] ) , query_groups ( ) , {{ [T::date] }} )", + note=( + "Same conditions, same live evidence and same fallback as " + "FIRST_VALUE. last_value_in_period and first_value_in_period " + "also validate in the identical three-argument shape and are " + "the period-completeness variants (see the reverse-direction " + "table) — out of this row's scope. Braces doubled on the axis " + "argument for the same str.format reason as FIRST_VALUE." + ), + ), + "NTH_VALUE(expr, n) OVER (...)": Construct( + "NTH_VALUE(expr, n) OVER (...)", Classification.PASSTHROUGH, + template="NTH_VALUE({0}, 2) OVER (ORDER BY {1})", + variant=Variant.NUMBER_AGGREGATE, + note=( + "ThoughtSpot's semi-additive functions reach only the first " + "and last values of the axis — live-confirmed 2026-07-30, " + "nth_value ( ... ) rejected with 'Search did not find " + "\"nth_value ( sum (\"'. n is a literal, baked into the " + "template, as NTILE's. Variant recorded here is the " + "documented default (sql_number_aggregate_op); the typed " + "sibling applies for a non-numeric expr, same reason as LAG's " + "note — NTH_VALUE returns its argument's own type, not an " + "aggregate." + ), + ), + "OVER (PARTITION BY ... ORDER BY ...) clause": Construct( + "OVER (PARTITION BY ... ORDER BY ...) clause", Classification.PASSTHROUGH, + template="per-clause-shape — see note", + variant=Variant.NUMBER_AGGREGATE, + note=( + "CONVENTION_DIVERGENCE: the generic OVER syntax template is a " + "fenced code block, not a table. Reclassified direct -> " + "passthrough 2026-07-30. The previous verdict claimed a clean " + "structural rewrite — 'PARTITION BY attrs becomes the " + "group_aggregate grouping argument; ORDER BY becomes the " + "window function's trailing attribute arguments' — but that " + "holds for PARTITION BY alone and breaks the moment an " + "ORDER BY is present, which is most window use. There are two " + "disjoint targets and only one accepts a partition: an OVER " + "clause with a PARTITION BY and no ORDER BY/frame is " + "group_aggregate ( agg ( [m] ) , { [T::a] , [T::b] } , " + "query_filters ( ) ) and is lossless; an OVER clause with an " + "ORDER BY must target moving_*/cumulative_*, which have no " + "partition slot at all. Live-confirmed accepted: a " + "fixed single-column grouping { [T::pk] } inside " + "group_aggregate (as a moving_* and a cumulative_* argument), " + "and query_groups ( ) - { [attr] } / " + "query_groups ( ) + { [attr] } inside group_aggregate. " + "Live-confirmed rejected: moving_sum ( ... , [ord] , " + "{ [attr] } ) and moving_sum ( ... , [ord] , query_groups ( ) " + "), plus cumulative_sum ( ... , [ord] , { [attr] } ). Not " + "probed: a bare { } or a bare query_groups ( ) as the " + "group_aggregate grouping argument, and the query_groups ( ) " + "form of the cumulative_sum rejection — those three cells rest " + "on the formula reference, not this run. A partitioned, " + "ordered window therefore has no native home and the whole " + "clause is out of catalog scope for the general case — " + "template records the dispatch rather than one substitutable " + "body, same treatment as CAST's per-type table. Variant " + "recorded here is the documented default " + "(sql_number_aggregate_op); the typed sibling applies for a " + "non-numeric aggregate. The reverse direction is lossy for the " + "mirror-image reason — ThoughtSpot's ordered window functions " + "add the query's own dimensions to the partition dynamically, " + "which the specification cannot express." + ), + ), + "Frame clause — ROWS BETWEEN ... / RANGE BETWEEN ...": Construct( + "Frame clause — ROWS BETWEEN ... / RANGE BETWEEN ...", Classification.DIRECT, + template="per-frame-shape — see note", + note=( + "CONVENTION_DIVERGENCE: frame options are a bullet list under " + "the OVER syntax section, not a table. direct for the frame " + "boundaries only — deliberately scoped, so the partition loss " + "is counted once, on the OVER clause row, and not twice. " + "ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -> " + "cumulative_*. Bounded ROWS frames -> moving_* with " + "n PRECEDING -> positive n, CURRENT ROW -> 0, n FOLLOWING -> " + "negative -n. All four boundary shapes were live-confirmed, " + "2026-07-30 (moving_sum ( [m] , 2 , 0 , [ord] " + "), ( ... , 1 , -1 , ... ), ( ... , -1 , 1 , ... ), " + "cumulative_sum ( [m] , [ord] )), and the positional signature " + "is enforced — moving_sum ( [m] , [ord] ) is rejected with " + "'Function moving_sum expects 2nd argument to be Numeric'. " + "RANGE frames fall back to sql_number_aggregate_op (the same " + "variant the window-aggregation row below falls back to): " + "ThoughtSpot's frames are row-positional, not value-ranged — " + "live-verified on gapped dates, moving_* counts surviving rows " + "regardless of the calendar distance between them — so a " + "RANGE frame over a gapped sort column would silently return " + "different numbers. A frame reaches ThoughtSpot natively " + "only when the accompanying OVER clause declares no " + "PARTITION BY; otherwise it is emitted verbatim inside the " + "pass-through template the OVER row selects. Per-shape " + "dispatch out of catalog scope, same treatment as CAST's " + "per-type table." + ), + ), + "Window aggregation — AGG(expr) OVER (...)": Construct( + "Window aggregation — AGG(expr) OVER (...)", Classification.PASSTHROUGH, + template="SUM({0}) OVER (PARTITION BY {1} ORDER BY {2} " + "ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)", + variant=Variant.NUMBER_AGGREGATE, + note=( + "CONVENTION_DIVERGENCE: the Window Aggregations section is " + "prose and code examples, not a table. Reclassified direct -> " + "passthrough 2026-07-30, inheriting the OVER row's problem: " + "the specification allows every aggregate as a window " + "function, but every ordered ThoughtSpot target " + "(cumulative_*, moving_*) completes its partition from the " + "query. The unordered case remains lossless and is the " + "group_aggregate path on the OVER row. The native family is " + "also narrower than the specification's: cumulative_*/" + "moving_* cover SUM, AVG, MIN and MAX only — live-confirmed " + "2026-07-30 that moving_count, moving_stddev and " + "cumulative_count do not exist ('Search did not find " + "\"moving_count (\"' and siblings) — so a windowed COUNT, " + "MEDIAN, STDDEV or VARIANCE has a partitioned form via " + "group_count/group_stddev/group_variance and no ordered or " + "framed form of any kind. The frame is an exemplar, the same " + "convention as NTILE's literal 4 (see the Construct.template " + "docstring): ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW " + "is the cumulative-aggregate boundary the Frame clause row " + "above maps cumulative_* to, and is one concrete, valid frame " + "among the ones a real occurrence could carry — a caller " + "rebuilds the frame per occurrence, same as any other " + "exemplar row. The mapping document's own cell for this row " + "writes the frame as a literal ellipsis ('ROWS BETWEEN …'), " + "which is prose shorthand for 'a frame clause goes here', not " + "renderable SQL — transcribing it verbatim rendered warehouse " + "syntax errors at query time, so this template supplies a " + "concrete, valid frame instead. Variant recorded here is the " + "documented default (sql_number_aggregate_op); the typed " + "sibling applies for a non-numeric aggregate. The argument must still be " + "an aggregate — a raw column reference cannot be nested inside window " + "aggregation." + ), + ), + } +) + +#: Constructs the mapping document (docs/ossie/ts-ossie-function-mapping.md in the +#: thoughtspot-agent-skills repo) counts separately ("one row per +#: construct") that core-spec/expression_language.md does not give a discrete +#: table row of their own. Each entry records WHY it diverges. This is NOT an +#: escape hatch for missing coverage: the 137 names in spec_construct_names() are +#: still parsed from the live upstream file, so an upstream addition still fails +#: the build. It exists because the mapping document's 146-row census counts by a +#: different unit (one row per construct, including constructs the spec only +#: describes in prose) than spec_construct_names() counts by (one entry per +#: parseable table row / heading). Verified directly against the mapping +#: document's actual row list — see catalog.py's docstring for the +#: reconciliation (137 + 9 == 146). +#: +#: Two mechanisms an earlier pass mistakenly guessed would appear here do NOT: +#: `CEIL(x)`/`CEILING(x)`, `TRUNC(x, d)`/`TRUNCATE(x, d)` and `TRUE`/`FALSE` are +#: each ONE row in the mapping document too (not split), matching spec_name's +#: single merged entry — a spelling-convention question addressed throughout +#: this file (see the module docstring's "Spelling" note), not a divergence. +CONVENTION_DIVERGENCES: dict[str, str] = { + "-x / +x (unary)": ( + "unary +/- is named only in the 'Operator Precedence' list " + "(core-spec/expression_language.md:142), never a table row" + ), + "CASE expr WHEN v1 THEN r1 ... END (simple)": ( + "simple CASE is described only in the CASE Expression code fence " + "(core-spec/expression_language.md:508-513) alongside searched CASE; " + "the top-level summary table's single bare 'CASE WHEN' token covers " + "the searched form and does not extend to this one" + ), + "Parentheses — expression grouping": ( + "its Supported SQL Constructs row (core-spec/expression_language.md:120) " + "carries no backtick token in either cell, the only marker the " + "top-table extraction keys on" + ), + "DISTINCT aggregate modifier": ( + "the DISTINCT modifier is described only in the Conditional " + "Aggregations prose/code block (core-spec/expression_language.md:219-230), " + "never a table row" + ), + "Column / metric reference — field, dataset.field": ( + "its Supported SQL Constructs row (core-spec/expression_language.md:108) " + "carries no backtick token in either cell, same reason as Parentheses" + ), + "EXISTS_IN()": ( + "named only in the Reason column of the excluded 'Not Supported in " + "Expressions' table (core-spec/expression_language.md:131), never in a " + "table of its own" + ), + "OVER (PARTITION BY ... ORDER BY ...) clause": ( + "the generic OVER syntax template (core-spec/expression_language.md:548-560) " + "is a fenced code block, not a table" + ), + "Frame clause — ROWS BETWEEN ... / RANGE BETWEEN ...": ( + "frame options are a bullet list under the OVER syntax section " + "(core-spec/expression_language.md:556-560), not a table" + ), + "Window aggregation — AGG(expr) OVER (...)": ( + "the Window Aggregations section (core-spec/expression_language.md:583-599) " + "is prose and code examples, not a table" + ), +} + + +# -------------------------------------------------------------------------- +# spec_construct_names(): the upstream-spec oracle. +# -------------------------------------------------------------------------- + +_HEADING_RE = re.compile(r"^(#{1,6})\s+(.*)$") +_CODE_SPAN_RE = re.compile(r"`([^`]+)`") + +# A section whose entire content is about something *other than* an Ossie +# construct in its own right. Matched case-insensitively as a substring of +# the heading text at any level, so a subsection ("### Common Dialect +# Variations" under "## Dialect Extensions") is caught the same way as a +# top-level one ("## Cross-Reference: Tool Mappings"). +_EXCLUDED_SECTION_MARKERS = ( + "not supported", # "Not Supported in Expressions": SELECT/FROM/GROUP BY/... are + # explicitly things Ossie expressions do NOT support - the opposite of a construct. + "dialect variation", # "Common Dialect Variations": other engines' spellings of + # constructs already counted from their Ossie-standard table. + "cross-reference", # "Cross-Reference: Tool Mappings": Tableau / Looker Studio / DAX + # spellings, not Ossie constructs. +) + +# A standalone H3 section that defines exactly one construct via prose/a code +# fence rather than a table - e.g. "### CAST (REQUIRED)". Matches only when +# the heading's own name is a single token (letters, digits, underscore, or a +# markdown-escaped underscore "\_"), which is what distinguishes "CAST" or +# "TRY\_CAST" from a descriptive multi-word heading like "Alternative +# Extraction Syntax" or "Null-Safe Comparison" (those need bespoke handling +# below, because more than one construct - or a construct whose name isn't +# the heading text - lives in their body). +_SINGLE_CONSTRUCT_HEADING_RE = re.compile( + r"^### ([A-Za-z0-9_]+(?:\\_[A-Za-z0-9_]+)*) \((?:REQUIRED|RECOMMENDED|EXPERIMENTAL)\)\s*$", + re.MULTILINE, +) + + +def _find_spec_path() -> Path: + """Walk up from this file to the repository root and locate the upstream spec.""" + here = Path(__file__).resolve() + for parent in here.parents: + candidate = parent / "core-spec" / "expression_language.md" + if candidate.is_file(): + return candidate + raise FileNotFoundError( + f"core-spec/expression_language.md not found by walking up from {here}" + ) + + +def _split_table_row(line: str) -> list[str]: + """Split a markdown table row on '|', but never inside a backtick code span. + + The document contains at least one cell whose code span itself contains a + pipe character - the `||` concatenation operator, written as + `` `str1 || str2` `` - which a naive `str.split("|")` would shred into + extra empty cells. Backticks otherwise never appear as literal (non-code) + content in this document's tables, so "toggle on backtick" is a safe, + general rule rather than a special case for that one row. + """ + line = line.strip() + if line.startswith("|"): + line = line[1:] + if line.endswith("|"): + line = line[:-1] + cells: list[str] = [] + current: list[str] = [] + in_code_span = False + for ch in line: + if ch == "`": + in_code_span = not in_code_span + current.append(ch) + elif ch == "|" and not in_code_span: + cells.append("".join(current).strip()) + current = [] + else: + current.append(ch) + cells.append("".join(current).strip()) + return cells + + +def _clean(cell: str) -> str: + """Strip markdown code-span backticks, leaving the underlying syntax text.""" + return cell.replace("`", "").strip() + + +def _is_section_excluded(heading_stack: dict[int, str]) -> bool: + return any( + marker in heading.lower() + for heading in heading_stack.values() + for marker in _EXCLUDED_SECTION_MARKERS + ) + + +def _extract_tables(text: str) -> tuple[set[str], set[str], list[tuple[str, str]]]: + """One pass over the document collecting constructs from ordinary tables. + + Returns: + names: constructs whose spec_name is a `Syntax` column value. + identifier_tokens: the identifying (first) column's backtick tokens for every + such table row, used to de-duplicate against the top-level summary table. + summary_rows: (construct_cell, notes_cell) pairs from the top-level + "Supported SQL Constructs" table, processed by the caller once every + detailed table has been seen. + """ + names: set[str] = set() + identifier_tokens: set[str] = set() + summary_rows: list[tuple[str, str]] = [] + + heading_stack: dict[int, str] = {} + lines = text.splitlines() + i, n = 0, len(lines) + while i < n: + line = lines[i] + + heading_match = _HEADING_RE.match(line) + if heading_match: + level = len(heading_match.group(1)) + for lvl in [lvl for lvl in heading_stack if lvl >= level]: + del heading_stack[lvl] + heading_stack[level] = heading_match.group(2).strip() + i += 1 + continue + + stripped = line.strip() + + # A fenced code block is never itself a table; skip its body outright. + # (The few constructs defined only inside a fence are picked up by the + # bespoke passes in spec_construct_names(), keyed by heading shape.) + if stripped.startswith("```"): + i += 1 + while i < n and not lines[i].strip().startswith("```"): + i += 1 + i += 1 + continue + + if not stripped.startswith("|"): + i += 1 + continue + + if _is_section_excluded(heading_stack): + # Skip the whole table without even parsing it. + while i < n and lines[i].strip().startswith("|"): + i += 1 + continue + + table_lines = [] + while i < n and lines[i].strip().startswith("|"): + table_lines.append(lines[i]) + i += 1 + if len(table_lines) < 2: + continue # a header with no separator row is not a real table + + header = [_split_table_row(table_lines[0])] + header_lower = [c.lower() for c in header[0]] + data_rows = [_split_table_row(r) for r in table_lines[2:]] + + # A table whose identifying column is literally "Token" is a + # format-token argument vocabulary (TO_CHAR/TO_DATE's `format` argument). + if header_lower and header_lower[0] == "token": + continue + + if "syntax" in header_lower: + syntax_idx = header_lower.index("syntax") + form_idx = header_lower.index("form") if "form" in header_lower else None + for row in data_rows: + if len(row) <= syntax_idx: + continue + if ( + form_idx is not None + and len(row) > form_idx + and row[form_idx].strip().lower() == "cast" + ): + # A "Cast" row in the Date/Time Construction table is a worked + # example of the already-catalogued generic CAST(...) construct, + # not a new one. + continue + for token in _CODE_SPAN_RE.findall(row[0]) if row else []: + identifier_tokens.add(token.strip().upper()) + syntax_value = _clean(row[syntax_idx]) + if syntax_value: + names.add(syntax_value) + elif header_lower[:2] == ["construct", "notes"]: + # The top-level "Supported SQL Constructs" table. Its rows range from + # genuine one-off operators/keywords (BETWEEN, CASE WHEN, IN / NOT IN) + # to category headers elaborated in detail elsewhere (Aggregate + # functions, Window functions) - processed once every detailed table + # has been seen, so it can tell the two apart (see _extract_summary_rows). + for row in data_rows: + if row: + summary_rows.append((row[0], row[1] if len(row) > 1 else "")) + # Any other table shape ("Database Support", "Decomposability Reference", + # the working-group roster, a comparison-of-quoting-styles example) names + # no new construct and is left unread. + + return names, identifier_tokens, summary_rows + + +def _extract_summary_rows( + summary_rows: list[tuple[str, str]], identifier_tokens: set[str] +) -> set[str]: + """Pull genuine constructs out of the top "Supported SQL Constructs" table. + + A row counts only if its Construct cell or its Notes cell carries a + backtick-quoted token - the document's own marker for "this cell names a + real piece of syntax" - which is how e.g. `BETWEEN`, `` `IN` / `NOT IN` `` + and `` `CASE WHEN` `` are told apart from plain category labels like + "Column and Metric references" or "Aggregate functions" (elaborated in + detailed tables elsewhere, and carrying no backtick markup of their own). + + A token already seen as a detailed table's identifying column (e.g. `LIKE` + from the Pattern Matching table) is skipped here to avoid counting the same + construct twice under two different spellings. + """ + names: set[str] = set() + for construct_cell, notes_cell in summary_rows: + tokens = _CODE_SPAN_RE.findall(construct_cell) + if not tokens: + for part in notes_cell.split(","): + tokens.extend(_CODE_SPAN_RE.findall(part)) + for token in tokens: + token = token.strip() + if token.upper() in identifier_tokens: + continue + names.add(token) + return names + + +def _extract_extraction_syntax_functions(text: str) -> set[str]: + """EXTRACT and DATE_PART: named in a code fence, not a table. + + The "Alternative Extraction Syntax" section is the only place either + function is named; the bullet list immediately below it enumerates the + date parts they accept (an argument vocabulary, not a construct). + That list needs no special exclusion - it is a bullet list, not a table, + so `_extract_tables()` never looks at it in the first place. + """ + section = re.search( + r"### Alternative Extraction Syntax.*?\n(.*?)\n###", text, re.S + ) + if not section: + return set() + return set(re.findall(r"\b([A-Z_]+)\(", section.group(1))) + + +def _extract_null_safe_comparison_operators(text: str) -> set[str]: + """IS DISTINCT FROM / IS NOT DISTINCT FROM: named only inside a code fence.""" + section = re.search(r"### Null-Safe Comparison.*?\n(.*?)\n---", text, re.S) + if not section: + return set() + return {m.group(0) for m in re.finditer(r"\bIS (?:NOT )?DISTINCT FROM\b", section.group(1))} + + +def _extract_single_construct_headings(text: str) -> set[str]: + """A standalone H3 whose own name (not a table) is the one construct it defines. + + Structural, not name-based: matches any "### {token} ({compliance level})" + heading whose section body contains no pipe-table. CAST and TRY_CAST are + the only two headings in the current document shaped this way. + """ + names: set[str] = set() + for match in _SINGLE_CONSTRUCT_HEADING_RE.finditer(text): + token = match.group(1).replace("\\_", "_") + start = match.end() + next_heading = re.search(r"^#{1,6} ", text[start:], re.M) + body = text[start : start + next_heading.start()] if next_heading else text[start:] + if "|" not in body: + names.add(token) + return names + + +def spec_construct_names() -> set[str]: + """The construct inventory of the upstream expression-language specification. + + Reads `core-spec/expression_language.md` fresh on every call - the file is + small and this is a test-time oracle, not a runtime hot path. + """ + text = _find_spec_path().read_text(encoding="utf-8") + + table_names, identifier_tokens, summary_rows = _extract_tables(text) + names = set(table_names) + names |= _extract_summary_rows(summary_rows, identifier_tokens) + names |= _extract_extraction_syntax_functions(text) + names |= _extract_null_safe_comparison_operators(text) + names |= _extract_single_construct_headings(text) + return names diff --git a/converters/thoughtspot/src/ossie_thoughtspot/expressions/emit.py b/converters/thoughtspot/src/ossie_thoughtspot/expressions/emit.py new file mode 100644 index 00000000..ef66bb16 --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/expressions/emit.py @@ -0,0 +1,229 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +"""Render a catalog `Construct` into an actual ThoughtSpot formula. + +Three emitters, one per `Classification` (see `_types.py`): + +- `emit_direct` — substitutes `args` into the construct's native ThoughtSpot + template positionally. A `direct` row may itself be + a composition of native functions, not only a rename — that + composition is baked into `construct.template` by the + catalog, not by this function. +- `emit_passthrough` — renders a `sql_*_op` call. The row's `variant` + fixes both the emitted function name and, through it, the + emitted column's type and measure/attribute role. Every call + raises a WARNING issue naming the function and the + object, because the body is raw, dialect-specific warehouse + SQL, opaque to ThoughtSpot's query planner. A call + that would carry a runtime ThoughtSpot parameter is refused + outright — it cannot resolve to static SQL, so it is not + portable in either direction, and the caller must route it + elsewhere (a THOUGHTSPOT-only dialect entry) instead of + obtaining a formula string from this function. Pass + `partition_column` when the passthrough carries a + `PARTITION BY` and the wrapped result guarantees that column + reaches ThoughtSpot's GROUP BY regardless of what the user's + search selects — enforced by a template/kwarg cross-check in + both directions, not left to caller convention. +- `emit_unmappable` — no formula exists; raises an ERROR issue and returns nothing + (the two-bucket rule: never a silent drop — the caller is + responsible for preserving the construct in custom_extensions). + +Two details that are easy to get subtly wrong (both pinned by tests in test_emit.py): + +- `emit_passthrough` does NOT substitute `args` into the SQL body. The body is + rendered as a quoted *template string*, followed by the arguments as separate + `sql_*_op` positional arguments — ThoughtSpot resolves the `{0}`, `{1}`, ... + placeholders itself at formula-evaluation time. Substituting them here would + produce a formula that looks right and is wrong. +- `emit_direct` DOES substitute positionally (via `str.format`), and rejects an + argument-count mismatch rather than silently dropping or reusing an argument, + which would compute the wrong thing while still importing cleanly. +""" +import json +import re + +from ._types import Classification, Construct +from ..issues import IssueLog, Severity + +_PLACEHOLDER_RE = re.compile(r"\{(\d+)\}") + + +def _placeholder_count(template: str) -> int: + """How many distinct positional `{n}` placeholders a template declares. + + Assumes contiguous 0-based indices (`{0}`, `{1}`, ...), which is the only + shape `str.format(*args)` accepts positionally and the only shape any + catalog template uses. + """ + indices = {int(m) for m in _PLACEHOLDER_RE.findall(template)} + return max(indices) + 1 if indices else 0 + + +def emit_direct(construct: Construct, args: list[str]) -> str: + """Render a DIRECT construct: substitute `args` into its template positionally. + + Raises ValueError if `construct` is not classified DIRECT, or if `args` does + not have exactly the number of positional arguments the template declares — + silently dropping or reusing an argument would produce a formula that imports + cleanly and computes the wrong thing. + """ + if construct.classification is not Classification.DIRECT: + raise ValueError( + f"{construct.spec_name}: emit_direct called on a " + f"{construct.classification.value} construct, not direct" + ) + expected = _placeholder_count(construct.template) + if len(args) != expected: + plural = "argument" if expected == 1 else "arguments" + raise ValueError( + f"{construct.spec_name} expects {expected} {plural}, got {len(args)}" + ) + return construct.template.format(*args) + + +def emit_passthrough( + construct: Construct, + args: list[str], + log: IssueLog, + *, + object_ref: str, + has_parameter: bool = False, + partition_column: str | None = None, +) -> str: + """Render a PASSTHROUGH construct as a `sql_*_op` call and log a warning. + + `has_parameter=True` refuses the call outright: a `sql_*_op` whose + arguments include a ThoughtSpot parameter cannot resolve to static SQL, so it + is not portable in either direction. The caller must not obtain a formula + string from this function in that case — it routes the construct to a + THOUGHTSPOT-only dialect entry instead. + + `partition_column`: when the pass-through's SQL carries a `PARTITION BY`, + pass the column it partitions on and the result comes back wrapped in + `group_aggregate ( , query_groups ( ) + { } , + query_filters ( ) )`, so the partition column reaches ThoughtSpot's GROUP BY + even when the user's search omits it. This is enforced, not left to caller + convention: a template that carries `PARTITION BY` (case-insensitive) but no + `partition_column` raises, and a `partition_column` supplied for a template + with no `PARTITION BY` raises too — a miscopied catalog row fails + loudly here instead of silently emitting an unwrapped, only-sometimes- + correct pass-through. + + The exemplar convention: not every `construct.template` this function renders is a + complete, general-purpose body. Roughly a third of the catalog's PASSTHROUGH rows + (`PERCENTILE_CONT`/`DISC`'s `0.75`, `APPROX_PERCENTILE`'s `0.5`, `NTILE`'s `4`, + `NTH_VALUE`'s `2`, `LAG`/`LEAD`'s offset `1`, `TO_TIMESTAMP`/`TO_CHAR`'s fixed formats, + `DENSE_RANK`/`CUME_DIST`'s fixed `ORDER BY`, typed literals, and window aggregation's + `SUM`, among others) bake ONE caller-supplied value into the template as a literal + while still declaring a satisfiable arity, rather than exposing that value as its own + `{n}` placeholder. This function renders such a row exactly as written — it has no way + to tell an exemplar from a genuinely complete template, since both pass the arg-count + check the same way. The catalog holds an exemplar for documentation and testing; a + caller translating a real occurrence with a different value for that slot must rebuild + the template for that occurrence rather than reuse the catalog row's rendering + verbatim. Each exemplar row's `note` names the baked-in value. + """ + if construct.classification is not Classification.PASSTHROUGH: + raise ValueError( + f"{construct.spec_name}: emit_passthrough called on a " + f"{construct.classification.value} construct, not passthrough" + ) + if has_parameter: + raise ValueError( + f"{construct.spec_name}: a passthrough cannot carry a runtime parameter " + "— it cannot resolve to static SQL" + ) + + # Mirrors emit_direct's own arg-count guard: a mismatch means either a caller + # passing the wrong number of resolved operands, or a template with a hardcoded + # literal (e.g. a fixed date) that declares zero placeholders — either way this + # would otherwise render as a call whose args outnumber (or fall short of) what + # the template's own {0}, {1}, ... placeholders consume, silently appending an + # unused argument or leaving a placeholder unfilled instead of failing loudly. + expected = _placeholder_count(construct.template) + if len(args) != expected: + plural = "argument" if expected == 1 else "arguments" + raise ValueError( + f"{construct.spec_name} expects {expected} {plural}, got {len(args)}" + ) + + # Enforced rather than left to caller convention: every passthrough row + # that needs the group_aggregate wrap carries the literal string "PARTITION BY" + # in its SQL template (ROW_NUMBER, LAG, LEAD, the OVER fallback, window + # aggregation, and the RANK/PERCENT_RANK/CUME_DIST fallbacks all do). Checking + # the template against the kwarg in both directions turns "the catalog author + # must remember to pass this" into something this function refuses to get wrong. + carries_partition_by = bool(re.search(r"partition\s+by", construct.template, re.IGNORECASE)) + if carries_partition_by and partition_column is None: + raise ValueError( + f"{construct.spec_name}: template carries PARTITION BY but no " + "partition_column was supplied — the group_aggregate wrapper is required" + ) + if partition_column is not None and not carries_partition_by: + raise ValueError( + f"{construct.spec_name}: partition_column was supplied but the template " + "carries no PARTITION BY — there is nothing to wrap" + ) + + # variant is guaranteed non-None for a PASSTHROUGH row by Construct.__post_init__. + variant = construct.variant + quoted_template = json.dumps(construct.template) + body = " , ".join([quoted_template, *args]) + call = f"{variant.value} ( {body} )" + + log.add( + code="TS-EXPR-PASSTHROUGH", + severity=Severity.WARNING, + message=( + f"{construct.spec_name} is emitted as a {variant.value} pass-through: " + "raw warehouse SQL, opaque to ThoughtSpot's query planner. Review before use." + ), + object_ref=object_ref, + ) + + if partition_column is not None: + return ( + f"group_aggregate ( {call} , " + f"query_groups ( ) + {{ {partition_column} }} , query_filters ( ) )" + ) + return call + + +def emit_unmappable(construct: Construct, log: IssueLog, *, object_ref: str) -> None: + """Raise an ERROR issue for an UNMAPPABLE construct. Never a silent drop. + + Returns nothing — the caller is responsible for preserving the construct in + `custom_extensions` for roundtrip; that stash is out of this function's scope. + """ + if construct.classification is not Classification.UNMAPPABLE: + raise ValueError( + f"{construct.spec_name}: emit_unmappable called on a " + f"{construct.classification.value} construct, not unmappable" + ) + log.add( + code="TS-EXPR-UNMAPPABLE", + severity=Severity.ERROR, + message=( + f"{construct.spec_name} has no ThoughtSpot representation; " + "preserved in custom_extensions for roundtrip." + ), + object_ref=object_ref, + ) + return None diff --git a/converters/thoughtspot/src/ossie_thoughtspot/expressions/reverse.py b/converters/thoughtspot/src/ossie_thoughtspot/expressions/reverse.py new file mode 100644 index 00000000..ce17f0f5 --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/expressions/reverse.py @@ -0,0 +1,940 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The reverse-direction inventory: ThoughtSpot functions with no counterpart in the +Ossie specification. + +`CATALOG` (catalog.py) is the specification's own inventory, Ossie construct -> ThoughtSpot +rendering. This module is the other half of a bidirectional converter: ThoughtSpot's own +native functions that the 146-row `CATALOG` never targets, sourced from the "Reverse +direction (ThoughtSpot -> Ossie)" section of +docs/ossie/ts-ossie-function-mapping.md (thoughtspot-agent-skills repo, not vendored here) — +its three sub-sections (conditional aggregates and arithmetic helpers; window, LOD and +semi-additive functions; runtime, display and calendar concepts). + +One governing idea shapes the whole module: prefer composition over the stash. Most of ThoughtSpot's +apparently-proprietary functions are sugar over constructs the specification already has +(`sum_if` -> `SUM(CASE WHEN ...)`, `safe_divide` -> `COALESCE(a / NULLIF(b, 0), 0)`, +`group_sum` over a fixed grain -> `SUM(x) OVER (PARTITION BY attr)`). The stash +(`custom_extensions` + issue) is for what genuinely has no expression — a short +list dominated by *runtime* concepts (parameters, signed-in-user identity, display markup, +fiscal calendars), not by missing mathematics. + +Four dispositions, not the forward module's three +------------------------------------------------------------------------------------------ +`Classification` (catalog.py's DIRECT/PASSTHROUGH/UNMAPPABLE) does not fit this direction +cleanly, so this module defines its own `ReverseDisposition` rather than bending it (as +instructed): a reverse row can compose fully, compose *partially* (with a real fidelity +loss that still deserves an issue and a preserved verbatim rendering), resolve to +the Ossie `dialects[]` mechanism instead of a portable expression at all, or have no +expression whatsoever. + + COMPOSE a full Ossie expression is produced. May still carry a caveat issue (locale + dependence, a week-start-day or DAYOFWEEK-base assumption) without being lossy + in the way PARTIAL is — the composition is exact, the caveat is about the + *specification's* own portability, not about this construct's translation. + PARTIAL a real Ossie expression is produced, but it is provably incomplete (the + `moving_*`/`cumulative_*` family: the frame and order translate exactly, the + partition does not — a ThoughtSpot window formula cannot declare its own + PARTITION BY, and the specification has no way to express that limitation). + Always logs a WARNING and the caller should pair the composed expression with a THOUGHTSPOT dialect + entry (`thoughtspot_dialect_entry`) carrying the verbatim original — and an + ANSI_SQL sibling (`portable_dialect_entry`) for the composed expression itself, + since it *is* portable, just incomplete: a consumer that does not implement the + THOUGHTSPOT dialect still gets something it can execute. + DIALECT the construct's natural home is the Ossie `dialects[]` mechanism, not a + portable expression — the ten `sql_*_op` / `sql_*_aggregate_op` names. No + ANSI_SQL sibling is ever emitted for these (the document is explicit: raw + warehouse SQL's portability is exactly what is unknown). + STASH no Ossie expression exists at all. Always logs an ERROR (mirroring + `emit_unmappable`'s severity choice for the same "no representation, preserved + only for roundtrip" shape) and the caller should attach a THOUGHTSPOT + dialect entry with the verbatim call. + +Argument abstraction level +------------------------------------------------------------------------------------------ +`translate_thoughtspot(name, args, log, *, object_ref, ...)` takes `args` as already- +extracted operand strings, exactly the abstraction level `emit_direct`/`emit_passthrough` +take in the forward direction (never raw ThoughtSpot formula text with nested calls or +brace/quote syntax to parse) — no expression parser exists yet, so there is nothing to parse +from either direction yet. A caller with a real parsed formula tree supplies the resolved +operand strings positionally. + +`translate_thoughtspot` carries `object_ref` as a required keyword-only parameter beyond the +plain `(name, args, log)` shape: `IssueLog.add` requires it, the same shape +`emit_passthrough`/`emit_unmappable` already use for the identical reason. A second keyword- +only parameter, `connection_dialect`, is added for the DIALECT family only (see +`_dispatch_sql_op`) — the document itself says resolving these needs the connection's own +dialect, which is not derivable from a bare function name and argument list. + +Two constructs are cross-cutting rather than name-keyed, so they are not `REVERSE` entries +looked up by `name` at all: + +- **The fiscal-calendar argument.** The document describes this as "the rest of the fiscal + family" without enumerating every date function it can decorate, so `translate_thoughtspot` + checks for a trailing literal `fiscal` argument up front, for any `name` at all, before + falling through to a normal lookup. +- **The runtime-parameter reference.** A bracketed name (`[Discount Threshold]`) is + syntactically identical to an ordinary column reference (`[Table::Column]`) — this module + has no model metadata to tell the two apart, so there is no way to safely auto-detect it in + `translate_thoughtspot`. `stash_runtime_parameter` is exposed separately; the caller (which + does have the model's declared parameter list) invokes it directly once it has confirmed + the name in hand is a declared parameter and not a column. + +Two more are shape-dependent rather than purely name-keyed, and use `ReverseConstruct`'s +`dispatch_fn` escape hatch (full control over composing vs. stashing, bypassing the +declarative `template`/`compose_fn` path entirely): `group_aggregate` and its four named +shorthands (`group_sum`, `group_count`, `group_stddev`, `group_variance`), whose disposition +depends on the shape of the grouping and filter arguments (see `_compose_grouped`); and the +ten `sql_*_op` names, whose disposition depends on whether the caller can supply +`connection_dialect` (see `_dispatch_sql_op`). `concat` is a third, narrower case: plain +`concat` has a spec counterpart already in `CATALOG` and is not this module's concern at all +(returns `None`, no issue) — only the ThoughtSpot hyperlink-markup content pattern inside its +string arguments (`{caption}` / `{/caption}`) is reverse-inventory territory. +""" +import re +from dataclasses import dataclass +from enum import Enum +from typing import Callable + +from ..constants import DIALECT, PORTABLE_DIALECT +from ..issues import IssueLog, Severity + + +class ReverseDisposition(str, Enum): + """How a ThoughtSpot-only construct reaches an Ossie document. See the module + docstring's "Four dispositions" section for the full reasoning behind each. + """ + + COMPOSE = "compose" + PARTIAL = "partial" + DIALECT = "dialect" + STASH = "stash" + + +# A dispatch_fn takes (args, log, object_ref, connection_dialect) and returns the composed +# Ossie expression, or None if it decides — internally, based on argument shape — to stash +# instead. It owns its own issue logging; disposition on such a row is documentation only. +DispatchFn = Callable[[list, IssueLog, str, str | None], str | None] +ComposeFn = Callable[[list], str] + + +@dataclass(frozen=True) +class ReverseConstruct: + """One row of the reverse-direction inventory. + + `thoughtspot_name` the construct as ThoughtSpot spells it, e.g. "sum_if". May be a + synthetic, non-callable key for a construct the document describes + by content pattern rather than by name (e.g. the hyperlink-markup + row) — always documented as such at the registration site. + `disposition` see `ReverseDisposition`. + `template` for a plain positional COMPOSE/PARTIAL row: the Ossie expression + with {0}, {1}, ... placeholders, rendered via `str.format`. Mutually + exclusive with `compose_fn` and unused when `dispatch_fn` is set. + `compose_fn` for a COMPOSE/PARTIAL row whose composition is not a simple + positional substitution (variable arity, a transform on an + argument's literal value). Takes precedence over `template`. + `dispatch_fn` full override: decides composing vs. stashing itself from argument + shape, and does its own issue logging. When set, `disposition` + above is documentation only and no other field is validated. + `issue_code` `IssueLog.add(code=...)` for this row's issue — never a + bare "untranslatable" message. + `issue_severity` WARNING for a COMPOSE-with-caveat or PARTIAL row (something usable + is still produced); ERROR for STASH (nothing is — mirrors + `emit_unmappable`'s choice for the same "no representation" shape). + `issue_message` may contain a `{name}` placeholder, filled with the actual + ThoughtSpot name/reference at translation time — the same message + template serves every case sharing one reason (the five identity + functions, the fiscal-calendar family). + `note` the row's caveat, traceable to the document, same convention as + catalog.py's `Construct.note`. + """ + + thoughtspot_name: str + disposition: ReverseDisposition + template: str | None = None + compose_fn: ComposeFn | None = None + dispatch_fn: DispatchFn | None = None + issue_code: str = "" + issue_severity: Severity = Severity.WARNING + issue_message: str = "" + note: str = "" + + def __post_init__(self) -> None: + if self.dispatch_fn is not None: + # Full custom control - no further shape validation applies (see class docstring). + return + needs_body = self.disposition in (ReverseDisposition.COMPOSE, ReverseDisposition.PARTIAL) + has_body = self.template is not None or self.compose_fn is not None + if needs_body and not has_body: + raise ValueError( + f"{self.thoughtspot_name}: a {self.disposition.value} row needs a " + "template or compose_fn" + ) + if not needs_body and has_body: + raise ValueError( + f"{self.thoughtspot_name}: a {self.disposition.value} row must not carry " + "a template or compose_fn" + ) + if self.disposition in (ReverseDisposition.PARTIAL, ReverseDisposition.STASH) and not self.issue_message: + raise ValueError( + f"{self.thoughtspot_name}: a {self.disposition.value} row must carry an " + "issue message — never a bare 'untranslatable'" + ) + + +REVERSE: dict[str, ReverseConstruct] = {} + +_PLACEHOLDER_RE = re.compile(r"\{(\d+)\}") + + +def _placeholder_count(template: str) -> int: + indices = {int(m) for m in _PLACEHOLDER_RE.findall(template)} + return max(indices) + 1 if indices else 0 + + +def _render(construct: ReverseConstruct, args: list[str]) -> str: + if construct.compose_fn is not None: + return construct.compose_fn(args) + expected = _placeholder_count(construct.template) + if len(args) != expected: + plural = "argument" if expected == 1 else "arguments" + raise ValueError(f"{construct.thoughtspot_name} expects {expected} {plural}, got {len(args)}") + return construct.template.format(*args) + + +def _apply_stash(construct: ReverseConstruct, name: str, log: IssueLog, *, object_ref: str) -> None: + log.add( + code=construct.issue_code, + severity=construct.issue_severity, + message=construct.issue_message.format(name=name), + object_ref=object_ref, + ) + return None + + +# -------------------------------------------------------------------------- +# Conditional aggregates and arithmetic helpers — 28 names, all COMPOSE. +# Source: docs/ossie/ts-ossie-function-mapping.md, "Reverse direction -> +# Conditional aggregates and arithmetic helpers". +# -------------------------------------------------------------------------- + +for _name, _agg in ( + ("sum_if", "SUM"), + ("count_if", "COUNT"), + ("average_if", "AVG"), + ("min_if", "MIN"), + ("max_if", "MAX"), + ("stddev_if", "STDDEV"), + ("variance_if", "VARIANCE"), +): + REVERSE[_name] = ReverseConstruct( + thoughtspot_name=_name, + disposition=ReverseDisposition.COMPOSE, + template=f"{_agg}(CASE WHEN {{0}} THEN {{1}} END)", + note=f"{_name} ( cond , x ) -> {_agg}(CASE WHEN cond THEN x END).", + ) + +REVERSE["unique_count_if"] = ReverseConstruct( + thoughtspot_name="unique_count_if", + disposition=ReverseDisposition.COMPOSE, + template="COUNT(DISTINCT CASE WHEN {0} THEN {1} END)", + note="unique_count_if ( cond , x ) -> COUNT(DISTINCT CASE WHEN cond THEN x END).", +) + +REVERSE["unique count"] = ReverseConstruct( + thoughtspot_name="unique count", + disposition=ReverseDisposition.COMPOSE, + template="COUNT(DISTINCT {0})", + note="ThoughtSpot's own spelling has a space, not an underscore.", +) + +REVERSE["safe_divide"] = ReverseConstruct( + thoughtspot_name="safe_divide", + disposition=ReverseDisposition.COMPOSE, + template="COALESCE({0} / NULLIF({1}, 0), 0)", + note="The zero-not-null result is preserved by the explicit COALESCE.", +) + +REVERSE["pow"] = ReverseConstruct( + thoughtspot_name="pow", disposition=ReverseDisposition.COMPOSE, template="POWER({0}, {1})", +) +REVERSE["log2"] = ReverseConstruct( + thoughtspot_name="log2", disposition=ReverseDisposition.COMPOSE, template="LOG(2, {0})", +) +REVERSE["strlen"] = ReverseConstruct( + thoughtspot_name="strlen", disposition=ReverseDisposition.COMPOSE, template="LENGTH({0})", +) +REVERSE["strpos"] = ReverseConstruct( + thoughtspot_name="strpos", + disposition=ReverseDisposition.COMPOSE, + template="POSITION({1} IN {0})", + note="ThoughtSpot strpos(s, sub) -> Ossie POSITION(sub IN s); operand order reverses.", +) +REVERSE["substr"] = ReverseConstruct( + thoughtspot_name="substr", + disposition=ReverseDisposition.COMPOSE, + template="SUBSTRING({0}, {1} + 1, {2})", + note="ThoughtSpot's substr is 0-based; the +1 is mandatory going this way.", +) +REVERSE["left"] = ReverseConstruct( + thoughtspot_name="left", disposition=ReverseDisposition.COMPOSE, template="LEFT({0}, {1})", +) +REVERSE["right"] = ReverseConstruct( + thoughtspot_name="right", disposition=ReverseDisposition.COMPOSE, template="RIGHT({0}, {1})", +) + +for _name in ("sin", "cos", "tan"): + REVERSE[_name] = ReverseConstruct( + thoughtspot_name=_name, + disposition=ReverseDisposition.COMPOSE, + template=f"{_name.upper()}(RADIANS({{0}}))", + note="ThoughtSpot trigonometry is in degrees; the conversion reverses.", + ) +for _name in ("asin", "acos", "atan"): + REVERSE[_name] = ReverseConstruct( + thoughtspot_name=_name, + disposition=ReverseDisposition.COMPOSE, + template=f"DEGREES({_name.upper()}({{0}}))", + note="ThoughtSpot's inverse trig functions return degrees.", + ) + +REVERSE["to_integer"] = ReverseConstruct( + thoughtspot_name="to_integer", disposition=ReverseDisposition.COMPOSE, template="CAST({0} AS INTEGER)", +) +REVERSE["to_double"] = ReverseConstruct( + thoughtspot_name="to_double", disposition=ReverseDisposition.COMPOSE, template="CAST({0} AS DOUBLE)", +) +REVERSE["to_string"] = ReverseConstruct( + thoughtspot_name="to_string", disposition=ReverseDisposition.COMPOSE, template="CAST({0} AS VARCHAR)", +) +REVERSE["to_date"] = ReverseConstruct( + thoughtspot_name="to_date", + disposition=ReverseDisposition.COMPOSE, + template="TO_DATE({0}, {1})", + issue_code="TS-EXPR-FORMAT-TOKENS-PASSTHROUGH", + issue_severity=Severity.INFO, + issue_message=( + "{name}'s format string is passed through verbatim, not mechanically translated " + "through the TO_DATE/TO_CHAR format-token table — no expression parser exists yet " + "to do that translation. TO_DATE(s, format) is EXPERIMENTAL on the Ossie side." + ), + note="Judgment call: format-token reversal is deferred until an expression parser exists to do the translation.", +) +REVERSE["if"] = ReverseConstruct( + thoughtspot_name="if", + disposition=ReverseDisposition.COMPOSE, + template="CASE WHEN {0} THEN {1} ELSE {2} END", + note="if ( c ) then a else b -> CASE WHEN c THEN a ELSE b END, or IF(c, a, b).", +) + + +# -------------------------------------------------------------------------- +# Window, LOD and semi-additive functions. +# Source: "Reverse direction -> Window, LOD and semi-additive functions". +# -------------------------------------------------------------------------- + +def _direction_keyword(literal: str) -> str: + bare = literal.strip().strip("'\"").lower() + if not bare.startswith(("asc", "desc")): + raise ValueError(f"unrecognised rank direction literal: {literal!r}") + return "DESC" if bare.startswith("desc") else "ASC" + + +def _compose_rank(args: list[str]) -> str: + if len(args) != 2: + raise ValueError(f"rank expects 2 arguments, got {len(args)}") + agg, direction = args + return f"RANK() OVER (ORDER BY {agg} {_direction_keyword(direction)})" + + +def _compose_rank_percentile(args: list[str]) -> str: + if len(args) != 2: + raise ValueError(f"rank_percentile expects 2 arguments, got {len(args)}") + agg, direction = args + return f"(1.0 - PERCENT_RANK() OVER (ORDER BY {agg} {_direction_keyword(direction)})) * 100" + + +REVERSE["rank"] = ReverseConstruct( + thoughtspot_name="rank", disposition=ReverseDisposition.COMPOSE, compose_fn=_compose_rank, + note="Global, ORDER-BY-only shape only — rank's arity is fixed at exactly two " + "(live-confirmed), so there is never a partition to lose in this direction.", +) +REVERSE["rank_percentile"] = ReverseConstruct( + thoughtspot_name="rank_percentile", + disposition=ReverseDisposition.COMPOSE, + compose_fn=_compose_rank_percentile, + note="Scale (0-100 -> 0-1) and inversion both reverse.", +) + + +def _frame_bound(offset: str) -> str: + n = int(offset.strip()) + if n > 0: + return f"{n} PRECEDING" + if n == 0: + return "CURRENT ROW" + return f"{-n} FOLLOWING" + + +_PARTITION_LOST_ISSUE = ( + "{name}'s emitted OVER clause has no PARTITION BY: ThoughtSpot completes the partition " + "dynamically from the query's own dimensions minus the order columns — a ThoughtSpot " + "window formula cannot declare its own PARTITION BY, and a static Ossie window has no " + "way to express that limitation. The composed expression is correct " + "only when the search returns exactly the grain this formula assumed. " + "Pair this with a THOUGHTSPOT dialect entry carrying the verbatim original." +) + + +def _compose_moving(agg: str) -> ComposeFn: + def _compose(args: list[str]) -> str: + if len(args) < 4: + raise ValueError( + f"moving_{agg.lower()} expects at least 4 arguments (m, start, end, order...), " + f"got {len(args)}" + ) + m, start, end, *order_cols = args + order_clause = ", ".join(order_cols) + return ( + f"{agg}({m}) OVER (ORDER BY {order_clause} " + f"ROWS BETWEEN {_frame_bound(start)} AND {_frame_bound(end)})" + ) + return _compose + + +def _compose_cumulative(agg: str) -> ComposeFn: + def _compose(args: list[str]) -> str: + if len(args) < 2: + raise ValueError( + f"cumulative_{agg.lower()} expects at least 2 arguments (m, order...), got {len(args)}" + ) + m, *order_cols = args + order_clause = ", ".join(order_cols) + return ( + f"{agg}({m}) OVER (ORDER BY {order_clause} " + "ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)" + ) + return _compose + + +for _agg in ("SUM", "AVERAGE", "MAX", "MIN"): + _ansi_agg = "AVG" if _agg == "AVERAGE" else _agg + REVERSE[f"moving_{_agg.lower()}"] = ReverseConstruct( + thoughtspot_name=f"moving_{_agg.lower()}", + disposition=ReverseDisposition.PARTIAL, + compose_fn=_compose_moving(_ansi_agg), + issue_code="TS-EXPR-PARTIAL-PARTITION", + issue_severity=Severity.WARNING, + issue_message=_PARTITION_LOST_ISSUE, + note="Frame and order translate exactly; the partition does not — a ThoughtSpot window formula cannot declare its own PARTITION BY, and the specification has no way to express that limitation.", + ) + REVERSE[f"cumulative_{_agg.lower()}"] = ReverseConstruct( + thoughtspot_name=f"cumulative_{_agg.lower()}", + disposition=ReverseDisposition.PARTIAL, + compose_fn=_compose_cumulative(_ansi_agg), + issue_code="TS-EXPR-PARTIAL-PARTITION", + issue_severity=Severity.WARNING, + issue_message=_PARTITION_LOST_ISSUE, + note="Frame and order translate exactly; the partition does not — a ThoughtSpot window formula cannot declare its own PARTITION BY, and the specification has no way to express that limitation.", + ) + + +def _compose_grouped( + agg_call: str, + grouping_arg: str, + filter_arg: str, + log: IssueLog, + *, + object_ref: str, + source_name: str, +) -> str | None: + """Shared shape dispatch for `group_aggregate` and its named shorthands. + + Three dispositions live under one ThoughtSpot spelling, distinguished only by the + grouping/filter arguments' shape (all three shapes are live-confirmed real, per the + document's "Window rows live-confirmed" section): + + - a `query_filters ( )`-only filter and a fixed `{ ... }` grouping (or `query_groups ( )` + alone) composes cleanly — this is the one ThoughtSpot windowing form that is clean in + this direction, because its partition is declared in the formula rather than completed + from the query; + - a `query_groups ( ) ± { attr }` dynamic grouping has no expression (the largest + reverse-direction fidelity gap); + - any filter argument other than `query_filters ( )` has no expression either (filter + scoping is excluded from Ossie expressions). + """ + grouping = grouping_arg.strip() + filt = filter_arg.strip() + if filt != "query_filters ( )": + log.add( + code="TS-EXPR-GROUP-FILTER-SCOPE", + severity=Severity.ERROR, + message=( + f"{source_name}'s filter argument ({filter_arg!r}) scopes the aggregate to " + "a filtered subset of the query; the specification excludes filter scoping " + "from expressions. Preserved verbatim for roundtrip." + ), + object_ref=object_ref, + ) + return None + if "query_groups ( )" in grouping and ("-" in grouping or "+" in grouping): + log.add( + code="TS-EXPR-GROUP-DYNAMIC-PARTITION", + severity=Severity.ERROR, + message=( + f"{source_name}'s grouping argument ({grouping_arg!r}) completes the " + "partition dynamically from the query's own dimensions, which the " + "specification cannot express — the largest reverse-direction " + "fidelity gap. Preserved verbatim for roundtrip." + ), + object_ref=object_ref, + ) + return None + if grouping == "query_groups ( )": + return agg_call + if grouping.startswith("{") and grouping.endswith("}"): + cols = grouping[1:-1].strip() + return f"{agg_call} OVER ()" if not cols else f"{agg_call} OVER (PARTITION BY {cols})" + raise ValueError(f"{source_name}: unrecognised grouping argument shape {grouping_arg!r}") + + +def _dispatch_group_aggregate( + args: list[str], log: IssueLog, object_ref: str, connection_dialect: str | None +) -> str | None: + if len(args) != 3: + raise ValueError(f"group_aggregate expects 3 arguments (agg, grouping, filter), got {len(args)}") + agg_call, grouping_arg, filter_arg = args + return _compose_grouped(agg_call, grouping_arg, filter_arg, log, object_ref=object_ref, source_name="group_aggregate") + + +REVERSE["group_aggregate"] = ReverseConstruct( + thoughtspot_name="group_aggregate", + disposition=ReverseDisposition.COMPOSE, + dispatch_fn=_dispatch_group_aggregate, + note="Shape dispatch on the grouping/filter arguments — see _compose_grouped.", +) + +_GROUP_SHORTHAND_AGGREGATES = { + # Only the shorthands the document names explicitly (its own worked example, "group_sum", + # and line ~420's "group_count / group_stddev / group_variance") — no group_average, + # group_max or group_min is invented, since the document never names them. + "group_sum": "SUM", + "group_count": "COUNT", + "group_stddev": "STDDEV", + "group_variance": "VARIANCE", +} + + +def _make_group_shorthand_dispatch(agg: str, source_name: str) -> DispatchFn: + def _dispatch(args: list[str], log: IssueLog, object_ref: str, connection_dialect: str | None) -> str | None: + if len(args) != 3: + raise ValueError(f"{source_name} expects 3 arguments (m, grouping, filter), got {len(args)}") + m, grouping_arg, filter_arg = args + return _compose_grouped(f"{agg}({m})", grouping_arg, filter_arg, log, object_ref=object_ref, source_name=source_name) + return _dispatch + + +for _name, _agg in _GROUP_SHORTHAND_AGGREGATES.items(): + REVERSE[_name] = ReverseConstruct( + thoughtspot_name=_name, + disposition=ReverseDisposition.COMPOSE, + dispatch_fn=_make_group_shorthand_dispatch(_agg, _name), + note=( + f"Shorthand for group_aggregate({_agg.lower()}(m), ...) — same shape dispatch. " + "Judgment call: the (m, grouping, filter) 3-argument shape is assumed by analogy " + "with group_aggregate's live-confirmed form; the shorthand family's own arity " + "was not independently live-tested." + ), + ) + +_SEMI_ADDITIVE_ISSUE = ( + "{name} declares a genuine partition and order axis, and that window clause round-trips " + "faithfully — but semi-additivity is a roll-up declaration (do not re-sum this measure " + "across the axis), not an expression, and the specification has no such declaration. " + "Preserved verbatim for roundtrip." +) +for _name in ("last_value", "first_value", "last_value_in_period", "first_value_in_period"): + REVERSE[_name] = ReverseConstruct( + thoughtspot_name=_name, + disposition=ReverseDisposition.STASH, + issue_code="TS-EXPR-SEMI-ADDITIVE", + issue_severity=Severity.ERROR, + issue_message=_SEMI_ADDITIVE_ISSUE, + note="The window clause itself round-trips; only the roll-up declaration is lost.", + ) + + +def _dispatch_sql_op( + args: list[str], log: IssueLog, object_ref: str, connection_dialect: str | None +) -> str | None: + """The ten `sql_*_op` / `sql_*_aggregate_op` names. `args[0]` is the unquoted template + body (this module's argument abstraction level — see the module docstring), `args[1:]` + are the already-resolved column expressions the template's `{0}`, `{1}`, ... refer to. + + Without a known `connection_dialect` there is nothing to build a `dialects[]` entry + for, and — per the document — the converter must not guess a dialect label, so this + stashes (ERROR) exactly like any other total loss. With one, the body is rendered as + static SQL for that dialect's entry and logged as a WARNING (a pass-through is always + reviewable raw SQL) — never paired with an ANSI_SQL sibling, since the document is + explicit that this template's portability is exactly what is unknown. + """ + if not args: + raise ValueError("a sql_*_op call needs at least its template-body argument") + body_template, *cols = args + if connection_dialect is None: + log.add( + code="TS-EXPR-DIALECT-UNKNOWN", + severity=Severity.ERROR, + message=( + "sql_*_op resolves to a dialects[] entry for the connection's own dialect, " + "which could not be derived from TML here; the converter must not guess a " + "dialect label. Preserved verbatim for roundtrip." + ), + object_ref=object_ref, + ) + return None + try: + body = body_template.format(*cols) + except (IndexError, KeyError) as exc: + raise ValueError( + f"sql_*_op template {body_template!r} does not match {len(cols)} argument(s)" + ) from exc + log.add( + code="TS-EXPR-DIALECT-PASSTHROUGH", + severity=Severity.WARNING, + message=( + f"Raw {connection_dialect} SQL, emitted as a dialects[] entry for that dialect; " + "opaque to any consumer that does not implement it. No ANSI_SQL sibling is " + "emitted — this template's portability is exactly what is unknown. Review " + "before use." + ), + object_ref=object_ref, + ) + return body + + +for _name in ( + "sql_string_op", "sql_int_op", "sql_double_op", "sql_bool_op", "sql_date_op", + "sql_date_time_op", "sql_string_aggregate_op", "sql_int_aggregate_op", + "sql_number_aggregate_op", "sql_date_time_aggregate_op", +): + REVERSE[_name] = ReverseConstruct( + thoughtspot_name=_name, + disposition=ReverseDisposition.DIALECT, + dispatch_fn=_dispatch_sql_op, + note="Resolves to the Ossie dialects[] mechanism for the connection's own dialect, " + "not a portable expression — the right home for raw warehouse SQL.", + ) + + +# -------------------------------------------------------------------------- +# Runtime, display and calendar concepts. +# Source: "Reverse direction -> Runtime, display and calendar concepts". +# -------------------------------------------------------------------------- + +REVERSE[""] = ReverseConstruct( + thoughtspot_name="", + disposition=ReverseDisposition.STASH, + issue_code="TS-EXPR-RUNTIME-PARAMETER", + issue_severity=Severity.ERROR, + issue_message=( + "Runtime parameter reference {name} is resolved per-query from user input; the " + "definitions are stashed at model level (owned by the construct-mapping document). " + "The expression itself stops being portable once it references a parameter. " + "Preserved verbatim for roundtrip." + ), + note="Synthetic key — not a callable name. See stash_runtime_parameter().", +) + + +def stash_runtime_parameter(parameter_name: str, log: IssueLog, *, object_ref: str) -> None: + """The 'Runtime parameter reference' reverse-direction row. + + Not auto-detected inside `translate_thoughtspot`: a bracketed name + (`[Discount Threshold]`) is syntactically identical to an ordinary column reference + (`[Table::Column]`), and this module has no model metadata to distinguish the two. The + caller — which does have the model's declared parameter list — invokes this directly + once it has confirmed `parameter_name` names a declared parameter, not a column. + """ + return _apply_stash(REVERSE[""], parameter_name, log, object_ref=object_ref) + + +_RUNTIME_IDENTITY_ISSUE = ( + "{name} resolves signed-in-user identity at query time; an interchange document that " + "carried it would describe an access-control decision, not semantics. " + "Preserved verbatim for roundtrip." +) +for _name in ("ts_username", "ts_groups", "ts_groups_int", "ts_org", "ts_email_domain", "ts_var"): + REVERSE[_name] = ReverseConstruct( + thoughtspot_name=_name, + disposition=ReverseDisposition.STASH, + issue_code="TS-EXPR-RUNTIME-IDENTITY", + issue_severity=Severity.ERROR, + issue_message=_RUNTIME_IDENTITY_ISSUE, + ) + +_HYPERLINK_MARKUP_TOKENS = ("{caption}", "{/caption}") + + +def _has_hyperlink_markup(args: list[str]) -> bool: + return any(token in a for a in args for token in _HYPERLINK_MARKUP_TOKENS) + + +REVERSE["concat (hyperlink markup)"] = ReverseConstruct( + thoughtspot_name="concat (hyperlink markup)", + disposition=ReverseDisposition.STASH, + issue_code="TS-EXPR-HYPERLINK-MARKUP", + issue_severity=Severity.ERROR, + issue_message=( + "{name}'s string arguments carry ThoughtSpot's {{caption}}/{{/caption}} hyperlink " + "display markup; concat itself maps (it has a spec counterpart, CONCAT), but a " + "consumer that rendered the tags literally would show them to users. Preserved " + "verbatim for roundtrip." + ), + note="Synthetic key, reached only via the content-pattern check in translate_thoughtspot " + "-- plain concat (no markup) is out of this module's scope entirely.", +) + +_FISCAL_MARKERS = {"fiscal", "'fiscal'"} + + +def _is_fiscal_variant(args: list[str]) -> bool: + return bool(args) and args[-1].strip().lower() in _FISCAL_MARKERS + + +_FISCAL_ISSUE_MESSAGE = ( + "{name}'s trailing 'fiscal' argument has no expression: the specification has no " + "fiscal-calendar concept, and the fiscal year's start month is model-level metadata no " + "per-expression rewrite can recover. Emitting the calendar-year composition " + "instead would be silently wrong for any organisation whose year does not start in " + "January. Preserved verbatim for roundtrip." +) + + +def _stash_fiscal_variant(name: str, log: IssueLog, *, object_ref: str) -> None: + log.add( + code="TS-EXPR-FISCAL-CALENDAR", + severity=Severity.ERROR, + message=_FISCAL_ISSUE_MESSAGE.format(name=name), + object_ref=object_ref, + ) + return None + + +_LOCALE_ISSUE_MESSAGE = ( + "{name} composes via TO_CHAR, which is EXPERIMENTAL on the Ossie side, and its name " + "tokens are locale-dependent by the specification's own admission. Review the target " + "locale before relying on this column." +) +for _name, _fmt in (("month", "MONTH"), ("year_name", "YYYY"), ("day_of_week", "DAY")): + REVERSE[_name] = ReverseConstruct( + thoughtspot_name=_name, + disposition=ReverseDisposition.COMPOSE, + template=f"TO_CHAR({{0}}, '{_fmt}')", + issue_code="TS-EXPR-LOCALE-DEPENDENT", + issue_severity=Severity.WARNING, + issue_message=_LOCALE_ISSUE_MESSAGE, + note="Name-returning form, distinct from month_number/year/day_number_of_week.", + ) + +REVERSE["month_number_of_quarter"] = ReverseConstruct( + thoughtspot_name="month_number_of_quarter", + disposition=ReverseDisposition.COMPOSE, + template="MOD(MONTH({0}) - 1, 3) + 1", +) +REVERSE["day_number_of_quarter"] = ReverseConstruct( + thoughtspot_name="day_number_of_quarter", + disposition=ReverseDisposition.COMPOSE, + template="DATEDIFF(day, DATE_TRUNC('quarter', {0}), {0}) + 1", +) + +_WEEK_START_ISSUE = ( + "{name} is correct only if the target engine's week start agrees with the " + "specification's fixed Monday start; ThoughtSpot's week start is an instance setting. " + "Verify alignment before relying on this column." +) +REVERSE["week_number_of_month"] = ReverseConstruct( + thoughtspot_name="week_number_of_month", + disposition=ReverseDisposition.COMPOSE, + template="DATEDIFF(week, DATE_TRUNC('month', {0}), {0}) + 1", + issue_code="TS-EXPR-WEEK-START-ASSUMED", + issue_severity=Severity.WARNING, + issue_message=_WEEK_START_ISSUE, +) +REVERSE["week_number_of_quarter"] = ReverseConstruct( + thoughtspot_name="week_number_of_quarter", + disposition=ReverseDisposition.COMPOSE, + template="DATEDIFF(week, DATE_TRUNC('quarter', {0}), {0}) + 1", + issue_code="TS-EXPR-WEEK-START-ASSUMED", + issue_severity=Severity.WARNING, + issue_message=_WEEK_START_ISSUE, +) +REVERSE["is_weekend"] = ReverseConstruct( + thoughtspot_name="is_weekend", + disposition=ReverseDisposition.COMPOSE, + template="DATE_PART('dayofweek', {0}) IN (6, 7)", + issue_code="TS-EXPR-DAYOFWEEK-BASE", + issue_severity=Severity.WARNING, + issue_message=( + "{name}'s member list (6, 7) uses ThoughtSpot's own DAYOFWEEK base (1 = Monday); " + "the specification does not fix a base and engines disagree — confirm " + "the target engine's base agrees before relying on this column." + ), +) +REVERSE["start_of_hour"] = ReverseConstruct( + thoughtspot_name="start_of_hour", disposition=ReverseDisposition.COMPOSE, + template="DATE_TRUNC('hour', {0})", +) +REVERSE["start_of_min"] = ReverseConstruct( + thoughtspot_name="start_of_min", disposition=ReverseDisposition.COMPOSE, + template="DATE_TRUNC('minute', {0})", +) +REVERSE["date"] = ReverseConstruct( + thoughtspot_name="date", disposition=ReverseDisposition.COMPOSE, + template="DATE_TRUNC('day', {0})", +) +REVERSE["time"] = ReverseConstruct( + thoughtspot_name="time", disposition=ReverseDisposition.COMPOSE, + template="CAST({0} AS TIME)", +) + + +def _compose_variadic(fn: str) -> ComposeFn: + def _compose(args: list[str]) -> str: + if not args: + raise ValueError(f"{fn} expects at least one argument") + return f"{fn}({', '.join(args)})" + return _compose + + +REVERSE["greatest"] = ReverseConstruct( + thoughtspot_name="greatest", + disposition=ReverseDisposition.COMPOSE, + compose_fn=_compose_variadic("GREATEST"), + note="Never MAX — that would turn a row-wise attribute into an aggregate measure.", +) +REVERSE["least"] = ReverseConstruct( + thoughtspot_name="least", + disposition=ReverseDisposition.COMPOSE, + compose_fn=_compose_variadic("LEAST"), + note="Never MIN, for the same reason.", +) + + +# -------------------------------------------------------------------------- +# The dispatcher. +# -------------------------------------------------------------------------- + +def translate_thoughtspot( + name: str, + args: list[str], + log: IssueLog, + *, + object_ref: str, + connection_dialect: str | None = None, +) -> str | None: + """Translate one ThoughtSpot-only construct call into an Ossie expression, or stash it. + + Returns the composed Ossie expression string, or `None` when the construct stashes (an + issue is always logged in that case) or when `name` has no entry in this + module's reverse inventory at all (nothing is logged — that name is either a plain + column/measure reference or a construct with a spec counterpart already covered by the + forward `CATALOG`, neither of which is this module's concern). + + See the module docstring for the two cross-cutting checks below (fiscal-calendar + argument, concat hyperlink markup) and for why `object_ref` and `connection_dialect` are + keyword-only additions beyond the plain `(name, args, log)` signature. + """ + if _is_fiscal_variant(args): + _stash_fiscal_variant(name, log, object_ref=object_ref) + return None + + if name == "concat" and _has_hyperlink_markup(args): + return _apply_stash(REVERSE["concat (hyperlink markup)"], name, log, object_ref=object_ref) + + construct = REVERSE.get(name) + if construct is None: + return None + + if construct.dispatch_fn is not None: + return construct.dispatch_fn(args, log, object_ref, connection_dialect) + + if construct.disposition is ReverseDisposition.STASH: + return _apply_stash(construct, name, log, object_ref=object_ref) + + expression = _render(construct, args) + if construct.issue_message: + log.add( + code=construct.issue_code, + severity=construct.issue_severity, + message=construct.issue_message.format(name=name), + object_ref=object_ref, + ) + return expression + + +# -------------------------------------------------------------------------- +# Dialect-entry and custom_extensions helpers. +# +# translate_thoughtspot's own return type is `str | None`, so it cannot itself hand back a +# dialects[] entry or a custom_extensions payload — those are object-level document +# concerns, one level above a single expression. These three helpers are what a caller +# operating at the object level combines with translate_thoughtspot's result to guarantee +# a lossless roundtrip. +# -------------------------------------------------------------------------- + +def thoughtspot_dialect_entry(name: str, args: list[str]) -> dict[str, str]: + """The verbatim ThoughtSpot call, reconstructed textually (this module never + has the original formula's exact whitespace, only the parsed name/args) so a PARTIAL or + STASH construct still round-trips losslessly through a THOUGHTSPOT dialect entry even + where no full — or no — portable Ossie expression exists. + """ + inner = f" {' , '.join(args)} " if args else " " + return {"dialect": DIALECT, "expression": f"{name} ({inner})"} + + +def portable_dialect_entry(expression: str) -> dict[str, str]: + """Pair the THOUGHTSPOT dialect entry with a PORTABLE_DIALECT (ANSI_SQL) sibling + wherever the expression alongside it is itself portable, so a consumer that does not + implement the THOUGHTSPOT dialect still gets something it can execute. Applies to + PARTIAL rows (the frame/order composition is genuine, portable ANSI SQL, just an + incomplete window) — never to a pure STASH (there is no portable expression to pair) + and never to the `sql_*_op` DIALECT family (the document is explicit: no ANSI_SQL + sibling is emitted there, because that template's portability is exactly what is + unknown). + """ + return {"dialect": PORTABLE_DIALECT, "expression": expression} + + +def custom_extensions_fragment(column: str, name: str, args: list[str]) -> dict[str, dict[str, str]]: + """The payload fragment this module contributes toward an object's + `custom_extensions[VENDOR_KEY]` entry (`stash.write_stash`) for one construct + this module could not fully compose. + + `write_stash(obj, payload)` treats `payload` as the *contents* of the object's + THOUGHTSPOT entry, not as `{VENDOR_KEY: contents}` — `write_stash` already owns the + vendor-key wrapping. So this fragment must be keyed by `column`, the caller's + Ossie metric/column name, not by `VENDOR_KEY`: this module operates at the + single-expression level and has no access to the enclosing object, so the caller merges + fragments across an object's columns — `{**fragment_for_col_a, **fragment_for_col_b}` — + before calling `stash.write_stash` once per object. Keying by `VENDOR_KEY` instead would + make that merge lossy (`{**f1, **f2}` collapses to whichever fragment merged last) and + would nest the vendor key inside its own entry when passed to `write_stash` directly. + """ + inner = f" {' , '.join(args)} " if args else " " + return {column: {"reverse_thoughtspot_call": f"{name} ({inner})"}} diff --git a/converters/thoughtspot/src/ossie_thoughtspot/formula.py b/converters/thoughtspot/src/ossie_thoughtspot/formula.py new file mode 100644 index 00000000..5bd7b634 --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/formula.py @@ -0,0 +1,337 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""A shallow tokenizer for ThoughtSpot formulas. + +Deliberately not a parser. It answers only what the two conversion directions need: is an +expression a single outer call and what are its parts, where are its column references, and +what does it look like with those references rewritten. Building an expression tree would be +the SQL-parser decision this converter does not take — expressions pass through under a +tagged dialect rather than being translated across dialects, matching every other converter +in this repository and the specification's stated default. + +Three ThoughtSpot syntax features drive the implementation and are why an off-the-shelf SQL +tokenizer is not usable here: column references are bracketed and doubly-colon-qualified +(`[TABLE::Column]`), grouping uses braces (`{ }`), and a bare bracketed name with no `::` is +a runtime parameter rather than a column. + +**Quoting note — doubling works, backslash-escaping is out of scope on purpose.** +ThoughtSpot's own convention for an embedded quote in a string literal is doubling it +(`'it''s'`), and `_scan` handles that correctly even though it has no explicit doubling +case: the character that closes a quote and the character that immediately reopens it are +both reported as quoted, so nothing in between ever reads as outside the literal. A +backslash before a quote is *not* an escape in ThoughtSpot's grammar — it is an ordinary +character — so `'a\'b'` genuinely ends the literal at the escaped quote, and `_scan` +splitting there is correct behaviour for this language, not a bug to fix. +""" +from __future__ import annotations + +import re +from typing import Callable + +from . import identifiers + +_CALL_HEAD = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*(?:\s+[A-Za-z_][A-Za-z0-9_]*)*)\s*\(") +#: Same call-head shape as `_CALL_HEAD`, but unanchored (no `^`) so it matches a call +#: starting anywhere in the text, and guarded on the left by a negative lookbehind so +#: a match can never start mid-identifier (e.g. inside "ground" when scanning for a +#: call literally named "round"). Used by `find_call_names` to find every call in an +#: expression, not just the single outer one `split_call` answers about. +_CALL_HEAD_ANYWHERE = re.compile( + r"(? list[str]: + parts: list[str] = [] + start = 0 + for i, ch, depth, in_quote in _scan(text): + if ch == "," and depth == 0 and not in_quote: + parts.append(text[start:i].strip()) + start = i + 1 + tail = text[start:].strip() + if tail or parts: + parts.append(tail) + return parts + + +def split_call(expression: str) -> tuple[str, list[str]] | None: + """`sum ( [A::x] , 2 )` -> `("sum", ["[A::x]", "2"])`; `None` if not a single outer call. + + Returns `None` — never a partial answer — for anything that merely *contains* a call, + such as `sum ( [A::x] ) + 1`. A caller that received `("sum", ["[A::x]"])` for that + input would silently drop the `+ 1`, which is precisely the class of silent loss this + converter exists to prevent. + """ + text = expression.strip() + head = _CALL_HEAD.match(text) + if head is None: + return None + name = head.group(1) + if any(word.lower() in _KEYWORDS for word in name.split()): + # `true and count (...)` is an operator expression whose last operand happens to + # look like a call head, not a call named "true and count". `unique count (...)` + # is unaffected — none of its words are in the blocklist. + return None + open_at = head.end() - 1 + close_at = None + for i, ch, depth, in_quote in _scan(text[open_at:]): + if ch == ")" and depth == 0 and not in_quote: + close_at = open_at + i + break + if close_at is None: + return None + if close_at != len(text) - 1: + return None + inner = text[open_at + 1 : close_at].strip() + if not inner: + return head.group(1), [] + return head.group(1), _split_top_level_commas(inner) + + +def find_call_names(expression: str) -> list[str]: + """Every function-call name in `expression`, at any nesting depth, that is + not also an operator/control-flow keyword — duplicates kept. + + `split_call` deliberately answers only about the single *outer* call — exactly + what building a rendering around a whole expression needs. This answers a + different question a safety check needs instead: whether a call with a + particular name appears *anywhere* inside the expression, however deeply + nested — `round ( sum ( [T::x] ) , 2 )` has `round` as its outer call but + `sum` buried one level inside it, and a caller checking only the outer call + would miss that the expression already aggregates. + + A name is only reported at a genuine call site: immediately followed by `(`, + not inside a quoted string literal, and not inside the opaque body of a + `[...]` reference — so a display name or string literal that happens to + contain text like `sum (` is never mistaken for a real call. + + **What the keyword exclusion actually costs.** A leading run of + operator/control-flow keyword words (`and`, `or`, `not`, `if`, ...) is + stripped from a matched run before it is reported: `true and count ( ... )` + reports `count`, not the bogus "true and count". But `not` and `if` are + *also* genuine ThoughtSpot catalog function names — the catalog holds + `not ( expr )` and `if ( ... ) then ...` — and the keyword blocklist cannot + tell a real call from an operator use of the same word. So a bare + `not ( [A::x] )` or `if ( ... )` reports **nothing** here, even though it is + a real call. This is deliberate and unfixed: this function's one caller + (`_contains_aggregate_call`) only cares about aggregate names, and neither + `not` nor `if` is one, so the loss costs that caller nothing. A caller with + a different need could not rely on this function to find every real call. + + **Weaker than `split_call`'s own keyword handling.** `split_call` rejects + its *whole* candidate the moment *any* word in it is a keyword, wherever + that word sits, because there its only job is to say whether the entire + expression is one call — being wrong in either direction there is a + correctness bug. This function only strips a *leading* run: a keyword + appearing after a genuine word is not stripped, and the whole multi-word + run — keyword included — is reported as one (bogus) name instead. For + example `flag and sum ( x )` reports the single name `"flag and sum"`, not + `sum` — silently missing the real call. This shape does not arise from + valid ThoughtSpot formula grammar (a bare word cannot precede `and` like + that), which is why it is left as is rather than fixed, but it is not the + guarantee `split_call` makes, and this docstring says so rather than + implying otherwise. + """ + opaque = {i for i, _ch, _d, in_quote in _scan(expression) if in_quote} + for start, end, _body in _bracketed_spans(expression): + opaque.update(range(start, end)) + + names: list[str] = [] + for match in _CALL_HEAD_ANYWHERE.finditer(expression): + if match.start() in opaque: + continue + words = match.group(1).split() + while words and words[0].lower() in _KEYWORDS: + words = words[1:] + if words: + names.append(" ".join(words)) + return names + + +def _bracketed_spans(expression: str) -> list[tuple[int, int, str]]: + """Every `[...]` span that is not inside a quoted literal, as `(start, end, body)`.""" + quoted = {i for i, _ch, _d, in_quote in _scan(expression) if in_quote} + return [ + (m.start(), m.end(), m.group(1)) + for m in _BRACKETED.finditer(expression) + if m.start() not in quoted + ] + + +def find_column_refs(expression: str) -> list[tuple[str, str]]: + """Every `[TABLE::Column]` reference, in order, duplicates kept. + + A bracketed name with no `::` is a runtime parameter, not a column — see + `find_parameter_refs`. Splitting delegates to `identifiers.split_column_ref` rather + than a bare `str.split("::", 1)`, so an ambiguous reference (more than one `::` + delimiter) raises `ValueError` instead of silently taking the first one — consistent + with every other reader of this reference shape, and because silently misreading one + reference in an otherwise-valid expression is worse than failing the whole call. + """ + return [ + identifiers.split_column_ref(f"[{body}]") + for _s, _e, body in _bracketed_spans(expression) + if "::" in body + ] + + +#: The prefix a bracketed name with no `::` carries when it is a formula +#: cross-reference (`[formula_Name]`) rather than a genuine +#: runtime parameter (`[Discount Threshold]`) — the two are the same +#: textual shape (a bracketed name, no table qualifier) and are told apart +#: only by this prefix. Shared here because both conversion directions have +#: to agree on the convention: the Ossie -> TML model builder mints every +#: formula's id with this prefix and rewrites cross-references that carry +#: it, and TML -> Ossie's own parameter finder has to recognise the same +#: prefix or it misclassifies a formula composing another formula as an +#: expression referencing a nonexistent runtime parameter. +FORMULA_REFERENCE_PREFIX = "formula_" + + +def is_formula_reference(body: str) -> bool: + """Whether a bracketed name with no `::` (see `find_parameter_refs` and + `find_formula_refs`) is a formula cross-reference rather than a genuine + runtime parameter.""" + return body.startswith(FORMULA_REFERENCE_PREFIX) + + +def find_parameter_refs(expression: str) -> list[str]: + """Every bracketed name with no table qualifier that is **not** a formula + cross-reference — a genuine ThoughtSpot runtime parameter. + + Ossie has no equivalent, so an expression carrying one is not portable and the caller + raises an issue rather than emitting a portable sibling. A formula cross-reference + (`[formula_Name]`) has the same bracketed, unqualified shape but is a different + construct entirely — see `find_formula_refs` and `is_formula_reference` — and must + not be reported here as a parameter that does not exist. + """ + return [ + body for _s, _e, body in _bracketed_spans(expression) + if "::" not in body and not is_formula_reference(body) + ] + + +def find_formula_refs(expression: str) -> list[str]: + """Every bracketed name with no table qualifier that **is** a formula + cross-reference — the complement of `find_parameter_refs` within the + "no `::`" bracket set. + + A formula composing another formula (`sum ( [formula_Margin] )`) is a + first-class ThoughtSpot construct, not a runtime parameter — see + `FORMULA_REFERENCE_PREFIX`. It is still not portable: a faithful ANSI_SQL + sibling would require inlining the referenced formula's own expression, + which this converter does not attempt. + """ + return [ + body for _s, _e, body in _bracketed_spans(expression) + if "::" not in body and is_formula_reference(body) + ] + + +def is_bare_column_ref(expression: str) -> tuple[str, str] | None: + """`(table, column)` when the whole expression is one column reference, else `None`. + + The common case by a wide margin: most fields are physical columns, and this is what + lets those fields carry a portable sibling for free. + """ + text = expression.strip() + spans = _bracketed_spans(text) + if len(spans) != 1: + return None + start, end, body = spans[0] + if start != 0 or end != len(text) or "::" not in body: + return None + return identifiers.split_column_ref(f"[{body}]") + + +def rewrite_column_refs( + expression: str, rename: Callable[[str, str], str] +) -> str: + """Replace each `[TABLE::Column]` with `rename(table, column)`, byte-preserving elsewhere. + + Everything between references — whitespace, literals, operators — is copied verbatim, so + an expression whose references are unchanged is returned unchanged. Parameter references + and bracketed text inside quoted literals are left alone. + """ + out: list[str] = [] + cursor = 0 + for start, end, body in _bracketed_spans(expression): + if "::" not in body: + continue + table, column = identifiers.split_column_ref(f"[{body}]") + out.append(expression[cursor:start]) + out.append(rename(table, column)) + cursor = end + out.append(expression[cursor:]) + return "".join(out) diff --git a/converters/thoughtspot/src/ossie_thoughtspot/identifiers.py b/converters/thoughtspot/src/ossie_thoughtspot/identifiers.py new file mode 100644 index 00000000..3408afeb --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/identifiers.py @@ -0,0 +1,143 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Identifier derivation and column-reference rewriting. + +ThoughtSpot has one `name` per column, serving as display name, search token and +cross-document key at once. Ossie splits identifier from label, so the +identifier has to be derived — and derivation collides. + +**Known limitation — non-Latin scripts, not diacritics.** An +earlier revision of this module documented ASCII-only folding as a stated +boundary rather than fixing it, on the grounds that a transliteration policy +is a product decision. That reasoning holds for *transliteration* (e.g. +Japanese -> romaji) but not for Unicode canonical *decomposition*, which is +stdlib and needs no policy choice. `normalise` now applies NFKD decomposition +first (`unicodedata.normalize("NFKD", s)`), which separates a base letter from +its combining diacritical marks, then drops non-ASCII before the existing +lowercase-and-substitute folding. A character with no ASCII decomposition +under NFKD (Cyrillic, CJK, and similarly non-Latin scripts) is still dropped, +not transliterated, exactly as before — and a name with no ASCII +alphanumerics surviving still raises `ValueError`. The residual limitation is +narrower than before: `"Café"` -> `"cafe"`, `"Ürün"` -> `"urun"`, and +`"Zürich"` -> `"zurich"` now fold correctly, while a CJK-only name (e.g. +`"北京市"`) still raises. There is also a real open question NFKD does not +settle: some accented Latin folds to a *conventional* ASCII expansion rather +than the bare decomposed letter — German `"Müller"` decomposes to `"Muller"` +here, not the conventional `"Mueller"` — and choosing between them is still a +product decision left to a later change. + +`normalise` itself still raises on a name with no surviving ASCII alphanumerics -- +that has not changed. What changed is who is still allowed to let it propagate. +`tml_to_ossie.py`'s model/field/metric name conversions each catch it and fall +back to a different, still-usable identifier instead (see that module's +`_field_or_metric_identifier` and its model-scope counterpart in `convert`) -- +a display name with no ASCII form is common enough for a non-Latin-script +customer that treating it as fatal dropped their entire model's worth of +fields and metrics, not just one name. A caller with no such fallback of its +own is still expected to let the exception propagate. +""" +import re +import unicodedata + +_NON_ALNUM = re.compile(r"[^0-9a-z]+") +_COLUMN_REF = re.compile(r"^\[(?P[^\]]+?)::(?P[^\]]+)\]$") + + +def normalise(display_name: str) -> str: + """Fold a ThoughtSpot display name to an Ossie identifier. + + Diacritics are folded via NFKD decomposition before the ASCII + lowercase-and-substitute step — see the module docstring's "Known + limitation" note. A character with no ASCII decomposition (non-Latin + scripts) is dropped, not transliterated; a name with no ASCII + alphanumerics surviving still raises. + """ + ascii_form = unicodedata.normalize("NFKD", display_name).encode("ascii", "ignore").decode("ascii") + folded = _NON_ALNUM.sub("_", ascii_form.strip().lower()).strip("_") + if not folded: + raise ValueError(f"{display_name!r} normalises to an empty identifier") + if folded[0].isdigit(): + # A leading digit is not a valid identifier in most consumers' grammars. + folded = f"n_{folded}" + return folded + + +class Allocator: + """Allocates unique identifiers, resolving collisions with a numeric suffix. + + Collision detection folds case, because Ossie resolves regular identifiers + case-insensitively (`core-spec/expression_language.md:77`) even though + `validation/validate.py` only rejects exact-string duplicates. Detecting on + the exact string would emit a document that validates and is still ambiguous. + """ + + def __init__(self) -> None: + self._taken: set[str] = set() + + def allocate(self, display_name: str) -> str: + base = normalise(display_name) + candidate, suffix = base, 1 + while candidate.casefold() in self._taken: + suffix += 1 + candidate = f"{base}_{suffix}" + self._taken.add(candidate.casefold()) + return candidate + + +def split_column_ref(ref: str) -> tuple[str, str]: + """`[TABLE::Column]` -> `("TABLE", "Column")`. + + Raises if `ref` doesn't match the `[TABLE::Column]` shape at all, and also + if it is *ambiguous* — rather than silently taking the first delimiter and + mis-splitting a table or column name that itself contains `::` (e.g. one + produced by `format_column_ref("A::B", "C")`). Two distinct ambiguity + shapes are checked: more than one non-overlapping `::` delimiter in the + whole reference (`str.count` is non-overlapping, which correctly catches + two separated delimiters), and a captured column that itself starts with + `:` — the signature of a *run* of three or more consecutive colons, + which `str.count("::") > 1` cannot see because the run has only one + non-overlapping match. `format_column_ref("ORDERS:", "Col")` produces + `"[ORDERS:::Col]"`, which is exactly as ambiguous as + `format_column_ref("ORDERS", ":Col")` (same string, same encoding + collision) and must not silently mis-split to `("ORDERS", ":Col")`. + Whether the right fix is an escaping scheme or a different delimiter is a + real design question against live ThoughtSpot display names, left to a + later change; loud failure is the correct interim behaviour. + + A table or column name containing a single `:` round-trips correctly — + `split_column_ref(format_column_ref("A:B", "x")) == ("A:B", "x")`. The + table group matches lazily up to the *first* `::`, not a character class + that excludes colons outright; only a `::` occurring inside either part + is the genuinely ambiguous case the checks above catch. + """ + stripped = ref.strip() + match = _COLUMN_REF.match(stripped) + if match is None: + raise ValueError(f"{ref!r} is not a ThoughtSpot column reference") + column = match.group("column") + if stripped.count("::") > 1 or column.startswith(":"): + raise ValueError( + f"{ref!r} is an ambiguous ThoughtSpot column reference: " + "contains more than one '::' delimiter" + ) + return match.group("table"), column + + +def format_column_ref(table: str, column: str) -> str: + """`("TABLE", "Column")` -> `[TABLE::Column]`.""" + return f"[{table}::{column}]" diff --git a/converters/thoughtspot/src/ossie_thoughtspot/issues.py b/converters/thoughtspot/src/ossie_thoughtspot/issues.py new file mode 100644 index 00000000..adb08ab0 --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/issues.py @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Structured, never-silent loss reporting. + +Discussion apache/ossie#325 treats a silently dropped field as a contract +violation rather than a documentation gap, so every declared loss produces an +issue here at conversion time. The same discussion treats a warning storm as a +defect, which is why severity is first-class and why callers can summarise via +count_by_severity() instead of printing every line. +""" +from collections import Counter +from dataclasses import dataclass, field +from enum import Enum + + +class Severity(str, Enum): + INFO = "INFO" + WARNING = "WARNING" + ERROR = "ERROR" + + +@dataclass(frozen=True) +class ConverterIssue: + """One declared loss or degradation, traceable to a specific object. + + object_ref is mandatory: an issue a reader cannot trace to an object cannot + be acted on, which is the complaint #325 raises about warning noise. + """ + + code: str + severity: Severity + message: str + object_ref: str + remedy: str | None = None + + def as_dict(self) -> dict[str, str | None]: + return { + "code": self.code, + "severity": self.severity.value, + "message": self.message, + "object_ref": self.object_ref, + "remedy": self.remedy, + } + + +@dataclass +class IssueLog: + """Ordered collection of issues raised during one conversion.""" + + issues: list[ConverterIssue] = field(default_factory=list) + + def add( + self, + *, + code: str, + severity: Severity, + message: str, + object_ref: str, + remedy: str | None = None, + ) -> None: + self.issues.append( + ConverterIssue( + code=code, + severity=severity, + message=message, + object_ref=object_ref, + remedy=remedy, + ) + ) + + def extend(self, other: "IssueLog") -> None: + self.issues.extend(other.issues) + + def as_dicts(self) -> list[dict[str, str | None]]: + return [i.as_dict() for i in self.issues] + + def has_errors(self) -> bool: + return any(i.severity is Severity.ERROR for i in self.issues) + + def count_by_severity(self) -> dict[str, int]: + return dict(Counter(i.severity.value for i in self.issues)) diff --git a/converters/thoughtspot/src/ossie_thoughtspot/keys.py b/converters/thoughtspot/src/ossie_thoughtspot/keys.py new file mode 100644 index 00000000..2563e1e2 --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/keys.py @@ -0,0 +1,153 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""primary_key / unique_keys derivation. + +TML declares no keys, so every key we emit is manufactured from the +join graph. Upstream PR #330 checks that a relationship's to_columns covers a +declared key, and converters/databricks turns a declared key into a +`rely.at_most_one_match` join hint — so a fabricated key becomes another +vendor's wrong numbers, not just a cosmetic error in ours. + +Orientation is re-checked downstream, so do not rely on ours surviving. +`converters/databricks` (`ossie_to_metric_view.py:446-478`) swaps `from`/`to` +and their column arrays — via `_warn()` (`ossie_to_metric_view.py:53`), not +silently — when the *from* side covers a key and the *to* side does not, +carrying any `custom_extensions` payload onto the reversed relationship. +""" +from dataclasses import dataclass + +from .issues import IssueLog, Severity + +_TO_ONE = frozenset({"MANY_TO_ONE", "ONE_TO_ONE"}) + + +@dataclass(frozen=True) +class Relationship: + """The subset of a relationship that key derivation needs. + + `frozen=True` implies hashability, but `to_columns` may hold a `list` + (unhashable), so `hash(instance)` is not reliably safe here — do not put + a `Relationship` in a set or use it as a dict key without checking the + concrete `to_columns` type first. + """ + + name: str + to_dataset: str + to_columns: tuple[str, ...] | list[str] + cardinality: str + has_residual_predicates: bool + + +def _qualifies(rel: Relationship) -> bool: + """Key evidence requires a to-one join whose condition is wholly + equality, and columns actually present to name as the key.""" + return ( + rel.cardinality in _TO_ONE + and not rel.has_residual_predicates + and bool(rel.to_columns) + ) + + +def derive_keys( + dataset_name: str, relationships: list[Relationship], log: IssueLog +) -> tuple[list[str] | None, list[list[str]]]: + """Return (primary_key, unique_keys) for one dataset. + + primary_key is emitted only when the qualifying relationships agree on a + single column set — where they disagree, choosing one is a guess, so the + candidates go to unique_keys and no primary key is declared. + """ + inbound = [r for r in relationships if r.to_dataset == dataset_name] + qualifying = [r for r in inbound if _qualifies(r)] + + seen: list[list[str]] = [] + for rel in qualifying: + cols = list(rel.to_columns) + if cols not in seen: + seen.append(cols) + + # Explain every disqualified relationship's non-key status. + # + # An empty to_columns is a hard schema failure, not a coverage warning: + # upstream's schema requires to_columns to be a non-empty list + # (minItems: 1), so such a relationship cannot be emitted at all. It is + # reported unconditionally, at ERROR severity, with its own remedy. + # + # For every other disqualification (residual predicates, wrong + # cardinality), the base "not a declared key" statement is our own, + # unconditional explanation of why we did not derive a key from this + # relationship. Upstream's *separate* to_columns coverage check + # (`validation/validate.py:159-165`) only warns when a declared key + # exists for this dataset AND this relationship's columns fail to cover + # any of it — `declared_keys and not any(set(key) <= to_column_set for + # key in declared_keys)`. So predicting that warning is only added when + # that same condition genuinely holds here: a key was derived (`seen`) + # and none of the derived keys is a subset of this relationship's + # to_columns. The canonical SCD-2 residual (as-of) join, whose + # to_columns exactly covers the derived key, passes upstream's check + # clean — predicting a warning for it would be wrong. + for rel in inbound: + if _qualifies(rel): + continue + + if not rel.to_columns: + log.add( + code="TS_KEY_COVERAGE", + severity=Severity.ERROR, + message=( + f"Relationship {rel.name!r} targets {dataset_name!r} with an empty " + f"to_columns. Ossie's schema requires to_columns to be a non-empty " + f"list (minItems: 1), so this relationship cannot be emitted as-is." + ), + object_ref=f"relationship:{rel.name}", + remedy=( + "Not expected. Populate to_columns with the join columns on the " + "'to' dataset, or drop the relationship — an empty to_columns fails " + "Ossie schema validation outright; it is not a coverage warning." + ), + ) + continue + + if rel.has_residual_predicates: + reason = "its condition carries residual (non-equality) predicates" + else: + reason = f"its cardinality is {rel.cardinality}" + + message = ( + f"Relationship {rel.name!r} targets {dataset_name!r} on columns that are " + f"not a declared key, because {reason}." + ) + to_column_set = set(rel.to_columns) + if seen and not any(set(key) <= to_column_set for key in seen): + message += ( + " Ossie validation will report a to_columns coverage warning for it." + ) + + log.add( + code="TS_KEY_COVERAGE", + severity=Severity.WARNING, + message=message, + object_ref=f"relationship:{rel.name}", + remedy=( + "The relationship is genuinely not a key join; declaring a key to " + "silence a coverage warning would assert uniqueness that does not hold." + ), + ) + + primary_key = seen[0] if len(seen) == 1 else None + return primary_key, seen diff --git a/converters/thoughtspot/src/ossie_thoughtspot/ossie_to_thoughtspot.py b/converters/thoughtspot/src/ossie_thoughtspot/ossie_to_thoughtspot.py new file mode 100644 index 00000000..8e9e371e --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/ossie_to_thoughtspot.py @@ -0,0 +1,2070 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Build a ThoughtSpot Table or SQL View TML document from one Ossie dataset. + +An Ossie semantic model becomes 1+N TML documents: one Model document plus one +Table (or SQL View) document per dataset. The Model references each table by +name, so the tables have to exist first — this module builds the "N" half. +Building the Model document itself (formulas, surfaced columns, joins) is a +separate module, because deciding which of a dataset's fields become a +physical column here versus a Model formula there needs the same test either +way: a field whose expression is a single, unqualified column reference is +physical; anything else — a function call, an operator, several references — +is computed and has no physical column to hold it. Only the first kind is +handled here. + +Two things are unrecoverable *from the field's own expression alone*, and +both are handled by falling back to a documented default rather than +guessing, with the fallback always reported: + +* **The warehouse column's own physical name, for a field that came from a + real ThoughtSpot table.** A round-tripped field's bracketed reference + (e.g. ``[ORDERS::Order Date]``) carries the table column's *display* name + only, not its own `db_column_name` — a Model's `column_id` is matched by + display name, never by warehouse name. When the two genuinely differed, + the forward direction now stashes the true warehouse name separately + (`FIELD_STASH_DB_COLUMN_NAME`, Table-backed columns only), and that value + is used whenever present. Only when it is genuinely absent — a + hand-authored bracket, or a document produced before this key existed — + does this fall back to assuming the display name and the warehouse name + agree, which is correct in the common case and is reported as an + assumption otherwise, because a wrong guess here names a column the + warehouse may not have. A hand-authored field instead carries a bare, + unqualified SQL identifier for its own physical column (e.g. + ``order_date``), which genuinely *is* its warehouse name, not a stand-in + for one, so no assumption or issue is needed there. Either way, + ``db_column_name`` is written — always, even when it is identical to the + column's display name, because some ThoughtSpot instances reject an import + that omits it. +* **Whether an Ossie `Time` field's underlying warehouse column is really a + full timestamp.** The datatype map gives `Time` a conditional mapping — + `VARCHAR` normally, `DATE_TIME` when the column is timestamp-backed — but + that condition needs a fact this module has no way to observe. A `Time` + value can only ever reach this function from a hand-authored document in + the first place: the forward direction never emits `Time` at all (nothing + round-trips into it), so there is never a stashed ThoughtSpot column behind + it to inspect, and an Ossie `Field` carries no storage-format signal + besides `datatype` itself. There is nothing here to condition on, so the + unconditional default (`VARCHAR`) is what gets written, and the datatype's + declared loss is still reported so the choice is visible rather than silent. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Callable, Sequence + +from . import datatypes, formula, identifiers, stash +from .constants import ( + DATASET_STASH_ALIAS, + DATASET_STASH_CONNECTION_NAME, + DATASET_STASH_SOURCE_PARTS, + DATASET_STASH_SOURCE_PARTS_DB, + DATASET_STASH_SOURCE_PARTS_DB_TABLE, + DATASET_STASH_SOURCE_PARTS_SCHEMA, + DATASET_STASH_SQL_OUTPUT_COLUMNS, + DATASET_STASH_TABLE_NAME, + DATASET_STASH_TABLE_PROPERTIES, + DATASET_STASH_TML_OBJECT, + DATASET_STASH_TML_OBJECT_WITNESS, + DATASET_STASH_UNSURFACED_COLUMNS, + DIALECT, + FIELD_STASH_COLUMN_PROPERTIES, + FIELD_STASH_DATA_TYPE, + FIELD_STASH_DATA_TYPE_WITNESS, + FIELD_STASH_DB_COLUMN_NAME, + FIELD_STASH_DB_COLUMN_NAME_WITNESS, + METRIC_SHAPE_COLUMN_AGGREGATION, + METRIC_SHAPE_FORMULA, + METRIC_SHAPE_SCALAR_FORMULA_PLUS_AGGREGATION, + METRIC_STASH_SHAPE, + MODEL_STASH_ACTION_OBJECT_ASSOCIATIONS, + MODEL_STASH_COLUMN_GROUPS, + MODEL_STASH_CONSTRAINTS, + MODEL_STASH_FILTERS, + MODEL_STASH_LESSON_PLANS, + MODEL_STASH_MODEL_JOINS_WITH, + MODEL_STASH_MODEL_PROPERTIES, + MODEL_STASH_PARAMETERS, + MODEL_STASH_UNATTRIBUTED_FORMULAS, + MODEL_STASH_UNREPRESENTABLE_JOINS, + PORTABLE_DIALECT, + RELATIONSHIP_STASH_CARDINALITY, + RELATIONSHIP_STASH_ENDPOINTS_SWAPPED, + RELATIONSHIP_STASH_ENDPOINTS_SWAPPED_WITNESS, + RELATIONSHIP_STASH_JOIN_SHAPE, + RELATIONSHIP_STASH_ON_EXPRESSION, + RELATIONSHIP_STASH_ON_EXPRESSION_WITNESS, + RELATIONSHIP_STASH_REFERENCING_JOIN, + RELATIONSHIP_STASH_TYPE, + STASH_TML_NAME, +) +from .errors import ConversionError +from .expressions import CATALOG, Classification, emit_direct, emit_passthrough, emit_unmappable +from .issues import IssueLog, Severity +from .tml import DocumentSet, TmlDocument, block_scalar + +#: A plain ANSI SQL regular identifier (unquoted) or a double-quoted one, per +#: the specification's own identifier grammar — up to 128 characters, and a +#: quoted identifier's content is the literal column name with the quotes +#: stripped. This is what a hand-authored field's own physical-column +#: expression looks like: no dataset qualifier (a field's expression runs +#: against its own dataset's source), no operators, no function calls. +_BARE_IDENTIFIER_RE = re.compile(r'^(?:[A-Za-z_][A-Za-z0-9_]{0,127}|"[^"]{1,128}")$') + +#: Any source string containing whitespace outside of a quoted identifier +#: reads as a query rather than a `db.schema.table` reference — a real +#: three-part identifier never contains one there, and any genuine SQL query +#: does (at minimum a `SELECT` and a target). Whitespace *inside* a quoted +#: identifier (`SALES.PUBLIC."ORDER TABLE"`) is a legitimate table name and +#: must not trip this — see `_split_three_part_identifier`, which is always +#: tried first for exactly that reason. +_WHITESPACE_RE = re.compile(r"\s") + +#: A plain, unquoted ANSI SQL identifier segment. +_PLAIN_IDENTIFIER_SEGMENT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") + + +def _split_three_part_identifier(source: str) -> list[str] | None: + """`source` split on top-level `.` into its parts, or `None` when it does + not parse as a dotted identifier sequence at all. + + Each part is either a plain unquoted identifier or a double-quoted one — + which may itself contain a `.`, whitespace, or any other character + except a literal quote, e.g. `"ORDER TABLE"`. Detecting the three-part + shape this way, before ever asking whether `source` merely *contains* + whitespace, is what keeps a quoted identifier with a space in it + (`SALES.PUBLIC."ORDER TABLE"`) from being misread as a query: the + quoted part's own whitespace is never inspected outside the quotes that + scope it. A genuine query fails this parse almost immediately -- its + first keyword is followed by a space, not a `.` or the end of the + string -- and falls through to the whitespace check instead. + """ + parts: list[str] = [] + i, n = 0, len(source) + if n == 0: + return None + while True: + if i >= n: + return None # a trailing '.' with nothing after it + if source[i] == '"': + end = source.find('"', i + 1) + if end == -1 or end == i + 1: + return None # unterminated or empty quoted identifier + parts.append(source[i + 1:end]) + i = end + 1 + else: + match = _PLAIN_IDENTIFIER_SEGMENT_RE.match(source, i) + if match is None: + return None + parts.append(match.group(0)) + i = match.end() + if i == n: + return parts + if source[i] != ".": + return None + i += 1 + + +def _bare_sql_identifier(expression: str) -> str | None: + """The plain column name `expression` names, or `None` if it is not one + single unqualified identifier.""" + text = expression.strip() + if _BARE_IDENTIFIER_RE.match(text) is None: + return None + if text.startswith('"') and text.endswith('"'): + return text[1:-1] + return text + + +def _physical_identity(field: dict, log: IssueLog, *, object_ref: str) -> tuple[str, str] | None: + """`(display name, warehouse identifier)` for a physical field, or `None` + when the field is computed and has no single physical column to become. + + A THOUGHTSPOT-dialect entry, when present, is authoritative and is + checked first: it is the verbatim expression a prior TML -> Ossie trip + preserved, so a bare `[TABLE::Column]` reference names the table's own + column exactly, and anything else in that dialect is unambiguously a + formula — no other dialect is worth consulting once a THOUGHTSPOT entry + says "computed". Only when there is no THOUGHTSPOT entry at all (a + hand-authored document) does a bare, unqualified SQL identifier in any + other dialect count as a physical reference instead. + """ + dialects = ((field.get("expression") or {}).get("dialects")) or [] + if not dialects: + log.add( + code="TS-FIELD-NO-EXPRESSION", + severity=Severity.WARNING, + message="field has no expression dialects; it cannot become a table column", + object_ref=object_ref, + ) + return None + + ts_entry = next((d for d in dialects if d.get("dialect") == DIALECT), None) + if ts_entry is not None: + ts_expr = ts_entry.get("expression", "") + try: + bare = formula.is_bare_column_ref(ts_expr) + except ValueError as exc: + # split_column_ref raises for two distinct reasons -- the + # bracket's table or column part itself contains "::", making the + # delimiter genuinely ambiguous, or the text inside the brackets + # never matched the [TABLE::Column] shape at all (e.g. an empty + # table part) -- and correctly refuses to guess in either case + # rather than silently mis-splitting. That refusal must not + # propagate as an uncaught exception out of a document + # conversion: it is reported and the field is treated the same + # as any other THOUGHTSPOT expression that is not a single + # column reference (see the `bare is None` case just below). + log.add( + code="TS-FIELD-COLUMN-REF-MALFORMED", + severity=Severity.ERROR, + message=( + f"expression {ts_expr!r} is not a usable ThoughtSpot column " + f"reference ({exc}); it cannot become a table column and is " + f"instead carried into the model as a formula, verbatim, " + f"which will fail to import until it is fixed" + ), + object_ref=object_ref, + ) + return None + if bare is None: + # A THOUGHTSPOT expression that is not a single column reference + # is a formula -- computed fields are the Model document's + # concern, not the table's. + return None + _table, column = bare + # The bracket's own column part is the table's *display* name (what + # a Model column_id must match), not necessarily its warehouse + # db_column_name -- a physical column is matched by display name + # only. When the forward direction saw the two differ, it stashes + # the true warehouse name on the field, witnessed against the + # display name it was recorded for: trustworthy only when the + # field still names the same physical column, since a user + # retargeting the bracket reference to a different column leaves a + # stash that now names the WRONG column's warehouse name -- one + # that would otherwise be silently applied to this one. + field_stash = stash.read_stash(field) + was_stashed = FIELD_STASH_DB_COLUMN_NAME in field_stash + stashed_db_column_name = stash.restore( + field_stash, FIELD_STASH_DB_COLUMN_NAME, None, + witness=column, witness_key=FIELD_STASH_DB_COLUMN_NAME_WITNESS, + ) + if isinstance(stashed_db_column_name, str) and stashed_db_column_name: + return column, stashed_db_column_name + if was_stashed: + log.add( + code="TS-FIELD-DB-COLUMN-NAME-STALE", + severity=Severity.WARNING, + message=( + f"a stashed warehouse column name was recorded for a " + f"different physical column than this field's current " + f"{column!r}; the field was retargeted since the stash " + f"was written, so the stash is dropped and " + f"db_column_name is set equal to the display name " + f"instead, which will name a column the warehouse does " + f"not have if the two differ" + ), + object_ref=object_ref, + ) + return column, column + # No stash to consult -- a hand-authored bracket, or a document + # produced before this key existed. Falling back to the display + # name is correct whenever the two originally agreed (the common + # case), but it is a genuine assumption, not a fact: a wrong guess + # here emits a Table column bound to a warehouse name that may not + # exist, so it is reported rather than made silently. + log.add( + code="TS-FIELD-DB-COLUMN-NAME-ASSUMED", + severity=Severity.WARNING, + message=( + f"no stashed warehouse column name was found for {column!r}; " + f"db_column_name is set equal to the display name, which will " + f"name a column the warehouse does not have if the two " + f"originally differed" + ), + object_ref=object_ref, + ) + return column, column + + display_name = field.get("label") or field.get("name") + for entry in dialects: + identifier = _bare_sql_identifier(entry.get("expression", "")) + if identifier is None: + continue + if not display_name: + log.add( + code="TS-FIELD-NO-NAME", + severity=Severity.WARNING, + message="field has neither a label nor a name; it cannot become a table column", + object_ref=object_ref, + ) + return None + return display_name, identifier + return None + + +def _field_datatype(field: dict, log: IssueLog, *, object_ref: str) -> str: + """The `db_column_properties.data_type` for one physical field. + + A field with no declared `datatype` still gets one: ThoughtSpot treats + the whole `db_column_properties` block as compulsory, so an absent value + is inferred (`datatypes.to_tml(None)`) rather than the key being omitted. + """ + datatype = field.get("datatype") + if datatype is not None: + loss = datatypes.declared_loss(datatype) + if loss is not None: + log.add( + code="TS-FIELD-DATATYPE-DECLARED-LOSS", + severity=Severity.WARNING, + message=f"datatype {datatype!r} does not round-trip exactly: {loss}", + object_ref=object_ref, + ) + + field_stash = stash.read_stash(field) + was_stashed = FIELD_STASH_DATA_TYPE in field_stash + # The exact ThoughtSpot spelling a prior TML -> Ossie trip recorded + # (BOOL vs BOOLEAN, FLOAT vs DOUBLE) wins over a freshly derived one only + # when the witness -- the Ossie datatype it was recorded against -- + # still matches this field's current `datatype`. A field whose declared + # type was edited since (Boolean -> String, say) makes the stashed + # spelling stale: "BOOL" names a warehouse type for the datatype that + # *was* there, not the one that is there now. + stashed_spelling = stash.restore( + field_stash, FIELD_STASH_DATA_TYPE, None, + witness=datatype, witness_key=FIELD_STASH_DATA_TYPE_WITNESS, + ) + if isinstance(stashed_spelling, str) and stashed_spelling: + return stashed_spelling + if was_stashed: + log.add( + code="TS-FIELD-DATA-TYPE-STASH-STALE", + severity=Severity.WARNING, + message=( + f"a warehouse spelling was stashed for a different datatype " + f"than this field's current {datatype!r}; the field was " + f"edited since the stash was written, so the stash is " + f"dropped and the canonical spelling is derived instead" + ), + object_ref=object_ref, + ) + + try: + return datatypes.to_tml(datatype) + except ValueError: + log.add( + code="TS-FIELD-DATATYPE-UNKNOWN", + severity=Severity.WARNING, + message=( + f"datatype {datatype!r} is not a recognised Ossie datatype; " + f"INT64 is inferred instead" + ), + object_ref=object_ref, + ) + return datatypes.to_tml(None) + + +def _field_object_ref(field: dict) -> str: + return f"field:{field.get('label') or field.get('name') or ''}" + + +def _physical_table_column(field: dict, log: IssueLog) -> dict | None: + """One Table `columns[]` entry for `field`, or `None` when it is computed. + + `description` is deliberately never copied here. Every field that + reaches this function is, by construction, also surfaced as a Model + `columns[]` ATTRIBUTE entry (`_build_field`, which writes the same + description there) -- a Table-only physical column never becomes a + `field` at all; it survives verbatim through + `DATASET_STASH_UNSURFACED_COLUMNS` instead. So the Model entry is the + only correct home for a Model-surfaced field's description; writing it + here too would assert something on the Table document its own source + never carried. + """ + object_ref = _field_object_ref(field) + identity = _physical_identity(field, log, object_ref=object_ref) + if identity is None: + return None + name, db_column_name = identity + return { + "name": name, + # Always present, even equal to `name` -- some ThoughtSpot instances + # reject an import that omits it. + "db_column_name": db_column_name, + "db_column_properties": {"data_type": _field_datatype(field, log, object_ref=object_ref)}, + } + + +def _physical_sql_view_column(field: dict, output_aliases: dict, log: IssueLog) -> dict | None: + """One SQL View `sql_view_columns[]` entry for `field`, or `None` when it + is computed. + + `output_aliases` is the dataset's stashed `field name -> sql_output_column` + map. It wins when present, because a query output alias is not something + the field's own expression can be relied on to reconstruct; the bare + identifier `_physical_identity` finds is only a fallback for a + hand-authored field with no such record. + """ + object_ref = _field_object_ref(field) + identity = _physical_identity(field, log, object_ref=object_ref) + if identity is None: + return None + name, fallback_identifier = identity + sql_output_column = output_aliases.get(field.get("name")) or fallback_identifier + # `description` is never copied here -- see the matching note on + # _physical_table_column just above; the same reasoning applies + # unchanged to a SQL View's own physical column. + return { + "name": name, + "sql_output_column": sql_output_column, + "db_column_properties": {"data_type": _field_datatype(field, log, object_ref=object_ref)}, + } + + +def _derive_kind(source: str) -> tuple[str, bool]: + """`(kind, malformed)` guessed from `source` alone. + + A genuine three-part dotted identifier — quoted parts included, so a + quoted identifier's own internal whitespace is never mistaken for a + query — reads as a table reference. Failing that, whitespace anywhere + else in `source` reads as a query: a real `db.schema.table` reference + never contains any outside a quoted part, and a real SQL query always + does. Anything else is neither shape clearly enough to guess, so it is + reported malformed and a table is still produced -- `_source_parts` is + what actually raises the issue for it, so the same root cause is never + reported twice. + """ + parts = _split_three_part_identifier(source) + if parts is not None and len(parts) == 3 and all(parts): + return "table", False + if _WHITESPACE_RE.search(source): + return "sql_view", False + return "table", True + + +def _decide_kind(dataset: dict, payload: dict, log: IssueLog, *, object_ref: str) -> str: + """Whether `dataset` becomes a `table:` or `sql_view:` document. + + A stashed `tml_object` (written whenever this dataset came from a prior + TML -> Ossie trip) is authoritative -- it also determines which shape + `unsurfaced_columns` was captured in, so trusting it keeps that list + valid -- but only when its witness (the `source` it was stashed + alongside) still matches this dataset's CURRENT `source`. A user who + rewrites `source` from a table reference to a query (or back) since the + stash was written leaves a `tml_object` that now describes the wrong + shape; using it anyway would misread `source` under the old rules (a + query parsed as db/schema/table, or vice versa). A hand-authored dataset + has no stash at all, and falls through to `_derive_kind` either way. + """ + source = dataset.get("source") or "" + was_stashed = DATASET_STASH_TML_OBJECT in payload + stashed_kind = stash.restore( + payload, DATASET_STASH_TML_OBJECT, None, + witness=source, witness_key=DATASET_STASH_TML_OBJECT_WITNESS, + ) + if stashed_kind in ("table", "sql_view"): + return stashed_kind + if was_stashed: + log.add( + code="TS-DATASET-TML-OBJECT-STALE", + severity=Severity.WARNING, + message=( + "a stashed document kind (table/sql_view) no longer matches " + "this dataset's current source; the source was rewritten " + "since the stash was written, so the stash is dropped and " + "the kind is re-derived from the current source instead" + ), + object_ref=object_ref, + ) + kind, _malformed = _derive_kind(source) + return kind + + +def _source_parts(dataset: dict, payload: dict, log: IssueLog, *, object_ref: str) -> tuple[str, str, str]: + """`(db, schema, db_table)` for a Table document. + + A stashed `source_parts` entry is used only when it still reconstructs + the dataset's current `source` exactly -- the dataset may have been + hand-edited since the stash was written, and a plain split of the live + `source` is the correct behaviour once that has happened, not a stale + three-way split nobody asked for any more. + """ + source = dataset.get("source") or "" + stashed = payload.get(DATASET_STASH_SOURCE_PARTS) + if isinstance(stashed, dict): + db, schema, db_table = ( + stashed.get(DATASET_STASH_SOURCE_PARTS_DB, ""), + stashed.get(DATASET_STASH_SOURCE_PARTS_SCHEMA, ""), + stashed.get(DATASET_STASH_SOURCE_PARTS_DB_TABLE, ""), + ) + if ".".join((db, schema, db_table)) == source: + return db, schema, db_table + log.add( + code="TS-DATASET-SOURCE-PARTS-STALE", + severity=Severity.WARNING, + message=( + "the stashed source_parts no longer reconstruct this dataset's " + "current source; the source is re-split instead" + ), + object_ref=object_ref, + ) + + parts = _split_three_part_identifier(source) + if parts is not None and len(parts) == 3 and all(parts): + return parts[0], parts[1], parts[2] + + log.add( + code="TS-DATASET-SOURCE-MALFORMED", + severity=Severity.WARNING, + message=( + f"source {source!r} does not split into three non-empty db/schema/table " + f"parts; it is kept verbatim as db_table with db and schema left blank" + ), + object_ref=object_ref, + ) + return "", "", source + + +def _connection_name( + payload: dict, connection_name: str | None, log: IssueLog, *, object_ref: str +) -> str | None: + name = payload.get(DATASET_STASH_CONNECTION_NAME) or connection_name + if name: + return name + log.add( + code="TS-DATASET-CONNECTION-MISSING", + severity=Severity.WARNING, + message=( + "no connection name is available for this table -- none was stashed " + "and none was supplied by the caller; the connection is omitted from " + "the document and the import will fail until one is added" + ), + object_ref=object_ref, + remedy="Set the Table document's connection.name to a valid Connection display name before import.", + ) + return None + + +def _table_name(dataset: dict, payload: dict) -> str: + return ( + payload.get(STASH_TML_NAME) + or payload.get(DATASET_STASH_TABLE_NAME) + or dataset.get("name") + or "" + ) + + +def _shared_body(dataset: dict, payload: dict, connection: str | None) -> dict: + body: dict = {} + if connection: + body["connection"] = {"name": connection} + description = dataset.get("description") + if description: + body["description"] = description + table_properties = payload.get(DATASET_STASH_TABLE_PROPERTIES) + if table_properties: + body["properties"] = table_properties + return body + + +def _unsurfaced_columns_still_unsurfaced( + unsurfaced: list[dict] | None, live_column_names: set[str] +) -> list[dict]: + """`unsurfaced` (the verbatim DATASET_STASH_UNSURFACED_COLUMNS entries), + with any entry now covered by a live field dropped. + + DATASET_STASH_UNSURFACED_COLUMNS is INFORMATION_ONLY in its per-entry + *content* -- a physical column's own db_column_name/data_type has no + Ossie counterpart to check it against -- but its *membership* is a + different question with a different answer: whether a given entry is + still unsurfaced is exactly the complement of what the live document's + fields now cover, and that complement can change. A field added (or + retargeted onto) a column that was unsurfaced when the stash was + written makes that column surfaced now; blindly re-appending it here + would emit it a second time under the field-derived entry's own name -- + a duplicate Table/SQL-View column name, which does not import. Filtering + here is silent by design: nothing was lost (the column is still present, + once, under the live field's own build), so there is nothing to name in + an issue -- see STASH_KEYS_WITH_DERIVABLE_MEMBERSHIP for why this is a + distinct question from the value-classification table above it. + """ + if not unsurfaced: + return [] + return [c for c in unsurfaced if c.get("name") not in live_column_names] + + +def _build_table_body( + dataset: dict, payload: dict, connection: str | None, log: IssueLog, *, object_ref: str +) -> dict: + db, schema, db_table = _source_parts(dataset, payload, log, object_ref=object_ref) + body: dict = {"name": _table_name(dataset, payload), "db": db, "schema": schema, "db_table": db_table} + body.update(_shared_body(dataset, payload, connection)) + + columns: list[dict] = [] + for field in dataset.get("fields") or []: + column = _physical_table_column(field, log) + if column is not None: + columns.append(column) + unsurfaced = payload.get(DATASET_STASH_UNSURFACED_COLUMNS) + columns.extend( + _unsurfaced_columns_still_unsurfaced(unsurfaced, {c["name"] for c in columns}) + ) + body["columns"] = columns + return body + + +def _build_sql_view_body( + dataset: dict, payload: dict, connection: str | None, log: IssueLog, *, object_ref: str +) -> dict: + body: dict = {"name": _table_name(dataset, payload), "sql_query": dataset.get("source") or ""} + body.update(_shared_body(dataset, payload, connection)) + + output_aliases = payload.get(DATASET_STASH_SQL_OUTPUT_COLUMNS) or {} + columns: list[dict] = [] + for field in dataset.get("fields") or []: + column = _physical_sql_view_column(field, output_aliases, log) + if column is not None: + columns.append(column) + unsurfaced = payload.get(DATASET_STASH_UNSURFACED_COLUMNS) + columns.extend( + _unsurfaced_columns_still_unsurfaced(unsurfaced, {c["name"] for c in columns}) + ) + body["sql_view_columns"] = columns + return body + + +def build_table(dataset: dict, log: IssueLog, *, connection_name: str | None = None) -> TmlDocument: + """One Ossie dataset -> one ThoughtSpot `table:`/`sql_view:` TML document. + + `connection_name` is the fallback used when the dataset carries no + stashed `connection_name` of its own -- Ossie has no connection concept, + so a hand-authored dataset has nowhere else to record which warehouse + Connection the table belongs to. When neither is available the + connection is omitted and an issue names the gap, rather than a + connection name being invented. + + A `source` that is a query becomes a `sql_view:` document, its columns + under `sql_view_columns[]`; anything that at least looks like a + `db.schema.table` reference becomes a `table:` document. Only a field + whose own expression is a single physical column reference becomes a + column here -- a computed field has no single warehouse column to name, + and is left for the Model document to turn into a formula instead. + """ + name = dataset.get("name") or "" + object_ref = f"dataset:{name}" + payload = stash.read_stash(dataset) + connection = _connection_name(payload, connection_name, log, object_ref=object_ref) + + if dataset.get("ai_context"): + # Neither a Table nor a SQL View document has any synonym or + # instruction field at all -- there is nowhere in TML for this to + # go, in either direction, so the loss is unconditional rather than + # a fallback that might be avoided with more information. + log.add( + code="TS-DATASET-AI-CONTEXT-UNSUPPORTED", + severity=Severity.WARNING, + message=( + "dataset ai_context has no home in a Table or SQL View " + "document; it is not carried into the table" + ), + object_ref=object_ref, + ) + + if _decide_kind(dataset, payload, log, object_ref=object_ref) == "sql_view": + body = _build_sql_view_body(dataset, payload, connection, log, object_ref=object_ref) + return TmlDocument(kind="sql_view", body=body, guid=None) + + body = _build_table_body(dataset, payload, connection, log, object_ref=object_ref) + return TmlDocument(kind="table", body=body, guid=None) + + +# --------------------------------------------------------------------------- +# build_model: the Model TML document. +# +# Everything below builds `model:` from one Ossie `semantic_model` entry plus +# the Table/SQL-View documents `build_table` already produced for its +# datasets. Order of business: name/description/ai_context, then a resolver +# any computed field or metric's portable (ANSI_SQL) expression needs +# (`resolve_field`, built once from every dataset's physical fields), then +# fields and metrics (which allocate model-wide unique display names), +# then unattributed formulas, then relationships/unrepresentable +# joins folded into each dataset's inline `joins[]`, then model-scope stash. +# --------------------------------------------------------------------------- + + +class _DisplayNameAllocator: + """Assigns unique TML display names across `columns[]` and `formulas[]` + combined, preserving each candidate's own text exactly whenever + it is not colliding with one already assigned. + + `identifiers.Allocator` is not reused directly here: it folds every + candidate to a normalised (lowercase, underscore-joined) identifier even + on its very first use, which is correct for an *Ossie* identifier + (TML -> Ossie's own `field.name`) but wrong for a TML display name -- + `Ossie -> TML` must use a field's `label` (or a metric's own + `name`, when there is no `label`) verbatim in the ordinary, non-colliding + case. This class reuses `identifiers.normalise` as the fold key -- the + exact case/punctuation-insensitive comparison Ossie identifier resolution + requires, and the same + one `identifiers.Allocator` computes internally -- and appends a numeric + suffix to the *original* text, never the folded one, only once a + collision is actually found. + """ + + def __init__(self) -> None: + self._taken: set[str] = set() + + def allocate(self, display_name: str, log: IssueLog, *, object_ref: str) -> str: + try: + fold_base = identifiers.normalise(display_name) + except ValueError: + # A name with no ASCII alphanumerics at all -- normalise() raises + # rather than returning one. Falls back to a plain casefold so + # this allocator still has *some* fold key to dedupe against, + # rather than propagating the exception into a model build. + fold_base = display_name.strip().casefold() or "field" + fold, candidate, suffix = fold_base, display_name, 1 + while fold in self._taken: + suffix += 1 + candidate = f"{display_name}_{suffix}" + fold = f"{fold_base}_{suffix}" + self._taken.add(fold) + if candidate != display_name: + # The rename is correct -- uniqueness is required -- but + # it changes text the user chose and will see in the product, and + # silence here is exactly the kind of quiet difference this + # package otherwise always reports. + log.add( + code="TS-MODEL-DISPLAY-NAME-COLLISION", + severity=Severity.WARNING, + message=( + f"display name {display_name!r} collides with one already " + f"assigned in this model; it is emitted as {candidate!r} " + f"instead to satisfy ThoughtSpot's uniqueness requirement" + ), + object_ref=object_ref, + ) + return candidate + + +def _normalise_or_self(text: str) -> str: + """`identifiers.normalise(text)`, or `text` itself when it has no ASCII + alphanumerics for `normalise` to fold onto -- the same fallback + `_DisplayNameAllocator.allocate` and `_formula_id_from` already use, so + all three agree on what "the fold key" is for a piece of text with no + normal form.""" + try: + return identifiers.normalise(text) + except ValueError: + return text + + +def _restore_tml_name( + payload: dict, live_identifier: str, log: IssueLog, *, object_ref: str +) -> str: + """The witness check for STASH_TML_NAME (metric and model scope): the exact ThoughtSpot + display name a prior TML -> Ossie trip stashed when identifier normalisation + changed it, restored only when it is still current. + + Self-verifying rather than a separately stored witness (the same shape + `_source_parts` already uses for DATASET_STASH_SOURCE_PARTS): the + stashed name's own normalised form IS the check, since that is exactly + the fold the forward direction applied to produce `live_identifier` in + the first place. If they still agree, nobody has renamed the Ossie + identifier since the stash was written, and the exact display name is + restored; if they disagree, the identifier was renamed and the stash + describes a name that no longer belongs to this object, so it is + dropped and the live identifier is used instead. + """ + stashed = payload.get(STASH_TML_NAME) + if not isinstance(stashed, str) or not stashed: + return live_identifier + if _normalise_or_self(stashed) == live_identifier: + return stashed + log.add( + code="TS-STASH-TML-NAME-STALE", + severity=Severity.WARNING, + message=( + f"a stashed display name {stashed!r} no longer matches this " + f"object's current identifier {live_identifier!r}; it was " + f"renamed since the stash was written, so the stashed name is " + f"dropped and the current identifier is used instead" + ), + object_ref=object_ref, + ) + return live_identifier + + +def _formula_id_from(display_name: str) -> str: + """`formulas[].id` for a formula surfaced under `display_name`. + + Real ThoughtSpot display names carry spaces and mixed case + (``"Net Amount"``); ids do not (``formula_net_amount``). Deriving the id + from the *normalised* form of the display name -- the same fold + `_DisplayNameAllocator` already dedupes on -- rather than embedding the + display name verbatim is what lets a THOUGHTSPOT-verbatim cross-reference + elsewhere in the model (`[formula_net_amount]`) resolve + against a formula this converter itself is generating: the reference was + written against ThoughtSpot's own slug-shaped id convention, and a + verbatim, unnormalised id (``formula_Net Amount``) would silently break + it while still importing (a stray space in an id is otherwise legal). + Calls `_normalise_or_self` rather than repeating its try/except, so the + id-minting side and `_rewrite_formula_references`'s reference-matching + side cannot independently drift onto two different fold rules. + """ + return f"{formula.FORMULA_REFERENCE_PREFIX}{_normalise_or_self(display_name)}" + + +#: TML aggregation enum value -> the catalog `spec_name` whose DIRECT template +#: is ThoughtSpot's own native rendering of it. Mirrors tml_to_ossie.py's own +#: `_AGGREGATION_CATALOG_SPEC` (kept local rather than imported across modules +#: for a private name) -- both derive `_CALL_NAME_TO_AGGREGATION` below from +#: the same catalog rows, so "what native call names an aggregate" cannot +#: silently drift between the read and write directions. +_METRIC_AGGREGATION_CATALOG_SPEC = { + "SUM": "SUM(expr)", "COUNT": "COUNT(expr)", "AVERAGE": "AVG(expr)", + "MIN": "MIN(expr)", "MAX": "MAX(expr)", "COUNT_DISTINCT": "COUNT(DISTINCT expr)", + "STD_DEVIATION": "STDDEV(expr)", "VARIANCE": "VARIANCE(expr)", +} + +#: The inverse: ThoughtSpot's own native aggregate call name (as rendered by +#: `emit_direct`) -> the TML `aggregation` enum value it corresponds to. +#: Derived, not hand-typed, for the same reason tml_to_ossie.py derives +#: `_AGGREGATE_CALL_NAMES` from the catalog rather than listing native names +#: by hand. +_CALL_NAME_TO_AGGREGATION: dict[str, str] = { + formula.split_call(emit_direct(CATALOG[_spec], ["x"]))[0].lower(): _agg + for _agg, _spec in _METRIC_AGGREGATION_CATALOG_SPEC.items() +} + + +def _outer_aggregation_of(ts_expr: str) -> str | None: + """The TML `aggregation` enum value matching `ts_expr`'s own outer call, + or `None` when there is no outer call or it is not a recognised native + aggregate. + + Used two ways: to decompose a `scalar_formula_plus_aggregation`-shaped + metric's composed expression back into its scalar inner expression plus + the aggregation that wraps it, and — for every other shape — to set the + surfacing column's `aggregation` as the documented convention the worked + shape shows (inert at query time when the formula's own expr already + aggregates, but present on real ThoughtSpot-authored documents). + """ + call = formula.split_call(ts_expr) + if call is None: + return None + name, args = call + if len(args) != 1: + return None + return _CALL_NAME_TO_AGGREGATION.get(name.lower()) + + +def _decompose_scalar_aggregate(ts_expr: str) -> tuple[str, str] | None: + """`(aggregation, inner scalar expr)` for a composed aggregate call, or + `None` when `ts_expr`'s outer call is not a recognised native aggregate + over a single argument. + + The scalar-formula-plus-aggregation pattern (`scalar_formula_plus_aggregation`): the Ossie metric's + THOUGHTSPOT-dialect entry already holds the *composed* text (e.g. + ``average ( [A::x] - [A::y] )``, built by tml_to_ossie's own + `_compose_aggregate_entries`) — this is the inverse, recovering the bare + scalar `[A::x] - [A::y]` and the `AVERAGE` that wraps it. + """ + call = formula.split_call(ts_expr) + if call is None: + return None + name, args = call + if len(args) != 1: + return None + aggregation = _CALL_NAME_TO_AGGREGATION.get(name.lower()) + if aggregation is None: + return None + return aggregation, args[0] + + +def _maybe_block_scalar(expr: str) -> str: + """Wrap `expr` for `>-` emission whenever it contains a brace, + otherwise return it untouched.""" + if "{" in expr or "}" in expr: + return block_scalar(expr) + return expr + + +def _rewrite_formula_references( + expr: str, + formula_id_by_normalised_name: dict[str, str], + log: IssueLog, + *, + object_ref: str, +) -> str: + """Rewrite every bare `[formula_X]` cross-reference in `expr` to the id + this build actually assigned the referenced formula. + + `_formula_id_from` regenerates every formula's id from the *normalised* + form of its own display name -- real ThoughtSpot ids are slug-shaped, + display names are not. A cross-reference embedded in a verbatim + THOUGHTSPOT-dialect expression was written against the *source* + document's own id text, which need not match the id this build just + minted for the same formula (the source id could use different casing, + punctuation, or spacing than this converter's own convention) -- and + ThoughtSpot does not fail an unresolvable bracket reference at parse + time, it parses it as search tokens instead, so a stale reference is a + guaranteed import failure discovered only later, not a warning. + + `formula_id_by_normalised_name` must be keyed by `_normalise_or_self` + applied to each formula's own final display name -- the exact same fold + `_formula_id_from` uses to mint the id in the first place, passed in by + the caller rather than recomputed here, so the two can never + independently drift the way two separately-typed normalisation steps + could (this package just finished centralising stash-key spellings for + the identical reason). + + A reference matching nothing being built in this model is left in the + text untouched -- there is nothing safe to substitute -- and logged as + an ERROR: the emitted document will fail to import on this reference + until it is fixed, and that has to be visible, not silently shipped. + """ + out: list[str] = [] + cursor = 0 + for start, end, body in formula._bracketed_spans(expr): + if "::" in body or not formula.is_formula_reference(body): + continue + referenced_name = body[len(formula.FORMULA_REFERENCE_PREFIX):] + target_id = formula_id_by_normalised_name.get(_normalise_or_self(referenced_name)) + out.append(expr[cursor:start]) + if target_id is None: + log.add( + code="TS-MODEL-FORMULA-REFERENCE-UNRESOLVED", + severity=Severity.ERROR, + message=( + f"expression references {body!r}, which does not match any " + f"formula this model emits; the reference is left as written " + f"and the resulting document will fail to import (ThoughtSpot " + f"parses an unresolvable bracket reference as search tokens, " + f"not a parse error) until it is fixed" + ), + object_ref=object_ref, + ) + out.append(expr[start:end]) + else: + out.append(f"[{target_id}]") + cursor = end + out.append(expr[cursor:]) + return "".join(out) + + +#: A bare `dataset.field` reference, per the specification's own dot-notation +#: convention (`core-spec/expression_language.md:98`) -- the shape a +#: hand-authored ANSI_SQL expression uses to name another Ossie field, e.g. +#: `orders.amount`. Distinct from the warehouse-qualified dot form +#: tml_to_ossie.py's own `resolve()` closure builds for a *round-tripped* +#: document's portable sibling (`TABLE.db_column_name`) -- that form is never +#: read back here: a round-tripped Ossie document always carries a THOUGHTSPOT +#: entry too, which `to_thoughtspot_expression` prefers unconditionally, so +#: this pattern is only ever exercised for a document with no such entry. +_ANSI_DATASET_FIELD_RE = re.compile(r"^\s*([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*)\s*$") + + +def _match_ansi_call(name: str, args: list[str]) -> tuple[str, list[str]] | None: + """The CATALOG key and (possibly rewritten) argument list matching a + single ANSI_SQL call `name(args)`, or `None` when nothing in the catalog + matches this call structurally. + + Deliberately narrow: only the single-argument aggregate family + (`SUM(expr)`, `COUNT(expr)`, ..., and the `COUNT(DISTINCT expr)` special + case) is matched. This is the shape a metric's portable expression + realistically takes (the scalar-formula-plus-aggregation pattern's own + composed shape), and the catalog's other + families spell their placeholder differently per row (`ABS(x)`, + `LOWER(str)`, ...) — matching those too would need a full per-row arity + index this module does not build, so anything else falls through to "no + catalog construct matches structurally" rather than a guess. + """ + upper = name.upper() + if upper == "COUNT" and len(args) == 1 and args[0].strip().upper().startswith("DISTINCT "): + inner = args[0].strip()[len("DISTINCT "):].strip() + if "COUNT(DISTINCT expr)" in CATALOG: + return "COUNT(DISTINCT expr)", [inner] + return None + if len(args) == 1: + key = f"{upper}(expr)" + if key in CATALOG: + return key, args + return None + + +def _translate_ansi_sql( + expr: str, + resolve_field: Callable[[str], tuple[str, str] | None], + log: IssueLog, + *, + object_ref: str, +) -> str | None: + """One ANSI_SQL expression -> a ThoughtSpot formula string, or `None`. + + Handles exactly two structural shapes, recursively: a bare + `dataset.field` reference (rewritten via `resolve_field`), and a single + catalog-matched function call wrapping arguments of either shape. Anything + else raises an issue and returns `None` — the caller stashes rather than + this function guessing a rendering. Never re-renders one SQL dialect into + another: a construct the catalog does not structurally match is left + alone, not approximated. + """ + match = _ANSI_DATASET_FIELD_RE.match(expr) + if match is not None: + key = f"{match.group(1)}.{match.group(2)}" + resolved = resolve_field(key) + if resolved is None: + log.add( + code="TS-EXPR-ANSI-UNRESOLVED", + severity=Severity.WARNING, + message=( + f"reference {key!r} does not resolve to a known field in this " + f"model; no ThoughtSpot expression is produced for it" + ), + object_ref=object_ref, + ) + return None + table, column = resolved + return identifiers.format_column_ref(table, column) + + call = formula.split_call(expr) + if call is None: + log.add( + code="TS-EXPR-ANSI-UNSTRUCTURED", + severity=Severity.WARNING, + message=( + f"ANSI_SQL expression {expr!r} is neither a bare dataset.field " + f"reference nor a single function call this converter's catalog " + f"matches structurally; it is not re-rendered rather than guessed" + ), + object_ref=object_ref, + ) + return None + + name, args = call + matched = _match_ansi_call(name, args) + if matched is None: + log.add( + code="TS-EXPR-ANSI-UNMATCHED", + severity=Severity.WARNING, + message=( + f"{name}(...) in {expr!r} has no catalog construct this converter " + f"matches structurally; it is not re-rendered rather than guessed" + ), + object_ref=object_ref, + ) + return None + + spec_key, inner_args = matched + construct = CATALOG[spec_key] + translated: list[str] = [] + for arg in inner_args: + piece = _translate_ansi_sql(arg, resolve_field, log, object_ref=object_ref) + if piece is None: + return None + translated.append(piece) + + if construct.classification is Classification.DIRECT: + return emit_direct(construct, translated) + if construct.classification is Classification.PASSTHROUGH: + return emit_passthrough(construct, translated, log, object_ref=object_ref) + emit_unmappable(construct, log, object_ref=object_ref) + return None + + +def to_thoughtspot_expression( + entries: Sequence[dict], + resolve_field: Callable[[str], tuple[str, str] | None], + log: IssueLog, + *, + object_ref: str, +) -> str | None: + """One Ossie `expression.dialects[]` list -> a ThoughtSpot formula string, + or `None`. + + Mirrors the reference converters' own `pick_expression`, and the same + dialect-selection order `tml_to_ossie.py`'s own `expression_entries` uses + in reverse: the THOUGHTSPOT entry, when present, is + authoritative and is returned **verbatim** — it is the exact `expr` text a + prior `TML -> Ossie` trip preserved untouched (tml_to_ossie.py's own + `expression_entries`), and every reference inside it already names this + document's own table/alias (a dataset's Ossie `name` is the + `model_tables[]` name-or-alias verbatim, so it round-trips unchanged) and + this document's own physical column display names (a Table document's + column `name` is copied from that same bracket text by `build_table`). + So nothing inside it needs rewriting for a document this converter + produced to return exactly, and none is attempted — `resolve_field` is + simply unused on this path. + + Only when there is no THOUGHTSPOT entry at all — a hand-authored document, + or the worked-shape example in the construct-mapping document, both of + which carry only an ANSI_SQL sibling — does this fall through to + `_translate_ansi_sql`, which structurally matches a bare `dataset.field` + reference or a single catalog-recognised function call and rewrites via + `resolve_field`. Anything else raises an issue and returns `None` (the + caller stashes rather than guessing); one dialect is never re-rendered + into another. + """ + by_dialect = {e.get("dialect"): e.get("expression") for e in entries if isinstance(e, dict)} + + ts_expr = by_dialect.get(DIALECT) + if isinstance(ts_expr, str) and ts_expr: + return ts_expr + + ansi_expr = by_dialect.get(PORTABLE_DIALECT) + if not isinstance(ansi_expr, str) or not ansi_expr: + log.add( + code="TS-EXPR-NO-USABLE-DIALECT", + severity=Severity.ERROR, + message=( + "expression carries no THOUGHTSPOT entry and no ANSI_SQL entry this " + "converter can translate; no ThoughtSpot expression can be produced for it" + ), + object_ref=object_ref, + ) + return None + + return _translate_ansi_sql(ansi_expr, resolve_field, log, object_ref=object_ref) + + +def _field_physical_display_name(field: dict) -> str | None: + """The physical Table column's own display name `field` maps to, or + `None` when `field` is computed. + + Mirrors `_physical_identity`'s own classification (THOUGHTSPOT-entry + priority, else any dialect's bare SQL identifier) without its logging or + its `db_column_name` lookup: this module's job here is only to classify + physical-vs-computed and to name the display column, and `build_table` + (called separately, on the same field, from the same log) already reports + any db_column_name assumption -- calling `_physical_identity` again here + would double-report the same finding under a second `object_ref`. + + An ambiguous bracket (the table or column part itself contains "::") is + treated the same as "not a bare column reference" rather than left to + raise: `_physical_identity`, called on this same field from `build_table` + before this function ever runs, already reports the ambiguity once -- + reporting it again here would be the same double report this function's + own docstring already rules out for db_column_name. + """ + dialects = ((field.get("expression") or {}).get("dialects")) or [] + ts_entry = next((d for d in dialects if d.get("dialect") == DIALECT), None) + if ts_entry is not None: + try: + bare = formula.is_bare_column_ref(ts_entry.get("expression", "")) + except ValueError: + return None + return bare[1] if bare is not None else None + + display_name = field.get("label") or field.get("name") + for entry in dialects: + if _bare_sql_identifier(entry.get("expression", "")) is not None: + return display_name + return None + + +def _physical_columns_of(table_doc: TmlDocument | None) -> list[dict]: + if table_doc is None: + return [] + key = "sql_view_columns" if table_doc.kind == "sql_view" else "columns" + return table_doc.body.get(key) or [] + + +def _restore_ai_context(properties: dict, ai_context: object, log: IssueLog, *, object_ref: str) -> None: + """Fold an Ossie `ai_context` value (string or `{synonyms, instructions, + examples}`) into `properties`, mutating it in place (`synonyms` and + `synonym_type` live under `properties`, never at the column root). + + `examples` has no TML equivalent and raises an issue rather than + being dropped silently. + """ + if ai_context is None: + return + if isinstance(ai_context, str): + if ai_context: + properties["ai_context"] = ai_context + return + if not isinstance(ai_context, dict): + return + + synonyms = ai_context.get("synonyms") + if synonyms: + properties["synonyms"] = list(synonyms) + properties["synonym_type"] = "USER_DEFINED" + instructions = ai_context.get("instructions") + if instructions: + properties["ai_context"] = instructions + if ai_context.get("examples"): + log.add( + code="TS-AI-CONTEXT-EXAMPLES-UNSUPPORTED", + severity=Severity.WARNING, + message=( + "ai_context.examples has no ThoughtSpot TML equivalent; it is " + "not carried into the model" + ), + object_ref=object_ref, + ) + + +#: Properties this converter must never write as `true` into a +#: generated model, even when the stash carries the value verbatim. The +#: stash is the Ossie document's own record of what the source TML held and +#: is untouched by this filter (a forward conversion must still be able to +#: recover the flag); only the *emitted* TML side ever drops it. A message +#: per key, not one generic message, because the reasoning differs for +#: each: a hidden column cannot be surfaced again without a manual edit on +#: the target instance, and re-asserting was_auto_generated on a column this +#: build did not itself generate would misrepresent its provenance. +_NEVER_EMIT_TRUE_PROPERTY_MESSAGES = { + "is_hidden": ( + "the source column had is_hidden=true, but a generated model must never " + "set it -- a hidden column cannot be surfaced again without a manual edit " + "on the target instance, so silently regenerating one would lock it there " + "again; it is dropped from the emitted column rather than written" + ), + "was_auto_generated": ( + "the source column had was_auto_generated=true, but this build did not " + "auto-generate the regenerated column -- re-asserting the flag would " + "misrepresent its provenance; it is dropped from the emitted column " + "rather than written" + ), +} + + +def _drop_never_emit_true_properties( + extra_properties: dict, log: IssueLog, *, object_ref: str +) -> dict: + """`extra_properties` (a restored `column_properties` stash) with + `is_hidden`/`was_auto_generated` removed before it is merged into the + emitted `properties` dict. + + Only a `true` value is dropped-and-logged: it is the one value this + converter forbids the *generated* TML from carrying, and a generated model + silently losing a column's visibility (or misreporting its provenance) + is a real, actionable difference the model owner needs to see, not a + stylistic omission -- hence WARNING, matching this module's other + declared-loss codes (TS-MODEL-FIELD-DATATYPE-UNWRITABLE, + TS-MODEL-DATASET-KEY-UNUSED), rather than the INFO severity reserved for + a benign structural note. A stashed `false` is simply omitted, logging + nothing: `false` (or absent) is ThoughtSpot's own default for both + properties, so leaving the key out of the emitted document loses no + information at all. + """ + filtered = dict(extra_properties) + for key, message in _NEVER_EMIT_TRUE_PROPERTY_MESSAGES.items(): + if key not in filtered: + continue + value = filtered.pop(key) + if value is True: + log.add( + code="TS-MODEL-PROPERTY-NEVER-EMITTED", + severity=Severity.WARNING, + message=message, + object_ref=object_ref, + ) + return filtered + + +def _build_field( + field: dict, + dataset_prefix: str, + table_doc: TmlDocument | None, + allocator: _DisplayNameAllocator, + resolve_field: Callable[[str], tuple[str, str] | None], + log: IssueLog, +) -> tuple[dict, dict | None] | None: + """One Ossie field -> `(columns[] entry, formulas[] entry or None)`, or + `None` when the field cannot be surfaced at all. + + A physical field becomes a `column_id` entry, validated against the + dataset's own already-built Table document so a broken reference is + caught here rather than shipped as an import-time 404. A computed field + becomes a `formulas[]` + `formula_id` pair, never a bare `column_id`. + """ + payload = stash.read_stash(field) + display_name = field.get("label") or field.get("name") or "" + object_ref = f"field:{display_name}" + name = allocator.allocate(display_name, log, object_ref=object_ref) + properties: dict = {"column_type": "ATTRIBUTE"} + formulas_entry: dict | None = None + + physical_column_name = _field_physical_display_name(field) + if physical_column_name is not None: + exists = any( + c.get("name") == physical_column_name for c in _physical_columns_of(table_doc) + ) + if not exists: + log.add( + code="TS-MODEL-COLUMN-ID-MISSING", + severity=Severity.ERROR, + message=( + f"field {display_name!r} maps to physical column " + f"{physical_column_name!r} on dataset {dataset_prefix!r}, but no " + f"such column exists on its Table document; the field is not " + f"surfaced in the model rather than referencing a column that " + f"does not exist" + ), + object_ref=object_ref, + ) + return None + columns_entry = { + "name": name, + "column_id": f"{dataset_prefix}::{physical_column_name}", + "properties": properties, + } + else: + expr = to_thoughtspot_expression( + (field.get("expression") or {}).get("dialects") or [], + resolve_field, log, object_ref=object_ref, + ) + if expr is None: + log.add( + code="TS-MODEL-FIELD-UNTRANSLATABLE", + severity=Severity.ERROR, + message=( + f"field {display_name!r}'s expression could not be translated " + f"into any ThoughtSpot-importable form; it is not included in " + f"the model" + ), + object_ref=object_ref, + ) + return None + formula_id = _formula_id_from(name) + # `expr` is stored raw here -- not yet rewritten for cross-references + # to other formulas, and not yet block-scalar-wrapped. Both happen + # once, uniformly, in build_model's own final pass over the fully + # assembled formulas[] list, which is the earliest point every + # formula's final id is known (see _rewrite_formula_references). + formulas_entry = {"id": formula_id, "name": name, "expr": expr} + columns_entry = {"name": name, "formula_id": formula_id, "properties": properties} + if field.get("datatype") is not None: + log.add( + code="TS-MODEL-FIELD-DATATYPE-UNWRITABLE", + severity=Severity.WARNING, + message=( + f"field {display_name!r} is formula-backed and declares a " + f"datatype, but Model TML has no data_type key on a " + f"formula-backed columns[] entry; it is not carried into the " + f"model" + ), + object_ref=object_ref, + ) + + extra_properties = payload.get(FIELD_STASH_COLUMN_PROPERTIES) or {} + properties.update(_drop_never_emit_true_properties(extra_properties, log, object_ref=object_ref)) + _restore_ai_context(properties, field.get("ai_context"), log, object_ref=object_ref) + + description = field.get("description") + if description: + columns_entry["description"] = description + + return columns_entry, formulas_entry + + +#: Every `shape` value METRIC_STASH_SHAPE's own vocabulary defines (see +#: constants.py) -- checked against, not enumerated a second time, so a +#: future fourth shape only needs adding there for this set to pick it up. +_KNOWN_METRIC_SHAPES = frozenset( + {METRIC_SHAPE_COLUMN_AGGREGATION, METRIC_SHAPE_SCALAR_FORMULA_PLUS_AGGREGATION, METRIC_SHAPE_FORMULA} +) + + +def _build_metric( + metric: dict, + allocator: _DisplayNameAllocator, + resolve_field: Callable[[str], tuple[str, str] | None], + log: IssueLog, +) -> tuple[dict, dict] | None: + """One Ossie metric -> `(formulas[] entry, columns[] entry)`, or `None` + when it cannot be translated at all. + + Always a formula, never `column_id` + `aggregation` -- Ossie's own + Metric schema has no `column_id` field regardless, so this is the only + shape available. The stash's `shape` (default METRIC_SHAPE_FORMULA, the + documented contract for an absent key) selects only between the two + formula-based emissions: `scalar_formula_plus_aggregation` + decomposes the composed expression back into a scalar `expr` plus a + load-bearing `properties.aggregation`; every other shape — the default, + and `column_aggregation`, whose Ossie-side THOUGHTSPOT text is *already* + the same aggregate-in-expr shape the default is — is emitted as-is, with + `properties.aggregation` set only as the inert convention real + ThoughtSpot-authored documents carry (see the worked shape example). + """ + payload = stash.read_stash(metric) + live_name = metric.get("name") or "" + display_name = _restore_tml_name(payload, live_name, log, object_ref=f"metric:{live_name}") + object_ref = f"metric:{display_name}" + name = allocator.allocate(display_name, log, object_ref=object_ref) + formula_id = _formula_id_from(name) + + ts_expr = to_thoughtspot_expression( + (metric.get("expression") or {}).get("dialects") or [], + resolve_field, log, object_ref=object_ref, + ) + if ts_expr is None: + log.add( + code="TS-MODEL-METRIC-UNTRANSLATABLE", + severity=Severity.ERROR, + message=( + f"metric {display_name!r}'s expression could not be translated " + f"into any ThoughtSpot-importable form; it is not included in the " + f"model" + ), + object_ref=object_ref, + ) + return None + + shape = payload.get(METRIC_STASH_SHAPE, METRIC_SHAPE_FORMULA) + if shape not in _KNOWN_METRIC_SHAPES: + log.add( + code="TS-MODEL-METRIC-SHAPE-UNKNOWN", + severity=Severity.WARNING, + message=( + f"metric {display_name!r} is stashed with shape {shape!r}, which is " + f"not one of the shapes this converter recognises " + f"({sorted(_KNOWN_METRIC_SHAPES)!r}); treated as the default " + f"({METRIC_SHAPE_FORMULA!r}) rather than silently misapplied" + ), + object_ref=object_ref, + ) + shape = METRIC_SHAPE_FORMULA + properties: dict = {"column_type": "MEASURE"} + formula_expr = ts_expr + + if shape == METRIC_SHAPE_SCALAR_FORMULA_PLUS_AGGREGATION: + decomposed = _decompose_scalar_aggregate(ts_expr) + if decomposed is None: + log.add( + code="TS-MODEL-METRIC-SHAPE-MISMATCH", + severity=Severity.WARNING, + message=( + f"metric {display_name!r} is stashed as " + f"scalar_formula_plus_aggregation but its composed expression " + f"{ts_expr!r} has no recognised single-argument outer aggregate " + f"call; it is emitted as a plain formula instead" + ), + object_ref=object_ref, + ) + else: + properties["aggregation"], formula_expr = decomposed + + if "aggregation" not in properties: + conventional = _outer_aggregation_of(ts_expr) + if conventional is not None: + properties["aggregation"] = conventional + # Ossie's own Metric object has nowhere to record whether the + # *source* TML's surfacing column carried this property or + # omitted it -- both collapse identically on the way in, so this + # converter cannot tell them apart and always re-derives it. The + # value is a documented no-op here (the expr already aggregates, + # per the worked shape example and the domain-review note this + # module's docstrings already cite), so it changes no number -- + # but it is still a difference a byte-for-byte reader would see, + # and a round trip whose whole point is fidelity should not make + # that judgment silently on the reader's behalf. INFO, not + # WARNING: nothing is wrong, this is FYI only, matching the + # severity expression_entries already uses for an equally benign + # structural note (TS-EXPR-THOUGHTSPOT-ONLY). + log.add( + code="TS-MODEL-METRIC-AGGREGATION-CONVENTION", + severity=Severity.INFO, + message=( + f"metric {display_name!r}'s formula already aggregates " + f"({conventional}); the surfacing column's aggregation is set " + f"to match, as the convention real ThoughtSpot-authored " + f"documents carry -- this is a no-op over an already-aggregate " + f"expression, not a change to the result, and Ossie has no way " + f"to record whether the source document set this property or " + f"omitted it" + ), + object_ref=object_ref, + ) + + if metric.get("datatype") is not None: + log.add( + code="TS-MODEL-METRIC-DATATYPE-UNWRITABLE", + severity=Severity.WARNING, + message=( + f"metric {display_name!r} declares a datatype, but Model TML has " + f"no data_type key anywhere for a formula-backed metric; it is not " + f"carried into the model" + ), + object_ref=object_ref, + ) + + extra_properties = payload.get(FIELD_STASH_COLUMN_PROPERTIES) or {} + properties.update(_drop_never_emit_true_properties(extra_properties, log, object_ref=object_ref)) + _restore_ai_context(properties, metric.get("ai_context"), log, object_ref=object_ref) + + # Raw, unwrapped `formula_expr` here -- see the matching comment in + # _build_field; both the cross-reference rewrite and the block-scalar + # wrap happen once, uniformly, in build_model's final pass. + formulas_entry = {"id": formula_id, "name": name, "expr": formula_expr} + columns_entry = {"name": name, "formula_id": formula_id, "properties": properties} + description = metric.get("description") + if description: + columns_entry["description"] = description + + return formulas_entry, columns_entry + + +def _build_field_index( + datasets: list[dict], +) -> dict[str, tuple[str, str]]: + """`"dataset.field" -> (TABLE, physical column display name)`, for every + physical field in every dataset -- the data `resolve_field` (the + `to_thoughtspot_expression` parameter) is built from. + + Deliberately not named `resolve` (see the module's Model-building + section and the task interfaces): `resolve` (tml_to_ossie.py) maps + `(TABLE, Column) -> "dataset.field"`; this is its inverse, same arity, + keyed the other way around, so a mixed-up argument would type-check and + produce silently wrong references. + """ + index: dict[str, tuple[str, str]] = {} + for dataset in datasets: + dataset_prefix = dataset.get("name") + if not dataset_prefix: + continue + for field in dataset.get("fields") or []: + field_name = field.get("name") + if not field_name: + continue + physical_column_name = _field_physical_display_name(field) + if physical_column_name is None: + continue + index[f"{dataset_prefix}.{field_name}"] = (dataset_prefix, physical_column_name) + return index + + +#: The two spellings a source join `type` can arrive as for what +#: ThoughtSpot calls `OUTER` (its own full outer join). Matched +#: case/whitespace-insensitively: the stash carries whatever spelling the +#: source TML happened to use, and neither variant -- nor any casing of +#: either -- is privileged. +_FULL_OUTER_SPELLING = "FULL_OUTER" + + +def _normalise_join_type(value: str) -> str: + """A source `FULL OUTER` / `FULL_OUTER` becomes `OUTER`, in every + context TML accepts a join `type` at all. ThoughtSpot accepts only + `INNER`, `LEFT_OUTER`, `RIGHT_OUTER`, `OUTER` and rejects both `FULL_OUTER` + spellings identically; `OUTER` *is* ThoughtSpot's own full outer join, so + this is a semantics-preserving rename, never a loss -- nothing is logged + for it, unlike every other rewrite in this module. Every other value + (already one of the four TML accepts, since it came from a real TML + export) passes through unchanged. + """ + if value.strip().upper().replace(" ", "_") == _FULL_OUTER_SPELLING: + return "OUTER" + return value + + +def _restore_relationship_condition( + from_prefix: str, to_prefix: str, from_columns: list[str], to_columns: list[str] +) -> str: + """The equality-only `on:` condition for a relationship with no stashed + `on_expression` -- reconstructed from `from_columns`/`to_columns` alone, + which is all a hand-authored relationship (no stash) has to go on.""" + pairs = zip(from_columns or [], to_columns or []) + return " and ".join( + f"{identifiers.format_column_ref(from_prefix, fc)} = " + f"{identifiers.format_column_ref(to_prefix, tc)}" + for fc, tc in pairs + ) + + +def _join_entry_for_relationship(rel: dict, log: IssueLog) -> tuple[str, dict, dict | None]: + """One Ossie relationship -> `(from_prefix, model join entry, + Table joins_with[] entry or None)`. + + A `"referencing"`- or `"referencing_with_inline_attrs"`-shaped join is + restored as such -- a `referencing_join` pointer on the Model entry plus + a matching `joins_with[]` entry for the caller to attach to the *Table* + document -- whenever the stashed `referencing_join` name still matches + this relationship's own current `name`. TML's inline join syntax has no + name field at all, so a relationship that instead falls through to the + inline branch below gets a fresh one synthesized from its own from/to + dataset names on the next TML -> Ossie pass; restoring the referencing + shape here is what avoids that rename. When the two names disagree -- + the relationship was renamed since the stash was written -- the stash + is stale: it is dropped, an issue records it, and the join is emitted + inline instead, exactly as a hand-authored relationship with no stash + at all would be. The same is true when there is no stashed + `referencing_join` to begin with. + + The witness-copy pattern governs `on_expression` here, named by example + elsewhere in this converter as the "verbatim on_expression" case. A plain + stash-if-present read would silently keep + serving the *old* condition (residual predicates included) after a user + retargets the relationship's `from_columns`/`to_columns` -- so the stash + is only trusted when the witness (a snapshot of those two arrays, taken + the moment the stash was written) still matches the live ones. A mismatch + means the relationship was edited since; the stash -- on_expression and + whatever residual narrowing it carried -- is dropped, an issue records + it, and the condition is re-derived from the current from_columns/ + to_columns alone, exactly as a hand-authored relationship with no stash + at all would be. + + The same witness pattern governs whether this relationship's endpoints + get un-swapped before any of the above runs. `TML -> Ossie` swaps a + `ONE_TO_MANY` join's `from`/`to`/`from_columns`/`to_columns` so the + emitted relationship satisfies core-spec/spec.yaml's many-side/one-side + convention (see `tml_to_ossie._relationship_from_join`) -- which means + recovering TML's own declared join direction here means undoing that + swap first, before `from_prefix`/`to_prefix`/`from_columns`/`to_columns` + are used for anything else in this function (the condition fallback, the + `with`/`destination` target, and the `from_prefix` the caller nests the + join under). The swap is undone only while + `RELATIONSHIP_STASH_ENDPOINTS_SWAPPED_WITNESS` still matches the + relationship's live from/to/from_columns/to_columns -- agreement means + nobody retargeted the relationship since the stash was written; + disagreement means it was, so the swap is left alone (the live shape is + trusted as-is, exactly as a hand-authored relationship with no stash at + all would be) and an issue records it. + """ + payload = stash.read_stash(rel) + live_from = rel.get("from") or "" + live_to = rel.get("to") or "" + live_from_columns = rel.get("from_columns") or [] + live_to_columns = rel.get("to_columns") or [] + + had_stashed_swap = RELATIONSHIP_STASH_ENDPOINTS_SWAPPED in payload + endpoints_swapped = stash.restore( + payload, RELATIONSHIP_STASH_ENDPOINTS_SWAPPED, False, + witness=[live_from, live_to, live_from_columns, live_to_columns], + witness_key=RELATIONSHIP_STASH_ENDPOINTS_SWAPPED_WITNESS, + ) + if had_stashed_swap and not endpoints_swapped: + log.add( + code="TS-JOIN-ENDPOINTS-SWAP-STALE", + severity=Severity.WARNING, + message=( + f"relationship {rel.get('name')!r} has a stashed endpoint swap, " + f"but its from/to/from_columns/to_columns no longer match what " + f"that swap was recorded against -- the relationship was " + f"retargeted since the stash was written, so the swap is not " + f"undone; the join is emitted from this relationship's current " + f"from/to exactly as a hand-authored relationship with no stash " + f"at all would be" + ), + object_ref=f"relationship:{rel.get('name')}", + ) + + if endpoints_swapped: + from_prefix, to_prefix = live_to, live_from + from_columns, to_columns = live_to_columns, live_from_columns + else: + from_prefix, to_prefix = live_from, live_to + from_columns, to_columns = live_from_columns, live_to_columns + + had_stashed_on_expression = RELATIONSHIP_STASH_ON_EXPRESSION in payload + on_expression = stash.restore( + payload, RELATIONSHIP_STASH_ON_EXPRESSION, None, + # Compared against the relationship's live from_columns/to_columns, + # never the un-swapped ones above: the stashed witness was written + # (tml_to_ossie.py) from the emitted -- i.e. already-swapped -- + # from_columns/to_columns, which is exactly what "live" means here. + witness=[live_from_columns, live_to_columns], + witness_key=RELATIONSHIP_STASH_ON_EXPRESSION_WITNESS, + ) + if not on_expression: + if had_stashed_on_expression: + log.add( + code="TS-JOIN-ON-EXPRESSION-STALE", + severity=Severity.WARNING, + message=( + f"relationship {rel.get('name')!r} has a stashed on_expression, " + f"but its from_columns/to_columns no longer match what that " + f"condition was derived from -- the relationship was retargeted " + f"since the stash was written, so the stashed condition (and any " + f"residual predicates it narrowed) is dropped; the plain equality " + f"condition is re-derived from the current from_columns/to_columns " + f"instead" + ), + object_ref=f"relationship:{rel.get('name')}", + ) + on_expression = _restore_relationship_condition(from_prefix, to_prefix, from_columns, to_columns) + join_type = _normalise_join_type(payload.get(RELATIONSHIP_STASH_TYPE) or "INNER") + cardinality = payload.get(RELATIONSHIP_STASH_CARDINALITY) or "MANY_TO_ONE" + + live_name = rel.get("name") + stashed_referencing_join = payload.get(RELATIONSHIP_STASH_REFERENCING_JOIN) + if stashed_referencing_join and stashed_referencing_join != live_name: + log.add( + code="TS-JOIN-REFERENCING-JOIN-STALE", + severity=Severity.WARNING, + message=( + f"relationship {live_name!r} has a stashed referencing_join " + f"{stashed_referencing_join!r}, but that no longer matches the " + f"relationship's own current name -- it was renamed since the " + f"stash was written, so the stashed Table joins_with[] reference " + f"is dropped; an inline join is emitted instead, named on the " + f"next TML -> Ossie pass from its from/to datasets like a " + f"hand-authored relationship would be" + ), + object_ref=f"relationship:{live_name}", + ) + stashed_referencing_join = None + + if stashed_referencing_join: + model_join_entry: dict = {"referencing_join": stashed_referencing_join} + if payload.get(RELATIONSHIP_STASH_JOIN_SHAPE) == "referencing_with_inline_attrs": + model_join_entry["type"] = join_type + model_join_entry["cardinality"] = cardinality + joins_with_entry = { + "name": stashed_referencing_join, + "destination": {"name": to_prefix}, + "on": on_expression, + "type": join_type, + "cardinality": cardinality, + } + return from_prefix, model_join_entry, joins_with_entry + + return from_prefix, { + "with": to_prefix, "on": on_expression, "type": join_type, "cardinality": cardinality, + }, None + + +def _join_entry_for_unrepresentable(entry: dict) -> tuple[str, dict]: + """One `unrepresentable_joins[]` stash entry -> `(from_prefix, inline join + entry)` -- these carry the verbatim `on_expression` unconditionally (they + exist only because their condition has no equality pair at all), so the + join is restored exactly rather than approximated.""" + from_prefix = entry.get("from") or "" + to_prefix = entry.get("to") or "" + on_expression = entry.get(RELATIONSHIP_STASH_ON_EXPRESSION) or "" + join_type = _normalise_join_type(entry.get(RELATIONSHIP_STASH_TYPE) or "INNER") + cardinality = entry.get(RELATIONSHIP_STASH_CARDINALITY) or "MANY_TO_ONE" + return from_prefix, { + "with": to_prefix, "on": on_expression, "type": join_type, "cardinality": cardinality, + } + + +def build_model(semantic_model: dict, tables: Sequence[TmlDocument], log: IssueLog) -> TmlDocument: + """One Ossie `semantic_model` entry -> one ThoughtSpot `model:` TML document. + + `tables` are the already-built Table/SQL-View documents for this model's + datasets (`build_table`, called once per dataset) -- consulted here, by + name, rather than re-derived, so a physical field's `column_id` always + references a column that genuinely exists on the document a Model import + would actually load (tables are emitted, and known, before the + model that references them). + """ + model_payload = stash.read_stash(semantic_model) + live_model_name = semantic_model.get("name") or "" + model_name = _restore_tml_name( + model_payload, live_model_name, log, object_ref=f"model:{live_model_name}" + ) + object_ref = f"model:{model_name}" + body: dict = {"name": model_name} + + description = semantic_model.get("description") + if description: + body["description"] = description + + if semantic_model.get("ai_context") is not None: + log.add( + code="TS-MODEL-AI-CONTEXT-UNSUPPORTED", + severity=Severity.WARNING, + message=( + "model-scope ai_context has no home in Model TML -- ThoughtSpot's " + "model-scope Spotter instructions are configured outside the TML " + "document; it is not carried into the model" + ), + object_ref=object_ref, + ) + + datasets = semantic_model.get("datasets") or [] + tables_by_name = {t.body.get("name"): t for t in tables} + + model_tables: list[dict] = [] + model_tables_by_prefix: dict[str, dict] = {} + table_doc_by_prefix: dict[str, TmlDocument | None] = {} + + for dataset in datasets: + dataset_prefix = dataset.get("name") or "" + ds_payload = stash.read_stash(dataset) + table_ref = _table_name(dataset, ds_payload) + alias = ds_payload.get(DATASET_STASH_ALIAS) + table_doc = tables_by_name.get(table_ref) + table_doc_by_prefix[dataset_prefix] = table_doc + if table_doc is None: + log.add( + code="TS-MODEL-TABLE-MISSING", + severity=Severity.ERROR, + message=( + f"dataset {dataset_prefix!r} references table {table_ref!r}, " + f"but no matching document was supplied in `tables`; the " + f"model_tables[] entry is still emitted by name, but none of " + f"this dataset's fields can be validated or surfaced" + ), + object_ref=f"dataset:{dataset_prefix}", + ) + + table_entry: dict = {"name": table_ref} + if alias: + table_entry["alias"] = alias + model_tables.append(table_entry) + model_tables_by_prefix[dataset_prefix] = table_entry + + resolve_field = _build_field_index(datasets).get + + allocator = _DisplayNameAllocator() + columns: list[dict] = [] + formulas: list[dict] = [] + + for dataset in datasets: + dataset_prefix = dataset.get("name") or "" + table_doc = table_doc_by_prefix.get(dataset_prefix) + for field in dataset.get("fields") or []: + built = _build_field(field, dataset_prefix, table_doc, allocator, resolve_field, log) + if built is None: + continue + columns_entry, formulas_entry = built + columns.append(columns_entry) + if formulas_entry is not None: + formulas.append(formulas_entry) + + for metric in semantic_model.get("metrics") or []: + built = _build_metric(metric, allocator, resolve_field, log) + if built is None: + continue + formulas_entry, columns_entry = built + formulas.append(formulas_entry) + columns.append(columns_entry) + + for entry in model_payload.get(MODEL_STASH_UNATTRIBUTED_FORMULAS) or []: + # A formula spanning two or more Ossie datasets has no single + # dataset to belong to, which is exactly why the forward direction + # could not turn it into an ordinary Ossie field -- but a TML + # formula's surfacing columns[] entry was never tied to a dataset + # in the first place (`formula_id` + `properties`, no + # `column_id`), so nothing here actually stops the formula from + # being surfaced normally. An earlier revision re-emitted only the + # bare formulas[] entry with no surfacing columns[] entry at all -- + # which, by ThoughtSpot's own visibility rule (a formulas[] entry + # with no columns[] entry referencing it is not surfaced), silently + # made a formula that WAS visible in the source unreachable in the + # rebuilt model, while the issue it raised said only that column + # properties were lost -- a materially smaller claim than what + # actually happened. Restoring the surfacing entry (using the + # stashed properties verbatim, filtered the same way every other + # surfaced field's properties are) fixes the cause rather than + # rewording the symptom, and needs no issue at all: nothing is lost + # once the formula is surfaced. + raw_name = entry.get("name") or "" + object_ref = f"formula:{raw_name}" + allocated_name = allocator.allocate(raw_name, log, object_ref=object_ref) + expr = entry.get("expr", "") + formula_id = _formula_id_from(allocated_name) + # Raw, unwrapped `expr` -- see the matching comment in _build_field. + formulas.append({"id": formula_id, "name": allocated_name, "expr": expr}) + stashed_properties = entry.get(FIELD_STASH_COLUMN_PROPERTIES) or {} + properties = _drop_never_emit_true_properties( + dict(stashed_properties), log, object_ref=object_ref + ) + properties.setdefault("column_type", "ATTRIBUTE") + columns.append({"name": allocated_name, "formula_id": formula_id, "properties": properties}) + + # Every formula's final id is only fully known once every field, metric + # and unattributed formula above has been assigned one -- a formula + # earlier in this list can be cross-referenced by one built later (or + # vice versa; declaration order inside model.formulas[] carries no + # ordering guarantee for this converter's own consumers). So the + # cross-reference rewrite and the block-scalar wrap + # both happen here, once, over the now-complete list, rather than + # per-formula while it was being built above. + formula_id_by_normalised_name = { + _normalise_or_self(entry["name"]): entry["id"] for entry in formulas + } + for entry in formulas: + rewritten = _rewrite_formula_references( + entry["expr"], formula_id_by_normalised_name, log, + object_ref=f"formula:{entry['name']}", + ) + entry["expr"] = _maybe_block_scalar(rewritten) + + covered_columns_by_dataset: dict[str, list[set]] = {} + + for rel in semantic_model.get("relationships") or []: + from_prefix, join_entry, joins_with_entry = _join_entry_for_relationship(rel, log) + target = model_tables_by_prefix.get(from_prefix) + if target is None: + log.add( + code="TS-MODEL-RELATIONSHIP-UNKNOWN-FROM", + severity=Severity.ERROR, + message=( + f"relationship {rel.get('name')!r} names `from` dataset " + f"{from_prefix!r}, which is not one of this model's datasets; " + f"the join is dropped" + ), + object_ref=f"relationship:{rel.get('name')}", + ) + continue + target.setdefault("joins", []).append(join_entry) + if joins_with_entry is not None: + # `table_doc_by_prefix` holds the same TmlDocument objects the + # caller's own `tables` sequence does -- `TmlDocument` is frozen, + # but its `body` dict is not, so appending here is visible in + # the final DocumentSet without build_model needing to return + # anything beyond the Model document it already does. + from_table_doc = table_doc_by_prefix.get(from_prefix) + if from_table_doc is not None: + from_table_doc.body.setdefault("joins_with", []).append(joins_with_entry) + to_prefix = rel.get("to") + to_columns = rel.get("to_columns") + if to_prefix and to_columns: + covered_columns_by_dataset.setdefault(to_prefix, []).append(set(to_columns)) + + for entry in model_payload.get(MODEL_STASH_UNREPRESENTABLE_JOINS) or []: + from_prefix, join_entry = _join_entry_for_unrepresentable(entry) + target = model_tables_by_prefix.get(from_prefix) + if target is None: + log.add( + code="TS-MODEL-RELATIONSHIP-UNKNOWN-FROM", + severity=Severity.ERROR, + message=( + f"an unrepresentable join names `from` dataset {from_prefix!r}, " + f"which is not one of this model's datasets; the join is dropped" + ), + object_ref=f"dataset:{from_prefix}", + ) + continue + target.setdefault("joins", []).append(join_entry) + + # Dataset-level mapping's `primary_key`/`unique_keys` rows: TML has no key + # declaration anywhere (neither Table nor Model), so a declared key's only + # possible home on the way back is a relationship whose `to_columns` + # cover it -- see the construct-mapping document's own worked example, + # where a single-dataset model's unused `primary_key` is exactly this + # loss. A key a relationship *does* cover needs no issue: the + # relationship (already restored above) carries the same fact. + for dataset in datasets: + dataset_prefix = dataset.get("name") or "" + covered = covered_columns_by_dataset.get(dataset_prefix, []) + declared_keys: list[tuple[str, list[str]]] = [] + primary_key = dataset.get("primary_key") + if primary_key: + declared_keys.append(("primary_key", list(primary_key))) + for index, unique_key in enumerate(dataset.get("unique_keys") or []): + if unique_key: + declared_keys.append((f"unique_keys[{index}]", list(unique_key))) + for key_label, key_columns in declared_keys: + key_set = set(key_columns) + if any(key_set <= c for c in covered): + continue + log.add( + code="TS-MODEL-DATASET-KEY-UNUSED", + severity=Severity.WARNING, + message=( + f"dataset {dataset_prefix!r} declares {key_label} " + f"{key_columns!r}, but no relationship's to_columns cover it; " + f"TML has no key declaration anywhere, so this key has nowhere " + f"to go and is dropped" + ), + object_ref=f"dataset:{dataset_prefix}", + ) + + body["model_tables"] = model_tables + if columns: + body["columns"] = columns + if formulas: + body["formulas"] = formulas + + model_properties = model_payload.get(MODEL_STASH_MODEL_PROPERTIES) + if model_properties: + body["properties"] = dict(model_properties) + + for stash_key in ( + MODEL_STASH_PARAMETERS, MODEL_STASH_FILTERS, MODEL_STASH_COLUMN_GROUPS, + MODEL_STASH_LESSON_PLANS, MODEL_STASH_ACTION_OBJECT_ASSOCIATIONS, MODEL_STASH_CONSTRAINTS, + ): + value = model_payload.get(stash_key) + if value: + body[stash_key] = value + + model_joins_with = model_payload.get(MODEL_STASH_MODEL_JOINS_WITH) + if model_joins_with: + # Restored under the bare TML key `joins_with` -- `model_` in the + # stash key only disambiguates it from a *Table* document's own, + # differently-scoped `joins_with[]` inside the same payload namespace. + body["joins_with"] = model_joins_with + + return TmlDocument(kind="model", body=body, guid=None) + + +# --------------------------------------------------------------------------- +# convert: the public Ossie -> TML entry point. +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class TmlConversion: + """The result of one Ossie -> TML conversion. + + `documents` is the full TML document set -- one Model document plus one + Table/SQL-View document per dataset, ready to serialise via + `tml.dump_document_set`. `issues` is every declared loss and degradation + raised while building it, mirroring `tml_to_ossie.OssieConversion`'s own + shape in reverse. + """ + + documents: DocumentSet + issues: IssueLog + + +def convert(ossie_document: dict) -> TmlConversion: + """Convert one Ossie document into one ThoughtSpot TML document set. + + Ossie's `semantic_model` is a list (`core-spec/spec.md:88-96`), but -- + mirroring `tml_to_ossie.convert`, which only ever *produces* a + single-entry list -- this converter only ever *consumes* one: "One Ossie + semantic model corresponds to 1 + N TML documents" is this document's own + opening rule, and there is no defined mapping for more than one model + sharing a single TML document set. Zero or more than one entry is a hard + failure naming what was found, not a best-effort pick of the first. + + Tables are built before the model (`build_table`, one per dataset) so + `build_model` can validate every physical field's `column_id` against a + Table document that genuinely exists -- the same ordering the model + document itself enforces on its output (tables emitted, and known, + before the model that references them). + + There is no separate `connection_name` parameter, unlike `build_table` + directly: a dataset with no stashed connection name and no way to supply + one here gets the same `TS-DATASET-CONNECTION-MISSING` issue `build_table` + already raises for that case, naming the gap rather than inventing a + connection. + """ + models = ossie_document.get("semantic_model") + if not isinstance(models, list) or not models: + raise ConversionError("the Ossie document has no semantic_model entry to convert") + if len(models) > 1: + names = ", ".join(str(m.get("name")) for m in models if isinstance(m, dict)) + raise ConversionError( + f"the Ossie document declares more than one semantic_model entry " + f"({names}); this converter handles exactly one model per document" + ) + semantic_model = models[0] + + log = IssueLog() + tables = [build_table(dataset, log) for dataset in semantic_model.get("datasets") or []] + model = build_model(semantic_model, tables, log) + return TmlConversion(documents=DocumentSet(model=model, tables=tuple(tables)), issues=log) diff --git a/converters/thoughtspot/src/ossie_thoughtspot/stash.py b/converters/thoughtspot/src/ossie_thoughtspot/stash.py new file mode 100644 index 00000000..f265c2f6 --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/stash.py @@ -0,0 +1,162 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""custom_extensions[THOUGHTSPOT] payload handling. + +The stash lives in the *Ossie* document, so it is written on the way in and read +on the way out. It follows that the stash can only carry what TML contains. +""" +import json +from typing import Any + +from .constants import STASH_VERSION, VENDOR_KEY +from .errors import ConversionError + +#: Instance-local identity never travels in a portable document. +_FORBIDDEN_KEYS = frozenset({"guid", "obj_id", "fqn"}) + + +def _object_label(obj: dict) -> str: + return str(obj.get("name", "")) + + +def find_forbidden_key(value: Any, forbidden: frozenset[str] | None = None) -> str | None: + """The first key from `forbidden` found anywhere inside `value`, at any + depth, or `None`. + + `forbidden` defaults to `_FORBIDDEN_KEYS` (`guid`/`obj_id`/`fqn`). A caller + with a wider identity vocabulary to check for — this + package's own `dataset_id`/`custom_file_guid` additions, documented + identity-shaped keys the default set does not name — passes its own set rather + than this module maintaining a second, wider copy of its own; the scan + itself is shared either way, so the two vocabularies cannot drift apart + the way two independently maintained scans could. + + `write_stash` is the single point every stashed payload passes through, + so this is the one place the check needs to live for no caller — present + or future — to bypass it by nesting identity content one level below a + payload's own top-level keys instead of putting it there directly. A + value copied wholesale from source data, rather than rebuilt field by + field, is exactly how that happens in practice — the documented + ThoughtSpot shape `geo_config.custom_file_guid` naming a custom map is + one real example. + """ + names = forbidden if forbidden is not None else _FORBIDDEN_KEYS + if isinstance(value, dict): + for key, v in value.items(): + if key in names: + return key + found = find_forbidden_key(v, names) + if found is not None: + return found + return None + if isinstance(value, list): + for item in value: + found = find_forbidden_key(item, names) + if found is not None: + return found + return None + return None + + +def read_stash(obj: dict) -> dict[str, Any]: + """Return this object's parsed THOUGHTSPOT payload, or {} if it has none.""" + for entry in obj.get("custom_extensions") or []: + if entry.get("vendor_name") != VENDOR_KEY: + continue + raw = entry.get("data") + if raw is None: + return {} + if not isinstance(raw, str): + # `data` is typed as a string; a nested object is a spec violation. + raise ConversionError( + f"custom_extensions data for {_object_label(obj)!r} is " + f"{type(raw).__name__}, expected a JSON string" + ) + try: + parsed = json.loads(raw) + except json.JSONDecodeError as exc: + # Name the object; never surface a bare json traceback. + raise ConversionError( + f"malformed THOUGHTSPOT custom_extensions payload on " + f"{_object_label(obj)!r}: {exc}" + ) from exc + version = parsed.get("_v") if isinstance(parsed, dict) else None + if version != STASH_VERSION: + # An unrecognised shape version is a hard failure, not a + # partial read — a future payload shape this converter has never + # seen would otherwise be silently misread as the current one. + raise ConversionError( + f"THOUGHTSPOT custom_extensions payload on " + f"{_object_label(obj)!r} has shape version {version!r}, " + f"which this converter does not recognise (expected " + f"{STASH_VERSION!r})" + ) + return parsed + return {} + + +def write_stash(obj: dict, payload: dict[str, Any]) -> dict: + """Merge `payload` into this object's THOUGHTSPOT entry, returning a new dict. + + Foreign-vendor entries are preserved untouched. An empty resulting + payload writes nothing at all. + """ + forbidden_key = find_forbidden_key(payload) + if forbidden_key is not None: + # Checked at any depth — see find_forbidden_key. + raise ConversionError( + f"refusing to stash instance-local identity key {forbidden_key!r} " + f"on {_object_label(obj)!r}" + ) + + merged = {**read_stash(obj), **payload} + if not merged: + return dict(obj) + + merged["_v"] = STASH_VERSION # stamp the shape version so a future reader can recognise it + others = [e for e in obj.get("custom_extensions") or [] if e.get("vendor_name") != VENDOR_KEY] + out = dict(obj) + # Exactly one own entry, merged rather than appended. + out["custom_extensions"] = [ + *others, + {"vendor_name": VENDOR_KEY, "data": json.dumps(merged, sort_keys=True)}, + ] + return out + + +def restore( + payload: dict[str, Any], + key: str, + derived: Any, + *, + witness: Any = None, + witness_key: str | None = None, +) -> Any: + """Stash-if-present-and-still-current-else-derive. + + `witness` is the live Ossie value and `witness_key` names the copy recorded + alongside the stashed value. When they disagree the Ossie document has been + edited since the stash was written, so the stash is stale for this key and + `derived` wins. Without a witness this degrades to stash-if-present, which is + correct only for values nothing downstream can edit. + """ + if key not in payload: + return derived + if witness_key is not None and payload.get(witness_key) != witness: + return derived + return payload[key] diff --git a/converters/thoughtspot/src/ossie_thoughtspot/tml.py b/converters/thoughtspot/src/ossie_thoughtspot/tml.py new file mode 100644 index 00000000..9f30efee --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/tml.py @@ -0,0 +1,247 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""TML's structural half: the 1+N document set, and the serialisation invariants. + +Deliberately holds no Ossie vocabulary — it is the ThoughtSpot file format and nothing else, +which is what makes it unit-testable without a fixture from the other side. + +Four invariants live here so no caller has to carry them. `guid` is read and never written, +at any depth: it belongs at the document root, and a nested one — anywhere in the body, not +only there — is *silently ignored* on import while ThoughtSpot creates a duplicate object +with the same name. A formula expression containing braces is emitted as a `>-` block scalar +or the YAML will not parse on re-read. Tables are emitted before the model, which references +each one by name, so ordering is load-bearing. And everything goes through the YAML 1.2 codec +so a column, synonym, or parameter value of `on`, `off`, `yes`, or `no` survives as the string +it is instead of being coerced to a boolean. + +Filenames minted for a document set are sanitised and length-capped: a table name is +user-controlled data and may contain characters a filesystem treats specially — a path +separator, a `..` component, a Windows-reserved device name, more bytes than a single path +component allows — so `dump_document_set` never writes one through unexamined, and never +lets two documents land on the same filename. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Sequence + +import yaml + +from . import _yaml +from .errors import ConversionError + +#: The TML root keys this converter handles. Ossie's scope is the semantic model, so +#: answers, liveboards and the rest are not merely unsupported but out of scope. +_KINDS = ("model", "table", "sql_view") + +#: Filename suffix per kind, matching ThoughtSpot's own export convention. +_SUFFIX = {"model": "model.tml", "table": "table.tml", "sql_view": "sql_view.tml"} + +#: Characters forbidden in a filename component on POSIX (`/`) or Windows +#: (`< > : " / \ | ? *` plus control characters). Anything else — including a plain +#: `.` — is left alone so an ordinary name is emitted unchanged. +_FORBIDDEN_FILENAME_CHARS = re.compile(r'[<>:"/\\|?*\x00-\x1f]') + +#: Windows device names that are reserved regardless of extension (`CON`, `CON.txt`, +#: `com1.bak`, ... are all reserved) and regardless of case. +_RESERVED_WINDOWS_NAMES = frozenset( + {"CON", "PRN", "AUX", "NUL"} + | {f"COM{i}" for i in range(1, 10)} + | {f"LPT{i}" for i in range(1, 10)} +) + +#: Most filesystems cap a single path component at 255 bytes. A ThoughtSpot table or +#: model name carries no length limit of its own, so a name at or past that boundary +#: has to be shortened before it becomes a filename, not left to fail at write time. +_MAX_FILENAME_BYTES = 255 + + +class _BlockScalar(str): + """A string the dumper must emit as a folded block scalar. See `block_scalar`.""" + + +def _represent_block(dumper: yaml.SafeDumper, data: _BlockScalar) -> yaml.ScalarNode: + return dumper.represent_scalar("tag:yaml.org,2002:str", str(data), style=">") + + +yaml.add_representer(_BlockScalar, _represent_block, Dumper=_yaml.Yaml12Dumper) + + +def block_scalar(text: str) -> str: + """Mark `text` for `>-` emission. Returns a `str`, so callers need not care.""" + return _BlockScalar(text) + + +@dataclass(frozen=True) +class TmlDocument: + kind: str + body: dict + guid: str | None + source: str | None = None + + +@dataclass(frozen=True) +class DocumentSet: + model: TmlDocument + tables: tuple[TmlDocument, ...] + + def table_by_name(self, name: str) -> TmlDocument | None: + """The table or SQL view whose `name` matches, or `None`. + + Model `model_tables[]` entries reference a table by this name (or by an `alias` + that the caller resolves first), so this is the join between the two documents. + """ + for table in self.tables: + if table.body.get("name") == name: + return table + return None + + +def load_document(text: str, *, source: str | None = None) -> TmlDocument: + """Parse one TML document. Raises `ConversionError` rather than a bare YAML error.""" + data = _yaml.load(text) + if not isinstance(data, dict): + raise ConversionError(f"{source or ''} is not a TML document: expected a mapping") + present = [kind for kind in _KINDS if kind in data] + if not present: + raise ConversionError( + f"{source or ''} is not a TML document this converter handles: " + f"expected one of {', '.join(_KINDS)} at the root" + ) + if len(present) > 1: + raise ConversionError( + f"{source or ''} declares more than one root kind ({', '.join(present)})" + ) + kind = present[0] + body = data[kind] + if not isinstance(body, dict): + raise ConversionError(f"{source or ''}: {kind} must be a mapping") + return TmlDocument(kind=kind, body=body, guid=data.get("guid"), source=source) + + +def load_document_set(texts: Sequence[tuple[str, str]]) -> DocumentSet: + """Load `(source, text)` pairs into exactly one model plus its tables, in any order.""" + documents = [load_document(text, source=source) for source, text in texts] + models = [d for d in documents if d.kind == "model"] + tables = tuple(d for d in documents if d.kind in ("table", "sql_view")) + if not models: + raise ConversionError("the document set contains no model document") + if len(models) > 1: + names = ", ".join(str(m.body.get("name")) for m in models) + raise ConversionError(f"the document set contains more than one model document: {names}") + return DocumentSet(model=models[0], tables=tables) + + +def _strip_nested_guids(value: object) -> object: + """A copy of `value` with every `guid` key removed, at every depth. + + A nested `guid` — on a column entry, a join, anywhere below the document root — is + silently ignored on import and ThoughtSpot creates a duplicate object rather than + updating the existing one, exactly like a root-level `guid` would; this closes that + off at every level rather than only the root. `fqn` is left alone: it is legitimate + inside a model's table references and stripping it would break them. The input is + never mutated — `TmlDocument.body` belongs to the caller, who may reasonably dump + the same document twice or inspect it afterwards. + """ + if isinstance(value, dict): + return {k: _strip_nested_guids(v) for k, v in value.items() if k != "guid"} + if isinstance(value, list): + return [_strip_nested_guids(v) for v in value] + if isinstance(value, tuple): + return tuple(_strip_nested_guids(v) for v in value) + return value + + +def dump_document(document: TmlDocument) -> str: + """Serialise one document. `guid` is stripped unconditionally, at every depth of + the body — not only at the document root.""" + return _yaml.dump({document.kind: _strip_nested_guids(document.body)}) + + +def _safe_filename_component(name: object) -> str: + """A ThoughtSpot table/model name, made safe to use as a filename stem. + + Every character forbidden by POSIX (`/`) or Windows (`< > : " / \\ | ? *` and + control characters) is replaced with `_`. Trailing dots and spaces are trimmed — + Windows drops them silently, which could otherwise make two distinct names collide + invisibly. A name that is empty, `.`, or `..` after that, and a Windows-reserved + device name (`CON`, `COM1`, ...) regardless of what follows the first dot, each get + a safe fallback. An ordinary name such as `ORDERS` or `store_sales` is returned + exactly as given. Length is not handled here — `dump_document_set` caps it once it + knows how much room the suffix and a possible disambiguating counter need. + """ + text = name if isinstance(name, str) else "" + cleaned = _FORBIDDEN_FILENAME_CHARS.sub("_", text).rstrip(" .") + if cleaned in ("", ".", ".."): + cleaned = "_unnamed" + elif cleaned.split(".", 1)[0].upper() in _RESERVED_WINDOWS_NAMES: + cleaned = f"_{cleaned}" + return cleaned + + +def _truncate_utf8(text: str, max_bytes: int) -> str: + """`text`, cut down to at most `max_bytes` UTF-8 bytes, never splitting a + multi-byte character in half.""" + encoded = text.encode("utf-8") + if len(encoded) <= max_bytes: + return text + cut = max_bytes + while cut > 0: + try: + return encoded[:cut].decode("utf-8") + except UnicodeDecodeError: + cut -= 1 + return "" + + +def dump_document_set(document_set: DocumentSet) -> list[tuple[str, str]]: + """`(filename, text)` for every document, tables first — the model references them + by name, so they must exist before it does. + + Each filename's stem is sanitised (`_safe_filename_component`) and truncated to + leave room, within the 255-byte filesystem component limit, for both the suffix and + a disambiguating counter. Every candidate filename is reserved as it is minted: if + it is already taken — two source names sanitising to the same stem, a truncated + long name colliding with another, or a counter-suffixed name happening to land on + some other document's plain name — the counter advances and a fresh candidate is + tried until one is free. No two documents in one `DocumentSet` can ever be handed + the same filename. + """ + documents = list(document_set.tables) + [document_set.model] + # However many documents there are, that is also the most candidates any single one + # could need to try before finding a free filename (there are only that many + # filenames already claimed to collide with) — one extra digit of headroom besides. + counter_reserve = len(f"-{len(documents) + 1}") + + used: set[str] = set() + out = [] + for document in documents: + suffix = _SUFFIX[document.kind] + max_stem_bytes = _MAX_FILENAME_BYTES - len(f".{suffix}") - counter_reserve + stem = _safe_filename_component(document.body.get("name", document.kind)) + stem = _truncate_utf8(stem, max_stem_bytes).rstrip(" .") or "_unnamed" + + candidate = f"{stem}.{suffix}" + counter = 1 + while candidate in used: + counter += 1 + candidate = f"{stem}-{counter}.{suffix}" + used.add(candidate) + out.append((candidate, dump_document(document))) + return out diff --git a/converters/thoughtspot/src/ossie_thoughtspot/tml_to_ossie.py b/converters/thoughtspot/src/ossie_thoughtspot/tml_to_ossie.py new file mode 100644 index 00000000..2033faa6 --- /dev/null +++ b/converters/thoughtspot/src/ossie_thoughtspot/tml_to_ossie.py @@ -0,0 +1,2315 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert a ThoughtSpot Model column into an Ossie field. + +A ThoughtSpot Model `columns[]` entry becomes an Ossie field when it declares +`column_type: ATTRIBUTE`. A `MEASURE` column belongs to a metric instead, converted +elsewhere — this module returns `None` for one rather than building a field that would +duplicate the metric conversion. + +Four properties of the mapping are easy to get subtly wrong, because getting them wrong +still produces a document that imports and looks plausible. + +The formula string carried in the THOUGHTSPOT dialect entry is the exact `expr` text from +the source document, untouched — never rebuilt from a parsed name/arguments shape. The +reverse direction reads this entry, and the two are compared by exact string equality, so +any reformatting here — even whitespace-only — breaks that comparison on the way back, +regardless of how well the rest of the conversion went. + +A portable ANSI_SQL sibling is only ever added next to that verbatim entry, and only when +it can be produced with certainty rather than a guess: a bare column reference is the one +shape handled here. Anything else — a function call, an expression combining several +references, a reference this document's resolver cannot place — is left as +THOUGHTSPOT-only, with an issue recording why, rather than emitting a translation nobody +checked. + +A field's identifier and its display label are two different values. `name` is a +normalised, portable identifier derived from the ThoughtSpot column's display name; +`label` carries that display name exactly as written. Writing the display name into +`name`, or the normalised form into `label`, silently breaks both. When the display +name has no ASCII form at all for `identifiers.normalise` to fold onto (a CJK-only, +Cyrillic-only, or Greek-only name), `name` falls back to a different, still-usable +identifier instead of raising — see `_field_or_metric_identifier` — while `label` +still carries the exact original, unaffected either way. + +Finally, a computed column is model-scoped in ThoughtSpot but has to live inside exactly +one dataset in Ossie. It is attributed to the dataset every one of its column references +resolves to. When those references disagree — two or more different datasets, one that +cannot be resolved at all, or none at all to go on — no dataset is obviously correct, so +none is guessed: the attribution fails, an issue records why, and the field is not built. +Preserving the formula for a caller's model-level stash is that caller's job from there — +this module only sees one column at a time and has no access to the enclosing document. + +A `MEASURE` column becomes a metric instead of a field, built by `convert_metric` below. +Its one genuinely tricky rule is easy to get backwards in a way that still imports cleanly +and produces wrong numbers: the surfacing column's `aggregation` is load-bearing on a +`column_id` metric and on a *scalar*-formula metric (the two compose — `AGG()`, never the bare scalar) but a no-op on a formula whose own outer call already +aggregates (`sum ( ... )`, `group_aggregate ( ... )`, ...) — a common, correct shape +ThoughtSpot's UI produces routinely, so discarding a redundant column aggregation there +is silent by design. Composing when the rule says no-op, or leaving bare when the rule +says compose, silently changes the grain the metric evaluates at while the model still +imports. There is a third, rarer case an outer-call check alone cannot see: an aggregate +*nested inside* a still-scalar outer call, as in `round ( sum ( ... ) , 2 )` — `round` is +not itself an aggregate, but the expression as a whole already is one. That case is the +one worth a warning, because it is the one shape where a reader might reasonably expect +composition and not get it. `_outer_call_is_aggregate` decides the first two cases; +`_contains_aggregate_call` — checked only once the outer call is not itself an aggregate — +decides the third. Both read ThoughtSpot's aggregate call names off the same expression +catalog `_compose_aggregate_entries` uses to build the composed rendering — one source for +every one of these jobs, so they cannot silently drift apart the way independently +hand-typed lists +could. And unlike a field, a metric has no `label`: when identifier normalisation changes the +identifier, the exact display name has nowhere to go but the `custom_extensions` stash. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Callable + +from . import datatypes, formula, identifiers, keys, stash +from .constants import ( + DATASET_STASH_ALIAS, + DATASET_STASH_CONNECTION_NAME, + DATASET_STASH_SOURCE_PARTS, + DATASET_STASH_SOURCE_PARTS_DB, + DATASET_STASH_SOURCE_PARTS_DB_TABLE, + DATASET_STASH_SOURCE_PARTS_SCHEMA, + DATASET_STASH_SQL_OUTPUT_COLUMNS, + DATASET_STASH_TABLE_NAME, + DATASET_STASH_TML_OBJECT, + DATASET_STASH_TML_OBJECT_WITNESS, + DATASET_STASH_UNSURFACED_COLUMNS, + DIALECT, + DOCUMENT_VERSION, + FIELD_STASH_COLUMN_PROPERTIES, + FIELD_STASH_DATA_TYPE, + FIELD_STASH_DATA_TYPE_WITNESS, + FIELD_STASH_DB_COLUMN_NAME, + FIELD_STASH_DB_COLUMN_NAME_WITNESS, + METRIC_SHAPE_COLUMN_AGGREGATION, + METRIC_SHAPE_FORMULA, + METRIC_SHAPE_SCALAR_FORMULA_PLUS_AGGREGATION, + METRIC_STASH_SHAPE, + MODEL_STASH_ACTION_OBJECT_ASSOCIATIONS, + MODEL_STASH_COLUMN_GROUPS, + MODEL_STASH_CONSTRAINTS, + MODEL_STASH_FILTERS, + MODEL_STASH_LESSON_PLANS, + MODEL_STASH_MODEL_JOINS_WITH, + MODEL_STASH_MODEL_PROPERTIES, + MODEL_STASH_PARAMETERS, + MODEL_STASH_UNATTRIBUTED_FORMULAS, + MODEL_STASH_UNREPRESENTABLE_JOINS, + PORTABLE_DIALECT, + RELATIONSHIP_STASH_CARDINALITY, + RELATIONSHIP_STASH_ENDPOINTS_SWAPPED, + RELATIONSHIP_STASH_ENDPOINTS_SWAPPED_WITNESS, + RELATIONSHIP_STASH_JOIN_SHAPE, + RELATIONSHIP_STASH_ON_EXPRESSION, + RELATIONSHIP_STASH_ON_EXPRESSION_WITNESS, + RELATIONSHIP_STASH_REFERENCING_JOIN, + RELATIONSHIP_STASH_TYPE, + STASH_TML_NAME, +) +from .errors import ConversionError +from .expressions import CATALOG, Variant, emit_direct +from .issues import IssueLog, Severity +from .tml import DocumentSet + + +def expression_entries( + expr: str, + resolve: Callable[[str, str], str | None], + log: IssueLog, + *, + object_ref: str, + kind: str = "field", +) -> list[dict[str, str]]: + """The dialect entries for one ThoughtSpot expression. + + The THOUGHTSPOT entry always comes first and always carries `expr` unmodified — + whatever else this function decides, that entry is what makes the expression + recoverable later, character for character. A second, ANSI_SQL entry is appended + only when the whole expression is a bare column reference the resolver can place in + a dataset. Every other shape — a runtime parameter, a formula cross-reference, an + unresolvable reference, a function call, a compound expression — gets an issue + instead of a guessed translation, and only the verbatim entry is returned. + + A runtime parameter and a formula cross-reference are the same textual shape (a + bracketed name with no `::` qualifier) but different constructs, and are reported + under different codes for it: `formula.find_formula_refs`/`find_parameter_refs` + share one definition of which is which (`formula.FORMULA_REFERENCE_PREFIX`) so this + function and the Ossie -> TML model builder — which mints exactly this prefix and + rewrites references that carry it — cannot silently disagree about the convention. + Both are logged when both are present in the same expression, each under its own + code, rather than one masking the other. + + `kind` names the Ossie object this expression belongs to ("field" or "metric") — + used only in issue text, so a metric's non-portability issue reads "...evaluate + this metric" rather than the field-shaped default. `object_ref` already carries + this distinction (`field:...` vs `metric:...`); `kind` exists so the message + itself agrees with it instead of contradicting it. + """ + entries: list[dict[str, str]] = [{"dialect": DIALECT, "expression": expr}] + + # `dict.fromkeys` dedupes while preserving first-seen order -- a + # parameter or cross-reference used twice in one expression (a + # discount applied on both sides of a ratio, say) is one fact worth + # reporting once, not a message that reads as two distinct unresolved + # names when only one name repeats. + formula_refs = list(dict.fromkeys(formula.find_formula_refs(expr))) + parameters = list(dict.fromkeys(formula.find_parameter_refs(expr))) + if formula_refs or parameters: + if formula_refs: + log.add( + code="TS-EXPR-FORMULA-REFERENCE", + severity=Severity.INFO, + message=( + f"expression references the formula(s) {', '.join(formula_refs)} " + f"by cross-reference; a portable sibling would require inlining " + f"the referenced formula's own expression, which this converter " + f"does not attempt, so only the THOUGHTSPOT dialect entry is " + f"emitted" + ), + object_ref=object_ref, + ) + if parameters: + log.add( + code="TS-EXPR-PARAM", + severity=Severity.WARNING, + message=( + f"expression references the ThoughtSpot runtime parameter(s) " + f"{', '.join(parameters)}, which have no Ossie equivalent; no " + f"portable expression is emitted" + ), + object_ref=object_ref, + ) + return entries + + bare = formula.is_bare_column_ref(expr) + if bare is not None: + target = resolve(*bare) + if target is None: + log.add( + code="TS-EXPR-UNRESOLVED", + severity=Severity.WARNING, + message=( + f"reference {identifiers.format_column_ref(*bare)} resolves to " + f"no dataset field; no portable expression is emitted" + ), + object_ref=object_ref, + ) + return entries + entries.append({"dialect": PORTABLE_DIALECT, "expression": target}) + return entries + + # Anything else is a function call or a multi-reference expression. Producing a + # portable sibling for one would need an expression tree this converter does not + # build — see the module docstring — so the non-portability is recorded instead + # of guessed at. + log.add( + code="TS-EXPR-THOUGHTSPOT-ONLY", + severity=Severity.INFO, + message=( + "expression is emitted in the THOUGHTSPOT dialect only; a consumer that " + f"does not implement it will not be able to evaluate this {kind}" + ), + object_ref=object_ref, + ) + return entries + + +def attribute_dataset( + expr: str, + resolve: Callable[[str, str], str | None], + log: IssueLog, + *, + object_ref: str, +) -> str | None: + """The single dataset every column reference in `expr` resolves to, or `None`. + + A computed column has to be placed inside exactly one Ossie dataset. That is safe + only when every reference in the expression agrees on the same one: no references + at all is no evidence to place it by, a reference the resolver cannot place is + missing evidence, and references landing in two or more datasets is contradictory + evidence. Each of those returns `None` and logs why, rather than falling back to + the first candidate found or any other default — a wrong guess here would silently + move a field into the wrong dataset, or invent a home for one that references + nothing at all. + + Runtime parameter references carry no dataset and take no part in this decision; + an expression can be fully attributed while still not being portable, and + `expression_entries` is what reports the latter. + """ + refs = formula.find_column_refs(expr) + if not refs: + log.add( + code="TS-FIELD-NO-REFERENCES", + severity=Severity.WARNING, + message=( + "expression contains no column references, so it cannot be " + "attributed to a dataset" + ), + object_ref=object_ref, + ) + return None + + datasets: list[str] = [] + unresolved: list[str] = [] + for table, column in refs: + target = resolve(table, column) + if target is None: + unresolved.append(identifiers.format_column_ref(table, column)) + continue + dataset = target.split(".", 1)[0] + if dataset not in datasets: + datasets.append(dataset) + + if unresolved: + log.add( + code="TS-FIELD-UNRESOLVED-REFERENCE", + severity=Severity.WARNING, + message=( + f"reference(s) {', '.join(unresolved)} resolve to no dataset field; " + f"the expression cannot be attributed with confidence" + ), + object_ref=object_ref, + ) + return None + + if len(datasets) > 1: + log.add( + code="TS-FIELD-UNATTRIBUTED", + severity=Severity.WARNING, + message=( + f"references resolve to {len(datasets)} different datasets " + f"({', '.join(datasets)}); a computed field spanning more than one " + f"dataset cannot be attributed, and should be preserved as an " + f"unattributed formula rather than emitted as a field" + ), + object_ref=object_ref, + ) + return None + + return datasets[0] + + +def _physical_datatype( + table_name: str, + column_name: str, + table_lookup: Callable[[str], dict | None], + log: IssueLog, + *, + object_ref: str, + kind: str = "field", +) -> str | None: + """The Ossie datatype for a physical column, or `None` when it cannot be found. + + Matched by the physical column's own display name — what a Model `column_id` + suffix names — not by its warehouse column name. + + `None` covers two different situations, and only one of them is a loss worth + logging. A column with no `data_type` at all has nothing to drop — `datatype` + is optional in Ossie, and `datatypes.to_ossie` documents omission as a + legitimate answer, so this stays silent. A column whose `data_type` *is* + present but unrecognised by `datatypes.to_ossie` is different: the warehouse + told us the type and it is about to be dropped on the floor, so that case + logs an issue naming the type before returning `None`. + + `kind` ("field" or "metric") names the Ossie object being built, both in the + issue code (`TS-FIELD-...` vs `TS-METRIC-...`) and in the message text, so a + metric calling this does not raise a `TS-FIELD-*` code or say "field" about + itself — `object_ref` already says `metric:...`, and the code and message + need to agree with it. + """ + code_prefix = f"TS-{kind.upper()}" + table = table_lookup(table_name) + physical = None + if table is not None: + for candidate in table.get("columns", []): + if candidate.get("name") == column_name: + physical = candidate + break + if physical is None: + log.add( + code=f"{code_prefix}-PHYSICAL-COLUMN-MISSING", + severity=Severity.WARNING, + message=( + f"physical column {column_name!r} was not found on table " + f"{table_name!r}; no datatype is emitted for this {kind}" + ), + object_ref=object_ref, + ) + return None + data_type = (physical.get("db_column_properties") or {}).get("data_type") + if data_type is None: + return None + ossie_type = datatypes.to_ossie(data_type) + if ossie_type is None: + log.add( + code=f"{code_prefix}-DATATYPE-UNMAPPED", + severity=Severity.WARNING, + message=( + f"physical column {column_name!r} on table {table_name!r} has " + f"warehouse data_type {data_type!r}, which has no Ossie " + f"equivalent; no datatype is emitted for this {kind}" + ), + object_ref=object_ref, + ) + return ossie_type + + +def _ai_context(properties: dict) -> dict | str | None: + """Fold synonyms and free-text AI context into one Ossie `ai_context` value. + + Synonyms need the object form to have somewhere to live; free-text context on its + own stays a bare string, the simpler of the two shapes Ossie accepts. + """ + synonyms = properties.get("synonyms") + instructions = properties.get("ai_context") + if synonyms and instructions: + return {"synonyms": list(synonyms), "instructions": instructions} + if synonyms: + return {"synonyms": list(synonyms)} + if instructions: + return instructions + return None + + +def _physical_db_column_name( + table_name: str, column_name: str, table_lookup: Callable[[str], dict | None] +) -> str | None: + """The warehouse `db_column_name` of the physical column matching + `column_name` (its own display name — what a Model `column_id` suffix + names) on `table_name`, or `None` when the table, the column, or a + `db_column_name` on it can't be found. + + Used only as a fallback identifier basis — see + `_field_or_metric_identifier` — for a column whose *display* name has no + ASCII form: the underlying warehouse column name is almost always ASCII + even then, and unique within its table by construction, so it survives + where the display name doesn't. + """ + table = table_lookup(table_name) + if table is None: + return None + physical = next( + (p for p in table.get("columns", []) if p.get("name") == column_name), None + ) + if physical is None: + return None + return physical.get("db_column_name") + + +def _field_or_metric_identifier( + display_name: str, + physical_hint: str | None, + allocator: identifiers.Allocator, + log: IssueLog, + *, + kind: str, + object_ref: str, +) -> str: + """The Ossie identifier for a field or metric's ThoughtSpot display name. + + The common case is `identifiers.normalise(display_name)`, unchanged. The + exceptional case — `display_name` has no ASCII alphanumerics for + `normalise` to fold onto (a CJK-only, Cyrillic-only, Greek-only, or + punctuation-only name) — used to propagate as `ValueError` out of + `convert_field`/`convert_metric` entirely, caught by `convert()`'s own + try/except and misreported as a malformed *column reference* (the + exception is the same type `identifiers.split_column_ref` raises for a + genuinely ambiguous reference, and `convert()` could not tell the two + apart from outside). That also meant the column was dropped rather than + converted — the same graceful-degradation gap `TS-MODEL-NAME-UNNORMALISABLE` + already closed at Model scope, here extended to Field/Metric scope, and + reported under its own code so the two failures are never conflated again. + + The fallback identifier, in preference order: + + 1. `physical_hint` — normally the underlying warehouse column's own + `db_column_name` (see `_physical_db_column_name`), for a + column_id-backed column. A warehouse identifier is almost always + ASCII even when the display name labelling it is not, and it is + unique within its own table by construction (two columns cannot + share one warehouse name) — so no collision-avoidance is needed for + this branch; it is naturally distinct the same way an ordinary, + successfully-normalised identifier is (this converter does not + collision-check those either, a pre-existing and separate gap — see + `_index_attribute_columns`). + 2. A fixed placeholder — `kind` itself, i.e. `"field"` or `"metric"` — + allocated through `allocator`. Reached only when there is no + `physical_hint` at all (a formula-backed column with no physical + grounding) or the hint itself also has no ASCII form. `allocator` is + shared by the caller across every column that can reach this branch, + so two columns that would otherwise both become `"field"` instead + become `"field"` and `"field_2"` — distinct, per the `Allocator` + collision-suffix contract in `identifiers.py`. + + Either fallback always differs from `display_name`, so a caller that + already stashes the original display name whenever the identifier + differs from it (metrics do; fields carry it in `label` instead, which + is populated independently of this call and needs no stash) picks this + case up for free — nothing here writes a stash entry itself, only logs + the WARNING naming what happened. + """ + try: + return identifiers.normalise(display_name) + except ValueError: + pass + + fallback: str | None = None + if physical_hint: + try: + fallback = identifiers.normalise(physical_hint) + except ValueError: + fallback = None + source = "the underlying warehouse column name" if fallback is not None else "a placeholder" + if fallback is None: + fallback = allocator.allocate(kind) + + log.add( + code=f"TS-{kind.upper()}-NAME-UNNORMALISABLE", + severity=Severity.WARNING, + message=( + f"{kind} name {display_name!r} has no ASCII alphanumerics for " + f"normalise() to fold onto; {source} is used as its identifier " + f"instead: {fallback!r}" + ), + object_ref=object_ref, + ) + return fallback + + +def convert_field( + column: dict, + formulas: dict[str, dict], + table_lookup: Callable[[str], dict | None], + resolve: Callable[[str, str], str | None], + log: IssueLog, + allocator: identifiers.Allocator | None = None, +) -> dict | None: + """Convert one Model `columns[]` entry into an Ossie field, or `None`. + + `column` is a ThoughtSpot Model `columns[]` entry. A physical column carries + `column_id` (`TABLE::Column Name`); a computed column carries `formula_id` + instead, naming an entry in the model's `formulas[]` list. `formulas` is that + list reshaped into a lookup keyed by each entry's `id`, value the whole entry, + so `formulas[column["formula_id"]]["expr"]` is the expression text — this + function reads the expression from there, never from the column itself. A + `formula_id` absent from `formulas`, a column with neither key, or a + `column_type` that is not `ATTRIBUTE`, produces no field. + + `allocator` scopes fallback-identifier collision avoidance when this + column's display name has no ASCII form — see + `_field_or_metric_identifier`. The caller (`convert()`) shares one + `Allocator` across every field in the model so two colliding fallbacks + never collide with each other; a caller that omits it (every existing + single-column test in this suite) gets a fresh, private one, which is + exactly as correct for a call that only ever converts one column at a + time. + """ + properties = column.get("properties") or {} + if properties.get("column_type") != "ATTRIBUTE": + return None + if allocator is None: + allocator = identifiers.Allocator() + + display_name = column["name"] + object_ref = f"field:{display_name}" + + if "column_id" in column: + table_name, column_name = identifiers.split_column_ref(f"[{column['column_id']}]") + field_name = _field_or_metric_identifier( + display_name, + _physical_db_column_name(table_name, column_name, table_lookup), + allocator, log, kind="field", object_ref=object_ref, + ) + field: dict = {"name": field_name, "label": display_name} + expr = identifiers.format_column_ref(table_name, column_name) + field["expression"] = { + "dialects": expression_entries(expr, resolve, log, object_ref=object_ref) + } + datatype = _physical_datatype( + table_name, column_name, table_lookup, log, object_ref=object_ref + ) + if datatype is not None: + field["datatype"] = datatype + elif "formula_id" in column: + formula_id = column["formula_id"] + formula_entry = formulas.get(formula_id) + if formula_entry is None: + log.add( + code="TS-FIELD-FORMULA-MISSING", + severity=Severity.WARNING, + message=( + f"column {display_name!r} has formula_id {formula_id!r}, which " + f"matches no formulas[] entry; no field can be built" + ), + object_ref=object_ref, + ) + return None + if "expr" not in formula_entry: + log.add( + code="TS-FIELD-FORMULA-MISSING", + severity=Severity.WARNING, + message=( + f"column {display_name!r} has formula_id {formula_id!r}, whose " + f"formulas[] entry has no expr; no field can be built" + ), + object_ref=object_ref, + ) + return None + expr = formula_entry["expr"] + dataset = attribute_dataset(expr, resolve, log, object_ref=object_ref) + if dataset is None: + return None + field_name = _field_or_metric_identifier( + display_name, None, allocator, log, kind="field", object_ref=object_ref, + ) + field: dict = {"name": field_name, "label": display_name} + field["expression"] = { + "dialects": expression_entries(expr, resolve, log, object_ref=object_ref) + } + else: + log.add( + code="TS-FIELD-NO-SOURCE", + severity=Severity.WARNING, + message=( + "column has neither a physical column_id nor a formula_id; " + "no field can be built" + ), + object_ref=object_ref, + ) + return None + + description = column.get("description") + if description: + field["description"] = description + + ai_context = _ai_context(properties) + if ai_context is not None: + field["ai_context"] = ai_context + + return field + + +#: TML column aggregation -> the Ossie aggregate applied to the column expression. +#: `NONE` means the column carries no aggregate at all, which is distinct from absent. +_AGGREGATION = { + "SUM": "SUM", "COUNT": "COUNT", "AVERAGE": "AVG", "MIN": "MIN", "MAX": "MAX", + "COUNT_DISTINCT": "COUNT_DISTINCT", "STD_DEVIATION": "STDDEV", "VARIANCE": "VARIANCE", + "NONE": None, +} + +#: Aggregations that always report themselves as Integer, regardless of the underlying +#: physical column's own warehouse type — a COUNT of DOUBLEs is still a whole number +#: of rows. +_COUNT_AGGREGATIONS = frozenset({"COUNT", "COUNT_DISTINCT"}) + +#: The three TML shapes a metric can arrive as (the stash's `shape` key), so a +#: return trip can reproduce the source shape instead of collapsing every metric +#: into the same one. The values themselves live in constants.py +#: (METRIC_SHAPE_*) — shared with ossie_to_thoughtspot.py, the reader. +#: `METRIC_SHAPE_FORMULA` is also what a document with no stash at all defaults +#: to on the way back — a plain formulas[] entry, aggregate already baked into +#: its expr — so it is the one value never worth writing to the stash: writing +#: it or omitting it produces the same reconstruction either way. + +#: TML column aggregation -> the catalog `spec_name` whose DIRECT template is +#: ThoughtSpot's own native rendering of that aggregate (`"sum ( {0} )"`, +#: `"unique count ( {0} )"`, ...). Reused for two different jobs: composing the +#: THOUGHTSPOT dialect entry for a load-bearing aggregation (`_compose_aggregate_entries`) +#: and — via `_AGGREGATE_CALL_NAMES` below — recognising when a formula's own outer call +#: is *already* one of these. Both jobs read the same catalog rows, so "what do we +#: render" and "is this already rendered" cannot silently disagree the way two +#: independently hand-typed lists could. +_AGGREGATION_CATALOG_SPEC = { + "SUM": "SUM(expr)", "COUNT": "COUNT(expr)", "AVERAGE": "AVG(expr)", + "MIN": "MIN(expr)", "MAX": "MAX(expr)", "COUNT_DISTINCT": "COUNT(DISTINCT expr)", + "STD_DEVIATION": "STDDEV(expr)", "VARIANCE": "VARIANCE(expr)", +} + +#: MEDIAN(expr) has no TML `aggregation` enum counterpart at all, but `median ( ... )` +#: is a genuine native ThoughtSpot aggregate and must still be recognised as one when +#: it appears in an expression — see `_contains_aggregate_call`. +_NATIVE_AGGREGATE_SPECS = (*_AGGREGATION_CATALOG_SPEC.values(), "MEDIAN(expr)") + +#: `group_aggregate` is ThoughtSpot's own construct for a grouped/windowed +#: aggregation — the *performant* pattern the catalog's window-function rows +#: prefer over a raw `sql_*_aggregate_op` pass-through — and it is not a target of +#: any TML `aggregation` enum value, so it cannot come from `_AGGREGATION_CATALOG_SPEC`. +#: There is exactly one such construct, so it is named directly rather than derived. +_GROUP_AGGREGATE_CALL = "group_aggregate" + +#: Every `Variant` that denotes an *aggregate* `sql_*_op` pass-through wrapper, +#: derived by filtering the enum on its own `_aggregate_op` naming convention +#: rather than listing `sql_int_aggregate_op` / `sql_number_aggregate_op` by hand, +#: so a future aggregate variant is covered the moment it is added to `_types.py` +#: without a second edit here. +_SQL_AGGREGATE_OP_CALLS = frozenset( + variant.value for variant in Variant if variant.value.endswith("_aggregate_op") +) + +#: Every ThoughtSpot call name that already aggregates: the native DIRECT catalog +#: templates (derived from the catalog itself, never retyped by hand, via the same +#: `emit_direct` the rest of this package uses to render them — see the task report +#: for why), plus `group_aggregate` and the `sql_*_aggregate_op` pass-through family. +#: This set alone is not the whole safety story — see `_contains_aggregate_call`, +#: which also scans for these names at *any* nesting depth, not only as an +#: expression's own outer call, because the catalog will always hold aggregate +#: constructs beyond whatever a fixed enumeration lists. +_AGGREGATE_CALL_NAMES = ( + frozenset( + formula.split_call(emit_direct(CATALOG[spec], ["x"]))[0].lower() + for spec in _NATIVE_AGGREGATE_SPECS + ) + | {_GROUP_AGGREGATE_CALL} + | _SQL_AGGREGATE_OP_CALLS +) + + +def _outer_call_is_aggregate(expr: str) -> bool: + """Whether `expr`'s own outer call (not something nested inside it) is a + native ThoughtSpot aggregate. + + `formula.split_call` returning `None` — not a single outer call, as in + `[A::x] - [B::y]` — means there is no outer call for it to be one. Matching + is case-insensitive (ThoughtSpot's formula functions are not case-sensitive) + and compares the whole call name as one unit, so a two-word name like + `unique count` is matched by both words together, never by either alone. + + This is the *documented no-op* case: `sum ( [T::x] )` with a column + aggregation of `SUM`, `AVERAGE`, or anything else is a real, common shape — + ThoughtSpot's UI sets an aggregation on a formula column routinely, whether + or not the formula's own expression already aggregates — and discarding a + redundant one here is expected behaviour, not a loss. See `convert_metric` + for why this case stays silent while `_contains_aggregate_call` below (an + aggregate *nested inside*, not as the outer call) is reported. + """ + call = formula.split_call(expr) + if call is None: + return False + name, _args = call + return name.lower() in _AGGREGATE_CALL_NAMES + + +def _contains_aggregate_call(expr: str) -> bool: + """Whether an aggregate call appears anywhere in `expr`, at any nesting depth. + + Checking only `expr`'s own outer call (`_outer_call_is_aggregate`) is not + enough: an aggregate can be buried inside a scalar wrapper the outer call + does not name at all — `round ( sum ( [T::x] ) , 2 )` has `round` as its + outer call, not `sum`, but the expression as a whole still aggregates. + `formula.find_call_names` finds every call at every depth, so this checks + the whole expression rather than the single outer position. Matching is + case-insensitive and compares each call's whole name, exactly as + `_outer_call_is_aggregate` does. + + Used only for the case `_outer_call_is_aggregate` already says `False` for: + see `convert_metric`, where an aggregate nested here (but not as the outer + call) is the one shape worth a warning — the outer-call case is silent by + design, and warning there too would fire on the common, correct case and + train readers to ignore the issue log. + """ + return any(name.lower() in _AGGREGATE_CALL_NAMES for name in formula.find_call_names(expr)) + + +def _compose_aggregate_entries( + inner_expr: str, + aggregation_raw: str, + resolve: Callable[[str, str], str | None], + log: IssueLog, + *, + object_ref: str, +) -> list[dict[str, str]]: + """Dialect entries for a load-bearing column aggregation wrapping `inner_expr`. + + `inner_expr` is `[TABLE::Column]` for a physical column, or the verbatim scalar + `formulas[].expr` text for a formula-backed one — in both cases the text the + column-level `aggregation` rolls up. There is no single TML string that already + represents "this column plus its aggregation": TML records the two as separate + values (the Metric-level `aggregation` row in the construct-mapping document), so + — unlike `expression_entries` — building the THOUGHTSPOT entry here is a genuine + construction, not a reconstruction of something that already existed as one + string. `inner_expr` itself still travels through untouched, inside the wrapper + `emit_direct` builds around it. + + The portable ANSI_SQL sibling is composed the same way, but only when + `inner_expr` itself produces one via `expression_entries` — wrapping a guess + around a non-portable inner expression would be exactly the kind of invented + translation this converter otherwise refuses to emit, and `expression_entries` + already logs why it can't when that happens. + """ + construct = CATALOG[_AGGREGATION_CATALOG_SPEC[aggregation_raw]] + ts_expr = emit_direct(construct, [inner_expr]) + entries: list[dict[str, str]] = [{"dialect": DIALECT, "expression": ts_expr}] + + # This helper only ever composes a metric's aggregation (never a field's), so + # "metric" is hardcoded here rather than threaded through as a parameter. + inner_entries = expression_entries( + inner_expr, resolve, log, object_ref=object_ref, kind="metric" + ) + inner_ansi = next( + (e["expression"] for e in inner_entries if e["dialect"] == PORTABLE_DIALECT), None + ) + if inner_ansi is not None: + if aggregation_raw == "COUNT_DISTINCT": + ansi_expr = f"COUNT(DISTINCT {inner_ansi})" + else: + ansi_expr = f"{_AGGREGATION[aggregation_raw]}({inner_ansi})" + entries.append({"dialect": PORTABLE_DIALECT, "expression": ansi_expr}) + return entries + + +def _metric_datatype( + table_name: str, + column_name: str, + aggregation_raw: str, + table_lookup: Callable[[str], dict | None], + log: IssueLog, + *, + object_ref: str, +) -> str | None: + """The Ossie datatype for a bare-aggregate-over-physical-column metric, or `None`. + + `COUNT` and `COUNT(DISTINCT ...)` always report `Integer`, regardless of the + underlying column's own warehouse type — counting DOUBLEs still counts whole + rows. Every other aggregation (including `NONE`, a bare unaggregated column) + reports the physical column's own mapped type, via the same `_physical_datatype` + a field uses, so the absent-vs-unmapped distinction it already makes (nothing to + log for a column with no declared type; an issue for one whose declared type has + no Ossie mapping) applies here unchanged. + """ + if aggregation_raw in _COUNT_AGGREGATIONS: + return "Integer" + return _physical_datatype( + table_name, column_name, table_lookup, log, object_ref=object_ref, kind="metric" + ) + + +def convert_metric( + column: dict, + formulas: dict[str, dict], + table_lookup: Callable[[str], dict | None], + resolve: Callable[[str, str], str | None], + log: IssueLog, + allocator: identifiers.Allocator | None = None, +) -> dict | None: + """Convert one Model `columns[]` entry into an Ossie metric, or `None`. + + `column` is a ThoughtSpot Model `columns[]` entry; only `column_type: MEASURE` + becomes a metric — an `ATTRIBUTE` column belongs to `convert_field` instead, and + building a metric for one here would produce two competing Ossie objects + surfacing the same TML column. + + As with `convert_field`, a physical column carries `column_id` (`TABLE::Column + Name`) and a computed one carries `formula_id`, resolved against `formulas` + (keyed by each entry's `id`) exactly the same way. Either shape composes with + `properties.aggregation` per the Metric-level `aggregation` row: load-bearing on + a `column_id` metric and on a *scalar* formula (the two compose into + `AGG()`). Two different shapes are a no-op instead, and only one + of them is reported: + + - The formula's own outer call already aggregates (`sum ( ... )`, + `group_aggregate ( ... )`, ...). This is the documented, common case — + ThoughtSpot's UI sets a column aggregation on a formula column routinely, + redundant or not — so the column-level value is discarded silently, without + logging anything. A warning here would fire on a large fraction of ordinary, + correct metrics and teach readers to stop reading the issue log. + - An aggregate is *nested* inside a still-scalar outer call + (`round ( sum ( ... ) , 2 )` — `round` is scalar, `sum` is buried one level + in). Composing here would silently double-aggregate an already-reduced + value, exactly as the first case would, but this is the one shape where a + reader might reasonably expect composition and not get it — so it is + reported. + + See `_outer_call_is_aggregate` and `_contains_aggregate_call` for how the two + are told apart, and the module docstring for why detection and composition + share one source. + + An unrecognised `aggregation` value (not one of TML's documented enum members) + is treated as `NONE` and logged — the value was present and could not be + understood, which is a loss worth reporting, unlike an absent `aggregation` key, + which defaults to `NONE` silently. + + Metrics have no `label` field (unlike fields): when identifier normalisation changes + the identifier, the exact ThoughtSpot display name is stashed as `tml_name` + rather than carried in a dedicated field. + + Which of the three TML shapes produced this metric — `column_aggregation`, + `scalar_formula_plus_aggregation`, or `formula` — is stashed as `shape`, so a + return trip can reproduce the source shape instead of collapsing all three + into one. `formula` is omitted rather than written: it is also what a + document with no stash defaults to on the way back, so writing it would + change nothing about the reconstruction while making the payload heavier. + + `allocator` is `convert_field`'s own parameter, mirrored here — see its + docstring. Metrics are model-scoped in Ossie (unlike fields, scoped per + dataset), so `convert()` shares a *different* `Allocator` across metrics + than it does across fields; a caller that omits it gets a fresh, private + one, correct for a call that only ever converts one column. + """ + properties = column.get("properties") or {} + if properties.get("column_type") != "MEASURE": + return None + if allocator is None: + allocator = identifiers.Allocator() + + display_name = column["name"] + object_ref = f"metric:{display_name}" + + aggregation_raw = properties.get("aggregation", "NONE") + if aggregation_raw not in _AGGREGATION: + log.add( + code="TS-METRIC-AGGREGATION-UNKNOWN", + severity=Severity.WARNING, + message=( + f"column {display_name!r} has aggregation {aggregation_raw!r}, which " + f"is not one of the TML aggregation values this converter recognises; " + f"treated as NONE" + ), + object_ref=object_ref, + ) + aggregation_raw = "NONE" + aggregation = _AGGREGATION[aggregation_raw] + + if "column_id" in column: + metric_shape = METRIC_SHAPE_COLUMN_AGGREGATION + table_name, column_name = identifiers.split_column_ref(f"[{column['column_id']}]") + metric_name = _field_or_metric_identifier( + display_name, + _physical_db_column_name(table_name, column_name, table_lookup), + allocator, log, kind="metric", object_ref=object_ref, + ) + metric: dict = {"name": metric_name} + field_ref = identifiers.format_column_ref(table_name, column_name) + if aggregation is None: + dialects = expression_entries( + field_ref, resolve, log, object_ref=object_ref, kind="metric" + ) + else: + dialects = _compose_aggregate_entries( + field_ref, aggregation_raw, resolve, log, object_ref=object_ref + ) + metric["expression"] = {"dialects": dialects} + datatype = _metric_datatype( + table_name, column_name, aggregation_raw, table_lookup, log, object_ref=object_ref + ) + if datatype is not None: + metric["datatype"] = datatype + elif "formula_id" in column: + formula_id = column["formula_id"] + formula_entry = formulas.get(formula_id) + if formula_entry is None: + log.add( + code="TS-METRIC-FORMULA-MISSING", + severity=Severity.WARNING, + message=( + f"column {display_name!r} has formula_id {formula_id!r}, which " + f"matches no formulas[] entry; no metric can be built" + ), + object_ref=object_ref, + ) + return None + if "expr" not in formula_entry: + log.add( + code="TS-METRIC-FORMULA-MISSING", + severity=Severity.WARNING, + message=( + f"column {display_name!r} has formula_id {formula_id!r}, whose " + f"formulas[] entry has no expr; no metric can be built" + ), + object_ref=object_ref, + ) + return None + expr = formula_entry["expr"] + metric_name = _field_or_metric_identifier( + display_name, None, allocator, log, kind="metric", object_ref=object_ref, + ) + metric: dict = {"name": metric_name} + if aggregation is None: + # Nothing to compose: the verbatim expr, untouched, is the whole metric. + metric_shape = METRIC_SHAPE_FORMULA + dialects = expression_entries( + expr, resolve, log, object_ref=object_ref, kind="metric" + ) + elif _outer_call_is_aggregate(expr): + # The documented no-op: the expression's own call already + # aggregates (sum ( ... ), group_aggregate ( ... ), ...), and + # ThoughtSpot's UI sets a column aggregation on a formula column + # like this routinely, whether or not it is redundant. Discarding + # it here is expected, not a loss, so nothing is logged -- + # warning on this common, correct shape would train readers to + # ignore the issue log entirely. + metric_shape = METRIC_SHAPE_FORMULA + dialects = expression_entries( + expr, resolve, log, object_ref=object_ref, kind="metric" + ) + elif _contains_aggregate_call(expr): + # Not the outer call, but an aggregate is nested somewhere inside + # (round ( sum ( ... ) , 2 ), or a sql_*_aggregate_op pass-through + # buried in a larger expression). This is the one shape where a + # reader might reasonably expect composition and not get it, so + # it is the one shape worth telling them about: composing here + # would silently double-aggregate an already-reduced value. + log.add( + code="TS-METRIC-AGGREGATION-ALREADY-AGGREGATED", + severity=Severity.WARNING, + message=( + f"column {display_name!r}'s expression already contains an " + f"aggregate; the column-level aggregation {aggregation_raw!r} " + f"was ignored to avoid double-aggregating" + ), + object_ref=object_ref, + ) + metric_shape = METRIC_SHAPE_FORMULA + dialects = expression_entries( + expr, resolve, log, object_ref=object_ref, kind="metric" + ) + else: + # A genuinely scalar expr: the column aggregation is load-bearing, so + # compose it. + metric_shape = METRIC_SHAPE_SCALAR_FORMULA_PLUS_AGGREGATION + dialects = _compose_aggregate_entries( + expr, aggregation_raw, resolve, log, object_ref=object_ref + ) + metric["expression"] = {"dialects": dialects} + # A formula carries no declared type anywhere in TML — neither columns[] nor + # formulas[] has a data_type key — so datatype is always omitted + # here, and never logged: there was never a value here to lose. + else: + log.add( + code="TS-METRIC-NO-SOURCE", + severity=Severity.WARNING, + message=( + "column has neither a physical column_id nor a formula_id; " + "no metric can be built" + ), + object_ref=object_ref, + ) + return None + + stash_payload: dict = {} + if metric_name != display_name: + stash_payload[STASH_TML_NAME] = display_name + if metric_shape != METRIC_SHAPE_FORMULA: + stash_payload[METRIC_STASH_SHAPE] = metric_shape + metric = _write_stash_safely(metric, stash_payload, log, object_ref) + + description = column.get("description") + if description: + metric["description"] = description + + ai_context = _ai_context(properties) + if ai_context is not None: + metric["ai_context"] = ai_context + + return metric + + +# --------------------------------------------------------------------------- +# Assembly: datasets, the cross-model resolver, relationships, and convert() +# --------------------------------------------------------------------------- +# +# Everything above converts one column at a time and takes `resolve` as a +# given. Nothing above can build `resolve` itself -- it maps a raw TML +# reference to "dataset.field", and that mapping cannot exist until every +# dataset in the model is known. That is the one job only this section can +# do, and everything else here exists to support it: datasets have to be +# built first (so their names -- the model_tables[] alias-or-name, never +# normalised -- are known), then `resolve` is a closure over that, then +# fields and metrics are converted through it, then joins become +# relationships, then keys are derived from the relationships that qualify. +# +# A malformed reference anywhere in a TML document (an ambiguous `column_id` +# or join condition -- `identifiers.split_column_ref` raising on purpose +# rather than mis-splitting) is caught per object: the object is skipped, an +# issue names it, and the rest of the model still converts. Nothing here lets +# one bad reference abort the whole conversion. + + +def _index_attribute_columns( + columns: list[dict], log: IssueLog +) -> set[tuple[str, str]]: + """`{(TABLE, physical column display name)}`, for every ATTRIBUTE + `columns[]` entry bound to a valid physical `column_id`. + + This is the set `resolve()` gates cross-references against: a bare + `[TABLE::Column]` reference is portable only when it names a column the + model actually surfaces as a field. It has to be built ahead of Phase 3 + (fields and metrics) because a formula processed early in that phase can + reference a column defined later in the same `columns[]` list -- the gate + needs to already know about every ATTRIBUTE column, not just the ones + converted so far. + + Membership only -- no identifier value. An earlier revision stored + `identifiers.normalise(column["name"])` as the value here, on the theory + that `resolve()`'s caller needed to know the field's actual identifier + up front. It didn't: every real consumer of the value only ever tested + membership (see `resolve()` in `convert()`), and the one place that did + read the *value* (Phase 3.5's `sql_output_columns` stash) now reads + `built_field_names`, populated from the field `convert_field` actually + built, in `convert()`'s Phase 3 -- the one place that identifier is + truly known, rather than a second, independent recomputation of it here + that had to somehow stay in exact sync. That sync was already fragile + (see the note on `convert_field`'s own `allocator` parameter) and it + silently broke the moment a column's display name failed to normalise: + this index used to drop such a column from the gate entirely, on the + theory that `convert_field` would drop the field too -- true before + `convert_field` grew a fallback identifier, and a real correctness bug + once it did, because a cross-reference to that column then resolved to + nothing even though the field genuinely exists, and the column's own + physical entry was reported as unsurfaced despite having a real field. + Tracking membership only, independent of how the identifier is derived, + closes both without needing the two call sites to agree on anything. + + A malformed `column_id` is caught here, per column, rather than aborting + the whole model: the column is left out of the index -- any expression + that references it resolves to nothing, which every caller already + treats as an ordinary unresolved reference -- and an issue names it. + """ + index: set[tuple[str, str]] = set() + for column in columns: + properties = column.get("properties") or {} + if properties.get("column_type") != "ATTRIBUTE": + continue + column_id = column.get("column_id") + if not column_id: + continue + try: + table_name, physical_name = identifiers.split_column_ref(f"[{column_id}]") + except ValueError as exc: + log.add( + code="TS-COLUMN-ID-MALFORMED", + severity=Severity.WARNING, + message=( + f"column {column.get('name', '')!r} has a malformed " + f"column_id {column_id!r} ({exc}); it cannot be resolved by any " + f"expression that references it" + ), + object_ref=f"field:{column.get('name', '')}", + ) + continue + index.add((table_name, physical_name)) + return index + + +def _raw_physical_columns(body: dict, kind: str) -> list[dict]: + """The verbatim physical-column list for a Table or SQL View document -- + `columns[]` for a `table:`, `sql_view_columns[]` for a `sql_view:`. + + Per the mapping document's SQL View row, a SQL View's columns live under + a different key entirely, not merely a differently-shaped entry under + the same one -- reading `.get("columns")` unconditionally finds nothing + on a SQL View document and every one of its columns silently vanishes + (no datatype, no unsurfaced_columns entry, nothing). This is the single + place that knows which key each kind uses; every reader of "this + dataset's physical columns" goes through here or through + `_normalized_physical_columns` below, never `body.get("columns")` directly. + """ + key = "sql_view_columns" if kind == "sql_view" else "columns" + return body.get(key) or [] + + +def _normalize_physical_column(entry: dict, kind: str) -> dict: + """One physical column entry, reshaped so datatype lookup + (`_physical_datatype` in the field/metric converters) can read `name` / + `db_column_name` / `db_column_properties` the same way regardless of + which document kind it came from. + + A Table column already has exactly this shape. A SQL View column binds + its physical reference via `sql_output_column` instead of + `db_column_name` -- "each bound to a query output alias via + sql_output_column", per the mapping document -- but is otherwise + documented as playing the same role, so `name` and + `db_column_properties` carry over unchanged. + """ + if kind != "sql_view": + return entry + return { + "name": entry.get("name"), + "db_column_name": entry.get("sql_output_column"), + "db_column_properties": entry.get("db_column_properties"), + } + + +def _normalized_physical_columns(body: dict, kind: str) -> list[dict]: + """`_raw_physical_columns`, each entry passed through + `_normalize_physical_column` -- the shape `table_lookup` hands to + `_physical_datatype`.""" + return [_normalize_physical_column(entry, kind) for entry in _raw_physical_columns(body, kind)] + + +#: The TML `db_column_properties.data_type` spelling `datatypes.to_tml` would +#: emit by default for each Ossie datatype whose TML source has more than one +#: valid spelling (the datatype map's Boolean and Float rows). Stashing the +#: canonical spelling itself would be noise -- the reverse direction's own +#: default already produces it; only the non-canonical spelling (`BOOL`, +#: `FLOAT`) is worth recording. +_CANONICAL_TML_SPELLING = {"Boolean": "BOOLEAN", "Float": "DOUBLE"} + + +def _physical_column_stash( + column_id: str, + ossie_datatype: str | None, + physical_columns_by_prefix: dict[str, list[dict]], + dataset_stashes: dict[str, dict], +) -> dict: + """Field/metric-level stash additions a physical column needs that + nothing else in this module records: + + * `data_type` -- the exact warehouse spelling, only when it is not the + canonical one `datatypes.to_tml` would emit by default for + `ossie_datatype` (see `_CANONICAL_TML_SPELLING`). Documented in the + datatype map's Boolean row: "the connection's spelling is recorded in + the field stash's data_type key so the return trip re-emits the same + one" -- the same reasoning applies to Float's DOUBLE/FLOAT pair. + + * `db_column_name` -- the exact warehouse column name, only when it + differs from the column's own display name. The forward direction + matches a physical column by display name only, so a round-tripped + bracket reference (`[TABLE::Column]`) carries the display name, never + the warehouse name, and the reverse direction has no other way to + recover it -- today it defaults to assuming the two are equal and + logs that assumption. This key is not in the pinned payload schema; + it closes a gap the schema itself does not yet cover. Only ever + stashed for a Table-backed column: a SQL View's own physical binding + (`sql_output_column`) already has its own dataset-level stash key + (`sql_output_columns`), so recording the same fact again here under a + different name would be redundant. + """ + try: + table_name, physical_name = identifiers.split_column_ref(f"[{column_id}]") + except ValueError: + return {} + physical = next( + (p for p in physical_columns_by_prefix.get(table_name, []) if p.get("name") == physical_name), + None, + ) + if physical is None: + return {} + + payload: dict = {} + is_table = dataset_stashes.get(table_name, {}).get(DATASET_STASH_TML_OBJECT) != "sql_view" + db_column_name = physical.get("db_column_name") + if is_table and db_column_name is not None and db_column_name != physical_name: + payload[FIELD_STASH_DB_COLUMN_NAME] = db_column_name + # The witness: the column's own display name (the bracket's column + # part) this warehouse name was recorded against, so the reverse + # direction can tell whether the field still names the same + # physical column before trusting a warehouse name that may + # describe a different one now. + payload[FIELD_STASH_DB_COLUMN_NAME_WITNESS] = physical_name + + raw_data_type = (physical.get("db_column_properties") or {}).get("data_type") + canonical = _CANONICAL_TML_SPELLING.get(ossie_datatype) if ossie_datatype else None + if raw_data_type is not None and canonical is not None and raw_data_type != canonical: + payload[FIELD_STASH_DATA_TYPE] = raw_data_type + # The witness: the Ossie datatype this spelling was derived from, so + # the reverse direction can tell a genuine edit (the field now + # declares a different datatype) from an unedited round trip before + # trusting a warehouse-specific spelling for a type it may no longer + # describe. + payload[FIELD_STASH_DATA_TYPE_WITNESS] = ossie_datatype + + return payload + + +#: Every `properties` key `convert_field` reads on the ATTRIBUTE path. +#: Anything else in a column's `properties` dict is unconsumed and, per the +#: fail-closed rule `_unconsumed_properties` implements, is stashed rather +#: than silently dropped. +_FIELD_CONSUMED_PROPERTIES = frozenset({"column_type", "synonyms", "ai_context"}) + +#: Same, for `convert_metric`'s MEASURE path -- one key more than the field +#: set: `aggregation` is load-bearing only for a metric. +_METRIC_CONSUMED_PROPERTIES = _FIELD_CONSUMED_PROPERTIES | {"aggregation"} + + +#: Identity-shaped keys that must never reach the portable document at any +#: depth -- broader than `stash._FORBIDDEN_KEYS` (`guid`/`obj_id`/ +#: `fqn`, which `stash.write_stash` scans every payload for regardless of +#: caller). `_unconsumed_properties` is the one place in this module that +#: copies a property's *value* wholesale rather than rebuilding it field by +#: field, so it is also the one place the two further identity keys this +#: converter also tracks -- `dataset_id`, and `geo_config. +#: custom_file_guid` naming a custom map -- are worth checking for +#: specifically, ahead of `write_stash`'s own narrower check: the scan is +#: `stash.find_forbidden_key`'s, shared rather than reimplemented here, only +#: the wider vocabulary to check it against is local to this one call site. +_DEEP_IDENTITY_KEYS = stash._FORBIDDEN_KEYS | {"dataset_id", "custom_file_guid"} + + +def _unconsumed_properties( + properties: dict, consumed: frozenset[str], log: IssueLog, object_ref: str +) -> dict: + """Every key in a column's `properties` dict that the converter did not + read, minus anything carrying instance-local identity at any + depth. + + Deliberately the complement of `consumed`, not an enumeration of the + ThoughtSpot-only property names this module happens to know about today + (`index_type`, `value_casing`, ...): an enumeration silently drops the + next property ThoughtSpot adds, where the complement preserves it and is + correct by construction. `consumed` is what `convert_field`/ + `convert_metric` actually read, reused here rather than duplicated, so + the two lists cannot drift apart the way two independently maintained + ones could. + + A property whose value contains a forbidden key anywhere inside it is + dropped here -- with a WARNING logged naming it, so a per-column loss + stays a survivable one rather than the hard `ConversionError` + `stash.write_stash` would otherwise raise for it -- rather than + aborting the whole column's conversion over one contaminated property. + `write_stash` still re-checks (against its own narrower vocabulary) + whatever reaches it, so this is a caller earning its place with a softer + landing for a known case, not the only thing standing between identity + content and the output. + """ + remainder: dict = {} + for key, value in properties.items(): + if key in consumed: + continue + # Wrapping `{key: value}` rather than scanning `value` alone catches + # both shapes in one call: the property's own name being forbidden + # (a scalar `properties: {"guid": "..."}`, unlikely but not ruled + # out) and a forbidden key nested inside its value. + if stash.find_forbidden_key({key: value}, _DEEP_IDENTITY_KEYS) is not None: + log.add( + code="TS-PROPERTY-IDENTITY-DROPPED", + severity=Severity.WARNING, + message=( + f"property {key!r} contains instance-local identity " + f"content; it is dropped rather than carried into the " + f"portable document" + ), + object_ref=object_ref, + ) + continue + remainder[key] = value + return remainder + + +def _write_stash_safely(obj: dict, payload: dict, log: IssueLog, object_ref: str) -> dict: + """`stash.write_stash(obj, payload)`, catching its identity guard and turning a + would-be hard failure into a survivable, logged drop. + + The payload content this module stashes is TML data read out of a + source file, not something the converter itself constructed -- an + identity key surfacing somewhere inside it is expected input, not a + programming error, and expected input must not abort the whole + conversion the way every other loss in this module does not. The guard + itself still lives at `stash.write_stash`, and still raises: that is + what makes it impossible to bypass, present caller or future one. This + is the one place that catches the raise and keeps going, generalising + the same choice `_unconsumed_properties` already makes for column + properties to every other stash site, rather than repeating a bespoke + pre-filter at each one. + + A payload can have more than one contaminated top-level key, so this + retries after removing one at a time rather than assuming a single + pass suffices. If `stash.write_stash` ever raises for a reason other + than a forbidden key found in `payload` itself (a malformed *existing* + stash entry on `obj`, surfaced via its internal `read_stash` call, is + the one other case it can raise for) there is no payload key to blame, + and the exception is left to propagate rather than being swallowed. + + Every call to `stash.write_stash` in this module goes through this + function -- including `convert_metric`'s own `tml_name`/`shape` payload, + which is hardcoded scalars today and so never actually exercises the + catch, but a future change that puts TML-derived content into it would + otherwise silently reinstate the abort-the-whole-conversion behaviour + this function exists to remove. A new call to `stash.write_stash` + added anywhere in this module should be a call to this function instead, + not a second bespoke exception. + """ + cleaned = dict(payload) + while True: + try: + return stash.write_stash(obj, cleaned) + except ConversionError: + offender = next( + ( + key for key, value in cleaned.items() + if stash.find_forbidden_key({key: value}) is not None + ), + None, + ) + if offender is None: + raise + log.add( + code="TS-STASH-IDENTITY-DROPPED", + severity=Severity.WARNING, + message=( + f"stash field {offender!r} contains instance-local identity " + f"content; it is dropped rather than carried into the " + f"portable document" + ), + object_ref=object_ref, + ) + del cleaned[offender] + + +def _field_owner_dataset( + column: dict, formulas: dict[str, dict], resolve: Callable[[str, str], str | None] +) -> str | None: + """Which dataset a *successfully built* field belongs in. + + Only ever called after `convert_field` has already returned a non-`None` + field for this exact column, which makes every path here provably safe: + the physical branch re-parses the same `column_id` `convert_field` just + parsed without raising, and the formula branch re-runs `attribute_dataset` + on the same expression `convert_field` just attributed successfully -- + and `attribute_dataset`'s success path never logs (only its failure paths + do), so repeating it here adds nothing to the issue log. + """ + if "column_id" in column: + table_name, _column_name = identifiers.split_column_ref(f"[{column['column_id']}]") + return table_name + formula_entry = formulas.get(column.get("formula_id")) + if formula_entry is None or "expr" not in formula_entry: + return None + return attribute_dataset(formula_entry["expr"], resolve, IssueLog(), object_ref="") + + +def _build_dataset(prefix: str, entry: dict, table_doc, log: IssueLog) -> tuple[dict, dict]: + """One `model_tables[]` entry, paired with its Table/SQL-View document, + into `(base Ossie dataset dict, its custom_extensions[THOUGHTSPOT] payload)`. + + The base dict carries `name`/`source`/`description` only -- no `fields` + key yet. The caller fills that in once every dataset (and therefore the + resolver) exists, and calls `stash.write_stash` with the returned payload + once fields are attached, so key order in the final dict reads naturally + even though this function runs long before fields are known. + + `prefix` becomes the dataset's Ossie `name` verbatim: `entry["alias"]` + when present, else `entry["name"]`, never run through + `identifiers.normalise` -- the Dataset-level mapping requires it to match + the model_tables[] reference name exactly, case-sensitive, since that is + also the prefix every `column_id`/join reference in this dataset uses. + """ + body = table_doc.body + table_ref = entry.get("name") + alias = entry.get("alias") + kind = table_doc.kind + + ds_stash: dict = {DATASET_STASH_TML_OBJECT: kind} + if alias: + ds_stash[DATASET_STASH_ALIAS] = alias + ds_stash[DATASET_STASH_TABLE_NAME] = table_ref + + connection_name = (body.get("connection") or {}).get("name") + if connection_name: + ds_stash[DATASET_STASH_CONNECTION_NAME] = connection_name + + if kind == "sql_view": + # Not separately stashed: `source` (below, the dataset's own live + # field) already carries this same query text, so a stash entry + # here would be a pure duplicate nothing ever reads back. + source = body.get("sql_query") or "" + else: + db = body.get("db") or "" + schema = body.get("schema") or "" + db_table = body.get("db_table") or table_ref or "" + if any("." in part for part in (db, schema, db_table)): + # A dotted source string would be ambiguous -- keep the parts too. + ds_stash[DATASET_STASH_SOURCE_PARTS] = { + DATASET_STASH_SOURCE_PARTS_DB: db, + DATASET_STASH_SOURCE_PARTS_SCHEMA: schema, + DATASET_STASH_SOURCE_PARTS_DB_TABLE: db_table, + } + source = ".".join((db, schema, db_table)) + + # The witness for DATASET_STASH_TML_OBJECT: the same `source` about to + # be written onto the dataset itself. Ossie -> TML compares its own + # current `source` against this snapshot before trusting the stashed + # kind -- a `source` rewritten from a query to a table reference (or + # back) since this was written makes the stashed kind stale. + ds_stash[DATASET_STASH_TML_OBJECT_WITNESS] = source + + dataset: dict = {"name": prefix, "source": source} + description = body.get("description") + if description: + dataset["description"] = description + + if body.get("rls_rules"): + # Row-level security policy is instance-local (it names groups + # that only exist on the source instance) and is never carried into + # the portable document. Per ThoughtSpot domain review this is now + # the primary RLS mechanism customers are migrating onto, so this is + # an ERROR-severity issue naming the table, not a quiet declared loss. + log.add( + code="TS-DATASET-RLS-RULES", + severity=Severity.ERROR, + message=( + f"table {table_ref!r} has row-level security rules (rls_rules); " + f"these reference instance-local groups and are not carried into " + f"the portable document -- data that was previously restricted is " + f"unrestricted until row-level security is reapplied on the " + f"target instance" + ), + object_ref=f"dataset:{prefix}", + remedy=( + "Reapply the table's row-level security rules manually on the " + "target instance after import." + ), + ) + + return dataset, ds_stash + + +#: One equality pair, and nothing but: two bracketed references either side of +#: a bare `=`. Anything else -- `>=`/`>`/`<`/`<=`, a literal on either side, or +#: a genuine `=` between something that isn't two whole `[TABLE::Column]` +#: references -- does not match, and is therefore a residual predicate. +_EQUALITY_PAIR_RE = re.compile(r"^\s*(\[[^\]]+\])\s*=\s*(\[[^\]]+\])\s*$") +_AND_RE = re.compile(r"\band\b", re.IGNORECASE) + + +def _split_top_level_and(text: str) -> list[str]: + """Split a join condition on its top-level ` and ` operators. + + Reuses `formula._scan` -- the same quote/bracket-depth tracker every + other reference-aware split in this package is built on -- so a literal + "and" inside a quoted literal, or inside a `[TABLE::Column]` body (a + table or column display name can genuinely contain the word, e.g. + `[Research and Development::Col]`), is never mistaken for the boolean + operator. An empty or whitespace-only `text` yields no parts. + """ + if not text or not text.strip(): + return [] + context = {i: (depth, in_quote) for i, _ch, depth, in_quote in formula._scan(text)} + parts: list[str] = [] + start = 0 + for match in _AND_RE.finditer(text): + depth, in_quote = context.get(match.start(), (0, False)) + if depth == 0 and not in_quote: + parts.append(text[start : match.start()].strip()) + start = match.end() + parts.append(text[start:].strip()) + return [p for p in parts if p] + + +def _parse_join_condition( + on_expression: str, from_prefix: str, to_prefix: str +) -> tuple[list[tuple[str, str]], list[str]]: + """Split a join condition into equality pairs and residual predicates. + + Per the mapping document's *Non-equality joins* section: the condition is + split on its top-level `and`s; a part that is exactly `[FROM::a] = [TO::x]` + (in either orientation -- the equality is symmetric in TML, so the pair is + reoriented to `(from_col, to_col)` regardless of which side of `=` each + reference was written on) becomes one pair. Everything else -- `>=`, `>`, + `<`, `<=`, a comparison against a literal, or an equality naming some + table other than `from_prefix`/`to_prefix` -- is a residual predicate, + kept verbatim. + + Raises `ValueError` (via `identifiers.split_column_ref`) on an ambiguous + column reference. The caller (`_relationship_from_join`) catches this per + relationship rather than letting it abort the whole conversion. + """ + equality_pairs: list[tuple[str, str]] = [] + residuals: list[str] = [] + for part in _split_top_level_and(on_expression): + match = _EQUALITY_PAIR_RE.match(part) + if match is None: + residuals.append(part) + continue + left_table, left_column = identifiers.split_column_ref(match.group(1)) + right_table, right_column = identifiers.split_column_ref(match.group(2)) + if left_table == from_prefix and right_table == to_prefix: + equality_pairs.append((left_column, right_column)) + elif left_table == to_prefix and right_table == from_prefix: + equality_pairs.append((right_column, left_column)) + else: + # An equality pair, but not one naming both sides of *this* join -- + # cannot be expressed as one of its from_columns/to_columns. + residuals.append(part) + return equality_pairs, residuals + + +def _unrepresentable_entry( + from_prefix: str, + to_prefix: str, + on_expression: str, + join_type: str | None, + cardinality: str | None, + join_shape: str, + referencing_join: str | None, +) -> dict: + """One `unrepresentable_joins[]` entry -- everything schema-required, plus + whatever else about the join is known, verbatim.""" + entry: dict = { + "from": from_prefix, + "to": to_prefix, + RELATIONSHIP_STASH_ON_EXPRESSION: on_expression, + RELATIONSHIP_STASH_JOIN_SHAPE: join_shape, + } + if join_type: + entry[RELATIONSHIP_STASH_TYPE] = join_type + if cardinality: + entry[RELATIONSHIP_STASH_CARDINALITY] = cardinality + if referencing_join: + entry[RELATIONSHIP_STASH_REFERENCING_JOIN] = referencing_join + return entry + + +def _relationship_from_join( + *, + name: str, + from_prefix: str, + to_prefix: str, + on_expression: str | None, + join_type: str | None, + cardinality: str | None, + join_shape: str, + referencing_join: str | None, + log: IssueLog, +) -> tuple[dict | None, dict | None, bool]: + """One join -> `(relationship, unrepresentable_entry, has_residual_predicates)`. + + Exactly one of `relationship`/`unrepresentable_entry` is non-`None` (or + both `None` when there is no condition at all to report). Implements the + *Non-equality joins* table: at least one equality pair emits a + `Relationship`, with any residual predicates riding along in its own + `custom_extensions` rather than withholding the relationship; zero + equality pairs -- including when the condition could not be parsed at all + -- emits nothing, because Ossie's schema requires `from_columns`/ + `to_columns` non-empty, and the condition goes to the model-scope + `unrepresentable_joins` stash instead. + + A `ONE_TO_MANY` join additionally has its endpoints swapped once a + `Relationship` is built -- see the comment at the swap site for why. An + `unrepresentable_entry` is never swapped: it carries no live Ossie + Relationship object of its own for the spec's from/to convention to + apply to, so `from`/`to` there stay exactly TML's own, unswapped. + """ + object_ref = f"relationship:{name}" + if not on_expression or not on_expression.strip(): + log.add( + code="TS-JOIN-NO-CONDITION", + severity=Severity.WARNING, + message=( + f"join {name!r} from {from_prefix!r} to {to_prefix!r} has no " + f"condition; it cannot be represented as a relationship" + ), + object_ref=object_ref, + ) + return None, None, False + + try: + equality_pairs, residuals = _parse_join_condition(on_expression, from_prefix, to_prefix) + except ValueError as exc: + log.add( + code="TS-JOIN-MALFORMED", + severity=Severity.WARNING, + message=( + f"join {name!r} condition {on_expression!r} could not be parsed " + f"({exc}); it is preserved verbatim as an unrepresentable join " + f"rather than as a relationship" + ), + object_ref=object_ref, + ) + entry = _unrepresentable_entry( + from_prefix, to_prefix, on_expression, join_type, cardinality, + join_shape, referencing_join, + ) + return None, entry, False + + if not equality_pairs: + log.add( + code="TS-JOIN-UNREPRESENTABLE", + severity=Severity.WARNING, + message=( + f"join {name!r} from {from_prefix!r} to {to_prefix!r} has no " + f"equality pair in its condition ({on_expression!r}); Ossie requires " + f"from_columns/to_columns to be non-empty, so no relationship is " + f"emitted for it" + ), + object_ref=object_ref, + ) + entry = _unrepresentable_entry( + from_prefix, to_prefix, on_expression, join_type, cardinality, + join_shape, referencing_join, + ) + return None, entry, False + + relationship: dict = { + "name": name, + "from": from_prefix, + "to": to_prefix, + "from_columns": [pair[0] for pair in equality_pairs], + "to_columns": [pair[1] for pair in equality_pairs], + } + + # core-spec/spec.yaml requires a Relationship's `from` to name the many + # side and `to` the one side. TML's own `from`/`to` -- the model_tables[] + # entry a join is declared under, and its `with`/`destination` target -- + # do not themselves encode which side is which; `cardinality` does, and + # ONE_TO_MANY is the one value where TML's `from` names the one side and + # `to` names the many side: the wrong way around for Ossie's spec. So a + # ONE_TO_MANY join's endpoints are swapped here to compensate; MANY_TO_ONE + # and ONE_TO_ONE are already oriented correctly and are left alone. + endpoints_swapped = cardinality == "ONE_TO_MANY" + if endpoints_swapped: + relationship["from"], relationship["to"] = relationship["to"], relationship["from"] + relationship["from_columns"], relationship["to_columns"] = ( + relationship["to_columns"], relationship["from_columns"] + ) + if join_shape == "inline": + # An inline join's name is synthesized (TML's inline syntax has + # no name field), so it is re-derived from the swapped from/to -- + # reading the same self-describing "{many}_to_{one}" way every + # other relationship's derived name already does. A `referencing` + # (or hybrid) shape's name is the Table joins_with[] entry's own + # identifier, unrelated to from/to naming, and is left as-is. + relationship["name"] = f"{relationship['from']}_to_{relationship['to']}" + name = relationship["name"] + object_ref = f"relationship:{name}" + + rel_stash: dict = {RELATIONSHIP_STASH_JOIN_SHAPE: join_shape} + if join_type: + rel_stash[RELATIONSHIP_STASH_TYPE] = join_type + if cardinality: + rel_stash[RELATIONSHIP_STASH_CARDINALITY] = cardinality + if referencing_join: + rel_stash[RELATIONSHIP_STASH_REFERENCING_JOIN] = referencing_join + if endpoints_swapped: + rel_stash[RELATIONSHIP_STASH_ENDPOINTS_SWAPPED] = True + # The witness: from/to/from_columns/to_columns exactly as emitted + # above (i.e. already swapped), so the reverse direction can tell + # whether the relationship has been retargeted since this stash was + # written before undoing the swap to recover TML's original from/to. + rel_stash[RELATIONSHIP_STASH_ENDPOINTS_SWAPPED_WITNESS] = [ + relationship["from"], relationship["to"], + relationship["from_columns"], relationship["to_columns"], + ] + has_residuals = bool(residuals) + if has_residuals: + # The residual predicates themselves are not stashed separately: they + # are already fully contained in the verbatim on_expression stashed + # below, and nothing reads them back on the way to TML. + rel_stash[RELATIONSHIP_STASH_ON_EXPRESSION] = on_expression + # The witness: from_columns/to_columns exactly as emitted above, so + # the reverse direction can tell whether the relationship has been + # retargeted since this stash was written before trusting the + # verbatim on_expression (and the residual narrowing riding with it). + rel_stash[RELATIONSHIP_STASH_ON_EXPRESSION_WITNESS] = [ + relationship["from_columns"], relationship["to_columns"], + ] + log.add( + code="TS-JOIN-RESIDUAL-PREDICATES", + severity=Severity.WARNING, + message=( + f"relationship {name!r} carries residual predicate(s) beyond its " + f"equality pairs; a consumer that reads only from_columns/" + f"to_columns will join more rows than ThoughtSpot does" + ), + object_ref=object_ref, + ) + relationship = _write_stash_safely(relationship, rel_stash, log, object_ref) + return relationship, None, has_residuals + + +def _convert_join( + from_prefix: str, join: dict, from_table_body: dict, known_datasets: frozenset[str], + log: IssueLog, +) -> tuple[dict | None, dict | None, keys.Relationship | None]: + """One `model_tables[].joins[]` entry -> `(relationship, + unrepresentable_entry, key_candidate)`. + + Handles both TML join shapes: `inline` (fully defined here -- `with`/ + `on`/`type`/`cardinality`) and `referencing` (`referencing_join` names an + entry in the *Table*'s own `joins_with[]`, which supplies `destination`/ + `on`/`type`/`cardinality`; a `type`/`cardinality` also present on this + entry overrides the Table's and marks the shape + `referencing_with_inline_attrs`, the real hybrid the 2026-07-30 census + found on 12 of 493 joins). + + `known_datasets` is checked against the resolved target the same way the + caller already checks the source before calling this at all: a target + naming a dataset this model never built -- a table document missing, a + duplicate alias, or simply a typo -- is dropped with a WARNING rather + than emitted. Upstream's own validator hard-fails a document with a + relationship pointing at an unknown dataset (`exit 1`, not a warning), + so emitting one anyway would make the *whole* document unusable by any + downstream tool that runs it; dropping the one broken relationship keeps + everything else in the model valid and usable, which is the more useful + failure of the two. + + The cardinality-orientation rule itself is applied inside + `_relationship_from_join`, not here: a `ONE_TO_MANY` join's endpoints are + swapped there so the *emitted* relationship's `from`/`to` always lands on + the many-side/one-side arrangement core-spec/spec.yaml requires, + regardless of which side TML happened to declare the join from. Because + that swap already happened by the time `relationship` comes back here, + the key candidate handed to `keys.derive_keys` is read directly off the + (possibly-swapped) `relationship["to"]`/`relationship["to_columns"]` -- + for a swapped `ONE_TO_MANY` join this is TML's original FROM side (now + the relationship's `to`), which is exactly the side a key belongs to; + for every other cardinality it is unchanged from before, since nothing + swapped. Only the cardinality label itself still needs translating: + `keys._qualifies` recognises `MANY_TO_ONE`/`ONE_TO_ONE`, never the TML + spelling `ONE_TO_MANY` a swapped relationship is still stashed under, so + `ONE_TO_MANY` is relabelled `MANY_TO_ONE` for the candidate. `MANY_TO_MANY` + needs no such handling -- it is excluded by `keys._qualifies` on either + side, which is already correct. + """ + referencing_join = join.get("referencing_join") + if referencing_join: + candidates = from_table_body.get("joins_with") or [] + matched = next((jw for jw in candidates if jw.get("name") == referencing_join), None) + if matched is None: + log.add( + code="TS-JOIN-REFERENCING-MISSING", + severity=Severity.WARNING, + message=( + f"model_tables[] entry {from_prefix!r} references joins_with " + f"{referencing_join!r}, which is not defined on its table; the " + f"join is skipped" + ), + object_ref=f"relationship:{referencing_join}", + ) + return None, None, None + to_prefix = (matched.get("destination") or {}).get("name") + on_expression = matched.get("on") + join_type = join.get("type", matched.get("type")) + cardinality = join.get("cardinality", matched.get("cardinality")) + join_shape = ( + "referencing_with_inline_attrs" + if ("type" in join or "cardinality" in join) + else "referencing" + ) + name = referencing_join + else: + to_prefix = join.get("with") + on_expression = join.get("on") + join_type = join.get("type") + cardinality = join.get("cardinality") + join_shape = "inline" + name = f"{from_prefix}_to_{to_prefix}" if to_prefix else f"{from_prefix}_to_" + + if not to_prefix: + log.add( + code="TS-JOIN-NO-TARGET", + severity=Severity.WARNING, + message=f"join {name!r} from {from_prefix!r} names no target dataset; it is skipped", + object_ref=f"relationship:{name}", + ) + return None, None, None + + if to_prefix not in known_datasets: + log.add( + code="TS-JOIN-UNKNOWN-TARGET", + severity=Severity.WARNING, + message=( + f"join {name!r} from {from_prefix!r} targets {to_prefix!r}, which " + f"is not one of this model's datasets; the relationship is " + f"dropped rather than emitted pointing at a dataset that does not " + f"exist" + ), + object_ref=f"relationship:{name}", + ) + return None, None, None + + relationship, unrepresentable, has_residuals = _relationship_from_join( + name=name, + from_prefix=from_prefix, + to_prefix=to_prefix, + on_expression=on_expression, + join_type=join_type, + cardinality=cardinality, + join_shape=join_shape, + referencing_join=referencing_join, + log=log, + ) + + candidate = None + if relationship is not None: + candidate_cardinality = "MANY_TO_ONE" if cardinality == "ONE_TO_MANY" else (cardinality or "") + candidate = keys.Relationship( + name=relationship["name"], + to_dataset=relationship["to"], + to_columns=relationship["to_columns"], + cardinality=candidate_cardinality, + has_residual_predicates=has_residuals, + ) + return relationship, unrepresentable, candidate + + +@dataclass(frozen=True) +class OssieConversion: + """The result of one TML -> Ossie conversion. + + `model` is the full Ossie document -- `{"version": ..., "semantic_model": + [...]}` -- ready to dump as YAML. `issues` is every declared loss and + degradation raised while building it: nothing in `model` is missing + something TML held without a matching entry here. + """ + + model: dict + issues: IssueLog + + +def convert(document_set: DocumentSet) -> OssieConversion: + """Convert one ThoughtSpot TML document set into one Ossie semantic model. + + Order matters and mirrors the module docstring above: datasets first (so + their names -- the model_tables[] alias-or-name, verbatim -- exist), + then the cross-model resolver (needs every dataset's name and every + ATTRIBUTE column's identifier), then fields and metrics (need `resolve`), + then relationships (need nothing new, but key derivation needs every + relationship gathered first), then keys. + + A malformed reference anywhere -- an ambiguous `column_id`, an ambiguous + reference inside a join condition -- is caught per object: that object is + skipped, an issue names it and why, and every other object still + converts. A field that could not be attributed to a dataset is either + entirely omitted (a physical column, or a formula-produced field with no + attribution -- there is nothing else to build) or, when it is a + formula-backed ATTRIBUTE column whose formula genuinely exists, preserved + verbatim in the model-scope `unattributed_formulas` stash rather than + dropped outright. + """ + log = IssueLog() + model_body = document_set.model.body + + model_display_name = model_body.get("name") or "" + if not model_display_name: + semantic_model_name = "model" + else: + try: + semantic_model_name = identifiers.normalise(model_display_name) + except ValueError: + # A model name with no ASCII alphanumerics at all (a CJK-only + # name, one that is punctuation-only) has nothing for `normalise` + # to fold onto. Falling back to a fixed placeholder identifier, + # reported, keeps the document convertible instead of aborting + # the whole model over one unfoldable name -- the exact text is + # still recovered via the STASH_TML_NAME stash just below, since + # the placeholder never equals the original display name. + semantic_model_name = "model" + log.add( + code="TS-MODEL-NAME-UNNORMALISABLE", + severity=Severity.WARNING, + message=( + f"model name {model_display_name!r} has no ASCII " + f"alphanumerics for normalise() to fold onto; the " + f"semantic model is named 'model' instead" + ), + object_ref=f"model:{model_display_name}", + ) + semantic_model: dict = {"name": semantic_model_name, "datasets": []} + model_stash: dict = {} + if semantic_model_name != model_display_name: + model_stash[STASH_TML_NAME] = model_display_name + + description = model_body.get("description") + if description: + semantic_model["description"] = description + + # -- Phase 1: datasets -------------------------------------------------- + dataset_order: list[str] = [] + dataset_bodies: dict[str, dict] = {} + dataset_stashes: dict[str, dict] = {} + table_docs: dict[str, dict] = {} + physical_columns_by_prefix: dict[str, list[dict]] = {} + fields_by_dataset: dict[str, list] = {} + seen_prefixes: set[str] = set() + + model_tables = model_body.get("model_tables") or [] + for entry in model_tables: + table_ref = entry.get("name") + prefix = entry.get("alias") or table_ref + if not prefix: + log.add( + code="TS-DATASET-NO-NAME", + severity=Severity.WARNING, + message="a model_tables[] entry has no name and no alias; it cannot become a dataset", + object_ref="dataset:", + ) + continue + object_ref = f"dataset:{prefix}" + if prefix in seen_prefixes: + log.add( + code="TS-DATASET-DUPLICATE-PREFIX", + severity=Severity.WARNING, + message=( + f"more than one model_tables[] entry resolves to the reference " + f"name {prefix!r}; only the first is converted" + ), + object_ref=object_ref, + ) + continue + table_doc = document_set.table_by_name(table_ref) if table_ref else None + if table_doc is None: + log.add( + code="TS-DATASET-TABLE-MISSING", + severity=Severity.WARNING, + message=( + f"model_tables[] entry {prefix!r} references table {table_ref!r}, " + f"which has no matching table/sql_view document; the dataset is " + f"skipped" + ), + object_ref=object_ref, + ) + continue + + dataset_dict, ds_stash = _build_dataset(prefix, entry, table_doc, log) + seen_prefixes.add(prefix) + dataset_order.append(prefix) + dataset_bodies[prefix] = dataset_dict + dataset_stashes[prefix] = ds_stash + table_docs[prefix] = table_doc.body + physical_columns_by_prefix[prefix] = _normalized_physical_columns( + table_doc.body, table_doc.kind + ) + fields_by_dataset[prefix] = [] + + def table_lookup(name: str) -> dict | None: + columns = physical_columns_by_prefix.get(name) + if columns is None: + return None + return {"columns": columns} + + # -- Phase 2: the cross-model resolver ----------------------------------- + model_columns = model_body.get("columns") or [] + attribute_index = _index_attribute_columns(model_columns, log) + + def resolve(table: str, column: str) -> str | None: + # The mapping document is explicit for a bare-identifier field: "the + # identifier is the *physical* column; the display name comes from + # label/name." So the ANSI_SQL sibling this feeds -- built to be + # directly executable against the warehouse -- has to carry the + # actual warehouse column reference (db_column_name, or a SQL + # View's sql_output_column), never the Ossie field's own + # display-derived identifier, which is not a column that exists on + # the underlying table at all. `attribute_index` still gates + # whether this reference is one the model actually surfaces as a + # field -- that scope is unchanged -- only the value returned once + # it passes that gate changes. + if table not in dataset_bodies: + return None + if (table, column) not in attribute_index: + return None + physical = next( + (p for p in physical_columns_by_prefix.get(table, []) if p.get("name") == column), + None, + ) + if physical is None: + return None + warehouse_reference = physical.get("db_column_name") + if warehouse_reference is None: + return None + return f"{table}.{warehouse_reference}" + + # -- Phase 3: fields and metrics ------------------------------------------ + formulas: dict[str, dict] = { + f["id"]: f for f in (model_body.get("formulas") or []) if f.get("id") + } + metrics: list[dict] = [] + # Field identifiers are scoped per dataset in Ossie (Field.name is unique + # "within the dataset"); metrics are scoped to the whole model (Metric.name + # is unique across `metrics[]`, which is a single flat list here, not one + # per dataset). One allocator per scope, shared by every fallback + # identifier `convert_field`/`convert_metric` allocate in this model, so + # two columns that would otherwise both fall back to the same placeholder + # (e.g. two formula-backed, non-Latin-named fields with no physical + # grounding to fall back on) get distinct identifiers instead of + # colliding. Sharing one field allocator across every dataset rather than + # one per dataset is a stricter guarantee than the schema requires, not a + # looser one -- a model-wide-unique fallback name is trivially also + # dataset-unique -- and it avoids threading a per-dataset registry through + # a call site that does not otherwise need to know which dataset it is in + # until after the identifier is already computed. + field_name_allocator = identifiers.Allocator() + metric_name_allocator = identifiers.Allocator() + # `(TABLE, physical column display name) -> the Ossie field identifier + # convert_field actually assigned it`, populated below as fields are + # built. Phase 3.5 needs this for the SQL View `sql_output_columns` stash + # -- see `_index_attribute_columns` for why it is no longer read from + # `attribute_index` itself. + built_field_names: dict[tuple[str, str], str] = {} + + for column in model_columns: + display_name = column.get("name", "") + properties = column.get("properties") or {} + try: + field = convert_field( + column, formulas, table_lookup, resolve, log, field_name_allocator + ) + metric = None if field is not None else convert_metric( + column, formulas, table_lookup, resolve, log, metric_name_allocator + ) + except ValueError as exc: + log.add( + code="TS-COLUMN-REF-MALFORMED", + severity=Severity.WARNING, + message=f"column {display_name!r} could not be converted: {exc}", + object_ref=f"field:{display_name}", + ) + continue + + if field is not None: + if "column_id" in column: + # Safe to re-parse without a try/except: convert_field just + # parsed this same column_id successfully (that's how `field` + # came to exist at all), so it cannot raise here. + field_table, field_column = identifiers.split_column_ref( + f"[{column['column_id']}]" + ) + built_field_names[(field_table, field_column)] = field["name"] + extra_properties = _unconsumed_properties( + properties, _FIELD_CONSUMED_PROPERTIES, log, f"field:{display_name}" + ) + field_stash_payload: dict = {} + if extra_properties: + field_stash_payload[FIELD_STASH_COLUMN_PROPERTIES] = extra_properties + if "column_id" in column: + field_stash_payload.update(_physical_column_stash( + column["column_id"], field.get("datatype"), + physical_columns_by_prefix, dataset_stashes, + )) + if field_stash_payload: + field = _write_stash_safely(field, field_stash_payload, log, f"field:{display_name}") + owner = _field_owner_dataset(column, formulas, resolve) + if owner is not None and owner in fields_by_dataset: + fields_by_dataset[owner].append(field) + else: + log.add( + code="TS-FIELD-DATASET-MISSING", + severity=Severity.WARNING, + message=( + f"field {display_name!r} resolves to dataset {owner!r}, " + f"which was not built; the field is dropped" + ), + object_ref=f"field:{display_name}", + ) + continue + + if metric is not None: + extra_properties = _unconsumed_properties( + properties, _METRIC_CONSUMED_PROPERTIES, log, f"metric:{display_name}" + ) + metric_stash_payload: dict = {} + if extra_properties: + metric_stash_payload[FIELD_STASH_COLUMN_PROPERTIES] = extra_properties + if "column_id" in column: + metric_stash_payload.update(_physical_column_stash( + column["column_id"], metric.get("datatype"), + physical_columns_by_prefix, dataset_stashes, + )) + if metric_stash_payload: + metric = _write_stash_safely(metric, metric_stash_payload, log, f"metric:{display_name}") + metrics.append(metric) + continue + + # Neither a field nor a metric was built. + column_type = properties.get("column_type") + if column_type not in ("ATTRIBUTE", "MEASURE"): + # A column_type this converter does not recognise at all (TML + # requires one of the two) is a malformed column, not a case + # convert_field/convert_metric already explained -- name it + # rather than silently skipping it. + log.add( + code="TS-COLUMN-TYPE-UNKNOWN", + severity=Severity.WARNING, + message=( + f"column {display_name!r} has column_type {column_type!r}, " + f"which is neither ATTRIBUTE nor MEASURE; it is not converted" + ), + object_ref=f"field:{display_name}", + ) + continue + + # The one case worth preserving: an ATTRIBUTE formula that genuinely + # exists (has an expr) but could not be attributed to a single + # dataset -- convert_field already logged why via attribute_dataset. + if column_type == "ATTRIBUTE" and "formula_id" in column: + formula_entry = formulas.get(column["formula_id"]) + if formula_entry is not None and "expr" in formula_entry: + unattributed: dict = { + "name": formula_entry.get("name") or display_name, + "expr": formula_entry["expr"], + } + if properties: + unattributed[FIELD_STASH_COLUMN_PROPERTIES] = properties + model_stash.setdefault(MODEL_STASH_UNATTRIBUTED_FORMULAS, []).append(unattributed) + + # -- Phase 3.5: unsurfaced physical columns, and SQL View output aliases -- + # A Table/SQL-View column with no Ossie FIELD of its own is not part of + # the semantic model *as a field*, but has to be preserved verbatim + # (Dataset-level mapping, "fields" row) so the source document can be + # regenerated exactly on the way back. `_raw_physical_columns` reads + # whichever key this dataset's document kind actually uses (`columns[]` + # or `sql_view_columns[]`) -- the RAW entries, not the datatype-lookup + # shape `_normalized_physical_column` builds, since regenerating a SQL + # View column needs its own `sql_output_column` key back, not a + # `db_column_name` this converter invented for lookup purposes. + # + # This is deliberately keyed on `attribute_index` -- which physical + # columns became an ATTRIBUTE *field* -- and not on every column_id any + # Model `columns[]` entry names (ATTRIBUTE and MEASURE alike). An + # earlier revision used the broader set, reasoning that a + # `column_aggregation`-shape metric surfaces its physical column just as + # much as an ATTRIBUTE field does. That is true as far as it goes, but + # nothing else preserves that column's definition: a metric has no + # `column_id` field in Ossie at all -- it carries only the composed + # THOUGHTSPOT-dialect expression, verbatim, with the bracket reference + # inside it -- so the physical column it names was silently dropped from + # both `fields` and `unsurfaced_columns`. `build_table` on the way back + # then regenerated a Table with no such column, while `build_model` + # still emitted a metric formula referencing it: a dangling + # `[TABLE::Column]` reference in an otherwise-valid document, the same + # "portable expression naming a column that does not exist" failure + # mode a plain round trip is the only way to catch. A physical column + # referenced only by a metric is therefore captured here exactly like + # one referenced by nothing at all -- redundant with the metric's own + # verbatim expression, but redundancy is what makes the Table document + # regenerable independently of which metrics happen to reference it. + # + # The SQL View alias lookup just below reads `built_field_names`, not + # `attribute_index`, for the *value* half of the same fact (which Ossie + # field identifier a physical column became) -- see + # `_index_attribute_columns` for why the two are no longer the same + # object. + referenced_columns = set(attribute_index) + for prefix in dataset_order: + kind = "sql_view" if dataset_stashes[prefix].get(DATASET_STASH_TML_OBJECT) == "sql_view" else "table" + raw_columns = _raw_physical_columns(table_docs.get(prefix) or {}, kind) + unsurfaced = [ + column for column in raw_columns + if (prefix, column.get("name")) not in referenced_columns + ] + if unsurfaced: + dataset_stashes[prefix][DATASET_STASH_UNSURFACED_COLUMNS] = unsurfaced + + if kind == "sql_view": + # Every SURFACED field on a SQL View needs its own + # sql_output_column recorded (DatasetLevel schema's + # sql_output_columns key: "field name -> sql_output_column + # alias") -- there is no safe way to re-derive a query output + # alias from an Ossie field's own identifier the way a Table's + # db_column_name might be guessed at, so this is always + # necessary, not just when the alias happens to differ from the + # field's name. + output_aliases = {} + for column in raw_columns: + field_name = built_field_names.get((prefix, column.get("name"))) + if field_name is not None and column.get("sql_output_column") is not None: + output_aliases[field_name] = column["sql_output_column"] + if output_aliases: + dataset_stashes[prefix][DATASET_STASH_SQL_OUTPUT_COLUMNS] = output_aliases + + # -- Phase 4: relationships ------------------------------------------------ + relationships: list[dict] = [] + key_candidates: list[keys.Relationship] = [] + known_datasets = frozenset(dataset_bodies) + + for entry in model_tables: + table_ref = entry.get("name") + from_prefix = entry.get("alias") or table_ref + if from_prefix not in dataset_bodies: + continue # the dataset itself failed to build; already logged + for join in entry.get("joins") or []: + relationship, unrepresentable, candidate = _convert_join( + from_prefix, join, table_docs.get(from_prefix) or {}, known_datasets, log + ) + if relationship is not None: + relationships.append(relationship) + if unrepresentable is not None: + model_stash.setdefault(MODEL_STASH_UNREPRESENTABLE_JOINS, []).append(unrepresentable) + if candidate is not None: + key_candidates.append(candidate) + + # -- Phase 5: keys ----------------------------------------------------- + for prefix in dataset_order: + primary_key, unique_keys = keys.derive_keys(prefix, key_candidates, log) + if primary_key: + dataset_bodies[prefix]["primary_key"] = primary_key + if unique_keys: + dataset_bodies[prefix]["unique_keys"] = unique_keys + + # -- Phase 6: assemble datasets ------------------------------------------ + datasets_out: list[dict] = [] + for prefix in dataset_order: + dataset_dict = dataset_bodies[prefix] + if fields_by_dataset[prefix]: + dataset_dict["fields"] = fields_by_dataset[prefix] + dataset_dict = _write_stash_safely(dataset_dict, dataset_stashes[prefix], log, f"dataset:{prefix}") + datasets_out.append(dataset_dict) + semantic_model["datasets"] = datasets_out + + if relationships: + semantic_model["relationships"] = relationships + if metrics: + semantic_model["metrics"] = metrics + + # -- Phase 7: model-scope stash ------------------------------------------- + raw_properties = model_body.get("properties") or {} + model_properties: dict = {} + for key_name in ("is_bypass_rls", "join_progressive"): + if key_name in raw_properties: + model_properties[key_name] = raw_properties[key_name] + spotter = raw_properties.get("spotter_config") + if isinstance(spotter, dict) and "is_spotter_enabled" in spotter: + model_properties["spotter_config"] = { + "is_spotter_enabled": spotter["is_spotter_enabled"] + } + if model_properties: + model_stash[MODEL_STASH_MODEL_PROPERTIES] = model_properties + + for key_name in ( + MODEL_STASH_PARAMETERS, MODEL_STASH_FILTERS, MODEL_STASH_COLUMN_GROUPS, + MODEL_STASH_LESSON_PLANS, MODEL_STASH_ACTION_OBJECT_ASSOCIATIONS, + ): + value = model_body.get(key_name) + if value: + model_stash[key_name] = value + constraints = model_body.get(MODEL_STASH_CONSTRAINTS) + if constraints: + model_stash[MODEL_STASH_CONSTRAINTS] = constraints + model_joins_with = model_body.get("joins_with") + if model_joins_with: + model_stash[MODEL_STASH_MODEL_JOINS_WITH] = model_joins_with + + if model_body.get("aggregated_models"): + # Aggregate-model routing associations are GUIDs of other Model + # objects -- instance-local, so they are never stashed. Stripping + # them silently disables the routing with no error, so the issue is + # the only signal a reader gets. + log.add( + code="TS-MODEL-AGGREGATED-MODELS", + severity=Severity.WARNING, + message=( + "model has aggregated_models query-routing associations, which " + "reference instance-local Model GUIDs; they are not carried into " + "the portable document, so aggregate-aware routing will not be " + "active after import" + ), + object_ref=f"model:{semantic_model_name}", + remedy="Reconfigure aggregate-model routing manually on the target instance after import.", + ) + + semantic_model = _write_stash_safely(semantic_model, model_stash, log, f"model:{semantic_model_name}") + + document = {"version": DOCUMENT_VERSION, "semantic_model": [semantic_model]} + return OssieConversion(model=document, issues=log) diff --git a/converters/thoughtspot/tests/conftest.py b/converters/thoughtspot/tests/conftest.py new file mode 100644 index 00000000..b8a423d3 --- /dev/null +++ b/converters/thoughtspot/tests/conftest.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) diff --git a/converters/thoughtspot/tests/expressions/test_catalog_aggregate.py b/converters/thoughtspot/tests/expressions/test_catalog_aggregate.py new file mode 100644 index 00000000..9e00f222 --- /dev/null +++ b/converters/thoughtspot/tests/expressions/test_catalog_aggregate.py @@ -0,0 +1,138 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Catalog coverage: Aggregate functions + Type conversion. + +Source: the `Aggregate functions` and `Type conversion` sections of +docs/ossie/ts-ossie-function-mapping.md (thoughtspot-agent-skills repo, not +vendored here). 20 rows total — 14 direct / 6 passthrough / 0 unmappable. + +Construct names are spelled exactly as `spec_construct_names()` extracts them +(see catalog.py's module docstring, "Spelling" section) — in particular `CAST` +and `TRY_CAST`, not `CAST(expression AS target_type)` / `TRY_CAST(expression AS +target_type)`, which is how the mapping document's own row headers write them. +""" +from ossie_thoughtspot.expressions import CATALOG +from ossie_thoughtspot.expressions._types import Classification, Variant + +EXPECTED: dict[str, Classification] = { + # -- Aggregate functions (18 rows: 12 direct / 6 passthrough) -------------- + "SUM(expr)": Classification.DIRECT, + "COUNT(expr)": Classification.DIRECT, + "COUNT(*)": Classification.DIRECT, + "COUNT(DISTINCT expr)": Classification.DIRECT, + "AVG(expr)": Classification.DIRECT, + "MIN(expr)": Classification.DIRECT, + "MAX(expr)": Classification.DIRECT, + "STDDEV(expr)": Classification.DIRECT, + "STDDEV_POP(expr)": Classification.PASSTHROUGH, + "STDDEV_SAMP(expr)": Classification.DIRECT, + "VARIANCE(expr)": Classification.DIRECT, + "VAR_POP(expr)": Classification.PASSTHROUGH, + "VAR_SAMP(expr)": Classification.DIRECT, + "MEDIAN(expr)": Classification.DIRECT, + "PERCENTILE_CONT(p) WITHIN GROUP (ORDER BY expr)": Classification.PASSTHROUGH, + "PERCENTILE_DISC(p) WITHIN GROUP (ORDER BY expr)": Classification.PASSTHROUGH, + "APPROX_COUNT_DISTINCT(expr)": Classification.PASSTHROUGH, + "APPROX_PERCENTILE(expr, p)": Classification.PASSTHROUGH, + # -- Type conversion (2 rows: 2 direct) ------------------------------------ + "CAST": Classification.DIRECT, + "TRY_CAST": Classification.DIRECT, +} + +#: Expected `Variant` for every passthrough row in this family. Getting +#: this wrong is the failure mode with no safety net: the wrong variant emits a +#: column that imports cleanly and aggregates wrongly, and nothing downstream +#: catches it. +EXPECTED_VARIANTS: dict[str, Variant] = { + "STDDEV_POP(expr)": Variant.NUMBER_AGGREGATE, + "VAR_POP(expr)": Variant.NUMBER_AGGREGATE, + "PERCENTILE_CONT(p) WITHIN GROUP (ORDER BY expr)": Variant.NUMBER_AGGREGATE, + "PERCENTILE_DISC(p) WITHIN GROUP (ORDER BY expr)": Variant.NUMBER_AGGREGATE, + "APPROX_COUNT_DISTINCT(expr)": Variant.INT_AGGREGATE, + "APPROX_PERCENTILE(expr, p)": Variant.NUMBER_AGGREGATE, +} + + +def test_row_count_for_this_family(): + ours = [c for c in CATALOG.values() if c.spec_name in EXPECTED] + assert len(ours) == 20 + + +def test_classifications(): + for name, expected in EXPECTED.items(): + assert CATALOG[name].classification is expected, name + + +def test_passthrough_count_and_variants(): + passthrough_names = {n for n, c in EXPECTED.items() if c is Classification.PASSTHROUGH} + assert len(passthrough_names) == 6 + assert passthrough_names == set(EXPECTED_VARIANTS) + for name, variant in EXPECTED_VARIANTS.items(): + assert CATALOG[name].variant is variant, name + + +def test_direct_count(): + direct_names = {n for n, c in EXPECTED.items() if c is Classification.DIRECT} + assert len(direct_names) == 14 + + +def test_no_unmappable_rows_in_this_family(): + assert not any(c is Classification.UNMAPPABLE for c in EXPECTED.values()) + + +# -------------------------------------------------------------------------- +# Rows the document only explains via its surrounding prose. +# -------------------------------------------------------------------------- + +def test_count_star_uses_a_non_null_column_not_a_literal_star(): + # ThoughtSpot has no count(*); the row is emitted as count() over a column + # the converter believes is non-null (the model's declared primary_key). + row = CATALOG["COUNT(*)"] + assert row.classification is Classification.DIRECT + assert "count" in row.template.lower() + + +def test_count_distinct_uses_a_space_not_an_underscore(): + # count_distinct(...) is rejected by the ThoughtSpot formula parser. + row = CATALOG["COUNT(DISTINCT expr)"] + assert "unique count" in row.template + assert "count_distinct" not in row.template.lower() + + +def test_stddev_and_variance_samp_aliases_map_to_the_sample_form(): + # STDDEV_SAMP / VAR_SAMP are specification aliases for STDDEV / VARIANCE — + # both are sample statistics in ThoughtSpot, so both alias rows stay direct. + assert CATALOG["STDDEV_SAMP(expr)"].template == CATALOG["STDDEV(expr)"].template + assert CATALOG["VAR_SAMP(expr)"].template == CATALOG["VARIANCE(expr)"].template + + +def test_population_statistics_are_passthrough_because_ts_stddev_is_sample_only(): + # STDDEV / VARIANCE are sample-only in ThoughtSpot; there is no population + # form, so STDDEV_POP / VAR_POP cannot reuse the sample-form template. + assert CATALOG["STDDEV_POP(expr)"].template != CATALOG["STDDEV(expr)"].template + assert CATALOG["VAR_POP(expr)"].template != CATALOG["VARIANCE(expr)"].template + + +def test_try_cast_shares_casts_mapping(): + # ThoughtSpot's to_integer/to_double/to_string already return NULL on + # failure, which is exactly TRY_CAST semantics — so CAST and TRY_CAST + # share a mapping, and it is CAST that is the imprecise one, not TRY_CAST. + cast = CATALOG["CAST"] + try_cast = CATALOG["TRY_CAST"] + assert cast.classification is Classification.DIRECT + assert try_cast.classification is Classification.DIRECT diff --git a/converters/thoughtspot/tests/expressions/test_catalog_covers_the_spec.py b/converters/thoughtspot/tests/expressions/test_catalog_covers_the_spec.py new file mode 100644 index 00000000..11216d21 --- /dev/null +++ b/converters/thoughtspot/tests/expressions/test_catalog_covers_the_spec.py @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The catalog must cover the specification's construct inventory, one to one. + +This test reads the UPSTREAM core-spec/expression_language.md rather than any +document of our own. Oracling against our own mapping notes would only prove we +are self-consistent; reading the spec means a construct added upstream fails this +build instead of silently going unsupported. + +spec_construct_names() and the mapping document's 146-row census count by +different units — one parseable table row/heading vs. one construct, which +also counts a handful of constructs the spec only describes in prose. +CONVENTION_DIVERGENCES (catalog.py) is the exact, reasoned list of the 9 where +that difference shows up; test_the_two_counts_reconcile pins the arithmetic so +the two counts cannot drift apart silently. +""" +from ossie_thoughtspot.expressions import CATALOG, CONVENTION_DIVERGENCES, spec_construct_names + + +def test_every_spec_construct_has_a_catalog_entry(): + missing = spec_construct_names() - set(CATALOG) + assert missing == set(), f"constructs in the spec with no catalog entry: {sorted(missing)}" + + +def test_no_catalog_entry_invents_a_construct_the_spec_does_not_have(): + # CONVENTION_DIVERGENCES is the one deliberate exception: constructs the + # mapping document counts as their own row that core-spec/expression_language.md + # never gives a discrete table row of their own (see catalog.py for why, per + # entry). Everything else in CATALOG must trace to a real spec row. + invented = set(CATALOG) - spec_construct_names() - set(CONVENTION_DIVERGENCES) + assert invented == set(), ( + f"catalog entries not found in the spec or CONVENTION_DIVERGENCES: {sorted(invented)}" + ) + + +def test_the_two_counts_reconcile(): + # The spec's parseable rows (137) plus the deliberate divergences (9) must + # equal the mapping document's own census (146). If this drifts, either + # spec_construct_names() regressed or CONVENTION_DIVERGENCES needs an entry + # added or removed - it must not be "fixed" by changing the 146 constant. + assert len(spec_construct_names()) + len(CONVENTION_DIVERGENCES) == 146 + + +def test_the_total_matches_the_mapping_document_census(): + # 146 is the figure the mapping document's coverage summary reports, arrived at + # one row per construct (argument vocabularies are not constructs). + assert len(CATALOG) == 146 diff --git a/converters/thoughtspot/tests/expressions/test_catalog_datetime.py b/converters/thoughtspot/tests/expressions/test_catalog_datetime.py new file mode 100644 index 00000000..367e25ea --- /dev/null +++ b/converters/thoughtspot/tests/expressions/test_catalog_datetime.py @@ -0,0 +1,238 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Catalog coverage: Date/time functions. + +Source: the `Date/time functions` section of +docs/ossie/ts-ossie-function-mapping.md (thoughtspot-agent-skills repo, not +vendored here). 24 rows total — 17 direct / 7 passthrough / 0 unmappable. + +Construct names are spelled exactly as `spec_construct_names()` extracts them +from the UPSTREAM core-spec/expression_language.md (see catalog.py's module +docstring, "Spelling" section) — several diverge from the mapping document's +own row header: + +- The three "current" rows keep the spec's own joining word, "or": e.g. + `"CURRENT_DATE or CURRENT_DATE()"`, not `CURRENT_DATE` / `CURRENT_DATE()`. +- `EXTRACT` and `DATE_PART` are bare tokens (from the "Alternative Extraction + Syntax" code fence), not `EXTRACT(part FROM date_expr)` or + `DATE_PART('part', date_expr)`. +- The typed-literal and EXPERIMENTAL rows match only the backticked portion of + the mapping document's row header — the trailing `(typed literal)` / + `(EXPERIMENTAL)` annotation sits outside the backtick span and is not part + of the key: `"DATE '2024-01-15'"`, `"TIMESTAMP_NTZ '2024-01-15 10:30:00'"`, + `"TIME '10:30:00'"`, `"TO_DATE(string, format)"`, + `"TO_TIMESTAMP(string, format)"`, `"TO_CHAR(date_expr, format)"`. +""" +from ossie_thoughtspot.expressions import CATALOG +from ossie_thoughtspot.expressions._types import Classification, Variant + +EXPECTED: dict[str, Classification] = { + # -- Current date/time (3 rows: 3 direct) ---------------------------------- + "CURRENT_DATE or CURRENT_DATE()": Classification.DIRECT, + "CURRENT_TIMESTAMP or CURRENT_TIMESTAMP()": Classification.DIRECT, + "CURRENT_TIME or CURRENT_TIME()": Classification.DIRECT, + # -- Date/time extraction (8 rows: 6 direct / 2 passthrough) --------------- + "YEAR(date_expr)": Classification.DIRECT, + "QUARTER(date_expr)": Classification.DIRECT, + "MONTH(date_expr)": Classification.DIRECT, + "DAY(date_expr)": Classification.DIRECT, + "DAYOFYEAR(date_expr)": Classification.DIRECT, + "HOUR(timestamp_expr)": Classification.DIRECT, + "MINUTE(timestamp_expr)": Classification.PASSTHROUGH, + "SECOND(timestamp_expr)": Classification.PASSTHROUGH, + # -- Alternative extraction syntax (2 rows: 2 direct) ----------------------- + "EXTRACT": Classification.DIRECT, + "DATE_PART": Classification.DIRECT, + # -- Truncation and arithmetic (3 rows: 3 direct) --------------------------- + "DATE_TRUNC(part, date_expr)": Classification.DIRECT, + "DATEADD(part, amount, date_expr)": Classification.DIRECT, + "DATEDIFF(part, start_date, end_date)": Classification.DIRECT, + # -- Construction: typed literals (3 rows: 1 direct / 2 passthrough) ------- + "DATE '2024-01-15'": Classification.DIRECT, + "TIMESTAMP_NTZ '2024-01-15 10:30:00'": Classification.PASSTHROUGH, + "TIME '10:30:00'": Classification.PASSTHROUGH, + # -- Construction: parse functions (2 rows: 1 direct / 1 passthrough) ------ + "TO_DATE(string)": Classification.DIRECT, + "TO_TIMESTAMP(string)": Classification.PASSTHROUGH, + # -- Construction from format strings, EXPERIMENTAL (2 rows: 1 direct / 1 passthrough) -- + "TO_DATE(string, format)": Classification.DIRECT, + "TO_TIMESTAMP(string, format)": Classification.PASSTHROUGH, + # -- Formatting, EXPERIMENTAL (1 row: 1 passthrough) ------------------------ + "TO_CHAR(date_expr, format)": Classification.PASSTHROUGH, +} + +#: Expected `Variant` for every passthrough row in this family. Getting +#: this wrong is the failure mode with no safety net: the wrong variant emits a +#: column that imports cleanly and aggregates wrongly, and nothing downstream +#: catches it. Taken individually from the document, not inferred. +EXPECTED_VARIANTS: dict[str, Variant] = { + "MINUTE(timestamp_expr)": Variant.INT, + "SECOND(timestamp_expr)": Variant.INT, + "TIMESTAMP_NTZ '2024-01-15 10:30:00'": Variant.DATE_TIME, + "TIME '10:30:00'": Variant.DATE_TIME, + "TO_TIMESTAMP(string)": Variant.DATE_TIME, + "TO_TIMESTAMP(string, format)": Variant.DATE_TIME, + "TO_CHAR(date_expr, format)": Variant.STRING, +} + + +def test_row_count_for_this_family(): + ours = [c for c in CATALOG.values() if c.spec_name in EXPECTED] + assert len(ours) == 24 + + +def test_classifications(): + for name, expected in EXPECTED.items(): + assert CATALOG[name].classification is expected, name + + +def test_passthrough_count_and_variants(): + passthrough_names = {n for n, c in EXPECTED.items() if c is Classification.PASSTHROUGH} + assert len(passthrough_names) == 7 + assert passthrough_names == set(EXPECTED_VARIANTS) + for name, variant in EXPECTED_VARIANTS.items(): + assert CATALOG[name].variant is variant, name + + +def test_direct_count(): + direct_names = {n for n, c in EXPECTED.items() if c is Classification.DIRECT} + assert len(direct_names) == 17 + + +def test_no_unmappable_rows_in_this_family(): + assert not any(c is Classification.UNMAPPABLE for c in EXPECTED.values()) + + +# -------------------------------------------------------------------------- +# Rows the document only explains via its surrounding prose. +# -------------------------------------------------------------------------- + +def test_month_uses_month_number_not_month_name(): + # ThoughtSpot's month() returns the month NAME ("January"); month_number() + # returns 1-12, which is what the specification's MONTH(date_expr) means. + # Mapping to month() would silently change the column's type. + row = CATALOG["MONTH(date_expr)"] + assert row.template == "month_number ( {0} )" + + +def test_quarter_uses_quarter_number_not_quarter(): + row = CATALOG["QUARTER(date_expr)"] + assert row.template == "quarter_number ( {0} )" + + +def test_hour_uses_hour_of_day_not_hour(): + row = CATALOG["HOUR(timestamp_expr)"] + assert row.template == "hour_of_day ( {0} )" + + +def test_dayofyear_uses_day_number_of_year_not_day_of_year(): + row = CATALOG["DAYOFYEAR(date_expr)"] + assert row.template == "day_number_of_year ( {0} )" + + +def test_current_time_is_a_composition_of_time_and_now(): + # ThoughtSpot has no current-time function; time ( now ( ) ) is exact. + row = CATALOG["CURRENT_TIME or CURRENT_TIME()"] + assert row.template == "time ( now ( ) )" + + +def test_no_native_minute_or_second_extractor(): + # There is no MINUTE/SECOND-of-hour extractor in ThoughtSpot at all — + # add_minutes/diff_minutes exist but neither extracts. + minute = CATALOG["MINUTE(timestamp_expr)"] + second = CATALOG["SECOND(timestamp_expr)"] + assert minute.classification is Classification.PASSTHROUGH + assert second.classification is Classification.PASSTHROUGH + assert minute.variant is Variant.INT + assert second.variant is Variant.INT + + +def test_extract_and_date_part_collapse_to_the_same_rewrite(): + # The two spellings are identical treatment per the specification. + extract = CATALOG["EXTRACT"] + date_part = CATALOG["DATE_PART"] + assert extract.classification is Classification.DIRECT + assert date_part.classification is Classification.DIRECT + assert extract.template == date_part.template + + +def test_no_native_date_trunc(): + # ThoughtSpot has no date_trunc; the row is still DIRECT because the + # start_of_* family covers 7 of 8 precisions (only 'second' falls back). + row = CATALOG["DATE_TRUNC(part, date_expr)"] + assert row.classification is Classification.DIRECT + assert "date_trunc" not in row.template.lower() + + +def test_dateadd_argument_order_is_documented_as_reversed_from_thoughtspot(): + # ThoughtSpot is add_days ( [d] , n ); the specification is + # DATEADD(day, n, d). Getting this backwards is a silent-wrong-answer bug. + row = CATALOG["DATEADD(part, amount, date_expr)"] + assert row.classification is Classification.DIRECT + assert "argument order" in row.note.lower() + + +def test_datediff_argument_order_is_documented_as_reversed(): + # ThoughtSpot is diff_days ( [end] , [start] ) - end first. Getting this + # wrong silently negates every duration in the model. + row = CATALOG["DATEDIFF(part, start_date, end_date)"] + assert row.classification is Classification.DIRECT + assert "argument order" in row.note.lower() + assert "end" in row.note.lower() + + +def test_date_literal_must_be_wrapped_in_to_date(): + # A bare '2024-01-15' in a ThoughtSpot formula parses as arithmetic + # (2024 - 1 - 15), so the typed literal must always be wrapped. + row = CATALOG["DATE '2024-01-15'"] + assert row.classification is Classification.DIRECT + assert "to_date" in row.template.lower() + + +def test_timestamp_ntz_and_time_literals_have_no_native_construction(): + # to_date() returns a DATE and drops the time part, so there is no native + # way to construct a wall-clock TIMESTAMP or a TIME value. + timestamp_ntz = CATALOG["TIMESTAMP_NTZ '2024-01-15 10:30:00'"] + time_literal = CATALOG["TIME '10:30:00'"] + assert timestamp_ntz.classification is Classification.PASSTHROUGH + assert time_literal.classification is Classification.PASSTHROUGH + assert timestamp_ntz.variant is Variant.DATE_TIME + assert time_literal.variant is Variant.DATE_TIME + + +def test_to_date_single_arg_supplies_the_iso_format_model(): + # ThoughtSpot's to_date is strictly two-argument; the converter supplies + # 'yyyy-MM-dd' for the specification's single-argument ISO form. + row = CATALOG["TO_DATE(string)"] + assert row.classification is Classification.DIRECT + assert "yyyy-mm-dd" in row.template.lower() + + +def test_to_timestamp_single_arg_is_passthrough_because_to_date_drops_time(): + row = CATALOG["TO_TIMESTAMP(string)"] + assert row.classification is Classification.PASSTHROUGH + assert row.variant is Variant.DATE_TIME + + +def test_to_char_prefers_no_native_general_formatter(): + # ThoughtSpot has no general date formatter; single-token formats (YYYY, + # MONTH, DAY) do have native equivalents the converter should prefer, but + # the general TO_CHAR(date_expr, format) row itself is passthrough. + row = CATALOG["TO_CHAR(date_expr, format)"] + assert row.classification is Classification.PASSTHROUGH + assert row.variant is Variant.STRING diff --git a/converters/thoughtspot/tests/expressions/test_catalog_math_conditional.py b/converters/thoughtspot/tests/expressions/test_catalog_math_conditional.py new file mode 100644 index 00000000..fe1f6991 --- /dev/null +++ b/converters/thoughtspot/tests/expressions/test_catalog_math_conditional.py @@ -0,0 +1,253 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Catalog coverage: Mathematical and Conditional functions. + +Source: the `Mathematical functions` and `Conditional functions` sections of +docs/ossie/ts-ossie-function-mapping.md (thoughtspot-agent-skills repo, not +vendored here). 34 rows total — 32 direct / 2 passthrough / 0 unmappable. + +Nearly everything here is direct, several by composition: SIGN is an +`if` chain with a mandatory `else 0` (ThoughtSpot rejects an `if` with no +`else`); RADIANS/DEGREES are bare arithmetic (no native function); PI is a +literal at the precision ThoughtSpot's own documented composites use. +ThoughtSpot trigonometry is in degrees while the specification is in radians, +so every forward trig function multiplies by 180/pi and every inverse trig +function divides by it — the opposite conversion, easy to get backwards. +GREATEST/LEAST are deliberately not MAX/MIN: ThoughtSpot's max/min are +aggregate-only, so mapping the row-wise N-ary forms onto them would both +collapse the column to one value and flip it from attribute to measure. + +Only two rows are passthrough: TRUNC/TRUNCATE (no native truncation, and +neither floor nor round is a safe substitute) and ATAN2 (quadrant-aware and +defined where x = 0, so it is not a two-argument ATAN composition). + +Construct names in this family are spelled identically to the mapping +document's own row headers, with the same alias-merge convention as CEIL/ +CEILING and TRUNC/TRUNCATE (see catalog.py's module docstring, "Spelling" +section) — the merged spelling (`CEIL(x)`, `TRUNC(x, d)`) is what +`spec_construct_names()` actually extracts, confirmed live before writing +this file. +""" +from ossie_thoughtspot.expressions import CATALOG +from ossie_thoughtspot.expressions._types import Classification, Variant + +EXPECTED: dict[str, Classification] = { + # Mathematical functions (25 rows: 23 direct / 2 passthrough) + "ABS(x)": Classification.DIRECT, + "ROUND(x, d)": Classification.DIRECT, + "FLOOR(x)": Classification.DIRECT, + "CEIL(x)": Classification.DIRECT, + "TRUNC(x, d)": Classification.PASSTHROUGH, + "MOD(x, y)": Classification.DIRECT, + "SIGN(x)": Classification.DIRECT, + "POWER(x, y)": Classification.DIRECT, + "SQRT(x)": Classification.DIRECT, + "EXP(x)": Classification.DIRECT, + "LN(x)": Classification.DIRECT, + "LOG(base, x)": Classification.DIRECT, + "LOG10(x)": Classification.DIRECT, + "SIN(x)": Classification.DIRECT, + "COS(x)": Classification.DIRECT, + "TAN(x)": Classification.DIRECT, + "ASIN(x)": Classification.DIRECT, + "ACOS(x)": Classification.DIRECT, + "ATAN(x)": Classification.DIRECT, + "ATAN2(y, x)": Classification.PASSTHROUGH, + "RADIANS(degrees)": Classification.DIRECT, + "DEGREES(radians)": Classification.DIRECT, + "PI()": Classification.DIRECT, + "GREATEST(x, y, ...)": Classification.DIRECT, + "LEAST(x, y, ...)": Classification.DIRECT, + # Conditional functions (9 rows: all direct) + "IF(condition, true_result, false_result)": Classification.DIRECT, + "IFF(condition, true_result, false_result)": Classification.DIRECT, + "NULLIF(expr1, expr2)": Classification.DIRECT, + "COALESCE(expr1, expr2, ...)": Classification.DIRECT, + "IFNULL(expr, default)": Classification.DIRECT, + "NVL(expr, default)": Classification.DIRECT, + "NVL2(expr, not_null_result, null_result)": Classification.DIRECT, + "ZEROIFNULL(expr)": Classification.DIRECT, + "NULLIFZERO(expr)": Classification.DIRECT, +} + +#: Expected `Variant` for every passthrough row in this family. Getting +#: this wrong is the failure mode with no safety net: the wrong variant emits a +#: column that imports cleanly and then aggregates or types wrongly, and +#: nothing downstream catches it. +EXPECTED_VARIANTS: dict[str, Variant] = { + "TRUNC(x, d)": Variant.DOUBLE, + "ATAN2(y, x)": Variant.DOUBLE, +} + + +def test_row_count_for_this_family(): + ours = [c for c in CATALOG.values() if c.spec_name in EXPECTED] + assert len(ours) == 34 + + +def test_classifications(): + for name, expected in EXPECTED.items(): + assert CATALOG[name].classification is expected, name + + +def test_passthrough_count_and_variants(): + passthrough_names = {n for n, c in EXPECTED.items() if c is Classification.PASSTHROUGH} + assert len(passthrough_names) == 2 + assert passthrough_names == set(EXPECTED_VARIANTS) + for name, variant in EXPECTED_VARIANTS.items(): + assert CATALOG[name].variant is variant, name + + +def test_direct_count(): + direct_names = {n for n, c in EXPECTED.items() if c is Classification.DIRECT} + assert len(direct_names) == 32 + + +def test_no_unmappable_rows_in_this_family(): + assert not any(c is Classification.UNMAPPABLE for c in EXPECTED.values()) + + +# -------------------------------------------------------------------------- +# Rows the document only explains via its surrounding prose. +# -------------------------------------------------------------------------- + +def test_sign_is_an_if_chain_with_a_mandatory_final_else(): + # No native `sign`, but the three-way result composes exactly from `if`. + # ThoughtSpot rejects an `if` chain with no `else` — the `else 0` is not + # optional decoration, it is required for the formula to import at all. + row = CATALOG["SIGN(x)"] + assert row.classification is Classification.DIRECT + assert "else 0" in row.template + + +def test_power_uses_pow_not_power(): + # `power` is rejected by the ThoughtSpot formula parser; the function is + # spelled `pow`. + row = CATALOG["POWER(x, y)"] + assert row.template.startswith("pow (") + + +def test_trig_functions_convert_degrees_because_thoughtspot_is_degrees_native(): + # ThoughtSpot trigonometry is in degrees; the specification is in radians. + # A bare sin(x) would return the sine of x *degrees* and be wrong for + # every non-zero input, so SIN/COS/TAN all multiply by 180/pi. + for name in ("SIN(x)", "COS(x)", "TAN(x)"): + row = CATALOG[name] + assert row.classification is Classification.DIRECT + assert "180" in row.template and "3.14159265358979" in row.template + + +def test_inverse_trig_functions_convert_the_other_way(): + # ASIN/ACOS/ATAN return degrees from ThoughtSpot's native functions but + # the specification expects radians, so these divide by 180/pi instead of + # multiplying by it — the opposite direction from SIN/COS/TAN. + for name in ("ASIN(x)", "ACOS(x)", "ATAN(x)"): + row = CATALOG[name] + assert row.classification is Classification.DIRECT + assert "/ 180" in row.template + + +def test_atan2_is_passthrough_not_a_two_argument_atan(): + # atan2 is quadrant-aware and defined where x = 0; it is not simply a + # two-argument form of atan, so no native composition is attempted. + row = CATALOG["ATAN2(y, x)"] + assert row.classification is Classification.PASSTHROUGH + assert row.variant is Variant.DOUBLE + + +def test_radians_and_degrees_are_bare_arithmetic(): + # No native radians/degrees function; the conversion is exact, dialect-free + # arithmetic, not a passthrough. + radians = CATALOG["RADIANS(degrees)"] + degrees = CATALOG["DEGREES(radians)"] + assert radians.classification is Classification.DIRECT + assert degrees.classification is Classification.DIRECT + assert radians.variant is None + assert degrees.variant is None + + +def test_pi_is_a_literal_at_the_documented_composite_precision(): + # No native pi(). The literal matches the precision ThoughtSpot's own + # documented composites (SIN/COS/TAN etc.) already use in this family. + row = CATALOG["PI()"] + assert row.classification is Classification.DIRECT + assert row.template.strip() == "3.14159265358979" + + +def test_greatest_and_least_are_not_max_and_min(): + # ThoughtSpot's max/min are aggregate-only; greatest/least are the + # row-wise N-ary functions. Mapping GREATEST to max would both collapse + # the column to one value and flip it from attribute to measure. + greatest = CATALOG["GREATEST(x, y, ...)"] + least = CATALOG["LEAST(x, y, ...)"] + assert greatest.classification is Classification.DIRECT + assert least.classification is Classification.DIRECT + assert "greatest (" in greatest.template + assert "least (" in least.template + + +def test_trunc_has_no_safe_native_substitute(): + # floor only agrees with TRUNC for x >= 0 and d = 0; round disagrees at + # every half-value. Neither is a safe substitute, hence passthrough. + row = CATALOG["TRUNC(x, d)"] + assert row.classification is Classification.PASSTHROUGH + assert row.variant is Variant.DOUBLE + + +def test_if_requires_parenthesized_condition(): + # The parentheses around the condition are mandatory for TML import — + # without them the parser reports "Expecting keyword '('". Applies to + # every condition shape, including a bare BOOL column reference. + row = CATALOG["IF(condition, true_result, false_result)"] + assert row.classification is Classification.DIRECT + assert row.template.startswith("if (") + + +def test_iff_is_an_alias_for_if(): + assert CATALOG["IFF(condition, true_result, false_result)"].template == ( + CATALOG["IF(condition, true_result, false_result)"].template + ) + + +def test_coalesce_is_a_right_nested_ifnull_chain(): + # ThoughtSpot's ifnull is strictly two-argument, so an N-ary COALESCE + # becomes a right-nested chain rather than a flat N-ary call. + row = CATALOG["COALESCE(expr1, expr2, ...)"] + assert row.classification is Classification.DIRECT + assert row.template.count("ifnull (") == 2 + + +def test_nvl_is_alias_for_two_argument_coalesce_via_ifnull(): + assert CATALOG["NVL(expr, default)"].template == CATALOG["IFNULL(expr, default)"].template + + +def test_nvl2_has_no_native_three_way_null_function(): + # No native three-way null function; the composition using isnotnull is + # exact. + row = CATALOG["NVL2(expr, not_null_result, null_result)"] + assert row.classification is Classification.DIRECT + assert "isnotnull (" in row.template + + +def test_zeroifnull_and_nullifzero_are_mirror_images(): + zeroifnull = CATALOG["ZEROIFNULL(expr)"] + nullifzero = CATALOG["NULLIFZERO(expr)"] + assert zeroifnull.classification is Classification.DIRECT + assert nullifzero.classification is Classification.DIRECT + assert "ifnull (" in zeroifnull.template + assert "nullif (" in nullifzero.template diff --git a/converters/thoughtspot/tests/expressions/test_catalog_operators.py b/converters/thoughtspot/tests/expressions/test_catalog_operators.py new file mode 100644 index 00000000..dc070f9f --- /dev/null +++ b/converters/thoughtspot/tests/expressions/test_catalog_operators.py @@ -0,0 +1,233 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Catalog coverage: Operators and constructs. + +Source: the `Operators and constructs` section of +docs/ossie/ts-ossie-function-mapping.md (thoughtspot-agent-skills repo, not +vendored here). 33 rows total - 30 direct / 2 passthrough / 1 unmappable. + +This is the only family with an `unmappable` row in the whole 146-row catalog: +`EXISTS_IN()` is named at :131 as the sanctioned way to filter on a subquery, +but the specification never defines it anywhere - no signature, no argument +order, no semantics. `str ILIKE pattern` is passthrough because +case-insensitive matching has no native form (and the usual `lower` +workaround is itself a passthrough); `DISTINCT` as an aggregate modifier is +passthrough because ThoughtSpot has exactly one distinct-aware aggregate +(`unique count`, i.e. `COUNT(DISTINCT)`, already its own row) and nothing +else. `str LIKE pattern` stays direct despite ThoughtSpot having no native +`starts_with`/`ends_with`: the prefix/suffix/contains compositions use only +native functions. + +Six of the family's 33 rows have no discrete row of their own in the upstream +core-spec/expression_language.md - they are named only in prose, a bullet +list, or an "Operator Precedence"/"Not Supported" table with no backtick +marker - so they are keyed via `CONVENTION_DIVERGENCES` rather than +`spec_construct_names()`: unary `-x`/`+x`, the simple `CASE` form, +`Parentheses`, the `DISTINCT` modifier, the column/metric reference, and +`EXISTS_IN()` itself. The other 27 rows key on `spec_construct_names()`'s own +extraction - confirmed live before writing this file - which for this family +means the BARE operator/keyword token, not the mapping document's `a op b` +worked-example header: `+`, `-`, `*`, `/`, `%`, `=`, `<>`, `!=`, `<`, `>`, +`<=`, `>=`, `BETWEEN`, `IN`, `NOT IN`, `NOT expr`, `IS NULL`, `IS NOT NULL`, +`IS DISTINCT FROM`, `IS NOT DISTINCT FROM`, `CASE WHEN`, `TRUE, FALSE`, +`expr1 AND expr2`, `expr1 OR expr2`, `str LIKE pattern`, `str ILIKE pattern`, +`str1 || str2`. +""" +from ossie_thoughtspot.expressions import CATALOG +from ossie_thoughtspot.expressions._types import Classification, Variant + +EXPECTED: dict[str, Classification] = { + # Arithmetic operators (spec_construct_names() extracts the bare symbol, + # not the document's "a + b" worked-example header) + "+": Classification.DIRECT, + "-": Classification.DIRECT, + "*": Classification.DIRECT, + "/": Classification.DIRECT, + "%": Classification.DIRECT, + "-x / +x (unary)": Classification.DIRECT, # CONVENTION_DIVERGENCES + # Comparison operators (same bare-symbol extraction) + "=": Classification.DIRECT, + "<>": Classification.DIRECT, + "!=": Classification.DIRECT, + "<": Classification.DIRECT, + ">": Classification.DIRECT, + "<=": Classification.DIRECT, + ">=": Classification.DIRECT, + # Logical operators (Boolean Functions table's own expr1/expr2 placeholders) + "expr1 AND expr2": Classification.DIRECT, + "expr1 OR expr2": Classification.DIRECT, + "NOT expr": Classification.DIRECT, + # Set/range/pattern operators + "BETWEEN": Classification.DIRECT, + "IN": Classification.DIRECT, + "NOT IN": Classification.DIRECT, + "str LIKE pattern": Classification.DIRECT, + "str ILIKE pattern": Classification.PASSTHROUGH, + # Null tests + "IS NULL": Classification.DIRECT, + "IS NOT NULL": Classification.DIRECT, + "IS DISTINCT FROM": Classification.DIRECT, + "IS NOT DISTINCT FROM": Classification.DIRECT, + # CASE (both forms are rowed here, not under Conditional functions) + "CASE WHEN": Classification.DIRECT, + "CASE expr WHEN v1 THEN r1 ... END (simple)": Classification.DIRECT, # CONVENTION_DIVERGENCES + # Concatenation, grouping, literals + "str1 || str2": Classification.DIRECT, + "Parentheses — expression grouping": Classification.DIRECT, # CONVENTION_DIVERGENCES + "TRUE, FALSE": Classification.DIRECT, + # Aggregate modifier + "DISTINCT aggregate modifier": Classification.PASSTHROUGH, # CONVENTION_DIVERGENCES + # References + "Column / metric reference — field, dataset.field": Classification.DIRECT, # CONVENTION_DIVERGENCES + # The one unmappable row in the whole 146-row catalog + "EXISTS_IN()": Classification.UNMAPPABLE, # CONVENTION_DIVERGENCES +} + +#: Expected `Variant` for every passthrough row in this family. +EXPECTED_VARIANTS: dict[str, Variant] = { + "str ILIKE pattern": Variant.BOOL, + "DISTINCT aggregate modifier": Variant.NUMBER_AGGREGATE, +} + + +def test_row_count_for_this_family(): + ours = [c for c in CATALOG.values() if c.spec_name in EXPECTED] + assert len(ours) == 33 + + +def test_classifications(): + for name, expected in EXPECTED.items(): + assert CATALOG[name].classification is expected, name + + +def test_direct_count(): + direct_names = {n for n, c in EXPECTED.items() if c is Classification.DIRECT} + assert len(direct_names) == 30 + + +def test_passthrough_count_and_variants(): + passthrough_names = {n for n, c in EXPECTED.items() if c is Classification.PASSTHROUGH} + assert len(passthrough_names) == 2 + assert passthrough_names == set(EXPECTED_VARIANTS) + for name, variant in EXPECTED_VARIANTS.items(): + assert CATALOG[name].variant is variant, name + + +def test_unmappable_count(): + unmappable_names = {n for n, c in EXPECTED.items() if c is Classification.UNMAPPABLE} + assert unmappable_names == {"EXISTS_IN()"} + + +# -------------------------------------------------------------------------- +# Rows the document only explains via its surrounding prose. +# -------------------------------------------------------------------------- + +def test_exists_in_is_unmappable_with_no_template_and_no_variant(): + # The single unmappable row in the entire 146-row catalog. Named at :131 + # as the sanctioned way to filter on a subquery, but defined nowhere in + # the specification - no signature, no argument order, no semantics - + # so there is nothing to translate, let alone compose. + row = CATALOG["EXISTS_IN()"] + assert row.classification is Classification.UNMAPPABLE + assert row.template is None + assert row.variant is None + + +def test_ilike_is_passthrough_because_case_fold_has_no_native_form(): + # Case-insensitive matching has no native form, and the usual workaround + # (fold both sides with `lower`) is itself a passthrough - there is + # nothing native to compose from. + row = CATALOG["str ILIKE pattern"] + assert row.classification is Classification.PASSTHROUGH + assert row.variant is Variant.BOOL + + +def test_like_stays_direct_despite_no_native_starts_with_ends_with(): + # Unlike ILIKE, LIKE's prefix/suffix/contains compositions use only + # native functions (strpos/substr/contains), so it stays direct. + row = CATALOG["str LIKE pattern"] + assert row.classification is Classification.DIRECT + + +def test_distinct_modifier_is_passthrough_except_count_distinct(): + # ThoughtSpot has exactly one distinct-aware aggregate - unique count, + # i.e. COUNT(DISTINCT) - which already has its own catalog row. Every + # other DISTINCT aggregate (e.g. SUM(DISTINCT ...)) is a passthrough. + row = CATALOG["DISTINCT aggregate modifier"] + assert row.classification is Classification.PASSTHROUGH + assert row.variant is Variant.NUMBER_AGGREGATE + assert "sql_number_aggregate_op (" not in row.template + + +def test_both_case_forms_are_direct_with_a_mandatory_typed_else(): + # No native CASE; both forms compose as an `else if` chain. The final + # `else` is mandatory and must be type-matched - omitting it raises + # "Unknown data type" at import. + searched = CATALOG["CASE WHEN"] + simple = CATALOG["CASE expr WHEN v1 THEN r1 ... END (simple)"] + assert searched.classification is Classification.DIRECT + assert simple.classification is Classification.DIRECT + assert "else if" in searched.template + assert "else if" in simple.template + + +def test_in_and_not_in_use_the_curly_brace_list_form(): + # Live-verified 2026-07-29: the round-paren form is rejected. The + # curly-brace delimiter is the confirmed syntax. + in_row = CATALOG["IN"] + not_in_row = CATALOG["NOT IN"] + assert in_row.classification is Classification.DIRECT + assert not_in_row.classification is Classification.DIRECT + assert "{" in in_row.template and "}" in in_row.template + assert "not (" in not_in_row.template + + +def test_is_distinct_from_tests_both_null_before_either_null(): + # No native null-safe comparison, but the three-case truth table is + # exactly expressible. The nesting order matters: both-null must be + # tested before either-null, or the either-null branch would also catch + # the both-null case first. + row = CATALOG["IS DISTINCT FROM"] + assert row.classification is Classification.DIRECT + both_null_idx = row.template.index("and") + either_null_idx = row.template.index("or") + assert both_null_idx < either_null_idx + + +def test_not_expr_is_a_function_form_not_a_prefix_operator(): + # "not [x]" does not parse; NOT is a function call with parentheses. + row = CATALOG["NOT expr"] + assert row.classification is Classification.DIRECT + assert row.template.startswith("not (") + + +def test_concatenation_operator_and_function_share_one_target(): + # ThoughtSpot has no concatenation operator at all - `+` is numeric-only + # - so `||` and CONCAT(...) both land on concat ( ). + row = CATALOG["str1 || str2"] + assert row.classification is Classification.DIRECT + assert row.template.startswith("concat (") + + +def test_boolean_literals_row_is_the_merged_true_false_entry(): + # The Boolean Functions table's Syntax cell merges TRUE and FALSE into + # one comma-joined entry, matching what spec_construct_names() extracts - + # not two separate catalog rows. + row = CATALOG["TRUE, FALSE"] + assert row.classification is Classification.DIRECT + assert "true" in row.template and "false" in row.template diff --git a/converters/thoughtspot/tests/expressions/test_catalog_string.py b/converters/thoughtspot/tests/expressions/test_catalog_string.py new file mode 100644 index 00000000..6fbb8a0a --- /dev/null +++ b/converters/thoughtspot/tests/expressions/test_catalog_string.py @@ -0,0 +1,191 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Catalog coverage: String functions. + +Source: the `String functions` section of docs/ossie/ts-ossie-function-mapping.md +(thoughtspot-agent-skills repo, not vendored here). 21 rows total — 10 direct / +11 passthrough / 0 unmappable. + +This family is over half passthrough, and the reasons are counter-intuitive: +TRIM/LTRIM/RTRIM/REPLACE/LOWER/UPPER are passthrough because ThoughtSpot has no +native trim/replace/lower/upper function at all — not because they behave +differently. STARTSWITH/ENDSWITH are the opposite surprise: direct despite +having no native function, because the composition out of strpos/substr/strlen +is exact and uses only native functions. + +Construct names in this family are spelled identically to the mapping +document's own row headers — none of this family's keys diverge the way CAST/ +TRY_CAST or the Date/time typed literals did (see catalog.py's module +docstring, "Spelling" section, and `spec_construct_names()` itself). +""" +from ossie_thoughtspot.expressions import CATALOG +from ossie_thoughtspot.expressions._types import Classification, Variant + +EXPECTED: dict[str, Classification] = { + "CONCAT(str1, str2, ...)": Classification.DIRECT, + "LENGTH(str)": Classification.DIRECT, + "LOWER(str)": Classification.PASSTHROUGH, + "UPPER(str)": Classification.PASSTHROUGH, + "TRIM(str)": Classification.PASSTHROUGH, + "LTRIM(str)": Classification.PASSTHROUGH, + "RTRIM(str)": Classification.PASSTHROUGH, + "LEFT(str, n)": Classification.DIRECT, + "RIGHT(str, n)": Classification.DIRECT, + "SUBSTRING(str, start, length)": Classification.DIRECT, + "REPLACE(str, from, to)": Classification.PASSTHROUGH, + "SPLIT_PART(str, delimiter, part)": Classification.PASSTHROUGH, + "POSITION(substr IN str)": Classification.DIRECT, + "CHARINDEX(substr, str)": Classification.DIRECT, + "CONTAINS(str, substr)": Classification.DIRECT, + "STARTSWITH(str, prefix)": Classification.DIRECT, + "ENDSWITH(str, suffix)": Classification.DIRECT, + "REGEXP_LIKE(str, pattern)": Classification.PASSTHROUGH, + "REGEXP_EXTRACT(str, pattern)": Classification.PASSTHROUGH, + "REGEXP_REPLACE(str, pattern, replacement)": Classification.PASSTHROUGH, + "REGEXP_COUNT(str, pattern)": Classification.PASSTHROUGH, +} + +#: Expected `Variant` for every passthrough row in this family. Getting +#: this wrong is the failure mode with no safety net: the wrong variant emits a +#: column that imports cleanly and then aggregates or types wrongly, and +#: nothing downstream catches it. +EXPECTED_VARIANTS: dict[str, Variant] = { + "LOWER(str)": Variant.STRING, + "UPPER(str)": Variant.STRING, + "TRIM(str)": Variant.STRING, + "LTRIM(str)": Variant.STRING, + "RTRIM(str)": Variant.STRING, + "REPLACE(str, from, to)": Variant.STRING, + "SPLIT_PART(str, delimiter, part)": Variant.STRING, + "REGEXP_LIKE(str, pattern)": Variant.BOOL, + "REGEXP_EXTRACT(str, pattern)": Variant.STRING, + "REGEXP_REPLACE(str, pattern, replacement)": Variant.STRING, + "REGEXP_COUNT(str, pattern)": Variant.INT, +} + + +def test_row_count_for_this_family(): + ours = [c for c in CATALOG.values() if c.spec_name in EXPECTED] + assert len(ours) == 21 + + +def test_classifications(): + for name, expected in EXPECTED.items(): + assert CATALOG[name].classification is expected, name + + +def test_passthrough_count_and_variants(): + passthrough_names = {n for n, c in EXPECTED.items() if c is Classification.PASSTHROUGH} + assert len(passthrough_names) == 11 + assert passthrough_names == set(EXPECTED_VARIANTS) + for name, variant in EXPECTED_VARIANTS.items(): + assert CATALOG[name].variant is variant, name + + +def test_direct_count(): + direct_names = {n for n, c in EXPECTED.items() if c is Classification.DIRECT} + assert len(direct_names) == 10 + + +def test_no_unmappable_rows_in_this_family(): + assert not any(c is Classification.UNMAPPABLE for c in EXPECTED.values()) + + +# -------------------------------------------------------------------------- +# Rows the document only explains via its surrounding prose. +# -------------------------------------------------------------------------- + +def test_the_whole_trim_family_is_passthrough_not_just_two_sided_trim(): + # Live-verified 2026-07-29: ThoughtSpot has no + # native trim at all, rejected with `Search did not find "trim ("`. TRIM, + # LTRIM and RTRIM are all passthrough for the same reason, not because a + # two-sided trim exists and the one-sided forms don't compose from it. + for name in ("TRIM(str)", "LTRIM(str)", "RTRIM(str)"): + row = CATALOG[name] + assert row.classification is Classification.PASSTHROUGH + assert row.variant is Variant.STRING + + +def test_lower_and_upper_have_no_native_equivalent(): + # No native lower/upper in ThoughtSpot — the most-used functions in the + # whole passthrough set, per the document's own framing. + assert CATALOG["LOWER(str)"].classification is Classification.PASSTHROUGH + assert CATALOG["UPPER(str)"].classification is Classification.PASSTHROUGH + + +def test_replace_was_direct_on_documentation_but_moved_on_live_verification(): + # Live-verified 2026-07-29: rejected with + # `Search did not find "replace ("`. The row was direct on documentation + # alone; the live pass moved it to the documented pass-through fallback. + row = CATALOG["REPLACE(str, from, to)"] + assert row.classification is Classification.PASSTHROUGH + assert row.variant is Variant.STRING + + +def test_startswith_and_endswith_are_direct_despite_no_native_function(): + # No native starts_with/ends_with (live-verified 2026-07-29), + # but both compositions use only native functions + # (strpos/substr/strlen), so they stay direct rather than + # passthrough. + for name in ("STARTSWITH(str, prefix)", "ENDSWITH(str, suffix)"): + row = CATALOG[name] + assert row.classification is Classification.DIRECT + assert row.variant is None + assert "sql_" not in row.template + + +def test_substring_index_base_shift_is_present_in_the_template(): + # ANSI SUBSTRING is 1-based; ThoughtSpot's substr is 0-based. The -1 shift + # is mandatory and is the single most likely off-by-one in the mapping. + row = CATALOG["SUBSTRING(str, start, length)"] + assert row.classification is Classification.DIRECT + assert "- 1" in row.template + + +def test_position_and_charindex_reverse_operand_order(): + # ThoughtSpot's strpos takes the haystack first; the specification's + # POSITION(substr IN str) and CHARINDEX(substr, str) both put the needle + # first, so both templates reverse the operand order onto strpos. + position = CATALOG["POSITION(substr IN str)"] + charindex = CATALOG["CHARINDEX(substr, str)"] + assert position.classification is Classification.DIRECT + assert charindex.classification is Classification.DIRECT + assert position.template == charindex.template + + +def test_no_regular_expression_support_of_any_kind(): + # ThoughtSpot has no regex engine at all — every REGEXP_* row is + # passthrough, with no native fallback for any of them. + for name in ( + "REGEXP_LIKE(str, pattern)", + "REGEXP_EXTRACT(str, pattern)", + "REGEXP_REPLACE(str, pattern, replacement)", + "REGEXP_COUNT(str, pattern)", + ): + assert CATALOG[name].classification is Classification.PASSTHROUGH + + +def test_regexp_like_is_bool_variant_not_string(): + # REGEXP_LIKE returns a boolean, so its pass-through variant is + # sql_bool_op, not sql_string_op like its REGEXP_* siblings. + assert CATALOG["REGEXP_LIKE(str, pattern)"].variant is Variant.BOOL + + +def test_regexp_count_is_int_variant(): + # REGEXP_COUNT returns an integer count, so its variant is sql_int_op. + assert CATALOG["REGEXP_COUNT(str, pattern)"].variant is Variant.INT diff --git a/converters/thoughtspot/tests/expressions/test_catalog_window.py b/converters/thoughtspot/tests/expressions/test_catalog_window.py new file mode 100644 index 00000000..f40ee42a --- /dev/null +++ b/converters/thoughtspot/tests/expressions/test_catalog_window.py @@ -0,0 +1,270 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Catalog coverage: Window functions - the last family, completing the catalog. + +Source: the `Window functions` section of docs/ossie/ts-ossie-function-mapping.md +(thoughtspot-agent-skills repo, not vendored here), plus the "Window rows +live-confirmed - 2026-07-30" section that records the 52-probe evidence behind the +classifications. 14 rows total - 5 direct / 9 passthrough / 0 unmappable. + +This is the hardest family. Three constraints govern it: + +- A raw aggregate cannot be nested inside a ThoughtSpot window function. +- The ORDER BY column must be a physical column reference, not a formula. +- A ThoughtSpot window formula cannot declare its own PARTITION BY; the + partition is always completed from the query's own dimensions. This is why nine + of fourteen rows are passthrough, and why LAG, LEAD, the OVER clause and window + aggregation moved direct -> passthrough after 52 live probes on 2026-07-30. + `rank` / `rank_percentile` have arity fixed at exactly two, proven by rejection + on a live instance (`Function rank expects only 2 arguments`). + +`FIRST_VALUE` / `LAST_VALUE` are the section's one exception: they take a genuine +explicit partition argument and a genuine explicit order axis, so the formula does +define its own window, and they stay direct along with `RANK`, `PERCENT_RANK` and +the frame-clause boundaries (whose native reach is now proven by rejection rather +than asserted). + +`PERCENT_RANK` is direct via `rank_percentile`, but `CUME_DIST` is deliberately +NOT: `PERCENT_RANK` divides by n-1 and starts at 0; `CUME_DIST` divides by n and +ends at 1. They agree on no row of a tie-free window except the last, so +`rank_percentile` is not a substitute and `CUME_DIST` stays passthrough with no +native fallback at all. + +Three of the family's 14 rows have no discrete row of their own in the upstream +core-spec/expression_language.md - the generic `OVER (...)` syntax template and its +frame-clause bullet list are a fenced code block, and window aggregation is prose - +so they are keyed via `CONVENTION_DIVERGENCES` rather than `spec_construct_names()`: +the `OVER` clause, the frame clause, and window aggregation. The other 11 rows key +on `spec_construct_names()`'s own extraction from the "Ranking Functions" and +"Offset Functions" tables' `Syntax` column - confirmed live before writing this +file. +""" +from ossie_thoughtspot.expressions import CATALOG +from ossie_thoughtspot.expressions._types import Classification, Variant +from ossie_thoughtspot.expressions.emit import emit_direct, emit_passthrough +from ossie_thoughtspot.issues import IssueLog + +EXPECTED: dict[str, Classification] = { + # Ranking functions (spec_construct_names() extracts the Syntax-column value) + "ROW_NUMBER() OVER (...)": Classification.PASSTHROUGH, + "RANK() OVER (...)": Classification.DIRECT, + "DENSE_RANK() OVER (...)": Classification.PASSTHROUGH, + "NTILE(n) OVER (...)": Classification.PASSTHROUGH, + "PERCENT_RANK() OVER (...)": Classification.DIRECT, + "CUME_DIST() OVER (...)": Classification.PASSTHROUGH, + # Offset functions + "LAG(expr, offset, default) OVER (...)": Classification.PASSTHROUGH, + "LEAD(expr, offset, default) OVER (...)": Classification.PASSTHROUGH, + "FIRST_VALUE(expr) OVER (...)": Classification.DIRECT, + "LAST_VALUE(expr) OVER (...)": Classification.DIRECT, + "NTH_VALUE(expr, n) OVER (...)": Classification.PASSTHROUGH, + # Structural rows (CONVENTION_DIVERGENCES - no discrete spec table row) + "OVER (PARTITION BY ... ORDER BY ...) clause": Classification.PASSTHROUGH, + "Frame clause — ROWS BETWEEN ... / RANGE BETWEEN ...": Classification.DIRECT, + "Window aggregation — AGG(expr) OVER (...)": Classification.PASSTHROUGH, +} + +#: Expected `Variant` for every passthrough row in this family. +EXPECTED_VARIANTS: dict[str, Variant] = { + "ROW_NUMBER() OVER (...)": Variant.INT_AGGREGATE, + "DENSE_RANK() OVER (...)": Variant.INT_AGGREGATE, + "NTILE(n) OVER (...)": Variant.INT_AGGREGATE, + "CUME_DIST() OVER (...)": Variant.NUMBER_AGGREGATE, + "LAG(expr, offset, default) OVER (...)": Variant.NUMBER_AGGREGATE, + "LEAD(expr, offset, default) OVER (...)": Variant.NUMBER_AGGREGATE, + "NTH_VALUE(expr, n) OVER (...)": Variant.NUMBER_AGGREGATE, + "OVER (PARTITION BY ... ORDER BY ...) clause": Variant.NUMBER_AGGREGATE, + "Window aggregation — AGG(expr) OVER (...)": Variant.NUMBER_AGGREGATE, +} + + +def test_row_count_for_this_family(): + ours = [c for c in CATALOG.values() if c.spec_name in EXPECTED] + assert len(ours) == 14 + + +def test_classifications(): + for name, expected in EXPECTED.items(): + assert CATALOG[name].classification is expected, name + + +def test_direct_count(): + direct_names = {n for n, c in EXPECTED.items() if c is Classification.DIRECT} + assert len(direct_names) == 5 + + +def test_passthrough_count_and_variants(): + passthrough_names = {n for n, c in EXPECTED.items() if c is Classification.PASSTHROUGH} + assert len(passthrough_names) == 9 + assert passthrough_names == set(EXPECTED_VARIANTS) + for name, variant in EXPECTED_VARIANTS.items(): + assert CATALOG[name].variant is variant, name + + +def test_no_unmappable_rows_in_this_family(): + unmappable_names = {n for n, c in EXPECTED.items() if c is Classification.UNMAPPABLE} + assert unmappable_names == set() + + +# -------------------------------------------------------------------------- +# The window-formula PARTITION BY constraint this family turns on. Locking in the four rows the July rework +# moved off `direct`, and the two structural rows (partition/frame) that are +# NOT swept up by the same reclassification. +# -------------------------------------------------------------------------- + +def test_the_four_rows_e13_reclassified_are_passthrough_not_direct(): + # LAG, LEAD, the OVER clause and window aggregation all moved direct -> + # passthrough on 2026-07-30 because a ThoughtSpot window formula cannot + # declare its own PARTITION BY. A regression here (restoring one to + # direct because a native idiom exists) would silently reintroduce a + # formula that only happens to be correct when the search's own + # dimensions match the intended partition. + for name in ( + "LAG(expr, offset, default) OVER (...)", + "LEAD(expr, offset, default) OVER (...)", + "OVER (PARTITION BY ... ORDER BY ...) clause", + "Window aggregation — AGG(expr) OVER (...)", + ): + assert CATALOG[name].classification is Classification.PASSTHROUGH, name + + +def test_first_value_and_last_value_are_the_surviving_exception(): + # The only window rows whose direct verdict survived the 2026-07-30 rework: + # first_value/last_value take a genuine explicit partition AND order axis, + # so the formula does define its own window. + for name in ("FIRST_VALUE(expr) OVER (...)", "LAST_VALUE(expr) OVER (...)"): + row = CATALOG[name] + assert row.classification is Classification.DIRECT + assert "query_groups" in row.template + + +def test_frame_clause_stays_direct_scoped_to_boundaries_only(): + # direct for the frame boundaries alone - the partition loss is counted + # once, on the OVER clause row, not twice. + row = CATALOG["Frame clause — ROWS BETWEEN ... / RANGE BETWEEN ..."] + assert row.classification is Classification.DIRECT + + +# -------------------------------------------------------------------------- +# rank / rank_percentile: arity fixed at exactly two, proven by rejection. +# -------------------------------------------------------------------------- + +def test_rank_and_percent_rank_carry_no_partition_argument(): + # rank and rank_percentile are both fixed at exactly two arguments + # (live-confirmed by rejection: "Function rank expects only 2 + # arguments"), so neither template may carry a PARTITION BY - there is no + # argument slot for one, in any spelling. + for name in ("RANK() OVER (...)", "PERCENT_RANK() OVER (...)"): + row = CATALOG[name] + assert row.classification is Classification.DIRECT + assert "partition" not in row.template.lower() + + +def test_percent_rank_is_direct_via_rank_percentile_with_both_adjustments(): + # Two adjustments are both required: the scale (ThoughtSpot 0-100, + # specification 0-1) and the inversion. Dropping either produces a + # plausible-looking column that is wrong everywhere. + row = CATALOG["PERCENT_RANK() OVER (...)"] + assert "rank_percentile" in row.template + assert "100" in row.template + assert row.template.strip().startswith("1 -") + + +def test_cume_dist_is_not_substituted_by_rank_percentile(): + # The document is explicit that this is NOT a permissible substitution: + # PERCENT_RANK divides by n-1 and starts at 0; CUME_DIST divides by n and + # ends at 1. Guards against "restoring" this row because the two names + # look equivalent. + row = CATALOG["CUME_DIST() OVER (...)"] + assert row.classification is Classification.PASSTHROUGH + assert "rank_percentile" not in row.template + + +# -------------------------------------------------------------------------- +# Which templates carry PARTITION BY and therefore require partition_column +# at emission time. Exactly the rows the document gives a PARTITION BY clause to. +# -------------------------------------------------------------------------- + +def test_only_the_documented_rows_carry_partition_by(): + partitioned = { + "ROW_NUMBER() OVER (...)", + "LAG(expr, offset, default) OVER (...)", + "LEAD(expr, offset, default) OVER (...)", + "Window aggregation — AGG(expr) OVER (...)", + } + for name in EXPECTED_VARIANTS: + carries = "partition by" in CATALOG[name].template.lower() + assert carries == (name in partitioned), name + + +# -------------------------------------------------------------------------- +# Brace escaping: FIRST_VALUE/LAST_VALUE are DIRECT rows whose ThoughtSpot +# rendering uses `{ ... }` list syntax for the axis argument. DIRECT templates +# render via str.format (emit_direct), so a literal brace must be doubled or +# the call raises "unexpected '{' in field name" - exactly the IN/NOT IN +# brace-escaping bug the catalog hit earlier. This exercises emit_direct +# directly, not just a substring check on the template text, so it would have +# caught that bug. +# -------------------------------------------------------------------------- + +def test_first_value_and_last_value_render_with_single_braces(): + for name in ("FIRST_VALUE(expr) OVER (...)", "LAST_VALUE(expr) OVER (...)"): + row = CATALOG[name] + rendered = emit_direct(row, []) + assert "{{" not in rendered and "}}" not in rendered + assert "{" in rendered and "}" in rendered + + +# -------------------------------------------------------------------------- +# The window-aggregation template previously carried a literal U+2026 +# ellipsis ("ROWS BETWEEN …") — the mapping document's own prose shorthand for +# "a frame clause goes here", not renderable SQL. It passed __post_init__, the +# partition check and declared a satisfiable 3-argument arity, so +# emit_passthrough rendered it as-is: a warehouse SQL syntax error far from the +# converter. The fix supplies a concrete, valid exemplar frame instead (the +# same convention as NTILE's literal 4). +# -------------------------------------------------------------------------- + +def test_window_aggregation_template_has_no_literal_ellipsis(): + row = CATALOG["Window aggregation — AGG(expr) OVER (...)"] + assert "…" not in row.template + + +def test_window_aggregation_renders_valid_sql_via_emit_passthrough(): + row = CATALOG["Window aggregation — AGG(expr) OVER (...)"] + log = IssueLog() + out = emit_passthrough( + row, ["[T::Amount]", "[T::Region]", "[T::OrderDate]"], log, + object_ref="metric:RunningTotal", partition_column="[T::Region]", + ) + assert "…" not in out + assert "ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW" in out + assert out.startswith("group_aggregate (") + + +def test_every_direct_row_in_this_family_has_natural_arity_zero(): + # Every direct row in this family records the document's own worked + # example (symbolic bracket names like [m]/[dim]/[ord]/[attr]/[T::date]), + # not a numbered {0}/{1} substitution slot - the same out-of-scope-dispatch + # treatment as CASE WHEN's c1/r1 names and CAST's per-type table. So each + # renders with zero arguments. + for name, classification in EXPECTED.items(): + if classification is not Classification.DIRECT: + continue + row = CATALOG[name] + assert emit_direct(row, []) == row.template.replace("{{", "{").replace("}}", "}") diff --git a/converters/thoughtspot/tests/expressions/test_emit.py b/converters/thoughtspot/tests/expressions/test_emit.py new file mode 100644 index 00000000..d0c717c0 --- /dev/null +++ b/converters/thoughtspot/tests/expressions/test_emit.py @@ -0,0 +1,271 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +"""Tests for the expression emitters. + +Three emitter signatures, one per `Classification`: + + emit_direct(construct, args) -> str + emit_passthrough(construct, args, log, *, object_ref, has_parameter=False) -> str + emit_unmappable(construct, log, *, object_ref) -> None + +`object_ref` is required on both issue-raising emitters: an issue names +the function, the object and the reason — an emitter that cannot name the object +structurally cannot satisfy it. +""" +import re + +import pytest + +from ossie_thoughtspot.expressions import CATALOG +from ossie_thoughtspot.expressions._types import Classification, Construct, Variant +from ossie_thoughtspot.expressions.emit import _placeholder_count, emit_direct, emit_passthrough, emit_unmappable +from ossie_thoughtspot.issues import IssueLog, Severity + +SUM = Construct("SUM(expr)", Classification.DIRECT, template="sum ( {0} )") +STDDEV_POP = Construct( + "STDDEV_POP(expr)", Classification.PASSTHROUGH, + template="STDDEV_POP({0})", variant=Variant.NUMBER_AGGREGATE, +) + + +def test_emit_direct_substitutes_positionally(): + assert emit_direct(SUM, ["[ORDERS::Amount]"]) == "sum ( [ORDERS::Amount] )" + + +def test_emit_direct_rejects_an_argument_count_mismatch(): + # Silently dropping or reusing an argument would produce a formula that imports + # and computes the wrong thing. + with pytest.raises(ValueError, match="expects 1 argument"): + emit_direct(SUM, ["a", "b"]) + + +def test_emit_direct_refuses_a_non_direct_construct(): + with pytest.raises(ValueError, match="STDDEV_POP.*direct"): + emit_direct(STDDEV_POP, ["[x]"]) + + +def test_emit_passthrough_wraps_the_body_in_its_variant(): + log = IssueLog() + out = emit_passthrough(STDDEV_POP, ["[ORDERS::Amount]"], log, object_ref="metric:Revenue") + assert out == 'sql_number_aggregate_op ( "STDDEV_POP({0})" , [ORDERS::Amount] )' + + +def test_emit_passthrough_always_raises_a_warning_issue(): + # The mapping document requires every pass-through to surface, because it embeds + # raw warehouse SQL and is opaque to ThoughtSpot's query planner. + log = IssueLog() + emit_passthrough(STDDEV_POP, ["[ORDERS::Amount]"], log, object_ref="metric:Revenue") + assert log.count_by_severity() == {"WARNING": 1} + issue = log.as_dicts()[0] + assert "STDDEV_POP" in issue["message"] # names the function + assert issue["object_ref"] # names the object + + +def test_emit_passthrough_refuses_a_runtime_parameter(): + # A sql_*_op whose arguments include a ThoughtSpot parameter cannot resolve to + # static SQL, so it is not portable in either direction. + log = IssueLog() + with pytest.raises(ValueError, match="runtime parameter"): + emit_passthrough( + STDDEV_POP, ["[Threshold Parameter]"], log, + object_ref="metric:Revenue", has_parameter=True, + ) + # The raise must precede any log.add — a refused call must never also emit a + # misleading WARNING that tells the user to "review" a formula they never got. + assert log.as_dicts() == [] + + +def test_emit_passthrough_refuses_a_non_passthrough_construct(): + with pytest.raises(ValueError, match="SUM.*passthrough"): + emit_passthrough(SUM, ["[x]"], IssueLog(), object_ref="metric:Revenue") + + +def test_emit_passthrough_rejects_an_argument_count_mismatch(): + # emit_direct already refuses a mismatch (test_emit_direct_rejects_an_argument_count_ + # mismatch above); emit_passthrough previously had no equivalent guard. Reproduces the + # concrete failure on a real catalog row: TIMESTAMP_NTZ's template is a zero-placeholder + # exemplar (its literal 2024-01-15 date is baked in — see the Construct.template + # docstring's "exemplar" case), so passing it a real value silently appended an unused + # sql_date_time_op argument the template never consumes, keeping the hardcoded date in + # the rendered SQL instead of failing loudly. + literal_timestamp = CATALOG["TIMESTAMP_NTZ '2024-01-15 10:30:00'"] + log = IssueLog() + with pytest.raises(ValueError, match="expects 0 arguments"): + emit_passthrough( + literal_timestamp, ["'2026-03-04 09:00:00'"], log, object_ref="metric:X", + ) + # Same discipline as the runtime-parameter refusal above: no misleading WARNING for a call that + # was refused. + assert log.as_dicts() == [] + + +def test_emit_passthrough_renders_the_exemplar_with_its_own_natural_arity(): + literal_timestamp = CATALOG["TIMESTAMP_NTZ '2024-01-15 10:30:00'"] + log = IssueLog() + out = emit_passthrough(literal_timestamp, [], log, object_ref="metric:X") + assert out == 'sql_date_time_op ( "CAST(\'2024-01-15 10:30:00\' AS TIMESTAMP)" )' + + +def test_emit_unmappable_raises_an_issue_and_returns_nothing(): + c = Construct("EXISTS_IN(x)", Classification.UNMAPPABLE) + log = IssueLog() + assert emit_unmappable(c, log, object_ref="metric:Revenue") is None + issue = log.as_dicts()[0] + assert issue["severity"] == Severity.ERROR.value + assert "EXISTS_IN" in issue["message"] and "metric:Revenue" == issue["object_ref"] + + +def test_emit_unmappable_refuses_a_mappable_construct(): + with pytest.raises(ValueError, match="SUM.*unmappable"): + emit_unmappable(SUM, IssueLog(), object_ref="metric:Revenue") + + +# -------------------------------------------------------------------------- +# A pass-through carrying PARTITION BY is wrapped in group_aggregate so the +# partition column reaches GROUP BY even when the user's search omits it. +# -------------------------------------------------------------------------- + +ROW_NUMBER = Construct( + "ROW_NUMBER() OVER (...)", Classification.PASSTHROUGH, + template="ROW_NUMBER() OVER (PARTITION BY {0} ORDER BY {1})", + variant=Variant.INT_AGGREGATE, +) + + +def test_emit_passthrough_wraps_a_partitioned_call_in_group_aggregate(): + log = IssueLog() + out = emit_passthrough( + ROW_NUMBER, ["[T::Region]", "[T::OrderDate]"], log, + object_ref="metric:RowNum", partition_column="[T::Region]", + ) + assert out == ( + 'group_aggregate ( sql_int_aggregate_op ( ' + '"ROW_NUMBER() OVER (PARTITION BY {0} ORDER BY {1})" , ' + '[T::Region] , [T::OrderDate] ) , ' + 'query_groups ( ) + { [T::Region] } , query_filters ( ) )' + ) + + +def test_emit_passthrough_without_a_partition_column_is_unwrapped(): + log = IssueLog() + out = emit_passthrough(STDDEV_POP, ["[x]"], log, object_ref="metric:Revenue") + assert not out.startswith("group_aggregate") + + +def test_emit_passthrough_requires_partition_column_when_template_carries_partition_by(): + # Enforced rather than left to convention: ROW_NUMBER's template carries a + # literal PARTITION BY, so omitting partition_column must fail loudly rather + # than silently emit an unwrapped, only-sometimes-correct pass-through. + log = IssueLog() + with pytest.raises(ValueError, match="PARTITION BY"): + emit_passthrough( + ROW_NUMBER, ["[T::Region]", "[T::OrderDate]"], log, + object_ref="metric:RowNum", + ) + + +def test_emit_passthrough_refuses_a_partition_column_for_a_template_with_no_partition_by(): + # Symmetric check: STDDEV_POP's template has no PARTITION BY, so supplying + # partition_column anyway is equally a mistake (a miscopied catalog row) + # and must also fail loudly. + log = IssueLog() + with pytest.raises(ValueError, match="PARTITION BY"): + emit_passthrough( + STDDEV_POP, ["[x]"], log, + object_ref="metric:Revenue", partition_column="[T::Region]", + ) + + +def test_emit_passthrough_detects_partition_by_with_irregular_whitespace(): + # A plain substring match on "partition by" misses "PARTITION BY" (two + # spaces) or a newline between the words, which would silently leave the + # guard defeated in both directions. Regex with \s+ must still catch it. + irregular = Construct( + "IRREGULAR_WHITESPACE(expr)", Classification.PASSTHROUGH, + template="SOME_FUNC({0}) OVER (PARTITION BY {0} ORDER BY {1})", + variant=Variant.NUMBER_AGGREGATE, + ) + log = IssueLog() + with pytest.raises(ValueError, match="PARTITION BY"): + emit_passthrough(irregular, ["[dim]", "[ord]"], log, object_ref="metric:X") + + +# -------------------------------------------------------------------------- +# Catalog-wide sweep: every DIRECT row must actually render, not merely read +# correctly. This is the check that caught the IN/NOT IN brace-escaping bug: +# both templates embedded ThoughtSpot's literal `{ ... }` set syntax unescaped +# in a Python format string, so the call with the CORRECT, natural-arity +# argument count (the one a real caller makes) crashed with `ValueError: +# unexpected '{' in field name` — a non-obvious failure, not a clean domain +# error, and invisible to any test that only inspects `construct.template` as +# a string (e.g. `"{" in row.template`) rather than executing it. A catalog +# author can transcribe a document cell containing a literal brace, +# parenthesis, or any other str.format metacharacter for any future family +# and reintroduce exactly this shape of bug; this sweep is general over every +# DIRECT row in CATALOG, not scoped to the rows that caught it originally, +# precisely so that it does. +# -------------------------------------------------------------------------- + +def test_every_direct_catalog_row_renders_with_its_own_natural_arity(): + failures = [] + for name, construct in CATALOG.items(): + if construct.classification is not Classification.DIRECT: + continue + arity = _placeholder_count(construct.template) + args = [f"arg{i}" for i in range(arity)] + try: + emit_direct(construct, args) + except Exception as exc: # noqa: BLE001 - want to report every failure, not stop at the first + failures.append(f"{name!r} ({arity} args): {exc!r}") + assert not failures, "DIRECT rows that fail to render with their own natural arity:\n" + "\n".join( + failures + ) + + +# -------------------------------------------------------------------------- +# The passthrough counterpart to the sweep above: every PASSTHROUGH row must +# actually render with its own natural arity, not merely read correctly. A +# catalog edit that desyncs a template's `{n}` placeholders from its intended +# arity — on any passthrough row, not just the one pinned regression case +# above — would otherwise go uncaught until something later tried to emit +# that specific row. Whether a row needs `partition_column` is derived from +# its own template (the same `PARTITION BY` check emit_passthrough itself +# makes), not hardcoded, so a row that gains or loses a PARTITION BY +# stays in sync with this sweep automatically. +# -------------------------------------------------------------------------- + +def test_every_passthrough_catalog_row_renders_with_its_own_natural_arity(): + failures = [] + for name, construct in CATALOG.items(): + if construct.classification is not Classification.PASSTHROUGH: + continue + arity = _placeholder_count(construct.template) + args = [f"arg{i}" for i in range(arity)] + carries_partition_by = bool(re.search(r"partition\s+by", construct.template, re.IGNORECASE)) + partition_column = "[partition_col]" if carries_partition_by else None + log = IssueLog() + try: + emit_passthrough( + construct, args, log, object_ref="metric:sweep", partition_column=partition_column + ) + except Exception as exc: # noqa: BLE001 - want to report every failure, not stop at the first + failures.append(f"{name!r} ({arity} args): {exc!r}") + assert not failures, "PASSTHROUGH rows that fail to render with their own natural arity:\n" + "\n".join( + failures + ) diff --git a/converters/thoughtspot/tests/expressions/test_reverse.py b/converters/thoughtspot/tests/expressions/test_reverse.py new file mode 100644 index 00000000..c4793ad5 --- /dev/null +++ b/converters/thoughtspot/tests/expressions/test_reverse.py @@ -0,0 +1,605 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The reverse-direction inventory (ThoughtSpot -> Ossie). + +Source: the "Reverse direction (ThoughtSpot -> Ossie)" section of +docs/ossie/ts-ossie-function-mapping.md (thoughtspot-agent-skills repo, not +vendored here), all three sub-sections: conditional aggregates and arithmetic +helpers; window, LOD and semi-additive functions; runtime, display and +calendar concepts. + +One assertion group per ThoughtSpot function: does it compose, or +does it stash (custom_extensions + issue)? Composers assert the +exact emitted Ossie expression. Stash/partial entries assert the issue's +code, severity and object_ref, since every stash issue names the function, +the object and the reason. +""" +from ossie_thoughtspot.expressions.reverse import ( + REVERSE, + ReverseConstruct, + ReverseDisposition, + custom_extensions_fragment, + portable_dialect_entry, + stash_runtime_parameter, + thoughtspot_dialect_entry, + translate_thoughtspot, +) +from ossie_thoughtspot.issues import IssueLog, Severity + +OBJ = "metrics.revenue" + + +def _log() -> IssueLog: + return IssueLog() + + +# --------------------------------------------------------------------------- +# Conditional aggregates and arithmetic helpers (28 names) — all COMPOSE. +# --------------------------------------------------------------------------- + +def test_sum_if_composes(): + log = _log() + assert translate_thoughtspot("sum_if", ["cond", "x"], log, object_ref=OBJ) == ( + "SUM(CASE WHEN cond THEN x END)" + ) + assert log.issues == [] + + +def test_count_if_composes(): + log = _log() + assert translate_thoughtspot("count_if", ["cond", "x"], log, object_ref=OBJ) == ( + "COUNT(CASE WHEN cond THEN x END)" + ) + assert log.issues == [] + + +def test_unique_count_if_composes(): + log = _log() + assert translate_thoughtspot("unique_count_if", ["cond", "x"], log, object_ref=OBJ) == ( + "COUNT(DISTINCT CASE WHEN cond THEN x END)" + ) + + +def test_average_min_max_stddev_variance_if_compose(): + log = _log() + cases = { + "average_if": "AVG(CASE WHEN cond THEN x END)", + "min_if": "MIN(CASE WHEN cond THEN x END)", + "max_if": "MAX(CASE WHEN cond THEN x END)", + "stddev_if": "STDDEV(CASE WHEN cond THEN x END)", + "variance_if": "VARIANCE(CASE WHEN cond THEN x END)", + } + for name, expected in cases.items(): + assert translate_thoughtspot(name, ["cond", "x"], log, object_ref=OBJ) == expected + assert log.issues == [] + + +def test_unique_count_with_a_space_composes(): + # "A space, not an underscore" — the forward mapping document's own words + # (Aggregate functions, COUNT(DISTINCT expr)). + log = _log() + assert translate_thoughtspot("unique count", ["x"], log, object_ref=OBJ) == "COUNT(DISTINCT x)" + + +def test_safe_divide_composes(): + log = _log() + assert translate_thoughtspot("safe_divide", ["a", "b"], log, object_ref=OBJ) == ( + "COALESCE(a / NULLIF(b, 0), 0)" + ) + assert log.issues == [] + + +def test_pow_log2_strlen_strpos_substr_left_right_compose(): + log = _log() + assert translate_thoughtspot("pow", ["base", "exp"], log, object_ref=OBJ) == "POWER(base, exp)" + assert translate_thoughtspot("log2", ["x"], log, object_ref=OBJ) == "LOG(2, x)" + assert translate_thoughtspot("strlen", ["s"], log, object_ref=OBJ) == "LENGTH(s)" + # ThoughtSpot's strpos(s, sub) reverses to Ossie POSITION(sub IN s). + assert translate_thoughtspot("strpos", ["s", "sub"], log, object_ref=OBJ) == "POSITION(sub IN s)" + # substr's 0-based start needs the +1 going this way (mirror of the forward -1). + assert translate_thoughtspot("substr", ["s", "0", "3"], log, object_ref=OBJ) == ( + "SUBSTRING(s, 0 + 1, 3)" + ) + assert translate_thoughtspot("left", ["s", "3"], log, object_ref=OBJ) == "LEFT(s, 3)" + assert translate_thoughtspot("right", ["s", "3"], log, object_ref=OBJ) == "RIGHT(s, 3)" + assert log.issues == [] + + +def test_trig_functions_reverse_the_degree_radian_conversion(): + log = _log() + assert translate_thoughtspot("sin", ["x"], log, object_ref=OBJ) == "SIN(RADIANS(x))" + assert translate_thoughtspot("cos", ["x"], log, object_ref=OBJ) == "COS(RADIANS(x))" + assert translate_thoughtspot("tan", ["x"], log, object_ref=OBJ) == "TAN(RADIANS(x))" + assert translate_thoughtspot("asin", ["x"], log, object_ref=OBJ) == "DEGREES(ASIN(x))" + assert translate_thoughtspot("acos", ["x"], log, object_ref=OBJ) == "DEGREES(ACOS(x))" + assert translate_thoughtspot("atan", ["x"], log, object_ref=OBJ) == "DEGREES(ATAN(x))" + assert log.issues == [] + + +def test_to_integer_to_double_to_string_compose_losslessly(): + log = _log() + assert translate_thoughtspot("to_integer", ["x"], log, object_ref=OBJ) == "CAST(x AS INTEGER)" + assert translate_thoughtspot("to_double", ["x"], log, object_ref=OBJ) == "CAST(x AS DOUBLE)" + assert translate_thoughtspot("to_string", ["x"], log, object_ref=OBJ) == "CAST(x AS VARCHAR)" + assert log.issues == [] + + +def test_to_date_composes_with_an_info_issue_for_the_untranslated_format(): + log = _log() + result = translate_thoughtspot("to_date", ["s", "'yyyy-MM-dd'"], log, object_ref=OBJ) + assert result == "TO_DATE(s, 'yyyy-MM-dd')" + assert len(log.issues) == 1 + assert log.issues[0].severity is Severity.INFO + assert "to_date" in log.issues[0].message + assert log.issues[0].object_ref == OBJ + + +def test_if_composes_as_case_when(): + log = _log() + assert translate_thoughtspot("if", ["c", "a", "b"], log, object_ref=OBJ) == ( + "CASE WHEN c THEN a ELSE b END" + ) + assert log.issues == [] + + +# --------------------------------------------------------------------------- +# Window, LOD and semi-additive functions +# --------------------------------------------------------------------------- + +def test_rank_composes_global_order_only(): + log = _log() + assert translate_thoughtspot("rank", ["SUM(m)", "'desc'"], log, object_ref=OBJ) == ( + "RANK() OVER (ORDER BY SUM(m) DESC)" + ) + assert translate_thoughtspot("rank", ["SUM(m)", "'asc'"], log, object_ref=OBJ) == ( + "RANK() OVER (ORDER BY SUM(m) ASC)" + ) + assert log.issues == [] + + +def test_rank_percentile_composes_with_scale_and_inversion_reversed(): + log = _log() + assert translate_thoughtspot("rank_percentile", ["SUM(m)", "'asc'"], log, object_ref=OBJ) == ( + "(1.0 - PERCENT_RANK() OVER (ORDER BY SUM(m) ASC)) * 100" + ) + + +def test_moving_sum_composes_the_frame_and_loses_the_partition(): + log = _log() + result = translate_thoughtspot("moving_sum", ["m", "2", "0", "ord"], log, object_ref=OBJ) + assert result == "SUM(m) OVER (ORDER BY ord ROWS BETWEEN 2 PRECEDING AND CURRENT ROW)" + assert len(log.issues) == 1 + issue = log.issues[0] + assert issue.severity is Severity.WARNING + assert "moving_sum" in issue.message + assert "partition" in issue.message.lower() + assert issue.object_ref == OBJ + + +def test_moving_average_max_min_compose_with_sign_conventions(): + log = _log() + # (m, 1, -1, ord): 1 PRECEDING .. 1 FOLLOWING (negative end flips to FOLLOWING). + assert translate_thoughtspot("moving_average", ["m", "1", "-1", "ord"], log, object_ref=OBJ) == ( + "AVG(m) OVER (ORDER BY ord ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)" + ) + assert translate_thoughtspot("moving_max", ["m", "-1", "1", "ord"], log, object_ref=OBJ) == ( + "MAX(m) OVER (ORDER BY ord ROWS BETWEEN 1 FOLLOWING AND 1 PRECEDING)" + ) + assert translate_thoughtspot("moving_min", ["m", "0", "0", "ord"], log, object_ref=OBJ) == ( + "MIN(m) OVER (ORDER BY ord ROWS BETWEEN CURRENT ROW AND CURRENT ROW)" + ) + + +def test_moving_sum_accepts_multiple_order_columns(): + log = _log() + result = translate_thoughtspot("moving_sum", ["m", "1", "-1", "ord1", "ord2"], log, object_ref=OBJ) + assert result == "SUM(m) OVER (ORDER BY ord1, ord2 ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)" + + +def test_cumulative_sum_composes_the_running_total_and_loses_the_partition(): + log = _log() + result = translate_thoughtspot("cumulative_sum", ["m", "ord"], log, object_ref=OBJ) + assert result == ( + "SUM(m) OVER (ORDER BY ord ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)" + ) + assert len(log.issues) == 1 + assert log.issues[0].severity is Severity.WARNING + + +def test_cumulative_average_max_min_compose(): + log = _log() + assert translate_thoughtspot("cumulative_average", ["m", "ord"], log, object_ref=OBJ) == ( + "AVG(m) OVER (ORDER BY ord ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)" + ) + assert translate_thoughtspot("cumulative_max", ["m", "ord"], log, object_ref=OBJ) == ( + "MAX(m) OVER (ORDER BY ord ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)" + ) + assert translate_thoughtspot("cumulative_min", ["m", "ord"], log, object_ref=OBJ) == ( + "MIN(m) OVER (ORDER BY ord ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)" + ) + + +def test_cumulative_sum_accepts_multiple_order_columns(): + log = _log() + result = translate_thoughtspot("cumulative_sum", ["m", "ord1", "ord2"], log, object_ref=OBJ) + assert result == ( + "SUM(m) OVER (ORDER BY ord1, ord2 ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)" + ) + + +def test_group_aggregate_with_query_groups_alone_becomes_a_plain_aggregate(): + log = _log() + result = translate_thoughtspot( + "group_aggregate", ["SUM(m)", "query_groups ( )", "query_filters ( )"], log, object_ref=OBJ + ) + assert result == "SUM(m)" + assert log.issues == [] + + +def test_group_aggregate_with_a_fixed_list_becomes_partition_by(): + log = _log() + result = translate_thoughtspot( + "group_aggregate", ["SUM(m)", "{ a , b }", "query_filters ( )"], log, object_ref=OBJ + ) + assert result == "SUM(m) OVER (PARTITION BY a , b)" + assert log.issues == [] + + +def test_group_aggregate_with_an_empty_list_becomes_over_with_no_partition(): + log = _log() + result = translate_thoughtspot( + "group_aggregate", ["SUM(m)", "{ }", "query_filters ( )"], log, object_ref=OBJ + ) + assert result == "SUM(m) OVER ()" + + +def test_group_aggregate_with_a_dynamic_partition_stashes(): + log = _log() + result = translate_thoughtspot( + "group_aggregate", + ["SUM(m)", "query_groups ( ) - { a }", "query_filters ( )"], + log, + object_ref=OBJ, + ) + assert result is None + assert len(log.issues) == 1 + issue = log.issues[0] + assert issue.severity is Severity.ERROR + assert "group_aggregate" in issue.message + assert issue.object_ref == OBJ + + log2 = _log() + result2 = translate_thoughtspot( + "group_aggregate", + ["SUM(m)", "query_groups ( ) + { a }", "query_filters ( )"], + log2, + object_ref=OBJ, + ) + assert result2 is None + assert len(log2.issues) == 1 + + +def test_group_aggregate_with_a_non_default_filter_stashes(): + log = _log() + result = translate_thoughtspot( + "group_aggregate", ["SUM(m)", "{ a }", "{ c = 'v' }"], log, object_ref=OBJ + ) + assert result is None + assert len(log.issues) == 1 + assert log.issues[0].severity is Severity.ERROR + assert "filter" in log.issues[0].message.lower() + + +def test_group_sum_shorthand_composes_like_group_aggregate(): + log = _log() + result = translate_thoughtspot("group_sum", ["m", "{ a }", "query_filters ( )"], log, object_ref=OBJ) + assert result == "SUM(m) OVER (PARTITION BY a)" + assert log.issues == [] + + +def test_group_count_stddev_variance_shorthands_compose(): + log = _log() + assert translate_thoughtspot( + "group_count", ["m", "{ a }", "query_filters ( )"], log, object_ref=OBJ + ) == "COUNT(m) OVER (PARTITION BY a)" + assert translate_thoughtspot( + "group_stddev", ["m", "{ a }", "query_filters ( )"], log, object_ref=OBJ + ) == "STDDEV(m) OVER (PARTITION BY a)" + assert translate_thoughtspot( + "group_variance", ["m", "{ a }", "query_filters ( )"], log, object_ref=OBJ + ) == "VARIANCE(m) OVER (PARTITION BY a)" + + +def test_group_sum_shorthand_also_stashes_a_dynamic_partition(): + log = _log() + result = translate_thoughtspot( + "group_sum", ["m", "query_groups ( ) - { a }", "query_filters ( )"], log, object_ref=OBJ + ) + assert result is None + assert len(log.issues) == 1 + assert log.issues[0].severity is Severity.ERROR + + +def test_last_value_and_kin_always_stash_the_semi_additivity_loss(): + for name in ("last_value", "first_value", "last_value_in_period", "first_value_in_period"): + log = _log() + result = translate_thoughtspot( + name, ["SUM(m)", "query_groups ( )", "{ date }"], log, object_ref=OBJ + ) + assert result is None, name + assert len(log.issues) == 1, name + issue = log.issues[0] + assert issue.severity is Severity.ERROR + assert name in issue.message + assert issue.object_ref == OBJ + + +def test_sql_op_family_is_registered_with_dialect_disposition(): + names = [ + "sql_string_op", "sql_int_op", "sql_double_op", "sql_bool_op", + "sql_date_op", "sql_date_time_op", "sql_string_aggregate_op", + "sql_int_aggregate_op", "sql_number_aggregate_op", "sql_date_time_aggregate_op", + ] + for name in names: + assert name in REVERSE, name + assert REVERSE[name].disposition is ReverseDisposition.DIALECT, name + + +def test_sql_op_with_a_known_connection_dialect_composes_the_raw_body(): + log = _log() + result = translate_thoughtspot( + "sql_string_op", ["LOWER({0})", "s"], log, object_ref=OBJ, connection_dialect="SNOWFLAKE" + ) + assert result == "LOWER(s)" + assert len(log.issues) == 1 + assert log.issues[0].severity is Severity.WARNING + assert "SNOWFLAKE" in log.issues[0].message + + +def test_sql_op_with_an_unknown_connection_dialect_stashes(): + log = _log() + result = translate_thoughtspot("sql_string_op", ["LOWER({0})", "s"], log, object_ref=OBJ) + assert result is None + assert len(log.issues) == 1 + assert log.issues[0].severity is Severity.ERROR + + +# --------------------------------------------------------------------------- +# Runtime, display and calendar concepts +# --------------------------------------------------------------------------- + +def test_runtime_identity_functions_stash(): + for name in ("ts_username", "ts_groups", "ts_groups_int", "ts_org", "ts_email_domain"): + log = _log() + result = translate_thoughtspot(name, [], log, object_ref=OBJ) + assert result is None, name + assert len(log.issues) == 1, name + assert log.issues[0].severity is Severity.ERROR + assert name in log.issues[0].message + + +def test_ts_var_stashes(): + log = _log() + result = translate_thoughtspot("ts_var", ["'my_var'"], log, object_ref=OBJ) + assert result is None + assert len(log.issues) == 1 + assert log.issues[0].severity is Severity.ERROR + + +def test_stash_runtime_parameter_is_reached_outside_the_main_dispatcher(): + # A bracketed parameter name is syntactically identical to a column reference, so + # this is not auto-detected by translate_thoughtspot — the caller (which has model + # metadata) must call this directly. See reverse.py's module docstring. + log = _log() + result = stash_runtime_parameter("[Discount Threshold]", log, object_ref=OBJ) + assert result is None + assert len(log.issues) == 1 + assert log.issues[0].severity is Severity.ERROR + assert "[Discount Threshold]" in log.issues[0].message + + +def test_concat_with_hyperlink_markup_stashes(): + log = _log() + result = translate_thoughtspot( + "concat", ['"{caption}"', '"text"', '"{/caption}"', "url"], log, object_ref=OBJ + ) + assert result is None + assert len(log.issues) == 1 + assert log.issues[0].severity is Severity.ERROR + assert "concat" in log.issues[0].message + + +def test_concat_without_markup_is_not_this_modules_concern(): + # Plain concat has a spec counterpart (CONCAT) and is already in the forward + # CATALOG — this module returns None (nothing to do) and raises no issue. + log = _log() + result = translate_thoughtspot("concat", ["a", "b"], log, object_ref=OBJ) + assert result is None + assert log.issues == [] + + +def test_fiscal_calendar_variants_stash_regardless_of_the_underlying_function(): + for name, args in ( + ("year", ["d", "'fiscal'"]), + ("quarter_number", ["d", "'fiscal'"]), + ("diff_months", ["e", "s", "'fiscal'"]), + ): + log = _log() + result = translate_thoughtspot(name, args, log, object_ref=OBJ) + assert result is None, name + assert len(log.issues) == 1, name + assert log.issues[0].severity is Severity.ERROR + assert name in log.issues[0].message + + +def test_plain_date_functions_without_a_fiscal_argument_are_not_this_modules_concern(): + # year(d) alone has a spec counterpart (YEAR) via the forward catalog; this module + # only owns the *fiscal*-argument variant. + log = _log() + result = translate_thoughtspot("year", ["d"], log, object_ref=OBJ) + assert result is None + assert log.issues == [] + + +def test_name_returning_date_functions_compose_with_a_locale_issue(): + for name, expected in ( + ("month", "TO_CHAR(d, 'MONTH')"), + ("year_name", "TO_CHAR(d, 'YYYY')"), + ("day_of_week", "TO_CHAR(d, 'DAY')"), + ): + log = _log() + result = translate_thoughtspot(name, ["d"], log, object_ref=OBJ) + assert result == expected, name + assert len(log.issues) == 1, name + assert log.issues[0].severity is Severity.WARNING + + +def test_month_number_of_quarter_composes(): + log = _log() + assert translate_thoughtspot("month_number_of_quarter", ["d"], log, object_ref=OBJ) == ( + "MOD(MONTH(d) - 1, 3) + 1" + ) + assert log.issues == [] + + +def test_day_number_of_quarter_composes(): + log = _log() + assert translate_thoughtspot("day_number_of_quarter", ["d"], log, object_ref=OBJ) == ( + "DATEDIFF(day, DATE_TRUNC('quarter', d), d) + 1" + ) + assert log.issues == [] + + +def test_week_number_of_month_and_quarter_compose_with_a_week_start_issue(): + log = _log() + assert translate_thoughtspot("week_number_of_month", ["d"], log, object_ref=OBJ) == ( + "DATEDIFF(week, DATE_TRUNC('month', d), d) + 1" + ) + assert len(log.issues) == 1 + assert log.issues[0].severity is Severity.WARNING + + log2 = _log() + assert translate_thoughtspot("week_number_of_quarter", ["d"], log2, object_ref=OBJ) == ( + "DATEDIFF(week, DATE_TRUNC('quarter', d), d) + 1" + ) + assert len(log2.issues) == 1 + + +def test_is_weekend_composes_with_a_dayofweek_base_issue(): + log = _log() + result = translate_thoughtspot("is_weekend", ["d"], log, object_ref=OBJ) + assert result == "DATE_PART('dayofweek', d) IN (6, 7)" + assert len(log.issues) == 1 + assert log.issues[0].severity is Severity.WARNING + assert "dayofweek" in log.issues[0].message.lower() or "DAYOFWEEK" in log.issues[0].message + + +def test_start_of_hour_min_date_time_compose_losslessly(): + log = _log() + assert translate_thoughtspot("start_of_hour", ["d"], log, object_ref=OBJ) == "DATE_TRUNC('hour', d)" + assert translate_thoughtspot("start_of_min", ["d"], log, object_ref=OBJ) == "DATE_TRUNC('minute', d)" + assert translate_thoughtspot("date", ["d"], log, object_ref=OBJ) == "DATE_TRUNC('day', d)" + assert translate_thoughtspot("time", ["d"], log, object_ref=OBJ) == "CAST(d AS TIME)" + assert log.issues == [] + + +def test_greatest_and_least_compose_n_ary(): + log = _log() + assert translate_thoughtspot("greatest", ["x", "y"], log, object_ref=OBJ) == "GREATEST(x, y)" + assert translate_thoughtspot("least", ["x", "y", "z"], log, object_ref=OBJ) == "LEAST(x, y, z)" + assert log.issues == [] + + +# --------------------------------------------------------------------------- +# Names with no reverse-inventory entry at all — the "not this module's job" contract. +# --------------------------------------------------------------------------- + +def test_unrecognised_name_returns_none_with_no_issue(): + log = _log() + assert translate_thoughtspot("some_future_function", ["x"], log, object_ref=OBJ) is None + assert log.issues == [] + + +# --------------------------------------------------------------------------- +# Dialect-entry and stash-payload helpers. +# --------------------------------------------------------------------------- + +def test_thoughtspot_dialect_entry_reconstructs_the_verbatim_call(): + entry = thoughtspot_dialect_entry("ts_username", []) + assert entry == {"dialect": "THOUGHTSPOT", "expression": "ts_username ( )"} + + entry2 = thoughtspot_dialect_entry("moving_sum", ["m", "2", "0", "ord"]) + assert entry2 == {"dialect": "THOUGHTSPOT", "expression": "moving_sum ( m , 2 , 0 , ord )"} + + +def test_portable_dialect_entry_pairs_with_ansi_sql(): + entry = portable_dialect_entry("SUM(m) OVER (ORDER BY ord ROWS BETWEEN 2 PRECEDING AND CURRENT ROW)") + assert entry == { + "dialect": "ANSI_SQL", + "expression": "SUM(m) OVER (ORDER BY ord ROWS BETWEEN 2 PRECEDING AND CURRENT ROW)", + } + + +def test_custom_extensions_fragment_is_keyed_by_the_ossie_column_name(): + # Not by VENDOR_KEY: write_stash(obj, payload) treats payload as the *contents* of + # the object's THOUGHTSPOT entry, so a fragment keyed by VENDOR_KEY would nest the + # vendor key inside its own entry instead of producing a valid write_stash payload. + fragment = custom_extensions_fragment("revenue_per_user", "ts_username", []) + assert fragment == {"revenue_per_user": {"reverse_thoughtspot_call": "ts_username ( )"}} + + +def test_custom_extensions_fragment_merges_losslessly_across_columns(): + # The whole point of keying by column: {**f1, **f2} must keep both columns' calls, + # not collapse to whichever fragment merged last (the VENDOR_KEY-keyed bug this + # replaces would silently drop the first column here). + f1 = custom_extensions_fragment("revenue_per_user", "ts_username", []) + f2 = custom_extensions_fragment("org_label", "ts_org", []) + merged = {**f1, **f2} + assert merged == { + "revenue_per_user": {"reverse_thoughtspot_call": "ts_username ( )"}, + "org_label": {"reverse_thoughtspot_call": "ts_org ( )"}, + } + + # And the merged fragment is a valid write_stash payload: write_stash treats its + # `payload` argument as the entry's own contents, so merged must round-trip through + # it without collapsing either column. + from ossie_thoughtspot.stash import read_stash, write_stash + + obj = write_stash({"name": "revenue_model"}, merged) + assert read_stash(obj) == {**merged, "_v": 1} + + +# --------------------------------------------------------------------------- +# Inventory shape. +# --------------------------------------------------------------------------- + +def test_every_reverse_construct_is_traceable_to_the_mapping_document(): + # Every entry must declare a disposition and, for COMPOSE/PARTIAL rows without a + # dispatch_fn, a template or compose_fn — ReverseConstruct.__post_init__ enforces the + # combination; this test just confirms every registered row survived construction + # (a failure here means the module itself failed to import). + assert len(REVERSE) > 0 + for name, construct in REVERSE.items(): + assert isinstance(construct, ReverseConstruct) + assert construct.thoughtspot_name == name + + +def test_reverse_inventory_census(): + # Pins the count so a silent addition/removal is visible in review, the same + # discipline the forward CATALOG's 146-row census test applies. + assert len(REVERSE) == 79 diff --git a/converters/thoughtspot/tests/expressions/test_types.py b/converters/thoughtspot/tests/expressions/test_types.py new file mode 100644 index 00000000..6ee8c274 --- /dev/null +++ b/converters/thoughtspot/tests/expressions/test_types.py @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Construct's __post_init__ validation invariants. + +A DIRECT or PASSTHROUGH row with no template would pass every other check and +the coverage gate, and only surface when something later tries to emit it - +so this is enforced at construction time instead. +""" +import pytest + +from ossie_thoughtspot.expressions import Classification, Construct, Variant + + +def test_direct_construct_requires_a_template(): + with pytest.raises(ValueError, match="SUM\\(expr\\).*direct.*template"): + Construct(spec_name="SUM(expr)", classification=Classification.DIRECT, template=None) + + +def test_direct_construct_rejects_an_empty_string_template(): + with pytest.raises(ValueError, match="template"): + Construct(spec_name="SUM(expr)", classification=Classification.DIRECT, template="") + + +def test_passthrough_construct_requires_a_template(): + with pytest.raises(ValueError, match="template"): + Construct( + spec_name="STDDEV_POP(expr)", + classification=Classification.PASSTHROUGH, + variant=Variant.NUMBER_AGGREGATE, + template=None, + ) + + +def test_unmappable_construct_needs_no_template(): + # Should not raise: UNMAPPABLE is the one classification allowed no template. + Construct(spec_name="EXISTS_IN()", classification=Classification.UNMAPPABLE) + + +def test_direct_construct_with_a_template_is_valid(): + # Should not raise. + Construct(spec_name="ABS(x)", classification=Classification.DIRECT, template="abs ( {0} )") + + +def test_passthrough_construct_with_a_template_and_variant_is_valid(): + # Should not raise. The template holds only the bare inner SQL body — + # emit_passthrough builds the variant(...) wrapper itself (see + # test_passthrough_construct_rejects_a_template_that_wraps_itself below). + Construct( + spec_name="STDDEV_POP(expr)", + classification=Classification.PASSTHROUGH, + template="STDDEV_POP({0})", + variant=Variant.NUMBER_AGGREGATE, + ) + + +def test_passthrough_construct_rejects_a_template_that_wraps_itself(): + # An early catalog draft made exactly this mistake: it stored a + # passthrough template as the FULL wrapped form (copied verbatim from the + # mapping document's ThoughtSpot-column cell) instead of the bare inner + # call. emit_passthrough builds the `variant ( "..." , args )` wrapper + # itself, so a template that already contains it double-wraps at emission + # time — a bug invisible from a static read of the catalog file. Pin the + # regression so a future catalog family can't reintroduce it. + with pytest.raises(ValueError, match="STDDEV_POP.*double-wrap"): + Construct( + spec_name="STDDEV_POP(expr)", + classification=Classification.PASSTHROUGH, + template='sql_number_aggregate_op ( "STDDEV_POP({0})" , {0} )', + variant=Variant.NUMBER_AGGREGATE, + ) diff --git a/converters/thoughtspot/tests/fixtures/minimal/customers.table.tml b/converters/thoughtspot/tests/fixtures/minimal/customers.table.tml new file mode 100644 index 00000000..76df3fdd --- /dev/null +++ b/converters/thoughtspot/tests/fixtures/minimal/customers.table.tml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +table: + name: customers + db: MINIMAL + schema: PUBLIC + db_table: CUSTOMERS + connection: + name: Minimal Connection + columns: + - name: customer_id + db_column_name: customer_id + db_column_properties: + data_type: INT64 + - name: customer_name + db_column_name: customer_name + db_column_properties: + data_type: VARCHAR diff --git a/converters/thoughtspot/tests/fixtures/minimal/expected.ossie.yaml b/converters/thoughtspot/tests/fixtures/minimal/expected.ossie.yaml new file mode 100644 index 00000000..033cfab1 --- /dev/null +++ b/converters/thoughtspot/tests/fixtures/minimal/expected.ossie.yaml @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated by running tml_to_ossie.convert() over the TML fixtures in this +# directory, then reviewed by hand against the construct mapping this +# converter implements. See test_fixtures.py. +version: 0.2.0.dev0 +semantic_model: +- name: minimal_orders_model + datasets: + - name: orders + source: MINIMAL.PUBLIC.ORDERS + fields: + - name: order_id + label: order_id + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[orders::order_id]' + - dialect: ANSI_SQL + expression: orders.order_id + datatype: Integer + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "connection_name": "Minimal Connection", "tml_object": "table", + "tml_object_source_witness": "MINIMAL.PUBLIC.ORDERS", "unsurfaced_columns": + [{"db_column_name": "customer_id", "db_column_properties": {"data_type": "INT64"}, + "name": "customer_id"}, {"db_column_name": "order_total", "db_column_properties": + {"data_type": "DOUBLE"}, "name": "order_total"}]}' + - name: customers + source: MINIMAL.PUBLIC.CUSTOMERS + primary_key: &id001 + - customer_id + unique_keys: + - *id001 + fields: + - name: customer_name + label: customer_name + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[customers::customer_name]' + - dialect: ANSI_SQL + expression: customers.customer_name + datatype: String + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "connection_name": "Minimal Connection", "tml_object": "table", + "tml_object_source_witness": "MINIMAL.PUBLIC.CUSTOMERS", "unsurfaced_columns": + [{"db_column_name": "customer_id", "db_column_properties": {"data_type": "INT64"}, + "name": "customer_id"}]}' + description: Smallest fixture exercising the one-model-plus-N-tables split. + relationships: + - name: orders_to_customers + from: orders + to: customers + from_columns: + - customer_id + to_columns: + - customer_id + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "cardinality": "MANY_TO_ONE", "join_shape": "referencing", + "referencing_join": "orders_to_customers", "type": "INNER"}' + metrics: + - name: total_order_amount + expression: + dialects: + - dialect: THOUGHTSPOT + expression: sum ( [orders::order_total] ) diff --git a/converters/thoughtspot/tests/fixtures/minimal/minimal_orders_model.model.tml b/converters/thoughtspot/tests/fixtures/minimal/minimal_orders_model.model.tml new file mode 100644 index 00000000..c1cb70f8 --- /dev/null +++ b/converters/thoughtspot/tests/fixtures/minimal/minimal_orders_model.model.tml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The smallest document set that exercises the one-model-plus-N-tables +# split: one model, two tables, one relationship, two fields and one +# metric. +model: + name: minimal_orders_model + description: "Smallest fixture exercising the one-model-plus-N-tables split." + model_tables: + - name: orders + joins: + - referencing_join: orders_to_customers + - name: customers + columns: + - name: order_id + column_id: orders::order_id + properties: + column_type: ATTRIBUTE + - name: customer_name + column_id: customers::customer_name + properties: + column_type: ATTRIBUTE + - name: total_order_amount + formula_id: formula_total_order_amount + properties: + column_type: MEASURE + formulas: + - id: formula_total_order_amount + name: total_order_amount + expr: "sum ( [orders::order_total] )" diff --git a/converters/thoughtspot/tests/fixtures/minimal/orders.table.tml b/converters/thoughtspot/tests/fixtures/minimal/orders.table.tml new file mode 100644 index 00000000..768eba06 --- /dev/null +++ b/converters/thoughtspot/tests/fixtures/minimal/orders.table.tml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +table: + name: orders + db: MINIMAL + schema: PUBLIC + db_table: ORDERS + connection: + name: Minimal Connection + columns: + - name: order_id + db_column_name: order_id + db_column_properties: + data_type: INT64 + - name: customer_id + db_column_name: customer_id + db_column_properties: + data_type: INT64 + - name: order_total + db_column_name: order_total + db_column_properties: + data_type: DOUBLE + joins_with: + - name: orders_to_customers + destination: + name: customers + 'on': "[orders::customer_id] = [customers::customer_id]" + type: INNER + cardinality: MANY_TO_ONE diff --git a/converters/thoughtspot/tests/fixtures/tpcds/customer.table.tml b/converters/thoughtspot/tests/fixtures/tpcds/customer.table.tml new file mode 100644 index 00000000..8aa4310c --- /dev/null +++ b/converters/thoughtspot/tests/fixtures/tpcds/customer.table.tml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +table: + name: customer + db: TPCDS + schema: PUBLIC + db_table: CUSTOMER + connection: + name: TPC-DS Snowflake + columns: + - name: c_customer_sk + db_column_name: c_customer_sk + db_column_properties: + data_type: INT64 + - name: c_customer_id + db_column_name: c_customer_id + db_column_properties: + data_type: VARCHAR + - name: c_first_name + db_column_name: c_first_name + db_column_properties: + data_type: VARCHAR + - name: c_last_name + db_column_name: c_last_name + db_column_properties: + data_type: VARCHAR + - name: c_email_address + db_column_name: c_email_address + db_column_properties: + data_type: VARCHAR + # A non-Latin display name (Japanese kana-spelled name) over an ASCII + # warehouse column name -- exercises the fallback identifier + # `identifiers.normalise` cannot derive one from the display name alone. + - name: カナ名 + db_column_name: c_kana_name + db_column_properties: + data_type: VARCHAR diff --git a/converters/thoughtspot/tests/fixtures/tpcds/date_dim.table.tml b/converters/thoughtspot/tests/fixtures/tpcds/date_dim.table.tml new file mode 100644 index 00000000..267dcdcb --- /dev/null +++ b/converters/thoughtspot/tests/fixtures/tpcds/date_dim.table.tml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +table: + name: date_dim + db: TPCDS + schema: PUBLIC + db_table: DATE_DIM + connection: + name: TPC-DS Snowflake + columns: + - name: d_date_sk + db_column_name: d_date_sk + db_column_properties: + data_type: INT64 + - name: d_date + db_column_name: d_date + db_column_properties: + data_type: DATE + - name: d_year + db_column_name: d_year + db_column_properties: + data_type: INT64 + - name: d_quarter_name + db_column_name: d_quarter_name + db_column_properties: + data_type: VARCHAR + - name: d_moy + db_column_name: d_moy + db_column_properties: + data_type: INT64 diff --git a/converters/thoughtspot/tests/fixtures/tpcds/expected.ossie.yaml b/converters/thoughtspot/tests/fixtures/tpcds/expected.ossie.yaml new file mode 100644 index 00000000..3005aaa3 --- /dev/null +++ b/converters/thoughtspot/tests/fixtures/tpcds/expected.ossie.yaml @@ -0,0 +1,581 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated by running tml_to_ossie.convert() over the TML fixtures in this +# directory, then reviewed by hand against the construct mapping this +# converter implements. See test_fixtures.py. +# +# store's unique_keys derives to [s_store_sk], not [s_store_id]: TML has no +# native key-declaration syntax, so every key here comes from the join +# graph, and every relationship that targets store (store_sales_to_store, +# and store_returns_sv_to_store -- a ONE_TO_MANY join declared from store's +# own side, whose emitted endpoints are swapped so store lands on `to`) +# joins on s_store_sk. That is a correct, expected divergence from a source +# format (such as one with its own native key declarations) that could +# declare s_store_id as a key independently of any join. +version: 0.2.0.dev0 +semantic_model: +- name: tpcds_retail_model + datasets: + - name: store_sales + source: TPCDS.PUBLIC.STORE_SALES + primary_key: &id001 + - ss_item_sk + - ss_ticket_number + unique_keys: + - *id001 + fields: + - name: ss_sold_date_sk + label: ss_sold_date_sk + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store_sales::ss_sold_date_sk]' + - dialect: ANSI_SQL + expression: store_sales.ss_sold_date_sk + datatype: Integer + - name: ss_item_sk + label: ss_item_sk + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store_sales::ss_item_sk]' + - dialect: ANSI_SQL + expression: store_sales.ss_item_sk + datatype: Integer + - name: ss_customer_sk + label: ss_customer_sk + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store_sales::ss_customer_sk]' + - dialect: ANSI_SQL + expression: store_sales.ss_customer_sk + datatype: Integer + - name: ss_store_sk + label: ss_store_sk + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store_sales::ss_store_sk]' + - dialect: ANSI_SQL + expression: store_sales.ss_store_sk + datatype: Integer + - name: ss_quantity + label: ss_quantity + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store_sales::ss_quantity]' + - dialect: ANSI_SQL + expression: store_sales.ss_quantity + datatype: Integer + - name: ss_sales_price + label: ss_sales_price + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store_sales::ss_sales_price]' + - dialect: ANSI_SQL + expression: store_sales.ss_sales_price + datatype: Decimal + - name: ss_ext_sales_price + label: ss_ext_sales_price + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store_sales::ss_ext_sales_price]' + - dialect: ANSI_SQL + expression: store_sales.ss_ext_sales_price + datatype: Decimal + - name: ss_net_profit + label: ss_net_profit + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store_sales::ss_net_profit]' + - dialect: ANSI_SQL + expression: store_sales.ss_net_profit + datatype: Decimal + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "connection_name": "TPC-DS Snowflake", "tml_object": "table", + "tml_object_source_witness": "TPCDS.PUBLIC.STORE_SALES", "unsurfaced_columns": + [{"db_column_name": "ss_ticket_number", "db_column_properties": {"data_type": + "INT64"}, "name": "ss_ticket_number"}]}' + - name: date_dim + source: TPCDS.PUBLIC.DATE_DIM + primary_key: &id002 + - d_date_sk + unique_keys: + - *id002 + fields: + - name: d_date_sk + label: d_date_sk + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[date_dim::d_date_sk]' + - dialect: ANSI_SQL + expression: date_dim.d_date_sk + datatype: Integer + - name: d_date + label: d_date + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[date_dim::d_date]' + - dialect: ANSI_SQL + expression: date_dim.d_date + datatype: Date + - name: d_year + label: d_year + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[date_dim::d_year]' + - dialect: ANSI_SQL + expression: date_dim.d_year + datatype: Integer + - name: d_quarter_name + label: d_quarter_name + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[date_dim::d_quarter_name]' + - dialect: ANSI_SQL + expression: date_dim.d_quarter_name + datatype: String + - name: d_moy + label: d_moy + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[date_dim::d_moy]' + - dialect: ANSI_SQL + expression: date_dim.d_moy + datatype: Integer + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "connection_name": "TPC-DS Snowflake", "tml_object": "table", + "tml_object_source_witness": "TPCDS.PUBLIC.DATE_DIM"}' + - name: customer + source: TPCDS.PUBLIC.CUSTOMER + primary_key: &id003 + - c_customer_sk + unique_keys: + - *id003 + fields: + - name: c_customer_sk + label: c_customer_sk + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[customer::c_customer_sk]' + - dialect: ANSI_SQL + expression: customer.c_customer_sk + datatype: Integer + - name: c_customer_id + label: c_customer_id + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[customer::c_customer_id]' + - dialect: ANSI_SQL + expression: customer.c_customer_id + datatype: String + - name: c_first_name + label: c_first_name + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[customer::c_first_name]' + - dialect: ANSI_SQL + expression: customer.c_first_name + datatype: String + - name: c_last_name + label: c_last_name + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[customer::c_last_name]' + - dialect: ANSI_SQL + expression: customer.c_last_name + datatype: String + - name: customer_full_name + label: customer_full_name + expression: + dialects: + - dialect: THOUGHTSPOT + expression: concat ( concat ( [customer::c_first_name] , ' ' ) , [customer::c_last_name] + ) + - name: c_email_address + label: c_email_address + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[customer::c_email_address]' + - dialect: ANSI_SQL + expression: customer.c_email_address + datatype: String + - name: c_kana_name + label: カナ名 + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[customer::カナ名]' + - dialect: ANSI_SQL + expression: customer.c_kana_name + datatype: String + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "db_column_name": "c_kana_name", "db_column_name_display_name_witness": + "\u30ab\u30ca\u540d"}' + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "connection_name": "TPC-DS Snowflake", "tml_object": "table", + "tml_object_source_witness": "TPCDS.PUBLIC.CUSTOMER"}' + - name: item + source: TPCDS.PUBLIC.ITEM + primary_key: &id004 + - i_item_sk + unique_keys: + - *id004 + fields: + - name: i_item_sk + label: i_item_sk + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[item::i_item_sk]' + - dialect: ANSI_SQL + expression: item.i_item_sk + datatype: Integer + - name: i_item_id + label: i_item_id + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[item::i_item_id]' + - dialect: ANSI_SQL + expression: item.i_item_id + datatype: String + - name: i_item_desc + label: i_item_desc + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[item::i_item_desc]' + - dialect: ANSI_SQL + expression: item.i_item_desc + datatype: String + - name: i_brand + label: i_brand + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[item::i_brand]' + - dialect: ANSI_SQL + expression: item.i_brand + datatype: String + - name: i_category + label: i_category + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[item::i_category]' + - dialect: ANSI_SQL + expression: item.i_category + datatype: String + - name: i_current_price + label: i_current_price + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[item::i_current_price]' + - dialect: ANSI_SQL + expression: item.i_current_price + datatype: Decimal + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "connection_name": "TPC-DS Snowflake", "tml_object": "table", + "tml_object_source_witness": "TPCDS.PUBLIC.ITEM"}' + - name: store + source: TPCDS.PUBLIC.STORE + primary_key: &id005 + - s_store_sk + unique_keys: + - *id005 + fields: + - name: s_store_sk + label: s_store_sk + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store::s_store_sk]' + - dialect: ANSI_SQL + expression: store.s_store_sk + datatype: Integer + - name: s_store_id + label: s_store_id + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store::s_store_id]' + - dialect: ANSI_SQL + expression: store.s_store_id + datatype: String + - name: s_store_name + label: s_store_name + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store::s_store_name]' + - dialect: ANSI_SQL + expression: store.STORE_NM + datatype: String + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "db_column_name": "STORE_NM", "db_column_name_display_name_witness": + "s_store_name"}' + - name: s_city + label: s_city + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store::s_city]' + - dialect: ANSI_SQL + expression: store.s_city + datatype: String + - name: s_state + label: s_state + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store::s_state]' + - dialect: ANSI_SQL + expression: store.s_state + datatype: String + - name: s_number_employees + label: s_number_employees + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store::s_number_employees]' + - dialect: ANSI_SQL + expression: store.s_number_employees + datatype: Integer + - name: 'on' + label: 'on' + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store::on]' + - dialect: ANSI_SQL + expression: store.on + datatype: Boolean + description: Whether the store is currently active and open for business. + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "data_type": "BOOL", "data_type_ossie_datatype_witness": + "Boolean"}' + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "connection_name": "TPC-DS Snowflake", "tml_object": "table", + "tml_object_source_witness": "TPCDS.PUBLIC.STORE"}' + - name: store_returns_sv + source: SELECT sr_item_sk, sr_ticket_number, sr_return_amt AS RETURN_AMT, sr_return_quantity, + sr_store_sk FROM tpcds.public.store_returns + fields: + - name: sr_item_sk + label: sr_item_sk + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store_returns_sv::sr_item_sk]' + - dialect: ANSI_SQL + expression: store_returns_sv.sr_item_sk + datatype: Integer + - name: sr_ticket_number + label: sr_ticket_number + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store_returns_sv::sr_ticket_number]' + - dialect: ANSI_SQL + expression: store_returns_sv.sr_ticket_number + datatype: Integer + - name: sr_return_amt + label: sr_return_amt + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[store_returns_sv::sr_return_amt]' + - dialect: ANSI_SQL + expression: store_returns_sv.RETURN_AMT + datatype: Decimal + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "connection_name": "TPC-DS Snowflake", "sql_output_columns": + {"sr_item_sk": "sr_item_sk", "sr_return_amt": "RETURN_AMT", "sr_ticket_number": + "sr_ticket_number"}, "tml_object": "sql_view", "tml_object_source_witness": + "SELECT sr_item_sk, sr_ticket_number, sr_return_amt AS RETURN_AMT, sr_return_quantity, + sr_store_sk FROM tpcds.public.store_returns", "unsurfaced_columns": [{"db_column_properties": + {"data_type": "INT64"}, "name": "sr_return_quantity", "sql_output_column": + "sr_return_quantity"}, {"db_column_properties": {"data_type": "INT64"}, "name": + "sr_store_sk", "sql_output_column": "sr_store_sk"}]}' + description: TPC-DS retail semantic model used as a shared test fixture. + relationships: + - name: store_sales_to_date + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - d_date_sk + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "cardinality": "MANY_TO_ONE", "join_shape": "referencing", + "referencing_join": "store_sales_to_date", "type": "INNER"}' + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "cardinality": "MANY_TO_ONE", "join_shape": "referencing", + "referencing_join": "store_sales_to_customer", "type": "INNER"}' + - name: store_sales_to_item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "cardinality": "MANY_TO_ONE", "join_shape": "referencing", + "referencing_join": "store_sales_to_item", "type": "INNER"}' + - name: store_sales_to_store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "cardinality": "MANY_TO_ONE", "join_shape": "referencing", + "referencing_join": "store_sales_to_store", "type": "INNER"}' + - name: store_returns_sv_to_store + from: store_returns_sv + to: store + from_columns: + - sr_store_sk + to_columns: + - s_store_sk + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "cardinality": "ONE_TO_MANY", "endpoints_swapped": true, "endpoints_swapped_witness": + ["store_returns_sv", "store", ["sr_store_sk"], ["s_store_sk"]], "join_shape": + "inline", "type": "INNER"}' + - name: store_returns_sv_to_store_sales + from: store_returns_sv + to: store_sales + from_columns: + - sr_item_sk + - sr_ticket_number + to_columns: + - ss_item_sk + - ss_ticket_number + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "cardinality": "MANY_TO_ONE", "join_shape": "inline", "type": + "INNER"}' + - name: store_returns_sv_to_item + from: store_returns_sv + to: item + from_columns: + - sr_item_sk + to_columns: + - i_item_sk + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "cardinality": "MANY_TO_ONE", "join_shape": "inline", "on_expression": + "[store_returns_sv::sr_item_sk] = [item::i_item_sk] and [store_returns_sv::sr_return_amt] + <= [item::i_current_price]", "on_expression_equality_witness": [["sr_item_sk"], + ["i_item_sk"]], "type": "INNER"}' + metrics: + - name: total_sales + expression: + dialects: + - dialect: THOUGHTSPOT + expression: sum ( [store_sales::ss_ext_sales_price] ) + - name: total_profit + expression: + dialects: + - dialect: THOUGHTSPOT + expression: sum ( [store_sales::ss_net_profit] ) + - name: customer_lifetime_value + expression: + dialects: + - dialect: THOUGHTSPOT + expression: sum ( [store_sales::ss_ext_sales_price] ) / unique count ( [customer::c_customer_sk] + ) + - name: sales_by_brand + expression: + dialects: + - dialect: THOUGHTSPOT + expression: sum ( [store_sales::ss_ext_sales_price] ) + - name: store_productivity + expression: + dialects: + - dialect: THOUGHTSPOT + expression: sum ( [store_sales::ss_ext_sales_price] ) / nullif ( sum ( [store::s_number_employees] + ) , 0 ) + - name: total_return_quantity + expression: + dialects: + - dialect: THOUGHTSPOT + expression: sum ( [store_returns_sv::sr_return_quantity] ) + datatype: Integer + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "shape": "column_aggregation"}' + - name: avg_price_adjustment + expression: + dialects: + - dialect: THOUGHTSPOT + expression: average ( [store_sales::ss_ext_sales_price] - [store_sales::ss_sales_price] + ) + custom_extensions: + - vendor_name: THOUGHTSPOT + data: '{"_v": 1, "shape": "scalar_formula_plus_aggregation"}' + - name: profit_margin + expression: + dialects: + - dialect: THOUGHTSPOT + expression: '[formula_total_profit] / [formula_total_sales]' + - name: prior_period_profit + expression: + dialects: + - dialect: THOUGHTSPOT + expression: last_value ( sum ( [store_sales::ss_net_profit] ) , query_groups + ( ) , { [date_dim::d_date] } ) diff --git a/converters/thoughtspot/tests/fixtures/tpcds/item.table.tml b/converters/thoughtspot/tests/fixtures/tpcds/item.table.tml new file mode 100644 index 00000000..2a5bbd5c --- /dev/null +++ b/converters/thoughtspot/tests/fixtures/tpcds/item.table.tml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +table: + name: item + db: TPCDS + schema: PUBLIC + db_table: ITEM + connection: + name: TPC-DS Snowflake + columns: + - name: i_item_sk + db_column_name: i_item_sk + db_column_properties: + data_type: INT64 + - name: i_item_id + db_column_name: i_item_id + db_column_properties: + data_type: VARCHAR + - name: i_item_desc + db_column_name: i_item_desc + db_column_properties: + data_type: VARCHAR + - name: i_brand + db_column_name: i_brand + db_column_properties: + data_type: VARCHAR + - name: i_category + db_column_name: i_category + db_column_properties: + data_type: VARCHAR + - name: i_current_price + db_column_name: i_current_price + db_column_properties: + data_type: DOUBLE diff --git a/converters/thoughtspot/tests/fixtures/tpcds/store.table.tml b/converters/thoughtspot/tests/fixtures/tpcds/store.table.tml new file mode 100644 index 00000000..0d96e627 --- /dev/null +++ b/converters/thoughtspot/tests/fixtures/tpcds/store.table.tml @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# `s_store_name`'s display name deliberately differs from its warehouse +# column name, so the portable expression built for it has to use the +# warehouse name rather than the display name. `"on"` is both a YAML 1.1 +# boolean token spelled out as a column name, and a BOOL-typed column (the +# Snowflake-specific spelling), quoted throughout so no YAML 1.1 reader can +# coerce it to a boolean. +table: + name: store + db: TPCDS + schema: PUBLIC + db_table: STORE + connection: + name: TPC-DS Snowflake + columns: + - name: s_store_sk + db_column_name: s_store_sk + db_column_properties: + data_type: INT64 + - name: s_store_id + db_column_name: s_store_id + db_column_properties: + data_type: VARCHAR + - name: s_store_name + db_column_name: STORE_NM + db_column_properties: + data_type: VARCHAR + - name: s_city + db_column_name: s_city + db_column_properties: + data_type: VARCHAR + - name: s_state + db_column_name: s_state + db_column_properties: + data_type: VARCHAR + - name: s_number_employees + db_column_name: s_number_employees + db_column_properties: + data_type: INT64 + - name: "on" + db_column_name: "on" + db_column_properties: + data_type: BOOL diff --git a/converters/thoughtspot/tests/fixtures/tpcds/store_returns_sv.sql_view.tml b/converters/thoughtspot/tests/fixtures/tpcds/store_returns_sv.sql_view.tml new file mode 100644 index 00000000..0fa828da --- /dev/null +++ b/converters/thoughtspot/tests/fixtures/tpcds/store_returns_sv.sql_view.tml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# A SQL View alongside the Tables above. `sr_return_amt`'s query output +# alias (`RETURN_AMT`) deliberately differs from its display name, and +# `sr_return_quantity` and `sr_store_sk` are deliberately left unsurfaced by +# the Model -- `sr_return_quantity` is only ever referenced through a +# metric's own formula, and `sr_store_sk` only through the `store` +# model_tables entry's ONE_TO_MANY join condition below (mirroring +# store_sales's own `ss_ticket_number`, referenced only from a join +# condition and never surfaced as a field). +sql_view: + name: store_returns_sv + sql_query: >- + SELECT sr_item_sk, sr_ticket_number, sr_return_amt AS RETURN_AMT, + sr_return_quantity, sr_store_sk FROM tpcds.public.store_returns + connection: + name: TPC-DS Snowflake + sql_view_columns: + - name: sr_item_sk + sql_output_column: sr_item_sk + db_column_properties: + data_type: INT64 + - name: sr_ticket_number + sql_output_column: sr_ticket_number + db_column_properties: + data_type: INT64 + - name: sr_return_amt + sql_output_column: RETURN_AMT + db_column_properties: + data_type: DOUBLE + - name: sr_return_quantity + sql_output_column: sr_return_quantity + db_column_properties: + data_type: INT64 + - name: sr_store_sk + sql_output_column: sr_store_sk + db_column_properties: + data_type: INT64 diff --git a/converters/thoughtspot/tests/fixtures/tpcds/store_sales.table.tml b/converters/thoughtspot/tests/fixtures/tpcds/store_sales.table.tml new file mode 100644 index 00000000..9ceaa90d --- /dev/null +++ b/converters/thoughtspot/tests/fixtures/tpcds/store_sales.table.tml @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# TPC-DS fact table. `ss_ticket_number` is part of this table's natural +# composite key (together with `ss_item_sk`) but is deliberately not +# surfaced by the Model below, so it exercises a physical column the +# semantic model does not expose as a field. +table: + name: store_sales + db: TPCDS + schema: PUBLIC + db_table: STORE_SALES + connection: + name: TPC-DS Snowflake + columns: + - name: ss_sold_date_sk + db_column_name: ss_sold_date_sk + db_column_properties: + data_type: INT64 + - name: ss_item_sk + db_column_name: ss_item_sk + db_column_properties: + data_type: INT64 + - name: ss_customer_sk + db_column_name: ss_customer_sk + db_column_properties: + data_type: INT64 + - name: ss_store_sk + db_column_name: ss_store_sk + db_column_properties: + data_type: INT64 + - name: ss_quantity + db_column_name: ss_quantity + db_column_properties: + data_type: INT64 + - name: ss_sales_price + db_column_name: ss_sales_price + db_column_properties: + data_type: DOUBLE + - name: ss_ext_sales_price + db_column_name: ss_ext_sales_price + db_column_properties: + data_type: DOUBLE + - name: ss_net_profit + db_column_name: ss_net_profit + db_column_properties: + data_type: DOUBLE + - name: ss_ticket_number + db_column_name: ss_ticket_number + db_column_properties: + data_type: INT64 + joins_with: + - name: store_sales_to_date + destination: + name: date_dim + 'on': "[store_sales::ss_sold_date_sk] = [date_dim::d_date_sk]" + type: INNER + cardinality: MANY_TO_ONE + - name: store_sales_to_customer + destination: + name: customer + 'on': "[store_sales::ss_customer_sk] = [customer::c_customer_sk]" + type: INNER + cardinality: MANY_TO_ONE + - name: store_sales_to_item + destination: + name: item + 'on': "[store_sales::ss_item_sk] = [item::i_item_sk]" + type: INNER + cardinality: MANY_TO_ONE + - name: store_sales_to_store + destination: + name: store + 'on': "[store_sales::ss_store_sk] = [store::s_store_sk]" + type: INNER + cardinality: MANY_TO_ONE diff --git a/converters/thoughtspot/tests/fixtures/tpcds/tpcds_retail_model.model.tml b/converters/thoughtspot/tests/fixtures/tpcds/tpcds_retail_model.model.tml new file mode 100644 index 00000000..84b9df1a --- /dev/null +++ b/converters/thoughtspot/tests/fixtures/tpcds/tpcds_retail_model.model.tml @@ -0,0 +1,312 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The TPC-DS retail model shared across this converter's sibling test +# fixtures: 5 core datasets (store_sales, date_dim, customer, item, store), +# their 4 core relationships and 5 core metrics, plus one additional SQL +# View dataset (store_returns_sv) and three additional relationships added +# deliberately to exercise a composite-key join, a non-equality join +# condition, and a ONE_TO_MANY join declared from its "one" side (store has +# many store_returns) -- TML's own from/to naming does not encode which side +# of a join is which, cardinality does, and ONE_TO_MANY is the one value +# where Ossie's spec (core-spec/spec.yaml: `from` is the many side, `to` is +# the one side) requires the emitted relationship's endpoints to be the +# reverse of how this join is declared here -- plus four additional metrics +# covering the remaining metric shapes and formula constructs this converter +# has to handle, and one non-Latin (CJK) display name on `customer` with no +# ASCII form to fold onto (see tml_to_ossie.py's +# `_field_or_metric_identifier`). +model: + name: tpcds_retail_model + description: "TPC-DS retail semantic model used as a shared test fixture." + model_tables: + - name: store_sales + joins: + - referencing_join: store_sales_to_date + - referencing_join: store_sales_to_customer + - referencing_join: store_sales_to_item + - referencing_join: store_sales_to_store + - name: date_dim + - name: customer + - name: item + - name: store + joins: + # A ONE_TO_MANY join declared from the "one" side: a store has many + # store_returns. Exercises the endpoint swap this converter applies + # to a ONE_TO_MANY join -- the emitted Ossie relationship's `from` + # is store_returns_sv (the many side) and `to` is store (the one + # side), the reverse of this join's own declared from/to. + - with: store_returns_sv + 'on': "[store::s_store_sk] = [store_returns_sv::sr_store_sk]" + type: INNER + cardinality: ONE_TO_MANY + - name: store_returns_sv + joins: + # A composite-key equality join: store_sales's own natural key is + # (ss_item_sk, ss_ticket_number), so this is the one relationship + # in the model whose from_columns/to_columns each carry two + # columns. + - with: store_sales + 'on': "[store_returns_sv::sr_item_sk] = [store_sales::ss_item_sk] and [store_returns_sv::sr_ticket_number] = [store_sales::ss_ticket_number]" + type: INNER + cardinality: MANY_TO_ONE + # A non-equality join condition: one equality pair narrowed by a + # residual `<=` predicate, so the relationship is still emitted + # (with the residual predicate riding along) but does not qualify + # as key evidence. + - with: item + 'on': "[store_returns_sv::sr_item_sk] = [item::i_item_sk] and [store_returns_sv::sr_return_amt] <= [item::i_current_price]" + type: INNER + cardinality: MANY_TO_ONE + columns: + # -- store_sales: physical attributes. ss_ticket_number is deliberately + # -- not listed here, so it stays a Table-only physical column. + - name: ss_sold_date_sk + column_id: store_sales::ss_sold_date_sk + properties: + column_type: ATTRIBUTE + - name: ss_item_sk + column_id: store_sales::ss_item_sk + properties: + column_type: ATTRIBUTE + - name: ss_customer_sk + column_id: store_sales::ss_customer_sk + properties: + column_type: ATTRIBUTE + - name: ss_store_sk + column_id: store_sales::ss_store_sk + properties: + column_type: ATTRIBUTE + - name: ss_quantity + column_id: store_sales::ss_quantity + properties: + column_type: ATTRIBUTE + - name: ss_sales_price + column_id: store_sales::ss_sales_price + properties: + column_type: ATTRIBUTE + - name: ss_ext_sales_price + column_id: store_sales::ss_ext_sales_price + properties: + column_type: ATTRIBUTE + - name: ss_net_profit + column_id: store_sales::ss_net_profit + properties: + column_type: ATTRIBUTE + # -- date_dim + - name: d_date_sk + column_id: date_dim::d_date_sk + properties: + column_type: ATTRIBUTE + - name: d_date + column_id: date_dim::d_date + properties: + column_type: ATTRIBUTE + - name: d_year + column_id: date_dim::d_year + properties: + column_type: ATTRIBUTE + - name: d_quarter_name + column_id: date_dim::d_quarter_name + properties: + column_type: ATTRIBUTE + - name: d_moy + column_id: date_dim::d_moy + properties: + column_type: ATTRIBUTE + # -- customer. customer_full_name is a computed attribute (a formula, + # -- not a column_id), attributed to this dataset because both of its + # -- column references resolve here. + - name: c_customer_sk + column_id: customer::c_customer_sk + properties: + column_type: ATTRIBUTE + - name: c_customer_id + column_id: customer::c_customer_id + properties: + column_type: ATTRIBUTE + - name: c_first_name + column_id: customer::c_first_name + properties: + column_type: ATTRIBUTE + - name: c_last_name + column_id: customer::c_last_name + properties: + column_type: ATTRIBUTE + - name: customer_full_name + formula_id: formula_customer_full_name + properties: + column_type: ATTRIBUTE + - name: c_email_address + column_id: customer::c_email_address + properties: + column_type: ATTRIBUTE + # A non-Latin (CJK) display name: identifiers.normalise has no ASCII + # form to fold it onto, so this field falls back to its physical + # column's own warehouse name (c_kana_name -> "c_kana_name") instead of + # being dropped -- see identifiers.py / tml_to_ossie.py's + # _field_or_metric_identifier. + - name: カナ名 + column_id: customer::カナ名 + properties: + column_type: ATTRIBUTE + # -- item + - name: i_item_sk + column_id: item::i_item_sk + properties: + column_type: ATTRIBUTE + - name: i_item_id + column_id: item::i_item_id + properties: + column_type: ATTRIBUTE + - name: i_item_desc + column_id: item::i_item_desc + properties: + column_type: ATTRIBUTE + - name: i_brand + column_id: item::i_brand + properties: + column_type: ATTRIBUTE + - name: i_category + column_id: item::i_category + properties: + column_type: ATTRIBUTE + - name: i_current_price + column_id: item::i_current_price + properties: + column_type: ATTRIBUTE + # -- store. s_store_name's db_column_name differs (STORE_NM); "on" is + # -- a YAML 1.1 boolean token and a Snowflake BOOL column. + - name: s_store_sk + column_id: store::s_store_sk + properties: + column_type: ATTRIBUTE + - name: s_store_id + column_id: store::s_store_id + properties: + column_type: ATTRIBUTE + - name: s_store_name + column_id: store::s_store_name + properties: + column_type: ATTRIBUTE + - name: s_city + column_id: store::s_city + properties: + column_type: ATTRIBUTE + - name: s_state + column_id: store::s_state + properties: + column_type: ATTRIBUTE + - name: s_number_employees + column_id: store::s_number_employees + properties: + column_type: ATTRIBUTE + - name: "on" + column_id: "store::on" + description: "Whether the store is currently active and open for business." + properties: + column_type: ATTRIBUTE + # -- store_returns_sv. sr_return_quantity is deliberately not listed + # -- here, so it stays a SQL View-only physical column, referenced only + # -- from total_return_quantity's formula below. + - name: sr_item_sk + column_id: store_returns_sv::sr_item_sk + properties: + column_type: ATTRIBUTE + - name: sr_ticket_number + column_id: store_returns_sv::sr_ticket_number + properties: + column_type: ATTRIBUTE + - name: sr_return_amt + column_id: store_returns_sv::sr_return_amt + properties: + column_type: ATTRIBUTE + # -- the 5 core TPC-DS metrics + - name: total_sales + formula_id: formula_total_sales + properties: + column_type: MEASURE + - name: total_profit + formula_id: formula_total_profit + properties: + column_type: MEASURE + - name: customer_lifetime_value + formula_id: formula_customer_lifetime_value + properties: + column_type: MEASURE + - name: sales_by_brand + formula_id: formula_sales_by_brand + properties: + column_type: MEASURE + - name: store_productivity + formula_id: formula_store_productivity + properties: + column_type: MEASURE + # -- additional metrics: one of each remaining metric shape, a + # -- cross-referencing formula, and a brace-carrying window formula. + - name: total_return_quantity + column_id: store_returns_sv::sr_return_quantity + properties: + column_type: MEASURE + aggregation: SUM + - name: avg_price_adjustment + formula_id: formula_avg_price_adjustment + properties: + column_type: MEASURE + aggregation: AVERAGE + - name: profit_margin + formula_id: formula_profit_margin + properties: + column_type: MEASURE + - name: prior_period_profit + formula_id: formula_prior_period_profit + properties: + column_type: MEASURE + formulas: + - id: formula_customer_full_name + name: customer_full_name + expr: "concat ( concat ( [customer::c_first_name] , ' ' ) , [customer::c_last_name] )" + - id: formula_total_sales + name: total_sales + expr: "sum ( [store_sales::ss_ext_sales_price] )" + - id: formula_total_profit + name: total_profit + expr: "sum ( [store_sales::ss_net_profit] )" + - id: formula_customer_lifetime_value + name: customer_lifetime_value + expr: "sum ( [store_sales::ss_ext_sales_price] ) / unique count ( [customer::c_customer_sk] )" + - id: formula_sales_by_brand + name: sales_by_brand + expr: "sum ( [store_sales::ss_ext_sales_price] )" + - id: formula_store_productivity + name: store_productivity + expr: "sum ( [store_sales::ss_ext_sales_price] ) / nullif ( sum ( [store::s_number_employees] ) , 0 )" + - id: formula_avg_price_adjustment + name: avg_price_adjustment + expr: "[store_sales::ss_ext_sales_price] - [store_sales::ss_sales_price]" + # A formula cross-referencing two other formulas by id -- the id form + # (`[formula_]`), never the display-name form. + - id: formula_profit_margin + name: profit_margin + expr: "[formula_total_profit] / [formula_total_sales]" + # A brace-carrying window formula. The `{ }` reset-group argument means + # this has to be a folded block scalar (`>-`), or the YAML will not + # parse. + - id: formula_prior_period_profit + name: prior_period_profit + expr: >- + last_value ( sum ( [store_sales::ss_net_profit] ) , query_groups ( ) , { [date_dim::d_date] } ) diff --git a/converters/thoughtspot/tests/test_cli.py b/converters/thoughtspot/tests/test_cli.py new file mode 100644 index 00000000..779e9713 --- /dev/null +++ b/converters/thoughtspot/tests/test_cli.py @@ -0,0 +1,341 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the ``ossie-thoughtspot`` command-line entry point. + +Exercises `cli.main` directly (not a subprocess) for speed, plus one test that +resolves the declared `pyproject.toml` console-script string dynamically, so a +typo there fails here instead of surfacing only after a user installs the +package. +""" +from __future__ import annotations + +import importlib +import json +import re +from pathlib import Path + +import pytest + +from ossie_thoughtspot import _yaml, cli, ossie_to_thoughtspot, tml, tml_to_ossie +from ossie_thoughtspot.errors import ConversionError + +PACKAGE_ROOT = Path(__file__).resolve().parents[1] +FIXTURES_ROOT = PACKAGE_ROOT / "tests" / "fixtures" + + +def _tml_paths(fixture_name: str) -> list[Path]: + return sorted((FIXTURES_ROOT / fixture_name).glob("*.tml")) + + +def _tml_argv(fixture_name: str) -> list[str]: + return [str(p) for p in _tml_paths(fixture_name)] + + +def _write_ossie_yaml_from_fixture(fixture_name: str, target: Path) -> None: + """An Apache Ossie YAML document converted from a TML fixture set -- the + natural input `to-tml` expects, rather than a hand-authored one.""" + texts = [(str(p), p.read_text(encoding="utf-8")) for p in _tml_paths(fixture_name)] + document_set = tml.load_document_set(texts) + result = tml_to_ossie.convert(document_set) + target.write_text(_yaml.dump(result.model), encoding="utf-8") + + +def _inject_rls_rules(src_dir: Path, dst_dir: Path, *, table_filename: str) -> None: + """Copy a fixture directory, adding `rls_rules` to one table document. + + `test_fixtures.py`'s own check forbids `rls_rules` in the committed fixtures + themselves -- an ERROR-severity issue (`TS-DATASET-RLS-RULES`) needs its own, + disposable copy rather than mutating a shared fixture. + """ + dst_dir.mkdir(parents=True, exist_ok=True) + for path in src_dir.glob("*.tml"): + text = path.read_text(encoding="utf-8") + if path.name == table_filename: + text += ' rls_rules:\n - name: rule1\n filter: "[customer_id] = 1"\n' + (dst_dir / path.name).write_text(text, encoding="utf-8") + + +# --------------------------------------------------------------------------- +# The console-script entry point +# --------------------------------------------------------------------------- + + +def test_console_script_entry_point_resolves(): + # Reads the declared target straight out of pyproject.toml -- not a + # hardcoded `from ossie_thoughtspot import cli` -- so a typo in the + # `[project.scripts]` string itself (wrong module, wrong attribute) fails + # this test rather than only a user's `pip install`. + text = (PACKAGE_ROOT / "pyproject.toml").read_text(encoding="utf-8") + match = re.search(r'(?m)^ossie-thoughtspot\s*=\s*"([^"]+)"', text) + assert match, "no ossie-thoughtspot console-script entry declared in pyproject.toml" + module_name, _, attr_name = match.group(1).partition(":") + module = importlib.import_module(module_name) + target = getattr(module, attr_name) + assert callable(target) + + +# --------------------------------------------------------------------------- +# --help +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("argv", [[], ["to-ossie"], ["to-tml"]]) +def test_help_works_for_both_subcommands(argv, capsys): + with pytest.raises(SystemExit) as exc_info: + cli.main([*argv, "--help"]) + assert exc_info.value.code == 0 + out, err = capsys.readouterr() + assert out # argparse writes --help to stdout + assert err == "" + + +# --------------------------------------------------------------------------- +# to-ossie +# --------------------------------------------------------------------------- + + +def test_to_ossie_writes_a_loadable_ossie_document(tmp_path): + out_path = tmp_path / "out.ossie.yaml" + issues_path = tmp_path / "issues.json" + code = cli.main( + ["to-ossie", *_tml_argv("tpcds"), "-o", str(out_path), "--issues", str(issues_path)] + ) + assert code == 0 + document = _yaml.load(out_path.read_text(encoding="utf-8")) + assert isinstance(document, dict) + assert document["semantic_model"][0]["name"] == "tpcds_retail_model" + + +def test_to_ossie_help_documents_the_overwrite_flag(capsys): + with pytest.raises(SystemExit): + cli.main(["to-ossie", "--help"]) + out, _ = capsys.readouterr() + assert "--force" in out + + +# --------------------------------------------------------------------------- +# to-tml +# --------------------------------------------------------------------------- + + +def test_to_tml_writes_expected_filenames(tmp_path): + ossie_path = tmp_path / "input.ossie.yaml" + _write_ossie_yaml_from_fixture("minimal", ossie_path) + out_dir = tmp_path / "out" + + code = cli.main(["to-tml", str(ossie_path), "-o", str(out_dir)]) + assert code == 0 + assert {p.name for p in out_dir.iterdir()} == { + "orders.table.tml", + "customers.table.tml", + "minimal_orders_model.model.tml", + } + + +def test_to_tml_writes_tables_before_the_model(tmp_path, monkeypatch): + ossie_path = tmp_path / "input.ossie.yaml" + _write_ossie_yaml_from_fixture("minimal", ossie_path) + out_dir = tmp_path / "out" + + write_order: list[str] = [] + resolved_out_dir = out_dir.resolve() + original_write_text = Path.write_text + + def _tracking_write_text(self, *args, **kwargs): + if self.parent == resolved_out_dir: + write_order.append(self.name) + return original_write_text(self, *args, **kwargs) + + monkeypatch.setattr(Path, "write_text", _tracking_write_text) + + code = cli.main(["to-tml", str(ossie_path), "-o", str(out_dir)]) + assert code == 0 + assert write_order[-1] == "minimal_orders_model.model.tml" + assert set(write_order[:-1]) == {"orders.table.tml", "customers.table.tml"} + + +# --------------------------------------------------------------------------- +# Issues: JSON, routed correctly, never mixed with document output +# --------------------------------------------------------------------------- + + +def test_issues_land_in_the_issues_file_as_json_and_not_in_the_document(tmp_path): + out_path = tmp_path / "out.ossie.yaml" + issues_path = tmp_path / "issues.json" + code = cli.main( + ["to-ossie", *_tml_argv("tpcds"), "-o", str(out_path), "--issues", str(issues_path)] + ) + assert code == 0 # tpcds carries WARNING/INFO issues only, never ERROR + + issues = json.loads(issues_path.read_text(encoding="utf-8")) + assert isinstance(issues, list) + assert len(issues) > 0 + assert {i["severity"] for i in issues} <= {"INFO", "WARNING", "ERROR"} + + document_text = out_path.read_text(encoding="utf-8") + for issue in issues: + assert issue["code"] not in document_text + assert issue["message"] not in document_text + + +def test_issues_default_to_stderr_and_stdout_stays_silent(tmp_path, capsys): + out_path = tmp_path / "out.ossie.yaml" + code = cli.main(["to-ossie", *_tml_argv("minimal"), "-o", str(out_path)]) + assert code == 0 + + out, err = capsys.readouterr() + assert out == "" # never interleaved with document output on stdout + issues = json.loads(err) + assert isinstance(issues, list) + + +# --------------------------------------------------------------------------- +# Exit code: tied to has_errors(), not to the mere presence of an issue +# --------------------------------------------------------------------------- + + +def test_exit_code_zero_on_a_clean_conversion(tmp_path): + # The minimal fixture's only issue is INFO-severity. + code = cli.main( + ["to-ossie", *_tml_argv("minimal"), "-o", str(tmp_path / "out.yaml")] + ) + assert code == 0 + + +def test_exit_code_zero_on_warnings_only(tmp_path): + # The tpcds fixture carries WARNING-severity issues but no ERROR. + code = cli.main(["to-ossie", *_tml_argv("tpcds"), "-o", str(tmp_path / "out.yaml")]) + assert code == 0 + + +def test_exit_code_one_on_an_error_severity_issue_but_the_document_is_still_written(tmp_path): + error_fixture = tmp_path / "error_fixture" + _inject_rls_rules( + FIXTURES_ROOT / "minimal", error_fixture, table_filename="orders.table.tml" + ) + out_path = tmp_path / "out.yaml" + issues_path = tmp_path / "issues.json" + + code = cli.main( + [ + "to-ossie", + *[str(p) for p in error_fixture.glob("*.tml")], + "-o", + str(out_path), + "--issues", + str(issues_path), + ] + ) + + assert code == 1 + issues = json.loads(issues_path.read_text(encoding="utf-8")) + assert any(i["severity"] == "ERROR" for i in issues) + # A conversion with a declared-loss ERROR is still a *successful* conversion + # that reported it -- the document is written regardless of exit code. + assert out_path.exists() + document = _yaml.load(out_path.read_text(encoding="utf-8")) + assert document["semantic_model"][0]["name"] == "minimal_orders_model" + + +# --------------------------------------------------------------------------- +# Overwrite behaviour +# --------------------------------------------------------------------------- + + +def test_refuses_to_overwrite_an_existing_output_file_without_force(tmp_path, capsys): + out_path = tmp_path / "out.yaml" + out_path.write_text("pre-existing content\n", encoding="utf-8") + + code = cli.main(["to-ossie", *_tml_argv("minimal"), "-o", str(out_path)]) + assert code == 1 + assert out_path.read_text(encoding="utf-8") == "pre-existing content\n" + _, err = capsys.readouterr() + assert "--force" in err + + +def test_force_allows_overwriting_an_existing_output_file(tmp_path): + out_path = tmp_path / "out.yaml" + out_path.write_text("pre-existing content\n", encoding="utf-8") + + code = cli.main(["to-ossie", *_tml_argv("minimal"), "-o", str(out_path), "--force"]) + assert code == 0 + assert "pre-existing content" not in out_path.read_text(encoding="utf-8") + + +# --------------------------------------------------------------------------- +# Two additional tests, chosen for what the filesystem-writing contract calls +# out as its two real hazards: an output path escaping the target directory, +# and a partial multi-file write when only some target files already exist. +# --------------------------------------------------------------------------- + + +def test_safe_target_path_refuses_to_escape_the_output_directory(tmp_path): + # `tml.dump_document_set` already sanitises every filename it mints, so a + # document name cannot reach `cli._safe_target_path` carrying a path + # separator through the normal `to-tml` flow. This attacks the guard + # directly, bypassing the dumper, because it is the last line of defence + # this module owns and the one thing it must never trust implicitly. + directory = tmp_path / "out" + directory.mkdir() + with pytest.raises(ConversionError): + cli._safe_target_path(directory, "../../etc/passwd") + + # A resolved symlink case too (this project has already gotten a naive + # string-prefix comparison wrong on macOS before): a candidate directory + # that is itself reached through a symlink must still be treated as + # legitimate, not rejected as "outside" its own resolved self. + real_target = tmp_path / "real" + real_target.mkdir() + link_dir = tmp_path / "link" + link_dir.symlink_to(real_target) + resolved = cli._safe_target_path(link_dir, "orders.table.tml") + assert resolved == (real_target / "orders.table.tml").resolve() + + +def test_to_tml_refuses_atomically_leaving_no_partial_output(tmp_path): + ossie_path = tmp_path / "input.ossie.yaml" + _write_ossie_yaml_from_fixture("minimal", ossie_path) + out_dir = tmp_path / "out" + out_dir.mkdir() + # Only one of the three eventual output files already exists. + (out_dir / "customers.table.tml").write_text("do not touch\n", encoding="utf-8") + + code = cli.main(["to-tml", str(ossie_path), "-o", str(out_dir)]) + + assert code == 1 + # The pre-existing file is untouched, and nothing else was written -- + # a conflict on one target must not let the other, non-conflicting + # targets be written anyway. + assert (out_dir / "customers.table.tml").read_text(encoding="utf-8") == "do not touch\n" + assert not (out_dir / "orders.table.tml").exists() + assert not (out_dir / "minimal_orders_model.model.tml").exists() + + +def test_ossie_to_thoughtspot_convert_agrees_with_the_cli_on_filenames(tmp_path): + # Cross-check the CLI's output against calling the library directly -- + # guards against the CLI silently diverging from `dump_document_set`'s + # own naming (e.g. by renaming files itself instead of using the names + # the dumper already sanitised and ordered). + ossie_path = tmp_path / "input.ossie.yaml" + _write_ossie_yaml_from_fixture("minimal", ossie_path) + ossie_document = _yaml.load(ossie_path.read_text(encoding="utf-8")) + expected = {name for name, _ in tml.dump_document_set(ossie_to_thoughtspot.convert(ossie_document).documents)} + + out_dir = tmp_path / "out" + cli.main(["to-tml", str(ossie_path), "-o", str(out_dir)]) + assert {p.name for p in out_dir.iterdir()} == expected diff --git a/converters/thoughtspot/tests/test_constants.py b/converters/thoughtspot/tests/test_constants.py new file mode 100644 index 00000000..bf1584a0 --- /dev/null +++ b/converters/thoughtspot/tests/test_constants.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from ossie_thoughtspot import constants + + +def test_vendor_key_and_dialect_are_distinct_constants(): + # Same value today, different upstream governance — the two constants + # must not collapse into one name. + assert constants.VENDOR_KEY == "THOUGHTSPOT" + assert constants.DIALECT == "THOUGHTSPOT" + # Both names must exist independently, so a later divergence touches one call site. + assert "VENDOR_KEY" in vars(constants) + assert "DIALECT" in vars(constants) + + +def test_dialect_is_registered_upstream(): + # apache/ossie#351 merged 2026-09-01: THOUGHTSPOT is a registered Dialect. + # ANSI_SQL is still emitted alongside it for portable expressions. + assert constants.DIALECT_IS_REGISTERED is True + assert constants.PORTABLE_DIALECT == "ANSI_SQL" + + +def test_spec_series_is_major_minor_not_an_exact_version(): + # Upstream's first release is proposed as 0.3.0; an exact pin on 0.2.0.dev0 would break. + assert constants.SPEC_SERIES == "0.2" + assert constants.STASH_VERSION == 1 diff --git a/converters/thoughtspot/tests/test_datatypes.py b/converters/thoughtspot/tests/test_datatypes.py new file mode 100644 index 00000000..47d2f311 --- /dev/null +++ b/converters/thoughtspot/tests/test_datatypes.py @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest +from ossie_thoughtspot.datatypes import ( + OSSIE_DATATYPES, declared_loss, to_ossie, to_tml, +) + + +class TestToTml: + @pytest.mark.parametrize("datatype,expected", [ + ("String", "VARCHAR"), ("Integer", "INT64"), ("Decimal", "DOUBLE"), + ("Float", "DOUBLE"), ("Boolean", "BOOLEAN"), ("Date", "DATE"), + ("Time", "VARCHAR"), ("DateTime", "DATE_TIME"), + ("DateTimeTz", "DATE_TIME"), ("Opaque", "VARCHAR"), + ]) + def test_every_ossie_datatype_maps(self, datatype, expected): + assert to_tml(datatype) == expected + + def test_the_map_covers_the_whole_enum(self): + # A datatype added to the Ossie enum must fail here, not silently + # convert to nothing. + for datatype in OSSIE_DATATYPES: + assert to_tml(datatype) + + def test_missing_datatype_infers_rather_than_raising(self): + # Ossie makes datatype optional; TML makes db_column_properties compulsory. + assert to_tml(None) == "INT64" + + def test_connection_spellings_are_selectable(self): + assert to_tml("Boolean", boolean_spelling="BOOL") == "BOOL" + assert to_tml("Float", float_spelling="FLOAT") == "FLOAT" + + def test_the_float_spelling_does_not_leak_into_decimal(self): + # Decimal is DOUBLE on every connection — only Float is BigQuery-sensitive. + assert to_tml("Decimal", float_spelling="FLOAT") == "DOUBLE" + + def test_an_unknown_datatype_raises(self): + with pytest.raises(ValueError, match="Nonsense"): + to_tml("Nonsense") + + def test_case_sensitive_datatype_is_unknown_rather_than_normalised(self): + # Ossie datatypes are spelled exactly as the enum ("Boolean", not + # "boolean" or "BOOLEAN"). A caller that passes a differently-cased + # variant — easy to do if the value came from a case-folding step + # upstream, or from a TML string mistaken for an Ossie one — must get + # a clear error, not a silent no-op or a wrong mapping. + with pytest.raises(ValueError, match="boolean"): + to_tml("boolean") + + +class TestToOssie: + @pytest.mark.parametrize("tml_type,expected", [ + ("VARCHAR", "String"), ("INT64", "Integer"), ("DOUBLE", "Decimal"), + ("FLOAT", "Float"), ("BOOL", "Boolean"), ("BOOLEAN", "Boolean"), + ("DATE", "Date"), ("DATE_TIME", "DateTime"), + ]) + def test_known_tml_types(self, tml_type, expected): + assert to_ossie(tml_type) == expected + + def test_an_unknown_tml_type_returns_none_rather_than_guessing(self): + # datatype is optional in Ossie, so omitting it is a legitimate answer + # and strictly better than inventing one. + assert to_ossie("GEOGRAPHY") is None + + def test_sql_type_names_are_not_accepted(self): + # ThoughtSpot rejects these itself: "DataType BIGINT does not match CDW DataType". + assert to_ossie("BIGINT") is None + + def test_empty_string_returns_none(self): + # A blank data_type is a plausible malformed-document artefact (a + # missing YAML value that parses as ""), and it is not a key in the + # map. It must return None like any other unmapped string, not raise. + assert to_ossie("") is None + + +class TestDeclaredLoss: + @pytest.mark.parametrize("datatype", ["Float", "Time", "DateTimeTz", "Opaque"]) + def test_the_four_lossy_types_are_named(self, datatype): + assert declared_loss(datatype) + + @pytest.mark.parametrize("datatype", ["String", "Integer", "Decimal", + "Boolean", "Date", "DateTime"]) + def test_the_lossless_types_are_not(self, datatype): + assert declared_loss(datatype) is None + + def test_round_trip_is_exact_for_every_non_lossy_type(self): + # The property that makes `declared_loss` trustworthy: if it says a type + # is lossless, TML -> Ossie -> TML really does return the same value. + for datatype in OSSIE_DATATYPES: + if declared_loss(datatype) is None: + assert to_ossie(to_tml(datatype)) == datatype diff --git a/converters/thoughtspot/tests/test_fixtures.py b/converters/thoughtspot/tests/test_fixtures.py new file mode 100644 index 00000000..156cd4ac --- /dev/null +++ b/converters/thoughtspot/tests/test_fixtures.py @@ -0,0 +1,295 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the shared TML fixture sets under tests/fixtures/. + +Two fixture sets live there. `tpcds/` mirrors the dataset, field, +relationship and metric names of the TPC-DS retail model every other +converter in this repository round-trips, so this converter is comparable +to its siblings rather than tested against a shape only it has seen; it +also deliberately carries the constructs that have broken in this +converter's own history and are easy for a fixture author to omit: a +display name differing from its db_column_name, a column name spelled as a +YAML 1.1 boolean token, a brace-carrying window formula, a formula +cross-reference, a connection-specific BOOL column, a SQL View with an +output alias differing from its column name, a physical column the Model +does not surface, a non-equality join condition, a composite-key +relationship, one metric of each of the three TML shapes this converter has +to compose, and a non-Latin (CJK) display name with no ASCII form for +`identifiers.normalise` to fold onto -- which reached a public PR before any +fixture had one, dropping every field and metric of a non-Latin-named model +outright (see `_field_or_metric_identifier` in tml_to_ossie.py). `minimal/` +is the smallest possible pair -- one model, two tables, one relationship -- +for debugging a failure without the larger fixture's noise. + +Each fixture directory holds the TML documents (`*.table.tml`, +`*.sql_view.tml`, `*.model.tml`) plus one `expected.ossie.yaml`: the Ossie +document `tml_to_ossie.convert()` produces from that TML, checked by hand +against the construct mapping this converter implements before being +committed here. +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from ossie_thoughtspot import _yaml, tml, tml_to_ossie +from ossie_thoughtspot.constants import ( + DATASET_STASH_UNSURFACED_COLUMNS, + DIALECT, + FIELD_STASH_DATA_TYPE, + METRIC_SHAPE_COLUMN_AGGREGATION, + METRIC_SHAPE_SCALAR_FORMULA_PLUS_AGGREGATION, + METRIC_STASH_SHAPE, + PORTABLE_DIALECT, + RELATIONSHIP_STASH_ON_EXPRESSION, + VENDOR_KEY, +) + +FIXTURES_ROOT = Path(__file__).resolve().parent / "fixtures" +FIXTURE_SETS = ("minimal", "tpcds") + +#: Every TML document kind `tml.load_document` accepts, mirrored here so a +#: fixture that accidentally ships a document of some other kind (a +#: `worksheet:`, say) is caught by the loading test rather than silently +#: skipped by whatever later step happens to ignore it. +_TML_KINDS = frozenset({"model", "table", "sql_view"}) + + +def _tml_paths(fixture_dir: Path) -> list[Path]: + return sorted(fixture_dir.glob("*.tml")) + + +def _load_document_set(fixture_dir: Path) -> tml.DocumentSet: + texts = [ + (str(path), path.read_text(encoding="utf-8")) for path in _tml_paths(fixture_dir) + ] + return tml.load_document_set(texts) + + +def _load_expected(fixture_dir: Path) -> dict: + text = (fixture_dir / "expected.ossie.yaml").read_text(encoding="utf-8") + document = _yaml.load(text) + assert isinstance(document, dict), ( + f"{fixture_dir / 'expected.ossie.yaml'} did not parse to a mapping" + ) + return document + + +@pytest.mark.parametrize("fixture_name", FIXTURE_SETS) +class TestFixtureSetsLoad: + def test_the_fixture_directory_has_tml_documents(self, fixture_name): + fixture_dir = FIXTURES_ROOT / fixture_name + paths = _tml_paths(fixture_dir) + assert paths, f"expected at least one .tml fixture in {fixture_dir}" + + def test_every_tml_fixture_loads(self, fixture_name): + fixture_dir = FIXTURES_ROOT / fixture_name + for path in _tml_paths(fixture_dir): + document = tml.load_document(path.read_text(encoding="utf-8"), source=str(path)) + assert document.kind in _TML_KINDS + # A fixture must never carry a root-level guid -- these are + # hand-authored, portable documents, not exports from a live + # instance. + assert document.guid is None + + def test_the_fixture_set_loads_as_one_document_set(self, fixture_name): + fixture_dir = FIXTURES_ROOT / fixture_name + document_set = _load_document_set(fixture_dir) + assert document_set.model.kind == "model" + assert document_set.tables, "expected at least one table/sql_view document" + + +@pytest.mark.parametrize("fixture_name", FIXTURE_SETS) +class TestExpectedOutputIsValid: + def test_expected_output_validates_against_the_upstream_schema(self, fixture_name): + jsonschema = pytest.importorskip("jsonschema") + schema_path = Path(__file__).resolve().parents[3] / "core-spec" / "ossie-schema.json" + with open(schema_path) as fh: + schema = json.load(fh) + expected = _load_expected(FIXTURES_ROOT / fixture_name) + jsonschema.Draft202012Validator(schema).validate(expected) + + +@pytest.mark.parametrize("fixture_name", FIXTURE_SETS) +class TestConversionMatchesExpected: + def test_converting_the_fixture_set_produces_the_expected_document(self, fixture_name): + fixture_dir = FIXTURES_ROOT / fixture_name + document_set = _load_document_set(fixture_dir) + result = tml_to_ossie.convert(document_set) + expected = _load_expected(fixture_dir) + assert result.model == expected + + +@pytest.fixture(scope="module") +def _tpcds_semantic_model() -> dict: + document_set = _load_document_set(FIXTURES_ROOT / "tpcds") + return tml_to_ossie.convert(document_set).model["semantic_model"][0] + + +class TestTpcdsFixtureCoversItsRequiredConstructs: + """Assertions naming the specific constructs the TPC-DS fixture set was + built to exercise, so a future edit that accidentally drops one of them + fails here with a clear message rather than only failing the (much + larger) exact-document comparison above.""" + + @pytest.fixture + def dataset(self, _tpcds_semantic_model): + return _tpcds_semantic_model + + def test_mirrors_the_tpcds_model_name(self, dataset): + assert dataset["name"] == "tpcds_retail_model" + + def test_mirrors_the_five_core_datasets(self, dataset): + names = {d["name"] for d in dataset["datasets"]} + assert {"store_sales", "date_dim", "customer", "item", "store"} <= names + + def test_mirrors_the_four_core_relationships(self, dataset): + names = {r["name"] for r in dataset["relationships"]} + assert { + "store_sales_to_date", "store_sales_to_customer", + "store_sales_to_item", "store_sales_to_store", + } <= names + + def test_mirrors_the_five_core_metrics(self, dataset): + names = {m["name"] for m in dataset["metrics"]} + assert { + "total_sales", "total_profit", "customer_lifetime_value", + "sales_by_brand", "store_productivity", + } <= names + + def test_a_computed_attribute_formula_is_present(self, dataset): + customer = next(d for d in dataset["datasets"] if d["name"] == "customer") + field = next(f for f in customer["fields"] if f["name"] == "customer_full_name") + assert "datatype" not in field # a formula-backed field declares no type + + def test_a_non_latin_display_name_falls_back_to_its_warehouse_column_name(self, dataset): + # The regression this fixture exists to catch: a CJK-only display + # name (カナ名, "kana name") has no ASCII form for + # identifiers.normalise to fold onto. Before _field_or_metric_identifier + # existed, this field -- and every other field/metric in a + # non-Latin-named model -- was silently dropped rather than falling + # back to a usable identifier. + customer = next(d for d in dataset["datasets"] if d["name"] == "customer") + field = next(f for f in customer["fields"] if f["label"] == "カナ名") + assert field["name"] == "c_kana_name" # its own warehouse column name, not a placeholder + + def test_the_non_latin_display_name_issue_names_the_right_cause(self): + # This must never be misreported as a malformed column *reference* + # -- the [customer::カナ名] bracket itself parses fine; it is the + # display name that has no ASCII form. + document_set = _load_document_set(FIXTURES_ROOT / "tpcds") + result = tml_to_ossie.convert(document_set) + codes = {i["code"] for i in result.issues.as_dicts()} + assert "TS-FIELD-NAME-UNNORMALISABLE" in codes + + def test_store_has_a_display_name_differing_from_its_db_column_name(self, dataset): + store = next(d for d in dataset["datasets"] if d["name"] == "store") + field = next(f for f in store["fields"] if f["name"] == "s_store_name") + dialects = {d["dialect"]: d["expression"] for d in field["expression"]["dialects"]} + assert dialects[PORTABLE_DIALECT] == "store.STORE_NM" + + def test_the_on_column_survives_as_a_string_not_a_boolean(self, dataset): + store = next(d for d in dataset["datasets"] if d["name"] == "store") + field = next(f for f in store["fields"] if f["name"] == "on") + assert field["name"] == "on" + assert field["datatype"] == "Boolean" + + def test_a_brace_carrying_window_formula_round_trips_verbatim(self, dataset): + metric = next(m for m in dataset["metrics"] if m["name"] == "prior_period_profit") + dialects = {d["dialect"]: d["expression"] for d in metric["expression"]["dialects"]} + assert dialects[DIALECT] == ( + "last_value ( sum ( [store_sales::ss_net_profit] ) , query_groups ( ) , " + "{ [date_dim::d_date] } )" + ) + + def test_a_formula_cross_reference_is_preserved(self, dataset): + metric = next(m for m in dataset["metrics"] if m["name"] == "profit_margin") + dialects = {d["dialect"]: d["expression"] for d in metric["expression"]["dialects"]} + assert dialects[DIALECT] == "[formula_total_profit] / [formula_total_sales]" + assert PORTABLE_DIALECT not in dialects # a cross-reference is never portable + + def test_a_bool_column_keeps_its_connection_specific_spelling(self, dataset): + store = next(d for d in dataset["datasets"] if d["name"] == "store") + field = next(f for f in store["fields"] if f["name"] == "on") + extensions = {e["vendor_name"]: json.loads(e["data"]) for e in field["custom_extensions"]} + assert extensions[VENDOR_KEY][FIELD_STASH_DATA_TYPE] == "BOOL" + + def test_a_sql_view_is_present_with_a_differing_output_alias(self, dataset): + sv = next(d for d in dataset["datasets"] if d["name"] == "store_returns_sv") + field = next(f for f in sv["fields"] if f["name"] == "sr_return_amt") + dialects = {d["dialect"]: d["expression"] for d in field["expression"]["dialects"]} + assert dialects[PORTABLE_DIALECT] == "store_returns_sv.RETURN_AMT" + + def test_a_physical_column_the_model_does_not_surface_is_stashed(self, dataset): + store_sales = next(d for d in dataset["datasets"] if d["name"] == "store_sales") + extensions = { + e["vendor_name"]: json.loads(e["data"]) for e in store_sales["custom_extensions"] + } + unsurfaced = { + c["name"] for c in extensions[VENDOR_KEY][DATASET_STASH_UNSURFACED_COLUMNS] + } + assert "ss_ticket_number" in unsurfaced + + def test_a_composite_key_relationship_is_present(self, dataset): + relationship = next( + r for r in dataset["relationships"] if r["name"] == "store_returns_sv_to_store_sales" + ) + assert relationship["to_columns"] == ["ss_item_sk", "ss_ticket_number"] + store_sales = next(d for d in dataset["datasets"] if d["name"] == "store_sales") + assert store_sales["primary_key"] == ["ss_item_sk", "ss_ticket_number"] + + def test_a_non_equality_join_condition_yields_a_relationship_with_residuals(self, dataset): + relationship = next( + r for r in dataset["relationships"] if r["name"] == "store_returns_sv_to_item" + ) + extensions = { + e["vendor_name"]: json.loads(e["data"]) for e in relationship["custom_extensions"] + } + assert extensions[VENDOR_KEY][RELATIONSHIP_STASH_ON_EXPRESSION] == ( + "[store_returns_sv::sr_item_sk] = [item::i_item_sk] and " + "[store_returns_sv::sr_return_amt] <= [item::i_current_price]" + ) + + def test_the_three_metric_shapes_are_all_present(self, dataset): + metrics_by_name = {m["name"]: m for m in dataset["metrics"]} + + # Bare aggregate over a physical column with no separate stash -- + # this is also the default TML shape (`formula`), which the writer + # never stashes. + assert "custom_extensions" not in metrics_by_name["total_sales"] + + # A physical column plus a load-bearing aggregation. + column_aggregation = metrics_by_name["total_return_quantity"] + extensions = { + e["vendor_name"]: json.loads(e["data"]) + for e in column_aggregation["custom_extensions"] + } + assert extensions[VENDOR_KEY][METRIC_STASH_SHAPE] == METRIC_SHAPE_COLUMN_AGGREGATION + + # A scalar formula plus a load-bearing aggregation, composed into + # one expression. + scalar_plus_agg = metrics_by_name["avg_price_adjustment"] + extensions = { + e["vendor_name"]: json.loads(e["data"]) for e in scalar_plus_agg["custom_extensions"] + } + assert ( + extensions[VENDOR_KEY][METRIC_STASH_SHAPE] + == METRIC_SHAPE_SCALAR_FORMULA_PLUS_AGGREGATION + ) diff --git a/converters/thoughtspot/tests/test_formula.py b/converters/thoughtspot/tests/test_formula.py new file mode 100644 index 00000000..8c58d346 --- /dev/null +++ b/converters/thoughtspot/tests/test_formula.py @@ -0,0 +1,368 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest +from ossie_thoughtspot.formula import ( + _scan, find_call_names, find_column_refs, find_formula_refs, find_parameter_refs, + is_bare_column_ref, is_formula_reference, rewrite_column_refs, split_call, +) + + +class TestSplitCall: + def test_simple_call(self): + assert split_call("sum ( [ORDERS::Amount] )") == ("sum", ["[ORDERS::Amount]"]) + + def test_no_space_before_paren(self): + assert split_call("sum([ORDERS::Amount])") == ("sum", ["[ORDERS::Amount]"]) + + def test_multiple_arguments(self): + name, args = split_call("concat ( [A::x] , '-' , [A::y] )") + assert name == "concat" + assert args == ["[A::x]", "'-'", "[A::y]"] + + def test_nested_call_is_one_argument(self): + name, args = split_call("sum ( if ( [A::x] > 0 ) then [A::x] else 0 )") + assert name == "sum" + assert args == ["if ( [A::x] > 0 ) then [A::x] else 0"] + + def test_comma_inside_nested_parens_does_not_split(self): + name, args = split_call("round ( divide ( [A::x] , [A::y] ) , 2 )") + assert args == ["divide ( [A::x] , [A::y] )", "2"] + + def test_comma_inside_a_quoted_literal_does_not_split(self): + name, args = split_call("concat ( [A::x] , ', ' , [A::y] )") + assert args == ["[A::x]", "', '", "[A::y]"] + + def test_brace_group_is_one_argument(self): + # The documented window shape: braces are ThoughtSpot's grouping syntax + # and no SQL parser handles them, which is half the reason we tokenize. + name, args = split_call( + "last_value ( sum ( [T::c] ) , query_groups ( ) , { [D::date] } )" + ) + assert name == "last_value" + assert args == ["sum ( [T::c] )", "query_groups ( )", "{ [D::date] }"] + + def test_empty_argument_list(self): + assert split_call("query_groups ( )") == ("query_groups", []) + + def test_not_a_call_returns_none(self): + assert split_call("[ORDERS::Amount]") is None + assert split_call("42") is None + + def test_expression_that_merely_contains_a_call_is_not_a_single_call(self): + # `sum(a) + 1` has a call in it but is not one — a caller that treated + # it as `("sum", ["a"])` would silently drop the `+ 1`. + assert split_call("sum ( [A::x] ) + 1") is None + + def test_two_calls_side_by_side_is_not_a_single_call(self): + assert split_call("sum ( [A::x] ) / count ( [A::y] )") is None + + def test_unbalanced_parens_return_none_rather_than_raising(self): + assert split_call("sum ( [A::x]") is None + + def test_a_keyword_prefix_is_not_mistaken_for_a_function_name(self): + # `_CALL_HEAD` allows unbounded space-separated words because some + # real ThoughtSpot function names are multi-word (`unique count`). + # But an operator/control-flow keyword can never be part of a + # function name, so `true and count (...)` is an operator expression + # ending in something that merely looks like a call head — not a + # call named "true and count". + assert split_call("true and count ( [B::y] )") is None + + def test_a_real_multi_word_function_name_still_works(self): + # The keyword blocklist must not catch legitimate multi-word names. + assert split_call("unique count ( [B::y] )") == ( + "unique count", + ["[B::y]"], + ) + + +class TestFindCallNames: + def test_a_single_call_reports_its_own_name(self): + assert find_call_names("sum ( [A::x] )") == ["sum"] + + def test_a_call_nested_inside_another_reports_both_at_every_depth(self): + # split_call only ever answers about the single outer call; this is the + # function that has to see through a scalar wrapper to what is inside it. + assert find_call_names("round ( sum ( [A::x] ) , 2 )") == ["round", "sum"] + + def test_several_sibling_arguments_each_report_their_own_call(self): + assert find_call_names( + "group_aggregate ( sum ( [T::x] ) , query_groups ( ) , query_filters ( ) )" + ) == ["group_aggregate", "sum", "query_groups", "query_filters"] + + def test_a_call_name_inside_a_quoted_string_literal_is_not_reported(self): + # The literal text "STDDEV_POP(" here is the quoted SQL body of a + # sql_number_aggregate_op pass-through, not a real ThoughtSpot call. + assert find_call_names( + "sql_number_aggregate_op ( 'STDDEV_POP({0})' , [T::x] )" + ) == ["sql_number_aggregate_op"] + + def test_a_compound_expression_with_no_call_at_all_reports_nothing(self): + assert find_call_names("[A::x] - [A::y]") == [] + + def test_a_multi_word_call_name_is_reported_as_one_unit(self): + assert find_call_names("unique count ( [A::x] )") == ["unique count"] + + def test_a_leading_keyword_is_stripped_but_the_real_call_after_it_is_still_found(self): + # `true and count ( ... )` is an operator expression ending in + # something that looks like a call head starting with "true and" — + # split_call correctly refuses to call the whole thing "true and + # count", but the nested `count(...)` call is still real and must + # still be found here, unlike in split_call's single-outer-call + # question where the whole expression is rejected instead. + assert find_call_names("true and count ( [A::x] )") == ["count"] + + def test_a_keyword_that_is_also_a_real_catalog_function_name_is_still_excluded(self): + # `not` is a genuine ThoughtSpot catalog function ("not ( expr )"), not + # merely an operator token -- but it is also in the keyword blocklist, + # needed so an expression like "true and count ( ... )" is not misread + # as one call named "true and count". The blocklist has no way to tell + # the two uses of "not" apart, so a real `not ( ... )` call reports + # nothing here. This is a deliberate, accepted cost: this function's + # one caller only looks for aggregate names, and neither `not` nor + # `if` (the other such collision) is one, so losing them here costs + # that caller nothing. + assert find_call_names("not ( [A::x] )") == [] + + +class TestFindColumnRefs: + def test_finds_each_reference_in_order(self): + assert find_column_refs("[A::x] + [B::y]") == [("A", "x"), ("B", "y")] + + def test_keeps_duplicates(self): + assert find_column_refs("[A::x] + [A::x]") == [("A", "x"), ("A", "x")] + + def test_ignores_a_parameter_reference(self): + # `[Growth Rate]` has no `::` — it is a runtime parameter, not a column. + assert find_column_refs("[A::x] * [Growth Rate]") == [("A", "x")] + + def test_no_references(self): + assert find_column_refs("42") == [] + + def test_an_ambiguous_reference_raises_rather_than_silently_misreading(self): + # `identifiers.split_column_ref` raises on a reference with more than + # one `::` delimiter rather than silently taking the first one. This + # module delegates to it instead of a bare `str.split("::", 1)`, so + # the same failure must surface here too — even when the ambiguous + # reference sits among otherwise-valid ones in a longer expression. + # Silently misreading one reference is worse than failing the call. + with pytest.raises(ValueError): + find_column_refs("[A::x] + [ORDERS:::Col] + [B::y]") + + +class TestFindParameterRefs: + def test_finds_bracketed_names_without_a_table_qualifier(self): + assert find_parameter_refs("[A::x] * [Growth Rate]") == ["Growth Rate"] + + def test_returns_empty_when_every_reference_is_qualified(self): + assert find_parameter_refs("[A::x] + [B::y]") == [] + + def test_a_formula_cross_reference_is_not_reported_as_a_parameter(self): + # A formula cross-reference is a bracketed name with no `::`, the + # same shape a runtime parameter has -- the only thing telling them + # apart is the formula_ prefix (FORMULA_REFERENCE_PREFIX), and this + # is the one place a `::`-less bracket must NOT be reported. + assert find_parameter_refs("sum ( [formula_Margin] )") == [] + + def test_a_genuine_parameter_is_still_reported_alongside_a_cross_reference(self): + assert find_parameter_refs( + "[formula_Margin] * [Growth Rate]" + ) == ["Growth Rate"] + + +class TestFindFormulaRefs: + def test_finds_a_formula_cross_reference(self): + assert find_formula_refs("sum ( [formula_Margin] )") == ["formula_Margin"] + + def test_does_not_find_a_genuine_parameter(self): + assert find_formula_refs("[A::x] * [Growth Rate]") == [] + + def test_does_not_find_a_qualified_column_reference(self): + assert find_formula_refs("[A::x] + [formula_Y]") == ["formula_Y"] + + def test_an_expression_with_both_reports_only_the_cross_reference(self): + assert find_formula_refs("[formula_Margin] * [Growth Rate]") == ["formula_Margin"] + + +class TestIsFormulaReference: + def test_a_formula_id_shaped_name_is_a_formula_reference(self): + assert is_formula_reference("formula_Margin") is True + + def test_an_ordinary_parameter_name_is_not(self): + assert is_formula_reference("Growth Rate") is False + + def test_a_name_that_merely_contains_the_word_formula_is_not(self): + # Only a leading formula_ prefix counts -- a coincidental substring + # elsewhere in the name must not trip this. + assert is_formula_reference("My Formula Budget") is False + + +class TestIsBareColumnRef: + def test_a_lone_reference(self): + assert is_bare_column_ref("[ORDERS::Amount]") == ("ORDERS", "Amount") + + def test_surrounding_whitespace_is_tolerated(self): + assert is_bare_column_ref(" [ORDERS::Amount] ") == ("ORDERS", "Amount") + + def test_anything_more_is_not_bare(self): + assert is_bare_column_ref("[ORDERS::Amount] + 1") is None + assert is_bare_column_ref("sum ( [ORDERS::Amount] )") is None + assert is_bare_column_ref("[Growth Rate]") is None + + +class TestRewriteColumnRefs: + def test_rewrites_every_reference(self): + out = rewrite_column_refs( + "[A::x] + [B::y]", lambda t, c: f"{t.lower()}.{c.lower()}" + ) + assert out == "a.x + b.y" + + def test_leaves_a_parameter_reference_untouched(self): + out = rewrite_column_refs( + "[A::x] * [Growth Rate]", lambda t, c: f"{t.lower()}.{c.lower()}" + ) + assert out == "a.x * [Growth Rate]" + + def test_preserves_everything_between_references_byte_for_byte(self): + src = "concat ( [A::x] , ', ' , [A::y] )" + out = rewrite_column_refs(src, lambda t, c: f"{t}.{c}") + assert out == "concat ( A.x , ', ' , A.y )" + + def test_rewriting_is_not_confused_by_a_bracket_inside_a_literal(self): + src = "concat ( [A::x] , '[not::a::ref]' )" + out = rewrite_column_refs(src, lambda t, c: f"{t}.{c}") + assert out == "concat ( A.x , '[not::a::ref]' )" + + def test_is_byte_preserving_when_rename_returns_the_reference_unchanged(self): + # A stronger check than "rewrites every reference": if rename hands + # back exactly the bracketed text it was asked to replace, the whole + # expression — including irregular internal spacing — must come back + # unchanged. This would catch an off-by-one in the span arithmetic + # that a same-length rewrite (e.g. "a.x") could hide. + src = " concat( [A::x] ,'-',[B::y] ) " + out = rewrite_column_refs( + src, lambda t, c: f"[{t}::{c}]" + ) + assert out == src + + +class TestAdditionalEdgeCases: + def test_doubled_quote_inside_a_quoted_literal_does_not_split_the_argument(self): + # ThoughtSpot (like standard SQL) escapes an embedded quote by doubling + # it: 'it''s' is meant as the single literal `it's`. `_scan` has no + # explicit doubling special-case — it just toggles `quote` on every + # matching quote char — but that toggle still yields `in_quote=True` + # on every character of the pair (the close and the immediate reopen + # are both reported as quoted), so a comma between two doubled quotes + # would still read as inside a literal and would not split. Verified + # empirically (not just by hand-trace) before asserting this: the + # whole doubled-quote literal survives as one untouched argument. + name, args = split_call("concat ( [A::x] , 'it''s' , [A::y] )") + assert name == "concat" + assert args == ["[A::x]", "'it''s'", "[A::y]"] + + def test_column_reference_with_a_space_in_the_column_name(self): + # Real ThoughtSpot display names routinely contain spaces + # ("Order Date", "Sales Amount"). `_BRACKETED` matches everything + # between `[` and `]` with no whitespace restriction, so this must + # work exactly like any other reference. + assert find_column_refs("[Orders::Order Date] + 1") == [ + ("Orders", "Order Date") + ] + assert is_bare_column_ref("[Orders::Order Date]") == ( + "Orders", + "Order Date", + ) + out = rewrite_column_refs( + "[Orders::Order Date]", lambda t, c: f"{t}.{c.replace(' ', '_')}" + ) + assert out == "Orders.Order_Date" + + +class TestQuoteInsideBracketBody: + """A `[...]` body is an opaque identifier, not code — a quote character inside one + (a display name like `Manager's Bonus`, entirely routine in real ThoughtSpot data) is + part of the name, not a string delimiter. Before the fix, `_scan` toggled quote state + on any `'`/`"` anywhere in the text, including inside brackets, which desynchronised + quote tracking for everything after the apostrophe — silently dropping a later + reference, leaving it unrewritten, or rejecting a valid single call. + """ + + def test_find_column_refs_does_not_lose_a_later_reference(self): + assert find_column_refs("[Managers::Manager's Bonus] + [B::y]") == [ + ("Managers", "Manager's Bonus"), + ("B", "y"), + ] + + def test_rewrite_column_refs_still_rewrites_a_later_reference(self): + out = rewrite_column_refs( + "[Managers::Manager's Bonus] + [B::y]", lambda t, c: f"{t}.{c}" + ) + assert out == "Managers.Manager's Bonus + B.y" + + def test_split_call_still_recognises_a_valid_single_call(self): + assert split_call("sum ( [Managers::Manager's Bonus] )") == ( + "sum", + ["[Managers::Manager's Bonus]"], + ) + + def test_apostrophe_name_nested_two_calls_deep(self): + # The bracket stack must un-suppress correctly on each `]` no matter + # how many enclosing `(` it is nested inside. + outer = split_call("sum ( count ( [Managers::Manager's Bonus] ) )") + assert outer is not None + name, args = outer + assert name == "sum" + assert args == ["count ( [Managers::Manager's Bonus] )"] + inner = split_call(args[0]) + assert inner == ("count", ["[Managers::Manager's Bonus]"]) + + def test_column_name_containing_a_double_quote(self): + assert find_column_refs('[A::Say "Hi"] + [B::y]') == [ + ("A", 'Say "Hi"'), + ("B", "y"), + ] + + +class TestScanBracketStackTypeAwarePop: + """`_scan`'s bracket-type stack must pop only when a closer matches the type of its top + entry, never unconditionally — an unconditional pop lets a mismatched or stray closer + desynchronise the stack, which can then incorrectly toggle quote suppression for + whatever follows. `_scan` is a shallow tokenizer over malformed input here, not a + validator, so these pin the actual observed output (checked by running the scanner + before writing the assertion, not the output one might expect) rather than any claim + that the malformed input is handled "correctly" in some absolute sense. + """ + + def test_a_mismatched_closer_does_not_pop_the_bracket_stack(self): + # `}` does not match the `[` on top of the stack, so the stack keeps + # treating everything after it as still inside the never-closed `[` + # body — quote-toggling for the trailing `'z'` literal stays + # suppressed rather than (incorrectly) starting a real quote. + by_index = {i: in_quote for i, _ch, _d, in_quote in _scan("[A::x} 'z'")} + assert by_index[7] is False # opening quote of 'z' + assert by_index[9] is False # closing quote of 'z' + + def test_a_stray_closer_with_nothing_open_does_not_crash_or_suppress_quoting(self): + # After the properly closed `[A::x]`, an extra `)` has an empty + # stack to pop from — no matching opener anywhere. It must not + # raise, and with nothing open afterwards the trailing 'z' literal + # is read as a real quoted string, not suppressed. + by_index = {i: in_quote for i, _ch, _d, in_quote in _scan("[A::x] ) 'z'")} + assert by_index[9] is True # opening quote of 'z' + assert by_index[11] is True # closing quote of 'z' diff --git a/converters/thoughtspot/tests/test_identifiers.py b/converters/thoughtspot/tests/test_identifiers.py new file mode 100644 index 00000000..92472f27 --- /dev/null +++ b/converters/thoughtspot/tests/test_identifiers.py @@ -0,0 +1,146 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + +from ossie_thoughtspot import identifiers + + +@pytest.mark.parametrize("display,expected", [ + ("Order Date", "order_date"), + ("Order-Date", "order_date"), + ("Total Sales (AUD)", "total_sales_aud"), + (" Leading and trailing ", "leading_and_trailing"), + ("Multiple spaces", "multiple_spaces"), + ("Already_snake", "already_snake"), + ("2024 Revenue", "n_2024_revenue"), +]) +def test_normalise(display, expected): + assert identifiers.normalise(display) == expected + + +def test_normalise_rejects_a_name_that_normalises_to_nothing(): + with pytest.raises(ValueError, match="normalises to an empty identifier"): + identifiers.normalise("!!!") + + +@pytest.mark.parametrize("display,expected", [ + # Diacritics are folded via NFKD decomposition (not an earlier + # ASCII-only drop) — see the module docstring's "Known limitation" note. + # These pin the *current* behaviour so a future change can't silently + # regress it; they do not bless the remaining non-Latin-script limitation + # as correct. + ("Café", "cafe"), + ("Ürün", "urun"), + ("Zürich", "zurich"), + ("İstanbul", "istanbul"), + ("naïve", "naive"), +]) +def test_normalise_folds_diacritics_known_limitation(display, expected): + assert identifiers.normalise(display) == expected + + +def test_normalise_on_a_cjk_only_name_is_non_latin_script_known_limitation(): + # NFKD decomposition has no ASCII form for non-Latin scripts, so a + # CJK-only name still raises — the narrower residual of the limitation. + with pytest.raises(ValueError, match="normalises to an empty identifier"): + identifiers.normalise("北京市") + + +def test_allocator_resolves_a_collision_with_a_numeric_suffix(): + # Two distinct display names folding onto one identifier. + alloc = identifiers.Allocator() + assert alloc.allocate("Order Date") == "order_date" + assert alloc.allocate("Order-Date") == "order_date_2" + assert alloc.allocate("Order.Date") == "order_date_3" + + +def test_allocator_folds_case_when_detecting_collisions(): + # Ossie resolves regular identifiers case-insensitively, so a case-only + # difference is ambiguous even though validate.py would accept it. + alloc = identifiers.Allocator() + assert alloc.allocate("Region") == "region" + assert alloc.allocate("REGION") == "region_2" + + +def test_split_and_format_column_refs_round_trip(): + assert identifiers.split_column_ref("[ORDERS::Order Date]") == ("ORDERS", "Order Date") + assert identifiers.format_column_ref("ORDERS", "Order Date") == "[ORDERS::Order Date]" + + +def test_split_column_ref_rejects_a_malformed_reference(): + with pytest.raises(ValueError, match="not a ThoughtSpot column reference"): + identifiers.split_column_ref("ORDERS::Order Date") + + +def test_split_column_ref_rejects_an_ambiguous_reference(): + # More than one '::' must raise rather than silently taking the + # first delimiter and mis-splitting table/column. + with pytest.raises(ValueError, match="ambiguous"): + identifiers.split_column_ref("[A::B::C]") + + +def test_split_column_ref_rejects_a_reference_formatted_from_a_delimiter_containing_name(): + # A table name that itself contains '::' formats + # into a reference that must fail loudly on split, not silently mis-split + # the table/column boundary. + ref = identifiers.format_column_ref("A::B", "C") + assert ref == "[A::B::C]" + with pytest.raises(ValueError, match="ambiguous"): + identifiers.split_column_ref(ref) + + +def test_split_column_ref_rejects_a_table_with_a_trailing_colon(): + # str.count("::") is non-overlapping, so a run of three consecutive + # colons ("ORDERS" + trailing ":" + the "::" delimiter) only counts as + # one match and previously slipped through, silently mis-splitting to + # ("ORDERS", ":Col") instead of raising. + ref = identifiers.format_column_ref("ORDERS:", "Col") + assert ref == "[ORDERS:::Col]" + with pytest.raises(ValueError, match="ambiguous"): + identifiers.split_column_ref(ref) + + +def test_split_column_ref_rejects_a_column_with_a_leading_colon(): + # The same three-colon-run string is equally producible from a column + # that itself starts with ':' — genuinely ambiguous either way. + ref = identifiers.format_column_ref("ORDERS", ":Col") + assert ref == "[ORDERS:::Col]" + with pytest.raises(ValueError, match="ambiguous"): + identifiers.split_column_ref(ref) + + +def test_split_column_ref_accepts_a_table_name_with_a_single_colon(): + # A single ':' in the table position is not the same as the genuinely + # ambiguous '::'/leading-colon shapes above — the table group matches + # lazily up to the first '::', it does not reject colons outright. + assert identifiers.split_column_ref("[A:B::x]") == ("A:B", "x") + + +def test_split_column_ref_accepts_a_column_name_with_a_single_colon(): + assert identifiers.split_column_ref("[A::x:y]") == ("A", "x:y") + + +def test_split_and_format_round_trip_a_table_name_containing_a_colon(): + # Regression: format_column_ref("A:B", "x") -> "[A:B::x]", which + # split_column_ref used to refuse (the table group excluded colons + # outright, a stricter grammar than the documented ambiguity rule). Not + # ambiguous — there is exactly one '::' — so it must round-trip. + table, column = "A:B", "x" + ref = identifiers.format_column_ref(table, column) + assert ref == "[A:B::x]" + assert identifiers.split_column_ref(ref) == (table, column) diff --git a/converters/thoughtspot/tests/test_issues.py b/converters/thoughtspot/tests/test_issues.py new file mode 100644 index 00000000..03cf9438 --- /dev/null +++ b/converters/thoughtspot/tests/test_issues.py @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +import pytest + +from ossie_thoughtspot.errors import ConversionError +from ossie_thoughtspot.issues import ConverterIssue, IssueLog, Severity + + +def test_issue_requires_an_object_reference(): + # An issue a reader cannot trace to an object is noise (#325). + with pytest.raises(TypeError): + ConverterIssue(code="TS001", severity=Severity.WARNING, message="something") + + +def test_as_dicts_is_json_serialisable_and_stable(): + log = IssueLog() + log.add( + code="TS_RLS_DROPPED", + severity=Severity.ERROR, + message="Row-level security rules dropped from 2 tables: ORDERS, CUSTOMERS", + object_ref="model:Sales", + remedy="Re-apply the rules in the target instance before use.", + ) + assert log.as_dicts() == [ + { + "code": "TS_RLS_DROPPED", + "severity": "ERROR", + "message": "Row-level security rules dropped from 2 tables: ORDERS, CUSTOMERS", + "object_ref": "model:Sales", + "remedy": "Re-apply the rules in the target instance before use.", + } + ] + assert json.loads(json.dumps(log.as_dicts())) == log.as_dicts() + + +def test_has_errors_distinguishes_severity(): + log = IssueLog() + log.add(code="TS_X", severity=Severity.WARNING, message="m", object_ref="o") + assert log.has_errors() is False + log.add(code="TS_Y", severity=Severity.ERROR, message="m", object_ref="o") + assert log.has_errors() is True + + +def test_count_by_severity_supports_summarising_instead_of_printing(): + # #325 treats a warning storm as a defect; a caller must be able to summarise. + log = IssueLog() + for i in range(30): + log.add(code="TS_W", severity=Severity.WARNING, message=f"m{i}", object_ref=f"col{i}") + log.add(code="TS_E", severity=Severity.ERROR, message="m", object_ref="o") + assert log.count_by_severity() == {"ERROR": 1, "WARNING": 30} + + +def test_conversion_error_is_distinct_from_an_issue(): + # A malformed stash is a hard error, not a loggable issue. + assert issubclass(ConversionError, Exception) diff --git a/converters/thoughtspot/tests/test_keys.py b/converters/thoughtspot/tests/test_keys.py new file mode 100644 index 00000000..f8c439e0 --- /dev/null +++ b/converters/thoughtspot/tests/test_keys.py @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from ossie_thoughtspot.issues import IssueLog, Severity +from ossie_thoughtspot.keys import Relationship, derive_keys + + +def rel(name, to_columns, cardinality="MANY_TO_ONE", residual=False): + return Relationship( + name=name, + to_dataset="customers", + to_columns=to_columns, + cardinality=cardinality, + has_residual_predicates=residual, + ) + + +def test_single_qualifying_relationship_yields_a_primary_key(): + log = IssueLog() + pk, uniques = derive_keys("customers", [rel("r1", ["customer_id"])], log) + assert pk == ["customer_id"] + assert uniques == [["customer_id"]] + assert log.as_dicts() == [] + + +def test_disagreeing_qualifying_relationships_yield_unique_keys_and_no_primary_key(): + # Choosing one would be a guess; both are real join targets. + log = IssueLog() + pk, uniques = derive_keys( + "customers", [rel("by_id", ["customer_id"]), rel("by_email", ["email"])], log + ) + assert pk is None + assert sorted(uniques) == [["customer_id"], ["email"]] + + +def test_residual_predicate_relationship_is_not_key_evidence(): + # The equality columns alone are not unique — the narrowing makes it to-one. + log = IssueLog() + pk, uniques = derive_keys("customers", [rel("asof", ["ccy"], residual=True)], log) + assert pk is None + assert uniques == [] + + +def test_many_to_many_is_not_key_evidence(): + log = IssueLog() + pk, uniques = derive_keys("customers", [rel("bridge", ["c_id"], cardinality="MANY_TO_MANY")], log) + assert pk is None + assert uniques == [] + + +def test_a_disqualified_sibling_raises_an_issue_naming_it(): + # "ccy" does not cover the derived key ("customer_id"), so + # upstream's to_columns coverage check (validate.py:159-165) genuinely + # will warn here — the claim is correct and must be present. + log = IssueLog() + pk, uniques = derive_keys( + "customers", [rel("by_id", ["customer_id"]), rel("asof", ["ccy"], residual=True)], log + ) + assert pk == ["customer_id"] + issues = log.as_dicts() + assert len(issues) == 1 + assert issues[0]["severity"] == Severity.WARNING.value + assert "asof" in issues[0]["message"] + assert "coverage warning" in issues[0]["message"] + + +def test_residual_join_whose_columns_cover_the_key_has_no_upstream_warning_claim(): + # The canonical SCD-2 shape — a residual (as-of) join whose + # to_columns exactly covers the derived key. Upstream's coverage check + # (validate.py:159-165) passes clean here, so the message must not + # predict a warning that will not fire. + log = IssueLog() + pk, uniques = derive_keys( + "customers", + [rel("by_id", ["customer_id"]), rel("scd2_asof", ["customer_id"], residual=True)], + log, + ) + assert pk == ["customer_id"] + issues = log.as_dicts() + assert len(issues) == 1 + assert issues[0]["severity"] == Severity.WARNING.value + assert "scd2_asof" in issues[0]["message"] + assert "coverage warning" not in issues[0]["message"] + + +def test_column_order_within_a_composite_key_is_preserved(): + log = IssueLog() + pk, _ = derive_keys("customers", [rel("r", ["region", "customer_id"])], log) + assert pk == ["region", "customer_id"] + + +def test_empty_to_columns_yields_no_key_and_raises_an_error(): + # An empty to_columns is a hard schema failure (minItems: 1) — such a + # relationship cannot be emitted at all, so there is no upstream check + # left to run and no coverage warning to predict. This is ERROR, not + # WARNING, and the remedy must not claim it is "Expected". + log = IssueLog() + pk, uniques = derive_keys("customers", [rel("blank", [])], log) + assert pk is None + assert uniques == [] + issues = log.as_dicts() + assert len(issues) == 1 + assert "blank" in issues[0]["message"] + assert issues[0]["severity"] == Severity.ERROR.value + assert "coverage warning" not in issues[0]["message"] + assert not issues[0]["remedy"].lower().startswith("expected") + + +def test_agreeing_qualifying_relationships_collapse_to_one_unique_key(): + # A dimension joined from several fact tables on the same foreign key is + # a common shape — it must not be mistaken for disagreement. + log = IssueLog() + pk, uniques = derive_keys( + "customers", [rel("from_orders", ["customer_id"]), rel("from_invoices", ["customer_id"])], log + ) + assert pk == ["customer_id"] + assert uniques == [["customer_id"]] + assert log.as_dicts() == [] diff --git a/converters/thoughtspot/tests/test_ossie_to_thoughtspot.py b/converters/thoughtspot/tests/test_ossie_to_thoughtspot.py new file mode 100644 index 00000000..1d5fcc76 --- /dev/null +++ b/converters/thoughtspot/tests/test_ossie_to_thoughtspot.py @@ -0,0 +1,793 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the public `convert` entry point (Ossie -> TML), inline-join +placement, and the stash-restoration witness. + +Three things are new here relative to the other `ossie_to_thoughtspot` +test modules: `convert()` itself (build_table/build_model already have +their own dedicated files), the two places this converter now applies its +stash-if-present-**and-still-current**-else-derive rule rather than +plain stash-if-present, and a round trip that drives the two public entry +points back to back (`tml_to_ossie.convert` then `ossie_to_thoughtspot. +convert`) rather than a hand-built Ossie fixture. +""" +import json + +import pytest + +from ossie_thoughtspot.constants import ( + FIELD_STASH_DATA_TYPE, + FIELD_STASH_DATA_TYPE_WITNESS, + RELATIONSHIP_STASH_CARDINALITY, + RELATIONSHIP_STASH_ENDPOINTS_SWAPPED, + RELATIONSHIP_STASH_ENDPOINTS_SWAPPED_WITNESS, + RELATIONSHIP_STASH_JOIN_SHAPE, + RELATIONSHIP_STASH_ON_EXPRESSION, + RELATIONSHIP_STASH_ON_EXPRESSION_WITNESS, + RELATIONSHIP_STASH_TYPE, +) +from ossie_thoughtspot.errors import ConversionError +from ossie_thoughtspot.issues import IssueLog +from ossie_thoughtspot.ossie_to_thoughtspot import TmlConversion, build_model, build_table, convert +from ossie_thoughtspot.tml import ( + DocumentSet, + TmlDocument, + dump_document, + dump_document_set, + load_document, + load_document_set, +) +from ossie_thoughtspot.tml_to_ossie import convert as tml_to_ossie_convert + +# --------------------------------------------------------------------------- +# Fixture builders -- the same conventions test_ossie_to_thoughtspot_model.py +# and test_ossie_to_thoughtspot_tables.py use, kept local rather than shared +# so each test module's fixtures stay self-contained. +# --------------------------------------------------------------------------- + + +def _stash_ext(**payload): + return [{"vendor_name": "THOUGHTSPOT", "data": json.dumps({"_v": 1, **payload})}] + + +def _dialects(*pairs): + return [{"dialect": d, "expression": e} for d, e in pairs] + + +def _field(name, dialects, *, label=None, datatype=None, description=None, field_stash=None): + field: dict = {"name": name} + if label is not None: + field["label"] = label + field["expression"] = {"dialects": dialects} + if datatype is not None: + field["datatype"] = datatype + if description is not None: + field["description"] = description + if field_stash is not None: + field["custom_extensions"] = _stash_ext(**field_stash) + return field + + +def _round_tripped(name, table, column, **kwargs): + """A field whose expression is the verbatim THOUGHTSPOT bracket a prior + TML -> Ossie trip would have produced -- the shape build_table/build_model + treat as authoritative over any ANSI_SQL sibling.""" + return _field(name, _dialects(("THOUGHTSPOT", f"[{table}::{column}]")), **kwargs) + + +def _hand_authored_physical(name, identifier=None, **kwargs): + """A field whose expression is a single bare SQL identifier and no + THOUGHTSPOT dialect entry at all -- the shape a hand-authored Ossie + document (never round-tripped through TML) uses for a physical column.""" + return _field(name, _dialects(("ANSI_SQL", identifier or name)), **kwargs) + + +def _metric(name, dialects, *, description=None, metric_stash=None): + metric: dict = {"name": name, "expression": {"dialects": dialects}} + if description is not None: + metric["description"] = description + if metric_stash is not None: + metric["custom_extensions"] = _stash_ext(**metric_stash) + return metric + + +def _dataset(name, source, fields=None, *, dataset_stash=None): + dataset: dict = {"name": name, "source": source} + if fields is not None: + dataset["fields"] = fields + if dataset_stash is not None: + dataset["custom_extensions"] = _stash_ext(**dataset_stash) + return dataset + + +def _semantic_model(name="test_model", datasets=None, metrics=None, relationships=None, model_stash=None): + model: dict = {"name": name, "datasets": datasets or []} + if metrics is not None: + model["metrics"] = metrics + if relationships is not None: + model["relationships"] = relationships + if model_stash is not None: + model["custom_extensions"] = _stash_ext(**model_stash) + return model + + +def _relationship(name, from_, to, from_columns, to_columns, *, rel_stash=None): + relationship: dict = { + "name": name, "from": from_, "to": to, + "from_columns": from_columns, "to_columns": to_columns, + } + if rel_stash is not None: + relationship["custom_extensions"] = _stash_ext(**rel_stash) + return relationship + + +def _ossie_document(*semantic_models): + return {"version": "0.2.0.dev0", "semantic_model": list(semantic_models)} + + +def _table_doc(name, columns, connection="My Snowflake"): + return TmlDocument( + kind="table", + body={ + "name": name, "db": "SALES", "schema": "PUBLIC", "db_table": name, + "connection": {"name": connection}, "columns": columns, + }, + guid=None, + ) + + +def _sql_view_doc(name, sql_query, columns, connection="My Snowflake"): + return TmlDocument( + kind="sql_view", + body={ + "name": name, "sql_query": sql_query, + "connection": {"name": connection}, "sql_view_columns": columns, + }, + guid=None, + ) + + +def _column(name, db_column_name=None, data_type="VARCHAR"): + return {"name": name, "db_column_name": db_column_name or name, + "db_column_properties": {"data_type": data_type}} + + +def _sql_view_column(name, sql_output_column=None, data_type="VARCHAR"): + return {"name": name, "sql_output_column": sql_output_column or name, + "db_column_properties": {"data_type": data_type}} + + +def _model_tml(name, model_tables, columns, formulas=None): + body: dict = {"name": name, "model_tables": model_tables, "columns": columns} + if formulas is not None: + body["formulas"] = formulas + return TmlDocument(kind="model", body=body, guid=None) + + +def _find_key(value, key): + """Whether `key` appears anywhere in `value`, at any depth -- the "no + guid anywhere" rule needs to look past the document root, since a nested + guid is exactly as import-breaking as a root one (tml.py strips guids + unconditionally at dump time, but build_model/build_table must also + never *emit* one in the first place).""" + if isinstance(value, dict): + return key in value or any(_find_key(v, key) for v in value.values()) + if isinstance(value, (list, tuple)): + return any(_find_key(v, key) for v in value) + return False + + +# --------------------------------------------------------------------------- +# convert(): the public entry point itself. +# --------------------------------------------------------------------------- + + +class TestConvertEntryPoint: + def test_convert_returns_a_document_set_with_a_model_and_its_tables(self): + orders = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _round_tripped("amount", "orders", "Amount", label="Amount"), + ]) + model = _semantic_model(datasets=[orders]) + result = convert(_ossie_document(model)) + + assert isinstance(result, TmlConversion) + assert isinstance(result.documents, DocumentSet) + assert result.documents.model.kind == "model" + assert [t.body["name"] for t in result.documents.tables] == ["orders"] + assert isinstance(result.issues, IssueLog) + + def test_no_semantic_model_at_all_is_a_hard_failure(self): + with pytest.raises(ConversionError): + convert({"version": "0.2.0.dev0", "semantic_model": []}) + + def test_missing_semantic_model_key_is_a_hard_failure(self): + with pytest.raises(ConversionError): + convert({"version": "0.2.0.dev0"}) + + def test_more_than_one_semantic_model_is_a_hard_failure_naming_both(self): + first = _semantic_model(name="first") + second = _semantic_model(name="second") + with pytest.raises(ConversionError, match="first"): + convert(_ossie_document(first, second)) + + def test_no_guid_appears_anywhere_in_the_emitted_document_set(self): + # The no-guid-anywhere rule, proven at the deepest fixture this file builds: a join, a + # formula cross-reference, a metric and a stashed foreign extension + # all present at once. + orders_ds = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _round_tripped("amount", "orders", "Amount", label="Amount"), + _round_tripped("cost", "orders", "Cost", label="Cost"), + ]) + customers_ds = _dataset("customers", "SALES.PUBLIC.CUSTOMERS", fields=[ + _round_tripped("id", "customers", "Id", label="Id"), + ]) + relationship = _relationship("orders_to_customers", "orders", "customers", ["Amount"], ["Id"]) + metric = _metric("total", _dialects(("THOUGHTSPOT", "sum ( [orders::Amount] )"))) + model = _semantic_model( + datasets=[orders_ds, customers_ds], metrics=[metric], relationships=[relationship], + ) + result = convert(_ossie_document(model)) + + assert not _find_key(result.documents.model.body, "guid") + for table in result.documents.tables: + assert not _find_key(table.body, "guid") + + def test_a_hand_authored_document_with_no_stash_at_all_converts(self): + # A genuinely hand-authored Ossie file: no custom_extensions + # anywhere, physical fields as bare identifiers, no THOUGHTSPOT + # dialect entries. This must still produce an importable document + # set -- the "else-derive" half of the witness rule: every stashed key needs a + # derivation or a documented default, since a hand-authored + # document has no stash to fall back on at all. + orders = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _hand_authored_physical("order_date", datatype="Date"), + _hand_authored_physical("amount", datatype="Decimal"), + ]) + customers = _dataset("customers", "SALES.PUBLIC.CUSTOMERS", fields=[ + _hand_authored_physical("id", datatype="Integer"), + ]) + relationship = _relationship("orders_to_customers", "orders", "customers", ["amount"], ["id"]) + model = _semantic_model( + name="hand_authored", datasets=[orders, customers], relationships=[relationship], + ) + result = convert(_ossie_document(model)) + + assert not result.issues.has_errors() + texts = dump_document_set(result.documents) + reloaded = load_document_set(texts) + assert reloaded.model.kind == "model" + assert {t.kind for t in reloaded.tables} == {"table"} + # A connection name was never supplied -- build_table names the gap + # rather than inventing one, per its own documented contract. + assert any(i["code"] == "TS-DATASET-CONNECTION-MISSING" for i in result.issues.as_dicts()) + + +# --------------------------------------------------------------------------- +# Inline join placement and type normalisation, through convert()'s own +# document -- build_model's join mechanics have their own dedicated tests in +# test_ossie_to_thoughtspot_model.py; these confirm the same invariants hold +# end to end through the public entry point. +# --------------------------------------------------------------------------- + + +class TestJoinPlacementThroughConvert: + def _model_with_join(self, **rel_kwargs): + orders = _dataset("orders", "SALES.PUBLIC.ORDERS") + customers = _dataset("customers", "SALES.PUBLIC.CUSTOMERS") + relationship = _relationship( + "orders_to_customers", "orders", "customers", ["Customer Id"], ["Id"], **rel_kwargs + ) + return _semantic_model(datasets=[orders, customers], relationships=[relationship]) + + def test_the_join_lives_on_the_source_entry_never_at_model_top_level(self): + result = convert(_ossie_document(self._model_with_join())) + body = result.documents.model.body + assert "joins" not in body + [orders_entry] = [t for t in body["model_tables"] if t["name"] == "orders"] + assert len(orders_entry["joins"]) == 1 + assert orders_entry["joins"][0]["with"] == "customers" + [customers_entry] = [t for t in body["model_tables"] if t["name"] == "customers"] + assert "joins" not in customers_entry + + def test_the_on_key_survives_dump_and_reload_as_a_plain_string(self): + # 'on' is a YAML 1.1 reserved word -- tml.py's codec has to quote it + # or a reload coerces the key itself, not just a value. + result = convert(_ossie_document(self._model_with_join())) + text = dump_document(result.documents.model) + assert "'on':" in text + reloaded = load_document(text) + [orders_entry] = [t for t in reloaded.body["model_tables"] if t["name"] == "orders"] + assert orders_entry["joins"][0]["on"] == "[orders::Customer Id] = [customers::Id]" + + @pytest.mark.parametrize("spelling", ["FULL_OUTER", "FULL OUTER", "full_outer"]) + def test_full_outer_becomes_outer_on_a_relationship_join(self, spelling): + model = self._model_with_join( + rel_stash={RELATIONSHIP_STASH_TYPE: spelling, RELATIONSHIP_STASH_CARDINALITY: "MANY_TO_ONE"}, + ) + result = convert(_ossie_document(model)) + [orders_entry] = [t for t in result.documents.model.body["model_tables"] if t["name"] == "orders"] + assert orders_entry["joins"][0]["type"] == "OUTER" + + def test_full_outer_becomes_outer_on_an_unrepresentable_join_too(self): + # The rename applies "in every context TML accepts a join type at + # all" -- unrepresentable_joins[] is the other one this module emits. + orders = _dataset("orders", "SALES.PUBLIC.ORDERS") + fx_rates = _dataset("fx_rates", "SALES.PUBLIC.FX_RATES") + model = _semantic_model( + datasets=[orders, fx_rates], + model_stash={ + "unrepresentable_joins": [{ + "from": "orders", "to": "fx_rates", + RELATIONSHIP_STASH_ON_EXPRESSION: "[orders::Order Date] >= [fx_rates::Effective Date]", + RELATIONSHIP_STASH_TYPE: "FULL_OUTER", + RELATIONSHIP_STASH_CARDINALITY: "MANY_TO_ONE", + }], + }, + ) + result = convert(_ossie_document(model)) + [orders_entry] = [t for t in result.documents.model.body["model_tables"] if t["name"] == "orders"] + assert orders_entry["joins"][0]["type"] == "OUTER" + + def test_missing_type_and_cardinality_default_rather_than_being_omitted(self): + # TML requires both keys on every join -- a document with + # neither stashed must still emit both, never leave one out. + result = convert(_ossie_document(self._model_with_join())) + [orders_entry] = [t for t in result.documents.model.body["model_tables"] if t["name"] == "orders"] + [join] = orders_entry["joins"] + assert join["type"] == "INNER" + assert join["cardinality"] == "MANY_TO_ONE" + assert set(join) == {"with", "on", "type", "cardinality"} + + +# --------------------------------------------------------------------------- +# Stash-if-present-and-still-current-else-derive, for a relationship's +# on_expression. The obvious reading ("use the stash if it is there") is +# wrong: it silently discards a retargeted relationship's edit. +# --------------------------------------------------------------------------- + + +class TestOnExpressionWitness: + _NARROWED_CONDITION = ( + "[orders::Currency] = [fx_rates::Currency] and " + "[orders::Order Date] >= [fx_rates::Effective Date]" + ) + + def _tables(self): + orders = _table_doc("orders", [ + _column("Order Date", "ORDER_DATE", "DATE"), + _column("Currency", "CURRENCY", "VARCHAR"), + ]) + fx_rates = _table_doc("fx_rates", [ + _column("Effective Date", "EFFECTIVE_DATE", "DATE"), + _column("Currency", "CURRENCY", "VARCHAR"), + ]) + return orders, fx_rates + + def _model(self, relationship): + return _semantic_model( + datasets=[ + _dataset("orders", "SALES.PUBLIC.ORDERS"), + _dataset("fx_rates", "SALES.PUBLIC.FX_RATES"), + ], + relationships=[relationship], + ) + + def test_a_witness_that_still_matches_restores_the_verbatim_condition(self): + relationship = _relationship( + "orders_to_fx", "orders", "fx_rates", ["Currency"], ["Currency"], + rel_stash={ + RELATIONSHIP_STASH_ON_EXPRESSION: self._NARROWED_CONDITION, + RELATIONSHIP_STASH_ON_EXPRESSION_WITNESS: [["Currency"], ["Currency"]], + RELATIONSHIP_STASH_TYPE: "INNER", + RELATIONSHIP_STASH_CARDINALITY: "MANY_TO_ONE", + }, + ) + orders, fx_rates = self._tables() + log = IssueLog() + doc = build_model(self._model(relationship), [orders, fx_rates], log) + + [orders_entry] = [t for t in doc.body["model_tables"] if t["name"] == "orders"] + assert orders_entry["joins"][0]["on"] == self._NARROWED_CONDITION + assert not [i for i in log.as_dicts() if i["code"] == "TS-JOIN-ON-EXPRESSION-STALE"] + + def test_a_witness_that_no_longer_matches_is_dropped_and_re_derived(self): + # from_columns/to_columns were retargeted after the stash was + # written -- the witness still names the OLD pairing (Currency). + relationship = _relationship( + "orders_to_fx", "orders", "fx_rates", ["Order Date"], ["Effective Date"], + rel_stash={ + RELATIONSHIP_STASH_ON_EXPRESSION: self._NARROWED_CONDITION, + RELATIONSHIP_STASH_ON_EXPRESSION_WITNESS: [["Currency"], ["Currency"]], + RELATIONSHIP_STASH_TYPE: "INNER", + RELATIONSHIP_STASH_CARDINALITY: "MANY_TO_ONE", + }, + ) + orders, fx_rates = self._tables() + log = IssueLog() + doc = build_model(self._model(relationship), [orders, fx_rates], log) + + [orders_entry] = [t for t in doc.body["model_tables"] if t["name"] == "orders"] + # Re-derived from the CURRENT from_columns/to_columns -- the stale + # verbatim text (and the residual narrowing it carried) is dropped, + # not silently kept. + assert orders_entry["joins"][0]["on"] == "[orders::Order Date] = [fx_rates::Effective Date]" + assert any(i["code"] == "TS-JOIN-ON-EXPRESSION-STALE" for i in log.as_dicts()) + + def test_no_stash_at_all_converts_using_the_plain_equality_condition(self): + # A hand-authored relationship with no custom_extensions at all must + # still convert, with no staleness issue raised -- there is nothing + # stale about a value that was never there in the first place. + relationship = _relationship("orders_to_fx", "orders", "fx_rates", ["Currency"], ["Currency"]) + orders, fx_rates = self._tables() + log = IssueLog() + doc = build_model(self._model(relationship), [orders, fx_rates], log) + + [orders_entry] = [t for t in doc.body["model_tables"] if t["name"] == "orders"] + assert orders_entry["joins"][0]["on"] == "[orders::Currency] = [fx_rates::Currency]" + assert not [i for i in log.as_dicts() if i["code"] == "TS-JOIN-ON-EXPRESSION-STALE"] + + +# --------------------------------------------------------------------------- +# A third witnessed construct: RELATIONSHIP_STASH_ENDPOINTS_SWAPPED. TML -> +# Ossie swaps a ONE_TO_MANY join's from/to/from_columns/to_columns so the +# emitted relationship satisfies core-spec/spec.yaml's many-side/one-side +# convention. Undoing that swap on the way back is itself governed by the +# same stash-if-present-and-still-current-else-derive rule: only while +# nothing has retargeted the relationship since the swap was stashed. +# --------------------------------------------------------------------------- + + +class TestEndpointsSwapWitness: + def _tables(self): + cust = _table_doc("CUST", [_column("ID", "ID", "INT64")]) + orders = _table_doc("ORDERS", [_column("CID", "CID", "INT64")]) + return cust, orders + + def _model(self, relationship): + return _semantic_model( + datasets=[ + _dataset("CUST", "SALES.PUBLIC.CUST"), + _dataset("ORDERS", "SALES.PUBLIC.ORDERS"), + ], + relationships=[relationship], + ) + + def test_a_witness_that_still_matches_undoes_the_swap(self): + # The live (already-swapped) relationship: from=ORDERS (many side), + # to=CUST (one side). Undoing the swap recovers TML's own + # declaration -- the join nested under CUST, targeting ORDERS. + relationship = _relationship( + "ORDERS_to_CUST", "ORDERS", "CUST", ["CID"], ["ID"], + rel_stash={ + RELATIONSHIP_STASH_CARDINALITY: "ONE_TO_MANY", + RELATIONSHIP_STASH_ENDPOINTS_SWAPPED: True, + RELATIONSHIP_STASH_ENDPOINTS_SWAPPED_WITNESS: ["ORDERS", "CUST", ["CID"], ["ID"]], + RELATIONSHIP_STASH_TYPE: "INNER", + RELATIONSHIP_STASH_JOIN_SHAPE: "inline", + }, + ) + cust, orders = self._tables() + log = IssueLog() + doc = build_model(self._model(relationship), [cust, orders], log) + + [cust_entry] = [t for t in doc.body["model_tables"] if t["name"] == "CUST"] + [orders_entry] = [t for t in doc.body["model_tables"] if t["name"] == "ORDERS"] + assert "joins" not in orders_entry + assert cust_entry["joins"] == [{ + "with": "ORDERS", + "on": "[CUST::ID] = [ORDERS::CID]", + "type": "INNER", + "cardinality": "ONE_TO_MANY", + }] + assert not [i for i in log.as_dicts() if i["code"] == "TS-JOIN-ENDPOINTS-SWAP-STALE"] + + def test_a_witness_that_no_longer_matches_leaves_the_swap_undone_and_logs(self): + # to_columns was retargeted after the stash was written -- the + # witness still names the OLD pairing (["ID"]). The swap is left + # alone: the join is emitted straight from the live (still-swapped) + # shape, exactly as a hand-authored relationship with no stash at + # all would be. + relationship = _relationship( + "ORDERS_to_CUST", "ORDERS", "CUST", ["CID"], ["OTHER_ID"], + rel_stash={ + RELATIONSHIP_STASH_CARDINALITY: "ONE_TO_MANY", + RELATIONSHIP_STASH_ENDPOINTS_SWAPPED: True, + RELATIONSHIP_STASH_ENDPOINTS_SWAPPED_WITNESS: ["ORDERS", "CUST", ["CID"], ["ID"]], + RELATIONSHIP_STASH_TYPE: "INNER", + RELATIONSHIP_STASH_JOIN_SHAPE: "inline", + }, + ) + cust, orders = self._tables() + log = IssueLog() + doc = build_model(self._model(relationship), [cust, orders], log) + + [orders_entry] = [t for t in doc.body["model_tables"] if t["name"] == "ORDERS"] + assert orders_entry["joins"] == [{ + "with": "CUST", + "on": "[ORDERS::CID] = [CUST::OTHER_ID]", + "type": "INNER", + "cardinality": "ONE_TO_MANY", + }] + assert any(i["code"] == "TS-JOIN-ENDPOINTS-SWAP-STALE" for i in log.as_dicts()) + + def test_no_endpoints_swapped_stash_uses_the_live_shape_directly(self): + # A relationship whose cardinality is ONE_TO_MANY but carries no + # endpoints_swapped stash at all (hand-authored, never round-tripped + # through TML -> Ossie) is emitted straight from its live from/to -- + # there is nothing to undo, and nothing stale to report either. + relationship = _relationship( + "CUST_to_ORDERS", "CUST", "ORDERS", ["ID"], ["CID"], + rel_stash={RELATIONSHIP_STASH_CARDINALITY: "ONE_TO_MANY", RELATIONSHIP_STASH_TYPE: "INNER"}, + ) + cust, orders = self._tables() + log = IssueLog() + doc = build_model(self._model(relationship), [cust, orders], log) + + [cust_entry] = [t for t in doc.body["model_tables"] if t["name"] == "CUST"] + assert cust_entry["joins"] == [{ + "with": "ORDERS", + "on": "[CUST::ID] = [ORDERS::CID]", + "type": "INNER", + "cardinality": "ONE_TO_MANY", + }] + assert not [i for i in log.as_dicts() if i["code"] == "TS-JOIN-ENDPOINTS-SWAP-STALE"] + + +# --------------------------------------------------------------------------- +# The same witness rule again, on a second construct: FIELD_STASH_DATA_TYPE. Reading +# _field_datatype revealed the exact same stash-if-present pattern to +# warn against for on_expression, just on a different key: a field whose +# `datatype` is edited after the stash was written (Boolean -> String, +# say) would silently keep emitting the OLD warehouse spelling (BOOL) for +# a column that is no longer Boolean at all. Worth its own test because it +# proves the fix is systemic -- every stash that shadows a live, editable +# Ossie value needs a witness -- not a one-off patch scoped to relationships. +# --------------------------------------------------------------------------- + + +class TestFieldDataTypeWitness: + def _dataset_with(self, field): + return _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[field]) + + def test_a_witness_that_still_matches_restores_the_stashed_spelling(self): + field = _hand_authored_physical( + "is_active", datatype="Boolean", + field_stash={FIELD_STASH_DATA_TYPE: "BOOL", FIELD_STASH_DATA_TYPE_WITNESS: "Boolean"}, + ) + table = build_table(self._dataset_with(field), IssueLog()) + assert table.body["columns"][0]["db_column_properties"]["data_type"] == "BOOL" + + def test_a_witness_that_no_longer_matches_is_dropped_and_re_derived(self): + # The field's datatype was edited (Boolean -> String) since the + # spelling was stashed -- BOOL now names a warehouse type this + # field no longer has. + field = _hand_authored_physical( + "is_active", datatype="String", + field_stash={FIELD_STASH_DATA_TYPE: "BOOL", FIELD_STASH_DATA_TYPE_WITNESS: "Boolean"}, + ) + log = IssueLog() + table = build_table(self._dataset_with(field), log) + assert table.body["columns"][0]["db_column_properties"]["data_type"] == "VARCHAR" + assert any(i["code"] == "TS-FIELD-DATA-TYPE-STASH-STALE" for i in log.as_dicts()) + + +# --------------------------------------------------------------------------- +# A full round trip through both public entry points. +# --------------------------------------------------------------------------- + + +class TestFullRoundTripBothEntryPoints: + """Build a rich TML document set by hand, run it forward + (tml_to_ossie.convert), touch the intermediate Ossie document the way + another tool legitimately might (append a foreign vendor's + custom_extensions entry), run it back through this module's convert, + and diff the result against the original. + + Covers: a Table and a SQL View, physical and computed fields, two + metric shapes (`formula` and `column_aggregation`), an equality join + and a non-equality join (with a residual predicate -- the exact + on_expression-witness path TestOnExpressionWitness exercises directly, + here exercised through a real round trip instead of a synthetic + fixture), a pre-existing foreign vendor extension, and a YAML 1.1 + boolean column name ("On"). + """ + + _FX_JOIN_CONDITION = ( + "[ORDERS::Currency] = [FX_RATES::Currency] and " + "[ORDERS::Order Date] >= [FX_RATES::Effective Date]" + ) + + def _original(self): + orders = _table_doc("ORDERS", [ + _column("Order Date", "O_ORDER_DATE", "DATE"), + _column("Amount", "O_AMOUNT", "DOUBLE"), + _column("Cost", "O_COST", "DOUBLE"), + _column("Currency", "O_CURRENCY", "VARCHAR"), + _column("Customer Id", "O_CUSTOMER_ID", "INT64"), + _column("On", "O_ON_FLAG", "VARCHAR"), + ]) + customers = _table_doc("CUSTOMERS", [ + _column("Id", "ID", "INT64"), + _column("Status", "C_STATUS", "VARCHAR"), + ]) + fx_rates = _sql_view_doc( + "FX_RATES", "SELECT CURRENCY, EFFECTIVE_DATE FROM RAW.FX", + [ + _sql_view_column("Currency", "CURRENCY", "VARCHAR"), + _sql_view_column("Effective Date", "EFFECTIVE_DATE", "DATE"), + ], + ) + model = _model_tml( + "Sales Analytics", + model_tables=[ + {"name": "ORDERS", "joins": [ + {"with": "CUSTOMERS", "on": "[ORDERS::Customer Id] = [CUSTOMERS::Id]", + "type": "LEFT_OUTER", "cardinality": "MANY_TO_ONE"}, + {"with": "FX_RATES", "on": self._FX_JOIN_CONDITION, + "type": "INNER", "cardinality": "MANY_TO_ONE"}, + ]}, + {"name": "CUSTOMERS"}, + {"name": "FX_RATES"}, + ], + columns=[ + {"name": "Order Date", "column_id": "ORDERS::Order Date", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "Amount", "column_id": "ORDERS::Amount", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "Cost", "column_id": "ORDERS::Cost", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "Currency", "column_id": "ORDERS::Currency", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "Customer Id", "column_id": "ORDERS::Customer Id", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "On", "column_id": "ORDERS::On", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "Status", "column_id": "CUSTOMERS::Status", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "Net Amount", "formula_id": "formula_net_amount", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "total_revenue", "formula_id": "formula_total_revenue", + "properties": {"column_type": "MEASURE", "aggregation": "SUM"}}, + {"name": "customer_count", "column_id": "CUSTOMERS::Id", + "properties": {"column_type": "MEASURE", "aggregation": "COUNT_DISTINCT"}}, + ], + formulas=[ + {"id": "formula_net_amount", "name": "Net Amount", + "expr": "[ORDERS::Amount] - [ORDERS::Cost]"}, + {"id": "formula_total_revenue", "name": "total_revenue", + "expr": "sum ( [ORDERS::Amount] )"}, + ], + ) + return DocumentSet(model=model, tables=(orders, customers, fx_rates)) + + def _round_trip(self): + original = self._original() + forward = tml_to_ossie_convert(original) + assert not forward.issues.has_errors() + + ossie_document = forward.model + orders_dataset = next( + d for d in ossie_document["semantic_model"][0]["datasets"] if d["name"] == "ORDERS" + ) + # Simulate another tool having already touched the intermediate + # Ossie document -- the scenario write_stash's foreign-vendor + # preservation guards against, and the only place a + # "foreign vendor extension" can meaningfully appear in a + # TML -> Ossie -> TML round trip, since TML itself has no + # extension mechanism at all for one to originate from. + foreign_entry = {"vendor_name": "DATABRICKS", "data": json.dumps({"note": "unrelated"})} + orders_dataset.setdefault("custom_extensions", []).append(foreign_entry) + + result = convert(ossie_document) + assert not result.issues.has_errors() + return original, ossie_document, foreign_entry, result + + def test_the_foreign_vendor_entry_is_never_touched(self): + _original, ossie_document, foreign_entry, _result = self._round_trip() + orders_dataset = next( + d for d in ossie_document["semantic_model"][0]["datasets"] if d["name"] == "ORDERS" + ) + assert foreign_entry in orders_dataset["custom_extensions"] + + def test_both_joins_restore_their_exact_original_condition_type_and_cardinality(self): + original, _ossie_document, _foreign, result = self._round_trip() + rebuilt_orders = next( + t for t in result.documents.model.body["model_tables"] if t["name"] == "ORDERS" + ) + original_orders = next( + t for t in original.model.body["model_tables"] if t["name"] == "ORDERS" + ) + rebuilt_joins = {j["with"]: j for j in rebuilt_orders["joins"]} + original_joins = {j["with"]: j for j in original_orders["joins"]} + + assert set(rebuilt_joins) == set(original_joins) + for target, original_join in original_joins.items(): + rebuilt_join = rebuilt_joins[target] + assert rebuilt_join["on"] == original_join["on"] + assert rebuilt_join["type"] == original_join["type"] + assert rebuilt_join["cardinality"] == original_join["cardinality"] + + def test_the_non_equality_joins_residual_narrowing_survived_the_full_trip(self): + # The concrete proof that TestOnExpressionWitness's synthetic case + # is not synthetic-only: an unedited FX_RATES relationship comes + # back with its ">=" narrowing intact, not collapsed to the bare + # equality pair a stale or absent stash would produce. + _original, _ossie_document, _foreign, result = self._round_trip() + rebuilt_orders = next( + t for t in result.documents.model.body["model_tables"] if t["name"] == "ORDERS" + ) + [fx_join] = [j for j in rebuilt_orders["joins"] if j["with"] == "FX_RATES"] + assert fx_join["on"] == self._FX_JOIN_CONDITION + assert ">=" in fx_join["on"] + + def test_formula_backed_fields_and_metrics_round_trip_their_expr_byte_identical(self): + original, _ossie_document, _foreign, result = self._round_trip() + original_formulas = {f["id"]: f["expr"] for f in original.model.body["formulas"]} + rebuilt_formulas = {f["id"]: f["expr"] for f in result.documents.model.body["formulas"]} + assert rebuilt_formulas["formula_net_amount"] == original_formulas["formula_net_amount"] + assert rebuilt_formulas["formula_total_revenue"] == original_formulas["formula_total_revenue"] + + def test_the_column_aggregation_metric_becomes_a_formula_a_declared_non_lossy_difference(self): + # A metric is always emitted as a formula, never column_id + + # aggregation, on the way back -- Ossie's Metric schema has no + # column_id field at all. This is the one deliberate structural + # difference the round trip produces; asserted explicitly here so + # it reads as "expected", not as an unnoticed regression. + _original, _ossie_document, _foreign, result = self._round_trip() + columns = result.documents.model.body["columns"] + customer_count = next(c for c in columns if c["name"] == "customer_count") + assert "column_id" not in customer_count + assert "formula_id" in customer_count + assert customer_count["properties"]["column_type"] == "MEASURE" + + def test_table_and_sql_view_documents_round_trip_their_connection_and_source(self): + original, _ossie_document, _foreign, result = self._round_trip() + rebuilt_by_name = {t.body["name"]: t for t in result.documents.tables} + + original_orders = next(t for t in original.tables if t.body["name"] == "ORDERS") + rebuilt_orders = rebuilt_by_name["ORDERS"] + assert rebuilt_orders.kind == "table" + assert rebuilt_orders.body["connection"] == original_orders.body["connection"] + assert (rebuilt_orders.body["db"], rebuilt_orders.body["schema"], rebuilt_orders.body["db_table"]) == ( + original_orders.body["db"], original_orders.body["schema"], original_orders.body["db_table"], + ) + + original_fx = next(t for t in original.tables if t.body["name"] == "FX_RATES") + rebuilt_fx = rebuilt_by_name["FX_RATES"] + assert rebuilt_fx.kind == "sql_view" + assert rebuilt_fx.body["sql_query"] == original_fx.body["sql_query"] + rebuilt_fx_columns = {c["name"]: c["sql_output_column"] for c in rebuilt_fx.body["sql_view_columns"]} + original_fx_columns = {c["name"]: c["sql_output_column"] for c in original_fx.body["sql_view_columns"]} + assert rebuilt_fx_columns == original_fx_columns + + def test_the_yaml_1_1_boolean_token_column_name_survives_dump_and_reload(self): + _original, _ossie_document, _foreign, result = self._round_trip() + text = dump_document(result.documents.model) + reloaded = load_document(text) + on_column = next(c for c in reloaded.body["columns"] if c["column_id"] == "ORDERS::On") + assert on_column["name"] == "On" + + def test_the_full_document_set_reloads_and_carries_no_guid(self): + _original, _ossie_document, _foreign, result = self._round_trip() + texts = dump_document_set(result.documents) + reloaded = load_document_set(texts) + assert reloaded.model.kind == "model" + assert len(reloaded.tables) == 3 + assert not _find_key(result.documents.model.body, "guid") + for table in result.documents.tables: + assert not _find_key(table.body, "guid") diff --git a/converters/thoughtspot/tests/test_ossie_to_thoughtspot_model.py b/converters/thoughtspot/tests/test_ossie_to_thoughtspot_model.py new file mode 100644 index 00000000..307c27e9 --- /dev/null +++ b/converters/thoughtspot/tests/test_ossie_to_thoughtspot_model.py @@ -0,0 +1,1557 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for `build_model` and `to_thoughtspot_expression`: the Model TML +document. + +Fixtures build raw Ossie `semantic_model`/dataset/field/metric dicts directly +(the same convention test_ossie_to_thoughtspot_tables.py uses for datasets), +except for the round-trip suite, which goes through `tml_to_ossie.convert` +first -- the strongest check available, because a hand-written Ossie fixture +can be unknowingly wrong about what the forward direction actually produces. +""" +import json + +from ossie_thoughtspot import formula as formula_module +from ossie_thoughtspot import stash as stash_module +from ossie_thoughtspot.constants import ( + FIELD_STASH_COLUMN_PROPERTIES, + METRIC_SHAPE_COLUMN_AGGREGATION, + METRIC_SHAPE_FORMULA, + METRIC_SHAPE_SCALAR_FORMULA_PLUS_AGGREGATION, + METRIC_STASH_SHAPE, + MODEL_STASH_UNATTRIBUTED_FORMULAS, + MODEL_STASH_UNREPRESENTABLE_JOINS, + RELATIONSHIP_STASH_CARDINALITY, + RELATIONSHIP_STASH_ON_EXPRESSION, + RELATIONSHIP_STASH_TYPE, +) +from ossie_thoughtspot.issues import IssueLog +from ossie_thoughtspot.ossie_to_thoughtspot import build_model, build_table, to_thoughtspot_expression +from ossie_thoughtspot.tml import DocumentSet, TmlDocument, dump_document, load_document +from ossie_thoughtspot.tml_to_ossie import convert as tml_to_ossie_convert + + +# --------------------------------------------------------------------------- +# Fixture builders. +# --------------------------------------------------------------------------- + +def _stash_ext(**payload): + return [{"vendor_name": "THOUGHTSPOT", "data": json.dumps({"_v": 1, **payload})}] + + +def _dialects(*pairs): + return [{"dialect": d, "expression": e} for d, e in pairs] + + +def _field(name, dialects, *, label=None, datatype=None, description=None, + ai_context=None, field_stash=None): + field: dict = {"name": name} + if label is not None: + field["label"] = label + field["expression"] = {"dialects": dialects} + if datatype is not None: + field["datatype"] = datatype + if description is not None: + field["description"] = description + if ai_context is not None: + field["ai_context"] = ai_context + if field_stash is not None: + field["custom_extensions"] = _stash_ext(**field_stash) + return field + + +def _metric(name, dialects, *, datatype=None, description=None, ai_context=None, + metric_stash=None): + metric: dict = {"name": name, "expression": {"dialects": dialects}} + if datatype is not None: + metric["datatype"] = datatype + if description is not None: + metric["description"] = description + if ai_context is not None: + metric["ai_context"] = ai_context + if metric_stash is not None: + metric["custom_extensions"] = _stash_ext(**metric_stash) + return metric + + +def _dataset(name, source, fields=None, **kwargs): + dataset: dict = {"name": name, "source": source} + if fields is not None: + dataset["fields"] = fields + dataset.update(kwargs) + return dataset + + +def _semantic_model(name="test_model", datasets=None, metrics=None, relationships=None, + model_stash=None, **kwargs): + model: dict = {"name": name, "datasets": datasets or []} + if metrics is not None: + model["metrics"] = metrics + if relationships is not None: + model["relationships"] = relationships + if model_stash is not None: + model["custom_extensions"] = _stash_ext(**model_stash) + model.update(kwargs) + return model + + +def _relationship(name, from_, to, from_columns, to_columns, *, rel_stash=None): + relationship: dict = { + "name": name, "from": from_, "to": to, + "from_columns": from_columns, "to_columns": to_columns, + } + if rel_stash is not None: + relationship["custom_extensions"] = _stash_ext(**rel_stash) + return relationship + + +def _table_doc(name, columns, connection="My Snowflake"): + return TmlDocument( + kind="table", + body={ + "name": name, "db": "SALES", "schema": "PUBLIC", "db_table": name, + "connection": {"name": connection}, "columns": columns, + }, + guid=None, + ) + + +def _column(name, db_column_name=None, data_type="VARCHAR"): + return {"name": name, "db_column_name": db_column_name or name, + "db_column_properties": {"data_type": data_type}} + + +def _resolve_field(index): + """A `resolve_field` closure over a plain `{"dataset.field": (table, column)}` dict.""" + return index.get + + +def _all_columns_and_formulas(body): + return body.get("columns") or [], body.get("formulas") or [] + + +def _dangling_formula_references(formulas): + """Every `[formula_X]` reference, across every emitted formula's `expr`, + that does not match any emitted `formulas[]` id -- empty when every + cross-reference resolves. Deliberately checks the *property* (does every + reference land somewhere real) rather than any specific id spelling, so + it survives a change of normalisation scheme. Reuses the package's own + bracket scanner (`formula._bracketed_spans`) rather than a parallel + regex, so this check cannot itself disagree with what the production + code considers a bracket reference. + """ + ids = {entry["id"] for entry in formulas} + dangling = [] + for entry in formulas: + for _start, _end, body in formula_module._bracketed_spans(entry["expr"]): + if "::" in body or not body.startswith("formula_"): + continue + if body not in ids: + dangling.append((entry["name"], body)) + return dangling + + +# --------------------------------------------------------------------------- +# Every formula is one formulas[] entry plus one columns[] entry. +# --------------------------------------------------------------------------- + +class TestFormulaPairing: + def test_a_computed_field_gets_a_formulas_entry_and_a_referencing_column(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE"), + _column("Cost", "COST", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _field("net", _dialects(("THOUGHTSPOT", "[orders::Amount] - [orders::Cost]")), + label="Net"), + ]) + model = _semantic_model(datasets=[dataset]) + log = IssueLog() + + doc = build_model(model, [orders], log) + columns, formulas = _all_columns_and_formulas(doc.body) + + assert len(formulas) == 1 + assert len(columns) == 1 + assert columns[0]["formula_id"] == formulas[0]["id"] + assert formulas[0]["expr"] == "[orders::Amount] - [orders::Cost]" + assert not log.as_dicts() + + def test_a_metric_gets_a_formulas_entry_and_a_referencing_column(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _field("amount", _dialects(("THOUGHTSPOT", "[orders::Amount]")), label="Amount"), + ]) + metric = _metric("total_revenue", _dialects(("THOUGHTSPOT", "sum ( [orders::Amount] )"))) + model = _semantic_model(datasets=[dataset], metrics=[metric]) + + doc = build_model(model, [orders], IssueLog()) + columns, formulas = _all_columns_and_formulas(doc.body) + + metric_column = next(c for c in columns if c["name"] == "total_revenue") + metric_formula = next(f for f in formulas if f["id"] == metric_column["formula_id"]) + assert metric_formula["expr"] == "sum ( [orders::Amount] )" + assert metric_column["properties"]["column_type"] == "MEASURE" + + +class TestFormulasNeverCarryAggregation: + def test_no_formulas_entry_ever_has_an_aggregation_key(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE"), + _column("Cost", "COST", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + metric_a = _metric("total", _dialects(("THOUGHTSPOT", "sum ( [orders::Amount] )"))) + metric_b = _metric( + "avg_net", _dialects(("THOUGHTSPOT", "average ( [orders::Amount] - [orders::Cost] )")), + metric_stash={METRIC_STASH_SHAPE: METRIC_SHAPE_SCALAR_FORMULA_PLUS_AGGREGATION}, + ) + model = _semantic_model(datasets=[dataset], metrics=[metric_a, metric_b]) + + doc = build_model(model, [orders], IssueLog()) + _columns, formulas = _all_columns_and_formulas(doc.body) + + assert formulas # sanity: something was built + for entry in formulas: + assert "aggregation" not in entry + + +class TestFormulaCrossReferenceUsesIdForm: + def test_a_formula_referencing_another_by_id_round_trips_the_reference(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE"), + _column("Cost", "COST", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _field("net_amount", _dialects(("THOUGHTSPOT", "[orders::Amount] - [orders::Cost]")), + label="Net Amount"), + _field( + "margin_pct", + _dialects(("THOUGHTSPOT", "[formula_net_amount] / [orders::Amount]")), + label="Margin Pct", + ), + ]) + model = _semantic_model(datasets=[dataset]) + + doc = build_model(model, [orders], IssueLog()) + columns, formulas = _all_columns_and_formulas(doc.body) + by_name = {f["name"]: f for f in formulas} + + net_amount_id = next(c for c in columns if c["name"] == "Net Amount")["formula_id"] + margin_expr = next(f for f in formulas if f["name"] == "Margin Pct")["expr"] + + # The id form, not the display-name form -- and it actually resolves + # against the id this same build assigned the referenced formula. + assert f"[{net_amount_id}]" in margin_expr + assert "[formula_net_amount]" == f"[{net_amount_id}]" + assert by_name # sanity + + +class TestFormulaReferenceRewriting: + """A formula id is regenerated from the *normalised* form of its own + display name (_formula_id_from), which can differ from whatever id text + the source document's own cross-references were written against. Every + embedded reference has to be rewritten to match, or it dangles -- + ThoughtSpot parses an unresolvable bracket reference as search tokens + rather than failing at parse time, so a stale reference is a guaranteed + import failure. Assert the property directly (every reference resolves + to an emitted id) rather than pinning specific id spellings, so these + survive a change of normalisation scheme. + """ + + def test_a_reference_whose_target_normalises_differently_is_rewritten(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE"), + _column("Cost", "COST", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _field( + "net_amount", _dialects(("THOUGHTSPOT", "[orders::Amount] - [orders::Cost]")), + label="Net-Amount", # normalises to "net_amount" + ), + _field( + "margin_pct", + # Written against the SOURCE's own id text ("Net-Amount", + # verbatim) -- not the normalised form this build mints. + _dialects(("THOUGHTSPOT", "[formula_Net-Amount] / [orders::Amount]")), + label="Margin Pct", + ), + ]) + model = _semantic_model(datasets=[dataset]) + log = IssueLog() + + doc = build_model(model, [orders], log) + _columns, formulas = _all_columns_and_formulas(doc.body) + + assert not _dangling_formula_references(formulas) + assert not [i for i in log.as_dicts() if i["code"] == "TS-MODEL-FORMULA-REFERENCE-UNRESOLVED"] + net_amount_id = next(f["id"] for f in formulas if f["name"] == "Net-Amount") + margin_expr = next(f["expr"] for f in formulas if f["name"] == "Margin Pct") + assert f"[{net_amount_id}]" in margin_expr + assert "[formula_Net-Amount]" not in margin_expr # the stale reference is gone + + def test_a_chain_of_three_resolves_regardless_of_declaration_order(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + # Declared in an order where the referenced formula comes AFTER + # its referencer, twice over -- proves the rewrite does not + # depend on build order. + _field( + "top", _dialects(("THOUGHTSPOT", "[formula_Middle] * 2")), label="Top", + ), + _field( + "middle", _dialects(("THOUGHTSPOT", "[formula_Bottom] + 1")), label="Middle", + ), + _field( + # A bare bracket reference would classify as a PHYSICAL + # field (no formula_id of its own) -- this must genuinely be + # computed so it gets an id the chain can resolve against. + "bottom", _dialects(("THOUGHTSPOT", "[orders::Amount] * 1")), label="Bottom", + ), + ]) + model = _semantic_model(datasets=[dataset]) + log = IssueLog() + + doc = build_model(model, [orders], log) + _columns, formulas = _all_columns_and_formulas(doc.body) + + assert not _dangling_formula_references(formulas) + assert not [i for i in log.as_dicts() if i["code"] == "TS-MODEL-FORMULA-REFERENCE-UNRESOLVED"] + by_name = {f["name"]: f for f in formulas} + bottom_id = by_name["Bottom"]["id"] + middle_id = by_name["Middle"]["id"] + assert f"[{middle_id}]" in by_name["Top"]["expr"] + assert f"[{bottom_id}]" in by_name["Middle"]["expr"] + + def test_a_reference_to_a_formula_that_does_not_exist_is_logged_not_silently_dangling(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _field( + "adjusted", _dialects(("THOUGHTSPOT", "[formula_Ghost] + [orders::Amount]")), + label="Adjusted", + ), + ]) + model = _semantic_model(datasets=[dataset]) + log = IssueLog() + + doc = build_model(model, [orders], log) + _columns, formulas = _all_columns_and_formulas(doc.body) + + issues = [i for i in log.as_dicts() if i["code"] == "TS-MODEL-FORMULA-REFERENCE-UNRESOLVED"] + assert len(issues) == 1 + assert issues[0]["severity"] == "ERROR" + assert "formula_Ghost" in issues[0]["message"] + # Nothing safe to substitute -- the unresolved reference is left + # exactly as written, not silently dropped or invented. + adjusted_expr = next(f["expr"] for f in formulas if f["name"] == "Adjusted") + assert "[formula_Ghost]" in adjusted_expr + + def test_a_reference_needing_no_normalisation_is_left_untouched(self): + # The common case: the source already used the slug-shaped + # convention this converter itself mints, so the rewrite is a no-op + # -- confirms the fix does not disturb the case that already worked. + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE"), + _column("Cost", "COST", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _field( + "net_amount", _dialects(("THOUGHTSPOT", "[orders::Amount] - [orders::Cost]")), + label="net_amount", + ), + _field( + "margin_pct", + _dialects(("THOUGHTSPOT", "[formula_net_amount] / [orders::Amount]")), + label="margin_pct", + ), + ]) + model = _semantic_model(datasets=[dataset]) + log = IssueLog() + + doc = build_model(model, [orders], log) + _columns, formulas = _all_columns_and_formulas(doc.body) + + assert not _dangling_formula_references(formulas) + margin_expr = next(f["expr"] for f in formulas if f["name"] == "margin_pct") + assert margin_expr == "[formula_net_amount] / [orders::Amount]" + assert not [i for i in log.as_dicts() if i["code"] == "TS-MODEL-FORMULA-REFERENCE-UNRESOLVED"] + + +# --------------------------------------------------------------------------- +# An ambiguous bracket reference must be reported, never crash the build. +# --------------------------------------------------------------------------- + +class TestAmbiguousColumnReferenceInModel: + """A field's own THOUGHTSPOT bracket carries "::" inside its table or + column part, so `identifiers.split_column_ref` refuses to guess which + "::" is the real delimiter. `build_table` (see + test_ossie_to_thoughtspot_tables.py's own TestAmbiguousColumnReference) + reports this once and omits the physical column; `build_model` reaches + the same ambiguous reference through a second, unlogging call + (`_field_physical_display_name`, to avoid a duplicate report under a + second object_ref) and must not raise either -- the field is emitted as + a formula carrying the ambiguous text verbatim, which will fail to + import until the ambiguity is fixed, exactly as any other unresolvable + THOUGHTSPOT-only construct already is. + """ + + def test_an_ambiguous_bracket_becomes_a_formula_rather_than_raising(self): + orders = _table_doc("A::B", [_column("y", "y", "INT64")]) + dataset = _dataset("A::B", "SALES.PUBLIC.WIDGETS", fields=[ + _field("x", _dialects(("THOUGHTSPOT", "[A::B::y]")), label="x"), + ]) + model = _semantic_model(name="probe", datasets=[dataset]) + doc = build_model(model, [orders], IssueLog()) + columns, formulas = _all_columns_and_formulas(doc.body) + assert columns == [ + {"name": "x", "formula_id": "formula_x", "properties": {"column_type": "ATTRIBUTE"}} + ] + assert formulas == [{"id": "formula_x", "name": "x", "expr": "[A::B::y]"}] + + +# --------------------------------------------------------------------------- +# Unique display names across columns[] and formulas[]. +# --------------------------------------------------------------------------- + +class TestDisplayNameCollisions: + def test_two_fields_from_different_datasets_with_the_same_label_get_distinct_names(self): + orders = _table_doc("orders", [_column("Status", "STATUS", "VARCHAR")]) + customers = _table_doc("customers", [_column("Status", "C_STATUS", "VARCHAR")]) + orders_ds = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _field("status", _dialects(("THOUGHTSPOT", "[orders::Status]")), label="Status"), + ]) + customers_ds = _dataset("customers", "SALES.PUBLIC.CUSTOMERS", fields=[ + _field("status", _dialects(("THOUGHTSPOT", "[customers::Status]")), label="Status"), + ]) + model = _semantic_model(datasets=[orders_ds, customers_ds]) + + doc = build_model(model, [orders, customers], IssueLog()) + columns, _formulas = _all_columns_and_formulas(doc.body) + names = [c["name"] for c in columns] + + assert len(names) == len(set(names)), f"duplicate display name(s) in {names!r}" + # Both columns still reference their own, unrenamed physical column. + column_ids = {c["column_id"] for c in columns} + assert column_ids == {"orders::Status", "customers::Status"} + + def test_the_rename_is_logged_not_silent(self): + # The rename itself is correct -- uniqueness is required -- but it + # changes a name the user chose, and that used to go unreported. + orders = _table_doc("orders", [_column("Status", "STATUS", "VARCHAR")]) + customers = _table_doc("customers", [_column("Status", "C_STATUS", "VARCHAR")]) + orders_ds = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _field("status", _dialects(("THOUGHTSPOT", "[orders::Status]")), label="Status"), + ]) + customers_ds = _dataset("customers", "SALES.PUBLIC.CUSTOMERS", fields=[ + _field("status", _dialects(("THOUGHTSPOT", "[customers::Status]")), label="Status"), + ]) + model = _semantic_model(datasets=[orders_ds, customers_ds]) + log = IssueLog() + + doc = build_model(model, [orders, customers], log) + columns, _formulas = _all_columns_and_formulas(doc.body) + renamed = next(c["name"] for c in columns if c["name"] != "Status") + + [issue] = [i for i in log.as_dicts() if i["code"] == "TS-MODEL-DISPLAY-NAME-COLLISION"] + assert "Status" in issue["message"] + assert renamed in issue["message"] + + def test_the_first_field_to_take_a_name_is_not_reported_as_a_collision(self): + orders = _table_doc("orders", [_column("Status", "STATUS", "VARCHAR")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _field("status", _dialects(("THOUGHTSPOT", "[orders::Status]")), label="Status"), + ]) + model = _semantic_model(datasets=[dataset]) + log = IssueLog() + + build_model(model, [orders], log) + + assert not any(i["code"] == "TS-MODEL-DISPLAY-NAME-COLLISION" for i in log.as_dicts()) + + def test_a_field_and_a_metric_with_the_same_display_name_also_get_distinct_names(self): + # Uniqueness spans columns[] AND formulas[] together, not just columns[] + # against columns[]. + orders = _table_doc("orders", [_column("Margin", "MARGIN", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _field("margin", _dialects(("THOUGHTSPOT", "[orders::Margin]")), label="Margin"), + ]) + metric = _metric("Margin", _dialects(("THOUGHTSPOT", "sum ( [orders::Margin] )"))) + model = _semantic_model(datasets=[dataset], metrics=[metric]) + + doc = build_model(model, [orders], IssueLog()) + columns, _formulas = _all_columns_and_formulas(doc.body) + names = [c["name"] for c in columns] + + # The field (a physical column_id entry) and the metric (a + # formula_id entry, whose own formulas[].name mirrors this same + # surfacing name by design -- see TestGeneratedModelWouldImport's + # helper) must not collide here. + assert len(names) == len(set(names)), f"duplicate display name(s) in {names!r}" + + +# --------------------------------------------------------------------------- +# column_type and synonyms under properties. +# --------------------------------------------------------------------------- + +class TestPropertiesPlacement: + def test_column_type_is_never_a_bare_root_key(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _field("amount", _dialects(("THOUGHTSPOT", "[orders::Amount]")), label="Amount"), + ]) + metric = _metric("total", _dialects(("THOUGHTSPOT", "sum ( [orders::Amount] )"))) + model = _semantic_model(datasets=[dataset], metrics=[metric]) + + doc = build_model(model, [orders], IssueLog()) + columns, _formulas = _all_columns_and_formulas(doc.body) + + for column in columns: + assert "column_type" not in column + assert column["properties"]["column_type"] in ("ATTRIBUTE", "MEASURE") + + def test_synonyms_and_synonym_type_land_under_properties(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _field( + "amount", _dialects(("THOUGHTSPOT", "[orders::Amount]")), label="Amount", + ai_context={"synonyms": ["revenue", "sales"]}, + ), + ]) + model = _semantic_model(datasets=[dataset]) + + doc = build_model(model, [orders], IssueLog()) + columns, _formulas = _all_columns_and_formulas(doc.body) + column = columns[0] + + assert "synonyms" not in column + assert column["properties"]["synonyms"] == ["revenue", "sales"] + assert column["properties"]["synonym_type"] == "USER_DEFINED" + + def test_synonym_type_is_set_whenever_synonyms_are_present_on_a_metric_too(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + metric = _metric( + "total_revenue", _dialects(("THOUGHTSPOT", "sum ( [orders::Amount] )")), + ai_context={"synonyms": ["revenue"]}, + ) + model = _semantic_model(datasets=[dataset], metrics=[metric]) + + doc = build_model(model, [orders], IssueLog()) + columns, _formulas = _all_columns_and_formulas(doc.body) + column = next(c for c in columns if c["name"] == "total_revenue") + + assert column["properties"]["synonym_type"] == "USER_DEFINED" + + +# --------------------------------------------------------------------------- +# Never is_hidden / was_auto_generated. +# --------------------------------------------------------------------------- + +class TestNeverEmitsHiddenOrAutoGenerated: + def test_is_hidden_and_was_auto_generated_are_never_emitted(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _field("amount", _dialects(("THOUGHTSPOT", "[orders::Amount]")), label="Amount"), + ]) + metric = _metric("total", _dialects(("THOUGHTSPOT", "sum ( [orders::Amount] )"))) + model = _semantic_model(datasets=[dataset], metrics=[metric]) + + doc = build_model(model, [orders], IssueLog()) + blob = json.dumps(doc.body) + + assert "is_hidden" not in blob + assert "was_auto_generated" not in blob + + +class TestHiddenFlagDroppedFromEmissionButKeptInTheStash: + """A hidden column cannot be surfaced again without a manual edit on the + target instance, so this converter forbids the *emitted* TML from ever carrying + `is_hidden: true` -- but the Ossie document's own vendor payload still + has to preserve it (the two are different artefacts: the stash is this + package's record of what the source held, the emission is what a fresh + import would create). Same treatment for `was_auto_generated`, which + reaches the identical `column_properties` catch-all whenever a source + TML sets it and is not explicitly consumed anywhere. + """ + + def test_a_stashed_is_hidden_true_is_absent_from_the_emitted_column(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + field = _field( + "amount", _dialects(("THOUGHTSPOT", "[orders::Amount]")), label="Amount", + field_stash={FIELD_STASH_COLUMN_PROPERTIES: {"is_hidden": True}}, + ) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[field]) + model = _semantic_model(datasets=[dataset]) + log = IssueLog() + + doc = build_model(model, [orders], log) + columns, _formulas = _all_columns_and_formulas(doc.body) + column = columns[0] + + assert "is_hidden" not in column["properties"] + issues = [i for i in log.as_dicts() if i["code"] == "TS-MODEL-PROPERTY-NEVER-EMITTED"] + assert len(issues) == 1 + assert issues[0]["severity"] == "WARNING" + assert "is_hidden" in issues[0]["message"] + assert "manual edit" in issues[0]["message"] + + def test_a_stashed_was_auto_generated_true_is_also_dropped_and_logged(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + field = _field( + "amount", _dialects(("THOUGHTSPOT", "[orders::Amount]")), label="Amount", + field_stash={FIELD_STASH_COLUMN_PROPERTIES: {"was_auto_generated": True}}, + ) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[field]) + model = _semantic_model(datasets=[dataset]) + log = IssueLog() + + doc = build_model(model, [orders], log) + columns, _formulas = _all_columns_and_formulas(doc.body) + column = columns[0] + + assert "was_auto_generated" not in column["properties"] + issues = [i for i in log.as_dicts() if i["code"] == "TS-MODEL-PROPERTY-NEVER-EMITTED"] + assert len(issues) == 1 + assert "was_auto_generated" in issues[0]["message"] + + def test_a_metric_with_a_stashed_is_hidden_true_is_also_covered(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + metric = _metric( + "total", _dialects(("THOUGHTSPOT", "sum ( [orders::Amount] )")), + metric_stash={FIELD_STASH_COLUMN_PROPERTIES: {"is_hidden": True}}, + ) + model = _semantic_model(datasets=[dataset], metrics=[metric]) + log = IssueLog() + + doc = build_model(model, [orders], log) + columns, _formulas = _all_columns_and_formulas(doc.body) + column = next(c for c in columns if c["name"] == "total") + + assert "is_hidden" not in column["properties"] + assert any(i["code"] == "TS-MODEL-PROPERTY-NEVER-EMITTED" for i in log.as_dicts()) + + def test_is_hidden_false_is_omitted_without_an_issue(self): + # false (or absent) is ThoughtSpot's own default, so leaving the key + # out of the emitted document is not a loss and is not worth a log. + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + field = _field( + "amount", _dialects(("THOUGHTSPOT", "[orders::Amount]")), label="Amount", + field_stash={FIELD_STASH_COLUMN_PROPERTIES: {"is_hidden": False}}, + ) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[field]) + model = _semantic_model(datasets=[dataset]) + log = IssueLog() + + doc = build_model(model, [orders], log) + columns, _formulas = _all_columns_and_formulas(doc.body) + + assert "is_hidden" not in columns[0]["properties"] + assert not [i for i in log.as_dicts() if i["code"] == "TS-MODEL-PROPERTY-NEVER-EMITTED"] + + def test_no_hidden_flag_at_all_is_untouched_and_logs_nothing(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + field = _field( + "amount", _dialects(("THOUGHTSPOT", "[orders::Amount]")), label="Amount", + field_stash={FIELD_STASH_COLUMN_PROPERTIES: {"index_type": "DONT_INDEX"}}, + ) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[field]) + model = _semantic_model(datasets=[dataset]) + log = IssueLog() + + doc = build_model(model, [orders], log) + columns, _formulas = _all_columns_and_formulas(doc.body) + + # The rest of the stashed property survives untouched -- only the + # never-emit keys are filtered, nothing else. + assert columns[0]["properties"]["index_type"] == "DONT_INDEX" + assert not [i for i in log.as_dicts() if i["code"] == "TS-MODEL-PROPERTY-NEVER-EMITTED"] + + def test_the_flag_still_reaches_the_ossie_stash_on_a_forward_conversion(self): + # The drop is emission-only: tml_to_ossie.py's own stash of an + # unconsumed column property is untouched by this fix, and still has + # to preserve is_hidden so a *future* Ossie -> TML build has + # something to see (and drop, and log) in the first place. + table_doc = TmlDocument( + kind="table", + body={ + "name": "ORDERS", "db": "SALES", "schema": "PUBLIC", "db_table": "ORDERS", + "connection": {"name": "My Snowflake"}, + "columns": [ + {"name": "Amount", "db_column_name": "O_AMOUNT", + "db_column_properties": {"data_type": "DOUBLE"}}, + ], + }, + guid=None, + ) + model_doc = TmlDocument( + kind="model", + body={ + "name": "Sales Analytics", + "model_tables": [{"name": "ORDERS"}], + "columns": [ + {"name": "Amount", "column_id": "ORDERS::Amount", + "properties": {"column_type": "ATTRIBUTE", "is_hidden": True}}, + ], + }, + guid=None, + ) + document_set = DocumentSet(model=model_doc, tables=(table_doc,)) + + ossie = tml_to_ossie_convert(document_set) + [dataset] = ossie.model["semantic_model"][0]["datasets"] + [field] = dataset["fields"] + + payload = stash_module.read_stash(field) + assert payload[FIELD_STASH_COLUMN_PROPERTIES]["is_hidden"] is True + + +# --------------------------------------------------------------------------- +# A brace-carrying expr is a block scalar. +# --------------------------------------------------------------------------- + +class TestBraceExpressionIsABlockScalar: + def test_a_brace_carrying_formula_is_wrapped_and_reloads_correctly(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE"), + _column("Region", "REGION", "VARCHAR")]) + expr = ( + "group_aggregate ( sum ( [orders::Amount] ) , " + "query_groups ( ) + { [orders::Region] } , query_filters ( ) )" + ) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + metric = _metric("grouped", _dialects(("THOUGHTSPOT", expr))) + model = _semantic_model(datasets=[dataset], metrics=[metric]) + + doc = build_model(model, [orders], IssueLog()) + + # tml.block_scalar marks the string for '>-' emission; the dump/reload + # round trip is the real proof it parses back byte-for-byte. + text = dump_document(doc) + assert ">-" in text + reloaded = load_document(text) + reloaded_formula = reloaded.body["formulas"][0] + assert reloaded_formula["expr"] == expr + + def test_a_formula_with_no_braces_is_not_wrapped(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + metric = _metric("total", _dialects(("THOUGHTSPOT", "sum ( [orders::Amount] )"))) + model = _semantic_model(datasets=[dataset], metrics=[metric]) + + doc = build_model(model, [orders], IssueLog()) + text = dump_document(doc) + + assert ">-" not in text + + +# --------------------------------------------------------------------------- +# Dialect selection. +# --------------------------------------------------------------------------- + +class TestDialectSelection: + def test_a_thoughtspot_entry_is_used_verbatim(self): + log = IssueLog() + result = to_thoughtspot_expression( + _dialects(("THOUGHTSPOT", "sum ( [A::x] )"), ("ANSI_SQL", "SUM(a.x)")), + _resolve_field({}), log, object_ref="metric:m", + ) + assert result == "sum ( [A::x] )" + assert not log.as_dicts() + + def test_an_ansi_sql_only_bare_reference_is_translated_via_resolve_field(self): + log = IssueLog() + index = {"orders.amount": ("orders", "Amount")} + result = to_thoughtspot_expression( + _dialects(("ANSI_SQL", "orders.amount")), _resolve_field(index), log, object_ref="field:f", + ) + assert result == "[orders::Amount]" + assert not log.as_dicts() + + def test_an_ansi_sql_only_aggregate_call_is_translated_structurally(self): + # The construct-mapping document's own worked shape: a hand-authored + # metric with only an ANSI_SQL sibling. + log = IssueLog() + index = {"orders.amount": ("orders", "Amount")} + result = to_thoughtspot_expression( + _dialects(("ANSI_SQL", "SUM(orders.amount)")), _resolve_field(index), log, object_ref="metric:m", + ) + assert result == "sum ( [orders::Amount] )" + assert not log.as_dicts() + + def test_count_distinct_is_translated_structurally(self): + log = IssueLog() + index = {"orders.id": ("orders", "Id")} + result = to_thoughtspot_expression( + _dialects(("ANSI_SQL", "COUNT(DISTINCT orders.id)")), + _resolve_field(index), log, object_ref="metric:m", + ) + assert result == "unique count ( [orders::Id] )" + + def test_an_unresolvable_ansi_sql_reference_raises_an_issue_and_stashes(self): + log = IssueLog() + result = to_thoughtspot_expression( + _dialects(("ANSI_SQL", "orders.unknown_field")), _resolve_field({}), log, object_ref="field:f", + ) + assert result is None + assert any(i["code"] == "TS-EXPR-ANSI-UNRESOLVED" for i in log.as_dicts()) + + def test_an_ansi_sql_expression_the_catalog_cannot_match_structurally_raises_an_issue(self): + log = IssueLog() + result = to_thoughtspot_expression( + _dialects(("ANSI_SQL", "orders.amount + orders.cost")), + _resolve_field({}), log, object_ref="field:f", + ) + assert result is None + assert any(i["code"] == "TS-EXPR-ANSI-UNSTRUCTURED" for i in log.as_dicts()) + + def test_an_ansi_sql_function_the_catalog_does_not_match_raises_an_issue(self): + log = IssueLog() + index = {"orders.amount": ("orders", "Amount"), "orders.cost": ("orders", "Cost")} + result = to_thoughtspot_expression( + _dialects(("ANSI_SQL", "MOD(orders.amount, orders.cost)")), + _resolve_field(index), log, object_ref="metric:m", + ) + assert result is None + assert any(i["code"] == "TS-EXPR-ANSI-UNMATCHED" for i in log.as_dicts()) + + def test_no_usable_dialect_at_all_raises_an_issue(self): + log = IssueLog() + result = to_thoughtspot_expression( + _dialects(("DATABRICKS", "amount")), _resolve_field({}), log, object_ref="field:f", + ) + assert result is None + assert any(i["code"] == "TS-EXPR-NO-USABLE-DIALECT" for i in log.as_dicts()) + + def test_thoughtspot_is_never_re_rendered_into_ansi_sql_or_vice_versa(self): + # Never re-render one dialect into another: an ANSI_SQL-only + # expression the catalog cannot match structurally must not fall + # back to guessing a translation from the (absent) THOUGHTSPOT side, + # and a present THOUGHTSPOT entry must not be second-guessed against + # a present-but-different ANSI_SQL sibling. + log = IssueLog() + # A THOUGHTSPOT entry wins even when an ANSI_SQL sibling exists and + # would translate to something textually different. + result = to_thoughtspot_expression( + _dialects(("THOUGHTSPOT", "average ( [A::x] )"), ("ANSI_SQL", "SUM(a.x)")), + _resolve_field({"a.x": ("A", "x")}), log, object_ref="metric:m", + ) + assert result == "average ( [A::x] )" + + +# --------------------------------------------------------------------------- +# A metric is always a formula, never column_id + aggregation. +# --------------------------------------------------------------------------- + +class TestMetricNeverEmitsColumnIdPlusAggregation: + def test_no_columns_entry_ever_has_both_column_id_and_aggregation(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + # This is exactly the shape stashed as column_aggregation, which a + # naive reversal might emit as column_id + aggregation (R4a: that + # would collide with a field sharing the same column_id). + metric = _metric( + "total", _dialects(("THOUGHTSPOT", "sum ( [orders::Amount] )")), + metric_stash={METRIC_STASH_SHAPE: METRIC_SHAPE_COLUMN_AGGREGATION}, + ) + model = _semantic_model(datasets=[dataset], metrics=[metric]) + + doc = build_model(model, [orders], IssueLog()) + columns, _formulas = _all_columns_and_formulas(doc.body) + metric_column = next(c for c in columns if c["name"] == "total") + + assert "column_id" not in metric_column + assert "formula_id" in metric_column + + +class TestMetricShapeDefault: + """The cross-task contract: tml_to_ossie.py deliberately omits the shape + stash key for the default (`formula`) shape, so an absent key must + default to that same shape here -- defaulting to anything else, or + raising, silently mis-converts the commonest metric shape.""" + + def test_a_metric_with_no_shape_stash_defaults_to_the_formula_shape(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + # No custom_extensions at all -- the exact contract under test. + metric = _metric("total", _dialects(("THOUGHTSPOT", "sum ( [orders::Amount] )"))) + assert "custom_extensions" not in metric + model = _semantic_model(datasets=[dataset], metrics=[metric]) + + doc = build_model(model, [orders], IssueLog()) + columns, formulas = _all_columns_and_formulas(doc.body) + metric_column = next(c for c in columns if c["name"] == "total") + metric_formula = next(f for f in formulas if f["id"] == metric_column["formula_id"]) + + # The formula shape: the composed expr is used as-is (never + # decomposed the way scalar_formula_plus_aggregation would be). + assert metric_formula["expr"] == "sum ( [orders::Amount] )" + + def test_the_default_and_an_explicit_formula_shape_stash_produce_the_same_result(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + implicit = _metric("total", _dialects(("THOUGHTSPOT", "sum ( [orders::Amount] )"))) + explicit = _metric( + "total", _dialects(("THOUGHTSPOT", "sum ( [orders::Amount] )")), + metric_stash={METRIC_STASH_SHAPE: METRIC_SHAPE_FORMULA}, + ) + + implicit_doc = build_model( + _semantic_model(datasets=[dataset], metrics=[implicit]), [orders], IssueLog(), + ) + explicit_doc = build_model( + _semantic_model(datasets=[dataset], metrics=[explicit]), [orders], IssueLog(), + ) + + implicit_columns, implicit_formulas = _all_columns_and_formulas(implicit_doc.body) + explicit_columns, explicit_formulas = _all_columns_and_formulas(explicit_doc.body) + assert implicit_columns[0]["properties"] == explicit_columns[0]["properties"] + assert implicit_formulas[0]["expr"] == explicit_formulas[0]["expr"] + + def test_a_scalar_formula_plus_aggregation_shape_is_decomposed_not_left_as_default(self): + # The one shape that must NOT collapse into the default: proves the + # default only kicks in for a genuinely absent/"formula" key. + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE"), + _column("Cost", "COST", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + metric = _metric( + "avg_net", _dialects(("THOUGHTSPOT", "average ( [orders::Amount] - [orders::Cost] )")), + metric_stash={METRIC_STASH_SHAPE: METRIC_SHAPE_SCALAR_FORMULA_PLUS_AGGREGATION}, + ) + model = _semantic_model(datasets=[dataset], metrics=[metric]) + + doc = build_model(model, [orders], IssueLog()) + columns, formulas = _all_columns_and_formulas(doc.body) + metric_column = next(c for c in columns if c["name"] == "avg_net") + metric_formula = next(f for f in formulas if f["id"] == metric_column["formula_id"]) + + assert metric_formula["expr"] == "[orders::Amount] - [orders::Cost]" + assert metric_column["properties"]["aggregation"] == "AVERAGE" + + +# --------------------------------------------------------------------------- +# TML import safety -- proving a generated Model would actually import. +# --------------------------------------------------------------------------- + +def _assert_would_import(body: dict) -> None: + columns, formulas = _all_columns_and_formulas(body) + formula_ids = {f["id"] for f in formulas} + surfaced_formula_ids = {c["formula_id"] for c in columns if "formula_id" in c} + + assert len(formula_ids) == len(formulas), "duplicate formulas[] id" + + # Uniqueness spans columns[] and formulas[] together, but a + # formula surfaced by exactly one columns[] entry shares its name with + # that entry *by design* (the worked shape example: formulas[].name == + # the surfacing columns[].name, both "total_revenue") -- that pairing is + # one logical object represented twice, not a collision. Only a formula + # with no surfacing column (an unattributed/orphan formula) contributes + # its own, separate name to the uniqueness check. + display_names = [] + for column in columns: + assert "column_type" not in column, "bare column_type at column root" + assert "properties" in column and "column_type" in column["properties"] + if "synonyms" in column: + raise AssertionError("synonyms present but not under properties") + properties = column["properties"] + if "synonyms" in properties: + assert properties.get("synonym_type") == "USER_DEFINED" + assert properties.get("is_hidden") is not True + assert properties.get("was_auto_generated") is not True + display_names.append(column["name"]) + if "formula_id" in column: + assert column["formula_id"] in formula_ids, "formula_id names no real formulas[] entry" + + for formula in formulas: + assert "aggregation" not in formula, "aggregation on a formulas[] entry" + if formula["id"] not in surfaced_formula_ids: + display_names.append(formula["name"]) + + assert len(display_names) == len(set(display_names)), ( + f"duplicate display name across columns[]/formulas[]: {display_names!r}" + ) + + +class TestGeneratedModelWouldImport: + def test_a_representative_model_satisfies_every_import_invariant(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE"), + _column("Cost", "COST", "DOUBLE")]) + customers = _table_doc("customers", [_column("Status", "C_STATUS", "VARCHAR")]) + orders_ds = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _field("amount", _dialects(("THOUGHTSPOT", "[orders::Amount]")), label="Amount"), + _field( + "net", _dialects(("THOUGHTSPOT", "[orders::Amount] - [orders::Cost]")), label="Status", + ), + ]) + customers_ds = _dataset("customers", "SALES.PUBLIC.CUSTOMERS", fields=[ + _field("status", _dialects(("THOUGHTSPOT", "[customers::Status]")), label="Status"), + ]) + metric = _metric( + "total", _dialects(("THOUGHTSPOT", "sum ( [orders::Amount] )")), + ai_context={"synonyms": ["revenue"]}, + ) + model = _semantic_model(datasets=[orders_ds, customers_ds], metrics=[metric]) + + doc = build_model(model, [orders, customers], IssueLog()) + _assert_would_import(doc.body) + + # And it genuinely re-parses as valid TML. + reloaded = load_document(dump_document(doc)) + assert reloaded.kind == "model" + + +# --------------------------------------------------------------------------- +# Round trip against the forward direction -- the strongest check available. +# --------------------------------------------------------------------------- + +def _model_tml(name, model_tables, columns, formulas=None, description=None): + body: dict = {"name": name, "model_tables": model_tables, "columns": columns} + if formulas is not None: + body["formulas"] = formulas + if description is not None: + body["description"] = description + return TmlDocument(kind="model", body=body, guid=None) + + +class TestRoundTripAgainstTheForwardDirection: + """Convert a rich, real TML document set forward (tml_to_ossie.convert), + then back (build_table + build_model), and inspect every difference + against the original -- a hand-written Ossie fixture built from reading + the rules can be unknowingly wrong about what the forward direction + actually produces; only a real round trip catches that. + + The fixture covers: physical and computed fields, a metric of each of + the three TML shapes, a formula cross-reference, a brace-carrying + formula (group_aggregate), a column name that is a YAML 1.1 boolean + token ("On"), and two display names that collide only after + normalisation (Status on two different datasets). + """ + + def _build(self): + orders = _table_doc("ORDERS", [ + _column("Order Date", "O_ORDERDATE", "DATE"), + _column("Amount", "O_AMOUNT", "DOUBLE"), + _column("Cost", "O_COST", "DOUBLE"), + _column("On", "O_ON_FLAG", "VARCHAR"), + _column("Status", "O_STATUS", "VARCHAR"), + ]) + customers = _table_doc("CUSTOMERS", [ + _column("Id", "ID", "INT64"), + _column("Status", "C_STATUS", "VARCHAR"), + ]) + model = _model_tml( + "Sales Analytics", + model_tables=[ + {"name": "ORDERS", "joins": [{ + "with": "CUSTOMERS", "on": "[ORDERS::Amount] = [CUSTOMERS::Id]", + "type": "INNER", "cardinality": "MANY_TO_ONE", + }]}, + {"name": "CUSTOMERS"}, + ], + columns=[ + {"name": "Order Date", "column_id": "ORDERS::Order Date", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "Amount", "column_id": "ORDERS::Amount", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "Cost", "column_id": "ORDERS::Cost", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "On", "column_id": "ORDERS::On", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "Status", "column_id": "ORDERS::Status", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "Status", "column_id": "CUSTOMERS::Status", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "Net Amount", "formula_id": "formula_net_amount", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "Margin Pct", "formula_id": "formula_margin_pct", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "total_revenue", "formula_id": "formula_total_revenue", + "properties": {"column_type": "MEASURE", "aggregation": "SUM"}}, + {"name": "average_net", "formula_id": "formula_average_net", + "properties": {"column_type": "MEASURE", "aggregation": "AVERAGE"}}, + {"name": "customer_count", "column_id": "CUSTOMERS::Id", + "properties": {"column_type": "MEASURE", "aggregation": "COUNT_DISTINCT"}}, + {"name": "grouped", "formula_id": "formula_grouped", + "properties": {"column_type": "MEASURE"}}, + ], + formulas=[ + {"id": "formula_net_amount", "name": "net_amount", + "expr": "[ORDERS::Amount] - [ORDERS::Cost]"}, + {"id": "formula_margin_pct", "name": "margin_pct", + "expr": "[formula_net_amount] / [ORDERS::Amount]"}, + {"id": "formula_total_revenue", "name": "total_revenue", + "expr": "sum ( [ORDERS::Amount] )"}, + {"id": "formula_average_net", "name": "average_net", + "expr": "[ORDERS::Amount] - [ORDERS::Cost]"}, + {"id": "formula_grouped", "name": "grouped", + "expr": ( + "group_aggregate ( sum ( [ORDERS::Amount] ) , " + "query_groups ( ) + { [ORDERS::Status] } , query_filters ( ) )" + )}, + ], + ) + document_set = DocumentSet(model=model, tables=(orders, customers)) + ossie = tml_to_ossie_convert(document_set) + semantic_model = ossie.model["semantic_model"][0] + + log = IssueLog() + rebuilt_tables = [build_table(ds, log) for ds in semantic_model["datasets"]] + rebuilt_model = build_model(semantic_model, rebuilt_tables, log) + return model, rebuilt_model, log + + def test_the_rebuilt_model_would_import(self): + _original, rebuilt, _log = self._build() + _assert_would_import(rebuilt.body) + reloaded = load_document(dump_document(rebuilt)) + assert reloaded.kind == "model" + + def test_the_relationship_is_reconstructed_as_an_inline_join(self): + _original, rebuilt, _log = self._build() + orders_entry = next(t for t in rebuilt.body["model_tables"] if t["name"] == "ORDERS") + [join] = orders_entry["joins"] + assert join["with"] == "CUSTOMERS" + assert join["on"] == "[ORDERS::Amount] = [CUSTOMERS::Id]" + assert join["type"] == "INNER" + assert join["cardinality"] == "MANY_TO_ONE" + + def test_the_formula_cross_reference_survives_the_round_trip(self): + _original, rebuilt, _log = self._build() + _columns, formulas = _all_columns_and_formulas(rebuilt.body) + by_name = {f["name"]: f for f in formulas} + net_amount_id = next(f["id"] for f in formulas if f["name"] == "Net Amount") + assert f"[{net_amount_id}]" in by_name["Margin Pct"]["expr"] + + def test_colliding_display_names_are_disambiguated(self): + _original, rebuilt, _log = self._build() + columns, _formulas = _all_columns_and_formulas(rebuilt.body) + status_columns = [c for c in columns if c.get("column_id", "").endswith("::Status")] + assert len(status_columns) == 2 + assert len({c["name"] for c in status_columns}) == 2 # renamed, not dropped + assert {c["column_id"] for c in status_columns} == {"ORDERS::Status", "CUSTOMERS::Status"} + + def test_the_collision_rename_is_logged_naming_both_names(self): + # The rename is correct (uniqueness is required), but it changes + # text the user chose -- silently, before this fix. The issue must + # name both the original, colliding name and what it was renamed to. + _original, _rebuilt, log = self._build() + collision_issues = [i for i in log.as_dicts() if i["code"] == "TS-MODEL-DISPLAY-NAME-COLLISION"] + assert len(collision_issues) == 1 + message = collision_issues[0]["message"] + assert "Status" in message + assert "Status_2" in message + + def test_a_yaml_1_1_boolean_token_column_name_survives_dump_and_reload(self): + _original, rebuilt, _log = self._build() + text = dump_document(rebuilt) + reloaded = load_document(text) + columns, _formulas = _all_columns_and_formulas(reloaded.body) + on_column = next(c for c in columns if c["column_id"] == "ORDERS::On") + assert on_column["name"] == "On" # not coerced to a boolean on either leg + + def test_the_scalar_formula_plus_aggregation_metric_round_trips_to_the_same_shape(self): + original, rebuilt, _log = self._build() + original_column = next(c for c in original.body["columns"] if c["name"] == "average_net") + columns, formulas = _all_columns_and_formulas(rebuilt.body) + rebuilt_column = next(c for c in columns if c["name"] == "average_net") + rebuilt_formula = next(f for f in formulas if f["id"] == rebuilt_column["formula_id"]) + + assert rebuilt_column["properties"]["aggregation"] == original_column["properties"]["aggregation"] + original_formula = next( + f for f in original.body["formulas"] if f["id"] == original_column["formula_id"] + ) + assert rebuilt_formula["expr"] == original_formula["expr"] + + def test_the_column_aggregation_metric_becomes_a_formula_never_column_id_plus_aggregation(self): + _original, rebuilt, _log = self._build() + columns, _formulas = _all_columns_and_formulas(rebuilt.body) + rebuilt_column = next(c for c in columns if c["name"] == "customer_count") + # customer_count arrived as column_id + aggregation (the + # "column_aggregation" shape) but must never be re-emitted that way. + assert "column_id" not in rebuilt_column + assert "formula_id" in rebuilt_column + assert rebuilt_column["properties"]["aggregation"] == "COUNT_DISTINCT" + + def test_the_brace_carrying_formula_round_trips_through_dump_and_reload(self): + original, rebuilt, _log = self._build() + original_formula = next(f for f in original.body["formulas"] if f["name"] == "grouped") + text = dump_document(rebuilt) + reloaded = load_document(text) + _columns, formulas = _all_columns_and_formulas(reloaded.body) + reloaded_formula = next(f for f in formulas if f["name"] == "grouped") + assert reloaded_formula["expr"] == original_formula["expr"] + + def test_no_unexpected_error_severity_issues_are_raised(self): + # customer_count's Ossie-side "Integer" datatype is the one + # declared, expected loss -- everything else in this fixture + # should convert cleanly both ways. + _original, _rebuilt, log = self._build() + errors = [i for i in log.as_dicts() if i["severity"] == "ERROR"] + assert not errors, errors + + +# --------------------------------------------------------------------------- +# Own tests, beyond everything specified above. +# --------------------------------------------------------------------------- +# +# 1. A dataset's declared primary_key that no relationship's to_columns cover +# has nowhere to go in TML (Dataset-level mapping's own worked example) -- +# chosen because it is the one Ossie-side construct in this module's whole +# remit that genuinely has no TML home at all, in either document, and the +# round trip above never exercises a key that ISN'T witnessed by a +# relationship (CUSTOMERS.Id always is). A silent drop here would be the +# quietest possible data loss this module could produce. +# 2. build_table is called separately per dataset and its resulting +# TmlDocuments are handed to build_model as a Sequence with no name index +# of their own -- a dataset whose table_ref matches nothing in `tables` +# (a caller bug, or a table that failed to build) must not raise an +# unhandled KeyError/AttributeError reaching into `tables_by_name`, and +# must not silently emit a column_id referencing a column that was never +# validated to exist. Chosen because "the caller passes tables in a +# different order/set than the datasets" is exactly the kind of interface +# mismatch the task brief calls out for `resolve`/`resolve_field`'s +# inverted arity -- the same class of bug, at the object level instead of +# the argument level. + +class TestModelScopeStashRestoration: + def test_every_model_scope_stash_key_is_restored_under_its_own_tml_name(self): + orders = _table_doc("ORDERS", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _field("amount", _dialects(("THOUGHTSPOT", "[orders::Amount]")), label="Amount"), + ]) + model_stash = { + "model_properties": { + "join_progressive": True, "spotter_config": {"is_spotter_enabled": True}, + }, + "parameters": [{"name": "Discount", "data_type": "DOUBLE"}], + "filters": [{"name": "Active Only", "expr": "[orders::Amount] > 0"}], + "column_groups": [{"name": "Financials", "columns": ["Amount"]}], + "lesson_plans": [{"name": "Getting Started"}], + "action_object_associations": [{"action_name": "Export", "object_name": "Amount"}], + "constraints": {"ORDERS": "rolling 90 days"}, + "model_joins_with": [{"name": "aug_join", "destination": {"name": "ORDERS"}, "on": "1=1"}], + } + model = _semantic_model(datasets=[dataset], model_stash=model_stash) + + doc = build_model(model, [orders], IssueLog()) + + assert doc.body["properties"] == model_stash["model_properties"] + assert doc.body["parameters"] == model_stash["parameters"] + assert doc.body["filters"] == model_stash["filters"] + assert doc.body["column_groups"] == model_stash["column_groups"] + assert doc.body["lesson_plans"] == model_stash["lesson_plans"] + assert doc.body["action_object_associations"] == model_stash["action_object_associations"] + assert doc.body["constraints"] == model_stash["constraints"] + # model_joins_with restores under the BARE TML key joins_with, not + # under its own (disambiguating) stash key name. + assert doc.body["joins_with"] == model_stash["model_joins_with"] + assert "model_joins_with" not in doc.body + assert "model_properties" not in doc.body + + +class TestTmlNameWitness: + """The witness check for STASH_TML_NAME at metric and model scope: the exact + ThoughtSpot display name a prior TML -> Ossie trip stashed (when identifier + normalisation changed the identifier) is trustworthy only while nobody + has renamed the live Ossie identifier since. Self-verifying: the + stashed name's own normalised form is compared against the live + identifier directly, with no separate stored witness needed.""" + + def test_a_metric_whose_identifier_still_matches_the_stash_uses_the_stashed_name(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + metric = _metric( + "total_revenue", _dialects(("THOUGHTSPOT", "sum ( [orders::Amount] )")), + metric_stash={"tml_name": "Total Revenue"}, + ) + model = _semantic_model(datasets=[_dataset("orders", "SALES.PUBLIC.ORDERS")], metrics=[metric]) + log = IssueLog() + + doc = build_model(model, [orders], log) + columns, _formulas = _all_columns_and_formulas(doc.body) + + assert columns[0]["name"] == "Total Revenue" + assert not any(i["code"] == "TS-STASH-TML-NAME-STALE" for i in log.as_dicts()) + + def test_a_renamed_metric_drops_the_stale_stashed_name(self): + # The metric's own `name` was changed (total_revenue -> gross_revenue) + # since the stash was written -- the stashed "Total Revenue" now + # names a metric that no longer exists under that identifier. + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + metric = _metric( + "gross_revenue", _dialects(("THOUGHTSPOT", "sum ( [orders::Amount] )")), + metric_stash={"tml_name": "Total Revenue"}, + ) + model = _semantic_model(datasets=[_dataset("orders", "SALES.PUBLIC.ORDERS")], metrics=[metric]) + log = IssueLog() + + doc = build_model(model, [orders], log) + columns, _formulas = _all_columns_and_formulas(doc.body) + + assert columns[0]["name"] == "gross_revenue" + assert any(i["code"] == "TS-STASH-TML-NAME-STALE" for i in log.as_dicts()) + + def test_a_model_whose_identifier_still_matches_the_stash_uses_the_stashed_name(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + model = _semantic_model( + name="sales_analytics", datasets=[dataset], model_stash={"tml_name": "Sales Analytics"}, + ) + log = IssueLog() + + doc = build_model(model, [orders], log) + + assert doc.body["name"] == "Sales Analytics" + assert not any(i["code"] == "TS-STASH-TML-NAME-STALE" for i in log.as_dicts()) + + def test_a_renamed_model_drops_the_stale_stashed_name(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + model = _semantic_model( + name="marketing_analytics", datasets=[dataset], model_stash={"tml_name": "Sales Analytics"}, + ) + log = IssueLog() + + doc = build_model(model, [orders], log) + + assert doc.body["name"] == "marketing_analytics" + assert any(i["code"] == "TS-STASH-TML-NAME-STALE" for i in log.as_dicts()) + + +class TestUnattributedFormulas: + """A formula whose references span two or more Ossie datasets could not + become an ordinary Ossie field on the way out (no single dataset owns + it), but nothing about a TML formula's own surfacing columns[] entry + ties it to a dataset in the first place (formula_id + properties, + no column_id) -- so it is restored fully surfaced, exactly like any + other formula, rather than re-emitted as an orphan formulas[] entry + with no columns[] entry pointing at it. An earlier revision did the + latter, which made the formula unreachable in the rebuilt model by + ThoughtSpot's own visibility rule (a formulas[] entry with no + referencing columns[] entry is not surfaced) while raising an issue + that claimed only its properties were lost -- describing a smaller + loss than the one that actually happened. + """ + + def test_an_unattributed_formula_is_restored_fully_surfaced(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + model = _semantic_model( + datasets=[dataset], + model_stash={ + MODEL_STASH_UNATTRIBUTED_FORMULAS: [ + {"name": "Cross Dataset Thing", "expr": "[ORDERS::Amount] + [CUSTOMERS::Fee]"}, + ], + }, + ) + log = IssueLog() + + doc = build_model(model, [orders], log) + columns, formulas = _all_columns_and_formulas(doc.body) + + assert len(formulas) == 1 + assert formulas[0]["name"] == "Cross Dataset Thing" + assert formulas[0]["expr"] == "[ORDERS::Amount] + [CUSTOMERS::Fee]" + assert formulas[0]["id"] == "formula_cross_dataset_thing" + # A columns[] entry references it -- surfaced, not orphaned, so + # ThoughtSpot's own visibility rule does not hide it. + [surfacing] = [c for c in columns if c.get("formula_id") == formulas[0]["id"]] + assert surfacing["name"] == "Cross Dataset Thing" + assert surfacing["properties"]["column_type"] == "ATTRIBUTE" + + def test_stashed_column_properties_on_an_unattributed_formula_are_restored(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + model = _semantic_model( + datasets=[dataset], + model_stash={ + MODEL_STASH_UNATTRIBUTED_FORMULAS: [ + {"name": "Cross Dataset Thing", "expr": "[ORDERS::Amount] + [CUSTOMERS::Fee]", + FIELD_STASH_COLUMN_PROPERTIES: {"index_type": "DONT_INDEX"}}, + ], + }, + ) + log = IssueLog() + + doc = build_model(model, [orders], log) + columns, _formulas = _all_columns_and_formulas(doc.body) + + [surfacing] = [c for c in columns if c["name"] == "Cross Dataset Thing"] + assert surfacing["properties"]["index_type"] == "DONT_INDEX" + # Nothing was lost -- the old "properties lost" issue no longer + # applies, because the properties are restored, not dropped. + assert not any( + i["code"] == "TS-MODEL-UNATTRIBUTED-FORMULA-PROPERTIES-LOST" + for i in log.as_dicts() + ) + + +# --------------------------------------------------------------------------- +# Inline joins: type/cardinality required, FULL_OUTER/FULL OUTER +# renamed to OUTER (semantics-preserving, never a loss). +# --------------------------------------------------------------------------- + +class TestJoinTypeRename: + """ThoughtSpot accepts only INNER, LEFT_OUTER, RIGHT_OUTER, OUTER for a + join `type` -- a stashed FULL_OUTER/"FULL OUTER" has to become OUTER + (OUTER *is* ThoughtSpot's own full outer join) or the generated document + is rejected on import. This is a rename, not a loss: no issue should be + raised for it, unlike every other rewrite this module performs. + """ + + def _built_join(self, join_type, log=None): + orders = _table_doc("orders", [_column("Customer Id", "CUSTOMER_ID", "INT64")]) + customers = _table_doc("customers", [_column("Id", "ID", "INT64")]) + orders_ds = _dataset("orders", "SALES.PUBLIC.ORDERS") + customers_ds = _dataset("customers", "SALES.PUBLIC.CUSTOMERS") + relationship = _relationship( + "orders_to_customers", "orders", "customers", ["Customer Id"], ["Id"], + rel_stash={RELATIONSHIP_STASH_TYPE: join_type, RELATIONSHIP_STASH_CARDINALITY: "MANY_TO_ONE"}, + ) + model = _semantic_model(datasets=[orders_ds, customers_ds], relationships=[relationship]) + doc = build_model(model, [orders, customers], log if log is not None else IssueLog()) + [orders_entry] = [t for t in doc.body["model_tables"] if t["name"] == "orders"] + [join] = orders_entry["joins"] + return join + + def test_full_outer_with_an_underscore_becomes_outer(self): + assert self._built_join("FULL_OUTER")["type"] == "OUTER" + + def test_full_outer_with_a_space_becomes_outer(self): + assert self._built_join("FULL OUTER")["type"] == "OUTER" + + def test_a_lowercase_full_outer_variant_also_becomes_outer(self): + assert self._built_join("full_outer")["type"] == "OUTER" + assert self._built_join("full outer")["type"] == "OUTER" + + def test_left_outer_passes_through_unchanged(self): + assert self._built_join("LEFT_OUTER")["type"] == "LEFT_OUTER" + + def test_the_rename_raises_no_issue_its_a_rename_not_a_loss(self): + log = IssueLog() + self._built_join("FULL_OUTER", log) + assert not log.as_dicts() + + def test_the_rename_also_applies_to_an_unrepresentable_joins_entry(self): + # The same rule governs every context this module emits a join + # `type` into -- unrepresentable_joins[] (a non-equality condition + # with no equality pair at all) is the other one. + orders = _table_doc("orders", [_column("Order Date", "ORDER_DATE", "DATE")]) + rates = _table_doc("fx_rates", [_column("Effective Date", "EFFECTIVE_DATE", "DATE")]) + orders_ds = _dataset("orders", "SALES.PUBLIC.ORDERS") + rates_ds = _dataset("fx_rates", "SALES.PUBLIC.FX_RATES") + model = _semantic_model( + datasets=[orders_ds, rates_ds], + model_stash={ + MODEL_STASH_UNREPRESENTABLE_JOINS: [{ + "from": "orders", "to": "fx_rates", + RELATIONSHIP_STASH_ON_EXPRESSION: "[orders::Order Date] >= [fx_rates::Effective Date]", + RELATIONSHIP_STASH_TYPE: "FULL_OUTER", + RELATIONSHIP_STASH_CARDINALITY: "MANY_TO_ONE", + }], + }, + ) + log = IssueLog() + + doc = build_model(model, [orders, rates], log) + + [orders_entry] = [t for t in doc.body["model_tables"] if t["name"] == "orders"] + [join] = orders_entry["joins"] + assert join["type"] == "OUTER" + assert not [i for i in log.as_dicts() if "FULL" in i["message"].upper()] + + def test_the_on_condition_key_is_quoted_and_survives_dump_and_reload(self): + # 'on' is a YAML 1.1 reserved word -- the generic YAML 1.2 codec + # (_yaml.py) is what actually has to quote it, since nothing in this + # module writes YAML text directly. Proven at the dump/reload + # boundary rather than trusted, because that is the only place this + # requirement can actually fail. + orders = _table_doc("orders", [_column("Customer Id", "CUSTOMER_ID", "INT64")]) + customers = _table_doc("customers", [_column("Id", "ID", "INT64")]) + orders_ds = _dataset("orders", "SALES.PUBLIC.ORDERS") + customers_ds = _dataset("customers", "SALES.PUBLIC.CUSTOMERS") + relationship = _relationship( + "orders_to_customers", "orders", "customers", ["Customer Id"], ["Id"], + ) + model = _semantic_model(datasets=[orders_ds, customers_ds], relationships=[relationship]) + + doc = build_model(model, [orders, customers], IssueLog()) + text = dump_document(doc) + + assert "'on':" in text + reloaded = load_document(text) + [orders_entry] = [t for t in reloaded.body["model_tables"] if t["name"] == "orders"] + [join] = orders_entry["joins"] + assert join["on"] == "[orders::Customer Id] = [customers::Id]" + + +class TestOwnTests: + def test_an_unused_primary_key_raises_an_issue_naming_the_dataset(self): + orders = _table_doc("orders", [_column("Amount", "AMOUNT", "DOUBLE")]) + dataset = _dataset( + "orders", "SALES.PUBLIC.ORDERS", fields=[ + _field("amount", _dialects(("THOUGHTSPOT", "[orders::Amount]")), label="Amount"), + ], + primary_key=["ORDER_ID"], + ) + model = _semantic_model(datasets=[dataset]) + log = IssueLog() + + doc = build_model(model, [orders], log) + + assert "ORDER_ID" not in json.dumps(doc.body) + issues = [i for i in log.as_dicts() if i["code"] == "TS-MODEL-DATASET-KEY-UNUSED"] + assert len(issues) == 1 + assert "orders" in issues[0]["message"] + assert "ORDER_ID" in issues[0]["message"] + + def test_a_primary_key_covered_by_a_relationship_raises_no_issue(self): + orders = _table_doc("orders", [_column("Customer Id", "CUSTOMER_ID", "INT64")]) + customers = _table_doc("customers", [_column("Id", "ID", "INT64")]) + orders_ds = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[ + _field( + "customer_id", _dialects(("THOUGHTSPOT", "[orders::Customer Id]")), + label="Customer Id", + ), + ]) + customers_ds = _dataset( + "customers", "SALES.PUBLIC.CUSTOMERS", + fields=[_field("id", _dialects(("THOUGHTSPOT", "[customers::Id]")), label="Id")], + primary_key=["Id"], + ) + relationship = { + "name": "orders_to_customers", "from": "orders", "to": "customers", + "from_columns": ["Customer Id"], "to_columns": ["Id"], + } + model = _semantic_model( + datasets=[orders_ds, customers_ds], relationships=[relationship], + ) + log = IssueLog() + + build_model(model, [orders, customers], log) + + assert not [i for i in log.as_dicts() if i["code"] == "TS-MODEL-DATASET-KEY-UNUSED"] + + def test_a_dataset_with_no_matching_table_document_does_not_crash(self): + # `tables` is a caller-supplied Sequence, matched by name -- a + # dataset whose expected table_ref has no corresponding document in + # `tables` (a caller bug, a table that failed to build) must degrade + # to a loud, per-dataset issue, never an unhandled exception, and + # must not surface a field referencing an unvalidated column. + dataset = _dataset( + "orders", "SALES.PUBLIC.ORDERS", fields=[ + _field("amount", _dialects(("THOUGHTSPOT", "[orders::Amount]")), label="Amount"), + ], + ) + model = _semantic_model(datasets=[dataset]) + log = IssueLog() + + doc = build_model(model, [], log) # no table documents supplied at all + + columns, _formulas = _all_columns_and_formulas(doc.body) + assert columns == [] # the field could not be validated, so it is dropped + assert doc.body["model_tables"] == [{"name": "orders"}] # still named, best-effort + assert any(i["code"] == "TS-MODEL-TABLE-MISSING" for i in log.as_dicts()) + assert any(i["code"] == "TS-MODEL-COLUMN-ID-MISSING" for i in log.as_dicts()) diff --git a/converters/thoughtspot/tests/test_ossie_to_thoughtspot_tables.py b/converters/thoughtspot/tests/test_ossie_to_thoughtspot_tables.py new file mode 100644 index 00000000..85f104d4 --- /dev/null +++ b/converters/thoughtspot/tests/test_ossie_to_thoughtspot_tables.py @@ -0,0 +1,746 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for `build_table`: one Ossie dataset -> one Table or SQL View document. + +Fixtures build raw Ossie dataset/field dicts directly, the same way +test_tml_to_ossie_fields.py builds raw TML column dicts -- `build_table`'s +input contract is the dataset dict, not any particular document it came from. +""" +import json + +import pytest + +from ossie_thoughtspot.constants import ( + DATASET_STASH_CONNECTION_NAME, + DATASET_STASH_SOURCE_PARTS, + DATASET_STASH_SOURCE_PARTS_DB, + DATASET_STASH_SOURCE_PARTS_DB_TABLE, + DATASET_STASH_SOURCE_PARTS_SCHEMA, + DATASET_STASH_SQL_OUTPUT_COLUMNS, + DATASET_STASH_TML_OBJECT, + DATASET_STASH_TML_OBJECT_WITNESS, + DATASET_STASH_UNSURFACED_COLUMNS, + FIELD_STASH_DATA_TYPE, + FIELD_STASH_DATA_TYPE_WITNESS, + FIELD_STASH_DB_COLUMN_NAME, + FIELD_STASH_DB_COLUMN_NAME_WITNESS, +) +from ossie_thoughtspot.issues import IssueLog +from ossie_thoughtspot.ossie_to_thoughtspot import build_table +from ossie_thoughtspot.tml import DocumentSet, TmlDocument, dump_document, load_document +from ossie_thoughtspot.tml_to_ossie import convert as tml_to_ossie_convert + + +def _dump(document): + return dump_document(document) + + +def _stash(**payload): + return [{"vendor_name": "THOUGHTSPOT", "data": json.dumps({"_v": 1, **payload})}] + + +def _field(name, expression, *, label=None, datatype=None, description=None, field_stash=None): + field: dict = {"name": name} + if label is not None: + field["label"] = label + field["expression"] = {"dialects": expression} + if datatype is not None: + field["datatype"] = datatype + if description is not None: + field["description"] = description + if field_stash is not None: + field["custom_extensions"] = _stash(**field_stash) + return field + + +def _physical(name, identifier=None, **kwargs): + """A field whose expression is a single bare SQL identifier -- the + hand-authored shape of a physical column.""" + return _field(name, [{"dialect": "ANSI_SQL", "expression": identifier or name}], **kwargs) + + +def _round_tripped_physical(name, table, column, **kwargs): + """A field whose expression is the THOUGHTSPOT-dialect verbatim bracket + reference a prior TML -> Ossie trip would have produced.""" + return _field(name, [{"dialect": "THOUGHTSPOT", "expression": f"[{table}::{column}]"}], **kwargs) + + +def _computed(name, expr, **kwargs): + return _field(name, [{"dialect": "THOUGHTSPOT", "expression": expr}], **kwargs) + + +def _dataset(name, source, fields=None, *, description=None, dataset_stash=None): + dataset: dict = {"name": name, "source": source} + if fields is not None: + dataset["fields"] = fields + if description is not None: + dataset["description"] = description + if dataset_stash is not None: + dataset["custom_extensions"] = _stash(**dataset_stash) + return dataset + + +class TestDbColumnName: + def test_every_column_carries_db_column_name(self): + dataset = _dataset( + "orders", "SALES.PUBLIC.ORDERS", + fields=[_physical("order_date"), _physical("amount", "o_amount")], + dataset_stash={DATASET_STASH_CONNECTION_NAME: "My Snowflake"}, + ) + table = build_table(dataset, IssueLog()) + assert table.kind == "table" + columns = table.body["columns"] + assert len(columns) == 2 + for column in columns: + assert "db_column_name" in column + assert columns[0] == { + "name": "order_date", + "db_column_name": "order_date", + "db_column_properties": {"data_type": "INT64"}, + } + assert columns[1]["db_column_name"] == "o_amount" + + def test_db_column_name_is_present_even_when_equal_to_name(self): + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[_physical("status")]) + table = build_table(dataset, IssueLog()) + column = table.body["columns"][0] + assert column["name"] == column["db_column_name"] == "status" + + def test_a_round_tripped_bracket_reference_supplies_db_column_name(self): + # A prior TML -> Ossie trip leaves the table's own physical column + # *display* name inside the verbatim THOUGHTSPOT bracket, not in + # label/name -- label/name are the Model's own display name, which + # a computed field's surfacing column can set independently. + field = _round_tripped_physical("order_date", "ORDERS", "O_ORDERDATE", label="Order Date") + dataset = _dataset("ORDERS", "SALES.PUBLIC.ORDERS", fields=[field]) + log = IssueLog() + table = build_table(dataset, log) + column = table.body["columns"][0] + assert column["name"] == "O_ORDERDATE" + assert column["db_column_name"] == "O_ORDERDATE" + # The default is reported, not silent -- see + # TestRoundTripAgainstTheForwardDirection for the case where it is + # wrong (the display name and the true db_column_name differed). + assert any(i["code"] == "TS-FIELD-DB-COLUMN-NAME-ASSUMED" for i in log.as_dicts()) + + def test_a_stashed_db_column_name_is_preferred_over_the_bracket_display_name(self): + # The bracket names the table's display name ("Order Date"); the + # field's own stash carries the true warehouse name separately when + # the forward direction saw the two differ, and that value wins -- + # no assumption, no issue -- as long as its witness (the display + # name it was recorded for) still matches. + field = _round_tripped_physical( + "order_date", "ORDERS", "Order Date", + field_stash={ + FIELD_STASH_DB_COLUMN_NAME: "O_ORDERDATE", + FIELD_STASH_DB_COLUMN_NAME_WITNESS: "Order Date", + }, + ) + dataset = _dataset("ORDERS", "SALES.PUBLIC.ORDERS", fields=[field]) + log = IssueLog() + table = build_table(dataset, log) + column = table.body["columns"][0] + assert column["name"] == "Order Date" + assert column["db_column_name"] == "O_ORDERDATE" + assert not [i for i in log.as_dicts() if i["code"] == "TS-FIELD-DB-COLUMN-NAME-ASSUMED"] + assert not [i for i in log.as_dicts() if i["code"] == "TS-FIELD-DB-COLUMN-NAME-STALE"] + + def test_a_stashed_db_column_name_whose_witness_no_longer_matches_is_dropped(self): + # The field was retargeted to a different physical column since the + # stash was written (Amount -> Total Amount, the exact scenario a + # retargeted reference produces) -- the stashed warehouse name + # describes the OLD column and must not be applied to the new one. + field = _round_tripped_physical( + "amount", "ORDERS", "Total Amount", + field_stash={ + FIELD_STASH_DB_COLUMN_NAME: "O_AMOUNT", + FIELD_STASH_DB_COLUMN_NAME_WITNESS: "Amount", + }, + ) + dataset = _dataset("ORDERS", "SALES.PUBLIC.ORDERS", fields=[field]) + log = IssueLog() + table = build_table(dataset, log) + column = table.body["columns"][0] + assert column["name"] == "Total Amount" + assert column["db_column_name"] == "Total Amount" + assert any(i["code"] == "TS-FIELD-DB-COLUMN-NAME-STALE" for i in log.as_dicts()) + + +class TestAmbiguousColumnReference: + """A THOUGHTSPOT-dialect bracket whose table or column part itself + contains "::" is genuinely ambiguous -- `split_column_ref` correctly + refuses to guess which "::" is the real delimiter rather than silently + mis-splitting one. That refusal must surface as a reported issue, not + an uncaught exception out of `build_table`.""" + + def test_an_ambiguous_bracket_is_reported_and_the_column_is_omitted(self): + # format_column_ref("A::B", "y") produces "[A::B::y]" -- two + # non-overlapping "::" delimiters, so identifiers.split_column_ref + # raises rather than picking one. + field = _round_tripped_physical("x", "A::B", "y") + dataset = _dataset("A::B", "SALES.PUBLIC.WIDGETS", fields=[field]) + log = IssueLog() + table = build_table(dataset, log) + assert table.body["columns"] == [] + issues = [i for i in log.as_dicts() if i["code"] == "TS-FIELD-COLUMN-REF-MALFORMED"] + assert len(issues) == 1 + assert "[A::B::y]" in issues[0]["message"] + + def test_an_empty_table_part_is_reported_the_same_way(self): + # format_column_ref("", "y") produces "[::y]" -- the bracket body + # never matches the [TABLE::Column] shape at all (no non-empty table + # part before a "::"), a different `split_column_ref` failure from + # the genuinely ambiguous case above, caught and reported the same + # way. + field = _round_tripped_physical("x", "", "y") + dataset = _dataset("", "SALES.PUBLIC.WIDGETS", fields=[field]) + log = IssueLog() + table = build_table(dataset, log) + assert table.body["columns"] == [] + issues = [i for i in log.as_dicts() if i["code"] == "TS-FIELD-COLUMN-REF-MALFORMED"] + assert len(issues) == 1 + assert "[::y]" in issues[0]["message"] + + +class TestDataTypeCompulsory: + def test_a_datatype_less_field_still_gets_a_data_type(self): + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[_physical("note")]) + log = IssueLog() + table = build_table(dataset, log) + assert table.body["columns"][0]["db_column_properties"] == {"data_type": "INT64"} + + def test_a_declared_datatype_maps_through(self): + dataset = _dataset( + "orders", "SALES.PUBLIC.ORDERS", fields=[_physical("amount", datatype="Integer")] + ) + table = build_table(dataset, IssueLog()) + assert table.body["columns"][0]["db_column_properties"]["data_type"] == "INT64" + + +class TestDeclaredLoss: + @pytest.mark.parametrize("datatype", ["Float", "Time", "DateTimeTz", "Opaque"]) + def test_each_declared_loss_datatype_raises_an_issue_naming_the_loss(self, datatype): + dataset = _dataset( + "orders", "SALES.PUBLIC.ORDERS", fields=[_physical("field_a", datatype=datatype)] + ) + log = IssueLog() + build_table(dataset, log) + issues = [i for i in log.as_dicts() if i["code"] == "TS-FIELD-DATATYPE-DECLARED-LOSS"] + assert len(issues) == 1 + assert datatype in issues[0]["message"] + + def test_a_lossless_datatype_raises_no_declared_loss_issue(self): + dataset = _dataset( + "orders", "SALES.PUBLIC.ORDERS", fields=[_physical("amount", datatype="Integer")] + ) + log = IssueLog() + build_table(dataset, log) + assert not [i for i in log.as_dicts() if i["code"] == "TS-FIELD-DATATYPE-DECLARED-LOSS"] + + +class TestSourceSplitting: + def test_a_three_part_source_splits_correctly(self): + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + log = IssueLog() + table = build_table(dataset, log) + assert table.kind == "table" + assert table.body["db"] == "SALES" + assert table.body["schema"] == "PUBLIC" + assert table.body["db_table"] == "ORDERS" + assert not [i for i in log.as_dicts() if "SOURCE" in i["code"]] + + @pytest.mark.parametrize("source", ["SALES.ORDERS", "ORDERS"]) + def test_a_two_or_one_part_source_raises_an_issue_rather_than_a_malformed_table(self, source): + dataset = _dataset("orders", source) + log = IssueLog() + table = build_table(dataset, log) + assert table.kind == "table" + assert any(i["code"] == "TS-DATASET-SOURCE-MALFORMED" for i in log.as_dicts()) + + def test_a_query_source_produces_a_sql_view_document(self): + dataset = _dataset("recent_orders", "SELECT * FROM orders WHERE recent = true") + log = IssueLog() + table = build_table(dataset, log) + assert table.kind == "sql_view" + assert table.body["sql_query"] == "SELECT * FROM orders WHERE recent = true" + assert not [i for i in log.as_dicts() if "SOURCE" in i["code"]] + + def test_a_stashed_tml_object_overrides_a_looks_like_a_query_source(self): + # A query that happens to be stored under a stashed sql_view kind + # must not be re-classified by the whitespace heuristic. The witness + # (the source it was stashed against) still matches, so the stash + # wins even though this particular source's derived kind agrees + # anyway -- see the next two tests for cases where it does not. + source = "SELECT * FROM orders" + dataset = _dataset( + "recent_orders", source, + dataset_stash={ + DATASET_STASH_TML_OBJECT: "sql_view", + DATASET_STASH_TML_OBJECT_WITNESS: source, + }, + ) + table = build_table(dataset, IssueLog()) + assert table.kind == "sql_view" + + def test_a_matching_witness_prefers_the_stash_over_a_disagreeing_derivation(self): + # The source LOOKS like a plain table reference (_derive_kind would + # call it "table"), but the stash says this dataset came from a + # sql_view -- and its witness still matches the live source, so the + # stash wins despite disagreeing with the heuristic. + source = "SALES.PUBLIC.ORDERS" + dataset = _dataset( + "orders", source, + dataset_stash={ + DATASET_STASH_TML_OBJECT: "sql_view", + DATASET_STASH_TML_OBJECT_WITNESS: source, + }, + ) + log = IssueLog() + table = build_table(dataset, log) + assert table.kind == "sql_view" + assert not any(i["code"] == "TS-DATASET-TML-OBJECT-STALE" for i in log.as_dicts()) + + def test_a_stale_tml_object_witness_is_dropped_and_the_kind_re_derived(self): + # The dataset's source has moved on since the stash was written (a + # query rewritten into a table reference) -- reusing the stale kind + # would silently misread the new source under the old rules. + dataset = _dataset( + "orders", "SALES.PUBLIC.ORDERS", + dataset_stash={ + DATASET_STASH_TML_OBJECT: "sql_view", + DATASET_STASH_TML_OBJECT_WITNESS: "SELECT * FROM orders", + }, + ) + log = IssueLog() + table = build_table(dataset, log) + assert table.kind == "table" + assert any(i["code"] == "TS-DATASET-TML-OBJECT-STALE" for i in log.as_dicts()) + + def test_a_stashed_source_parts_entry_is_used_when_it_still_agrees(self): + dataset = _dataset( + "orders", "SALES.PUBLIC.ORDERS", + dataset_stash={ + DATASET_STASH_SOURCE_PARTS: { + DATASET_STASH_SOURCE_PARTS_DB: "SALES", + DATASET_STASH_SOURCE_PARTS_SCHEMA: "PUBLIC", + DATASET_STASH_SOURCE_PARTS_DB_TABLE: "ORDERS", + } + }, + ) + table = build_table(dataset, IssueLog()) + assert (table.body["db"], table.body["schema"], table.body["db_table"]) == ( + "SALES", "PUBLIC", "ORDERS", + ) + + def test_a_stale_stashed_source_parts_entry_is_dropped_and_re_derived(self): + # The dataset's source has moved on since the stash was written -- + # reusing the stale parts would silently discard the edit. + dataset = _dataset( + "orders", "SALES.PUBLIC.RENAMED_ORDERS", + dataset_stash={ + DATASET_STASH_SOURCE_PARTS: { + DATASET_STASH_SOURCE_PARTS_DB: "SALES", + DATASET_STASH_SOURCE_PARTS_SCHEMA: "PUBLIC", + DATASET_STASH_SOURCE_PARTS_DB_TABLE: "ORDERS", + } + }, + ) + log = IssueLog() + table = build_table(dataset, log) + assert table.body["db_table"] == "RENAMED_ORDERS" + assert any(i["code"] == "TS-DATASET-SOURCE-PARTS-STALE" for i in log.as_dicts()) + + +class TestQuotedIdentifierIsNotMisreadAsAQuery: + """A quoted identifier segment may legitimately contain whitespace + (`"ORDER TABLE"`) -- classifying a source by "contains whitespace" + alone would misread it as a query and emit an unimportable sql_view + document with the whole dotted string as its query, with no issue to + say so.""" + + def test_a_quoted_identifier_with_a_space_is_still_a_table(self): + dataset = _dataset("orders", 'SALES.PUBLIC."ORDER TABLE"') + log = IssueLog() + table = build_table(dataset, log) + assert table.kind == "table" + assert (table.body["db"], table.body["schema"], table.body["db_table"]) == ( + "SALES", "PUBLIC", "ORDER TABLE", + ) + assert not [i for i in log.as_dicts() if "SOURCE" in i["code"]] + + def test_an_ordinary_three_part_name_is_unaffected(self): + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + table = build_table(dataset, IssueLog()) + assert table.kind == "table" + assert table.body["db_table"] == "ORDERS" + + def test_a_genuine_query_is_still_a_sql_view(self): + dataset = _dataset("recent_orders", "SELECT * FROM orders WHERE recent = true") + table = build_table(dataset, IssueLog()) + assert table.kind == "sql_view" + + +class TestConnectionDependentSpelling: + def test_boolean_spelling_is_taken_from_the_stash_when_present(self): + field = _physical( + "is_active", datatype="Boolean", + field_stash={FIELD_STASH_DATA_TYPE: "BOOL", FIELD_STASH_DATA_TYPE_WITNESS: "Boolean"}, + ) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[field]) + table = build_table(dataset, IssueLog()) + assert table.body["columns"][0]["db_column_properties"]["data_type"] == "BOOL" + + def test_boolean_spelling_defaults_when_no_stash_is_present(self): + field = _physical("is_active", datatype="Boolean") + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[field]) + table = build_table(dataset, IssueLog()) + assert table.body["columns"][0]["db_column_properties"]["data_type"] == "BOOLEAN" + + def test_float_spelling_is_taken_from_the_stash_when_present(self): + field = _physical( + "weight", datatype="Float", + field_stash={FIELD_STASH_DATA_TYPE: "FLOAT", FIELD_STASH_DATA_TYPE_WITNESS: "Float"}, + ) + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[field]) + log = IssueLog() + table = build_table(dataset, log) + assert table.body["columns"][0]["db_column_properties"]["data_type"] == "FLOAT" + # Still a declared loss -- the stash only fixes the spelling, not the + # Float/Decimal collapse itself. + assert any(i["code"] == "TS-FIELD-DATATYPE-DECLARED-LOSS" for i in log.as_dicts()) + + def test_float_spelling_defaults_when_no_stash_is_present(self): + field = _physical("weight", datatype="Float") + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[field]) + table = build_table(dataset, IssueLog()) + assert table.body["columns"][0]["db_column_properties"]["data_type"] == "DOUBLE" + + +class TestReload: + def test_the_emitted_table_document_reloads(self): + dataset = _dataset( + "orders", "SALES.PUBLIC.ORDERS", + fields=[_physical("order_date"), _physical("amount")], + dataset_stash={DATASET_STASH_CONNECTION_NAME: "My Snowflake"}, + ) + table = build_table(dataset, IssueLog()) + reloaded = load_document(_dump(table)) + assert reloaded.kind == "table" + assert reloaded.body["name"] == dataset["name"] + + def test_the_emitted_sql_view_document_reloads(self): + dataset = _dataset( + "recent_orders", "SELECT * FROM orders", + fields=[_physical("order_id")], + dataset_stash={DATASET_STASH_CONNECTION_NAME: "My Snowflake"}, + ) + table = build_table(dataset, IssueLog()) + reloaded = load_document(_dump(table)) + assert reloaded.kind == "sql_view" + assert reloaded.body["sql_query"] == "SELECT * FROM orders" + + +class TestConnectionFallback: + def test_a_connection_name_argument_is_used_when_nothing_is_stashed(self): + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + table = build_table(dataset, IssueLog(), connection_name="Fallback Connection") + assert table.body["connection"] == {"name": "Fallback Connection"} + + def test_a_stashed_connection_name_wins_over_the_argument(self): + dataset = _dataset( + "orders", "SALES.PUBLIC.ORDERS", dataset_stash={DATASET_STASH_CONNECTION_NAME: "Stashed Connection"} + ) + table = build_table(dataset, IssueLog(), connection_name="Fallback Connection") + assert table.body["connection"] == {"name": "Stashed Connection"} + + def test_no_connection_at_all_omits_the_block_and_raises_an_issue(self): + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS") + log = IssueLog() + table = build_table(dataset, log) + assert "connection" not in table.body + assert any(i["code"] == "TS-DATASET-CONNECTION-MISSING" for i in log.as_dicts()) + + +# --------------------------------------------------------------------------- +# Own tests: the two shapes judged most likely to hide a real bug. +# --------------------------------------------------------------------------- + +class TestComputedFieldsAreNotMisreadAsColumns: + """A dataset with a genuine computed field mixed in among physical ones is + exactly the input a Model-building step will hand this module in practice + (a dataset's Ossie fields are not pre-sorted into physical vs. computed). + Getting this wrong either drops a physical column or invents a column + for a formula -- both produce a Table document a Model can silently + reference incorrectly, one document kind an isolated single-field test + would never exercise.""" + + def test_a_computed_field_produces_no_table_column(self): + physical = _physical("amount") + computed = _computed("net_amount", "[ORDERS::amount] - [ORDERS::cost]") + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[physical, computed]) + table = build_table(dataset, IssueLog()) + names = {c["name"] for c in table.body["columns"]} + assert names == {"amount"} + + def test_a_thoughtspot_only_aggregate_expression_is_also_skipped(self): + # The THOUGHTSPOT dialect is present but is a function call, not a + # bare reference -- must not be mistaken for a bracketed physical + # column just because a bracket appears somewhere inside it. + computed = _computed("total", "sum ( [ORDERS::amount] )") + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[computed]) + table = build_table(dataset, IssueLog()) + assert table.body["columns"] == [] + + +class TestSqlViewColumnsUseOutputAliasNotDbColumnName: + """SQL View columns bind to a query output alias (`sql_output_column`), + never `db_column_name` -- a prior task shipped with SQL views completely + broken while every existing test used a table document, because nothing + exercised the SQL View column shape at all. These tests exist so that + failure mode cannot repeat silently here.""" + + def test_sql_view_columns_carry_sql_output_column_not_db_column_name(self): + field = _physical("customer_id", "cust_id_out") + dataset = _dataset("recent_orders", "SELECT cust_id_out FROM orders", fields=[field]) + table = build_table(dataset, IssueLog()) + column = table.body["sql_view_columns"][0] + assert column["sql_output_column"] == "cust_id_out" + assert "db_column_name" not in column + + def test_a_stashed_output_alias_wins_over_the_expressions_own_identifier(self): + field = _round_tripped_physical("customer_id", "recent_orders", "customer_id") + dataset = _dataset( + "recent_orders", "SELECT cust_id_out AS customer_id FROM orders", + fields=[field], + dataset_stash={ + DATASET_STASH_TML_OBJECT: "sql_view", + DATASET_STASH_SQL_OUTPUT_COLUMNS: {"customer_id": "cust_id_out"}, + }, + ) + table = build_table(dataset, IssueLog()) + column = table.body["sql_view_columns"][0] + assert column["sql_output_column"] == "cust_id_out" + + def test_unsurfaced_columns_land_in_sql_view_columns_not_columns(self): + dataset = _dataset( + "recent_orders", "SELECT a, b FROM orders", + dataset_stash={ + DATASET_STASH_TML_OBJECT: "sql_view", + DATASET_STASH_UNSURFACED_COLUMNS: [ + {"name": "b", "sql_output_column": "b", + "db_column_properties": {"data_type": "VARCHAR"}} + ], + }, + ) + table = build_table(dataset, IssueLog()) + assert "columns" not in table.body + assert table.body["sql_view_columns"] == [ + {"name": "b", "sql_output_column": "b", "db_column_properties": {"data_type": "VARCHAR"}} + ] + + +class TestUnsurfacedColumns: + def test_unsurfaced_table_columns_are_restored_verbatim(self): + dataset = _dataset( + "orders", "SALES.PUBLIC.ORDERS", + fields=[_physical("amount")], + dataset_stash={ + DATASET_STASH_UNSURFACED_COLUMNS: [ + {"name": "internal_flag", "db_column_name": "INTERNAL_FLAG", + "db_column_properties": {"data_type": "BOOLEAN"}} + ] + }, + ) + table = build_table(dataset, IssueLog()) + names = [c["name"] for c in table.body["columns"]] + assert names == ["amount", "internal_flag"] + + def test_a_field_retargeted_onto_a_previously_unsurfaced_column_is_not_duplicated(self): + # "Total Amount" was unsurfaced when the stash was written. The + # field was then retargeted onto it ([ORDERS::Amount] -> + # [ORDERS::Total Amount]) -- it is surfaced now, so blindly + # restoring the stale unsurfaced_columns entry would emit it twice + # under the same display name, which does not import. + field = _round_tripped_physical( + "amount", "ORDERS", "Total Amount", + field_stash={ + FIELD_STASH_DB_COLUMN_NAME: "O_AMOUNT", + FIELD_STASH_DB_COLUMN_NAME_WITNESS: "Amount", + }, + ) + dataset = _dataset( + "ORDERS", "SALES.PUBLIC.ORDERS", + fields=[field], + dataset_stash={ + DATASET_STASH_UNSURFACED_COLUMNS: [ + {"name": "Total Amount", "db_column_name": "O_TOTAL_AMOUNT", + "db_column_properties": {"data_type": "DOUBLE"}}, + ], + }, + ) + log = IssueLog() + table = build_table(dataset, log) + names = [c["name"] for c in table.body["columns"]] + assert names == ["Total Amount"] + assert len(names) == len(set(names)) + # The field itself was still retargeted (a real edit, correctly + # reported) -- only the now-redundant unsurfaced duplicate is + # dropped, and that drop is silent: nothing was lost, so there is + # nothing to name in a SECOND issue about it. + codes = [i["code"] for i in log.as_dicts()] + assert codes.count("TS-FIELD-DB-COLUMN-NAME-STALE") == 1 + assert not any("unsurfaced" in i["message"].lower() for i in log.as_dicts()) + + def test_an_unrelated_unsurfaced_column_is_unaffected_by_a_retarget_elsewhere(self): + # A collision on ONE column must not suppress an unrelated + # unsurfaced column that genuinely still has no live field. + field = _round_tripped_physical( + "amount", "ORDERS", "Total Amount", + field_stash={ + FIELD_STASH_DB_COLUMN_NAME: "O_AMOUNT", + FIELD_STASH_DB_COLUMN_NAME_WITNESS: "Amount", + }, + ) + dataset = _dataset( + "ORDERS", "SALES.PUBLIC.ORDERS", + fields=[field], + dataset_stash={ + DATASET_STASH_UNSURFACED_COLUMNS: [ + {"name": "Total Amount", "db_column_name": "O_TOTAL_AMOUNT", + "db_column_properties": {"data_type": "DOUBLE"}}, + {"name": "Internal Flag", "db_column_name": "INTERNAL_FLAG", + "db_column_properties": {"data_type": "BOOLEAN"}}, + ], + }, + ) + table = build_table(dataset, IssueLog()) + names = [c["name"] for c in table.body["columns"]] + assert names == ["Total Amount", "Internal Flag"] + + +class TestDatasetAiContextHasNoHomeInTml: + """Table TML (both kinds) has no synonym or instruction field at all -- + this loss is unconditional, so it must always be reported, not only when + some other recovery path happens to fail.""" + + def test_a_string_ai_context_raises_an_issue(self): + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[_physical("amount")]) + dataset["ai_context"] = "Use this table for revenue questions." + log = IssueLog() + build_table(dataset, log) + assert any(i["code"] == "TS-DATASET-AI-CONTEXT-UNSUPPORTED" for i in log.as_dicts()) + + def test_an_object_ai_context_also_raises_an_issue(self): + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[_physical("amount")]) + dataset["ai_context"] = {"synonyms": ["sales"]} + log = IssueLog() + build_table(dataset, log) + assert any(i["code"] == "TS-DATASET-AI-CONTEXT-UNSUPPORTED" for i in log.as_dicts()) + + def test_no_ai_context_raises_nothing(self): + dataset = _dataset("orders", "SALES.PUBLIC.ORDERS", fields=[_physical("amount")]) + log = IssueLog() + build_table(dataset, log) + assert not [i for i in log.as_dicts() if i["code"] == "TS-DATASET-AI-CONTEXT-UNSUPPORTED"] + + +class TestRoundTripAgainstTheForwardDirection: + """Feed a real TML Table document through the forward direction and back + through `build_table`, and compare against the original -- the strongest + check available, because a unit test built from a hand-written Ossie + fixture can be unknowingly wrong about what the forward direction + actually produces.""" + + def _model_and_table(self): + table_doc = TmlDocument( + kind="table", + body={ + "name": "ORDERS", + "db": "SALES", "schema": "PUBLIC", "db_table": "ORDERS", + "connection": {"name": "My Snowflake"}, + "columns": [ + {"name": "Order Date", "db_column_name": "O_ORDERDATE", + "db_column_properties": {"data_type": "DATE"}}, + {"name": "Amount", "db_column_name": "O_AMOUNT", + "db_column_properties": {"data_type": "DOUBLE"}}, + {"name": "Is Priority", "db_column_name": "O_IS_PRIORITY", + "db_column_properties": {"data_type": "BOOL"}}, + {"name": "Internal Note", "db_column_name": "O_NOTE", + "db_column_properties": {"data_type": "VARCHAR"}}, + ], + }, + guid=None, + ) + model_doc = TmlDocument( + kind="model", + body={ + "name": "Sales Analytics", + "model_tables": [{"name": "ORDERS"}], + "columns": [ + {"name": "Order Date", "column_id": "ORDERS::Order Date", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "Amount", "column_id": "ORDERS::Amount", + "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "Is Priority", "column_id": "ORDERS::Is Priority", + "properties": {"column_type": "ATTRIBUTE"}}, + # "Internal Note" is deliberately not surfaced -- it must + # come back as an unsurfaced column, not a field. + ], + }, + guid=None, + ) + return DocumentSet(model=model_doc, tables=(table_doc,)), table_doc + + def test_round_trip_reproduces_the_original_table_structurally(self): + document_set, original = self._model_and_table() + ossie = tml_to_ossie_convert(document_set) + [dataset] = ossie.model["semantic_model"][0]["datasets"] + + log = IssueLog() + rebuilt = build_table(dataset, log) + + assert rebuilt.kind == "table" + assert rebuilt.body["name"] == original.body["name"] + assert rebuilt.body["db"] == original.body["db"] + assert rebuilt.body["schema"] == original.body["schema"] + assert rebuilt.body["db_table"] == original.body["db_table"] + assert rebuilt.body["connection"] == original.body["connection"] + + by_name = {c["name"]: c for c in rebuilt.body["columns"]} + assert set(by_name) == {c["name"] for c in original.body["columns"]} + for original_column in original.body["columns"]: + rebuilt_column = by_name[original_column["name"]] + assert rebuilt_column["db_column_properties"]["data_type"] == ( + original_column["db_column_properties"]["data_type"] + ) + + # "Internal Note" was never surfaced by the Model, so it round-trips + # verbatim through unsurfaced_columns, db_column_name included. + assert by_name["Internal Note"]["db_column_name"] == "O_NOTE" + + # "Order Date", "Amount" and "Is Priority" WERE surfaced, and each + # has a display name that differs from its true db_column_name. The + # forward direction stashes that true name separately for exactly + # this case, and it round-trips exactly rather than falling back to + # the display-name assumption. A hand-written fixture that happens + # to set bracket-name equal to db_column_name would never expose a + # regression here; only a real forward-then-reverse round trip does. + assert by_name["Order Date"]["db_column_name"] == "O_ORDERDATE" + assert by_name["Amount"]["db_column_name"] == "O_AMOUNT" + assert by_name["Is Priority"]["db_column_name"] == "O_IS_PRIORITY" + assert not [i for i in log.as_dicts() if i["code"] == "TS-FIELD-DB-COLUMN-NAME-ASSUMED"] diff --git a/converters/thoughtspot/tests/test_packaging.py b/converters/thoughtspot/tests/test_packaging.py new file mode 100644 index 00000000..3351e1de --- /dev/null +++ b/converters/thoughtspot/tests/test_packaging.py @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Packaging invariants that upstream's PR checklist gates.""" +from pathlib import Path + +import ossie_thoughtspot + +ROOT = Path(__file__).resolve().parents[1] +LICENSE_MARKER = "Licensed to the Apache Software Foundation (ASF)" + + +def test_package_exposes_a_version(): + assert ossie_thoughtspot.__version__ == "0.1.0" + + +def test_every_source_file_carries_the_asf_header(): + sources = [ + *(ROOT / "src").rglob("*.py"), + *(ROOT / "tests").rglob("*.py"), + ] + assert sources, "expected at least one source file" + missing = [ + str(p.relative_to(ROOT)) + for p in sources + if LICENSE_MARKER not in p.read_text(encoding="utf-8") + ] + assert missing == [], f"ASF header missing from: {missing}" + + +def test_non_python_packaging_files_carry_the_asf_header(): + # The glob above only covers src/**/*.py and tests/**/*.py, so + # pyproject.toml, .gitignore, README.md, and the CI workflow were + # ungated. Each uses a different comment syntax ('#', HTML comment, + # YAML '#'), so this checks for the licence text itself, not an exact + # comment-prefixed line. + repo_root = ROOT.parent.parent + files = { + "pyproject.toml": ROOT / "pyproject.toml", + ".gitignore": ROOT / ".gitignore", + "README.md": ROOT / "README.md", + "CI workflow": repo_root / ".github" / "workflows" / "converter-thoughtspot-ci.yml", + } + missing = [ + label for label, path in files.items() + if not path.is_file() or LICENSE_MARKER not in path.read_text(encoding="utf-8") + ] + assert missing == [], f"ASF header missing from: {missing}" diff --git a/converters/thoughtspot/tests/test_readme.py b/converters/thoughtspot/tests/test_readme.py new file mode 100644 index 00000000..498f8391 --- /dev/null +++ b/converters/thoughtspot/tests/test_readme.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import re +from pathlib import Path + +README = Path(__file__).resolve().parents[1] / "README.md" + + +def test_readme_declares_both_directions(): + text = README.read_text(encoding="utf-8") + assert "ThoughtSpot TML -> Ossie" in text or "ThoughtSpot TML → Ossie" in text + assert "Ossie -> ThoughtSpot TML" in text or "Ossie → ThoughtSpot TML" in text + + +def test_readme_carries_a_coverage_matrix_with_rows(): + # The coverage information must appear as a matrix, not a prose + # limitations list. + text = README.read_text(encoding="utf-8") + assert "## Coverage matrix" in text + body = text.split("## Coverage matrix", 1)[1] + rows = re.findall(r"^\| *L\d+ *\|", body, flags=re.MULTILINE) + assert len(rows) >= 1, "coverage matrix has no L-numbered limitation rows" + + +def test_readme_states_the_dialect_registration(): + # The THOUGHTSPOT dialect was registered by apache/ossie#351 (merged 2026-09-01). + assert "351" in README.read_text(encoding="utf-8") diff --git a/converters/thoughtspot/tests/test_reference_docs_current.py b/converters/thoughtspot/tests/test_reference_docs_current.py new file mode 100644 index 00000000..879e48c0 --- /dev/null +++ b/converters/thoughtspot/tests/test_reference_docs_current.py @@ -0,0 +1,127 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Guard: the committed `docs/*.md` reference documents must equal what +`tools/generate_reference_docs.py` produces right now. + +`docs/expression-mapping.md`, `docs/reverse-inventory.md`, `docs/datatype-map.md` +and `docs/vendor-payload.md` are generated, not hand-authored — the code +(`expressions/catalog.py`, `expressions/reverse.py`, `datatypes.py`, +`constants.py`) is the single source of truth for the mapping they describe. +Committing the generated output makes the reference readable on GitHub without +running anything; this test is what keeps a committed file from silently going +stale after the code it was generated from changes underneath it. + +**Comparison is byte-exact (full string equality), deliberately.** A softer +comparison — ignoring whitespace, or checking only that each row's data is +*present* somewhere in the file — would tolerate the generator's own Markdown +formatting drifting out of sync with what is actually committed, which defeats +the point: the committed file must be reproducible from a single, deterministic +command, not merely "close enough". The generator has no non-deterministic +inputs (no timestamps, no unsorted set/dict iteration — every set-derived +listing in `tools/generate_reference_docs.py` is explicitly `sorted()`), so +byte-exact does not mean flaky. The trade-off this accepts: a purely cosmetic +change to the generator's Markdown layout (e.g. column order, a reworded +banner) requires regenerating and committing `docs/*.md` in the same change, +even though no *fact* in the tables moved — this is treated as a feature, not +a cost: it is the same discipline `test_shipped_references.py` already applies +to every other shipped file, and it is exactly what proves this test can fail +at all (see the module for how that was verified). +""" +from __future__ import annotations + +import importlib.util +import types +from pathlib import Path + +import pytest + +PACKAGE_ROOT = Path(__file__).resolve().parents[1] +GENERATOR_PATH = PACKAGE_ROOT / "tools" / "generate_reference_docs.py" +DOCS_DIR = PACKAGE_ROOT / "docs" + + +def _load_generator() -> types.ModuleType: + """Load `tools/generate_reference_docs.py` by file path rather than + `import`. `tools/` is not listed in `pyproject.toml`'s wheel `packages` + and carries no `[project.scripts]` entry point, so nothing under `src/` + reaches it this way either — this loader exists only so *this test* can + reach it, the same way it would reach any other standalone script. + """ + spec = importlib.util.spec_from_file_location("generate_reference_docs", GENERATOR_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def generator() -> types.ModuleType: + return _load_generator() + + +def test_generator_is_not_part_of_the_installed_package() -> None: + """The generator must not be a runtime dependency (see its own docstring): + not part of the wheel this package ships, not a new entry in + `dependencies`, and not a console-script entry point. A plain substring + check on the raw file, not a full TOML parse — `tomllib` needs Python + 3.11+, and this package's floor is 3.10 (`requires-python = ">=3.10"`). + """ + text = (PACKAGE_ROOT / "pyproject.toml").read_text(encoding="utf-8") + assert 'packages = ["src/ossie_thoughtspot"]' in text, ( + "wheel packages line changed shape -- update this check, and confirm " + "'tools' was not added to it" + ) + assert 'dependencies = [\n "PyYAML>=6.0",\n]' in text, ( + "runtime dependencies changed shape -- confirm PyYAML is still the only one" + ) + assert "[project.scripts]" in text + scripts_block = text.split("[project.scripts]", 1)[1].split("\n\n", 1)[0] + assert "generate_reference_docs" not in scripts_block + assert "generate-reference-docs" not in scripts_block + + +def test_generator_produces_exactly_the_files_docs_contains(generator) -> None: + expected_names = set(generator.DOCS) + on_disk = {p.name for p in DOCS_DIR.glob("*.md")} + assert on_disk == expected_names, ( + f"docs/ and tools/generate_reference_docs.py's DOCS registry disagree on the " + f"file set -- on disk but not generated: {sorted(on_disk - expected_names)}; " + f"generated but not on disk: {sorted(expected_names - on_disk)}" + ) + + +def test_committed_docs_match_generator_output_byte_for_byte(generator) -> None: + mismatches: list[str] = [] + for name, content in generator.generate_all().items(): + path = DOCS_DIR / name + if not path.exists(): + mismatches.append(f"docs/{name}: missing from docs/") + continue + on_disk = path.read_text(encoding="utf-8") + if on_disk != content: + mismatches.append( + f"docs/{name}: committed content does not match the generator's " + f"current output ({len(on_disk)} bytes on disk vs {len(content)} " + "bytes generated)" + ) + assert mismatches == [], ( + "One or more generated reference documents are stale relative to the code " + "they are generated from. Regenerate and commit the result:\n" + " uv run --python 3.13 python tools/generate_reference_docs.py\n\n" + + "\n".join(mismatches) + ) diff --git a/converters/thoughtspot/tests/test_roundtrip.py b/converters/thoughtspot/tests/test_roundtrip.py new file mode 100644 index 00000000..e36fd27d --- /dev/null +++ b/converters/thoughtspot/tests/test_roundtrip.py @@ -0,0 +1,731 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Round-trip tests, in both directions, over both fixture sets. + +A round trip that only compares documents is a weaker test than it looks. The +return leg of `TML -> Ossie -> TML` reads the expression back out of the +THOUGHTSPOT dialect entry, which `tml_to_ossie.py` always writes verbatim -- +so the original formula text comes back byte for byte whether or not the +*portable* ANSI_SQL sibling next to it was ever built correctly. A round trip +that only asserts the documents match can pass while translation is entirely +broken, because preservation and translation are proved by two different +code paths that happen to feed the same comparison. So this module asserts +them separately: `TestTmlRoundTripReproduces*` proves preservation (the +document set comes back, expressions held to exact string equality); +`TestTmlRoundTripTranslat*` proves translation, directly on the ANSI_SQL +siblings the intermediate Ossie document carries -- these fail on a +translation regression even when every preservation assertion still passes. + +A round trip that only compares documents is also blind to the issue log, +and the issue log is this converter's whole contract with whoever reads it: +a conversion that silently drops something and one that reports the same +drop correctly produce identical documents. Every assertion below that +checks a real difference also checks that the issue log named it -- object +and all -- rather than trusting a bare count. + +Not every difference this module finds is a defect. `test_minimal_model_ +reproduces_every_column_and_formula_except_the_aggregation_convention` and +its tpcds counterpart document a difference the converter's own code +already explains and justifies -- collapsing three ThoughtSpot Model +TML metric shapes into one on the way out. That is asserted as the +current, intentional behaviour. + +Two further differences this module found while it was first written had +no such justification anywhere in the source, and were deliberately NOT +asserted as correct -- baking in a value nobody has justified is exactly +how a real regression gets permanently disguised as a passing test. Both +have since been fixed in `ossie_to_thoughtspot.py`, and this module now +asserts the fix directly instead of excluding the field it touches: a +physical Table column no longer gains a `description` its own document +never had (see `_columns_by_name` and the dedicated description test), and +a relationship whose join was Table-referenced keeps its own `name` and +its Table `joins_with[]` entry across the round trip, restored from the +stash rather than resynthesized -- with a currency check, so a relationship +renamed after the stash was written falls back to the old, safe behaviour +instead of restoring a stale reference under the wrong name (see +`TestReferencingJoinRestoration`). +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from ossie_thoughtspot import _yaml, datatypes, ossie_to_thoughtspot, stash, tml, tml_to_ossie +from ossie_thoughtspot.constants import ( + DATASET_STASH_CONNECTION_NAME, + DIALECT, + DOCUMENT_VERSION, + PORTABLE_DIALECT, + RELATIONSHIP_STASH_CARDINALITY, + RELATIONSHIP_STASH_TYPE, +) +from ossie_thoughtspot.datatypes import OSSIE_DATATYPES + +FIXTURES_ROOT = Path(__file__).resolve().parent / "fixtures" +FIXTURE_SETS = ("minimal", "tpcds") + + +# --------------------------------------------------------------------------- +# Loading helpers -- small, deliberate duplicates of test_fixtures.py's own +# (this module's fixture-loading needs are the same, but its assertions +# are a different concern and belong in a different file). +# --------------------------------------------------------------------------- + +def _tml_paths(fixture_dir: Path) -> list[Path]: + return sorted(fixture_dir.glob("*.tml")) + + +def _load_document_set(fixture_dir: Path) -> tml.DocumentSet: + texts = [(str(p), p.read_text(encoding="utf-8")) for p in _tml_paths(fixture_dir)] + return tml.load_document_set(texts) + + +def _load_expected(fixture_dir: Path) -> dict: + text = (fixture_dir / "expected.ossie.yaml").read_text(encoding="utf-8") + document = _yaml.load(text) + assert isinstance(document, dict) + return document + + +def _tml_roundtrip(fixture_name: str): + """`(document_set, ossie_result, tml_result)` for one fixture set's own + `TML -> Ossie -> TML` round trip. `ossie_result` is the intermediate + Ossie document -- the one translation is checked against -- and + `tml_result` is the returned TML document set -- the one preservation + is checked against. Every test in this module reads one or the other of + these, never a third, independently-converted copy of either.""" + document_set = _load_document_set(FIXTURES_ROOT / fixture_name) + ossie_result = tml_to_ossie.convert(document_set) + tml_result = ossie_to_thoughtspot.convert(ossie_result.model) + return document_set, ossie_result, tml_result + + +def _ossie_roundtrip(fixture_name: str): + """`(expected_ossie, tml_result, ossie_result)` for one fixture set's own + `Ossie -> TML -> Ossie` round trip, starting from its checked-in + `expected.ossie.yaml` rather than from the TML fixtures.""" + expected = _load_expected(FIXTURES_ROOT / fixture_name) + tml_result = ossie_to_thoughtspot.convert(expected) + ossie_result = tml_to_ossie.convert(tml_result.documents) + return expected, tml_result, ossie_result + + +def _table_by_name(document_set: tml.DocumentSet, name: str) -> tml.TmlDocument: + return next(t for t in document_set.tables if t.body.get("name") == name) + + +def _dataset(model: dict, name: str) -> dict: + return next(d for d in model["datasets"] if d["name"] == name) + + +def _field(dataset: dict, name: str) -> dict: + return next(f for f in dataset["fields"] if f["name"] == name) + + +def _metric(model: dict, name: str) -> dict: + return next(m for m in model["metrics"] if m["name"] == name) + + +def _dialects(obj: dict) -> dict[str, str]: + return {d["dialect"]: d["expression"] for d in obj["expression"]["dialects"]} + + +def _issue_refs(issues, code: str) -> set[str]: + return {i.object_ref for i in issues.issues if i.code == code} + + +def _columns_by_name(body: dict) -> dict[str, dict]: + """A Table/SQL-View document's physical columns, keyed by `name` -- + order-insensitive, content-exact (`description` included: a physical + column reaching `_physical_table_column`/`_physical_sql_view_column` is + always Model-surfaced, so it never carries one -- see + `test_a_model_surfaced_fields_description_is_never_duplicated_onto_its_ + physical_column` below). + + `_build_table_body`/`_build_sql_view_body` in ossie_to_thoughtspot.py + unconditionally emit a dataset's surfaced fields first and its still- + unsurfaced physical columns after (see those two functions): a fixed + policy, not a reflection of whatever order the source document + happened to list its own columns in. The `minimal` fixture's own + `customers` table lists its unsurfaced `customer_id` column before its + surfaced `customer_name` field; `orders` lists its surfaced column + first -- both legitimate authoring choices TML places no meaning on, so + comparing column list *order* would fail on a difference the converter + never claims to preserve. + """ + key = "sql_view_columns" if "sql_view_columns" in body else "columns" + return {c["name"]: c for c in body.get(key, [])} + + +def _model_columns_by_name(body: dict) -> dict[str, dict]: + return {c["name"]: c for c in body.get("columns") or []} + + +def _model_formulas_by_id(body: dict) -> dict[str, dict]: + return {f["id"]: f for f in body.get("formulas") or []} + + +def _relationship_identity(rel: dict) -> tuple: + return (rel["from"], rel["to"], tuple(rel["from_columns"]), tuple(rel["to_columns"])) + + +# --------------------------------------------------------------------------- +# TML -> Ossie -> TML: preservation. +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("fixture_name", FIXTURE_SETS) +class TestTmlRoundTripReproducesTableDocuments: + def test_every_table_or_sql_view_document_is_present(self, fixture_name): + document_set, _, tml_result = _tml_roundtrip(fixture_name) + original_names = {t.body["name"] for t in document_set.tables} + new_names = {t.body["name"] for t in tml_result.documents.tables} + assert new_names == original_names + + def test_every_physical_column_survives_with_its_exact_content(self, fixture_name): + document_set, _, tml_result = _tml_roundtrip(fixture_name) + for original in document_set.tables: + new = _table_by_name(tml_result.documents, original.body["name"]) + assert new.kind == original.kind + assert _columns_by_name(new.body) == _columns_by_name(original.body) + + def test_shared_table_level_attributes_survive(self, fixture_name): + document_set, _, tml_result = _tml_roundtrip(fixture_name) + for original in document_set.tables: + new = _table_by_name(tml_result.documents, original.body["name"]) + # "joins_with" included: a Table-referenced join's own + # joins_with[] entry -- name, destination, condition, type, + # cardinality -- is restored from the relationship's stash + # rather than dropped (see TestReferencingJoinRestoration). + for key in ("name", "db", "schema", "db_table", "sql_query", "connection", "joins_with"): + if key in original.body or key in new.body: + assert new.body.get(key) == original.body.get(key), (original.body["name"], key) + + +def test_minimal_model_reproduces_every_column_and_formula_except_the_aggregation_convention(): + document_set, _, tml_result = _tml_roundtrip("minimal") + original = document_set.model.body + new = tml_result.documents.model.body + assert new["name"] == original["name"] + assert new.get("description") == original.get("description") + # The relationship's own referencing_join name survives too, so + # `model_tables[].joins[]` -- {"referencing_join": "orders_to_customers"} + # -- comes back exactly, not resynthesized as an inline join. + assert new["model_tables"] == original["model_tables"] + + orig_columns = _model_columns_by_name(original) + new_columns = _model_columns_by_name(new) + assert set(new_columns) == set(orig_columns) + + # "total_order_amount"'s formula ("sum ( ... )") already aggregates, so + # _build_metric (ossie_to_thoughtspot.py) sets the surfacing column's own + # `aggregation` to match, as the convention a real ThoughtSpot-authored + # document carries -- a documented no-op over an already-aggregate + # expression, not a change to what the metric evaluates to. + for name in set(orig_columns) - {"total_order_amount"}: + assert new_columns[name] == orig_columns[name], name + assert "aggregation" not in orig_columns["total_order_amount"]["properties"] + assert new_columns["total_order_amount"]["properties"]["aggregation"] == "SUM" + assert _issue_refs(tml_result.issues, "TS-MODEL-METRIC-AGGREGATION-CONVENTION") == { + "metric:total_order_amount" + } + + # formulas[] itself is untouched by the convention -- only the + # surfacing column's properties change. + assert _model_formulas_by_id(new) == _model_formulas_by_id(original) + + +def test_tpcds_model_reproduces_every_column_and_formula_except_the_metric_shape_collapse(): + document_set, _, tml_result = _tml_roundtrip("tpcds") + original = document_set.model.body + new = tml_result.documents.model.body + # All four of store_sales's referencing joins (including the one whose + # target dataset name, "date_dim", differs from its own name, + # "store_sales_to_date" -- the exact case that used to be resynthesized + # as "store_sales_to_date_dim") come back exactly. + assert new["model_tables"] == original["model_tables"] + + orig_columns = _model_columns_by_name(original) + new_columns = _model_columns_by_name(new) + assert set(new_columns) == set(orig_columns) + + # Three metrics keep their `formula` shape but gain the same + # aggregation-convention property as minimal's "total_order_amount" + # above. "total_return_quantity" is different: it arrives as + # `column_id` + a load-bearing `aggregation` (never a `formula` in the + # source document at all) -- Ossie's own Metric schema has no + # `column_id` field, so the only shape available on the way back is a + # formula, and the aggregate is composed into a brand new formulas[] + # entry rather than surviving as a column-level property. + convention_names = {"total_sales", "total_profit", "sales_by_brand"} + shape_collapsed = {"total_return_quantity"} + for name in set(orig_columns) - convention_names - shape_collapsed: + assert new_columns[name] == orig_columns[name], name + + for name in convention_names: + assert "aggregation" not in orig_columns[name]["properties"], name + assert new_columns[name]["properties"]["aggregation"] == "SUM", name + assert _issue_refs(tml_result.issues, "TS-MODEL-METRIC-AGGREGATION-CONVENTION") == { + f"metric:{name}" for name in convention_names | shape_collapsed + } + + assert orig_columns["total_return_quantity"]["column_id"] == "store_returns_sv::sr_return_quantity" + assert "column_id" not in new_columns["total_return_quantity"] + assert new_columns["total_return_quantity"]["formula_id"] == "formula_total_return_quantity" + assert new_columns["total_return_quantity"]["properties"]["aggregation"] == "SUM" + + orig_formulas = _model_formulas_by_id(original) + new_formulas = _model_formulas_by_id(new) + assert set(new_formulas) == set(orig_formulas) | {"formula_total_return_quantity"} + for formula_id in orig_formulas: + assert new_formulas[formula_id] == orig_formulas[formula_id], formula_id + assert new_formulas["formula_total_return_quantity"] == { + "id": "formula_total_return_quantity", + "name": "total_return_quantity", + "expr": "sum ( [store_returns_sv::sr_return_quantity] )", + } + + +class TestReferencingJoinRestoration: + """`_join_entry_for_relationship` (ossie_to_thoughtspot.py) restores a + Table-referenced join's own shape -- a `referencing_join` pointer on the + Model entry plus a matching `joins_with[]` entry on the Table document + -- from the relationship's `RELATIONSHIP_STASH_REFERENCING_JOIN` stash, + rather than always collapsing it to an inline join. TML's inline join + syntax has no name field at all, so without this a relationship whose + join round-trips through TML gets a fresh name synthesized from its own + from/to dataset names on the next pass -- unstable exactly when the + relationship's own name does not already match that pattern (a target + dataset named `date_dim` but a relationship named `..._to_date`, tpcds's + own `store_sales_to_date`). + """ + + def test_minimal_referencing_join_is_restored_with_its_own_name(self): + document_set, _, tml_result = _tml_roundtrip("minimal") + original_orders = _table_by_name(document_set, "orders") + new_orders = _table_by_name(tml_result.documents, "orders") + assert new_orders.body["joins_with"] == original_orders.body["joins_with"] + + new_join = tml_result.documents.model.body["model_tables"][0]["joins"][0] + assert new_join == {"referencing_join": "orders_to_customers"} + + def test_tpcds_four_referencing_joins_are_restored_with_their_own_names(self): + document_set, _, tml_result = _tml_roundtrip("tpcds") + original_store_sales = _table_by_name(document_set, "store_sales") + new_store_sales = _table_by_name(tml_result.documents, "store_sales") + assert new_store_sales.body["joins_with"] == original_store_sales.body["joins_with"] + + store_sales_entry = next( + t for t in tml_result.documents.model.body["model_tables"] if t["name"] == "store_sales" + ) + # The exact case a name synthesized from from/to dataset names gets + # wrong: the target dataset is "date_dim", not "date", so a + # resynthesized name would read "store_sales_to_date_dim". + assert {"referencing_join": "store_sales_to_date"} in store_sales_entry["joins"] + assert {j["referencing_join"] for j in store_sales_entry["joins"]} == { + "store_sales_to_date", "store_sales_to_customer", "store_sales_to_item", "store_sales_to_store", + } + + # The relationships already inline in the source document + # (store_returns_sv's own joins, one of them carrying a residual + # predicate) have no referencing_join to restore and are + # unaffected -- still emitted inline, verbatim. + store_returns_sv_entry = next( + t for t in tml_result.documents.model.body["model_tables"] + if t["name"] == "store_returns_sv" + ) + original_store_returns_sv_entry = next( + t for t in document_set.model.body["model_tables"] if t["name"] == "store_returns_sv" + ) + assert store_returns_sv_entry["joins"] == original_store_returns_sv_entry["joins"] + + def test_a_relationship_renamed_since_the_stash_was_written_falls_back_and_logs(self): + """A relationship's `name` can be edited directly in the Ossie + document (there is nothing to keep it in sync with the stash it was + written alongside). Restoring the stashed `referencing_join` under + that stale name would point the Table's `joins_with[]` reference at + a name the live relationship no longer answers to -- so the + currency check (RELATIONSHIP_STASH_REFERENCING_JOIN compared + directly against the relationship's own live `name`) drops it + instead, exactly as if no stash were present at all, and reports + why. The other three relationships from the same dataset, whose + stash is still current, are unaffected. + """ + expected = _load_expected(FIXTURES_ROOT / "tpcds") + model = expected["semantic_model"][0] + relationship = next(r for r in model["relationships"] if r["name"] == "store_sales_to_date") + relationship["name"] = "renamed_relationship" + + tml_result = ossie_to_thoughtspot.convert(expected) + store_sales = _table_by_name(tml_result.documents, "store_sales") + joins_with_names = {j["name"] for j in store_sales.body.get("joins_with") or []} + assert joins_with_names == {"store_sales_to_customer", "store_sales_to_item", "store_sales_to_store"} + + store_sales_entry = next( + t for t in tml_result.documents.model.body["model_tables"] if t["name"] == "store_sales" + ) + renamed_join = next(j for j in store_sales_entry["joins"] if j.get("with") == "date_dim") + assert renamed_join == { + "with": "date_dim", + "on": "[store_sales::ss_sold_date_sk] = [date_dim::d_date_sk]", + "type": "INNER", + "cardinality": "MANY_TO_ONE", + } + assert _issue_refs(tml_result.issues, "TS-JOIN-REFERENCING-JOIN-STALE") == { + "relationship:renamed_relationship" + } + + +def test_a_model_surfaced_fields_description_is_never_duplicated_onto_its_physical_column(): + """tpcds's `store.on` field carries a `description` on the Model's own + `columns[]` entry; the Table's own `on` column never has one. That + description must survive on the Model side and must NOT be invented on + the Table side -- `_physical_table_column`/`_physical_sql_view_column` + never read a field's `description` at all, so the only place a + Model-surfaced field's description can end up is the one place the + mapping rule puts it. + """ + document_set, _, tml_result = _tml_roundtrip("tpcds") + original_store = _table_by_name(document_set, "store") + assert "description" not in _columns_by_name(original_store.body)["on"] + + new_store = _table_by_name(tml_result.documents, "store") + assert "description" not in _columns_by_name(new_store.body)["on"] + + new_model_columns = _model_columns_by_name(tml_result.documents.model.body) + assert new_model_columns["on"]["description"] == "Whether the store is currently active and open for business." + + +def test_tpcds_one_to_many_join_round_trips_with_swapped_endpoints(): + """`store`'s join to `store_returns_sv` (tests/fixtures/tpcds/ + tpcds_retail_model.model.tml) is the fixture's only ONE_TO_MANY join -- + the one case where TML's own declared from/to is backwards relative to + core-spec/spec.yaml's many-side/one-side convention, so `TML -> Ossie` + swaps the emitted relationship's endpoints. Undoing that swap on the + `Ossie -> TML` leg must reproduce the original join exactly: nested + under the same dataset (`store`, the "one" side, not `store_returns_sv`, + the "many" side the swap moves the relationship's own `from` to), + same `with` target, same condition, same cardinality -- with no + endpoint-swap staleness issue logged. + """ + document_set, ossie_result, tml_result = _tml_roundtrip("tpcds") + + # The intermediate Ossie relationship: endpoints swapped relative to + # TML's declaration (`from` is the many side, `to` is the one side). + semantic_model = ossie_result.model["semantic_model"][0] + relationship = next( + r for r in semantic_model["relationships"] if r["name"] == "store_returns_sv_to_store" + ) + assert relationship["from"] == "store_returns_sv" + assert relationship["to"] == "store" + assert relationship["from_columns"] == ["sr_store_sk"] + assert relationship["to_columns"] == ["s_store_sk"] + + # The round-tripped TML: the join is nested back under `store`'s own + # model_tables entry, targeting `store_returns_sv`, exactly as declared. + original_store_entry = next( + t for t in document_set.model.body["model_tables"] if t["name"] == "store" + ) + new_store_entry = next( + t for t in tml_result.documents.model.body["model_tables"] if t["name"] == "store" + ) + assert new_store_entry["joins"] == original_store_entry["joins"] + assert new_store_entry["joins"] == [{ + "with": "store_returns_sv", + "on": "[store::s_store_sk] = [store_returns_sv::sr_store_sk]", + "type": "INNER", + "cardinality": "ONE_TO_MANY", + }] + + assert not _issue_refs(tml_result.issues, "TS-JOIN-ENDPOINTS-SWAP-STALE") + + +# --------------------------------------------------------------------------- +# TML -> Ossie -> TML: translation. Asserted directly on the intermediate +# Ossie document's ANSI_SQL siblings -- the half a preservation test, by +# construction, cannot see (see module docstring). +# --------------------------------------------------------------------------- + +def test_minimal_physical_field_translates_to_its_dataset_dot_column(): + _, ossie_result, _ = _tml_roundtrip("minimal") + model = ossie_result.model["semantic_model"][0] + field = _field(_dataset(model, "orders"), "order_id") + dialects = _dialects(field) + assert dialects[DIALECT] == "[orders::order_id]" + assert dialects[PORTABLE_DIALECT] == "orders.order_id" + + +def test_minimal_known_unportable_metric_has_no_portable_sibling_and_an_issue(): + _, ossie_result, _ = _tml_roundtrip("minimal") + model = ossie_result.model["semantic_model"][0] + metric = _metric(model, "total_order_amount") + assert PORTABLE_DIALECT not in _dialects(metric) + assert "metric:total_order_amount" in _issue_refs(ossie_result.issues, "TS-EXPR-THOUGHTSPOT-ONLY") + + +def test_tpcds_physical_fields_translate_to_their_warehouse_column_even_when_the_display_name_differs(): + _, ossie_result, _ = _tml_roundtrip("tpcds") + model = ossie_result.model["semantic_model"][0] + # s_store_name's display name differs from its warehouse column name + # (STORE_NM); the portable expression has to carry the warehouse name, + # not the display-derived Ossie identifier. + store_name = _field(_dataset(model, "store"), "s_store_name") + assert _dialects(store_name)[PORTABLE_DIALECT] == "store.STORE_NM" + # sr_return_amt is a SQL View column whose output alias (RETURN_AMT) + # differs from its own field name -- the same fact, for a query rather + # than a table. + return_amt = _field(_dataset(model, "store_returns_sv"), "sr_return_amt") + assert _dialects(return_amt)[PORTABLE_DIALECT] == "store_returns_sv.RETURN_AMT" + + +def test_tpcds_known_unportable_metrics_have_no_portable_sibling_and_an_issue(): + _, ossie_result, _ = _tml_roundtrip("tpcds") + model = ossie_result.model["semantic_model"][0] + + # A formula cross-reference: inlining the referenced formulas is out of + # scope, so only the THOUGHTSPOT dialect entry is emitted. + profit_margin = _metric(model, "profit_margin") + assert PORTABLE_DIALECT not in _dialects(profit_margin) + assert "metric:profit_margin" in _issue_refs(ossie_result.issues, "TS-EXPR-FORMULA-REFERENCE") + + # A bare aggregate call over a physical column: still a function call, + # not a bare column reference, so it is THOUGHTSPOT-only too. + total_sales = _metric(model, "total_sales") + assert PORTABLE_DIALECT not in _dialects(total_sales) + assert "metric:total_sales" in _issue_refs(ossie_result.issues, "TS-EXPR-THOUGHTSPOT-ONLY") + + # A physical column reference the model never surfaces as a field + # resolves to nothing at all, which is a different unportable reason + # (and a different code) from the two above. + total_return_quantity = _metric(model, "total_return_quantity") + assert PORTABLE_DIALECT not in _dialects(total_return_quantity) + assert "metric:total_return_quantity" in _issue_refs(ossie_result.issues, "TS-EXPR-UNRESOLVED") + + +def test_a_metrics_portable_expression_carries_its_column_level_aggregation(): + """A metric built from `column_id` + a load-bearing `aggregation` (never + a bare `formula`) composes a portable ANSI_SQL sibling that carries the + same aggregate (`_compose_aggregate_entries`, tml_to_ossie.py). + + Exercised here with a small, purpose-built document rather than either + fixture set: neither fixture's own `column_id` + `aggregation` metric + (tpcds's `total_return_quantity`, asserted unportable just above) takes + this shape over a column the model ALSO surfaces as a field, which is + what `_compose_aggregate_entries` requires before it can name a + warehouse column at all. + """ + table = tml.TmlDocument( + kind="table", + body={ + "name": "widgets", + "db": "TESTDB", + "schema": "PUBLIC", + "db_table": "WIDGETS", + "connection": {"name": "Test Connection"}, + "columns": [ + {"name": "amount", "db_column_name": "amount", "db_column_properties": {"data_type": "INT64"}}, + ], + }, + guid=None, + ) + model_doc = tml.TmlDocument( + kind="model", + body={ + "name": "aggregate_probe_model", + "model_tables": [{"name": "widgets"}], + "columns": [ + {"name": "amount", "column_id": "widgets::amount", "properties": {"column_type": "ATTRIBUTE"}}, + { + "name": "total_amount", + "column_id": "widgets::amount", + "properties": {"column_type": "MEASURE", "aggregation": "SUM"}, + }, + ], + }, + guid=None, + ) + document_set = tml.DocumentSet(model=model_doc, tables=(table,)) + + ossie_result = tml_to_ossie.convert(document_set) + metric = _metric(ossie_result.model["semantic_model"][0], "total_amount") + dialects = _dialects(metric) + assert dialects[DIALECT] == "sum ( [widgets::amount] )" + assert dialects[PORTABLE_DIALECT] == "SUM(widgets.amount)" + + # The composed aggregate survives being written back out as a formula + # too -- a metric is always a formula on the way back. + tml_result = ossie_to_thoughtspot.convert(ossie_result.model) + new_columns = _model_columns_by_name(tml_result.documents.model.body) + formulas = _model_formulas_by_id(tml_result.documents.model.body) + formula_id = new_columns["total_amount"]["formula_id"] + assert formulas[formula_id]["expr"] == "sum ( [widgets::amount] )" + + +# --------------------------------------------------------------------------- +# Ossie -> TML -> Ossie: the datatype map's own declared_loss taxonomy. +# --------------------------------------------------------------------------- + +_LOSSY_DATATYPES = sorted(dt for dt in OSSIE_DATATYPES if datatypes.declared_loss(dt)) +_LOSSLESS_DATATYPES = sorted(dt for dt in OSSIE_DATATYPES if not datatypes.declared_loss(dt)) + + +def _datatype_probe_document() -> dict: + """One Ossie dataset with one field per datatype in the closed + `OSSIE_DATATYPES` enum, so every entry in `datatypes._DECLARED_LOSS` + (and every entry NOT in it) is exercised in one round trip.""" + dataset = stash.write_stash( + { + "name": "widgets", + "source": "TESTDB.PUBLIC.WIDGETS", + "fields": [ + { + "name": f"col_{dt.lower()}", + "datatype": dt, + "expression": {"dialects": [{"dialect": DIALECT, "expression": f"[widgets::col_{dt.lower()}]"}]}, + } + for dt in sorted(OSSIE_DATATYPES) + ], + }, + {DATASET_STASH_CONNECTION_NAME: "Test Connection"}, + ) + return { + "version": DOCUMENT_VERSION, + "semantic_model": [{"name": "datatype_probe_model", "datasets": [dataset]}], + } + + +def test_the_declared_loss_datatypes_are_exactly_datetimetz_float_opaque_and_time(): + # Pins datatypes.py's own taxonomy so the two tests below -- which + # split their assertions on this same set -- fail loudly if a future + # change to _DECLARED_LOSS adds or removes a member, rather than + # silently checking a set that no longer matches the module they test. + assert _LOSSY_DATATYPES == ["DateTimeTz", "Float", "Opaque", "Time"] + + +def test_every_declared_loss_datatype_is_flagged_before_the_round_trip_changes_it(): + ossie_in = _datatype_probe_document() + tml_result = ossie_to_thoughtspot.convert(ossie_in) + flagged = _issue_refs(tml_result.issues, "TS-FIELD-DATATYPE-DECLARED-LOSS") + assert flagged == {f"field:col_{dt.lower()}" for dt in _LOSSY_DATATYPES} + + ossie_result = tml_to_ossie.convert(tml_result.documents) + new_dataset = ossie_result.model["semantic_model"][0]["datasets"][0] + new_by_name = {f["name"]: f.get("datatype") for f in new_dataset["fields"]} + # Exactly what datatypes.py's own _TO_TML/_TO_OSSIE maps predict -- the + # TML type each lossy datatype collapses into, mapped back. + assert new_by_name["col_datetimetz"] == "DateTime" + assert new_by_name["col_float"] == "Decimal" + assert new_by_name["col_opaque"] == "String" + assert new_by_name["col_time"] == "String" + for dt in _LOSSY_DATATYPES: + assert new_by_name[f"col_{dt.lower()}"] != dt + + +def test_every_lossless_datatype_returns_exactly(): + ossie_in = _datatype_probe_document() + tml_result = ossie_to_thoughtspot.convert(ossie_in) + assert _issue_refs(tml_result.issues, "TS-FIELD-DATATYPE-DECLARED-LOSS") == { + f"field:col_{dt.lower()}" for dt in _LOSSY_DATATYPES + } # none of the lossless fields are flagged + + ossie_result = tml_to_ossie.convert(tml_result.documents) + new_dataset = ossie_result.model["semantic_model"][0]["datasets"][0] + new_by_name = {f["name"]: f.get("datatype") for f in new_dataset["fields"]} + for dt in _LOSSLESS_DATATYPES: + assert new_by_name[f"col_{dt.lower()}"] == dt, dt + + +# --------------------------------------------------------------------------- +# Ossie -> TML -> Ossie, over both fixture sets' own expected.ossie.yaml. +# --------------------------------------------------------------------------- + +def _fields_with_datatype(model: dict): + for dataset in model["datasets"]: + for field in dataset.get("fields", []): + if "datatype" in field: + yield dataset["name"], field["name"], field["datatype"] + + +@pytest.mark.parametrize("fixture_name", FIXTURE_SETS) +def test_every_declared_field_datatype_returns_exactly(fixture_name): + """Both fixture sets only ever declare datatypes datatypes.py calls + lossless (String, Integer, Decimal, Boolean, Date -- see the four + declared_loss types covered by the synthetic probe above), so every + field with a declared datatype must come back unchanged.""" + expected, _, ossie_result = _ossie_roundtrip(fixture_name) + original = list(_fields_with_datatype(expected["semantic_model"][0])) + assert original, "expected at least one field with a declared datatype" + + new_by_key = { + (d["name"], f["name"]): f.get("datatype") + for d in ossie_result.model["semantic_model"][0]["datasets"] + for f in d.get("fields", []) + } + for dataset_name, field_name, expected_datatype in original: + assert new_by_key[(dataset_name, field_name)] == expected_datatype, (dataset_name, field_name) + + +def test_tpcds_metric_datatype_is_dropped_with_an_issue_naming_it(): + """A Metric's own `datatype` is a different kind of loss from anything + datatypes.py's own map declares: Model TML has no `data_type` key + anywhere for a formula-backed metric, so there is nowhere at all to + write one, regardless of what the datatype is. It is still the same + declared-and-reported shape -- a real loss, an issue naming it before + the round trip discards it.""" + expected = _load_expected(FIXTURES_ROOT / "tpcds") + tml_result = ossie_to_thoughtspot.convert(expected) + assert _issue_refs(tml_result.issues, "TS-MODEL-METRIC-DATATYPE-UNWRITABLE") == { + "metric:total_return_quantity" + } + + original_metric = _metric(expected["semantic_model"][0], "total_return_quantity") + assert original_metric["datatype"] == "Integer" + + ossie_result = tml_to_ossie.convert(tml_result.documents) + new_metric = _metric(ossie_result.model["semantic_model"][0], "total_return_quantity") + assert "datatype" not in new_metric + + +@pytest.mark.parametrize("fixture_name", FIXTURE_SETS) +def test_every_relationship_survives_by_from_to_columns_type_cardinality_and_name(fixture_name): + """`name` is included in this comparison -- see + TestReferencingJoinRestoration for why a relationship whose join was + Table-referenced now keeps its own name across the TML leg instead of + getting a fresh one synthesized from its from/to dataset names. + tpcds's own `store_sales_to_date` (target dataset `date_dim`, not + `date`) is the concrete case that used to come back renamed. + """ + expected, _, ossie_result = _ossie_roundtrip(fixture_name) + original_model = expected["semantic_model"][0] + new_model = ossie_result.model["semantic_model"][0] + + original_by_identity = { + _relationship_identity(r): r for r in original_model.get("relationships", []) + } + new_by_identity = {_relationship_identity(r): r for r in new_model.get("relationships", [])} + assert set(new_by_identity) == set(original_by_identity) + + for identity, original_rel in original_by_identity.items(): + new_rel = new_by_identity[identity] + assert new_rel["name"] == original_rel["name"], identity + original_payload = stash.read_stash(original_rel) + new_payload = stash.read_stash(new_rel) + assert new_payload.get(RELATIONSHIP_STASH_TYPE) == original_payload.get(RELATIONSHIP_STASH_TYPE) + assert new_payload.get(RELATIONSHIP_STASH_CARDINALITY) == original_payload.get( + RELATIONSHIP_STASH_CARDINALITY + ) diff --git a/converters/thoughtspot/tests/test_roundtrip_properties.py b/converters/thoughtspot/tests/test_roundtrip_properties.py new file mode 100644 index 00000000..0666fdba --- /dev/null +++ b/converters/thoughtspot/tests/test_roundtrip_properties.py @@ -0,0 +1,535 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Property-based round trip: `Ossie -> TML -> Ossie` over adversarial names. + +test_roundtrip.py proves the round trip on documents this project's own +authors wrote -- fixtures and hand-built probes. Every one of those reflects +what its author thought to include, which is exactly the weakness a +generator can correct: it draws dataset and field display names from a pool +that *deliberately* includes the cases most likely to break identifier +handling -- YAML 1.1 boolean tokens (`on`, `off`, `yes`, `no`, `y`, `n`, and +every case variant), names that collide only after `identifiers.normalise` +folds them, non-ASCII names (both the kind NFKD decomposition folds to ASCII +and the kind it cannot), names containing `::`, leading/trailing whitespace, +punctuation-only text, very long text, and the empty string. + +The property: the round trip is the identity on everything +`datatypes.declared_loss` calls lossless, and every difference beyond that +is covered by a reported issue. A silently dropped or silently mangled name +is a defect; the same name dropped *with* an issue naming it is a declared +loss, and the two are told apart here by checking the issue log, not by +trusting a document comparison alone. + +Two real, previously uncaught crashes were found while writing this +suite -- both now fixed (see ossie_to_thoughtspot.py's `_physical_identity`/ +`_field_physical_display_name` and tml_to_ossie.py's `convert`/ +`_index_attribute_columns`) and both exercised directly by the properties +below, so a regression reopens loudly rather than silently. Every +*remaining* difference this suite predicts is computed by `_field_survives` +and `_expected_datatype` below, both built from the exact `identifiers`/ +`datatypes` functions the converter itself calls -- an oracle, not a +hand-written parallel prediction that could quietly drift from what the +code actually does. + +Each document also crosses the real YAML 1.2 codec (`tml.dump_document`/ +`load_document` for the TML leg, `_yaml.dump`/`load` for the returned Ossie +document) rather than staying as Python objects the whole way through -- +that text round trip is what actually exercises the boolean-token guard for +a name like "on" or "Off", not merely the pure conversion functions. +""" +from __future__ import annotations + +import pytest + +pytest.importorskip("hypothesis") # skip cleanly if hypothesis is not installed + +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from ossie_thoughtspot import _yaml, datatypes, identifiers, ossie_to_thoughtspot, stash, tml, tml_to_ossie +from ossie_thoughtspot.constants import ( + DATASET_STASH_CONNECTION_NAME, + DIALECT, + DOCUMENT_VERSION, + FIELD_STASH_DB_COLUMN_NAME, + FIELD_STASH_DB_COLUMN_NAME_WITNESS, + STASH_TML_NAME, +) +from ossie_thoughtspot.datatypes import OSSIE_DATATYPES + +from test_roundtrip import _issue_refs # reuse rather than duplicate + +_SETTINGS = settings( + max_examples=100, # the Hypothesis default -- modest, per the project's own guidance + deadline=None, # generous for CI: a slow YAML round trip must not read as a bug + suppress_health_check=[ + HealthCheck.too_slow, HealthCheck.data_too_large, HealthCheck.filter_too_much, + ], +) + +# --------------------------------------------------------------------------- +# Adversarial name strategies. Each bucket below exists because a real defect +# in this converter, or its sibling reference converters, was traced to +# exactly this shape of name -- see the module docstring. +# --------------------------------------------------------------------------- + +#: YAML 1.1 resolves these bare scalars as booleans; TML/Ossie use them as +#: ordinary strings (see _yaml.py's own Yaml12Loader/Yaml12Dumper). Every +#: case variant is included, not just the lower-case spelling. +_YAML11_BOOL_WORDS = ("on", "off", "yes", "no", "y", "n") +_yaml_bool_names = st.sampled_from( + sorted({form(w) for w in _YAML11_BOOL_WORDS for form in (str.lower, str.upper, str.capitalize)}) +) + +#: Ordinary, well-behaved identifiers -- the baseline every fixture already covers. +_plain_names = st.from_regex(r"[A-Za-z][A-Za-z0-9 _]{0,14}", fullmatch=True) + +#: Diacritics that NFKD decomposition folds cleanly to ASCII (identifiers.py's +#: own documented case: "Café" -> "cafe"). +_nfkd_foldable_names = st.sampled_from( + ["Café", "Ürün", "Zürich", "Müller", "Crème Brûlée", "Naïve Café"] +) + +#: Non-Latin scripts NFKD decomposition has no ASCII expansion for at all -- +#: identifiers.normalise's own documented residual limitation. +_non_foldable_names = st.sampled_from(["北京市", "Москва", "東京都", "Привет мир", "日本語"]) + +#: A table or column name containing "::" -- ambiguous once embedded in a +#: [TABLE::Column] bracket (identifiers.split_column_ref must refuse to +#: guess which "::" is the real delimiter). +_double_colon_names = st.sampled_from(["A::B", "table::name", "x::y::z", "a::b::c::d"]) + +_whitespace_padded_names = st.sampled_from( + [" padded ", "\ttabbed\t", " leading", "trailing ", "\n\nnewlines\n\n"] +) + +#: Folds to nothing: identifiers.normalise has no ASCII alphanumerics to keep. +_punctuation_only_names = st.sampled_from(["!!!", "---", "***", "...", "###", "@@@"]) + +_long_names = st.text(alphabet=st.characters(whitelist_categories=("Lu", "Ll")), min_size=200, max_size=260) + +_empty_name = st.just("") + +#: The full adversarial pool -- every field label is drawn from this. +_adversarial_names = st.one_of( + _plain_names, _yaml_bool_names, _nfkd_foldable_names, _non_foldable_names, + _double_colon_names, _whitespace_padded_names, _punctuation_only_names, + _long_names, _empty_name, +) + +#: Dataset/model names use the same pool minus the empty string: an empty +#: dataset or model `name` hits `_table_name`/`build_model`'s own silent +#: "" placeholder fallback (no issue logged for either), a real, +#: separate gap this suite does not paper over by excluding the case -- +#: see the report for why it is not fixed here. Field labels keep the empty +#: string (`_build_field`'s `label or name` falls back to the field's own +#: always-present `name` with no placeholder and no silent gap), so "empty +#: names" is still exercised, just not doubled up on top of an already-known, +#: separately reported difference. +_adversarial_names_nonempty = st.one_of( + _plain_names, _yaml_bool_names, _nfkd_foldable_names, _non_foldable_names, + _double_colon_names, _whitespace_padded_names, _punctuation_only_names, _long_names, +) + + +# --------------------------------------------------------------------------- +# Oracles -- reuse the exact functions the converter itself calls, so a +# prediction here can never quietly diverge from what the code does. +# --------------------------------------------------------------------------- + +def _folds(text: str) -> bool: + try: + identifiers.normalise(text) + return True + except ValueError: + return False + + +def _bracket_is_usable(dataset_name: str, label: str) -> bool: + """Whether `[dataset_name::label]` is a reference `is_bare_column_ref` + (via `split_column_ref`) can actually parse, rather than raising on an + ambiguous or malformed bracket.""" + try: + identifiers.split_column_ref(identifiers.format_column_ref(dataset_name, label)) + return True + except ValueError: + return False + + +def _effective_label(label: str, name: str) -> str: + """`_build_field`'s own `field.get("label") or field.get("name")`.""" + return label or name + + +def _field_survives(dataset_name: str, label: str, name: str) -> bool: + """Whether this field is expected to still be present after the round + trip. + + Only one real gate remains: the bracket `[dataset_name::effective_label]` + must be a reference `split_column_ref` can parse. An ambiguous one is + caught, reported, and the field is carried into the model as an + unreadable formula that `tml_to_ossie.convert`'s own Phase 3 then also + fails to convert, via whichever of `identifiers.normalise`/ + `find_column_refs` hits the ambiguity first -- always the same + observable outcome, so this property does not need to distinguish which. + + A second gate used to exist here: the effective display name had to + itself fold (`identifiers.normalise`), because `convert_field`/ + `convert_metric` called it unconditionally and a raise there dropped the + column entirely. `_field_or_metric_identifier` closed that gap with a + fallback identifier (physical-hint-based, or an allocator-suffixed + placeholder) -- a field with an unfoldable name now always survives, + just under a different identifier than `identifiers.normalise` would + have produced. See the test body for what is, and isn't, pinned about + that fallback identifier's exact value. + """ + effective = _effective_label(label, name) + return _bracket_is_usable(dataset_name, effective) + + +def _expected_datatype(original: str | None) -> str: + """The Ossie datatype a physical field's own `datatype` becomes after + one round trip, computed via the same `datatypes.to_tml`/`to_ossie` + pair `ossie_to_thoughtspot.py`/`tml_to_ossie.py` call -- including the + undeclared case, which is not silent: `db_column_properties` is + compulsory in TML, so `to_tml(None)` infers INT64, which comes back as + "Integer" rather than staying absent (documented in + ossie_to_thoughtspot.py's own module docstring). + """ + return datatypes.to_ossie(datatypes.to_tml(original)) + + +def _fold_key(label: str, name: str) -> str: + """The same fold key `_DisplayNameAllocator.allocate` computes over one + field's effective display name -- used only to keep the generator's own + fields mutually non-colliding (deliberate collisions get their own, + dedicated property below, not this one).""" + effective = _effective_label(label, name) + if _folds(effective): + return identifiers.normalise(effective) + return effective.strip().casefold() or "field" + + +# --------------------------------------------------------------------------- +# Document construction. +# --------------------------------------------------------------------------- + +def _dataset_names(min_size: int, max_size: int): + return st.lists(_adversarial_names_nonempty, min_size=min_size, max_size=max_size, unique=True) + + +@st.composite +def _ossie_documents(draw): + model_name = draw(_adversarial_names_nonempty) + dataset_names = draw(_dataset_names(1, 2)) + + datasets = [] + # `_DisplayNameAllocator` (ossie_to_thoughtspot.py's build_model) is one + # instance shared across every dataset in the model -- display-name + # uniqueness is model-wide, not per-dataset. A fold-key set scoped to one + # dataset would let two fields in *different* datasets collide by + # accident, which is exactly the deliberate scenario + # TestDisplayNameCollisionAllocator exercises on purpose; this generator + # avoids it happening by accident here instead, so this property tests + # only the no-collision case cleanly. + seen_folds: set[str] = set() + for d_index, dataset_name in enumerate(dataset_names): + raw_labels = draw(st.lists(_adversarial_names, min_size=1, max_size=3)) + datatypes_drawn = draw( + st.lists( + st.one_of(st.none(), st.sampled_from(sorted(OSSIE_DATATYPES))), + min_size=len(raw_labels), max_size=len(raw_labels), + ) + ) + + fields = [] + for f_index, (label, datatype) in enumerate(zip(raw_labels, datatypes_drawn)): + name = f"field_{d_index}_{f_index}" + key = _fold_key(label, name) + if key in seen_folds: + continue # deliberate collisions are a separate, dedicated property below + seen_folds.add(key) + + effective = _effective_label(label, name) + bracket = identifiers.format_column_ref(dataset_name, effective) + field: dict = { + "name": name, "label": label, + "expression": {"dialects": [{"dialect": DIALECT, "expression": bracket}]}, + } + if datatype is not None: + field["datatype"] = datatype + fields.append(field) + + dataset = stash.write_stash( + {"name": dataset_name, "source": f"TESTDB.PUBLIC.T{d_index}", "fields": fields}, + {DATASET_STASH_CONNECTION_NAME: "Test Connection"}, + ) + datasets.append(dataset) + + return { + "version": DOCUMENT_VERSION, + "semantic_model": [{"name": model_name, "datasets": datasets}], + } + + +def _run_roundtrip(ossie_in: dict): + """`Ossie -> TML -> Ossie`, crossing the real YAML 1.2 codec on both legs + (not just the pure Python conversion functions) -- see the module + docstring for why that matters for a name like "on".""" + tml_result = ossie_to_thoughtspot.convert(ossie_in) + + reloaded_tables = tuple( + tml.load_document(tml.dump_document(t)) for t in tml_result.documents.tables + ) + reloaded_model = tml.load_document(tml.dump_document(tml_result.documents.model)) + ossie_result = tml_to_ossie.convert(tml.DocumentSet(model=reloaded_model, tables=reloaded_tables)) + + ossie_reloaded = _yaml.load(_yaml.dump(ossie_result.model)) + return tml_result, ossie_result, ossie_reloaded + + +def _has_issue(log, code: str) -> bool: + return len(_issue_refs(log, code)) > 0 + + +# --------------------------------------------------------------------------- +# The main property. +# --------------------------------------------------------------------------- + +class TestOssieRoundTripAdversarialNames: + @given(ossie_in=_ossie_documents()) + @_SETTINGS + def test_lossless_content_survives_and_every_other_difference_is_reported(self, ossie_in): + original_model = ossie_in["semantic_model"][0] + tml_result, ossie_result, ossie_reloaded = _run_roundtrip(ossie_in) + new_model = ossie_reloaded["semantic_model"][0] + + # -- Model name ------------------------------------------------- + # An Ossie `name` is a normalised identifier; TML's own `model: + # name:` is free text. ossie_to_thoughtspot.build_model writes the + # Ossie name into TML verbatim (no folding on that leg), so + # tml_to_ossie.convert's own top-level `identifiers.normalise` call + # is what actually derives the returned identifier -- the identity + # case (an already-normalised name) and the folding-but-different + # case (e.g. "A" -> "a") are the same formula, not two branches. + original_model_name = original_model["name"] + if _folds(original_model_name): + expected_name = identifiers.normalise(original_model_name) + assert new_model["name"] == expected_name + if expected_name != original_model_name: + # No issue here -- this is not a declared loss, it is a + # transformed-but-recoverable identifier, preserved via + # STASH_TML_NAME exactly as constants.py documents. + assert stash.read_stash(new_model).get(STASH_TML_NAME) == original_model_name + else: + # Reported fallback (see tml_to_ossie.convert's own guard) -- + # never a crash, never silent. + assert new_model["name"] == "model" + assert _has_issue(ossie_result.issues, "TS-MODEL-NAME-UNNORMALISABLE") + + new_datasets_by_name = {d["name"]: d for d in new_model["datasets"]} + + for original_dataset in original_model["datasets"]: + dataset_name = original_dataset["name"] + # A non-empty dataset name is never run through normalise() on + # either leg (constants.py's own STASH_TML_NAME docstring) -- + # it must come back byte for byte, unconditionally. + assert dataset_name in new_datasets_by_name + new_dataset = new_datasets_by_name[dataset_name] + new_fields_by_label = {f["label"]: f for f in (new_dataset.get("fields") or [])} + + for original_field in original_dataset.get("fields") or []: + # Read the field's own embedded `name` rather than + # recomputing "field_{d}_{f}" from this loop's own position: + # the generator's fold-key dedup can skip a raw label, which + # leaves a gap in the *position* index (e.g. field_0_0, + # field_0_2 with no field_0_1) that a freshly enumerated + # index here would not reproduce. + name = original_field["name"] + label = original_field.get("label", "") + effective = _effective_label(label, name) + original_datatype = original_field.get("datatype") + + if _field_survives(dataset_name, label, name): + assert effective in new_fields_by_label, ( + f"expected field {effective!r} to survive; issues=" + f"{[i.code for i in ossie_result.issues.issues]}" + ) + new_field = new_fields_by_label[effective] + if _folds(effective): + assert new_field["name"] == identifiers.normalise(effective) + else: + # The exact fallback identifier depends on the + # regenerated table's own db_column_name (itself + # just `effective` verbatim here -- no + # FIELD_STASH_DB_COLUMN_NAME is written by this + # generator, so ossie_to_thoughtspot.py's + # TS-FIELD-DB-COLUMN-NAME-ASSUMED path applies) and, + # once that also fails to fold, on allocator + # ordering across every unfoldable field in the + # model -- see _field_or_metric_identifier. This + # property only pins that SOME usable identifier + # was assigned and reported, not which one; + # TestUnnormalisableNamesAreCaughtNotFatal in + # test_tml_to_ossie.py pins the exact fallback for + # one concrete case. + assert new_field["name"] + assert _has_issue(ossie_result.issues, "TS-FIELD-NAME-UNNORMALISABLE") + assert new_field.get("datatype") == _expected_datatype(original_datatype) + if original_datatype is not None and datatypes.declared_loss(original_datatype): + assert _has_issue(tml_result.issues, "TS-FIELD-DATATYPE-DECLARED-LOSS") + else: + # Never silently vanished -- the bracket was unusable + # (reported on the Ossie -> TML leg, and again on the + # TML -> Ossie leg once the unreadable formula is + # reprocessed) -- `effective` is absent and an issue + # explains why. + assert effective not in new_fields_by_label + assert ( + _has_issue(tml_result.issues, "TS-FIELD-COLUMN-REF-MALFORMED") + or _has_issue(ossie_result.issues, "TS-COLUMN-REF-MALFORMED") + ) + + +# --------------------------------------------------------------------------- +# Names that collide only after normalisation. +# --------------------------------------------------------------------------- + +#: Base words chosen so every spelling variant below folds to the same +#: identifiers.normalise() output, but no two variants are textually equal. +_COLLISION_BASE_WORDS = ("Net Amount", "Customer ID", "Total-Sales", "on") + + +def _collision_variants(word: str) -> list[str]: + variants = [word, word.upper(), word.lower(), f" {word} ", word.replace(" ", "-").replace("_", "-")] + seen: list[str] = [] + for v in variants: + if v not in seen: + seen.append(v) + return seen + + +class TestDisplayNameCollisionAllocator: + @given( + word=st.sampled_from(_COLLISION_BASE_WORDS), + indices=st.lists(st.integers(min_value=0, max_value=4), min_size=2, max_size=2, unique=True), + ) + @_SETTINGS + def test_colliding_labels_get_distinct_names_and_both_survive(self, word, indices): + variants = _collision_variants(word) + indices = [i for i in indices if i < len(variants)] + if len(indices) < 2: + return # this word's variant list happened to be shorter than the drawn indices + label_a, label_b = variants[indices[0]], variants[indices[1]] + if label_a == label_b: + return # a genuine, if degenerate, tie -- covered by the plain-duplicate case instead + + dataset = stash.write_stash( + { + "name": "widgets", + "source": "TESTDB.PUBLIC.WIDGETS", + "fields": [ + { + "name": "field_a", "label": label_a, + "expression": {"dialects": [ + {"dialect": DIALECT, "expression": identifiers.format_column_ref("widgets", label_a)} + ]}, + }, + { + "name": "field_b", "label": label_b, + "expression": {"dialects": [ + {"dialect": DIALECT, "expression": identifiers.format_column_ref("widgets", label_b)} + ]}, + }, + ], + }, + {DATASET_STASH_CONNECTION_NAME: "Test Connection"}, + ) + ossie_in = { + "version": DOCUMENT_VERSION, + "semantic_model": [{"name": "collision_model", "datasets": [dataset]}], + } + + tml_result = ossie_to_thoughtspot.convert(ossie_in) + model_columns = tml_result.documents.model.body.get("columns") or [] + tml_names = [c["name"] for c in model_columns] + # The allocator's whole job: both survive, under distinct names. + assert len(tml_names) == len(set(tml_names)) == 2 + assert _has_issue(tml_result.issues, "TS-MODEL-DISPLAY-NAME-COLLISION") + + ossie_result = tml_to_ossie.convert(tml_result.documents) + new_fields = ossie_result.model["semantic_model"][0]["datasets"][0].get("fields") or [] + assert len(new_fields) == 2 + + +# --------------------------------------------------------------------------- +# A display name that differs from its warehouse column name. +# --------------------------------------------------------------------------- + +#: A smaller pool for this property: every entry must survive on its own +#: (fold successfully, no "::") since the mechanism under test -- +#: FIELD_STASH_DB_COLUMN_NAME kept distinct from the display name -- is only +#: reachable for a field that becomes a physical column at all. "::" names, +#: punctuation-only names and non-foldable names are exercised for survival +#: itself by the main property above; re-including them here would only +#: assert the same "dropped, reported" outcome under a different name. +_survivable_display_names = st.one_of( + _plain_names, _yaml_bool_names, _nfkd_foldable_names, _whitespace_padded_names, _long_names, +) +_warehouse_names = st.from_regex(r"[A-Z][A-Z_0-9]{0,10}", fullmatch=True) + + +class TestWarehouseColumnNameDiffersFromDisplayName: + @given(display_name=_survivable_display_names, warehouse_name=_warehouse_names) + @_SETTINGS + def test_the_table_column_carries_the_warehouse_name_not_the_display_name( + self, display_name, warehouse_name + ): + field = stash.write_stash( + { + "name": "field_0", + "label": display_name, + "expression": {"dialects": [ + {"dialect": DIALECT, "expression": identifiers.format_column_ref("widgets", display_name)} + ]}, + }, + { + FIELD_STASH_DB_COLUMN_NAME: warehouse_name, + FIELD_STASH_DB_COLUMN_NAME_WITNESS: display_name, + }, + ) + dataset = stash.write_stash( + {"name": "widgets", "source": "TESTDB.PUBLIC.WIDGETS", "fields": [field]}, + {DATASET_STASH_CONNECTION_NAME: "Test Connection"}, + ) + ossie_in = { + "version": DOCUMENT_VERSION, + "semantic_model": [{"name": "warehouse_probe_model", "datasets": [dataset]}], + } + + tml_result, ossie_result, ossie_reloaded = _run_roundtrip(ossie_in) + + table_column = tml_result.documents.tables[0].body["columns"][0] + assert table_column["name"] == display_name + if warehouse_name != display_name: + assert table_column["db_column_name"] == warehouse_name + assert not _has_issue(tml_result.issues, "TS-FIELD-DB-COLUMN-NAME-ASSUMED") + + new_dataset = ossie_reloaded["semantic_model"][0]["datasets"][0] + new_field = next(f for f in new_dataset["fields"] if f["label"] == display_name) + assert new_field["name"] == identifiers.normalise(display_name) diff --git a/converters/thoughtspot/tests/test_shipped_references.py b/converters/thoughtspot/tests/test_shipped_references.py new file mode 100644 index 00000000..02ab4f93 --- /dev/null +++ b/converters/thoughtspot/tests/test_shipped_references.py @@ -0,0 +1,249 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Guard: nothing shipped may cite a document a repository-only reader cannot see. + +This package was developed against internal design material that is not part of +this repository and will not be. Two shapes of unresolvable reference leak in as a +result, and this module checks for both every time the suite runs, so a new one +introduced by future work is caught immediately rather than found by the next +person who happens to grep for it by hand. + +**Identifier-shaped citations** — a short run of uppercase letters immediately +followed by digits, with an optional hyphen between the two, the shape a rule id +or a backlog-style item number is written in — are handled fail-closed: every +token of that shape actually present in a shipped file is collected, a curated +ALLOWED_TOKENS set of genuinely unrelated technical tokens (data types, encodings, +lint codes, ...) is subtracted, and a second set — MAPPING_DOC_RULE_IDS, held +permanently empty now that every mapping-document rule-id family it once +allowed has had its citation rewritten in place — is subtracted too, and +*anything left over fails the suite*. A blocklist can only catch an id someone +already thought to list; this can't be evaded that way, because the burden is +on a new token to justify itself, not on this file to have predicted it. + +**Ordinary-English process language** — internal task-tracking, multi-option +planning, and change-review vocabulary that reads as a normal sentence and so +has no identifier shape a scanner can key off — stays a hand-curated, +case-insensitive phrase blocklist for that reason. It trades recall for +precision in the other direction: it can miss a new phrasing, but it will not +fail to explain a hit. + +Both halves run over the *actual shipped surface* — every ``.py`` file under +``src/`` and ``tests/`` (this file included — a guard that exempts itself is not +a guard), ``README.md``, ``pyproject.toml``, and this package's own CI workflow +file if one is ever added directly under the package directory. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +PACKAGE_ROOT = Path(__file__).resolve().parents[1] + + +def _shipped_files() -> list[Path]: + """Every file this test module treats as "shipped" — see the module docstring.""" + patterns = ( + "src/**/*.py", + "tests/**/*.py", + "tools/**/*.py", + "docs/**/*.md", + "README.md", + "pyproject.toml", + # Only matches if this package ever grows its own workflow file directly + # under the package directory, per the module docstring; today it does not, + # and the repository's top-level CI is out of this package's scope. + "*.yml", + "*.yaml", + ) + seen: set[Path] = set() + files: list[Path] = [] + for pattern in patterns: + for path in sorted(PACKAGE_ROOT.glob(pattern)): + if path.is_file() and path not in seen: + seen.add(path) + files.append(path) + return files + + +# --------------------------------------------------------------------------- +# Half 1 — identifier-shaped tokens. Fail-closed: ALLOWED_TOKENS below is the +# complete list of tokens of this shape that are *not* a citation to unshipped +# material. Anything of this shape found in a shipped file and not in one of +# the two sets below (this one, or MAPPING_DOC_RULE_IDS further down, held +# empty) fails. +# --------------------------------------------------------------------------- + +#: An uppercase letter run (1-6 chars) followed by 1-4 digits, with an optional +#: hyphen between them — a short in-house rule code and a longer, hyphenated +#: backlog-style item number are both this same shape. Letters capped at 6 and +#: digits at 4 deliberately excludes the ASF licence header's own repeated +#: "LICENSE-2.0" URL fragment (a 7-letter run cannot start a match under this +#: cap) without needing a special-case exclusion for it. +TOKEN_SHAPE_RE = re.compile(r"\b[A-Z]{1,6}-?[0-9]{1,4}\b") + +#: Tokens of the id shape above that are genuinely unrelated technical terms — +#: not a citation to anything, mapping-document or otherwise. Each entry is +#: justified individually; an entry that is actually a citation to unshipped +#: material does not belong here at all — see MAPPING_DOC_RULE_IDS below. +ALLOWED_TOKENS: frozenset[str] = frozenset( + { + # ANSI/SQL function and format names emitted into translated expressions — + # real function and format-token spellings, not references to anything. + "ATAN2", # two-argument arctangent + "LOG10", # base-10 logarithm + "NVL2", # three-argument null-coalescing form + "HH24", # 24-hour hour component in a TO_TIMESTAMP format string + "INT64", # a datatype name written into TML's db_column_properties + "UTF-8", # the character encoding standard + "COM1", # a Windows-reserved device name, from filename-safety tests + "SCD-2", # "Slowly Changing Dimension type 2" — a data-warehousing term + "H3", # a Markdown heading level (### = h3), describing source structure + "P75", # the 75th percentile — a statistical term, not an identifier + "BLE001", # a ruff lint rule code, appearing only in a `# noqa:` comment + "TS001", # an arbitrary example issue code used as test fixture data + # Loss-category codes this repository defines and explains itself, in + # README's own coverage matrix — resolvable from inside this repository + # alone, unlike a citation to unshipped mapping-document material. + "L1", + "L2", + "L3", + "L4", + "L5", + "L6", + } +) + +# --------------------------------------------------------------------------- +# RESOLVED — kept empty, not deleted. +# +# The decision on every mapping-document rule-id family cited from this +# package (A/E/G/ID/KD/NM/R/X, and finally I) has now been made the same way: +# the internal mapping/invariant reference each one cited is not shipping, so +# every citation has been rewritten in place to state its substance directly +# (see README.md's "Rules" section for the full account). None remain +# allowed, so this set is empty — a citation of this shape now fails the +# suite like any other unresolvable reference. +# +# This name stays defined, rather than being deleted along with the families +# it used to hold, only because test_allowed_token_sets_do_not_overlap and +# the union in test_no_unresolvable_identifier_shaped_tokens still refer to +# it by name; removing it would mean rewriting those tests' structure, not +# just their comments. It is not a container to drop a new id into: a future +# citation of this shape gets the same treatment every prior one did (state +# the substance in place), and only a fresh, reasoned decision — recorded +# here the way this comment records the last one — may repopulate it. +# --------------------------------------------------------------------------- +MAPPING_DOC_RULE_IDS: frozenset[str] = frozenset() + + +def test_allowed_token_sets_do_not_overlap() -> None: + # MAPPING_DOC_RULE_IDS is empty today (see its comment above), but a + # token claimed there in the future must not also be claimed as an + # unrelated legitimate token in ALLOWED_TOKENS — that would hide which + # bucket it is really in, and defeat the point of separating the two. + overlap = ALLOWED_TOKENS & MAPPING_DOC_RULE_IDS + assert overlap == set(), f"tokens claimed in both allowlists: {sorted(overlap)}" + + +def test_no_unresolvable_identifier_shaped_tokens() -> None: + shipped = _shipped_files() + assert shipped, "expected at least one shipped file to scan" + + allowed = ALLOWED_TOKENS | MAPPING_DOC_RULE_IDS + offenders: list[str] = [] + for path in shipped: + text = path.read_text(encoding="utf-8") + for lineno, line in enumerate(text.splitlines(), start=1): + for match in TOKEN_SHAPE_RE.finditer(line): + token = match.group(0) + if token not in allowed: + rel = path.relative_to(PACKAGE_ROOT) + offenders.append(f"{rel}:{lineno}: {token!r} — {line.strip()!r}") + + assert offenders == [], ( + "Identifier-shaped token(s) found that are not in ALLOWED_TOKENS or " + "MAPPING_DOC_RULE_IDS. A reader of this repository alone cannot resolve " + "what they name. Either it is a genuinely unrelated technical token — add " + "it to ALLOWED_TOKENS with a one-line justification — or it is a new " + "citation to unshipped material, which needs a human decision (reword to " + "state the substance, or add to MAPPING_DOC_RULE_IDS with reason):\n" + + "\n".join(offenders) + ) + + +# --------------------------------------------------------------------------- +# Half 2 — ordinary English used in a process sense. No identifier shape +# describes this half, so it stays a hand-curated, case-insensitive blocklist +# of phrases that only make sense with access to material this repository does +# not ship: an internal task tracker, a set of named alternative plans, an +# original instructions document, a named human role in that process and the +# cycle of revising drafts against its feedback, and the internal agent-skill +# framework with its planning/tracking directories. +# +# Every pattern below leads with `\b` (a word-boundary assertion). That choice +# is deliberate, not incidental: it is also what keeps this module passing its +# own check below. In this file's own source text, each pattern's *raw +# characters* read literally as backslash-b-then-the-phrase (e.g. the actual +# bytes of the fourth pattern are `\btranscribed\b`), so the phrase is always +# immediately preceded by the letter "b" from that escape — a word character +# butted against another word character, which is never a word boundary. A +# pattern can therefore never match its own definition here. This was verified +# by running this suite against this file, not just reasoned about. +# --------------------------------------------------------------------------- +PROCESS_LANGUAGE_PATTERNS: tuple[re.Pattern[str], ...] = tuple( + re.compile(pattern, re.IGNORECASE) + for pattern in ( + r"\btask\s+\d+\b", + r"\bplan\s+[a-d]\b", + r"\bthe\s+brief\b", + r"\btranscribed\b", + r"\breviewers?\b", + r"\bfix\s+rounds?\b", + r"\breview\s+rounds?\b", + r"\bsuperpowers\b", + r"\bsdd/", + r"\bopen[- ]items?\b", + r"\bledger\b", + r"\bthe\s+findings?\b", + ) +) + + +def test_no_internal_process_language() -> None: + shipped = _shipped_files() + assert shipped, "expected at least one shipped file to scan" + + offenders: list[str] = [] + for path in shipped: + text = path.read_text(encoding="utf-8") + for lineno, line in enumerate(text.splitlines(), start=1): + for pattern in PROCESS_LANGUAGE_PATTERNS: + if pattern.search(line): + rel = path.relative_to(PACKAGE_ROOT) + offenders.append( + f"{rel}:{lineno}: matched {pattern.pattern!r} — {line.strip()!r}" + ) + + assert offenders == [], ( + "Ordinary-English process language found — a reference that only makes " + "sense with access to an internal document this repository does not " + "ship. Reword to state the substance directly:\n" + "\n".join(offenders) + ) diff --git a/converters/thoughtspot/tests/test_stash.py b/converters/thoughtspot/tests/test_stash.py new file mode 100644 index 00000000..6a37e75b --- /dev/null +++ b/converters/thoughtspot/tests/test_stash.py @@ -0,0 +1,189 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +import pytest + +from ossie_thoughtspot import stash +from ossie_thoughtspot.constants import ( + MODEL_STASH_ACTION_OBJECT_ASSOCIATIONS, + MODEL_STASH_COLUMN_GROUPS, + MODEL_STASH_CONSTRAINTS, + MODEL_STASH_FILTERS, + MODEL_STASH_LESSON_PLANS, + MODEL_STASH_MODEL_JOINS_WITH, + MODEL_STASH_PARAMETERS, + STASH_VERSION, + VENDOR_KEY, +) +from ossie_thoughtspot.errors import ConversionError + + +def test_write_stash_serialises_data_as_a_json_string_not_an_object(): + # ossie-schema.json types `data` as "string". + obj = stash.write_stash({}, {"join_type": "LEFT_OUTER"}) + entry = obj["custom_extensions"][0] + assert entry["vendor_name"] == VENDOR_KEY + assert isinstance(entry["data"], str) + assert json.loads(entry["data"])["join_type"] == "LEFT_OUTER" + + +def test_write_stash_stamps_the_shape_version(): + obj = stash.write_stash({}, {"k": "v"}) + assert json.loads(obj["custom_extensions"][0]["data"])["_v"] == STASH_VERSION + + +def test_write_stash_writes_nothing_for_an_empty_payload(): + # A converted document stays clean where ThoughtSpot added nothing. + assert stash.write_stash({}, {}) == {} + + +def test_write_stash_merges_into_the_existing_own_entry(): + # One entry per object, merged — never a second THOUGHTSPOT entry. + obj = stash.write_stash({}, {"a": 1}) + obj = stash.write_stash(obj, {"b": 2}) + own = [e for e in obj["custom_extensions"] if e["vendor_name"] == VENDOR_KEY] + assert len(own) == 1 + assert json.loads(own[0]["data"])["a"] == 1 + assert json.loads(own[0]["data"])["b"] == 2 + + +def test_foreign_vendor_entries_pass_through_untouched(): + obj = {"custom_extensions": [{"vendor_name": "DATABRICKS", "data": '{"x": 1}'}]} + out = stash.write_stash(obj, {"a": 1}) + foreign = [e for e in out["custom_extensions"] if e["vendor_name"] == "DATABRICKS"] + assert foreign == [{"vendor_name": "DATABRICKS", "data": '{"x": 1}'}] + + +def test_write_stash_refuses_identity_keys(): + # A portable document must not carry instance-local identity. + for key in ("guid", "obj_id", "fqn"): + with pytest.raises(ConversionError, match=key): + stash.write_stash({}, {key: "abc-123"}) + + +def test_read_stash_raises_a_named_error_on_malformed_json(): + # Never a bare json traceback. + obj = {"name": "orders", "custom_extensions": [{"vendor_name": VENDOR_KEY, "data": "{not json"}]} + with pytest.raises(ConversionError, match="orders"): + stash.read_stash(obj) + + +def test_read_stash_returns_empty_when_there_is_no_own_entry(): + assert stash.read_stash({"custom_extensions": [{"vendor_name": "OMNI", "data": "{}"}]}) == {} + + +def test_restore_returns_the_stashed_value_with_no_witness_key(): + # The degraded (stash-if-present) shape — the most common form in + # practice: no witness_key, so a present key always wins regardless of + # `witness`. Correct only for values nothing downstream can edit. + payload = {"some_key": "stashed_value"} + assert stash.restore(payload, "some_key", "DERIVED") == "stashed_value" + + +def test_restore_prefers_the_stash_when_the_witness_still_agrees(): + # The witness-agrees case. + payload = {"on_expression": "a = b", "ossie_expression": "a = b"} + assert stash.restore(payload, "on_expression", "DERIVED", + witness="a = b", witness_key="ossie_expression") == "a = b" + + +def test_restore_rederives_when_the_witness_has_changed(): + # The case a plain stash-if-present rule gets wrong: the user edited the + # Ossie document, so the stashed copy is stale and must not win. + payload = {"on_expression": "a = b", "ossie_expression": "a = b"} + assert stash.restore(payload, "on_expression", "DERIVED", + witness="a = c", witness_key="ossie_expression") == "DERIVED" + + +def test_restore_falls_back_to_derived_when_the_key_is_absent(): + assert stash.restore({}, "on_expression", "DERIVED") == "DERIVED" + + +class TestFindForbiddenKeyIsTheSingleChokePoint: + """write_stash is the one function every stashed payload passes through, + so the identity guard has to live there rather than at each caller -- + otherwise a caller that copies a whole sub-object verbatim (a model's + parameters[], filters[], ...) rather than rebuilding it field by field + can carry a forbidden key arbitrarily deep with nothing to catch it. + + Each payload shape below mirrors a real model-scope stash field this + package copies wholesale: a nested identity key inside any of them must + be caught the same way. The point of the last case is that this list + does not have to be exhaustive for the guard to work -- an entirely + unrelated, previously unseen key name is caught too, because the guard + scans by shape (any key named guid/obj_id/fqn) rather than by an + enumeration of known field names.""" + + SHAPES = { + MODEL_STASH_PARAMETERS: [{"name": "P", "default_value": {"obj_id": "p-1"}}], + MODEL_STASH_FILTERS: [{"column": "Region", "values": ["US", {"nested": {"fqn": "f-1"}}]}], + MODEL_STASH_COLUMN_GROUPS: [{"name": "Sales", "meta": {"guid": "g-1"}}], + MODEL_STASH_LESSON_PLANS: [{"lesson_id": 0, "extra": {"obj_id": "l-1"}}], + MODEL_STASH_ACTION_OBJECT_ASSOCIATIONS: [{"action_name": "A", "context": {"fqn": "a-1"}}], + MODEL_STASH_CONSTRAINTS: {"rolling": {"window": {"guid": "c-1"}}}, + MODEL_STASH_MODEL_JOINS_WITH: [{"name": "j", "destination": {"fqn": "j-1"}}], + # A field name this module has never heard of -- the fail-closed + # property itself: the guard must not depend on a list of known + # model-scope keys to check. + "a_future_property_nobody_has_named_yet": {"deeply": {"nested": {"obj_id": "u-1"}}}, + } + + @pytest.mark.parametrize("key,value", SHAPES.items(), ids=SHAPES.keys()) + def test_a_nested_identity_key_is_caught_regardless_of_which_field_carries_it(self, key, value): + with pytest.raises(ConversionError): + stash.write_stash({}, {key: value}) + + def test_find_forbidden_key_names_the_key_it_found(self): + assert stash.find_forbidden_key({"a": {"b": [{"obj_id": "x"}]}}) == "obj_id" + + def test_find_forbidden_key_returns_none_for_a_clean_payload(self): + assert stash.find_forbidden_key({"a": {"b": ["ordinary", "values"]}}) is None + + def test_find_forbidden_key_accepts_a_wider_vocabulary_than_the_default(self): + # tml_to_ossie.py's column-properties path checks a wider identity + # vocabulary than the default three names (this package's own + # dataset_id/custom_file_guid additions) -- find_forbidden_key has to + # support that without stash.py hard-coding a second, wider set. + wider = frozenset({"custom_file_guid"}) + assert stash.find_forbidden_key({"geo_config": {"custom_file_guid": "m-1"}}, wider) == "custom_file_guid" + assert stash.find_forbidden_key({"geo_config": {"custom_file_guid": "m-1"}}) is None + + +class TestReadStashShapeVersion: + def test_an_unrecognised_shape_version_raises_naming_the_object_and_version(self): + # A future payload shape must never be partially read as today's. + obj = {"name": "orders", "custom_extensions": [ + {"vendor_name": VENDOR_KEY, "data": json.dumps({"_v": 999, "alias": "X"})} + ]} + with pytest.raises(ConversionError, match="orders") as excinfo: + stash.read_stash(obj) + assert "999" in str(excinfo.value) + + def test_a_missing_shape_version_raises_too(self): + obj = {"name": "orders", "custom_extensions": [ + {"vendor_name": VENDOR_KEY, "data": json.dumps({"alias": "X"})} + ]} + with pytest.raises(ConversionError, match="orders"): + stash.read_stash(obj) + + def test_the_current_shape_version_reads_normally(self): + obj = {"name": "orders", "custom_extensions": [ + {"vendor_name": VENDOR_KEY, "data": json.dumps({"_v": STASH_VERSION, "alias": "X"})} + ]} + assert stash.read_stash(obj) == {"_v": STASH_VERSION, "alias": "X"} diff --git a/converters/thoughtspot/tests/test_stash_key_classification.py b/converters/thoughtspot/tests/test_stash_key_classification.py new file mode 100644 index 00000000..4f30ba51 --- /dev/null +++ b/converters/thoughtspot/tests/test_stash_key_classification.py @@ -0,0 +1,147 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The witness rule's enforcement point: every custom_extensions[THOUGHTSPOT] stash key +this converter reads on its Ossie -> TML direction has to declare, in +`constants.STASH_KEY_CLASSIFICATION`, whether it shadows a value this +converter could otherwise derive from the live Ossie document (and so needs +a witness and a currency check) or is information that exists nowhere else +(and so cannot go stale). Three keys were found reading the unsafe way +before this table existed — each found by generalising from the one before +it, never by a rule anyone consulted. This test is that rule, made +structural: a stash key read anywhere in ossie_to_thoughtspot.py that is +missing from the classification table fails here, so the question has to be +answered before the key is used, not left to memory. +""" +import re +from pathlib import Path + +from ossie_thoughtspot import constants + +_SRC = Path(__file__).resolve().parent.parent / "src" / "ossie_thoughtspot" + +#: Every top-level custom_extensions[THOUGHTSPOT] key constant in +#: constants.py -- excludes witness-copy constants (a witness is not itself +#: a key this converter classifies; it is the currency check FOR one) and +#: the nested source_parts.{db,schema,db_table} sub-keys, which are never +#: read as standalone top-level payload keys. +_KEY_CONSTANT_RE = re.compile(r'^([A-Z][A-Z_0-9]*)\s*=\s*"', re.MULTILINE) + + +def _all_stash_key_constants() -> list[str]: + text = (_SRC / "constants.py").read_text(encoding="utf-8") + names = _KEY_CONSTANT_RE.findall(text) + return [ + n for n in names + if ("_STASH" in n or n == "STASH_TML_NAME") + and not n.endswith("_WITNESS") + and "SOURCE_PARTS_" not in n + ] + + +def _keys_read_in_reverse_direction() -> set[str]: + """Every stash-key VALUE (the payload string, e.g. "db_column_name" -- + the same shape STASH_KEY_CLASSIFICATION is keyed by) whose constant is + imported into ossie_to_thoughtspot.py. The module only imports names it + actually uses (nothing in this package imports a constant it never + references), so import presence is a reliable proxy for "this key is + read on the Ossie -> TML direction".""" + text = (_SRC / "ossie_to_thoughtspot.py").read_text(encoding="utf-8") + return { + getattr(constants, name) for name in _all_stash_key_constants() + if re.search(rf"\b{name}\b", text) + } + + +def test_every_stash_key_constant_is_a_real_constants_attribute(): + # Guards the scan itself: a typo in _all_stash_key_constants' regex or + # in this file would otherwise silently check nothing. + for name in _all_stash_key_constants(): + assert hasattr(constants, name), name + + +def test_every_key_read_in_the_reverse_direction_is_classified(): + read_keys = _keys_read_in_reverse_direction() + assert read_keys, "expected at least one stash key to be read" + unclassified = sorted(read_keys - set(constants.STASH_KEY_CLASSIFICATION)) + assert unclassified == [], ( + f"stash key(s) read in ossie_to_thoughtspot.py with no entry in " + f"STASH_KEY_CLASSIFICATION: {unclassified} -- classify each as " + f"SHADOWS_DERIVABLE (needs a witness) or INFORMATION_ONLY before " + f"reading it" + ) + + +def test_the_classification_table_names_no_key_that_does_not_exist(): + # The inverse check: every classified key must be a real constant's + # value, so a renamed constant can't leave a stale string behind here. + known_values = {getattr(constants, n) for n in _all_stash_key_constants()} + unknown = sorted(k for k in constants.STASH_KEY_CLASSIFICATION if k not in known_values) + assert unknown == [] + + +def test_every_shadows_derivable_key_has_a_witness_constant_or_documented_self_check(): + """A SHADOWS_DERIVABLE key must be checkable for currency: either a + `_WITNESS` constant exists for it (the `stash.restore` path), or + it is one of the keys documented as self-verifying (its own stashed + value is reconstructed and compared against the live document directly, + the same shape DATASET_STASH_SOURCE_PARTS and STASH_TML_NAME use). + """ + self_verifying = { + "DATASET_STASH_SOURCE_PARTS", "STASH_TML_NAME", "RELATIONSHIP_STASH_REFERENCING_JOIN", + } + all_names = _all_stash_key_constants() + name_by_value = {getattr(constants, n): n for n in all_names} + for key, classification in constants.STASH_KEY_CLASSIFICATION.items(): + if classification is not constants.StashKeyClass.SHADOWS_DERIVABLE: + continue + name = name_by_value[key] + if name in self_verifying: + continue + witness_name = f"{name}_WITNESS" + assert hasattr(constants, witness_name), ( + f"{name} is classified SHADOWS_DERIVABLE but has no " + f"{witness_name} constant and is not listed as self-verifying" + ) + + +def test_derivable_membership_keys_are_information_only_in_value(): + """STASH_KEYS_WITH_DERIVABLE_MEMBERSHIP is a second, orthogonal axis on + top of StashKeyClass, not a replacement for it -- a key there still + needs a primary classification, and it can only sensibly be + INFORMATION_ONLY: a SHADOWS_DERIVABLE key's *value* is already checked + against a witness on every read, which would have caught a membership + problem too (the witness mismatch IS the "this entry no longer + applies" signal). A key found here classified SHADOWS_DERIVABLE would + mean the two axes were mixed up. + """ + for key in constants.STASH_KEYS_WITH_DERIVABLE_MEMBERSHIP: + assert key in constants.STASH_KEY_CLASSIFICATION, key + assert constants.STASH_KEY_CLASSIFICATION[key] is constants.StashKeyClass.INFORMATION_ONLY, key + + +def test_unsurfaced_columns_is_the_known_derivable_membership_case(): + # Regression pin -- the instance that motivated the second axis. + assert constants.DATASET_STASH_UNSURFACED_COLUMNS in constants.STASH_KEYS_WITH_DERIVABLE_MEMBERSHIP + + +def test_sql_output_columns_does_not_share_the_hybrid(): + # Checked directly, not assumed innocent: DATASET_STASH_SQL_OUTPUT_COLUMNS + # is consulted as a per-field dict lookup keyed by the live field's own + # name, never appended as a block the way unsurfaced_columns is, so a + # stale entry is simply never looked up rather than duplicated. + assert constants.DATASET_STASH_SQL_OUTPUT_COLUMNS not in constants.STASH_KEYS_WITH_DERIVABLE_MEMBERSHIP diff --git a/converters/thoughtspot/tests/test_tml.py b/converters/thoughtspot/tests/test_tml.py new file mode 100644 index 00000000..5bfa9af2 --- /dev/null +++ b/converters/thoughtspot/tests/test_tml.py @@ -0,0 +1,379 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + +from ossie_thoughtspot import _yaml +from ossie_thoughtspot.errors import ConversionError +from ossie_thoughtspot.tml import ( + DocumentSet, TmlDocument, block_scalar, dump_document, + dump_document_set, load_document, load_document_set, +) + +TABLE = """\ +guid: tbl-orders-001 +table: + name: ORDERS + db: SALES + schema: PUBLIC + columns: + - name: AMOUNT + db_column_name: AMOUNT + properties: + column_type: MEASURE + db_column_properties: + data_type: DOUBLE +""" + +MODEL = """\ +guid: model-001 +model: + name: Sales + model_tables: + - name: ORDERS + formulas: + - id: formula_Revenue + name: Revenue + expr: sum ( [ORDERS::AMOUNT] ) + columns: + - name: Revenue + formula_id: formula_Revenue + properties: + column_type: MEASURE +""" + + +def _table_named(name): + # Single-quoted YAML scalar: the only escape it needs is doubling a literal + # single quote, so a backslash in `name` (used for Windows-style path cases) + # survives unmangled — a double-quoted scalar would try to interpret it as an + # escape. + escaped = name.replace("'", "''") + return f"table:\n name: '{escaped}'\n db: SALES\n" + + +class TestLoad: + def test_detects_a_table_document(self): + doc = load_document(TABLE) + assert doc.kind == "table" + assert doc.body["name"] == "ORDERS" + assert doc.guid == "tbl-orders-001" + + def test_detects_a_model_document(self): + assert load_document(MODEL).kind == "model" + + def test_a_document_with_no_recognised_root_key_raises(self): + with pytest.raises(ConversionError, match="not a TML document"): + load_document("answer:\n name: Nope\n") + + def test_a_document_with_two_root_kinds_raises(self): + with pytest.raises(ConversionError, match="more than one"): + load_document("table:\n name: A\nmodel:\n name: B\n") + + def test_yaml_1_1_boolean_tokens_survive_as_strings(self): + # A column really can be called `on` — it must not be coerced to a boolean. + doc = load_document("table:\n name: T\n columns:\n - name: 'on'\n") + assert doc.body["columns"][0]["name"] == "on" + + +class TestLoadDocumentSet: + def test_splits_the_model_from_the_tables(self): + ds = load_document_set([("orders.table.tml", TABLE), ("sales.model.tml", MODEL)]) + assert ds.model.body["name"] == "Sales" + assert [t.body["name"] for t in ds.tables] == ["ORDERS"] + + def test_order_of_input_does_not_matter(self): + ds = load_document_set([("sales.model.tml", MODEL), ("orders.table.tml", TABLE)]) + assert ds.model.body["name"] == "Sales" + + def test_table_lookup_by_name(self): + ds = load_document_set([("o", TABLE), ("m", MODEL)]) + assert ds.table_by_name("ORDERS").body["db"] == "SALES" + assert ds.table_by_name("MISSING") is None + + def test_no_model_raises(self): + with pytest.raises(ConversionError, match="no model document"): + load_document_set([("o", TABLE)]) + + def test_two_models_raise(self): + with pytest.raises(ConversionError, match="more than one model"): + load_document_set([("m1", MODEL), ("m2", MODEL)]) + + +class TestDump: + def test_guid_is_never_written(self): + # The single most consequential invariant in this module. + out = dump_document(load_document(TABLE)) + assert "guid" not in out + assert "tbl-orders-001" not in out + + def test_the_kind_key_is_the_document_root(self): + out = dump_document(load_document(TABLE)) + assert out.startswith("table:") + + def test_a_brace_expression_is_written_as_a_block_scalar(self): + # A plain scalar containing `{ }` fails to parse on re-read. + doc = TmlDocument(kind="model", body={ + "name": "M", + "formulas": [{"id": "formula_X", "name": "X", + "expr": block_scalar("last_value ( sum ( [T::c] ) , { [D::d] } )")}], + }, guid=None) + out = dump_document(doc) + assert ">-" in out + assert _yaml.load(out)["model"]["formulas"][0]["expr"].strip() == ( + "last_value ( sum ( [T::c] ) , { [D::d] } )" + ) + + def test_an_on_key_is_quoted(self): + # `on` is a YAML 1.1 reserved word; unquoted it would come back as True. + doc = TmlDocument(kind="table", body={ + "name": "T", + "joins_with": [{"name": "j", "on": "[A::x] = [B::y]", + "type": "INNER", "cardinality": "MANY_TO_ONE"}], + }, guid=None) + out = dump_document(doc) + assert "'on':" in out + assert _yaml.load(out)["table"]["joins_with"][0]["on"] == "[A::x] = [B::y]" + + def test_round_trips_through_load(self): + doc = load_document(TABLE) + assert load_document(dump_document(doc)).body == doc.body + + +class TestDumpDocumentSet: + def test_tables_come_before_the_model(self): + # The model references tables by name, so they must exist first. + ds = load_document_set([("m", MODEL), ("o", TABLE)]) + names = [name for name, _text in dump_document_set(ds)] + assert names == ["ORDERS.table.tml", "Sales.model.tml"] + + def test_every_emitted_document_reloads(self): + ds = load_document_set([("o", TABLE), ("m", MODEL)]) + for _name, text in dump_document_set(ds): + load_document(text) + + +class TestFilenameSafety: + """A table or model name is user-controlled data, and `dump_document_set` turns it + into a filename. What matters is not the string shape but that joining the result + onto an output directory and resolving it can never land outside that directory — + checked with `Path.resolve()` on both sides so a symlinked temp directory (e.g. on + macOS, where `/tmp` itself is a symlink) can't produce a false positive. + """ + + @pytest.mark.parametrize("name", [ + "../../etc/passwd", + "/etc/passwd", + "..\\..\\x", + "A B", + "..", + "", + ]) + def test_stays_inside_the_output_directory(self, tmp_path, name): + ds = load_document_set([("t", _table_named(name)), ("m", MODEL)]) + out_dir = (tmp_path / "intended_output") + out_dir.mkdir() + resolved_out_dir = out_dir.resolve() + for filename, _text in dump_document_set(ds): + target = (out_dir / filename).resolve() + assert target.is_relative_to(resolved_out_dir) + + def test_a_normal_name_is_unchanged(self): + ds = load_document_set([("t", _table_named("ORDERS")), ("m", MODEL)]) + names = [name for name, _text in dump_document_set(ds)] + assert names[0] == "ORDERS.table.tml" + + def test_distinct_names_that_collide_after_sanitising_do_not_overwrite_each_other(self): + # `A/B` and `A\B` both lose their separator to the same replacement character. + ds = load_document_set([ + ("a", _table_named("A/B")), + ("b", _table_named("A\\B")), + ("m", MODEL), + ]) + names = [name for name, _text in dump_document_set(ds)] + table_names = names[:-1] + assert len(table_names) == len(set(table_names)) + assert table_names == ["A_B.table.tml", "A_B-2.table.tml"] + + +class TestFilenameCollisionSafety: + """Sanitising two distinct names onto the same stem is not enough to guarantee + distinct filenames by itself — a disambiguating counter has to be checked against + every filename actually being emitted in this call, not just against how many + times its own stem has been seen, or a counter-suffixed name can land on another + document's real name and one document silently overwrites the other on disk. Every + case below asserts only that the emitted filenames are pairwise distinct, not any + particular suffix, so it keeps holding if the disambiguation scheme changes. + """ + + def test_a_third_name_matching_the_second_names_disambiguated_filename(self): + # `A/B` and `A\B` both sanitise to `A_B` and would naively disambiguate to + # `A_B` and `A_B-2`; a third table literally named `A_B-2` must not be handed + # that same filename. + ds = load_document_set([ + ("a", _table_named("A/B")), + ("b", _table_named("A\\B")), + ("c", _table_named("A_B-2")), + ("m", MODEL), + ]) + names = [name for name, _text in dump_document_set(ds)] + assert len(names) == len(set(names)) + + def test_the_same_three_names_in_a_different_processing_order(self): + ds = load_document_set([ + ("c", _table_named("A_B-2")), + ("a", _table_named("A/B")), + ("b", _table_named("A\\B")), + ("m", MODEL), + ]) + names = [name for name, _text in dump_document_set(ds)] + assert len(names) == len(set(names)) + + def test_four_names_that_all_sanitise_to_the_same_stem(self): + ds = load_document_set([ + ("a", _table_named("A/B")), + ("b", _table_named("A\\B")), + ("c", _table_named("A:B")), + ("d", _table_named("A|B")), + ("m", MODEL), + ]) + names = [name for name, _text in dump_document_set(ds)] + assert len(names) == len(set(names)) + + def test_a_table_sharing_the_models_raw_name(self): + # A table's suffix (`.table.tml`) and the model's (`.model.tml`) differ, so this + # pair can't collide under the current suffix scheme — but both filenames are + # still minted from the same reservation set, and this proves that holds when + # the raw names match too, not only when they happen to differ. + ds = load_document_set([("t", _table_named("Sales")), ("m", MODEL)]) + names = [name for name, _text in dump_document_set(ds)] + assert len(names) == len(set(names)) + + +class TestFilenameLengthCap: + """A ThoughtSpot table or model name has no length limit of its own, but a + filename component does — 255 bytes on most filesystems. A name at or past that + boundary is truncated at mint time rather than left to fail when something tries + to write the file. + """ + + def test_a_very_long_name_is_truncated_to_fit(self): + ds = load_document_set([("t", _table_named("y" * 1000)), ("m", MODEL)]) + names = [name for name, _text in dump_document_set(ds)] + assert len(names[0].encode("utf-8")) <= 255 + + def test_two_long_names_sharing_a_prefix_stay_distinct_after_truncation(self): + # Identical for long enough that truncation collapses them to the same stem — + # the two names differ only in their last four characters, well past where a + # 255-byte cap cuts them off. + name_a = ("x" * 250) + "AAAA" + name_b = ("x" * 250) + "BBBB" + ds = load_document_set([ + ("a", _table_named(name_a)), + ("b", _table_named(name_b)), + ("m", MODEL), + ]) + names = [name for name, _text in dump_document_set(ds)] + assert len(names) == len(set(names)) + for name in names: + assert len(name.encode("utf-8")) <= 255 + + +class TestNestedGuidStripping: + """The guid rule applies at every depth of the body, not only the document root — + a nested guid is silently ignored on import, and ThoughtSpot creates a duplicate + object rather than updating the one that already exists. + """ + + def test_a_guid_one_level_deep_is_stripped(self): + doc = TmlDocument(kind="table", body={"name": "T", "guid": "should-not-survive"}, guid=None) + out = dump_document(doc) + assert "should-not-survive" not in out + assert "guid" not in out + + def test_a_guid_inside_a_list_of_column_entries_is_stripped(self): + doc = TmlDocument(kind="table", body={ + "name": "T", + "columns": [ + {"name": "A", "guid": "col-a-guid"}, + {"name": "B", "guid": "col-b-guid"}, + ], + }, guid=None) + out = dump_document(doc) + assert "col-a-guid" not in out + assert "col-b-guid" not in out + assert load_document(out).body["columns"] == [{"name": "A"}, {"name": "B"}] + + def test_a_document_with_no_guid_anywhere_is_unchanged(self): + doc = TmlDocument(kind="table", body={"name": "T", "columns": [{"name": "A"}]}, guid=None) + out = dump_document(doc) + assert load_document(out).body == doc.body + + def test_dump_document_does_not_mutate_the_callers_body(self): + body = { + "name": "T", + "guid": "root-level-in-body", + "columns": [{"name": "A", "guid": "col-guid"}], + } + doc = TmlDocument(kind="table", body=body, guid=None) + dump_document(doc) + assert body["guid"] == "root-level-in-body" + assert body["columns"][0]["guid"] == "col-guid" + + def test_fqn_is_left_alone(self): + doc = TmlDocument(kind="model", body={ + "name": "M", + "model_tables": [{"name": "T", "fqn": "db.schema.t", "guid": "should-strip"}], + }, guid=None) + out = dump_document(doc) + assert "db.schema.t" in out + assert "should-strip" not in out + + +class TestAdditionalCoverage: + """Two cases judged most likely to bite in practice. + + A document whose body is a list rather than a mapping exercises a defensive branch + in `load_document` that no other test here reaches — worth proving the guard + actually fires rather than trusting it by inspection. + + Duplicate table names in one document set are a plausible real-world input (the same + physical table re-emitted by an upstream step, or two directories merged without a + dedupe pass). `table_by_name` still silently returns only the first match on such a + duplicate — a known limitation of set assembly, not of this module's filename layer, + since nothing here can tell two identically-named tables apart. The filename + collision duplicate names used to also cause is gone: it is resolved by the same + scheme `dump_document_set` uses for any other filename collision (see + `TestFilenameCollisionSafety`). + """ + + def test_a_document_whose_body_is_a_list_raises(self): + with pytest.raises(ConversionError, match="must be a mapping"): + load_document("table:\n- name: A\n- name: B\n") + + def test_duplicate_table_names_still_shadow_on_lookup_but_no_longer_collide_on_dump(self): + table_a = "table:\n name: ORDERS\n db: SALES_A\n" + table_b = "table:\n name: ORDERS\n db: SALES_B\n" + ds = load_document_set([("a", table_a), ("b", table_b), ("m", MODEL)]) + + # Lookup still silently returns only the first match — a known limitation of + # set assembly, unrelated to the filename layer this module owns. + assert ds.table_by_name("ORDERS").body["db"] == "SALES_A" + + # But dumping no longer loses one of them to a filename collision. + names = [name for name, _text in dump_document_set(ds)] + assert len(names) == len(set(names)) diff --git a/converters/thoughtspot/tests/test_tml_to_ossie.py b/converters/thoughtspot/tests/test_tml_to_ossie.py new file mode 100644 index 00000000..57cffd59 --- /dev/null +++ b/converters/thoughtspot/tests/test_tml_to_ossie.py @@ -0,0 +1,1379 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the assembler: datasets, the cross-model resolver, relationships, +key derivation, the model-scope stash, and `convert()`'s public entry point. + +Fixtures build `TmlDocument`/`DocumentSet` objects directly (as +test_tml_to_ossie_fields.py and test_tml_to_ossie_metrics.py build raw column +dicts), rather than round-tripping through YAML text -- `convert()`'s input +contract is the dataclass, not the text format `tml.py` parses separately and +already tests on its own. +""" +import json +from pathlib import Path + +import pytest + +from ossie_thoughtspot import stash +from ossie_thoughtspot.constants import ( + DATASET_STASH_ALIAS, + DATASET_STASH_SQL_OUTPUT_COLUMNS, + DATASET_STASH_TABLE_NAME, + DATASET_STASH_TML_OBJECT, + DATASET_STASH_UNSURFACED_COLUMNS, + FIELD_STASH_COLUMN_PROPERTIES, + FIELD_STASH_DATA_TYPE, + FIELD_STASH_DB_COLUMN_NAME, + MODEL_STASH_ACTION_OBJECT_ASSOCIATIONS, + MODEL_STASH_COLUMN_GROUPS, + MODEL_STASH_CONSTRAINTS, + MODEL_STASH_FILTERS, + MODEL_STASH_LESSON_PLANS, + MODEL_STASH_MODEL_JOINS_WITH, + MODEL_STASH_PARAMETERS, + MODEL_STASH_UNATTRIBUTED_FORMULAS, + MODEL_STASH_UNREPRESENTABLE_JOINS, + RELATIONSHIP_STASH_CARDINALITY, + RELATIONSHIP_STASH_ENDPOINTS_SWAPPED, + RELATIONSHIP_STASH_ENDPOINTS_SWAPPED_WITNESS, + RELATIONSHIP_STASH_JOIN_SHAPE, + RELATIONSHIP_STASH_ON_EXPRESSION, + RELATIONSHIP_STASH_REFERENCING_JOIN, + RELATIONSHIP_STASH_TYPE, +) +from ossie_thoughtspot.errors import ConversionError +from ossie_thoughtspot.tml import DocumentSet, TmlDocument +from ossie_thoughtspot.tml_to_ossie import OssieConversion, convert + + +def _table(name, db="SALES", schema="PUBLIC", db_table=None, columns=None, + connection="My Snowflake", **extra): + body = { + "name": name, + "db": db, + "schema": schema, + "db_table": db_table or name, + "connection": {"name": connection}, + "columns": columns or [], + } + body.update(extra) + return TmlDocument(kind="table", body=body, guid=None) + + +def _column(name, db_column_name=None, data_type="VARCHAR"): + return { + "name": name, + "db_column_name": db_column_name or name, + "db_column_properties": {"data_type": data_type}, + } + + +def _sql_view(name, sql_query="SELECT 1", columns=None, connection="My Snowflake", **extra): + body = { + "name": name, + "sql_query": sql_query, + "connection": {"name": connection}, + "sql_view_columns": columns or [], + } + body.update(extra) + return TmlDocument(kind="sql_view", body=body, guid=None) + + +def _sql_view_column(name, sql_output_column=None, data_type="VARCHAR"): + return { + "name": name, + "sql_output_column": sql_output_column or name, + "db_column_properties": {"data_type": data_type}, + } + + +def _attribute(name, column_id): + return {"name": name, "column_id": column_id, "properties": {"column_type": "ATTRIBUTE"}} + + +def _model(name="Sales Analytics", model_tables=None, columns=None, formulas=None, **extra): + body: dict = {"name": name, "model_tables": model_tables or [], "columns": columns or []} + if formulas is not None: + body["formulas"] = formulas + body.update(extra) + return TmlDocument(kind="model", body=body, guid=None) + + +def _document_set(model_doc, *table_docs): + return DocumentSet(model=model_doc, tables=tuple(table_docs)) + + +def _own_stash(obj): + """The parsed THOUGHTSPOT custom_extensions payload on `obj`, or None.""" + for entry in obj.get("custom_extensions") or []: + if entry["vendor_name"] == "THOUGHTSPOT": + return json.loads(entry["data"]) + return None + + +class TestMinimalConversion: + def test_a_minimal_document_set_converts(self): + # The mapping document's *Worked shape*: one dataset, one attribute, + # one metric. + orders = _table( + "ORDERS", + columns=[ + _column("Order Date", "O_ORDERDATE", "DATE"), + _column("Amount", "O_TOTALPRICE", "DOUBLE"), + ], + ) + model = _model( + name="Sales Analytics", + model_tables=[{"name": "ORDERS"}], + columns=[ + _attribute("Order Date", "ORDERS::Order Date"), + _attribute("Amount", "ORDERS::Amount"), + {"name": "total_revenue", "formula_id": "formula_total_revenue", + "properties": {"column_type": "MEASURE", "aggregation": "SUM"}}, + ], + formulas=[{"id": "formula_total_revenue", "name": "total_revenue", + "expr": "sum ( [ORDERS::Amount] )"}], + ) + + result = convert(_document_set(model, orders)) + + assert isinstance(result, OssieConversion) + document = result.model + assert document["version"] == "0.2.0.dev0" + semantic_model = document["semantic_model"][0] + assert semantic_model["name"] == "sales_analytics" + + assert len(semantic_model["datasets"]) == 1 + dataset = semantic_model["datasets"][0] + assert dataset["name"] == "ORDERS" + assert dataset["source"] == "SALES.PUBLIC.ORDERS" + assert {f["name"] for f in dataset["fields"]} == {"order_date", "amount"} + assert "primary_key" not in dataset + assert "relationships" not in semantic_model + + assert len(semantic_model["metrics"]) == 1 + metric = semantic_model["metrics"][0] + assert metric["name"] == "total_revenue" + dialects = {d["dialect"]: d["expression"] for d in metric["expression"]["dialects"]} + assert dialects["THOUGHTSPOT"] == "sum ( [ORDERS::Amount] )" + + def test_dataset_source_is_db_schema_table(self): + orders = _table("ORDERS", db="SALES", schema="PUBLIC", db_table="ORDERS_FACT") + model = _model(model_tables=[{"name": "ORDERS"}]) + result = convert(_document_set(model, orders)) + dataset = result.model["semantic_model"][0]["datasets"][0] + assert dataset["source"] == "SALES.PUBLIC.ORDERS_FACT" + + +class TestAliasPrefix: + def test_an_alias_is_used_for_the_reference_prefix_when_present(self): + # model_tables[].alias overrides name in column_id prefixes; getting + # this wrong breaks every reference in an aliased model. A plain, + # unaliased table sits alongside it to prove that case still works. + ship_to = _table("ADDRESSES", columns=[_column("City", "CITY", "VARCHAR")]) + orders = _table("ORDERS", columns=[_column("Amount", "AMOUNT", "DOUBLE")]) + model = _model( + model_tables=[ + {"name": "ADDRESSES", "alias": "ShippingAddress"}, + {"name": "ORDERS"}, + ], + columns=[ + _attribute("Ship City", "ShippingAddress::City"), + _attribute("Amount", "ORDERS::Amount"), + {"name": "city_count", "formula_id": "formula_city_count", + "properties": {"column_type": "MEASURE", "aggregation": "COUNT_DISTINCT"}}, + ], + formulas=[{"id": "formula_city_count", "name": "city_count", + "expr": "[ShippingAddress::City]"}], + ) + + result = convert(_document_set(model, ship_to, orders)) + semantic_model = result.model["semantic_model"][0] + datasets = {d["name"]: d for d in semantic_model["datasets"]} + + assert set(datasets) == {"ShippingAddress", "ORDERS"} + + aliased = datasets["ShippingAddress"] + assert aliased["fields"][0]["name"] == "ship_city" + assert aliased["fields"][0]["datatype"] == "String" # table_lookup used the alias too + aliased_stash = _own_stash(aliased) + assert aliased_stash[DATASET_STASH_ALIAS] == "ShippingAddress" + assert aliased_stash[DATASET_STASH_TABLE_NAME] == "ADDRESSES" + + # The unaliased case is unaffected: dataset name is the plain table + # name, and there is no alias/table_name in its stash. + plain = datasets["ORDERS"] + assert plain["fields"][0]["name"] == "amount" + plain_stash = _own_stash(plain) or {} + assert DATASET_STASH_ALIAS not in plain_stash + + # The metric's expression resolves through the ALIAS, not "ADDRESSES" -- + # proof `resolve()` keys off the alias end to end, not just the + # column_id -> field mapping. The dataset qualifier preserves the + # alias's exact case (Dataset-level mapping's "name...exactly, + # case-sensitive" rule); the column itself is the WAREHOUSE name + # ("CITY", from _column("City", "CITY", ...)), not the Ossie + # field's own display-derived identifier ("ship_city") -- a bare + # reference's portable sibling names the physical column, per the + # mapping document. + metric = semantic_model["metrics"][0] + dialects = {d["dialect"]: d["expression"] for d in metric["expression"]["dialects"]} + assert dialects["ANSI_SQL"] == "COUNT(DISTINCT ShippingAddress.CITY)" + + # No unresolved-reference issue should have fired for the aliased column. + assert not any(i["code"] == "TS-EXPR-UNRESOLVED" for i in result.issues.as_dicts()) + + +class TestKeyDerivation: + def test_an_equality_join_derives_a_primary_key(self): + customers = _table("CUSTOMERS", columns=[_column("Id", "ID", "INT64")]) + orders = _table("ORDERS", columns=[_column("Customer Id", "CUSTOMER_ID", "INT64")]) + model = _model( + model_tables=[ + {"name": "ORDERS", "joins": [{ + "with": "CUSTOMERS", + "on": "[ORDERS::Customer Id] = [CUSTOMERS::Id]", + "type": "INNER", + "cardinality": "MANY_TO_ONE", + }]}, + {"name": "CUSTOMERS"}, + ], + ) + + result = convert(_document_set(model, orders, customers)) + semantic_model = result.model["semantic_model"][0] + customers_ds = next(d for d in semantic_model["datasets"] if d["name"] == "CUSTOMERS") + + assert customers_ds["primary_key"] == ["Id"] + assert customers_ds["unique_keys"] == [["Id"]] + + rel = semantic_model["relationships"][0] + assert rel["from"] == "ORDERS" + assert rel["to"] == "CUSTOMERS" + assert rel["from_columns"] == ["Customer Id"] + assert rel["to_columns"] == ["Id"] + rel_stash = _own_stash(rel) + assert rel_stash[RELATIONSHIP_STASH_TYPE] == "INNER" + assert rel_stash[RELATIONSHIP_STASH_CARDINALITY] == "MANY_TO_ONE" + assert rel_stash[RELATIONSHIP_STASH_JOIN_SHAPE] == "inline" + assert RELATIONSHIP_STASH_ON_EXPRESSION not in rel_stash + + def test_a_non_equality_join_derives_no_key_and_stashes_the_condition(self): + # A residual-predicate (as-of) join is to-one only + # because of the narrowing -- its equality columns alone are not + # unique, so no key is derived and no relationship is emitted at all + # (the condition has zero equality pairs). + rates = _table("FX_RATES", columns=[_column("Ccy", "CCY"), _column("Effective Date", "EFFECTIVE_DATE", "DATE")]) + orders = _table("ORDERS", columns=[_column("Order Date", "ORDER_DATE", "DATE")]) + on_expr = "[ORDERS::Order Date] >= [FX_RATES::Effective Date]" + model = _model( + model_tables=[ + {"name": "ORDERS", "joins": [{ + "with": "FX_RATES", "on": on_expr, + "type": "INNER", "cardinality": "MANY_TO_ONE", + }]}, + {"name": "FX_RATES"}, + ], + ) + + result = convert(_document_set(model, orders, rates)) + semantic_model = result.model["semantic_model"][0] + fx_ds = next(d for d in semantic_model["datasets"] if d["name"] == "FX_RATES") + + assert "primary_key" not in fx_ds + assert "unique_keys" not in fx_ds + assert "relationships" not in semantic_model + + model_stash = _own_stash(semantic_model) + unrep = model_stash[MODEL_STASH_UNREPRESENTABLE_JOINS][0] + assert unrep["from"] == "ORDERS" + assert unrep["to"] == "FX_RATES" + assert unrep[RELATIONSHIP_STASH_ON_EXPRESSION] == on_expr + assert any( + i["code"] == "TS-JOIN-UNREPRESENTABLE" and "FX_RATES" in i["message"] + for i in result.issues.as_dicts() + ) + + def test_a_composite_equality_join_derives_a_composite_key(self): + customers = _table("CUSTOMERS", columns=[_column("Region"), _column("Id", "ID", "INT64")]) + orders = _table("ORDERS", columns=[_column("Region"), _column("Customer Id", "CUSTOMER_ID", "INT64")]) + on_expr = "[ORDERS::Region] = [CUSTOMERS::Region] and [ORDERS::Customer Id] = [CUSTOMERS::Id]" + model = _model( + model_tables=[ + {"name": "ORDERS", "joins": [{ + "with": "CUSTOMERS", "on": on_expr, + "type": "INNER", "cardinality": "MANY_TO_ONE", + }]}, + {"name": "CUSTOMERS"}, + ], + ) + + result = convert(_document_set(model, orders, customers)) + semantic_model = result.model["semantic_model"][0] + customers_ds = next(d for d in semantic_model["datasets"] if d["name"] == "CUSTOMERS") + + assert customers_ds["primary_key"] == ["Region", "Id"] + rel = semantic_model["relationships"][0] + assert rel["from_columns"] == ["Region", "Customer Id"] + assert rel["to_columns"] == ["Region", "Id"] + + +class TestOneToManyEndpointSwap: + """core-spec/spec.yaml requires a Relationship's `from` to name the many + side and `to` the one side, but TML's own `from`/`to` -- the + model_tables[] entry a join is declared under, and its `with` target -- + do not encode which side is which; `cardinality` does. A `ONE_TO_MANY` + join is the one case where TML's `from` names the one side and `to` + names the many side: the wrong way around for Ossie's spec, so its + emitted endpoints are swapped to compensate. `MANY_TO_ONE`/`ONE_TO_ONE` + are already oriented correctly and must be left alone. + """ + + def test_one_to_many_swaps_the_relationships_endpoints(self): + # A customer has many orders: TML declares this from the "one" side + # (CUSTOMERS), naming ORDERS as `with` and ONE_TO_MANY as the + # cardinality -- so from=CUSTOMERS is the one side and to=ORDERS is + # the many side, backwards for Ossie's spec. + customers = _table("CUSTOMERS", columns=[_column("Id", "ID", "INT64")]) + orders = _table("ORDERS", columns=[_column("Customer Id", "CUSTOMER_ID", "INT64")]) + model = _model( + model_tables=[ + {"name": "CUSTOMERS", "joins": [{ + "with": "ORDERS", + "on": "[CUSTOMERS::Id] = [ORDERS::Customer Id]", + "type": "INNER", + "cardinality": "ONE_TO_MANY", + }]}, + {"name": "ORDERS"}, + ], + ) + + result = convert(_document_set(model, customers, orders)) + semantic_model = result.model["semantic_model"][0] + + rel = semantic_model["relationships"][0] + # Swapped: the many side (ORDERS) is `from`, the one side + # (CUSTOMERS) is `to` -- the reverse of how the join is declared. + assert rel["from"] == "ORDERS" + assert rel["to"] == "CUSTOMERS" + assert rel["from_columns"] == ["Customer Id"] + assert rel["to_columns"] == ["Id"] + # The inline join's synthesized name reflects the emitted (swapped) + # from/to, not the TML declaration order. + assert rel["name"] == "ORDERS_to_CUSTOMERS" + + rel_stash = _own_stash(rel) + assert rel_stash[RELATIONSHIP_STASH_CARDINALITY] == "ONE_TO_MANY" + assert rel_stash[RELATIONSHIP_STASH_ENDPOINTS_SWAPPED] is True + assert rel_stash[RELATIONSHIP_STASH_ENDPOINTS_SWAPPED_WITNESS] == [ + "ORDERS", "CUSTOMERS", ["Customer Id"], ["Id"], + ] + + # Key derivation follows the swap: the key belongs to the one side + # (CUSTOMERS), which is now `to`. + customers_ds = next(d for d in semantic_model["datasets"] if d["name"] == "CUSTOMERS") + assert customers_ds["primary_key"] == ["Id"] + assert customers_ds["unique_keys"] == [["Id"]] + orders_ds = next(d for d in semantic_model["datasets"] if d["name"] == "ORDERS") + assert "primary_key" not in orders_ds + + def test_many_to_one_is_not_swapped(self): + customers = _table("CUSTOMERS", columns=[_column("Id", "ID", "INT64")]) + orders = _table("ORDERS", columns=[_column("Customer Id", "CUSTOMER_ID", "INT64")]) + model = _model( + model_tables=[ + {"name": "ORDERS", "joins": [{ + "with": "CUSTOMERS", + "on": "[ORDERS::Customer Id] = [CUSTOMERS::Id]", + "type": "INNER", + "cardinality": "MANY_TO_ONE", + }]}, + {"name": "CUSTOMERS"}, + ], + ) + + result = convert(_document_set(model, orders, customers)) + rel = result.model["semantic_model"][0]["relationships"][0] + + assert rel["from"] == "ORDERS" + assert rel["to"] == "CUSTOMERS" + assert rel["from_columns"] == ["Customer Id"] + assert rel["to_columns"] == ["Id"] + assert rel["name"] == "ORDERS_to_CUSTOMERS" + + rel_stash = _own_stash(rel) + assert rel_stash[RELATIONSHIP_STASH_CARDINALITY] == "MANY_TO_ONE" + assert RELATIONSHIP_STASH_ENDPOINTS_SWAPPED not in rel_stash + assert RELATIONSHIP_STASH_ENDPOINTS_SWAPPED_WITNESS not in rel_stash + + def test_one_to_one_is_not_swapped(self): + people = _table("PEOPLE", columns=[_column("Id", "ID", "INT64")]) + profiles = _table("PROFILES", columns=[_column("Person Id", "PERSON_ID", "INT64")]) + model = _model( + model_tables=[ + {"name": "PROFILES", "joins": [{ + "with": "PEOPLE", + "on": "[PROFILES::Person Id] = [PEOPLE::Id]", + "type": "INNER", + "cardinality": "ONE_TO_ONE", + }]}, + {"name": "PEOPLE"}, + ], + ) + + result = convert(_document_set(model, profiles, people)) + rel = result.model["semantic_model"][0]["relationships"][0] + + assert rel["from"] == "PROFILES" + assert rel["to"] == "PEOPLE" + assert rel["from_columns"] == ["Person Id"] + assert rel["to_columns"] == ["Id"] + + rel_stash = _own_stash(rel) + assert rel_stash[RELATIONSHIP_STASH_CARDINALITY] == "ONE_TO_ONE" + assert RELATIONSHIP_STASH_ENDPOINTS_SWAPPED not in rel_stash + + def test_a_referencing_shaped_one_to_many_join_keeps_its_own_name(self): + # The hybrid shape (referencing_join plus an inline cardinality + # override): the name comes from the Table's own joins_with[] entry, + # not from/to dataset names, and is untouched by the endpoint swap. + customers = _table( + "CUSTOMERS", + columns=[_column("Id", "ID", "INT64")], + joins_with=[{ + "name": "customers_to_orders", + "destination": {"name": "ORDERS"}, + "on": "[CUSTOMERS::Id] = [ORDERS::Customer Id]", + "type": "INNER", + "cardinality": "MANY_TO_ONE", + }], + ) + orders = _table("ORDERS", columns=[_column("Customer Id", "CUSTOMER_ID", "INT64")]) + model = _model( + model_tables=[ + {"name": "CUSTOMERS", "joins": [{ + "referencing_join": "customers_to_orders", "cardinality": "ONE_TO_MANY", + }]}, + {"name": "ORDERS"}, + ], + ) + + result = convert(_document_set(model, customers, orders)) + rel = result.model["semantic_model"][0]["relationships"][0] + + assert rel["name"] == "customers_to_orders" + assert rel["from"] == "ORDERS" + assert rel["to"] == "CUSTOMERS" + assert rel["from_columns"] == ["Customer Id"] + assert rel["to_columns"] == ["Id"] + + rel_stash = _own_stash(rel) + assert rel_stash[RELATIONSHIP_STASH_CARDINALITY] == "ONE_TO_MANY" + assert rel_stash[RELATIONSHIP_STASH_ENDPOINTS_SWAPPED] is True + assert rel_stash[RELATIONSHIP_STASH_JOIN_SHAPE] == "referencing_with_inline_attrs" + assert rel_stash[RELATIONSHIP_STASH_REFERENCING_JOIN] == "customers_to_orders" + + +class TestUnattributedFormulas: + def test_a_multi_dataset_formula_is_not_attributed_and_raises_an_issue(self): + orders = _table("ORDERS", columns=[_column("Amount", "AMOUNT", "DOUBLE")]) + customers = _table("CUSTOMERS", columns=[_column("Discount", "DISCOUNT", "DOUBLE")]) + expr = "[ORDERS::Amount] - [CUSTOMERS::Discount]" + model = _model( + model_tables=[{"name": "ORDERS"}, {"name": "CUSTOMERS"}], + columns=[ + _attribute("Amount", "ORDERS::Amount"), + _attribute("Discount", "CUSTOMERS::Discount"), + {"name": "Net Amount", "formula_id": "formula_net", + "properties": {"column_type": "ATTRIBUTE"}}, + ], + formulas=[{"id": "formula_net", "name": "Net Amount", "expr": expr}], + ) + + result = convert(_document_set(model, orders, customers)) + semantic_model = result.model["semantic_model"][0] + + field_names = {f["name"] for d in semantic_model["datasets"] for f in d.get("fields", [])} + assert "net_amount" not in field_names + + model_stash = _own_stash(semantic_model) + unattributed = model_stash[MODEL_STASH_UNATTRIBUTED_FORMULAS] + assert len(unattributed) == 1 + assert unattributed[0]["name"] == "Net Amount" + assert unattributed[0]["expr"] == expr + + assert any(i["code"] == "TS-FIELD-UNATTRIBUTED" for i in result.issues.as_dicts()) + + +class TestStashProtocol: + def test_other_vendors_custom_extensions_pass_through_untouched(self): + # `convert()`'s own objects must stay compatible with a further + # write_stash call from another vendor's tooling -- exercised on a + # dataset dict `convert()` actually produced. + orders = _table("ORDERS") + model = _model(model_tables=[{"name": "ORDERS"}]) + result = convert(_document_set(model, orders)) + + dataset = result.model["semantic_model"][0]["datasets"][0] + dataset.setdefault("custom_extensions", []).append( + {"vendor_name": "SNOWFLAKE", "data": '{"x": 1}'} + ) + merged = stash.write_stash(dataset, {"extra": "value"}) + vendor_names = {e["vendor_name"] for e in merged["custom_extensions"]} + assert vendor_names == {"THOUGHTSPOT", "SNOWFLAKE"} + foreign = next(e for e in merged["custom_extensions"] if e["vendor_name"] == "SNOWFLAKE") + assert foreign == {"vendor_name": "SNOWFLAKE", "data": '{"x": 1}'} + + def test_no_guid_obj_id_or_fqn_appears_anywhere_in_the_output(self): + # Nested guids on the model_tables[] entry (fqn) and the model + # document root (guid) are present in the source and must never leak + # into the output -- not only into the stash, but anywhere at all. + orders = _table("ORDERS", columns=[_column("Amount", "AMOUNT", "DOUBLE")]) + customers = _table("CUSTOMERS", columns=[_column("Id", "ID", "INT64")]) + model_body = { + "name": "Sales", + "model_tables": [ + {"name": "ORDERS", "fqn": "abc-123-fqn", "obj_id": "obj-1", + "joins": [{"with": "CUSTOMERS", + "on": "[ORDERS::Amount] = [CUSTOMERS::Id]", + "cardinality": "MANY_TO_ONE"}]}, + {"name": "CUSTOMERS", "fqn": "def-456-fqn"}, + ], + "columns": [_attribute("Amount", "ORDERS::Amount")], + } + model = TmlDocument(kind="model", body=model_body, guid="model-guid-1") + + result = convert(_document_set(model, orders, customers)) + serialised = json.dumps(result.model) + for forbidden in ("guid", "obj_id", "fqn"): + assert forbidden not in serialised, forbidden + + def test_an_empty_payload_writes_no_stash_entry(self): + # A model with an already-normalised name and no ThoughtSpot-only + # model-scope properties stays clean at model scope. + orders = _table("ORDERS", columns=[_column("Amount", "AMOUNT", "DOUBLE")], connection="Snowflake") + model = _model( + name="sales", # already a valid identifier -- normalise() is a no-op + model_tables=[{"name": "ORDERS"}], + columns=[_attribute("Amount", "ORDERS::Amount")], + ) + result = convert(_document_set(model, orders)) + semantic_model = result.model["semantic_model"][0] + assert "custom_extensions" not in semantic_model + + +class TestSchemaValidation: + def test_the_output_validates_against_the_upstream_schema(self): + jsonschema = pytest.importorskip("jsonschema") + schema_path = Path(__file__).resolve().parents[3] / "core-spec" / "ossie-schema.json" + with open(schema_path) as fh: + schema = json.load(fh) + + orders = _table( + "ORDERS", + columns=[_column("Amount", "AMOUNT", "DOUBLE"), _column("Order Date", "ORDER_DATE", "DATE")], + ) + customers = _table("CUSTOMERS", columns=[_column("Id", "ID", "INT64")]) + model = _model( + model_tables=[ + {"name": "ORDERS", "joins": [{ + "with": "CUSTOMERS", + "on": "[ORDERS::Amount] = [CUSTOMERS::Id]", + "type": "INNER", + "cardinality": "MANY_TO_ONE", + }]}, + {"name": "CUSTOMERS"}, + ], + columns=[ + _attribute("Amount", "ORDERS::Amount"), + _attribute("Order Date", "ORDERS::Order Date"), + {"name": "total_revenue", "formula_id": "formula_rev", + "properties": {"column_type": "MEASURE", "aggregation": "SUM"}}, + ], + formulas=[{"id": "formula_rev", "name": "total_revenue", + "expr": "sum ( [ORDERS::Amount] )"}], + ) + + result = convert(_document_set(model, orders, customers)) + jsonschema.Draft202012Validator(schema).validate(result.model) + + +class TestOwnChoice: + """Two cases the required list doesn't name, chosen because they attack + code this task adds that nothing else exercises.""" + + def test_a_referencing_join_resolves_via_the_tables_joins_with(self): + # The OTHER TML join shape (Table joins_with[] + Model referencing_join) + # is real and documented but untouched by every other required test, + # which all use inline joins. If _convert_join's referencing-join + # branch has a bug, nothing else here would catch it. + customers = _table("CUSTOMERS", columns=[_column("Id", "ID", "INT64")]) + orders = _table( + "ORDERS", + columns=[_column("Customer Id", "CUSTOMER_ID", "INT64")], + joins_with=[{ + "name": "orders_to_customers", + "destination": {"name": "CUSTOMERS"}, + "on": "[ORDERS::Customer Id] = [CUSTOMERS::Id]", + "type": "INNER", + "cardinality": "MANY_TO_ONE", + }], + ) + model = _model( + model_tables=[ + {"name": "ORDERS", "joins": [{"referencing_join": "orders_to_customers"}]}, + {"name": "CUSTOMERS"}, + ], + ) + + result = convert(_document_set(model, orders, customers)) + semantic_model = result.model["semantic_model"][0] + + rel = semantic_model["relationships"][0] + assert rel["name"] == "orders_to_customers" + assert rel["from"] == "ORDERS" + assert rel["to"] == "CUSTOMERS" + assert rel["from_columns"] == ["Customer Id"] + assert rel["to_columns"] == ["Id"] + rel_stash = _own_stash(rel) + assert rel_stash[RELATIONSHIP_STASH_JOIN_SHAPE] == "referencing" + assert rel_stash[RELATIONSHIP_STASH_REFERENCING_JOIN] == "orders_to_customers" + assert rel_stash[RELATIONSHIP_STASH_TYPE] == "INNER" + assert rel_stash[RELATIONSHIP_STASH_CARDINALITY] == "MANY_TO_ONE" + + customers_ds = next(d for d in semantic_model["datasets"] if d["name"] == "CUSTOMERS") + assert customers_ds["primary_key"] == ["Id"] + + def test_a_malformed_join_condition_is_caught_and_the_conversion_continues(self): + # Lesson carried into this task: a malformed reference must not abort + # the whole model. This is the join-condition version of that rule -- + # untested anywhere else, since every other join test uses a clean + # condition. A triple-colon reference is ambiguous per + # identifiers.split_column_ref and raises inside _parse_join_condition. + orders = _table("ORDERS", columns=[_column("Amount", "AMOUNT", "DOUBLE")]) + customers = _table("CUSTOMERS", columns=[_column("Id", "ID", "INT64")]) + bad_condition = "[ORDERS:::Bad Ref] = [CUSTOMERS::Id]" + model = _model( + model_tables=[ + {"name": "ORDERS", "joins": [{ + "with": "CUSTOMERS", "on": bad_condition, "cardinality": "MANY_TO_ONE", + }]}, + {"name": "CUSTOMERS"}, + ], + columns=[_attribute("Amount", "ORDERS::Amount")], + ) + + result = convert(_document_set(model, orders, customers)) + semantic_model = result.model["semantic_model"][0] + + # The rest of the model still converts. + assert len(semantic_model["datasets"]) == 2 + orders_ds = next(d for d in semantic_model["datasets"] if d["name"] == "ORDERS") + assert orders_ds["fields"][0]["name"] == "amount" + assert "relationships" not in semantic_model + + model_stash = _own_stash(semantic_model) + unrep = model_stash[MODEL_STASH_UNREPRESENTABLE_JOINS][0] + assert unrep[RELATIONSHIP_STASH_ON_EXPRESSION] == bad_condition + assert any(i["code"] == "TS-JOIN-MALFORMED" for i in result.issues.as_dicts()) + + +class TestUnconsumedColumnProperties: + """Neither convert_field nor convert_metric preserves a ThoughtSpot-only + column property (`index_type`, `value_casing`, ...): they never returned + a fragment for the assembler to merge, so it vanished with no issue. The + assembler now stashes the complement of what the converter actually + reads, rather than an enumeration of known ThoughtSpot-only names.""" + + # db_column_name matches the display name exactly, so these tests' + # "no extra properties" premises are not disturbed by the separate + # db_column_name-preservation stash (TestPhysicalColumnStash covers + # that dimension on its own fixtures). + ORDERS = _table("ORDERS", columns=[_column("Amount", "Amount", "DOUBLE")]) + + def _convert_one(self, properties): + model = _model( + model_tables=[{"name": "ORDERS"}], + columns=[{"name": "Amount", "column_id": "ORDERS::Amount", "properties": properties}], + ) + return convert(_document_set(model, self.ORDERS)) + + def test_thoughtspot_only_properties_round_trip_into_column_properties(self): + result = self._convert_one( + {"column_type": "ATTRIBUTE", "index_type": "DONT_INDEX", "value_casing": "UPPER"} + ) + field = result.model["semantic_model"][0]["datasets"][0]["fields"][0] + assert _own_stash(field)[FIELD_STASH_COLUMN_PROPERTIES] == { + "index_type": "DONT_INDEX", "value_casing": "UPPER", + } + + def test_a_metric_with_only_consumed_properties_gets_no_column_properties_key(self): + result = self._convert_one({"column_type": "MEASURE", "aggregation": "SUM"}) + metric = result.model["semantic_model"][0]["metrics"][0] + stashed = _own_stash(metric) or {} + assert FIELD_STASH_COLUMN_PROPERTIES not in stashed + + def test_a_column_with_no_extra_properties_gets_no_extension_entry(self): + result = self._convert_one({"column_type": "ATTRIBUTE"}) + field = result.model["semantic_model"][0]["datasets"][0]["fields"][0] + assert "custom_extensions" not in field + + def test_an_unknown_invented_property_name_is_preserved(self): + # The fail-closed property itself: a name this converter has never + # heard of must still survive, because the rule is "everything not + # consumed", not "everything on a known list". + result = self._convert_one( + {"column_type": "ATTRIBUTE", "a_property_ossie_thoughtspot_has_never_seen": 42} + ) + field = result.model["semantic_model"][0]["datasets"][0]["fields"][0] + assert _own_stash(field)[FIELD_STASH_COLUMN_PROPERTIES] == { + "a_property_ossie_thoughtspot_has_never_seen": 42 + } + + def test_the_metric_side_behaves_the_same_as_the_field_side(self): + result = self._convert_one( + {"column_type": "MEASURE", "aggregation": "SUM", "index_type": "DONT_INDEX"} + ) + metric = result.model["semantic_model"][0]["metrics"][0] + assert _own_stash(metric)[FIELD_STASH_COLUMN_PROPERTIES] == {"index_type": "DONT_INDEX"} + + def test_identity_shaped_content_nested_in_a_property_value_is_dropped_not_stashed(self): + # Found while re-verifying the identity guard for this fix: the complement copies an + # unconsumed property's *value* wholesale, and a real, documented + # ThoughtSpot shape (geo_config naming a custom map) carries a GUID + # nested inside that value -- not as a top-level payload key, which + # is all stash.write_stash's own guard checks. Dropped, not stashed, + # with an issue -- silently widening what "column_properties" leaks + # would be worse than the original gap. + result = self._convert_one({ + "column_type": "ATTRIBUTE", + "index_type": "DONT_INDEX", + "geo_config": {"custom_file_guid": "map-guid-123", "geometryType": "polygon"}, + }) + field = result.model["semantic_model"][0]["datasets"][0]["fields"][0] + assert _own_stash(field)[FIELD_STASH_COLUMN_PROPERTIES] == {"index_type": "DONT_INDEX"} + serialised = json.dumps(result.model) + assert "guid" not in serialised + assert any(i["code"] == "TS-PROPERTY-IDENTITY-DROPPED" for i in result.issues.as_dicts()) + + +class TestUnknownRelationshipTarget: + def test_a_relationship_targeting_an_unknown_dataset_is_dropped_and_logged(self): + # Critical: only the FROM side of a join used to be checked against + # the datasets this model actually built. CUSTOMERS is referenced by + # the join but has no Table document, so its dataset never builds -- + # emitting a relationship pointing at it would produce a document + # upstream's own validator rejects outright. + orders = _table("ORDERS", columns=[_column("Customer Id", "CUSTOMER_ID", "INT64")]) + model = _model( + model_tables=[{"name": "ORDERS", "joins": [{ + "with": "CUSTOMERS", + "on": "[ORDERS::Customer Id] = [CUSTOMERS::Id]", + "cardinality": "MANY_TO_ONE", + }]}], + columns=[_attribute("Customer Id", "ORDERS::Customer Id")], + ) + + result = convert(_document_set(model, orders)) + semantic_model = result.model["semantic_model"][0] + + assert "relationships" not in semantic_model + # The rest of the model -- the one dataset that DID build -- is + # still useful rather than being discarded along with the bad join. + assert semantic_model["datasets"][0]["fields"][0]["name"] == "customer_id" + assert any( + i["code"] == "TS-JOIN-UNKNOWN-TARGET" and "CUSTOMERS" in i["message"] + for i in result.issues.as_dicts() + ) + + +class TestUnsurfacedColumns: + def test_a_physical_column_the_model_does_not_surface_is_stashed_verbatim(self): + orders = _table("ORDERS", columns=[ + _column("Amount", "AMOUNT", "DOUBLE"), + _column("Internal Flag", "INTERNAL_FLAG", "BOOLEAN"), + ]) + model = _model( + model_tables=[{"name": "ORDERS"}], + columns=[_attribute("Amount", "ORDERS::Amount")], + ) + + result = convert(_document_set(model, orders)) + dataset = result.model["semantic_model"][0]["datasets"][0] + + unsurfaced = _own_stash(dataset)[DATASET_STASH_UNSURFACED_COLUMNS] + assert len(unsurfaced) == 1 + assert unsurfaced[0]["name"] == "Internal Flag" + assert unsurfaced[0]["db_column_name"] == "INTERNAL_FLAG" + + def test_a_column_surfaced_only_as_a_measure_is_stashed_too(self): + # column_aggregation-shape metrics surface their physical column via + # column_id, but a Metric has no column_id field on the Ossie side + # at all -- it carries only the composed THOUGHTSPOT-dialect + # expression, bracket reference and all. An earlier revision treated + # this column as "surfaced enough" to skip unsurfaced_columns, on + # the reasoning that it is still part of the semantic model. True, + # but nothing else preserves its definition, so the reverse + # direction regenerated a Table missing it while the metric's own + # formula still referenced it -- a dangling column reference, + # caught only by round-tripping a real document through both public + # entry points. This column is now stashed exactly like one + # referenced by nothing at all, redundant with the metric's own + # expression but making the Table document regenerable on its own. + orders = _table("ORDERS", columns=[_column("Amount", "AMOUNT", "DOUBLE")]) + model = _model( + model_tables=[{"name": "ORDERS"}], + columns=[{"name": "Total", "column_id": "ORDERS::Amount", + "properties": {"column_type": "MEASURE", "aggregation": "SUM"}}], + ) + + result = convert(_document_set(model, orders)) + dataset = result.model["semantic_model"][0]["datasets"][0] + stashed = _own_stash(dataset) or {} + unsurfaced = stashed.get(DATASET_STASH_UNSURFACED_COLUMNS) or [] + assert [c["name"] for c in unsurfaced] == ["Amount"] + + def test_a_dataset_with_no_unsurfaced_columns_gets_no_such_key(self): + orders = _table("ORDERS", columns=[_column("Amount", "AMOUNT", "DOUBLE")]) + model = _model( + model_tables=[{"name": "ORDERS"}], + columns=[_attribute("Amount", "ORDERS::Amount")], + ) + result = convert(_document_set(model, orders)) + dataset = result.model["semantic_model"][0]["datasets"][0] + stashed = _own_stash(dataset) or {} + assert DATASET_STASH_UNSURFACED_COLUMNS not in stashed + + def test_unsurfaced_columns_populates_the_dataset_stash_on_its_own(self): + # A dataset's stash always carries at least tml_object, so the + # empty-payload guarantee is exercised at the model scope + # (test_an_empty_payload_writes_no_stash_entry), not here -- this + # confirms unsurfaced_columns itself lands correctly when nothing + # else about the column is surfaced at all. + orders = _table("ORDERS", columns=[_column("Amount", "AMOUNT", "DOUBLE")]) + model = _model(model_tables=[{"name": "ORDERS"}], columns=[]) + result = convert(_document_set(model, orders)) + dataset = result.model["semantic_model"][0]["datasets"][0] + stashed = _own_stash(dataset) + assert stashed is not None + assert stashed[DATASET_STASH_UNSURFACED_COLUMNS][0]["name"] == "Amount" + + +class TestModelScopeIdentityIsCaughtNotFatal: + """The boundary guard (stash.write_stash) still raises -- that is what + makes it impossible to bypass -- but the assembly catches it, drops the + one contaminated stash field, logs why, and keeps converting everything + else. A single stray identity value in an otherwise-fine model must not + turn the whole conversion into a traceback.""" + + def _model_with(self, **model_scope_fields): + orders = _table("ORDERS", columns=[_column("Amount", "AMOUNT", "DOUBLE")]) + model = _model( + model_tables=[{"name": "ORDERS"}], + columns=[_attribute("Amount", "ORDERS::Amount")], + **model_scope_fields, + ) + return orders, model + + def test_a_guid_nested_in_parameters_is_dropped_not_fatal(self): + orders, model = self._model_with( + parameters=[{"name": "P", "default_value": {"obj_id": "p-1"}}] + ) + result = convert(_document_set(model, orders)) + semantic_model = result.model["semantic_model"][0] + assert semantic_model["datasets"][0]["fields"][0]["name"] == "amount" + stashed = _own_stash(semantic_model) or {} + assert MODEL_STASH_PARAMETERS not in stashed + assert "obj_id" not in json.dumps(result.model) + assert any(i["code"] == "TS-STASH-IDENTITY-DROPPED" for i in result.issues.as_dicts()) + + def test_a_guid_nested_in_filters_is_dropped_not_fatal(self): + orders, model = self._model_with( + filters=[{"column": "Region", "values": [{"nested": {"fqn": "f-1"}}]}] + ) + result = convert(_document_set(model, orders)) + stashed = _own_stash(result.model["semantic_model"][0]) or {} + assert MODEL_STASH_FILTERS not in stashed + assert "fqn" not in json.dumps(result.model) + + def test_a_guid_nested_in_column_groups_is_dropped_not_fatal(self): + orders, model = self._model_with( + column_groups=[{"name": "Sales", "meta": {"guid": "g-1"}}] + ) + result = convert(_document_set(model, orders)) + stashed = _own_stash(result.model["semantic_model"][0]) or {} + assert MODEL_STASH_COLUMN_GROUPS not in stashed + assert "guid" not in json.dumps(result.model) + + def test_a_guid_nested_in_lesson_plans_is_dropped_not_fatal(self): + orders, model = self._model_with( + lesson_plans=[{"lesson_id": 0, "extra": {"obj_id": "l-1"}}] + ) + result = convert(_document_set(model, orders)) + stashed = _own_stash(result.model["semantic_model"][0]) or {} + assert MODEL_STASH_LESSON_PLANS not in stashed + assert "obj_id" not in json.dumps(result.model) + + def test_a_guid_nested_in_action_object_associations_is_dropped_not_fatal(self): + orders, model = self._model_with( + action_object_associations=[{"action_name": "A", "context": {"fqn": "a-1"}}] + ) + result = convert(_document_set(model, orders)) + stashed = _own_stash(result.model["semantic_model"][0]) or {} + assert MODEL_STASH_ACTION_OBJECT_ASSOCIATIONS not in stashed + assert "fqn" not in json.dumps(result.model) + + def test_a_guid_nested_in_constraints_is_dropped_not_fatal(self): + orders, model = self._model_with(constraints={"rolling": {"window": {"guid": "c-1"}}}) + result = convert(_document_set(model, orders)) + stashed = _own_stash(result.model["semantic_model"][0]) or {} + assert MODEL_STASH_CONSTRAINTS not in stashed + assert "guid" not in json.dumps(result.model) + + def test_a_guid_nested_in_model_joins_with_is_dropped_not_fatal(self): + orders, model = self._model_with( + joins_with=[{"name": "j", "destination": {"fqn": "j-1"}}] + ) + result = convert(_document_set(model, orders)) + stashed = _own_stash(result.model["semantic_model"][0]) or {} + assert MODEL_STASH_MODEL_JOINS_WITH not in stashed + assert "fqn" not in json.dumps(result.model) + + def test_other_model_scope_fields_survive_when_only_one_is_contaminated(self): + # Dropping the one bad key must not take the rest of the model + # stash down with it. + orders, model = self._model_with( + parameters=[{"name": "P", "default_value": {"obj_id": "p-1"}}], + filters=[{"column": "Region", "values": ["US"]}], + ) + result = convert(_document_set(model, orders)) + stashed = _own_stash(result.model["semantic_model"][0]) or {} + assert MODEL_STASH_PARAMETERS not in stashed + assert stashed[MODEL_STASH_FILTERS] == [{"column": "Region", "values": ["US"]}] + + +class TestUnnormalisableNamesAreCaughtNotFatal: + """`identifiers.normalise` raises when a display name has no ASCII + alphanumerics for it to fold onto (a CJK-only name, a punctuation-only + one). Three scopes can hit it: the model's own top-level name, a field, + and a metric. All three must degrade -- fall back to a usable + identifier, report it, and continue -- rather than take the whole + conversion down, or the column, over one unfoldable name. (Before + `_field_or_metric_identifier` existed, a field or metric hitting this + was dropped entirely and misreported as a malformed column *reference* + -- see `test_a_column_ref_and_an_unnormalisable_name_report_different_codes` + for the two now being told apart.) + """ + + def test_a_model_name_with_no_ascii_alphanumerics_falls_back_and_is_reported(self): + orders = _table("ORDERS", columns=[_column("Amount", "AMOUNT", "DOUBLE")]) + model = _model( + name="北京市", # CJK-only; NFKD folds none of it to ASCII + model_tables=[{"name": "ORDERS"}], + columns=[_attribute("Amount", "ORDERS::Amount")], + ) + result = convert(_document_set(model, orders)) + semantic_model = result.model["semantic_model"][0] + assert semantic_model["name"] == "model" + assert any(i["code"] == "TS-MODEL-NAME-UNNORMALISABLE" for i in result.issues.as_dicts()) + # The rest of the model still converts -- one unfoldable name does + # not take the whole document down. + assert semantic_model["datasets"][0]["fields"][0]["name"] == "amount" + + def test_an_attribute_columns_unnormalisable_name_falls_back_to_the_warehouse_column_name(self): + # Also exercises `_index_attribute_columns` (Phase 2, which runs + # over every ATTRIBUTE column before Phase 3 starts): before this + # fix, Phase 2 silently excluded a column like this one from the + # cross-reference gate on the theory that convert_field would drop + # the field too -- true then, a correctness bug once convert_field + # grew this fallback. There is no cross-reference in this fixture, + # but the field surviving Phase 3 at all already proves Phase 2 did + # not exclude it. + orders = _table("ORDERS", columns=[_column("Amount", "AMOUNT", "DOUBLE")]) + model = _model( + model_tables=[{"name": "ORDERS"}], + columns=[_attribute("!!!", "ORDERS::Amount")], + ) + result = convert(_document_set(model, orders)) + fields = result.model["semantic_model"][0]["datasets"][0]["fields"] + assert len(fields) == 1 + field = fields[0] + # The physical column's own warehouse name is the fallback basis -- + # not a bare placeholder -- see _field_or_metric_identifier. + assert field["name"] == "amount" + assert field["label"] == "!!!" # the exact display name, recoverable via `label` alone + assert any(i["code"] == "TS-FIELD-NAME-UNNORMALISABLE" for i in result.issues.as_dicts()) + assert not any(i["code"] == "TS-COLUMN-REF-MALFORMED" for i in result.issues.as_dicts()) + + def test_two_unnormalisable_fields_with_no_physical_hint_get_distinct_fallback_names(self): + # Neither formula-backed field has a column_id, so neither has a + # warehouse column name to fall back on -- both reach the + # allocator-suffixed placeholder, and must not collide into the same + # "field" identifier. The physical "Amount" field is only here so + # `resolve()` has something to attribute the two formulas' shared + # `[ORDERS::Amount]` reference to. + orders = _table("ORDERS", columns=[_column("Amount", "AMOUNT", "DOUBLE")]) + model = _model( + model_tables=[{"name": "ORDERS"}], + columns=[ + _attribute("Amount", "ORDERS::Amount"), + {"name": "顧客", "formula_id": "f1", "properties": {"column_type": "ATTRIBUTE"}}, + {"name": "名前", "formula_id": "f2", "properties": {"column_type": "ATTRIBUTE"}}, + ], + formulas=[ + {"id": "f1", "name": "顧客", "expr": "[ORDERS::Amount]"}, + {"id": "f2", "name": "名前", "expr": "[ORDERS::Amount]"}, + ], + ) + result = convert(_document_set(model, orders)) + fields = result.model["semantic_model"][0]["datasets"][0]["fields"] + assert len(fields) == 3 # "amount" plus the two non-Latin formula fields + fallback_names = {f["name"] for f in fields if f["label"] in ("顧客", "名前")} + assert len(fallback_names) == 2 # distinct, not collapsed into one "field" + assert fallback_names == {"field", "field_2"} + + def test_a_column_ref_and_an_unnormalisable_name_report_different_codes(self): + # Fix 2: split_column_ref('[顧客::名前]') parses fine -- the reference + # itself is not malformed -- so a field with a genuinely ambiguous + # column_id must still report TS-COLUMN-REF-MALFORMED, distinct from + # TS-FIELD-NAME-UNNORMALISABLE (a valid reference, unfoldable name). + # Before this fix both funnelled through one shared except-ValueError + # handler in convert() and were indistinguishable. + orders = _table("ORDERS", columns=[_column("Amount", "AMOUNT", "DOUBLE")]) + model = _model( + model_tables=[{"name": "ORDERS"}], + columns=[ + _attribute("名前", "ORDERS::Amount"), # unfoldable name, valid reference + # A column_id containing a run of "::" is ambiguous -- + # identifiers.split_column_ref refuses to guess. + _attribute("Ambiguous", "ORDERS:::Nested"), + ], + ) + result = convert(_document_set(model, orders)) + codes = {i["code"] for i in result.issues.as_dicts()} + assert "TS-FIELD-NAME-UNNORMALISABLE" in codes + assert "TS-COLUMN-REF-MALFORMED" in codes + fields = result.model["semantic_model"][0]["datasets"][0]["fields"] + assert len(fields) == 1 # the unfoldable-but-valid field survives + assert fields[0]["label"] == "名前" + + +class TestKeyDerivationEdgeCasesCommitted: + """Edge cases attacked and confirmed by hand during development, now + committed so the check runs on every future change instead of living + only in a one-off transcript.""" + + def test_a_mixed_equality_and_residual_join_emits_a_weaker_relationship_and_no_key(self): + customers = _table("CUSTOMERS", columns=[_column("Id", "ID", "INT64"), _column("Effective Date", "EFFECTIVE_DATE", "DATE")]) + orders = _table("ORDERS", columns=[_column("Customer Id", "CUSTOMER_ID", "INT64"), _column("Order Date", "ORDER_DATE", "DATE")]) + on_expr = ( + "[ORDERS::Customer Id] = [CUSTOMERS::Id] and " + "[ORDERS::Order Date] >= [CUSTOMERS::Effective Date]" + ) + model = _model( + model_tables=[ + {"name": "ORDERS", "joins": [{"with": "CUSTOMERS", "on": on_expr, + "type": "INNER", "cardinality": "MANY_TO_ONE"}]}, + {"name": "CUSTOMERS"}, + ], + ) + result = convert(_document_set(model, orders, customers)) + semantic_model = result.model["semantic_model"][0] + customers_ds = next(d for d in semantic_model["datasets"] if d["name"] == "CUSTOMERS") + + assert "primary_key" not in customers_ds + assert "unique_keys" not in customers_ds + + rel = semantic_model["relationships"][0] + assert rel["from_columns"] == ["Customer Id"] + assert rel["to_columns"] == ["Id"] + rel_stash = _own_stash(rel) + assert rel_stash[RELATIONSHIP_STASH_ON_EXPRESSION] == on_expr + assert any(i["code"] == "TS-JOIN-RESIDUAL-PREDICATES" for i in result.issues.as_dicts()) + assert any(i["code"] == "TS_KEY_COVERAGE" for i in result.issues.as_dicts()) + + def test_a_self_join_gives_each_alias_its_own_dataset_and_key(self): + employees = _table("EMPLOYEES", columns=[ + _column("Id", "ID", "INT64"), _column("Manager Id", "MANAGER_ID", "INT64"), + _column("Name", "NAME", "VARCHAR"), + ]) + model = _model( + name="OrgChart", + model_tables=[ + {"name": "EMPLOYEES", "alias": "Emp", "joins": [{ + "with": "Mgr", "on": "[Emp::Manager Id] = [Mgr::Id]", + "type": "INNER", "cardinality": "MANY_TO_ONE", + }]}, + {"name": "EMPLOYEES", "alias": "Mgr"}, + ], + columns=[ + _attribute("Emp Name", "Emp::Name"), + _attribute("Mgr Name", "Mgr::Name"), + ], + ) + result = convert(_document_set(model, employees)) + semantic_model = result.model["semantic_model"][0] + datasets = {d["name"]: d for d in semantic_model["datasets"]} + + assert set(datasets) == {"Emp", "Mgr"} + assert datasets["Emp"]["source"] == datasets["Mgr"]["source"] == "SALES.PUBLIC.EMPLOYEES" + assert datasets["Mgr"]["primary_key"] == ["Id"] + rel = semantic_model["relationships"][0] + assert rel["from"] == "Emp" + assert rel["to"] == "Mgr" + assert result.issues.as_dicts() == [] + + def test_a_top_level_or_in_a_join_condition_is_not_fabricated_into_an_equality_pair(self): + customers = _table("CUSTOMERS", columns=[_column("Id", "ID", "INT64"), _column("Legacy Id", "LEGACY_ID", "INT64")]) + orders = _table("ORDERS", columns=[_column("Customer Id", "CUSTOMER_ID", "INT64")]) + on_expr = "[ORDERS::Customer Id] = [CUSTOMERS::Id] or [ORDERS::Customer Id] = [CUSTOMERS::Legacy Id]" + model = _model( + model_tables=[ + {"name": "ORDERS", "joins": [{"with": "CUSTOMERS", "on": on_expr, "cardinality": "MANY_TO_ONE"}]}, + {"name": "CUSTOMERS"}, + ], + ) + result = convert(_document_set(model, orders, customers)) + semantic_model = result.model["semantic_model"][0] + + # No equality pair could be safely attributed -- the whole "or" + # expression is one residual, never split into a fabricated pair. + assert "relationships" not in semantic_model + model_stash = _own_stash(semantic_model) + assert model_stash[MODEL_STASH_UNREPRESENTABLE_JOINS][0][RELATIONSHIP_STASH_ON_EXPRESSION] == on_expr + + def test_many_to_many_is_not_key_evidence_through_the_full_pipeline(self): + customers = _table("CUSTOMERS", columns=[_column("Id", "ID", "INT64")]) + products = _table("PRODUCTS", columns=[_column("Customer Id", "CUSTOMER_ID", "INT64")]) + model = _model( + model_tables=[ + {"name": "PRODUCTS", "joins": [{ + "with": "CUSTOMERS", "on": "[PRODUCTS::Customer Id] = [CUSTOMERS::Id]", + "type": "INNER", "cardinality": "MANY_TO_MANY", + }]}, + {"name": "CUSTOMERS"}, + ], + ) + result = convert(_document_set(model, products, customers)) + semantic_model = result.model["semantic_model"][0] + customers_ds = next(d for d in semantic_model["datasets"] if d["name"] == "CUSTOMERS") + + assert "primary_key" not in customers_ds + assert "unique_keys" not in customers_ds + rel = semantic_model["relationships"][0] + assert _own_stash(rel)[RELATIONSHIP_STASH_CARDINALITY] == "MANY_TO_MANY" + + +class TestSqlViewColumns: + """A SQL View document's columns live under sql_view_columns[], not + columns[] -- a different key entirely, not a differently-shaped entry + under the same one. Reading the wrong key silently finds nothing for + every column of every SQL View: no datatype resolves, and every column + reads as unsurfaced regardless of whether the Model actually surfaces + it.""" + + def test_a_surfaced_sql_view_column_resolves_its_datatype(self): + vw = _sql_view("VW", columns=[ + _sql_view_column("CID", "c_id", "INT64"), + ]) + model = _model( + model_tables=[{"name": "VW"}], + columns=[_attribute("Cid", "VW::CID")], + ) + result = convert(_document_set(model, vw)) + field = result.model["semantic_model"][0]["datasets"][0]["fields"][0] + assert field["datatype"] == "Integer" + assert result.issues.as_dicts() == [] + + def test_an_unsurfaced_sql_view_column_is_stashed_verbatim(self): + vw = _sql_view("VW", columns=[ + _sql_view_column("CID", "c_id", "INT64"), + _sql_view_column("Never Surfaced", "x", "VARCHAR"), + ]) + model = _model( + model_tables=[{"name": "VW"}], + columns=[_attribute("Cid", "VW::CID")], + ) + result = convert(_document_set(model, vw)) + dataset = result.model["semantic_model"][0]["datasets"][0] + unsurfaced = _own_stash(dataset)[DATASET_STASH_UNSURFACED_COLUMNS] + assert len(unsurfaced) == 1 + assert unsurfaced[0]["name"] == "Never Surfaced" + # Verbatim -- the SQL View's own key name, not the datatype-lookup + # translation this converter builds internally for its own use. + assert unsurfaced[0]["sql_output_column"] == "x" + assert "db_column_name" not in unsurfaced[0] + + def test_sql_output_column_differing_from_name_is_stashed(self): + vw = _sql_view("VW", columns=[ + _sql_view_column("Customer Id", sql_output_column="cust_id_out", data_type="INT64"), + ]) + model = _model( + model_tables=[{"name": "VW"}], + columns=[_attribute("Customer Id", "VW::Customer Id")], + ) + result = convert(_document_set(model, vw)) + dataset = result.model["semantic_model"][0]["datasets"][0] + field_name = dataset["fields"][0]["name"] + assert field_name == "customer_id" + stashed = _own_stash(dataset) + assert stashed[DATASET_STASH_SQL_OUTPUT_COLUMNS] == {"customer_id": "cust_id_out"} + + def test_a_mixed_document_set_with_a_table_and_a_sql_view_both_convert(self): + orders = _table("ORDERS", columns=[_column("Amount", "AMOUNT", "DOUBLE")]) + vw = _sql_view("VW", columns=[_sql_view_column("CID", "c_id", "INT64")]) + model = _model( + model_tables=[{"name": "ORDERS"}, {"name": "VW"}], + columns=[ + _attribute("Amount", "ORDERS::Amount"), + _attribute("Cid", "VW::CID"), + ], + ) + result = convert(_document_set(model, orders, vw)) + datasets = {d["name"]: d for d in result.model["semantic_model"][0]["datasets"]} + + assert datasets["ORDERS"]["source"] == "SALES.PUBLIC.ORDERS" + assert datasets["ORDERS"]["fields"][0]["datatype"] == "Decimal" + + assert datasets["VW"]["source"] == "SELECT 1" + assert datasets["VW"]["fields"][0]["datatype"] == "Integer" + assert _own_stash(datasets["VW"])[DATASET_STASH_TML_OBJECT] == "sql_view" + assert result.issues.as_dicts() == [] + + +class TestPhysicalColumnReferences: + """The mapping document is explicit for a bare-identifier field: "the + identifier is the *physical* column; the display name comes from + label/name." resolve() used to build the ANSI_SQL sibling from the + Ossie field's own display-derived identifier instead -- confident, + well-formed SQL that names a column the warehouse does not have, + wrong in every model where a display name differs from its physical + column, which is the normal case in any curated model.""" + + def test_a_differing_db_column_name_is_used_in_the_portable_expression(self): + orders = _table("ORDERS", columns=[_column("Amount", "AMT_RAW", "DOUBLE")]) + model = _model(model_tables=[{"name": "ORDERS"}], columns=[_attribute("Amount", "ORDERS::Amount")]) + result = convert(_document_set(model, orders)) + field = result.model["semantic_model"][0]["datasets"][0]["fields"][0] + dialects = {d["dialect"]: d["expression"] for d in field["expression"]["dialects"]} + assert dialects["ANSI_SQL"] == "ORDERS.AMT_RAW" + + def test_an_equal_db_column_name_still_resolves(self): + orders = _table("ORDERS", columns=[_column("Amount", "Amount", "DOUBLE")]) + model = _model(model_tables=[{"name": "ORDERS"}], columns=[_attribute("Amount", "ORDERS::Amount")]) + result = convert(_document_set(model, orders)) + field = result.model["semantic_model"][0]["datasets"][0]["fields"][0] + dialects = {d["dialect"]: d["expression"] for d in field["expression"]["dialects"]} + assert dialects["ANSI_SQL"] == "ORDERS.Amount" + + def test_a_sql_view_reference_uses_sql_output_column_not_db_column_name(self): + vw = _sql_view("VW", columns=[_sql_view_column("CID", "c_id", "INT64")]) + model = _model(model_tables=[{"name": "VW"}], columns=[_attribute("Cid", "VW::CID")]) + result = convert(_document_set(model, vw)) + field = result.model["semantic_model"][0]["datasets"][0]["fields"][0] + dialects = {d["dialect"]: d["expression"] for d in field["expression"]["dialects"]} + assert dialects["ANSI_SQL"] == "VW.c_id" + + def test_a_computed_field_referencing_a_renamed_column_still_resolves(self): + orders = _table("ORDERS", columns=[_column("Amount", "AMT_RAW", "DOUBLE")]) + model = _model( + model_tables=[{"name": "ORDERS"}], + columns=[ + _attribute("Amount", "ORDERS::Amount"), + {"name": "Doubled", "formula_id": "formula_doubled", + "properties": {"column_type": "ATTRIBUTE"}}, + ], + formulas=[{"id": "formula_doubled", "name": "Doubled", "expr": "[ORDERS::Amount]"}], + ) + result = convert(_document_set(model, orders)) + fields = {f["name"]: f for f in result.model["semantic_model"][0]["datasets"][0]["fields"]} + doubled_dialects = {d["dialect"]: d["expression"] for d in fields["doubled"]["expression"]["dialects"]} + assert doubled_dialects["ANSI_SQL"] == "ORDERS.AMT_RAW" + assert result.issues.as_dicts() == [] + + +class TestPhysicalColumnStash: + """`data_type`'s connection-dependent spelling (BOOL/BOOLEAN, + DOUBLE/FLOAT) and a Table column's `db_column_name` are both + unrecoverable by the reverse direction unless the forward direction + records them -- the datatype map says so explicitly for the former; the + latter has no documented stash slot at all yet but is just as lost + without one, since the display name is all a round-tripped bracket + reference carries.""" + + def test_a_non_canonical_boolean_spelling_is_stashed(self): + orders = _table("ORDERS", columns=[_column("Is Active", "Is Active", "BOOL")]) + model = _model(model_tables=[{"name": "ORDERS"}], columns=[_attribute("Is Active", "ORDERS::Is Active")]) + result = convert(_document_set(model, orders)) + field = result.model["semantic_model"][0]["datasets"][0]["fields"][0] + assert field["datatype"] == "Boolean" + assert _own_stash(field)[FIELD_STASH_DATA_TYPE] == "BOOL" + + def test_the_canonical_boolean_spelling_is_not_stashed(self): + orders = _table("ORDERS", columns=[_column("Is Active", "Is Active", "BOOLEAN")]) + model = _model(model_tables=[{"name": "ORDERS"}], columns=[_attribute("Is Active", "ORDERS::Is Active")]) + result = convert(_document_set(model, orders)) + field = result.model["semantic_model"][0]["datasets"][0]["fields"][0] + assert field["datatype"] == "Boolean" + assert "custom_extensions" not in field + + def test_a_float_column_stashes_its_float_spelling(self): + orders = _table("ORDERS", columns=[_column("Rate", "Rate", "FLOAT")]) + model = _model(model_tables=[{"name": "ORDERS"}], columns=[_attribute("Rate", "ORDERS::Rate")]) + result = convert(_document_set(model, orders)) + field = result.model["semantic_model"][0]["datasets"][0]["fields"][0] + assert field["datatype"] == "Float" + assert _own_stash(field)[FIELD_STASH_DATA_TYPE] == "FLOAT" + + def test_a_differing_db_column_name_is_stashed_on_a_table_column(self): + orders = _table("ORDERS", columns=[_column("Amount", "AMT_RAW", "DOUBLE")]) + model = _model(model_tables=[{"name": "ORDERS"}], columns=[_attribute("Amount", "ORDERS::Amount")]) + result = convert(_document_set(model, orders)) + field = result.model["semantic_model"][0]["datasets"][0]["fields"][0] + assert _own_stash(field)[FIELD_STASH_DB_COLUMN_NAME] == "AMT_RAW" + + def test_an_equal_db_column_name_is_not_stashed(self): + orders = _table("ORDERS", columns=[_column("Amount", "Amount", "DOUBLE")]) + model = _model(model_tables=[{"name": "ORDERS"}], columns=[_attribute("Amount", "ORDERS::Amount")]) + result = convert(_document_set(model, orders)) + field = result.model["semantic_model"][0]["datasets"][0]["fields"][0] + assert "custom_extensions" not in field + + def test_a_sql_view_column_never_gets_a_db_column_name_stash(self): + # sql_output_columns (dataset-level) already carries this fact for + # a SQL View -- a field-level db_column_name would be a redundant + # second copy of the same information under a different name. + vw = _sql_view("VW", columns=[_sql_view_column("CID", "c_id", "INT64")]) + model = _model(model_tables=[{"name": "VW"}], columns=[_attribute("Cid", "VW::CID")]) + result = convert(_document_set(model, vw)) + field = result.model["semantic_model"][0]["datasets"][0]["fields"][0] + assert "custom_extensions" not in field + dataset_stash = _own_stash(result.model["semantic_model"][0]["datasets"][0]) + assert dataset_stash[DATASET_STASH_SQL_OUTPUT_COLUMNS] == {"cid": "c_id"} + + def test_a_metric_bound_to_a_physical_column_gets_the_same_stash(self): + orders = _table("ORDERS", columns=[_column("Amount", "AMT_RAW", "BOOL")]) + model = _model( + model_tables=[{"name": "ORDERS"}], + columns=[{"name": "Amount", "column_id": "ORDERS::Amount", + "properties": {"column_type": "MEASURE", "aggregation": "COUNT"}}], + ) + result = convert(_document_set(model, orders)) + metric = result.model["semantic_model"][0]["metrics"][0] + stashed = _own_stash(metric) + assert stashed[FIELD_STASH_DB_COLUMN_NAME] == "AMT_RAW" diff --git a/converters/thoughtspot/tests/test_tml_to_ossie_fields.py b/converters/thoughtspot/tests/test_tml_to_ossie_fields.py new file mode 100644 index 00000000..d2f017bd --- /dev/null +++ b/converters/thoughtspot/tests/test_tml_to_ossie_fields.py @@ -0,0 +1,594 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest +from ossie_thoughtspot.issues import IssueLog +from ossie_thoughtspot.tml_to_ossie import attribute_dataset, convert_field, expression_entries + + +def _resolve(table, column): + """Every reference lands in the dataset named after its table, lower-cased, + unless the table is named "MISSING" — then it resolves to nothing at all.""" + return None if table == "MISSING" else f"{table.lower()}.{column.lower()}" + + +class TestExpressionEntries: + def test_a_bare_reference_gets_both_dialects(self): + log = IssueLog() + out = expression_entries("[ORDERS::Amount]", _resolve, log, object_ref="f") + assert out == [ + {"dialect": "THOUGHTSPOT", "expression": "[ORDERS::Amount]"}, + {"dialect": "ANSI_SQL", "expression": "orders.amount"}, + ] + assert log.as_dicts() == [] + + def test_the_thoughtspot_entry_is_byte_for_byte_the_input(self): + # A reconstruction from a parsed (name, args) shape would normalise this to + # `sum ( [ORDERS::Amount] )` and break exact-string equality on the way back. + log = IssueLog() + out = expression_entries("sum([ORDERS::Amount])", _resolve, log, object_ref="f") + assert out[0] == {"dialect": "THOUGHTSPOT", "expression": "sum([ORDERS::Amount])"} + assert [e["dialect"] for e in out] == ["THOUGHTSPOT"] + + def test_a_computed_expression_gets_a_thoughtspot_entry_and_an_issue(self): + log = IssueLog() + out = expression_entries( + "sum ( [ORDERS::Amount] ) / count ( [ORDERS::Id] )", _resolve, log, object_ref="f" + ) + assert [e["dialect"] for e in out] == ["THOUGHTSPOT"] + assert len(log.as_dicts()) == 1 + + def test_a_parameter_reference_blocks_the_portable_sibling(self): + # A runtime parameter has no Ossie equivalent, so the expression is not + # portable however simple it looks. + log = IssueLog() + out = expression_entries("[ORDERS::Amount] * [Growth Rate]", _resolve, log, object_ref="f") + assert [e["dialect"] for e in out] == ["THOUGHTSPOT"] + assert any("parameter" in i["message"].lower() for i in log.as_dicts()) + assert not any(i["code"] == "TS-EXPR-FORMULA-REFERENCE" for i in log.as_dicts()) + + def test_a_formula_cross_reference_is_not_reported_as_a_parameter(self): + # `[formula_Margin]` has no `::`, the same bracketed shape a runtime + # parameter has -- but it is a formula composing another formula, a + # first-class ThoughtSpot construct, not something with no Ossie + # equivalent. The old bug: this fired TS-EXPR-PARAM and told a reader + # to go hunting for a parameter that does not exist. + log = IssueLog() + out = expression_entries("sum ( [formula_Margin] )", _resolve, log, object_ref="f") + assert [e["dialect"] for e in out] == ["THOUGHTSPOT"] + assert not any(i["code"] == "TS-EXPR-PARAM" for i in log.as_dicts()) + assert not any("parameter" in i["message"].lower() for i in log.as_dicts()) + + def test_the_cross_reference_issue_names_the_right_cause(self): + log = IssueLog() + expression_entries("sum ( [formula_Margin] )", _resolve, log, object_ref="f") + [issue] = [i for i in log.as_dicts() if i["code"] == "TS-EXPR-FORMULA-REFERENCE"] + assert "formula_Margin" in issue["message"] + assert "inlin" in issue["message"].lower() + + def test_an_expression_with_both_a_cross_reference_and_a_parameter_reports_both(self): + log = IssueLog() + out = expression_entries( + "[formula_Margin] * [Growth Rate]", _resolve, log, object_ref="f" + ) + assert [e["dialect"] for e in out] == ["THOUGHTSPOT"] + codes = {i["code"] for i in log.as_dicts()} + assert codes == {"TS-EXPR-FORMULA-REFERENCE", "TS-EXPR-PARAM"} + [param_issue] = [i for i in log.as_dicts() if i["code"] == "TS-EXPR-PARAM"] + assert "Growth Rate" in param_issue["message"] + assert "formula_Margin" not in param_issue["message"] + + def test_a_parameter_used_twice_is_named_once_not_twice(self): + # `[Growth Rate]` on both sides of the ratio is one fact worth + # reporting once -- listing it twice reads as two distinct + # unresolved parameters, not a single repeated reference. + log = IssueLog() + expression_entries( + "[Growth Rate] / [Growth Rate]", _resolve, log, object_ref="f" + ) + [issue] = [i for i in log.as_dicts() if i["code"] == "TS-EXPR-PARAM"] + assert issue["message"].count("Growth Rate") == 1 + + def test_a_repeated_cross_reference_is_also_named_once(self): + log = IssueLog() + expression_entries( + "[formula_Margin] + [formula_Margin]", _resolve, log, object_ref="f" + ) + [issue] = [i for i in log.as_dicts() if i["code"] == "TS-EXPR-FORMULA-REFERENCE"] + assert issue["message"].count("formula_Margin") == 1 + + def test_an_unresolvable_reference_blocks_the_portable_sibling(self): + log = IssueLog() + out = expression_entries("[MISSING::Col]", _resolve, log, object_ref="f") + assert [e["dialect"] for e in out] == ["THOUGHTSPOT"] + assert log.as_dicts() + + def test_the_thoughtspot_entry_is_always_present(self): + # The invariant the whole round trip rests on: whatever else happens, the + # original expression string survives, unmodified, as the first entry. + log = IssueLog() + for expr in ["[A::x]", "sum ( [A::x] )", "gibberish ( (", "[Param]"]: + out = expression_entries(expr, _resolve, log, object_ref="f") + assert out[0]["dialect"] == "THOUGHTSPOT" + assert out[0]["expression"] == expr + + +class TestExpressionEntriesVerbatimUnderAdversarialInput: + """The exact-string property is the one thing the whole round trip depends on. + Each case below distorts the input in one way real ThoughtSpot formulas can be + distorted — irregular internal spacing, edge whitespace, embedded structure, a + quoting convention — and checks the THOUGHTSPOT entry is untouched regardless.""" + + @pytest.mark.parametrize( + "expr", + [ + "sum( [ORDERS::Amount] ,2 )", # irregular internal whitespace + "[ORDERS::Amount] ", # trailing space + "[ORDERS::Amount]\t", # trailing tab + "sum(\n [ORDERS::Amount]\n)", # embedded newline + "concat ( [ORDERS::Name] , 'it''s a test' )", # doubled quote + "\t[ORDERS::Amount]\n", # leading tab, trailing newline + ], + ) + def test_verbatim_property_holds(self, expr): + log = IssueLog() + out = expression_entries(expr, _resolve, log, object_ref="f") + assert out[0] == {"dialect": "THOUGHTSPOT", "expression": expr} + # A dialect entry is a plain dict of Python strs; nothing en route (e.g. an + # implicit int/float coercion or a str subclass with different __eq__) could + # make the equality above pass while the object stored is not the original. + assert out[0]["expression"] is expr or out[0]["expression"] == expr + assert isinstance(out[0]["expression"], str) + + +class TestPortableSiblingTruthTable: + """When exactly does a portable ANSI_SQL sibling appear? One row per input shape, + plus a sweep confirming none of the non-portable shapes ever produces one anyway + (a wrong portable expression is worse than none).""" + + CASES = { + "bare reference": ("[ORDERS::Amount]", True), + "bare reference with surrounding whitespace": (" [ORDERS::Amount] ", True), + "reference the resolver cannot resolve": ("[MISSING::Col]", False), + "expression with a runtime parameter": ("[ORDERS::Amount] * [Growth Rate]", False), + "expression with a formula cross-reference": ("sum ( [formula_Margin] )", False), + "single function call": ("sum([ORDERS::Amount])", False), + "compound expression": ("[ORDERS::Amount] + [ORDERS::Tax]", False), + } + + @pytest.mark.parametrize("expr,expect_portable", CASES.values(), ids=CASES.keys()) + def test_portability_per_case(self, expr, expect_portable): + log = IssueLog() + out = expression_entries(expr, _resolve, log, object_ref="f") + dialects = [e["dialect"] for e in out] + if expect_portable: + assert dialects == ["THOUGHTSPOT", "ANSI_SQL"] + assert log.as_dicts() == [] + else: + assert dialects == ["THOUGHTSPOT"] + assert log.as_dicts() + + def test_no_non_portable_case_produces_a_portable_sibling(self): + # The inverse check: sweep every case this table declares non-portable and + # confirm none of them slipped an ANSI_SQL entry in anyway. + for expr, expect_portable in self.CASES.values(): + if expect_portable: + continue + log = IssueLog() + out = expression_entries(expr, _resolve, log, object_ref="f") + assert "ANSI_SQL" not in [e["dialect"] for e in out], expr + + +class TestAttributeDataset: + """Attacking the attribution rule: a computed field spanning two datasets must + not be attributed, and neither must the other shapes that offer no single, + confident answer.""" + + def test_references_spanning_two_datasets_are_not_attributed(self): + log = IssueLog() + + def resolve(table, column): + return f"other.{column.lower()}" if table == "OTHER" else f"orders.{column.lower()}" + + result = attribute_dataset( + "[ORDERS::Amount] + [OTHER::Fee]", resolve, log, object_ref="f" + ) + assert result is None + issues = log.as_dicts() + assert len(issues) == 1 + assert "different datasets" in issues[0]["message"] + + def test_no_references_at_all_is_not_attributed(self): + log = IssueLog() + result = attribute_dataset("1 + 1", _resolve, log, object_ref="f") + assert result is None + assert log.as_dicts() + assert "no column references" in log.as_dicts()[0]["message"] + + def test_references_to_the_same_dataset_via_different_tables_are_attributed(self): + # Two different ThoughtSpot tables can legitimately resolve into the *same* + # Ossie dataset (an alias, or two tables mapped onto one source) — that is + # real agreement, not a coincidence to be suspicious of. + log = IssueLog() + + def resolve(table, column): + return f"orders.{column.lower()}" # ORDERS and ORDERS_ALIAS both land here + + result = attribute_dataset( + "[ORDERS::Amount] + [ORDERS_ALIAS::Tax]", resolve, log, object_ref="f" + ) + assert result == "orders" + assert log.as_dicts() == [] + + def test_an_unresolvable_reference_is_not_attributed(self): + log = IssueLog() + result = attribute_dataset( + "[ORDERS::Amount] + [MISSING::Fee]", _resolve, log, object_ref="f" + ) + assert result is None + issues = log.as_dicts() + assert len(issues) == 1 + assert "[MISSING::Fee]" in issues[0]["message"] + + def test_a_single_resolvable_reference_is_attributed(self): + log = IssueLog() + result = attribute_dataset("[ORDERS::Amount]", _resolve, log, object_ref="f") + assert result == "orders" + assert log.as_dicts() == [] + + def test_parameters_take_no_part_in_attribution(self): + # A parameter reference carries no dataset. Attribution should succeed from + # the column references alone; portability is a separate question that + # expression_entries answers, not this function. + log = IssueLog() + result = attribute_dataset( + "[ORDERS::Amount] * [Growth Rate]", _resolve, log, object_ref="f" + ) + assert result == "orders" + assert log.as_dicts() == [] + + +class TestConvertField: + def _table(self, name): + return {"ORDERS": {"name": "ORDERS", "columns": [ + {"name": "AMOUNT", "db_column_name": "AMOUNT", + "db_column_properties": {"data_type": "DOUBLE"}}, + ]}}.get(name) + + def test_a_physical_column_becomes_a_field(self): + log = IssueLog() + field = convert_field( + {"name": "Order Amount", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "ATTRIBUTE"}}, + {}, self._table, _resolve, log, + ) + assert field["name"] == "order_amount" # the normalised identifier + assert field["label"] == "Order Amount" # the exact display name + assert field["datatype"] == "Decimal" # DOUBLE -> Decimal + + def test_description_round_trips_without_a_stash(self): + log = IssueLog() + field = convert_field( + {"name": "Amount", "column_id": "ORDERS::AMOUNT", "description": "How much", + "properties": {"column_type": "ATTRIBUTE"}}, + {}, self._table, _resolve, log, + ) + assert field["description"] == "How much" + + def test_an_absent_description_is_omitted_not_blank(self): + log = IssueLog() + field = convert_field( + {"name": "Amount", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "ATTRIBUTE"}}, + {}, self._table, _resolve, log, + ) + assert "description" not in field + + def test_synonyms_become_ai_context(self): + # The full shape, not just presence: a synonyms-only ai_context must be the + # bare {"synonyms": [...]} object, with no "instructions" key sitting empty + # beside it. + log = IssueLog() + field = convert_field( + {"name": "Amount", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "ATTRIBUTE", "synonyms": ["total", "value"], + "synonym_type": "USER_DEFINED"}}, + {}, self._table, _resolve, log, + ) + assert field["ai_context"] == {"synonyms": ["total", "value"]} + + def test_an_empty_synonyms_list_produces_no_ai_context(self): + log = IssueLog() + field = convert_field( + {"name": "Amount", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "ATTRIBUTE", "synonyms": []}}, + {}, self._table, _resolve, log, + ) + assert "ai_context" not in field + + def test_free_text_ai_context_alone_stays_a_bare_string(self): + # The other shape Ossie's ai_context oneOf accepts: free-text instructions + # with no synonyms must come through as the plain string itself, not + # wrapped in an object — a regression that wrapped it would still pass a + # presence-only check. + log = IssueLog() + field = convert_field( + {"name": "Amount", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "ATTRIBUTE", + "ai_context": "Prefer this over the raw column."}}, + {}, self._table, _resolve, log, + ) + assert field["ai_context"] == "Prefer this over the raw column." + assert isinstance(field["ai_context"], str) + + def test_synonyms_and_free_text_together_combine_into_one_object(self): + log = IssueLog() + field = convert_field( + {"name": "Amount", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "ATTRIBUTE", "synonyms": ["total"], + "ai_context": "Prefer this over the raw column."}}, + {}, self._table, _resolve, log, + ) + assert field["ai_context"] == { + "synonyms": ["total"], + "instructions": "Prefer this over the raw column.", + } + + def test_neither_synonyms_nor_free_text_omits_ai_context_entirely(self): + log = IssueLog() + field = convert_field( + {"name": "Amount", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "ATTRIBUTE"}}, + {}, self._table, _resolve, log, + ) + assert "ai_context" not in field + + def test_a_measure_column_is_not_a_field(self): + # MEASURE columns become metrics, handled elsewhere. + log = IssueLog() + assert convert_field( + {"name": "Amount", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "MEASURE"}}, + {}, self._table, _resolve, log, + ) is None + + def test_is_time_is_omitted_when_the_type_already_implies_it(self): + # ThoughtSpot has no temporal-role flag at all in this direction, so is_time + # is never written — writing it for a Date column would just be noise on + # top of the type-derived default. + log = IssueLog() + table = lambda n: {"name": "ORDERS", "columns": [ + {"name": "DT", "db_column_name": "DT", + "db_column_properties": {"data_type": "DATE"}}]} + field = convert_field( + {"name": "Order Date", "column_id": "ORDERS::DT", + "properties": {"column_type": "ATTRIBUTE"}}, + {}, table, _resolve, log, + ) + assert "dimension" not in field or "is_time" not in field.get("dimension", {}) + + def test_a_missing_physical_column_raises_an_issue_and_omits_the_datatype(self): + log = IssueLog() + field = convert_field( + {"name": "Ghost", "column_id": "ORDERS::NOPE", + "properties": {"column_type": "ATTRIBUTE"}}, + {}, self._table, _resolve, log, + ) + assert "datatype" not in field + assert log.as_dicts() + + def test_a_missing_table_also_omits_the_datatype_and_raises_an_issue(self): + # Distinct from the case above: here the whole table is absent, not just one + # column inside a table that was found. + log = IssueLog() + field = convert_field( + {"name": "Ghost", "column_id": "MISSING::Col", + "properties": {"column_type": "ATTRIBUTE"}}, + {}, self._table, _resolve, log, + ) + assert field is not None + assert "datatype" not in field + assert log.as_dicts() + + def test_the_identifier_and_the_label_are_never_swapped(self): + # A stronger anti-regression case than the basic one above: punctuation in + # the display name makes name/label divergence unmistakable if the two were + # ever accidentally swapped. + log = IssueLog() + field = convert_field( + {"name": "Gross Margin %!!", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "ATTRIBUTE"}}, + {}, self._table, _resolve, log, + ) + assert field["label"] == "Gross Margin %!!" + assert field["name"] != field["label"] + assert field["name"] == "gross_margin" + + def test_a_computed_field_spanning_two_datasets_is_not_built(self): + log = IssueLog() + + def resolve(table, column): + return f"other.{column.lower()}" if table == "OTHER" else f"orders.{column.lower()}" + + formulas = {"formula_Combined": {"id": "formula_Combined", + "expr": "[ORDERS::Amount] + [OTHER::Fee]"}} + field = convert_field( + {"name": "Combined", "formula_id": "formula_Combined", + "properties": {"column_type": "ATTRIBUTE"}}, + formulas, self._table, resolve, log, + ) + assert field is None + assert log.as_dicts() + + # -- Tests of my own, beyond everything specified above. -- + # + # 1. convert_field's handling of a *computed* (formula-backed) ATTRIBUTE column — + # attribution plus field construction end to end — has no coverage at all in + # the cases above; every one of them is a physical, column_id-backed field. + # That whole code path is new and untested, and is exactly the kind of thing + # that "reasoning about behaviour instead of running it" would get wrong. + def test_a_computed_field_with_references_in_one_dataset_is_built(self): + log = IssueLog() + formulas = {"formula_Net_Amount": {"id": "formula_Net_Amount", + "expr": "[ORDERS::Amount] - [ORDERS::Discount]"}} + field = convert_field( + {"name": "Net Amount", "formula_id": "formula_Net_Amount", + "properties": {"column_type": "ATTRIBUTE"}}, + formulas, self._table, _resolve, log, + ) + assert field is not None + assert field["name"] == "net_amount" + assert field["label"] == "Net Amount" + assert field["expression"]["dialects"] == [ + {"dialect": "THOUGHTSPOT", "expression": "[ORDERS::Amount] - [ORDERS::Discount]"}, + ] + # Not portable (a compound expression), but still attributed and built — + # attribution and portability are independent questions. + assert "datatype" not in field + assert log.as_dicts() # the non-portability issue from expression_entries + + # 2. A computed field that mixes an attributable column reference with a runtime + # parameter is the sharpest test of whether attribution and portability were + # kept genuinely independent, rather than one implementation accidentally + # leaning on the other (e.g. attribution silently failing because of the + # parameter, or the parameter warning silently being swallowed because + # attribution succeeded). + def test_a_computed_field_with_a_parameter_is_attributed_but_not_portable(self): + log = IssueLog() + formulas = {"formula_Grown_Amount": {"id": "formula_Grown_Amount", + "expr": "[ORDERS::Amount] * [Growth Rate]"}} + field = convert_field( + {"name": "Grown Amount", "formula_id": "formula_Grown_Amount", + "properties": {"column_type": "ATTRIBUTE"}}, + formulas, self._table, _resolve, log, + ) + assert field is not None # attribution succeeded from the one column reference + dialects = [e["dialect"] for e in field["expression"]["dialects"]] + assert dialects == ["THOUGHTSPOT"] # but it is not portable + assert any("parameter" in i["message"].lower() for i in log.as_dicts()) + + # 3. The `formulas` map lookup itself, per the task's three required cases. + # + # 3a. formula_id present in the map: the expr must survive verbatim, byte for + # byte, into the THOUGHTSPOT dialect entry — re-run through the new + # lookup-based path rather than assumed to still hold from the expr-stash + # tests above. + def test_a_formula_id_present_in_the_map_converts_with_the_verbatim_expr(self): + log = IssueLog() + weird_expr = "concat( [ORDERS::Amount] , 'it''s a test'\t)\n" + formulas = {"formula_Weird": {"id": "formula_Weird", "expr": weird_expr}} + field = convert_field( + {"name": "Weird", "formula_id": "formula_Weird", + "properties": {"column_type": "ATTRIBUTE"}}, + formulas, self._table, _resolve, log, + ) + assert field is not None + thoughtspot_entries = [ + e for e in field["expression"]["dialects"] if e["dialect"] == "THOUGHTSPOT" + ] + assert thoughtspot_entries == [{"dialect": "THOUGHTSPOT", "expression": weird_expr}] + + # 3b. formula_id with no matching entry in the map: must not raise (no + # KeyError), must log an issue naming the column and the missing id, and + # must return None rather than a field silently missing its expression. + def test_a_formula_id_missing_from_the_map_logs_and_returns_none(self): + log = IssueLog() + field = convert_field( + {"name": "Orphan", "formula_id": "formula_Nonexistent", + "properties": {"column_type": "ATTRIBUTE"}}, + {}, self._table, _resolve, log, + ) + assert field is None + issues = log.as_dicts() + assert len(issues) == 1 + assert "Orphan" in issues[0]["message"] + assert "formula_Nonexistent" in issues[0]["message"] + + # 3c. Neither column_id nor formula_id: decided to treat this the same as the + # pre-existing "no source" contract (a column with neither key was already + # handled before formula_id existed) — log an issue and return None, rather + # than inventing a new, silent no-op path for what is really the same + # "nothing to build this field from" situation. + def test_neither_column_id_nor_formula_id_logs_and_returns_none(self): + log = IssueLog() + field = convert_field( + {"name": "Nothing", "properties": {"column_type": "ATTRIBUTE"}}, + {}, self._table, _resolve, log, + ) + assert field is None + issues = log.as_dicts() + assert len(issues) == 1 + assert "column_id" in issues[0]["message"] + assert "formula_id" in issues[0]["message"] + + # 4. A formulas[] entry can be present under the id but still be malformed — + # missing its own `expr` key. Same class of problem as an absent id (there + # is still no expression text to read), so it gets the same treatment: an + # issue naming the column and the formula id, and None — not a KeyError. + def test_a_formula_entry_present_but_missing_expr_logs_and_returns_none(self): + log = IssueLog() + formulas = {"formula_Bad": {"id": "formula_Bad"}} # no "expr" key + field = convert_field( + {"name": "Malformed", "formula_id": "formula_Bad", + "properties": {"column_type": "ATTRIBUTE"}}, + formulas, self._table, _resolve, log, + ) + assert field is None + issues = log.as_dicts() + assert len(issues) == 1 + assert "Malformed" in issues[0]["message"] + assert "formula_Bad" in issues[0]["message"] + + +class TestPhysicalDatatypeLoss: + """`_physical_datatype`'s two `None` outcomes are not the same kind of + outcome, and only one of them is a loss worth logging — exercised through + `convert_field`'s column_id path, the only way this private helper runs.""" + + def test_an_absent_data_type_produces_no_issue(self): + # Nothing was ever declared, so there is nothing being dropped. datatype + # is optional in Ossie; silence here is the correct, unremarkable answer. + log = IssueLog() + table = lambda n: {"name": "ORDERS", "columns": [ + {"name": "NOTE", "db_column_name": "NOTE"}]} # no db_column_properties + field = convert_field( + {"name": "Note", "column_id": "ORDERS::NOTE", + "properties": {"column_type": "ATTRIBUTE"}}, + {}, table, _resolve, log, + ) + assert field is not None + assert "datatype" not in field + assert log.as_dicts() == [] + + def test_a_present_but_unmapped_data_type_logs_exactly_one_issue_naming_it(self): + # The warehouse told us the type (GEOGRAPHY, outside the Ossie enum) and + # to_ossie has no mapping for it — that is a genuine silent loss unless + # this logs it. + log = IssueLog() + table = lambda n: {"name": "ORDERS", "columns": [ + {"name": "LOC", "db_column_name": "LOC", + "db_column_properties": {"data_type": "GEOGRAPHY"}}]} + field = convert_field( + {"name": "Location", "column_id": "ORDERS::LOC", + "properties": {"column_type": "ATTRIBUTE"}}, + {}, table, _resolve, log, + ) + assert field is not None + assert "datatype" not in field + issues = log.as_dicts() + assert len(issues) == 1 + assert "GEOGRAPHY" in issues[0]["message"] diff --git a/converters/thoughtspot/tests/test_tml_to_ossie_metrics.py b/converters/thoughtspot/tests/test_tml_to_ossie_metrics.py new file mode 100644 index 00000000..afbec6fc --- /dev/null +++ b/converters/thoughtspot/tests/test_tml_to_ossie_metrics.py @@ -0,0 +1,503 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest +from ossie_thoughtspot import stash +from ossie_thoughtspot.constants import METRIC_STASH_SHAPE, STASH_TML_NAME +from ossie_thoughtspot.issues import IssueLog +from ossie_thoughtspot.tml_to_ossie import _contains_aggregate_call, convert_field, convert_metric + + +def _resolve(table, column): + """Every reference lands in the dataset named after its table, lower-cased, + unless the table is named "MISSING" — then it resolves to nothing at all.""" + return None if table == "MISSING" else f"{table.lower()}.{column.lower()}" + + +class TestConvertMetric: + def _table(self, name): + return {"ORDERS": {"name": "ORDERS", "columns": [ + {"name": "AMOUNT", "db_column_name": "AMOUNT", + "db_column_properties": {"data_type": "DOUBLE"}}, + ]}}.get(name) + + # -- Row 1 of the truth table: column_id + aggregation. -- + def test_physical_column_with_aggregation_becomes_an_aggregate_metric(self): + log = IssueLog() + metric = convert_metric( + {"name": "Total Amount", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "MEASURE", "aggregation": "SUM"}}, + {}, self._table, _resolve, log, + ) + assert metric is not None + dialects = metric["expression"]["dialects"] + assert {"dialect": "THOUGHTSPOT", "expression": "sum ( [ORDERS::AMOUNT] )"} in dialects + assert {"dialect": "ANSI_SQL", "expression": "SUM(orders.amount)"} in dialects + assert log.as_dicts() == [] + + # -- Row 2: formula_id -> a *scalar* expr + aggregation. The two compose. -- + def test_scalar_formula_composes_with_the_column_aggregation(self): + log = IssueLog() + formulas = {"formula_Net": {"id": "formula_Net", "expr": "[A::x] - [A::y]"}} + metric = convert_metric( + {"name": "Average Net", "formula_id": "formula_Net", + "properties": {"column_type": "MEASURE", "aggregation": "AVERAGE"}}, + formulas, self._table, _resolve, log, + ) + assert metric is not None + dialects = metric["expression"]["dialects"] + assert {"dialect": "THOUGHTSPOT", "expression": "average ( [A::x] - [A::y] )"} in dialects + # [A::x] - [A::y] is compound, not a bare reference, so it is not portable + # on its own — no ANSI_SQL sibling can be composed around it either, and + # an issue records why (from expression_entries's own non-portability check). + assert "ANSI_SQL" not in [d["dialect"] for d in dialects] + issues = log.as_dicts() + assert len(issues) == 1 + + def test_scalar_formula_composes_and_the_ansi_sql_sibling_appears_when_portable(self): + # The other half of the same rule: when the scalar formula IS a bare, + # resolvable reference, composing produces a real ANSI_SQL sibling too, not + # just a THOUGHTSPOT-only rendering. + log = IssueLog() + formulas = {"formula_Bare": {"id": "formula_Bare", "expr": "[ORDERS::AMOUNT]"}} + metric = convert_metric( + {"name": "Average Amount", "formula_id": "formula_Bare", + "properties": {"column_type": "MEASURE", "aggregation": "AVERAGE"}}, + formulas, self._table, _resolve, log, + ) + dialects = metric["expression"]["dialects"] + assert {"dialect": "THOUGHTSPOT", "expression": "average ( [ORDERS::AMOUNT] )"} in dialects + assert {"dialect": "ANSI_SQL", "expression": "AVG(orders.amount)"} in dialects + assert log.as_dicts() == [] + + # -- Row 3: formula_id -> an *aggregate* expr. The column aggregation is a no-op. -- + def test_aggregate_formula_ignores_the_column_aggregation(self): + # The documented no-op: the formula's own outer call already aggregates, + # and ThoughtSpot's UI sets a column aggregation on a formula column like + # this routinely, redundant or not -- so this must be silent, not just + # correct. A warning here would fire on a large fraction of ordinary, + # correct metrics. + log = IssueLog() + formulas = {"formula_Sum": {"id": "formula_Sum", "expr": "sum ( [A::x] )"}} + metric = convert_metric( + {"name": "Odd Max Of Sum", "formula_id": "formula_Sum", + "properties": {"column_type": "MEASURE", "aggregation": "MAX"}}, + formulas, self._table, _resolve, log, + ) + dialects = metric["expression"]["dialects"] + assert {"dialect": "THOUGHTSPOT", "expression": "sum ( [A::x] )"} in dialects + assert not any("MAX" in d["expression"] for d in dialects) + assert not any(i["severity"] == "WARNING" for i in log.as_dicts()) + + def test_count_distinct_maps_to_count_distinct(self): + log = IssueLog() + metric = convert_metric( + {"name": "Unique Customers", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "MEASURE", "aggregation": "COUNT_DISTINCT"}}, + {}, self._table, _resolve, log, + ) + dialects = metric["expression"]["dialects"] + assert {"dialect": "THOUGHTSPOT", "expression": "unique count ( [ORDERS::AMOUNT] )"} \ + in dialects + assert {"dialect": "ANSI_SQL", "expression": "COUNT(DISTINCT orders.amount)"} in dialects + + def test_none_aggregation_means_no_aggregate(self): + log = IssueLog() + metric = convert_metric( + {"name": "Raw Amount", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "MEASURE", "aggregation": "NONE"}}, + {}, self._table, _resolve, log, + ) + assert metric["expression"]["dialects"] == [ + {"dialect": "THOUGHTSPOT", "expression": "[ORDERS::AMOUNT]"}, + {"dialect": "ANSI_SQL", "expression": "orders.amount"}, + ] + assert log.as_dicts() == [] + + def test_std_deviation_and_variance_map(self): + log = IssueLog() + stddev_metric = convert_metric( + {"name": "Amount Stddev", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "MEASURE", "aggregation": "STD_DEVIATION"}}, + {}, self._table, _resolve, log, + ) + variance_metric = convert_metric( + {"name": "Amount Variance", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "MEASURE", "aggregation": "VARIANCE"}}, + {}, self._table, _resolve, log, + ) + assert {"dialect": "THOUGHTSPOT", "expression": "stddev ( [ORDERS::AMOUNT] )"} \ + in stddev_metric["expression"]["dialects"] + assert {"dialect": "ANSI_SQL", "expression": "STDDEV(orders.amount)"} \ + in stddev_metric["expression"]["dialects"] + assert {"dialect": "THOUGHTSPOT", "expression": "variance ( [ORDERS::AMOUNT] )"} \ + in variance_metric["expression"]["dialects"] + assert {"dialect": "ANSI_SQL", "expression": "VARIANCE(orders.amount)"} \ + in variance_metric["expression"]["dialects"] + + @pytest.mark.parametrize("expr", [ + "sum( [A::x] )", + "sum(\n [A::x]\n)", + "count ( [A::x] )", + ]) + def test_the_thoughtspot_entry_is_the_verbatim_formula_expr(self, expr): + # The no-op (aggregate-formula) shape must carry the exact source text, + # untouched — never a reconstruction — even under adversarial whitespace, + # and even though a *different* column-level aggregation is present and + # must be discarded rather than applied. + log = IssueLog() + formulas = {"formula_Weird": {"id": "formula_Weird", "expr": expr}} + metric = convert_metric( + {"name": "Weird", "formula_id": "formula_Weird", + "properties": {"column_type": "MEASURE", "aggregation": "MAX"}}, + formulas, self._table, _resolve, log, + ) + thoughtspot_entries = [ + d for d in metric["expression"]["dialects"] if d["dialect"] == "THOUGHTSPOT" + ] + assert thoughtspot_entries == [{"dialect": "THOUGHTSPOT", "expression": expr}] + + def test_a_metric_name_that_normalises_differently_stashes_the_exact_name(self): + log = IssueLog() + metric = convert_metric( + {"name": "Gross Margin %!!", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "MEASURE", "aggregation": "SUM"}}, + {}, self._table, _resolve, log, + ) + assert metric["name"] == "gross_margin" + assert "label" not in metric # metrics have no label field + assert stash.read_stash(metric)[STASH_TML_NAME] == "Gross Margin %!!" + + def test_a_metric_that_needs_neither_tml_name_nor_shape_stashes_nothing(self): + # A converted document stays clean where ThoughtSpot added nothing. + # Both conditions have to hold at once here: the name must normalise to + # itself, AND the shape must be the "formula" default — the one shape + # that needs no stash entry, because it is also what a document with no + # stash defaults to on the way back. + log = IssueLog() + formulas = {"formula_Revenue": {"id": "formula_Revenue", "expr": "sum ( [A::x] )"}} + metric = convert_metric( + {"name": "revenue", "formula_id": "formula_Revenue", + "properties": {"column_type": "MEASURE", "aggregation": "NONE"}}, + formulas, self._table, _resolve, log, + ) + assert metric["name"] == "revenue" + assert "custom_extensions" not in metric + + def test_shape_is_stashed_even_when_the_name_is_unchanged(self): + # A column_id metric is shape `column_aggregation`, not the default + # `formula` — it needs the stash entry regardless of whether the name + # also needed one, so an unchanged name must not suppress it. + log = IssueLog() + metric = convert_metric( + {"name": "amount", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "MEASURE", "aggregation": "SUM"}}, + {}, self._table, _resolve, log, + ) + assert metric["name"] == "amount" + payload = stash.read_stash(metric) + assert payload[METRIC_STASH_SHAPE] == "column_aggregation" + assert STASH_TML_NAME not in payload + + def test_each_shape_is_stashed_with_its_own_enum_value(self): + # Pins all three enum spellings the stash schema defines, and confirms + # each survives a read_stash round trip. The "formula" shape is the one + # value that is never written (see the empty-payload test above), so its + # absence here is itself the assertion for that row. + log = IssueLog() + column_aggregation_metric = convert_metric( + {"name": "Total Amount", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "MEASURE", "aggregation": "SUM"}}, + {}, self._table, _resolve, log, + ) + formulas = {"formula_Net": {"id": "formula_Net", "expr": "[A::x] - [A::y]"}} + scalar_plus_aggregation_metric = convert_metric( + {"name": "Average Net", "formula_id": "formula_Net", + "properties": {"column_type": "MEASURE", "aggregation": "AVERAGE"}}, + formulas, self._table, _resolve, log, + ) + formulas = {"formula_Sum": {"id": "formula_Sum", "expr": "sum ( [A::x] )"}} + formula_metric = convert_metric( + {"name": "Odd Max Of Sum", "formula_id": "formula_Sum", + "properties": {"column_type": "MEASURE", "aggregation": "MAX"}}, + formulas, self._table, _resolve, log, + ) + + assert stash.read_stash(column_aggregation_metric)[METRIC_STASH_SHAPE] == "column_aggregation" + assert ( + stash.read_stash(scalar_plus_aggregation_metric)[METRIC_STASH_SHAPE] + == "scalar_formula_plus_aggregation" + ) + assert METRIC_STASH_SHAPE not in stash.read_stash(formula_metric) + + def test_datatype_is_emitted_only_for_a_bare_aggregate_over_a_typed_column(self): + log = IssueLog() + + count_metric = convert_metric( + {"name": "Order Count", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "MEASURE", "aggregation": "COUNT"}}, + {}, self._table, _resolve, log, + ) + count_distinct_metric = convert_metric( + {"name": "Distinct Count", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "MEASURE", "aggregation": "COUNT_DISTINCT"}}, + {}, self._table, _resolve, log, + ) + sum_metric = convert_metric( + {"name": "Total", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "MEASURE", "aggregation": "SUM"}}, + {}, self._table, _resolve, log, + ) + formulas = {"formula_Sum": {"id": "formula_Sum", "expr": "sum ( [A::x] )"}} + formula_metric = convert_metric( + {"name": "Formula Total", "formula_id": "formula_Sum", + "properties": {"column_type": "MEASURE", "aggregation": "NONE"}}, + formulas, self._table, _resolve, log, + ) + + assert count_metric["datatype"] == "Integer" + assert count_distinct_metric["datatype"] == "Integer" + assert sum_metric["datatype"] == "Decimal" # the physical column's own mapped type + assert "datatype" not in formula_metric # a formula has no declared type anywhere + + def test_a_formula_id_with_no_matching_formulas_entry_raises_an_issue(self): + log = IssueLog() + metric = convert_metric( + {"name": "Orphan", "formula_id": "formula_Nonexistent", + "properties": {"column_type": "MEASURE", "aggregation": "SUM"}}, + {}, self._table, _resolve, log, + ) + assert metric is None + issues = log.as_dicts() + assert len(issues) == 1 + assert "Orphan" in issues[0]["message"] + assert "formula_Nonexistent" in issues[0]["message"] + + # -- Tests of my own, beyond everything specified above. -- + # + # 1. An unrecognised `aggregation` value must not raise KeyError. A missing + # formula_id is already guarded against a bare KeyError above; the + # _AGGREGATION lookup is exactly the same shape of hazard on a different + # dict: a malformed or newer-than-this-converter TML value hitting an + # unguarded lookup would crash the whole conversion instead of degrading + # one metric. Chosen because it is the most direct sibling of a failure + # mode already treated as important elsewhere in this suite, just applied + # to a different lookup. + def test_an_unrecognised_aggregation_value_logs_and_falls_back_to_none(self): + log = IssueLog() + metric = convert_metric( + {"name": "Mystery", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "MEASURE", "aggregation": "BOGUS"}}, + {}, self._table, _resolve, log, + ) + assert metric is not None + assert metric["expression"]["dialects"] == [ + {"dialect": "THOUGHTSPOT", "expression": "[ORDERS::AMOUNT]"}, + {"dialect": "ANSI_SQL", "expression": "orders.amount"}, + ] + issues = log.as_dicts() + assert len(issues) == 1 + assert "BOGUS" in issues[0]["message"] + + # 2. An ATTRIBUTE column must not become a metric. Building one for it here + # would surface the same TML column as two competing Ossie objects (a field + # from convert_field and a metric from here) once a future caller runs both + # functions over every columns[] entry, so this boundary needs to be + # enforced on this function's own side, not only on convert_field's + # ATTRIBUTE-only check. + def test_an_attribute_column_is_not_a_metric(self): + log = IssueLog() + assert convert_metric( + {"name": "Amount", "column_id": "ORDERS::AMOUNT", + "properties": {"column_type": "ATTRIBUTE"}}, + {}, self._table, _resolve, log, + ) is None + assert log.as_dicts() == [] + + # -- One more, mirroring convert_field's own coverage of the same shape. -- + def test_neither_column_id_nor_formula_id_logs_and_returns_none(self): + log = IssueLog() + metric = convert_metric( + {"name": "Nothing", "properties": {"column_type": "MEASURE"}}, + {}, self._table, _resolve, log, + ) + assert metric is None + issues = log.as_dicts() + assert len(issues) == 1 + assert "column_id" in issues[0]["message"] + assert "formula_id" in issues[0]["message"] + + # -- A guard against silent double aggregation, narrowed to the one shape + # -- worth a warning: an aggregate nested inside a still-scalar outer call. + # -- An aggregate *as the outer call* (sum(...), group_aggregate(...), ...) + # -- is the documented, common no-op ThoughtSpot's UI produces routinely, + # -- and must stay silent -- warning there would fire on a large fraction + # -- of ordinary, correct metrics. + @pytest.mark.parametrize("expr", [ + "sum ( [T::x] )", + "average ( [T::x] )", + "unique count ( [T::x] )", + "group_aggregate ( sum ( [T::x] ) , query_groups ( ) , query_filters ( ) )", + ]) + def test_an_aggregate_outer_call_is_silent_even_with_a_redundant_aggregation(self, expr): + log = IssueLog() + formulas = {"formula_X": {"id": "formula_X", "expr": expr}} + metric = convert_metric( + {"name": "X", "formula_id": "formula_X", + "properties": {"column_type": "MEASURE", "aggregation": "SUM"}}, + formulas, self._table, _resolve, log, + ) + thoughtspot_entries = [ + d for d in metric["expression"]["dialects"] if d["dialect"] == "THOUGHTSPOT" + ] + assert thoughtspot_entries == [{"dialect": "THOUGHTSPOT", "expression": expr}] + assert not any(i["severity"] == "WARNING" for i in log.as_dicts()) + + def test_an_aggregate_nested_inside_a_scalar_wrapper_does_not_get_double_aggregated(self): + # The case an outer-call-only check cannot catch: round's own outer call + # is scalar, but sum is buried one level inside it -- the one shape + # where a reader might expect composition and not get it, so it is the + # one shape worth a warning. + log = IssueLog() + formulas = {"formula_R": {"id": "formula_R", "expr": "round ( sum ( [T::x] ) , 2 )"}} + metric = convert_metric( + {"name": "Rounded", "formula_id": "formula_R", + "properties": {"column_type": "MEASURE", "aggregation": "AVERAGE"}}, + formulas, self._table, _resolve, log, + ) + thoughtspot_entries = [ + d for d in metric["expression"]["dialects"] if d["dialect"] == "THOUGHTSPOT" + ] + assert thoughtspot_entries == [ + {"dialect": "THOUGHTSPOT", "expression": "round ( sum ( [T::x] ) , 2 )"} + ] + warnings = [i for i in log.as_dicts() if i["severity"] == "WARNING"] + assert len(warnings) == 1 + assert warnings[0]["code"] == "TS-METRIC-AGGREGATION-ALREADY-AGGREGATED" + + def test_group_aggregate_nested_inside_a_scalar_wrapper_also_warns_once(self): + # Same shape as the round(sum(x), 2) case above, but with + # group_aggregate as the buried aggregate rather than a bare sum -- + # confirming the nested-detection path recognises the broadened set, + # not only the original eight TML-aggregation-mapped names. + log = IssueLog() + formulas = {"formula_GA": {"id": "formula_GA", "expr": ( + "round ( group_aggregate ( sum ( [T::x] ) , query_groups ( ) , " + "query_filters ( ) ) , 2 )" + )}} + metric = convert_metric( + {"name": "Rounded GA", "formula_id": "formula_GA", + "properties": {"column_type": "MEASURE", "aggregation": "AVERAGE"}}, + formulas, self._table, _resolve, log, + ) + thoughtspot_entries = [ + d for d in metric["expression"]["dialects"] if d["dialect"] == "THOUGHTSPOT" + ] + assert thoughtspot_entries == [ + {"dialect": "THOUGHTSPOT", "expression": formulas["formula_GA"]["expr"]} + ] + assert not any( + d["expression"].startswith("average (") for d in metric["expression"]["dialects"] + ) + warnings = [i for i in log.as_dicts() if i["severity"] == "WARNING"] + assert len(warnings) == 1 + assert warnings[0]["code"] == "TS-METRIC-AGGREGATION-ALREADY-AGGREGATED" + + def test_a_genuinely_scalar_formula_still_composes_despite_the_new_guard(self): + # The guard must not break the case this whole feature exists for. + log = IssueLog() + formulas = {"formula_Net": {"id": "formula_Net", "expr": "[A::x] - [A::y]"}} + metric = convert_metric( + {"name": "Average Net", "formula_id": "formula_Net", + "properties": {"column_type": "MEASURE", "aggregation": "AVERAGE"}}, + formulas, self._table, _resolve, log, + ) + thoughtspot_entries = [ + d for d in metric["expression"]["dialects"] if d["dialect"] == "THOUGHTSPOT" + ] + assert thoughtspot_entries == [ + {"dialect": "THOUGHTSPOT", "expression": "average ( [A::x] - [A::y] )"} + ] + assert not any( + i["code"] == "TS-METRIC-AGGREGATION-ALREADY-AGGREGATED" for i in log.as_dicts() + ) + + # -- Field/metric wording and codes must match the object being converted. -- + def test_a_missing_physical_column_on_a_metric_says_metric_not_field(self): + log = IssueLog() + metric = convert_metric( + {"name": "Ghost Metric", "column_id": "ORDERS::NOPE", + "properties": {"column_type": "MEASURE", "aggregation": "SUM"}}, + {}, self._table, _resolve, log, + ) + assert metric is not None + issues = log.as_dicts() + assert any(i["code"] == "TS-METRIC-PHYSICAL-COLUMN-MISSING" for i in issues) + assert all("field" not in i["message"] for i in issues) + assert all("TS-FIELD-" not in i["code"] for i in issues) + + def test_a_non_portable_metric_expression_says_metric_not_field(self): + log = IssueLog() + formulas = {"formula_Net": {"id": "formula_Net", "expr": "[A::x] - [A::y]"}} + metric = convert_metric( + {"name": "Net", "formula_id": "formula_Net", + "properties": {"column_type": "MEASURE", "aggregation": "NONE"}}, + formulas, self._table, _resolve, log, + ) + assert metric is not None + issues = log.as_dicts() + thoughtspot_only = [i for i in issues if i["code"] == "TS-EXPR-THOUGHTSPOT-ONLY"] + assert len(thoughtspot_only) == 1 + assert "metric" in thoughtspot_only[0]["message"] + assert "field" not in thoughtspot_only[0]["message"] + + def test_field_side_wording_and_codes_are_unchanged(self): + # The fix must not touch convert_field's own behaviour at all. + log = IssueLog() + field = convert_field( + {"name": "Ghost", "column_id": "ORDERS::NOPE", + "properties": {"column_type": "ATTRIBUTE"}}, + {}, self._table, _resolve, log, + ) + assert field is not None + issues = log.as_dicts() + assert len(issues) == 1 + assert issues[0]["code"] == "TS-FIELD-PHYSICAL-COLUMN-MISSING" + assert "field" in issues[0]["message"] + + +class TestContainsAggregateCall: + """Layer (a) + (b) at the unit level: the broadened set, checked at any depth.""" + + @pytest.mark.parametrize("expr", [ + "group_aggregate ( sum ( [T::x] ) , query_groups ( ) , query_filters ( ) )", + "sql_number_aggregate_op ( 'STDDEV_POP({0})' , [T::x] )", + "sql_int_aggregate_op ( 'COUNT({0})' , [T::x] )", + "round ( sum ( [A::x] ) , 2 )", + "sum ( [A::x] )", + "unique count ( [A::x] )", + ]) + def test_detected(self, expr): + assert _contains_aggregate_call(expr) is True + + @pytest.mark.parametrize("expr", [ + "[A::x] - [A::y]", + "least ( [A::x] , [A::y] )", + "[ORDERS::AMOUNT]", + ]) + def test_not_detected(self, expr): + assert _contains_aggregate_call(expr) is False diff --git a/converters/thoughtspot/tests/test_yaml.py b/converters/thoughtspot/tests/test_yaml.py new file mode 100644 index 00000000..8fce0a58 --- /dev/null +++ b/converters/thoughtspot/tests/test_yaml.py @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest +import yaml + +from ossie_thoughtspot import _yaml +from ossie_thoughtspot.errors import ConversionError + +YAML11_BOOL_TOKENS = ["y", "Y", "n", "N", "yes", "Yes", "YES", "no", "No", "NO", + "on", "On", "ON", "off", "Off", "OFF"] + + +@pytest.mark.parametrize("token", YAML11_BOOL_TOKENS) +def test_yaml11_bool_tokens_load_as_strings(token): + # PyYAML implements YAML 1.1 and would return True/False for these. + assert _yaml.load(f"value: {token}") == {"value": token} + + +@pytest.mark.parametrize("literal,expected", [("true", True), ("True", True), ("false", False)]) +def test_real_booleans_still_load_as_booleans(literal, expected): + assert _yaml.load(f"value: {literal}") == {"value": expected} + + +@pytest.mark.parametrize("token", YAML11_BOOL_TOKENS) +def test_yaml11_bool_tokens_are_quoted_on_dump(token): + # Unquoted, a YAML 1.1 reader downstream would resolve these back to booleans. + assert _yaml.load(_yaml.dump({"value": token})) == {"value": token} + assert f"'{token}'" in _yaml.dump({"value": token}) + + +def test_ordinary_strings_are_not_gratuitously_quoted(): + assert _yaml.dump({"value": "Region"}).strip() == "value: Region" + + +def test_round_trip_preserves_key_order(): + src = {"z": 1, "a": 2, "m": 3} + assert list(_yaml.load(_yaml.dump(src))) == ["z", "a", "m"] + + +PLAIN_PYYAML_MISREADS = ["yes", "Yes", "YES", "no", "No", "NO", + "on", "On", "ON", "off", "Off", "OFF"] + + +@pytest.mark.parametrize("token", PLAIN_PYYAML_MISREADS) +def test_loader_fixes_what_plain_pyyaml_gets_wrong(token): + """The loader is load-bearing exactly here: plain PyYAML returns a bool.""" + assert isinstance(yaml.safe_load(f"value: {token}")["value"], bool) + assert _yaml.load(f"value: {token}") == {"value": token} + + +PLAIN_PYYAML_LEAVES_BARE = ["y", "Y", "n", "N"] + + +@pytest.mark.parametrize("token", PLAIN_PYYAML_LEAVES_BARE) +def test_dumper_quotes_what_plain_pyyaml_leaves_bare(token): + """YAML 1.1 booleans PyYAML's own resolver omits, so plain SafeDumper emits them + bare. Another 1.1 reader would resolve them as booleans, which is why we quote.""" + assert f"'{token}'" in _yaml.dump({"value": token}) + assert f"'{token}'" not in yaml.dump({"value": token}, Dumper=yaml.SafeDumper, sort_keys=False) + + +def test_load_wraps_a_parser_error_in_conversion_error(): + # Never let a bare yaml.YAMLError escape — same never-a-bare-traceback + # contract stash.py holds for malformed custom_extensions JSON. + with pytest.raises(ConversionError, match="malformed YAML"): + _yaml.load("a: [1, 2\nb: 3") + + +def test_load_does_not_wrap_a_clean_document(): + assert _yaml.load("a: 1") == {"a": 1} + + +def test_dump_allow_unicode_round_trips_and_does_not_escape(): + # Without allow_unicode=True, PyYAML escapes non-ASCII as \xE9 etc. + text = _yaml.dump({"label": "Café"}) + assert "Café" in text + assert "\\x" not in text and "\\u" not in text + assert _yaml.load(text) == {"label": "Café"} diff --git a/converters/thoughtspot/tools/generate_reference_docs.py b/converters/thoughtspot/tools/generate_reference_docs.py new file mode 100644 index 00000000..d8a4b596 --- /dev/null +++ b/converters/thoughtspot/tools/generate_reference_docs.py @@ -0,0 +1,568 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Generate `docs/*.md` from the converter's own code. + +This package's expression mapping, reverse inventory, datatype map and vendor +payload were originally hand-authored design documents. The code now +*implements* that mapping, which makes the code the single source of truth — +so this script reads it back out into Markdown, rather than a document being +maintained by hand a second time alongside it. `tests/test_reference_docs_current.py` +regenerates on every test run and compares the result against the committed +`docs/*.md` files byte-for-byte, so the two cannot silently drift apart. + +**Not a runtime dependency.** This script is dev/tooling only: it is not +imported by anything under `src/`, it is not registered as a +`[project.scripts]` entry point, and it uses nothing beyond the Python +standard library plus this package's own modules — the package's only +*runtime* dependency stays PyYAML. + +Usage:: + + uv run --python 3.13 python tools/generate_reference_docs.py + +Regenerates every file in `DOCS` under `docs/`. Run it, then `git diff` — +an empty diff means the docs were already current. +""" +from __future__ import annotations + +import re +from collections import Counter +from pathlib import Path +from typing import Callable + +from ossie_thoughtspot import constants, datatypes +from ossie_thoughtspot.expressions import catalog, reverse +from ossie_thoughtspot.expressions._types import Classification + +PACKAGE_ROOT = Path(__file__).resolve().parents[1] + +# --------------------------------------------------------------------------- +# Markdown helpers — shared by every generate_*_doc() function below. +# --------------------------------------------------------------------------- + +_LICENSE_HEADER_MD = """""" + + +def _generated_banner(sources: list[str]) -> str: + source_lines = "\n".join(f" - `{s}`" for s in sources) + return ( + "" + ) + + +def _header(title: str, intro: str, sources: list[str]) -> str: + return f"{_LICENSE_HEADER_MD}\n\n{_generated_banner(sources)}\n\n# {title}\n\n{intro}\n" + + +def _clean(text: str) -> str: + """Collapse any embedded whitespace/newlines to single spaces and strip.""" + return " ".join(str(text).split()) + + +def _prose_cell(text: str | None) -> str: + """A plain-text table cell. Escapes a literal pipe with a backslash — the + documented GFM mechanism for a pipe that must not end the cell. + """ + if not text: + return "—" # em dash + return _clean(text).replace("|", "\\|") + + +def _code_cell(text: str | None) -> str: + """A code-styled table cell (backtick span). Falls back to `_prose_cell` + when the raw text itself contains a literal pipe (e.g. the spec construct + ``str1 || str2``): CommonMark does not process backslash escapes inside a + code span, so escaping the pipe *inside* the backticks would show the + backslash literally. Escaping it in plain text, outside a code span, is + the reliable mechanism instead — verified against this file's own single + affected row before relying on it. + """ + if not text: + return "—" + cleaned = _clean(text) + if "|" in cleaned: + return _prose_cell(cleaned) + return f"`{cleaned}`" + + +def _table(headers: list[str], rows: list[list[str]]) -> str: + lines = ["| " + " | ".join(headers) + " |", "|" + "|".join(["---"] * len(headers)) + "|"] + for row in rows: + lines.append("| " + " | ".join(row) + " |") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# 1. Expression mapping — expressions/catalog.py's CATALOG. +# --------------------------------------------------------------------------- + + +def generate_expression_mapping_doc() -> str: + sources = [ + "src/ossie_thoughtspot/expressions/catalog.py", + "src/ossie_thoughtspot/expressions/_types.py", + ] + intro = ( + "Every construct the Ossie expression language specification defines, mapped to " + "its ThoughtSpot rendering (`Ossie -> ThoughtSpot`, the direction " + "`expressions/catalog.py` drives). Rows follow the source's own definition " + "order, which groups related constructs together (aggregates, then type " + "conversion, date/time, string, math/conditional, operators, window functions) " + "— that grouping exists only as source comments, not as data the code carries, " + "so it is not reproduced as separate sections here." + ) + out = [_header("Ossie -> ThoughtSpot Expression Mapping", intro, sources)] + + counts = Counter(c.classification for c in catalog.CATALOG.values()) + total = len(catalog.CATALOG) + out.append("## Coverage\n") + coverage_rows = [ + [cls.value, str(counts.get(cls, 0)), f"{counts.get(cls, 0) / total:.0%}"] + for cls in Classification + ] + coverage_rows.append(["**Total**", f"**{total}**", "**100%**"]) + out.append(_table(["Classification", "Count", "Share"], coverage_rows)) + out.append("") + + out.append("## Constructs with no discrete specification table row\n") + out.append( + f"{len(catalog.CONVENTION_DIVERGENCES)} `CATALOG` rows are real, intended " + "constructs that `spec_construct_names()` cannot key on directly, because the " + "upstream specification describes them in prose or a code fence rather than a " + "table row with a `Syntax` column. Each is keyed via `CONVENTION_DIVERGENCES` " + "instead, with the reason recorded per construct.\n" + ) + out.append( + _table( + ["Construct", "Why it has no discrete spec table row"], + [ + [_code_cell(name), _prose_cell(reason)] + for name, reason in catalog.CONVENTION_DIVERGENCES.items() + ], + ) + ) + out.append("") + + out.append("## Every construct\n") + rows = [] + for name, c in catalog.CATALOG.items(): + if c.classification is Classification.UNMAPPABLE: + rendering = "—" + elif c.classification is Classification.PASSTHROUGH: + rendering = f"{_code_cell(c.template)} — pass-through via `{c.variant.value}`" + else: + rendering = _code_cell(c.template) + rows.append([_code_cell(name), c.classification.value, rendering, _prose_cell(c.note)]) + out.append( + _table(["Ossie construct", "Classification", "ThoughtSpot rendering", "Notes"], rows) + ) + out.append("") + + return "\n".join(out) + "\n" + + +# --------------------------------------------------------------------------- +# 2. Reverse inventory — expressions/reverse.py's REVERSE. +# --------------------------------------------------------------------------- + +_DISPOSITION_MEANING: dict[reverse.ReverseDisposition, str] = { + reverse.ReverseDisposition.COMPOSE: "A full, portable Ossie expression is produced.", + reverse.ReverseDisposition.PARTIAL: ( + "A real Ossie expression is produced, but it is provably incomplete." + ), + reverse.ReverseDisposition.DIALECT: ( + "Resolves to the Ossie `dialects[]` mechanism, not a portable expression." + ), + reverse.ReverseDisposition.STASH: ( + "No Ossie expression exists at all; preserved verbatim for round-trip only." + ), +} + + +def generate_reverse_inventory_doc() -> str: + sources = ["src/ossie_thoughtspot/expressions/reverse.py"] + intro = ( + "ThoughtSpot's own native functions with no counterpart in the Ossie " + "specification (`ThoughtSpot -> Ossie`, the reverse of the expression mapping " + "above), and how each reaches — or does not reach — a portable Ossie " + "expression. This inventory is not yet called from the shipped `TML -> Ossie` " + "conversion path; see `converters/thoughtspot/README.md`'s " + '"Expression translation" section for the converter\'s current, more ' + "conservative default." + ) + out = [_header("ThoughtSpot -> Ossie Reverse Inventory", intro, sources)] + + counts = Counter(c.disposition for c in reverse.REVERSE.values()) + total = len(reverse.REVERSE) + out.append("## Coverage\n") + coverage_rows = [ + [d.value, str(counts.get(d, 0)), f"{counts.get(d, 0) / total:.0%}", _DISPOSITION_MEANING[d]] + for d in reverse.ReverseDisposition + ] + coverage_rows.append(["**Total**", f"**{total}**", "**100%**", ""]) + out.append(_table(["Disposition", "Count", "Share", "Meaning"], coverage_rows)) + out.append("") + + out.append("## Cross-cutting dispatch, not name-keyed\n") + fiscal_markers = ", ".join(_code_cell(m) for m in sorted(reverse._FISCAL_MARKERS)) + hyperlink_tokens = " or ".join(_code_cell(t) for t in reverse._HYPERLINK_MARKUP_TOKENS) + out.append( + "Two checks apply before an ordinary lookup by name into `REVERSE`, so they are " + "not rows of the table below:\n\n" + f"- **Fiscal-calendar argument.** Any call whose last argument is {fiscal_markers} " + "stashes unconditionally, for any function name at all, before the name is " + "looked up.\n" + f"- **Hyperlink markup.** A `concat` call whose string arguments contain " + f"{hyperlink_tokens} is redirected to the `concat (hyperlink markup)` row below; " + "plain `concat` has a specification counterpart already covered by `CATALOG` " + "and is not this module's concern.\n" + ) + + out.append("## Every entry\n") + rows = [] + for name, c in reverse.REVERSE.items(): + fn = c.compose_fn or c.dispatch_fn + if c.template is not None: + composes_to = _code_cell(c.template) + elif fn is not None: + composes_to = f"dynamic — see `{fn.__qualname__}` in `reverse.py`" + else: + composes_to = "—" + + if c.issue_code: + issue = f"`{c.issue_code}` · {c.issue_severity.value}" + elif c.issue_message: + issue = c.issue_severity.value + else: + issue = "—" + + if c.note: + notes = c.note + elif c.issue_message: + notes = c.issue_message.format(name=f"`{name}`") + else: + notes = "" + + rows.append( + [_code_cell(name), c.disposition.value, composes_to, issue, _prose_cell(notes)] + ) + out.append( + _table(["ThoughtSpot construct", "Disposition", "Composes to", "Issue", "Notes"], rows) + ) + out.append("") + + return "\n".join(out) + "\n" + + +# --------------------------------------------------------------------------- +# 3. Datatype map — datatypes.py. +# --------------------------------------------------------------------------- + + +def _spelling_and_loss_note(ossie_type: str) -> str: + """Derived, not stated: calls `to_tml` with each spelling override and compares + the result to the default, so a type only reads as connection-dependent when the + function's own behaviour actually varies with the argument — e.g. `Decimal` + always renders `DOUBLE` regardless of `float_spelling`, while `Float` does not. + """ + default = datatypes.to_tml(ossie_type) + alt_bool = datatypes.to_tml(ossie_type, boolean_spelling="BOOL") + alt_float = datatypes.to_tml(ossie_type, float_spelling="FLOAT") + notes = [] + if alt_bool != default: + notes.append( + f"connection-dependent spelling — `{default}` by default, `{alt_bool}` " + "when the connection's own TML spells it that way" + ) + if alt_float != default: + notes.append( + f"connection-dependent spelling — `{default}` by default, `{alt_float}` " + "when the connection's own TML spells it that way" + ) + loss = datatypes.declared_loss(ossie_type) + if loss: + notes.append(loss) + return "; ".join(notes) if notes else "exact, single spelling" + + +def generate_datatype_map_doc() -> str: + sources = ["src/ossie_thoughtspot/datatypes.py"] + intro = ( + "The bidirectional Ossie <-> ThoughtSpot TML datatype map. The map is **not " + "injective** — several Ossie types collapse onto one TML spelling and cannot " + "be told apart on the way back; see \"Not injective\" below." + ) + out = [_header("Ossie <-> ThoughtSpot Datatype Map", intro, sources)] + + out.append("## The closed Ossie datatype enum\n") + out.append(", ".join(_code_cell(t) for t in sorted(datatypes.OSSIE_DATATYPES)) + "\n") + + out.append("## Ossie -> TML\n") + rows = [ + [_code_cell(t), _code_cell(datatypes.to_tml(t)), _spelling_and_loss_note(t)] + for t in sorted(datatypes.OSSIE_DATATYPES) + ] + out.append(_table(["Ossie datatype", "TML `data_type` (default)", "Notes"], rows)) + out.append( + f"\nA column with no declared `datatype` at all infers " + f"{_code_cell(datatypes.to_tml(None))} rather than raising — `datatype` is " + "optional in Ossie, but TML rejects a column with no `db_column_properties` " + "block at all.\n" + ) + + out.append("## TML -> Ossie\n") + rows = [ + [_code_cell(tml_type), _code_cell(datatypes.to_ossie(tml_type))] + for tml_type in sorted(datatypes._TO_OSSIE) + ] + out.append(_table(["TML `data_type`", "Ossie datatype"], rows)) + out.append( + "\nA TML `data_type` outside this map returns no Ossie datatype at all — " + "`datatype` is optional in Ossie, so omitting it is preferred over inventing one.\n" + ) + + out.append("## Not injective — declared losses\n") + rows = [ + [_code_cell(t), _prose_cell(reason)] + for t, reason in sorted(datatypes._DECLARED_LOSS.items()) + ] + out.append(_table(["Ossie datatype", "Why the round trip is lossy"], rows)) + out.append("") + + return "\n".join(out) + "\n" + + +# --------------------------------------------------------------------------- +# 4. Vendor payload — constants.py's custom_extensions[THOUGHTSPOT] vocabulary. +# --------------------------------------------------------------------------- + +_SCOPE_RE = re.compile(r"^(MODEL|DATASET|RELATIONSHIP|FIELD|METRIC)_STASH_") + + +def _scope_for_constant(name: str) -> str: + match = _SCOPE_RE.match(name) + return match.group(1).title() if match else "Shared" + + +def _stash_key_constant_names() -> list[str]: + """Every top-level `custom_extensions[THOUGHTSPOT]` key constant — excludes the + `_WITNESS` companion constants (a witness is not itself a key this converter + classifies; it is the currency check FOR one) and the nested + `source_parts.{db,schema,db_table}` sub-keys, which are never read as standalone + top-level payload keys. Same exclusion shape as + `tests/test_stash_key_classification.py`'s own scan, arrived at independently via + runtime introspection (`vars(constants)`) rather than a second text-regex reading + of the same file. + """ + names = [] + for name, value in vars(constants).items(): + if not name.isupper() or not isinstance(value, str): + continue + if name.endswith("_WITNESS") or "SOURCE_PARTS_" in name: + continue + if "_STASH_" in name or name == "STASH_TML_NAME": + names.append(name) + return names + + +def _treatment_text(key: str, cls: "constants.StashKeyClass", has_witness: bool) -> str: + if cls is constants.StashKeyClass.INFORMATION_ONLY: + text = ( + "Restored as-is whenever present — nothing on the Ossie side could " + "have diverged from it." + ) + if key in constants.STASH_KEYS_WITH_DERIVABLE_MEMBERSHIP: + text += ( + " Each list entry is additionally checked against live coverage before " + "being restored: an entry now covered by a live field is dropped rather " + "than duplicated." + ) + return text + if has_witness: + return ( + "Restored only if its witness companion key still matches the live " + "document's current value; a mismatch means the document changed since the " + "stash was written, so the value is re-derived instead." + ) + return ( + "Restored only if reconstructing it from the live document still agrees with " + "the stashed value (self-verifying — no separate witness key); " + "disagreement re-derives instead." + ) + + +def generate_vendor_payload_doc() -> str: + sources = ["src/ossie_thoughtspot/constants.py"] + intro = ( + "TML carries properties Ossie's core specification has no field for. " + "`TML -> Ossie` stashes each one under a single `custom_extensions` entry " + "attached to the Ossie object it came from; `Ossie -> TML` reads the same " + "entry back. This page is generated from `constants.py`'s own key vocabulary " + "and `STASH_KEY_CLASSIFICATION` — the table `test_stash_key_classification.py` " + "enforces every key read on the `Ossie -> TML` direction must appear in." + ) + out = [_header("The `custom_extensions[THOUGHTSPOT]` Payload", intro, sources)] + + out.append("## Envelope\n") + out.append( + f"Every `custom_extensions` entry this converter writes uses " + f"`vendor_name` {_code_cell(constants.VENDOR_KEY)}. `data` is a single " + f"JSON-encoded string (never a nested object) whose own top-level `_v` field " + f"is the shape version ({_code_cell(str(constants.STASH_VERSION))} today) — " + "bumped only when the payload's shape changes, never for a value change; an " + "unrecognised version is a hard failure rather than a silent misread.\n" + ) + + all_names = _stash_key_constant_names() + value_to_name = {getattr(constants, n): n for n in all_names} + + out.append("## Payload keys\n") + rows = [] + for key, cls in constants.STASH_KEY_CLASSIFICATION.items(): + name = value_to_name.get(key, "?") + scope = _scope_for_constant(name) + has_witness = hasattr(constants, f"{name}_WITNESS") + rows.append( + [ + _code_cell(key), + scope, + cls.value, + _treatment_text(key, cls, has_witness), + ] + ) + out.append(_table(["Key", "Scope", "Classification", "Treatment on the return trip"], rows)) + out.append("") + + witness_names = sorted( + n for n in vars(constants) if n.isupper() and n.endswith("_WITNESS") + ) + if witness_names: + out.append("## Witness companion keys\n") + out.append( + "A `SHADOWS_DERIVABLE` key's stashed value is checked for currency before " + "being restored; these are the witness copies that check does it against " + "(see `stash.restore`).\n" + ) + rows = [] + for wn in witness_names: + primary_name = wn[: -len("_WITNESS")] + primary_value = getattr(constants, primary_name, None) + rows.append( + [ + _code_cell(getattr(constants, wn)), + _code_cell(primary_value) if primary_value else "—", + ] + ) + out.append(_table(["Witness key", "Checks currency for"], rows)) + out.append("") + + source_parts_names = sorted( + n for n in vars(constants) if n.isupper() and "SOURCE_PARTS_" in n + ) + if source_parts_names: + out.append("## Nested keys under `source_parts`\n") + rows = [ + [_code_cell(getattr(constants, n)), _code_cell(f"source_parts.{getattr(constants, n)}")] + for n in source_parts_names + ] + out.append(_table(["Sub-key", "Full path"], rows)) + out.append("") + + metric_shape_names = sorted(n for n in vars(constants) if n.startswith("METRIC_SHAPE_")) + if metric_shape_names: + out.append("## `METRIC_STASH_SHAPE` value vocabulary\n") + rows = [[_code_cell(n), _code_cell(getattr(constants, n))] for n in metric_shape_names] + out.append(_table(["Constant", "Value"], rows)) + out.append("") + + if constants.STASH_ONLY_CARRIER_KEY_CLASSIFICATION: + out.append("## Reclassified on a stash-only carrier\n") + out.append( + "The same key name, reclassified when it is read off a stash-only carrier " + "(an `unrepresentable_joins[]` or `unattributed_formulas[]` entry) that has " + "no independent Relationship/Metric/Field object of its own to diverge " + "from.\n" + ) + rows = [] + for key, cls in constants.STASH_ONLY_CARRIER_KEY_CLASSIFICATION.items(): + primary_cls = constants.STASH_KEY_CLASSIFICATION.get(key) + rows.append( + [ + _code_cell(key), + primary_cls.value if primary_cls is not None else "—", + cls.value, + ] + ) + out.append( + _table(["Key", "Classification (primary carrier)", "Classification (stash-only carrier)"], rows) + ) + out.append("") + + return "\n".join(out) + "\n" + + +# --------------------------------------------------------------------------- +# Registry + entry point. +# --------------------------------------------------------------------------- + +DOCS: dict[str, Callable[[], str]] = { + "expression-mapping.md": generate_expression_mapping_doc, + "reverse-inventory.md": generate_reverse_inventory_doc, + "datatype-map.md": generate_datatype_map_doc, + "vendor-payload.md": generate_vendor_payload_doc, +} + + +def generate_all() -> dict[str, str]: + """{filename: content} for every document `DOCS` declares.""" + return {name: fn() for name, fn in DOCS.items()} + + +def main() -> None: + docs_dir = PACKAGE_ROOT / "docs" + docs_dir.mkdir(exist_ok=True) + for name, content in generate_all().items(): + (docs_dir / name).write_text(content, encoding="utf-8") + print(f"Wrote {len(DOCS)} file(s) to {docs_dir}") + + +if __name__ == "__main__": + main() diff --git a/converters/thoughtspot/uv.lock b/converters/thoughtspot/uv.lock new file mode 100644 index 00000000..ef26557a --- /dev/null +++ b/converters/thoughtspot/uv.lock @@ -0,0 +1,638 @@ +version = 1 +revision = 3 +requires-python = ">=3.10" +resolution-markers = [ + "python_full_version >= '3.11'", + "python_full_version < '3.11'", +] + +[[package]] +name = "apache-ossie-thoughtspot" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "hypothesis" }, + { name = "jsonschema" }, + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [{ name = "pyyaml", specifier = ">=6.0" }] + +[package.metadata.requires-dev] +dev = [ + { name = "hypothesis", specifier = ">=6.0" }, + { name = "jsonschema", specifier = ">=4.26.0" }, + { name = "pytest", specifier = ">=8.0" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, +] + +[[package]] +name = "hypothesis" +version = "6.167.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/c9/8cee74c1390b2932406faaab76980f18946f258fa5a8afca17189b3bc655/hypothesis-6.167.1.tar.gz", hash = "sha256:62eefcb4d2791423626e9901c3027a6e0c5ffda2ac0b44b3c7e797ab9d2d5a4c", size = 505849, upload-time = "2026-08-30T19:53:09.05Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/4d/3592ca336deafbd3e9b0f47dc4c727aa32d30e765ef6370da8ecd590d388/hypothesis-6.167.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d28118fd70e4e15ff9c308a98b312b544b6145ae45aaa3b566328c1fdee8058f", size = 785476, upload-time = "2026-08-30T19:51:02.154Z" }, + { url = "https://files.pythonhosted.org/packages/18/bf/e33c431148994cbcb3332c6df94b833ecfb4aa6a8e51ea4b83da55ddd581/hypothesis-6.167.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e517be7f82a0a917758cc489a88b826b5371f56381fd94b4a8a09ce82d8de406", size = 781033, upload-time = "2026-08-30T19:51:27.314Z" }, + { url = "https://files.pythonhosted.org/packages/94/a3/e0de9a82c7e790a1def0801076e0ef43110f98e95ed54a3554877d0cb66d/hypothesis-6.167.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26f8cec74c4fad7aeb0852cb34c2134b16db05d878ad3946a53337dace7016f4", size = 1117814, upload-time = "2026-08-30T19:53:04.109Z" }, + { url = "https://files.pythonhosted.org/packages/71/a4/8dd6bdc909324d1c39da1c86d65f75512ae049c159952af4cfe8feb5f8d4/hypothesis-6.167.1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fd202d02d129197a5e771f8a11c7d30559927284c23ec3a8bd4f37a7955964d1", size = 1141639, upload-time = "2026-08-30T19:51:50.399Z" }, + { url = "https://files.pythonhosted.org/packages/fa/bd/c13ed6145c360d0770415efd7d5a7e63c29905aeef52ab88004fe7e7f924/hypothesis-6.167.1-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8b1e393ab01b71f683ba2a783785871cc6b81a6e41017780c64a5bc0b99759ae", size = 1143334, upload-time = "2026-08-30T19:50:38.045Z" }, + { url = "https://files.pythonhosted.org/packages/c0/a8/060d79ed8504b54ced9ad16f33d674b1b98a9debe9733c02709d7dd5c71c/hypothesis-6.167.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c385c7741893404306f9e5559ab3835432e85e7c153e25f854c502c410bbcbb", size = 1163345, upload-time = "2026-08-30T19:53:06.583Z" }, + { url = "https://files.pythonhosted.org/packages/cb/f7/6e68e2b705f729a6f7b4f41030022b1a5264c5434d3bcd917233d6801c6a/hypothesis-6.167.1-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:94920ca1fae70c26b0bd3fabbeef9437ffc17a39fe85696fb9a86187d92f6dba", size = 1123029, upload-time = "2026-08-30T19:52:03.134Z" }, + { url = "https://files.pythonhosted.org/packages/a5/c0/d274fe37ed5ecd5ad8ed555edc1f5e2abc8e1c3be3d5404b7edd5cc353a8/hypothesis-6.167.1-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b8d90ded2ffdc7e56b5e571993f384b52fade0a7b424e614f999cc2491789970", size = 1154003, upload-time = "2026-08-30T19:50:55.053Z" }, + { url = "https://files.pythonhosted.org/packages/b4/2f/2b5bb386f43fc965eb86fd69fcb2bd62c08cb6d7c6708a40dc39b3b97440/hypothesis-6.167.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:40cd5de7dd252942a08480639f5850594b1aca4a463e8a7f15e1fb6c2c3760c1", size = 1293729, upload-time = "2026-08-30T19:52:59.481Z" }, + { url = "https://files.pythonhosted.org/packages/ac/32/22436b072d79011fe81abb933edcd2476057c7b971588c5f3caf07519a88/hypothesis-6.167.1-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:b9c33f921ddc7fea93660eca408b25fe755516e22ec7ab21cb9951031f1cd608", size = 1419248, upload-time = "2026-08-30T19:50:52.903Z" }, + { url = "https://files.pythonhosted.org/packages/9e/3e/abf39faaff0f78112112a82316a5c9fe472574480c1ecee526734775b812/hypothesis-6.167.1-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:495989cf0a5ee03f7f9598ee9efeaabf15fd861ec52b5a9d6435849453e17e5d", size = 1274903, upload-time = "2026-08-30T19:52:27.25Z" }, + { url = "https://files.pythonhosted.org/packages/29/83/69b89ed5692ba3aad117facfb9ce099633a22c35acc3d64829b72253ec8c/hypothesis-6.167.1-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:bc73c46ce8ff93b0eb220f2b75adcbd9fcc9112078a74d3522d2532ad8069bad", size = 1294185, upload-time = "2026-08-30T19:50:35.038Z" }, + { url = "https://files.pythonhosted.org/packages/2b/1e/55dfcbe45c72df0a5c5b86a6b7c9365121acab69c2cc060bd55336a48c8f/hypothesis-6.167.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:36e83e1d7e97aaacbf6cd778e14a841344f848a674b20dfe4fe997546a6a2151", size = 1330013, upload-time = "2026-08-30T19:52:54.841Z" }, + { url = "https://files.pythonhosted.org/packages/3b/a6/0a36cead4ccff58bedb5d1aa2f894f7a580c317424b4fa788c6b22232b2d/hypothesis-6.167.1-cp310-abi3-win32.whl", hash = "sha256:fb4d87454d2459c2ccb541a4c61c92ce13058b91305ed3304695a409a1d886e4", size = 671942, upload-time = "2026-08-30T19:51:32.732Z" }, + { url = "https://files.pythonhosted.org/packages/ec/5b/360285ed42109f5ef48d98ca9ffcf71d1477130c0ff1f1a8d535c8507259/hypothesis-6.167.1-cp310-abi3-win_amd64.whl", hash = "sha256:5e35f98b427bf438a946203426b485dd5b62485f3d5a69a0e0862870a545e518", size = 678637, upload-time = "2026-08-30T19:50:33.573Z" }, + { url = "https://files.pythonhosted.org/packages/79/2d/cc084c1a8bfa296048ec0461f0fa11731abe0097f5c2310c8d39d94c8dc4/hypothesis-6.167.1-cp310-abi3-win_arm64.whl", hash = "sha256:dd6a0808a2eb8b5b1ac06bca4244eee18ed2c0e7b105599e1662203d164317b5", size = 676657, upload-time = "2026-08-30T19:51:34.494Z" }, + { url = "https://files.pythonhosted.org/packages/4a/8e/e0f470823bc301a97a8e6156806f6322f5ab99a2f40b6c604261966b3393/hypothesis-6.167.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:64cf8b7ac9a0cc80dad8884f81a2e50f0b79956694927d40e21e0cfa48830b8a", size = 786180, upload-time = "2026-08-30T19:52:09.714Z" }, + { url = "https://files.pythonhosted.org/packages/6c/4d/6e9e430ded5e226f245cd7e55923c7675311fa2115bc4262b5802edfcbb2/hypothesis-6.167.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:69b92f037080cefb949c5f9683873abac12283e0dc77201df089369c1ef67e4c", size = 781901, upload-time = "2026-08-30T19:50:42.641Z" }, + { url = "https://files.pythonhosted.org/packages/bd/aa/8df2711daf3ace849045482492b7f178fb55b79201f332477f6eef590b46/hypothesis-6.167.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19b334180260de636a0b3017dd96d63709da0c4182c9cb28f52dd6cda78dc933", size = 1118135, upload-time = "2026-08-30T19:51:54.599Z" }, + { url = "https://files.pythonhosted.org/packages/8c/4f/6040b58ffc511013394ba027191dd7e2886c5ae39e0b7872ab15503803fe/hypothesis-6.167.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:769d0242531067e6daf16b6c9894fd9584139884de887023328e3c5658993597", size = 1163947, upload-time = "2026-08-30T19:50:45.908Z" }, + { url = "https://files.pythonhosted.org/packages/ca/66/24aedbae1b56e71308d0aef4415e37c6bb22068c77e95337cdf9d74cd8b1/hypothesis-6.167.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:22b6c3def5446016148523b74ef9da4948bec5ef05865d56c99ba187f4092663", size = 1294290, upload-time = "2026-08-30T19:52:45.383Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e8/a063e3ab97851f4425918f0fcd407d0f426184c26b09f3bda7369e3d38d7/hypothesis-6.167.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cbbb6bc17a5a04120a5bfd830335b8b301a22d21d2262804be330c235031b4c5", size = 1330694, upload-time = "2026-08-30T19:51:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/bd/f6/6ebb84d532c0d5b36b3bab7b477a167685b2210a0246c7101557bfa4751c/hypothesis-6.167.1-cp310-cp310-win_amd64.whl", hash = "sha256:cdc7e20161f21f14c2d7a057054521db0a8c4bbb647d2e50c90c84f28e4621a0", size = 678586, upload-time = "2026-08-30T19:52:31.618Z" }, + { url = "https://files.pythonhosted.org/packages/ff/14/2445b7b1a0c8db61c6812b74606ed7a4f41e3ea0e0c103313e25995b82d5/hypothesis-6.167.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:613bf10e6e490daaa88eb4bf06fb3aacf6572b887e2f9fa5d0bac1be96a18c00", size = 785945, upload-time = "2026-08-30T19:51:19.221Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f6/67926d308a9ab19bb7dfb5118832fa74b508f94871fadbb3370629decbfc/hypothesis-6.167.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f715d912560dd4df3daab4c1fd98c01132eea7ab292f5b9e1d28fc419fe63348", size = 781726, upload-time = "2026-08-30T19:52:57.179Z" }, + { url = "https://files.pythonhosted.org/packages/33/de/22ac0e272530b36ad840170bf661ff89df8648052bb6a37dba520b312f1f/hypothesis-6.167.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cdd2fc0232b47910e9621f8c1b5732381438055b03fcc69180e1ef3659b7e70", size = 1117929, upload-time = "2026-08-30T19:51:58.828Z" }, + { url = "https://files.pythonhosted.org/packages/4d/1f/b72b51bac9a7d330bcdda01dc0ab1abe76e1c29ff522effd2d1be4e23702/hypothesis-6.167.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:163e4cebd4b2380ff92f5d36bd04697e82b13973794440694da768521e1e2eab", size = 1163855, upload-time = "2026-08-30T19:52:33.689Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b4/02424328951f0244f7dd3f620775ed22d39dc234e92759f9d2980150252b/hypothesis-6.167.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2de4e67289f86e358732eb16b345f9d1f987c33e44b10d4ffa3ccd715dea9316", size = 1294177, upload-time = "2026-08-30T19:50:44.418Z" }, + { url = "https://files.pythonhosted.org/packages/d8/09/2262d6b362c81066ad451fda48633b2d3ea6cf2d0d673ee6554ad8f86c58/hypothesis-6.167.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:fcfd20792f62d65729f850ea50328c1f8b09874d5960b122715b9e6784a7a547", size = 1330267, upload-time = "2026-08-30T19:51:23.5Z" }, + { url = "https://files.pythonhosted.org/packages/70/5d/19e8c02eebfe89592dfd12372032e8868ad40e6d40b38e5ecc4935ce194f/hypothesis-6.167.1-cp311-cp311-win_amd64.whl", hash = "sha256:ebb841d21156039d7da0a41fa9de4ccf468510a4e4d8144f4fe2b3f31239ef3b", size = 678401, upload-time = "2026-08-30T19:51:44.77Z" }, + { url = "https://files.pythonhosted.org/packages/72/82/07987292cfb59c73ce6574e2912c015f678d5aa4d8c0712b78ae4415a535/hypothesis-6.167.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1937ae4e23f7dde6d4202d3d08c2633bcd535a091bdf866b8799abaabcb1e6f0", size = 787050, upload-time = "2026-08-30T19:51:10.782Z" }, + { url = "https://files.pythonhosted.org/packages/c7/e9/0d37051bec44ec433d87da03c9a7fe389b1b58210790c1f166af249bf941/hypothesis-6.167.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1434bbd25d05aaf75c4e6829b4cad8e9931b690f840d92b747b2e6e5af575922", size = 778617, upload-time = "2026-08-30T19:50:31.939Z" }, + { url = "https://files.pythonhosted.org/packages/13/2a/60c18a493215c22c9cfcb4574b381497bd1971eed9fb5f9831b26e73cffb/hypothesis-6.167.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af3c09428e553b1dd2f9abbc4738377c58bf6d74cb0b8b528cc1dde3a9cdfbe8", size = 1116743, upload-time = "2026-08-30T19:50:49.293Z" }, + { url = "https://files.pythonhosted.org/packages/12/1f/b6796f11d6502e0b1764aec99f2792ca60382b6a45e26bbe163dc5757980/hypothesis-6.167.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04f807b85d425a7005e8a24498ca832bf5590f0d306737471d94c842569cecef", size = 1162718, upload-time = "2026-08-30T19:52:47.544Z" }, + { url = "https://files.pythonhosted.org/packages/a8/6c/e3cf35474b799e299fa08980b6756f500d730781686916ab65f87cbc0613/hypothesis-6.167.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:803c6a98ff66cee4caf03245bfd00e442a907264031b994a3a650dc6e4786f51", size = 1292417, upload-time = "2026-08-30T19:50:56.822Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4c/875803c80c373a1628f8eb62f215ad23ce7b50ed61f884d6be0838ebea4a/hypothesis-6.167.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:21a7122ddf072906083e3704fe3961ecbc49d7d30a9b51bcd525d977b3afe65e", size = 1329035, upload-time = "2026-08-30T19:51:46.659Z" }, + { url = "https://files.pythonhosted.org/packages/90/a8/a8daed3796623884471dc0ee8ed63917b1e2b979b4074bcea19a964fcd71/hypothesis-6.167.1-cp312-cp312-win_amd64.whl", hash = "sha256:a2837c60d782eb0b8a910c541264675b9d11486e186af8c82a5e2920b5fe4fe8", size = 675966, upload-time = "2026-08-30T19:51:56.58Z" }, + { url = "https://files.pythonhosted.org/packages/4b/44/dca7b211804f60c789aced2792b1e7803ccd8b70b79041cbb92788df5d19/hypothesis-6.167.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:6478d19a7887731cc2afaa1ec15f62811c9ceb6fd18e5b7563e0a18399a9528f", size = 786947, upload-time = "2026-08-30T19:51:29.165Z" }, + { url = "https://files.pythonhosted.org/packages/b9/6a/3cffa138492c9e3d5f98f4ff8b467273dc87af6ca3c18084272d106bde10/hypothesis-6.167.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8f13167a4b81c93e7e051d1f02790814a6495fb79cacf3fb89560a796a2f7d00", size = 778584, upload-time = "2026-08-30T19:52:25.091Z" }, + { url = "https://files.pythonhosted.org/packages/7a/7d/e8039791aaca3b21557bc520a71cdb88751892f66fd1a0a459b59872e463/hypothesis-6.167.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ef7dd225f7df7d74d1c5a905592cd8b4cd348e6be639b189a43def8b0b5dd79", size = 1116749, upload-time = "2026-08-30T19:52:38.178Z" }, + { url = "https://files.pythonhosted.org/packages/ee/b8/f9b8d93bd6178870f0daa868ca99915f6d9df1f99dc7291e9ce2743a6dc5/hypothesis-6.167.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d7585429f2263d3ceeb3474bae3871024630a7a598e71eeb4b0dcf03e291623", size = 1162599, upload-time = "2026-08-30T19:52:11.72Z" }, + { url = "https://files.pythonhosted.org/packages/a1/0d/53d419094e6f8a7e7377c09de15ac23f842ab698ff07241f7b73e19bd559/hypothesis-6.167.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fb9194f450417cf35f66b6c72737cc6b8f21f20567ba4d39822c64f0d1075784", size = 1292230, upload-time = "2026-08-30T19:52:49.732Z" }, + { url = "https://files.pythonhosted.org/packages/5c/df/cf4c482323ae4f06b5326b5bdd89cf17d8232fdb3186c9913e0b19a5fa58/hypothesis-6.167.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c63a0d292a5dde3c0fe999892e76d8375a003ca40c0c00d763f3360f91be5b96", size = 1328899, upload-time = "2026-08-30T19:51:05.366Z" }, + { url = "https://files.pythonhosted.org/packages/49/05/780c4b0396491d294fda69a541cb1dedb37fb9eb2e3a696e85fe19064c40/hypothesis-6.167.1-cp313-cp313-win_amd64.whl", hash = "sha256:ff07f98a0b230632bb2836b5dad3e94d85c114ae155a316afd251c58760958ae", size = 675927, upload-time = "2026-08-30T19:50:58.472Z" }, + { url = "https://files.pythonhosted.org/packages/6a/f1/1e602f090dcb7e38655f1f7909482742891332275fc01f241e255cdfa514/hypothesis-6.167.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:fcfc2a78fc1025644f889a74684b3201f4652ce8e6694c2a01af0f100d0348cf", size = 787054, upload-time = "2026-08-30T19:50:40.88Z" }, + { url = "https://files.pythonhosted.org/packages/52/57/cfa930719c7af5a33627abba826a3fa2efa61a5f23e38d4111eace5dfe53/hypothesis-6.167.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:769bdd9aa0af08c063327730ab6dc18b7a23837a2912f2aeaab3912f11a7e3ad", size = 778721, upload-time = "2026-08-30T19:50:36.503Z" }, + { url = "https://files.pythonhosted.org/packages/01/37/4c4bc3d319eac85bfe17515c9786bf49e57181ca8757886110d2cfb13d10/hypothesis-6.167.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d75d44bdead6679b6ee9a7c90d10207db865ca0c77c5212103b5ff421379f99e", size = 1116972, upload-time = "2026-08-30T19:51:52.472Z" }, + { url = "https://files.pythonhosted.org/packages/33/45/0d61ceef2739e7b96ea1faa0f3d5aa5917c8156797993bf3acbadfcd7f0a/hypothesis-6.167.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:742be00d7bb53d10634e6435e5b98f51fcdbe7ed377d473ab7387d9499c87169", size = 1162776, upload-time = "2026-08-30T19:51:09.084Z" }, + { url = "https://files.pythonhosted.org/packages/4a/cf/756666ce2262e90fd61fec41a95548cceab94b0669381d8f0387cd89af93/hypothesis-6.167.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b6fcdc8d03b37a902262be13112d113bb4ac87edf3b08afb47f3d1210deb038a", size = 1292748, upload-time = "2026-08-30T19:51:17.22Z" }, + { url = "https://files.pythonhosted.org/packages/e8/b4/87eb3c695d6c37fb44f4d49f9faa2033af496e24965658942a1706e22620/hypothesis-6.167.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56e7841514276c308c2bb4d033cf01860d0fc8c76e2b79ce748a9f123eaf83b", size = 1329101, upload-time = "2026-08-30T19:51:36.313Z" }, + { url = "https://files.pythonhosted.org/packages/a7/3b/87faa4a86533eaaa19037741fb9cdde8647f7ffdf8fd4279828ac9d81f8b/hypothesis-6.167.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:bbf4f0cad201d0b8e821e82ad828b2aec99ce6d9967779eecb2cad4d4a93debd", size = 618079, upload-time = "2026-08-30T19:52:43.226Z" }, + { url = "https://files.pythonhosted.org/packages/e0/46/96b7ac9605887447d267b4b3a9ecf61c6caaabf39eef667173b0cc9222b3/hypothesis-6.167.1-cp314-cp314-win_amd64.whl", hash = "sha256:3e04f6001299708b6fd4512267b189c0b029ef1e34500deb4e4c9639023598d7", size = 675812, upload-time = "2026-08-30T19:52:52.298Z" }, + { url = "https://files.pythonhosted.org/packages/6e/f6/0337a91c50ce4be323c3d6aa852fcf08199ffbb1072da09fbe6d602f4dfe/hypothesis-6.167.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:47c99256df28555ecc2aed0e22ca17cd61c63c8c44207a07b4e402cc49661fae", size = 785525, upload-time = "2026-08-30T19:51:03.637Z" }, + { url = "https://files.pythonhosted.org/packages/5a/08/9bb52de855169d31888c7033ee2f94b94138fde021c1af9dbc7ba5e83cd5/hypothesis-6.167.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5d6614e88fd267bbd870e3ec02f8a5387897d2d573626a02f5ac06d81533afa6", size = 777142, upload-time = "2026-08-30T19:52:18.472Z" }, + { url = "https://files.pythonhosted.org/packages/85/79/f1a7e088e13a641357abb9b43d75c116c2a0902711b1a25a203864b96c9b/hypothesis-6.167.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27829aa89fe2e47c5c8d13b3ce31e0f53a01a98f76a4f99cfa3369ab35362f33", size = 1115311, upload-time = "2026-08-30T19:52:40.975Z" }, + { url = "https://files.pythonhosted.org/packages/e0/38/e28b1fc20bd3d67d43cf1aab7a15daa2a24ec01d17a153e82fcf38c882f3/hypothesis-6.167.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e936777d92ae27393b4a941839bbb43c1f339b5a0e394c7f3730454cdf091b3a", size = 1161238, upload-time = "2026-08-30T19:52:20.501Z" }, + { url = "https://files.pythonhosted.org/packages/5a/de/9b4fc7992166299e0fc5c13c8766919ae57d0fb9eed5319b7a3bad4f2f17/hypothesis-6.167.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:971ce0d8a367a37c4690b83a2e7f6ef832fa3543357eb6da0da27ba078e5088a", size = 1290974, upload-time = "2026-08-30T19:51:15.673Z" }, + { url = "https://files.pythonhosted.org/packages/cc/79/ca086eea02588212ab796ee4bd7fe6ed514e10d1a99967e478691608e8d9/hypothesis-6.167.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:af84ce2416be2a65bc0ea18e64d2dbb9796b7692593b5b2064d60ea1d52ec1e2", size = 1327969, upload-time = "2026-08-30T19:52:35.846Z" }, + { url = "https://files.pythonhosted.org/packages/26/79/1875380fa30e8411553e76b3e9695aca0845f9f265523f20f879bdea2b82/hypothesis-6.167.1-cp314-cp314t-win_amd64.whl", hash = "sha256:3b596efec5bd714588e3bb269544d993c5258c979f3a26f51fadf62c215d0e68", size = 675735, upload-time = "2026-08-30T19:52:22.821Z" }, + { url = "https://files.pythonhosted.org/packages/b0/45/59abecd75e52b9dfb5b3eb991276f54954c44917a1c83d148cfb3580bd39/hypothesis-6.167.1-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:c25c556d51d55d94988dc0a2c716d471ff19cf9632cd33f5e6db2914d802428a", size = 785097, upload-time = "2026-08-30T19:51:12.528Z" }, + { url = "https://files.pythonhosted.org/packages/01/7c/e6d978dc9564ba70352da60c00f55f6ad7d66d99ecbf336a228978206cb4/hypothesis-6.167.1-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:b57e950f9d5c93ca335bc612e8fa8fb49abb187c3fc9d5e7d9966d52eb27d747", size = 776798, upload-time = "2026-08-30T19:50:47.247Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a4/f8ecedcf96790aab69d750afe3fcbf503229d0bb4e0c32be655385a4fc8c/hypothesis-6.167.1-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0807ae8d399162827fc1c396ab4c697a41921c2a2baacfada439771e9dc2b867", size = 1115116, upload-time = "2026-08-30T19:52:16.029Z" }, + { url = "https://files.pythonhosted.org/packages/7e/97/8bca7c262ac4fcb1ee684c04e4ba75f26541d3a30417e3743d19912d257e/hypothesis-6.167.1-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:546fef39c7aadba74bf3e592585694a71340d1775e9b3274bb3f94106dbde4b7", size = 1137812, upload-time = "2026-08-30T19:51:25.507Z" }, + { url = "https://files.pythonhosted.org/packages/d6/07/9cb4a7fc2aa2b2ad063b446c378f0d7acfa5303e84afd1b1374ba23fd6f3/hypothesis-6.167.1-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a17a5618b6a5b84f17c8acb3bce37122647cf7a3e48b660a39d68a773bd627dd", size = 1140384, upload-time = "2026-08-30T19:52:05.264Z" }, + { url = "https://files.pythonhosted.org/packages/1c/18/813bf7efa18f11ce938a0da22a7518a54db46d4a181ebf4cb0a8061c263f/hypothesis-6.167.1-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ff5ad833480c1e34ae902cb52fc00802b08ac6c87bd22d7e5f04fd925869608", size = 1160569, upload-time = "2026-08-30T19:51:40.099Z" }, + { url = "https://files.pythonhosted.org/packages/52/1d/6658d9294ed33bb17da4acf06fe63b010b205ea1861f6921c38060793255/hypothesis-6.167.1-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:630eb37df80b5bc4ec6f391b13caaecc06942ff5da3aadebacf85f53bbc55757", size = 1120605, upload-time = "2026-08-30T19:53:01.848Z" }, + { url = "https://files.pythonhosted.org/packages/a6/81/a75821c0e879223a2635b8eded84ae874cb6c711b23e9930668008d0b13f/hypothesis-6.167.1-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bc4e65f7c43b187f7a40964706b5ded1073e0c1839e9fb5e041d7ed973bb65fe", size = 1149479, upload-time = "2026-08-30T19:51:30.972Z" }, + { url = "https://files.pythonhosted.org/packages/4e/f2/c8c3faf4ec796d6dbf36b84662806696434aef38616e5a65b46188c04262/hypothesis-6.167.1-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:e849f518cbc4e76ab15f2f1473c60dd3103da8d32399187325ceb84309105976", size = 1290423, upload-time = "2026-08-30T19:50:51.114Z" }, + { url = "https://files.pythonhosted.org/packages/a4/e3/e0903abe7e8634cedb5414931452e82daacdd3b8b46d6348bbefcaa45f2a/hypothesis-6.167.1-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:5c5a26d4d3dca0c84e01bde41df4cabaa5a373c7393f9eef372d19fe93b07ccd", size = 1415749, upload-time = "2026-08-30T19:51:48.456Z" }, + { url = "https://files.pythonhosted.org/packages/a6/9b/03f09c1ecac1dfb0f4cd7fcc6dc50d9c6ea8067b295a728e242650bafe32/hypothesis-6.167.1-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:4819adbc5911648f6bfaeb574f276add184b4b49f54731dbde46fd71256bb157", size = 1272086, upload-time = "2026-08-30T19:52:29.368Z" }, + { url = "https://files.pythonhosted.org/packages/7c/7c/7a63bfc0bfbaf000f71352c4faac72ff611376330a2ce2e9a1bf4668848a/hypothesis-6.167.1-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:3ad7206de9c398c8da5745b69b5ba2ef45100082eeb174656490bc4f262b112c", size = 1291553, upload-time = "2026-08-30T19:52:07.545Z" }, + { url = "https://files.pythonhosted.org/packages/e8/5c/8065bdab53bc81743ca68fc76ca53fc7531a5b3f01c0de4ba40467955d6a/hypothesis-6.167.1-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:96d5e8017a9508f06c8a61a6130cb0d0b4810847ed5c76923cb5cfb9952b31af", size = 1327734, upload-time = "2026-08-30T19:51:42.306Z" }, + { url = "https://files.pythonhosted.org/packages/0d/a0/5c15d480aea3a8e6e5c17c7cb1170171707ac643ffd319473bc194743ad8/hypothesis-6.167.1-cp315-abi3.abi3t-win32.whl", hash = "sha256:a4e4de36a397cba49d949d89cbc26135977c15f9d797caa95317962ceb5b5674", size = 669115, upload-time = "2026-08-30T19:51:14.192Z" }, + { url = "https://files.pythonhosted.org/packages/1b/36/4cf494bc96384189fedb7d3f272580315f2284a9f8a7f6a59796612eb76d/hypothesis-6.167.1-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:f6fe9c40ab14def363d9e7ab22863fa31652bd5e08f8495b34ff7bd0062b3f8d", size = 675438, upload-time = "2026-08-30T19:51:06.881Z" }, + { url = "https://files.pythonhosted.org/packages/b0/7f/db1a37e5f45be32c0e64f9ed1268eba56aeedcb2ef20d195fa60c6610347/hypothesis-6.167.1-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:627ce3bd166799a6c0ddcf1351049be5b9a772d5bce436216d42b41a935f42c0", size = 673123, upload-time = "2026-08-30T19:52:13.991Z" }, + { url = "https://files.pythonhosted.org/packages/53/bc/a77ee57eb8fb13f2b5bdfb4a1ea3f32713c50420f0208e84fbe590fad1ad/hypothesis-6.167.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:436027c9a00eb11a2ca3d608147ca2d0d623b4f02878c56a50fc3f3f58c2b41b", size = 786862, upload-time = "2026-08-30T19:51:38.287Z" }, + { url = "https://files.pythonhosted.org/packages/f8/3f/cc9c9120fad719e683914b9204b38f1a30721bc06344f465fc36427cc45e/hypothesis-6.167.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:35e90c121b1518d7428a45e6b0d5c6d06e0ed9eaa567f1106e1f09dae006d6da", size = 782711, upload-time = "2026-08-30T19:50:28.733Z" }, + { url = "https://files.pythonhosted.org/packages/98/a6/a48824ba4ad1257904bde4654099851febf0b4c3f018c8174b33d4ba0308/hypothesis-6.167.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c903b4f1c8531736fc7e8e537f47ef509756b731a32a5e5e7014e5291343acb", size = 1118685, upload-time = "2026-08-30T19:52:01.045Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f3/c6bcfb38c4b4cd22494902f5368b5815425f03eeb5159741c7d910a69af5/hypothesis-6.167.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27ca252991fdbe2ff5c611a1cc4d972d4e009eb45292c7802faa2190f995dc50", size = 1165389, upload-time = "2026-08-30T19:50:39.44Z" }, + { url = "https://files.pythonhosted.org/packages/3b/f6/d1d19d115a4c0aa35c87b9e5d570b0a7c9f42f816087849cf29c58664426/hypothesis-6.167.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6c91f6f2f15b8bc6e474b824f931247c39711231e7b1b2f68277726e0ae1c728", size = 679392, upload-time = "2026-08-30T19:51:00.264Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py", version = "0.30.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "rpds-py", version = "2026.6.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" }, + { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" }, + { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" }, + { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" }, + { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" }, + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py", version = "0.30.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "rpds-py", version = "2026.6.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "rpds-py" +version = "0.30.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/0c/0c411a0ec64ccb6d104dcabe0e713e05e153a9a2c3c2bd2b32ce412166fe/rpds_py-0.30.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:679ae98e00c0e8d68a7fda324e16b90fd5260945b45d3b824c892cec9eea3288", size = 370490, upload-time = "2025-11-30T20:21:33.256Z" }, + { url = "https://files.pythonhosted.org/packages/19/6a/4ba3d0fb7297ebae71171822554abe48d7cab29c28b8f9f2c04b79988c05/rpds_py-0.30.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4cc2206b76b4f576934f0ed374b10d7ca5f457858b157ca52064bdfc26b9fc00", size = 359751, upload-time = "2025-11-30T20:21:34.591Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7c/e4933565ef7f7a0818985d87c15d9d273f1a649afa6a52ea35ad011195ea/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:389a2d49eded1896c3d48b0136ead37c48e221b391c052fba3f4055c367f60a6", size = 389696, upload-time = "2025-11-30T20:21:36.122Z" }, + { url = "https://files.pythonhosted.org/packages/5e/01/6271a2511ad0815f00f7ed4390cf2567bec1d4b1da39e2c27a41e6e3b4de/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:32c8528634e1bf7121f3de08fa85b138f4e0dc47657866630611b03967f041d7", size = 403136, upload-time = "2025-11-30T20:21:37.728Z" }, + { url = "https://files.pythonhosted.org/packages/55/64/c857eb7cd7541e9b4eee9d49c196e833128a55b89a9850a9c9ac33ccf897/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f207f69853edd6f6700b86efb84999651baf3789e78a466431df1331608e5324", size = 524699, upload-time = "2025-11-30T20:21:38.92Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ed/94816543404078af9ab26159c44f9e98e20fe47e2126d5d32c9d9948d10a/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:67b02ec25ba7a9e8fa74c63b6ca44cf5707f2fbfadae3ee8e7494297d56aa9df", size = 412022, upload-time = "2025-11-30T20:21:40.407Z" }, + { url = "https://files.pythonhosted.org/packages/61/b5/707f6cf0066a6412aacc11d17920ea2e19e5b2f04081c64526eb35b5c6e7/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0e95f6819a19965ff420f65578bacb0b00f251fefe2c8b23347c37174271f3", size = 390522, upload-time = "2025-11-30T20:21:42.17Z" }, + { url = "https://files.pythonhosted.org/packages/13/4e/57a85fda37a229ff4226f8cbcf09f2a455d1ed20e802ce5b2b4a7f5ed053/rpds_py-0.30.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:a452763cc5198f2f98898eb98f7569649fe5da666c2dc6b5ddb10fde5a574221", size = 404579, upload-time = "2025-11-30T20:21:43.769Z" }, + { url = "https://files.pythonhosted.org/packages/f9/da/c9339293513ec680a721e0e16bf2bac3db6e5d7e922488de471308349bba/rpds_py-0.30.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e0b65193a413ccc930671c55153a03ee57cecb49e6227204b04fae512eb657a7", size = 421305, upload-time = "2025-11-30T20:21:44.994Z" }, + { url = "https://files.pythonhosted.org/packages/f9/be/522cb84751114f4ad9d822ff5a1aa3c98006341895d5f084779b99596e5c/rpds_py-0.30.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:858738e9c32147f78b3ac24dc0edb6610000e56dc0f700fd5f651d0a0f0eb9ff", size = 572503, upload-time = "2025-11-30T20:21:46.91Z" }, + { url = "https://files.pythonhosted.org/packages/a2/9b/de879f7e7ceddc973ea6e4629e9b380213a6938a249e94b0cdbcc325bb66/rpds_py-0.30.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:da279aa314f00acbb803da1e76fa18666778e8a8f83484fba94526da5de2cba7", size = 598322, upload-time = "2025-11-30T20:21:48.709Z" }, + { url = "https://files.pythonhosted.org/packages/48/ac/f01fc22efec3f37d8a914fc1b2fb9bcafd56a299edbe96406f3053edea5a/rpds_py-0.30.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7c64d38fb49b6cdeda16ab49e35fe0da2e1e9b34bc38bd78386530f218b37139", size = 560792, upload-time = "2025-11-30T20:21:50.024Z" }, + { url = "https://files.pythonhosted.org/packages/e2/da/4e2b19d0f131f35b6146425f846563d0ce036763e38913d917187307a671/rpds_py-0.30.0-cp310-cp310-win32.whl", hash = "sha256:6de2a32a1665b93233cde140ff8b3467bdb9e2af2b91079f0333a0974d12d464", size = 221901, upload-time = "2025-11-30T20:21:51.32Z" }, + { url = "https://files.pythonhosted.org/packages/96/cb/156d7a5cf4f78a7cc571465d8aec7a3c447c94f6749c5123f08438bcf7bc/rpds_py-0.30.0-cp310-cp310-win_amd64.whl", hash = "sha256:1726859cd0de969f88dc8673bdd954185b9104e05806be64bcd87badbe313169", size = 235823, upload-time = "2025-11-30T20:21:52.505Z" }, + { url = "https://files.pythonhosted.org/packages/4d/6e/f964e88b3d2abee2a82c1ac8366da848fce1c6d834dc2132c3fda3970290/rpds_py-0.30.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a2bffea6a4ca9f01b3f8e548302470306689684e61602aa3d141e34da06cf425", size = 370157, upload-time = "2025-11-30T20:21:53.789Z" }, + { url = "https://files.pythonhosted.org/packages/94/ba/24e5ebb7c1c82e74c4e4f33b2112a5573ddc703915b13a073737b59b86e0/rpds_py-0.30.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dc4f992dfe1e2bc3ebc7444f6c7051b4bc13cd8e33e43511e8ffd13bf407010d", size = 359676, upload-time = "2025-11-30T20:21:55.475Z" }, + { url = "https://files.pythonhosted.org/packages/84/86/04dbba1b087227747d64d80c3b74df946b986c57af0a9f0c98726d4d7a3b/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:422c3cb9856d80b09d30d2eb255d0754b23e090034e1deb4083f8004bd0761e4", size = 389938, upload-time = "2025-11-30T20:21:57.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/bb/1463f0b1722b7f45431bdd468301991d1328b16cffe0b1c2918eba2c4eee/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07ae8a593e1c3c6b82ca3292efbe73c30b61332fd612e05abee07c79359f292f", size = 402932, upload-time = "2025-11-30T20:21:58.47Z" }, + { url = "https://files.pythonhosted.org/packages/99/ee/2520700a5c1f2d76631f948b0736cdf9b0acb25abd0ca8e889b5c62ac2e3/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12f90dd7557b6bd57f40abe7747e81e0c0b119bef015ea7726e69fe550e394a4", size = 525830, upload-time = "2025-11-30T20:21:59.699Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ad/bd0331f740f5705cc555a5e17fdf334671262160270962e69a2bdef3bf76/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99b47d6ad9a6da00bec6aabe5a6279ecd3c06a329d4aa4771034a21e335c3a97", size = 412033, upload-time = "2025-11-30T20:22:00.991Z" }, + { url = "https://files.pythonhosted.org/packages/f8/1e/372195d326549bb51f0ba0f2ecb9874579906b97e08880e7a65c3bef1a99/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33f559f3104504506a44bb666b93a33f5d33133765b0c216a5bf2f1e1503af89", size = 390828, upload-time = "2025-11-30T20:22:02.723Z" }, + { url = "https://files.pythonhosted.org/packages/ab/2b/d88bb33294e3e0c76bc8f351a3721212713629ffca1700fa94979cb3eae8/rpds_py-0.30.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:946fe926af6e44f3697abbc305ea168c2c31d3e3ef1058cf68f379bf0335a78d", size = 404683, upload-time = "2025-11-30T20:22:04.367Z" }, + { url = "https://files.pythonhosted.org/packages/50/32/c759a8d42bcb5289c1fac697cd92f6fe01a018dd937e62ae77e0e7f15702/rpds_py-0.30.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:495aeca4b93d465efde585977365187149e75383ad2684f81519f504f5c13038", size = 421583, upload-time = "2025-11-30T20:22:05.814Z" }, + { url = "https://files.pythonhosted.org/packages/2b/81/e729761dbd55ddf5d84ec4ff1f47857f4374b0f19bdabfcf929164da3e24/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9a0ca5da0386dee0655b4ccdf46119df60e0f10da268d04fe7cc87886872ba7", size = 572496, upload-time = "2025-11-30T20:22:07.713Z" }, + { url = "https://files.pythonhosted.org/packages/14/f6/69066a924c3557c9c30baa6ec3a0aa07526305684c6f86c696b08860726c/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8d6d1cc13664ec13c1b84241204ff3b12f9bb82464b8ad6e7a5d3486975c2eed", size = 598669, upload-time = "2025-11-30T20:22:09.312Z" }, + { url = "https://files.pythonhosted.org/packages/5f/48/905896b1eb8a05630d20333d1d8ffd162394127b74ce0b0784ae04498d32/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3896fa1be39912cf0757753826bc8bdc8ca331a28a7c4ae46b7a21280b06bb85", size = 561011, upload-time = "2025-11-30T20:22:11.309Z" }, + { url = "https://files.pythonhosted.org/packages/22/16/cd3027c7e279d22e5eb431dd3c0fbc677bed58797fe7581e148f3f68818b/rpds_py-0.30.0-cp311-cp311-win32.whl", hash = "sha256:55f66022632205940f1827effeff17c4fa7ae1953d2b74a8581baaefb7d16f8c", size = 221406, upload-time = "2025-11-30T20:22:13.101Z" }, + { url = "https://files.pythonhosted.org/packages/fa/5b/e7b7aa136f28462b344e652ee010d4de26ee9fd16f1bfd5811f5153ccf89/rpds_py-0.30.0-cp311-cp311-win_amd64.whl", hash = "sha256:a51033ff701fca756439d641c0ad09a41d9242fa69121c7d8769604a0a629825", size = 236024, upload-time = "2025-11-30T20:22:14.853Z" }, + { url = "https://files.pythonhosted.org/packages/14/a6/364bba985e4c13658edb156640608f2c9e1d3ea3c81b27aa9d889fff0e31/rpds_py-0.30.0-cp311-cp311-win_arm64.whl", hash = "sha256:47b0ef6231c58f506ef0b74d44e330405caa8428e770fec25329ed2cb971a229", size = 229069, upload-time = "2025-11-30T20:22:16.577Z" }, + { url = "https://files.pythonhosted.org/packages/03/e7/98a2f4ac921d82f33e03f3835f5bf3a4a40aa1bfdc57975e74a97b2b4bdd/rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad", size = 375086, upload-time = "2025-11-30T20:22:17.93Z" }, + { url = "https://files.pythonhosted.org/packages/4d/a1/bca7fd3d452b272e13335db8d6b0b3ecde0f90ad6f16f3328c6fb150c889/rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05", size = 359053, upload-time = "2025-11-30T20:22:19.297Z" }, + { url = "https://files.pythonhosted.org/packages/65/1c/ae157e83a6357eceff62ba7e52113e3ec4834a84cfe07fa4b0757a7d105f/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28", size = 390763, upload-time = "2025-11-30T20:22:21.661Z" }, + { url = "https://files.pythonhosted.org/packages/d4/36/eb2eb8515e2ad24c0bd43c3ee9cd74c33f7ca6430755ccdb240fd3144c44/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd", size = 408951, upload-time = "2025-11-30T20:22:23.408Z" }, + { url = "https://files.pythonhosted.org/packages/d6/65/ad8dc1784a331fabbd740ef6f71ce2198c7ed0890dab595adb9ea2d775a1/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f", size = 514622, upload-time = "2025-11-30T20:22:25.16Z" }, + { url = "https://files.pythonhosted.org/packages/63/8e/0cfa7ae158e15e143fe03993b5bcd743a59f541f5952e1546b1ac1b5fd45/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1", size = 414492, upload-time = "2025-11-30T20:22:26.505Z" }, + { url = "https://files.pythonhosted.org/packages/60/1b/6f8f29f3f995c7ffdde46a626ddccd7c63aefc0efae881dc13b6e5d5bb16/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23", size = 394080, upload-time = "2025-11-30T20:22:27.934Z" }, + { url = "https://files.pythonhosted.org/packages/6d/d5/a266341051a7a3ca2f4b750a3aa4abc986378431fc2da508c5034d081b70/rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6", size = 408680, upload-time = "2025-11-30T20:22:29.341Z" }, + { url = "https://files.pythonhosted.org/packages/10/3b/71b725851df9ab7a7a4e33cf36d241933da66040d195a84781f49c50490c/rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51", size = 423589, upload-time = "2025-11-30T20:22:31.469Z" }, + { url = "https://files.pythonhosted.org/packages/00/2b/e59e58c544dc9bd8bd8384ecdb8ea91f6727f0e37a7131baeff8d6f51661/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5", size = 573289, upload-time = "2025-11-30T20:22:32.997Z" }, + { url = "https://files.pythonhosted.org/packages/da/3e/a18e6f5b460893172a7d6a680e86d3b6bc87a54c1f0b03446a3c8c7b588f/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e", size = 599737, upload-time = "2025-11-30T20:22:34.419Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e2/714694e4b87b85a18e2c243614974413c60aa107fd815b8cbc42b873d1d7/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394", size = 563120, upload-time = "2025-11-30T20:22:35.903Z" }, + { url = "https://files.pythonhosted.org/packages/6f/ab/d5d5e3bcedb0a77f4f613706b750e50a5a3ba1c15ccd3665ecc636c968fd/rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf", size = 223782, upload-time = "2025-11-30T20:22:37.271Z" }, + { url = "https://files.pythonhosted.org/packages/39/3b/f786af9957306fdc38a74cef405b7b93180f481fb48453a114bb6465744a/rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b", size = 240463, upload-time = "2025-11-30T20:22:39.021Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d2/b91dc748126c1559042cfe41990deb92c4ee3e2b415f6b5234969ffaf0cc/rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e", size = 230868, upload-time = "2025-11-30T20:22:40.493Z" }, + { url = "https://files.pythonhosted.org/packages/ed/dc/d61221eb88ff410de3c49143407f6f3147acf2538c86f2ab7ce65ae7d5f9/rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2", size = 374887, upload-time = "2025-11-30T20:22:41.812Z" }, + { url = "https://files.pythonhosted.org/packages/fd/32/55fb50ae104061dbc564ef15cc43c013dc4a9f4527a1f4d99baddf56fe5f/rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8", size = 358904, upload-time = "2025-11-30T20:22:43.479Z" }, + { url = "https://files.pythonhosted.org/packages/58/70/faed8186300e3b9bdd138d0273109784eea2396c68458ed580f885dfe7ad/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4", size = 389945, upload-time = "2025-11-30T20:22:44.819Z" }, + { url = "https://files.pythonhosted.org/packages/bd/a8/073cac3ed2c6387df38f71296d002ab43496a96b92c823e76f46b8af0543/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136", size = 407783, upload-time = "2025-11-30T20:22:46.103Z" }, + { url = "https://files.pythonhosted.org/packages/77/57/5999eb8c58671f1c11eba084115e77a8899d6e694d2a18f69f0ba471ec8b/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7", size = 515021, upload-time = "2025-11-30T20:22:47.458Z" }, + { url = "https://files.pythonhosted.org/packages/e0/af/5ab4833eadc36c0a8ed2bc5c0de0493c04f6c06de223170bd0798ff98ced/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2", size = 414589, upload-time = "2025-11-30T20:22:48.872Z" }, + { url = "https://files.pythonhosted.org/packages/b7/de/f7192e12b21b9e9a68a6d0f249b4af3fdcdff8418be0767a627564afa1f1/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6", size = 394025, upload-time = "2025-11-30T20:22:50.196Z" }, + { url = "https://files.pythonhosted.org/packages/91/c4/fc70cd0249496493500e7cc2de87504f5aa6509de1e88623431fec76d4b6/rpds_py-0.30.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e", size = 408895, upload-time = "2025-11-30T20:22:51.87Z" }, + { url = "https://files.pythonhosted.org/packages/58/95/d9275b05ab96556fefff73a385813eb66032e4c99f411d0795372d9abcea/rpds_py-0.30.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d", size = 422799, upload-time = "2025-11-30T20:22:53.341Z" }, + { url = "https://files.pythonhosted.org/packages/06/c1/3088fc04b6624eb12a57eb814f0d4997a44b0d208d6cace713033ff1a6ba/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7", size = 572731, upload-time = "2025-11-30T20:22:54.778Z" }, + { url = "https://files.pythonhosted.org/packages/d8/42/c612a833183b39774e8ac8fecae81263a68b9583ee343db33ab571a7ce55/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31", size = 599027, upload-time = "2025-11-30T20:22:56.212Z" }, + { url = "https://files.pythonhosted.org/packages/5f/60/525a50f45b01d70005403ae0e25f43c0384369ad24ffe46e8d9068b50086/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95", size = 563020, upload-time = "2025-11-30T20:22:58.2Z" }, + { url = "https://files.pythonhosted.org/packages/0b/5d/47c4655e9bcd5ca907148535c10e7d489044243cc9941c16ed7cd53be91d/rpds_py-0.30.0-cp313-cp313-win32.whl", hash = "sha256:b40fb160a2db369a194cb27943582b38f79fc4887291417685f3ad693c5a1d5d", size = 223139, upload-time = "2025-11-30T20:23:00.209Z" }, + { url = "https://files.pythonhosted.org/packages/f2/e1/485132437d20aa4d3e1d8b3fb5a5e65aa8139f1e097080c2a8443201742c/rpds_py-0.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15", size = 240224, upload-time = "2025-11-30T20:23:02.008Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/ffd128ed1146a153d928617b0ef673960130be0009c77d8fbf0abe306713/rpds_py-0.30.0-cp313-cp313-win_arm64.whl", hash = "sha256:d96c2086587c7c30d44f31f42eae4eac89b60dabbac18c7669be3700f13c3ce1", size = 230645, upload-time = "2025-11-30T20:23:03.43Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1b/b10de890a0def2a319a2626334a7f0ae388215eb60914dbac8a3bae54435/rpds_py-0.30.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a", size = 364443, upload-time = "2025-11-30T20:23:04.878Z" }, + { url = "https://files.pythonhosted.org/packages/0d/bf/27e39f5971dc4f305a4fb9c672ca06f290f7c4e261c568f3dea16a410d47/rpds_py-0.30.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e", size = 353375, upload-time = "2025-11-30T20:23:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/40/58/442ada3bba6e8e6615fc00483135c14a7538d2ffac30e2d933ccf6852232/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000", size = 383850, upload-time = "2025-11-30T20:23:07.825Z" }, + { url = "https://files.pythonhosted.org/packages/14/14/f59b0127409a33c6ef6f5c1ebd5ad8e32d7861c9c7adfa9a624fc3889f6c/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db", size = 392812, upload-time = "2025-11-30T20:23:09.228Z" }, + { url = "https://files.pythonhosted.org/packages/b3/66/e0be3e162ac299b3a22527e8913767d869e6cc75c46bd844aa43fb81ab62/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2", size = 517841, upload-time = "2025-11-30T20:23:11.186Z" }, + { url = "https://files.pythonhosted.org/packages/3d/55/fa3b9cf31d0c963ecf1ba777f7cf4b2a2c976795ac430d24a1f43d25a6ba/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa", size = 408149, upload-time = "2025-11-30T20:23:12.864Z" }, + { url = "https://files.pythonhosted.org/packages/60/ca/780cf3b1a32b18c0f05c441958d3758f02544f1d613abf9488cd78876378/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083", size = 383843, upload-time = "2025-11-30T20:23:14.638Z" }, + { url = "https://files.pythonhosted.org/packages/82/86/d5f2e04f2aa6247c613da0c1dd87fcd08fa17107e858193566048a1e2f0a/rpds_py-0.30.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9", size = 396507, upload-time = "2025-11-30T20:23:16.105Z" }, + { url = "https://files.pythonhosted.org/packages/4b/9a/453255d2f769fe44e07ea9785c8347edaf867f7026872e76c1ad9f7bed92/rpds_py-0.30.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0", size = 414949, upload-time = "2025-11-30T20:23:17.539Z" }, + { url = "https://files.pythonhosted.org/packages/a3/31/622a86cdc0c45d6df0e9ccb6becdba5074735e7033c20e401a6d9d0e2ca0/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94", size = 565790, upload-time = "2025-11-30T20:23:19.029Z" }, + { url = "https://files.pythonhosted.org/packages/1c/5d/15bbf0fb4a3f58a3b1c67855ec1efcc4ceaef4e86644665fff03e1b66d8d/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08", size = 590217, upload-time = "2025-11-30T20:23:20.885Z" }, + { url = "https://files.pythonhosted.org/packages/6d/61/21b8c41f68e60c8cc3b2e25644f0e3681926020f11d06ab0b78e3c6bbff1/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27", size = 555806, upload-time = "2025-11-30T20:23:22.488Z" }, + { url = "https://files.pythonhosted.org/packages/f9/39/7e067bb06c31de48de3eb200f9fc7c58982a4d3db44b07e73963e10d3be9/rpds_py-0.30.0-cp313-cp313t-win32.whl", hash = "sha256:3d4a69de7a3e50ffc214ae16d79d8fbb0922972da0356dcf4d0fdca2878559c6", size = 211341, upload-time = "2025-11-30T20:23:24.449Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4d/222ef0b46443cf4cf46764d9c630f3fe4abaa7245be9417e56e9f52b8f65/rpds_py-0.30.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f14fc5df50a716f7ece6a80b6c78bb35ea2ca47c499e422aa4463455dd96d56d", size = 225768, upload-time = "2025-11-30T20:23:25.908Z" }, + { url = "https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0", size = 362099, upload-time = "2025-11-30T20:23:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be", size = 353192, upload-time = "2025-11-30T20:23:29.151Z" }, + { url = "https://files.pythonhosted.org/packages/bf/c4/76eb0e1e72d1a9c4703c69607cec123c29028bff28ce41588792417098ac/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f", size = 384080, upload-time = "2025-11-30T20:23:30.785Z" }, + { url = "https://files.pythonhosted.org/packages/72/87/87ea665e92f3298d1b26d78814721dc39ed8d2c74b86e83348d6b48a6f31/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f", size = 394841, upload-time = "2025-11-30T20:23:32.209Z" }, + { url = "https://files.pythonhosted.org/packages/77/ad/7783a89ca0587c15dcbf139b4a8364a872a25f861bdb88ed99f9b0dec985/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87", size = 516670, upload-time = "2025-11-30T20:23:33.742Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3c/2882bdac942bd2172f3da574eab16f309ae10a3925644e969536553cb4ee/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18", size = 408005, upload-time = "2025-11-30T20:23:35.253Z" }, + { url = "https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad", size = 382112, upload-time = "2025-11-30T20:23:36.842Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8e/1da49d4a107027e5fbc64daeab96a0706361a2918da10cb41769244b805d/rpds_py-0.30.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07", size = 399049, upload-time = "2025-11-30T20:23:38.343Z" }, + { url = "https://files.pythonhosted.org/packages/df/5a/7ee239b1aa48a127570ec03becbb29c9d5a9eb092febbd1699d567cae859/rpds_py-0.30.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f", size = 415661, upload-time = "2025-11-30T20:23:40.263Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/caa143cf6b772f823bc7929a45da1fa83569ee49b11d18d0ada7f5ee6fd6/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65", size = 565606, upload-time = "2025-11-30T20:23:42.186Z" }, + { url = "https://files.pythonhosted.org/packages/64/91/ac20ba2d69303f961ad8cf55bf7dbdb4763f627291ba3d0d7d67333cced9/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f", size = 591126, upload-time = "2025-11-30T20:23:44.086Z" }, + { url = "https://files.pythonhosted.org/packages/21/20/7ff5f3c8b00c8a95f75985128c26ba44503fb35b8e0259d812766ea966c7/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53", size = 553371, upload-time = "2025-11-30T20:23:46.004Z" }, + { url = "https://files.pythonhosted.org/packages/72/c7/81dadd7b27c8ee391c132a6b192111ca58d866577ce2d9b0ca157552cce0/rpds_py-0.30.0-cp314-cp314-win32.whl", hash = "sha256:ee454b2a007d57363c2dfd5b6ca4a5d7e2c518938f8ed3b706e37e5d470801ed", size = 215298, upload-time = "2025-11-30T20:23:47.696Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl", hash = "sha256:95f0802447ac2d10bcc69f6dc28fe95fdf17940367b21d34e34c737870758950", size = 228604, upload-time = "2025-11-30T20:23:49.501Z" }, + { url = "https://files.pythonhosted.org/packages/e8/95/ab005315818cc519ad074cb7784dae60d939163108bd2b394e60dc7b5461/rpds_py-0.30.0-cp314-cp314-win_arm64.whl", hash = "sha256:613aa4771c99f03346e54c3f038e4cc574ac09a3ddfb0e8878487335e96dead6", size = 222391, upload-time = "2025-11-30T20:23:50.96Z" }, + { url = "https://files.pythonhosted.org/packages/9e/68/154fe0194d83b973cdedcdcc88947a2752411165930182ae41d983dcefa6/rpds_py-0.30.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb", size = 364868, upload-time = "2025-11-30T20:23:52.494Z" }, + { url = "https://files.pythonhosted.org/packages/83/69/8bbc8b07ec854d92a8b75668c24d2abcb1719ebf890f5604c61c9369a16f/rpds_py-0.30.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8", size = 353747, upload-time = "2025-11-30T20:23:54.036Z" }, + { url = "https://files.pythonhosted.org/packages/ab/00/ba2e50183dbd9abcce9497fa5149c62b4ff3e22d338a30d690f9af970561/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7", size = 383795, upload-time = "2025-11-30T20:23:55.556Z" }, + { url = "https://files.pythonhosted.org/packages/05/6f/86f0272b84926bcb0e4c972262f54223e8ecc556b3224d281e6598fc9268/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898", size = 393330, upload-time = "2025-11-30T20:23:57.033Z" }, + { url = "https://files.pythonhosted.org/packages/cb/e9/0e02bb2e6dc63d212641da45df2b0bf29699d01715913e0d0f017ee29438/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e", size = 518194, upload-time = "2025-11-30T20:23:58.637Z" }, + { url = "https://files.pythonhosted.org/packages/ee/ca/be7bca14cf21513bdf9c0606aba17d1f389ea2b6987035eb4f62bd923f25/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419", size = 408340, upload-time = "2025-11-30T20:24:00.2Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c7/736e00ebf39ed81d75544c0da6ef7b0998f8201b369acf842f9a90dc8fce/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551", size = 383765, upload-time = "2025-11-30T20:24:01.759Z" }, + { url = "https://files.pythonhosted.org/packages/4a/3f/da50dfde9956aaf365c4adc9533b100008ed31aea635f2b8d7b627e25b49/rpds_py-0.30.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8", size = 396834, upload-time = "2025-11-30T20:24:03.687Z" }, + { url = "https://files.pythonhosted.org/packages/4e/00/34bcc2565b6020eab2623349efbdec810676ad571995911f1abdae62a3a0/rpds_py-0.30.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5", size = 415470, upload-time = "2025-11-30T20:24:05.232Z" }, + { url = "https://files.pythonhosted.org/packages/8c/28/882e72b5b3e6f718d5453bd4d0d9cf8df36fddeb4ddbbab17869d5868616/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404", size = 565630, upload-time = "2025-11-30T20:24:06.878Z" }, + { url = "https://files.pythonhosted.org/packages/3b/97/04a65539c17692de5b85c6e293520fd01317fd878ea1995f0367d4532fb1/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856", size = 591148, upload-time = "2025-11-30T20:24:08.445Z" }, + { url = "https://files.pythonhosted.org/packages/85/70/92482ccffb96f5441aab93e26c4d66489eb599efdcf96fad90c14bbfb976/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40", size = 556030, upload-time = "2025-11-30T20:24:10.956Z" }, + { url = "https://files.pythonhosted.org/packages/20/53/7c7e784abfa500a2b6b583b147ee4bb5a2b3747a9166bab52fec4b5b5e7d/rpds_py-0.30.0-cp314-cp314t-win32.whl", hash = "sha256:dc824125c72246d924f7f796b4f63c1e9dc810c7d9e2355864b3c3a73d59ade0", size = 211570, upload-time = "2025-11-30T20:24:12.735Z" }, + { url = "https://files.pythonhosted.org/packages/d0/02/fa464cdfbe6b26e0600b62c528b72d8608f5cc49f96b8d6e38c95d60c676/rpds_py-0.30.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3", size = 226532, upload-time = "2025-11-30T20:24:14.634Z" }, + { url = "https://files.pythonhosted.org/packages/69/71/3f34339ee70521864411f8b6992e7ab13ac30d8e4e3309e07c7361767d91/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c2262bdba0ad4fc6fb5545660673925c2d2a5d9e2e0fb603aad545427be0fc58", size = 372292, upload-time = "2025-11-30T20:24:16.537Z" }, + { url = "https://files.pythonhosted.org/packages/57/09/f183df9b8f2d66720d2ef71075c59f7e1b336bec7ee4c48f0a2b06857653/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ee6af14263f25eedc3bb918a3c04245106a42dfd4f5c2285ea6f997b1fc3f89a", size = 362128, upload-time = "2025-11-30T20:24:18.086Z" }, + { url = "https://files.pythonhosted.org/packages/7a/68/5c2594e937253457342e078f0cc1ded3dd7b2ad59afdbf2d354869110a02/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3adbb8179ce342d235c31ab8ec511e66c73faa27a47e076ccc92421add53e2bb", size = 391542, upload-time = "2025-11-30T20:24:20.092Z" }, + { url = "https://files.pythonhosted.org/packages/49/5c/31ef1afd70b4b4fbdb2800249f34c57c64beb687495b10aec0365f53dfc4/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:250fa00e9543ac9b97ac258bd37367ff5256666122c2d0f2bc97577c60a1818c", size = 404004, upload-time = "2025-11-30T20:24:22.231Z" }, + { url = "https://files.pythonhosted.org/packages/e3/63/0cfbea38d05756f3440ce6534d51a491d26176ac045e2707adc99bb6e60a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9854cf4f488b3d57b9aaeb105f06d78e5529d3145b1e4a41750167e8c213c6d3", size = 527063, upload-time = "2025-11-30T20:24:24.302Z" }, + { url = "https://files.pythonhosted.org/packages/42/e6/01e1f72a2456678b0f618fc9a1a13f882061690893c192fcad9f2926553a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:993914b8e560023bc0a8bf742c5f303551992dcb85e247b1e5c7f4a7d145bda5", size = 413099, upload-time = "2025-11-30T20:24:25.916Z" }, + { url = "https://files.pythonhosted.org/packages/b8/25/8df56677f209003dcbb180765520c544525e3ef21ea72279c98b9aa7c7fb/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58edca431fb9b29950807e301826586e5bbf24163677732429770a697ffe6738", size = 392177, upload-time = "2025-11-30T20:24:27.834Z" }, + { url = "https://files.pythonhosted.org/packages/4a/b4/0a771378c5f16f8115f796d1f437950158679bcd2a7c68cf251cfb00ed5b/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:dea5b552272a944763b34394d04577cf0f9bd013207bc32323b5a89a53cf9c2f", size = 406015, upload-time = "2025-11-30T20:24:29.457Z" }, + { url = "https://files.pythonhosted.org/packages/36/d8/456dbba0af75049dc6f63ff295a2f92766b9d521fa00de67a2bd6427d57a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ba3af48635eb83d03f6c9735dfb21785303e73d22ad03d489e88adae6eab8877", size = 423736, upload-time = "2025-11-30T20:24:31.22Z" }, + { url = "https://files.pythonhosted.org/packages/13/64/b4d76f227d5c45a7e0b796c674fd81b0a6c4fbd48dc29271857d8219571c/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:dff13836529b921e22f15cb099751209a60009731a68519630a24d61f0b1b30a", size = 573981, upload-time = "2025-11-30T20:24:32.934Z" }, + { url = "https://files.pythonhosted.org/packages/20/91/092bacadeda3edf92bf743cc96a7be133e13a39cdbfd7b5082e7ab638406/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:1b151685b23929ab7beec71080a8889d4d6d9fa9a983d213f07121205d48e2c4", size = 599782, upload-time = "2025-11-30T20:24:35.169Z" }, + { url = "https://files.pythonhosted.org/packages/d1/b7/b95708304cd49b7b6f82fdd039f1748b66ec2b21d6a45180910802f1abf1/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ac37f9f516c51e5753f27dfdef11a88330f04de2d564be3991384b2f3535d02e", size = 562191, upload-time = "2025-11-30T20:24:36.853Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.11'", +] +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/1f/a2dca5ffdbf1d475ffc4e80e4d5d720ff3a00f691795910116960ee12511/rpds_py-2026.6.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7b689145a1485c335569bd056464f3243a29af7ed3871c7be31ad624ba239bc7", size = 342174, upload-time = "2026-06-30T07:14:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/4d/dc/323d08583c0832911768663d1944f0107fcd4088704858d84b5e06d105a0/rpds_py-2026.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911", size = 345513, upload-time = "2026-06-30T07:14:56.515Z" }, + { url = "https://files.pythonhosted.org/packages/0b/2a/e31989834d18d2f26ec1d2774c5b1eb3331df4ea8ada525175294c94b48a/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acc992ab27b15f852c76755eb2ab7dce86585ddadba6fa5946e58556088845b4", size = 373783, upload-time = "2026-06-30T07:14:57.736Z" }, + { url = "https://files.pythonhosted.org/packages/87/fe/e80107ee3639585c9941c17d6a42cd65325022f656c023191fce78c324c8/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f88d653e7b3b779d71ae7454e20dcc9b6bae903f33c269db9f2be41bda3f261", size = 378316, upload-time = "2026-06-30T07:14:59.077Z" }, + { url = "https://files.pythonhosted.org/packages/22/6f/81e3adf81acfb6fa694de2a6e4e7d8863121e3e0799e0a7725e6cf5679c4/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e52655eaf81e32593abedaa4bfe33170c8cfedf3365ed9be6e11e07f148f0278", size = 499423, upload-time = "2026-06-30T07:15:00.488Z" }, + { url = "https://files.pythonhosted.org/packages/2d/9a/41263969df0ce3d9af2a96d5005a288200af1989aed3354bfceb5fc0b21f/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dfcc8b909769d19db55c7cc9541eb64b9b774b1057ffffb4f1048070475bb9f9", size = 386077, upload-time = "2026-06-30T07:15:01.911Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/7e98f468bd50346faff5b10e5297374b443bfdddacc8e9fbc65984539597/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c1255b302953c86a486b81d330d5ee1d5bd937691ce271b6be0ef0e299eaab7", size = 371315, upload-time = "2026-06-30T07:15:03.317Z" }, + { url = "https://files.pythonhosted.org/packages/99/3c/2b973b4d371906a134b03decfea7f5d9835a2c6d263454392e15b64b5b18/rpds_py-2026.6.3-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:8d2294a31386bfa251d8c8a39472beee17db67d4f1a6eabea665d35c9a4461c3", size = 383502, upload-time = "2026-06-30T07:15:04.627Z" }, + { url = "https://files.pythonhosted.org/packages/98/2a/12e2799500af0a307bca76b63361c51f9fe479223561489c29eea1f2ee41/rpds_py-2026.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8f23ead891a3b762f35ab3b04623da7056545b48aa60d59957e6789914545da", size = 402673, upload-time = "2026-06-30T07:15:05.856Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e3/21e5872d165fe08be4f229e3d5ee9d90019c0bf0e5538de60dbd54009450/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:421aba32367055614287a4292b6a17f1939c9452299f7a0209c117e990b646d4", size = 549964, upload-time = "2026-06-30T07:15:07.159Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d0/5ee0fe36844297de8123bee27bc12078c1a7416ad9f1b8a8ca18d6b0c0ac/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1e5822dfc2f0d4ab7e745eaa6d85945069329beeccef965af3f3bb26058fcab6", size = 615446, upload-time = "2026-06-30T07:15:08.531Z" }, + { url = "https://files.pythonhosted.org/packages/b1/80/1ea5873cb683f2fbe5f21b23ea1f6d179ead19f3c5b249b7eb5dca568ef2/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:83e35b57523816c8613fd0776b40cd8bb9f596b37ddd2692eb4a6bb5ab2f8c93", size = 576975, upload-time = "2026-06-30T07:15:09.97Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e1/90ef639217a5ddb15b7f4f61b1c33911fd044ad03c311bafdd2bcab85582/rpds_py-2026.6.3-cp311-cp311-win32.whl", hash = "sha256:de3eceba0b683bcbb1ab93da016d0270df1f9ae7be716b40214c5dafac6ea45a", size = 204453, upload-time = "2026-06-30T07:15:11.324Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b7/b7a1695d7af36f521fb11e80d6d3adbd744f73b921859bd3c2a2c0dc706f/rpds_py-2026.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:2c54a076ca4d370980ab57bc0e31df57bbe8d41340436a90ef8b1219a3cbb127", size = 223219, upload-time = "2026-06-30T07:15:12.476Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a2/145afacf796e4506062825941176ad9445c2dcf2b3b6a1f13d3030a15e19/rpds_py-2026.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:168c733a7112e071bb7a66460e667edfcff06c017a3c523f7a8a8e08d0140804", size = 219137, upload-time = "2026-06-30T07:15:13.631Z" }, + { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, + { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, + { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, + { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, + { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, + { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, + { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, + { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, + { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, + { url = "https://files.pythonhosted.org/packages/b4/9c/f0d19ac587fd0e4ab6b72cda355e9c5a6166b01ef7e064e437aef8eb9fef/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4cf2d36a2357e4d07bb5a4f98801265327b48256867816cfd2ceb001e9754a8f", size = 349791, upload-time = "2026-06-30T07:17:33.315Z" }, + { url = "https://files.pythonhosted.org/packages/38/c7/1d49d204c9fd2ee6c537601dc4c1ba921e03363ca576bfab94a00254ac9a/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:30c6dc199b24a5e3e81d50da0f00858c5bbdb2617a750395687f4339c5818171", size = 352842, upload-time = "2026-06-30T07:17:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/ac/e5/c0b5dc93cd0d4c06ce1f438907649514e2ea077bcd911e3154a51e96c38e/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9891e594296ab9dada6551c8e7b387b2721f27a67eecd528412e8906247a7b90", size = 382094, upload-time = "2026-06-30T07:17:36.514Z" }, + { url = "https://files.pythonhosted.org/packages/0d/54/ec0e907b4ca8d541112db352409bd15f871c9b243e0c92c9b5a46ae96f01/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5c2dc92304aa48a4a60443b548bb12f12e119d4b72f314015e67b9e1be97fca", size = 388662, upload-time = "2026-06-30T07:17:38.235Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f4/921c22a4fd0f1c1ac13a3996ffbf0aa67951e2c8ad0d1d9574938a2932e8/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:127e08c0642d880cf32ca47ec2a4a77b901f7e2dd1ad9762adb13955d72ffcc9", size = 504896, upload-time = "2026-06-30T07:17:39.689Z" }, + { url = "https://files.pythonhosted.org/packages/0b/1b/a114b972cefa1ab1cdb3c7bb177cd3844a12826c507c722d3a73516dbbaf/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8bb68f03f395eb793220b45c097bd4d8c32944393da0fad8b999efac0868fc8c", size = 391545, upload-time = "2026-06-30T07:17:41.336Z" }, + { url = "https://files.pythonhosted.org/packages/4e/98/af9b3db77d47fcbe6c8c1f36e2c2147ec70292819e99c325f871584a1c11/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3450b693fde92133e9f51060568a4c31fcca76d5e53bbd611e689ca446517e9", size = 380059, upload-time = "2026-06-30T07:17:42.857Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ba/0efd8668b97c1d26a61566386c636a7a7a09829e474fdf807caa15a2c844/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:5e8d07bddee435a2ff6f1920e18feff28d0bc4533e42f4bf6927fbd073312c41", size = 393235, upload-time = "2026-06-30T07:17:44.637Z" }, + { url = "https://files.pythonhosted.org/packages/62/90/8c139ee9690f73b0829f32647de6f40d826f8f443af6fa72644f96351aac/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3a83ae6c67b7676b9878378547ca8e93ed77a580037bcbcd1d32f739e1e6089c", size = 413008, upload-time = "2026-06-30T07:17:46.225Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/0043896fdd7828ce09a1d9a8b06433714d0960fc4ff3fc4aa72b666b764e/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2bfd04c19ddbd6640de0b51894d764bd2758854d5b75bd102d2ef10cb9c293a9", size = 558118, upload-time = "2026-06-30T07:17:47.759Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/02355f0e134f783a8f9814c4680a1bd311d37671577a5964ea838573ff37/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:ca6546b66be9dc4738b1b043d5ebd5488c66c578c5ff0fd0e8065313fe3afb76", size = 623138, upload-time = "2026-06-30T07:17:49.355Z" }, + { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] diff --git a/core-spec/spec.md b/core-spec/spec.md index 307318e4..b2c0b09c 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -449,6 +449,7 @@ The following are well-known examples: | `HONEYDEW` | Honeydew-specific attributes | | `WISDOM` | WisdomAI-specific attributes | | `SIGMA` | Sigma Computing-specific attributes | +| `THOUGHTSPOT` | ThoughtSpot-specific attributes | ### Examples