From dce3ecdfb3093780a44f70324e81c8caa60e9b72 Mon Sep 17 00:00:00 2001 From: Markus Cozowicz Date: Sat, 5 Sep 2026 05:21:23 +0200 Subject: [PATCH 1/2] core-spec: move semantic_model out of the Enumerations section in spec.yaml spec.yaml is a multi-document YAML file whose --- separators group the specification into sections. semantic_model is a document-root key, but it sits after the # Enumerations separator, so it parses as a member of that section alongside the enum definitions: 0 -> ['version'] 1 -> ['dialects', 'datatypes', 'vendor_name', 'semantic_model'] 2 -> ['datasets'] Give the root keys their own section, matching how every other section in the file is delimited: 0 -> ['version'] 1 -> ['dialects', 'datatypes', 'vendor_name'] 2 -> ['semantic_model'] 3 -> ['datasets'] Comment-only restructuring: no key, value or enum member is added, removed or changed, and nothing in the repository parses this file. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- core-spec/spec.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 4678b004..8c9c0fa4 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -59,6 +59,11 @@ datatypes: vendor_name: string +--- +# Document root +# Keys allowed on the root object of an Ossie document, alongside the `version` +# declared at the top of this file. + # Top-level semantic model definition semantic_model: # Required: Unique identifier for the semantic model From 62e5ee5bc0e4f25822056437d62499309c167680 Mon Sep 17 00:00:00 2001 From: Markus Cozowicz Date: Sat, 5 Sep 2026 05:28:48 +0200 Subject: [PATCH 2/2] Core spec: allow ai_context on the document root The document root is the only node in the core spec with no ai_context. The key appears on SemanticModel, Dataset, Field, Metric and Relationship; the root is closed with only version and semantic_model. Since semantic_model is a list, guidance that governs every model in a document has nowhere to live. A producer emitting one model per data source must copy shared instructions into each model, where they drift, and a consumer cannot tell those copies apart from genuinely model-specific instruction. Add one optional key to the document root, reusing the existing AIContext definition by reference rather than introducing a new one. Scope only: document-level context applies to every semantic model in the document. How it composes with model-level ai_context is left undefined by this version, because "model-level takes precedence on conflict" has no meaning for the string form of AIContext. An ontology document already carries ai_context on its root and refers to this specification's AIContext definition, so document-wide context is an established shape in Ossie rather than a new one. A test pins both roots to the same definition so they cannot drift. Not a breaking change: no new definitions, required is unchanged, the key is optional, and additionalProperties stays false -- unknown root keys and a wrong-typed ai_context are still rejected. examples/tpcds_semantic_model.yaml validates unchanged. examples/multi_model_ai_context.yaml shows the motivating case: two models, one per data source, each queried in its own dialect, with the publisher's conventions stated once at the root. It is wired into validation CI so it cannot rot. Refs #322 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/validation-ci.yml | 5 ++ core-spec/ossie-schema.json | 4 ++ core-spec/spec.md | 51 ++++++++++++-- core-spec/spec.yaml | 13 ++++ docs/index.md | 4 +- examples/multi_model_ai_context.yaml | 99 ++++++++++++++++++++++++++++ python/src/ossie/models.py | 1 + python/tests/test_models.py | 58 ++++++++++++++++ 8 files changed, 227 insertions(+), 8 deletions(-) create mode 100644 examples/multi_model_ai_context.yaml diff --git a/.github/workflows/validation-ci.yml b/.github/workflows/validation-ci.yml index a5c670a3..6b8c1ddc 100644 --- a/.github/workflows/validation-ci.yml +++ b/.github/workflows/validation-ci.yml @@ -26,6 +26,7 @@ on: - "validation/**" - "core-spec/**" - "examples/tpcds_semantic_model.yaml" + - "examples/multi_model_ai_context.yaml" - ".github/workflows/validation-ci.yml" pull_request: branches: ["main"] @@ -33,6 +34,7 @@ on: - "validation/**" - "core-spec/**" - "examples/tpcds_semantic_model.yaml" + - "examples/multi_model_ai_context.yaml" - ".github/workflows/validation-ci.yml" jobs: @@ -64,3 +66,6 @@ jobs: - name: Validate canonical example run: uv run validation/validate.py examples/tpcds_semantic_model.yaml + + - name: Validate document-wide ai_context example + run: uv run validation/validate.py examples/multi_model_ai_context.yaml diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index c87af723..63984d65 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -10,6 +10,10 @@ "const": "0.2.0.dev0", "description": "Apache Ossie specification version" }, + "ai_context": { + "$ref": "#/$defs/AIContext", + "description": "Document-wide context for AI tools. Applies to every semantic model in this document." + }, "semantic_model": { "type": "array", "description": "Collection of semantic model definitions", diff --git a/core-spec/spec.md b/core-spec/spec.md index 2850edfe..279896c7 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -32,12 +32,13 @@ ## Table of Contents 1. [Enumerations](#enumerations) -2. [Semantic Model](#semantic-model) -3. [Datasets](#datasets) -4. [Relationships](#relationships) -5. [Fields](#fields) -6. [Metrics](#metrics) -7. [Examples](#examples) +2. [Document](#document) +3. [Semantic Model](#semantic-model) +4. [Datasets](#datasets) +5. [Relationships](#relationships) +6. [Fields](#fields) +7. [Metrics](#metrics) +8. [Examples](#examples) --- @@ -80,6 +81,44 @@ ontology specification's built-in value types; `Time`, `DateTimeTz`, and | `DateTimeTz` | Date and time with sufficient offset or timezone context to identify an instant. Preservation of a named timezone identifier is not guaranteed. | | `Opaque` | Known type outside the portable vocabulary; use `custom_extensions` for vendor-specific refinement. Omit `datatype` when the type is unknown or unspecified. | +## Document + +The root object of an Ossie file. A document carries a specification `version` and one or +more semantic models. + +### Schema + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `version` | string | Yes | Apache Ossie specification version | +| `ai_context` | string/object | No | Document-wide context for AI tools | +| `semantic_model` | array | Yes | Collection of semantic model definitions | + +### Document-wide `ai_context` + +A document may hold several semantic models — for example one per data source. Guidance that +governs all of them belongs to the document rather than to any one model, and stating it at +the root avoids copying it into every model, where the copies drift and a consumer cannot +tell them apart from genuinely model-specific instruction. + +```yaml +version: 0.2.0.dev0 +ai_context: + instructions: "Fiscal year starts in July. Never join across the finance and telemetry models." +semantic_model: + - name: finance + ai_context: + instructions: "Amounts are in USD." + datasets: [...] + - name: telemetry + datasets: [...] +``` + +Document-level `ai_context` applies to every semantic model in the document. How it composes +with model-level `ai_context` is not defined by this version of the specification. + +--- + ## Semantic Model The top-level container that represents a complete semantic model, including datasets, relationships, and metrics. diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 8c9c0fa4..844fbbfa 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -64,6 +64,19 @@ vendor_name: string # Keys allowed on the root object of an Ossie document, alongside the `version` # declared at the top of this file. +# Optional: Document-wide context for AI tools. +# Applies to every semantic model in this document. Use this for guidance that +# governs the document as a whole, rather than repeating it in every model. +# Can be a simple string or a structured object; see the authoritative AIContext +# definition in ossie-schema.json, which accepts both forms wherever ai_context +# appears. +ai_context: + instructions: string + synonyms: + - string + examples: + - string + # Top-level semantic model definition semantic_model: # Required: Unique identifier for the semantic model diff --git a/docs/index.md b/docs/index.md index 3836092d..c497156f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -57,7 +57,7 @@ The Ossie core specification (current version: **0.2.0.dev0**, latest released: | **Relationships** | Foreign key connections between datasets, supporting both simple and composite keys. | | **Metrics** | Quantitative measures (sums, averages, ratios, etc.) defined at the model level, capable of spanning multiple datasets. | | **Custom Extensions** | Vendor-specific metadata stored as JSON, allowing platforms to carry additional information without breaking core compatibility. | -| **AI Context** | Optional annotations at every level (model, dataset, field, relationship, metric) to help AI tools understand business meaning — including instructions, synonyms, and example queries. | +| **AI Context** | Optional annotations at every level (document, model, dataset, field, relationship, metric) to help AI tools understand business meaning — including instructions, synonyms, and example queries. | The specification supports multiple SQL dialects (`ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`, `MDX`, `TABLEAU`) so that expressions can be tailored to each platform while maintaining a common model structure. @@ -322,7 +322,7 @@ A practical guide for organizations looking to adopt Ossie. | **Relationship** | A foreign key connection between two datasets, defining how they can be joined. Relationships are always many-to-one (from the referencing dataset to the referenced dataset). | | **Dialect** | A specific SQL or expression language variant (e.g., `ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`). Ossie supports multiple dialects so expressions can be tailored to each platform. | | **Custom Extension** | Vendor-specific metadata attached to any Ossie construct as a JSON string. Extensions allow platforms to carry additional information without modifying the core specification. | -| **AI Context** | Optional annotations on any Ossie construct (model, dataset, field, relationship, metric) that provide additional context for AI tools — including natural language instructions, synonyms, and example queries. | +| **AI Context** | Optional annotations on any Ossie construct (document, model, dataset, field, relationship, metric) that provide additional context for AI tools — including natural language instructions, synonyms, and example queries. | | **Converter** | A tool that translates between the Ossie format and a specific vendor's semantic model format. Converters come in pairs: import (vendor → Ossie) and export (Ossie → vendor). | | **Hub-and-Spoke** | The architectural pattern used by Ossie, where the specification acts as the central format (hub) and vendor converters act as spokes, avoiding the need for point-to-point integrations. | | **Round-Trip Fidelity** | The ability to convert a model from one format to Ossie and back without losing information. Achieved by preserving vendor-specific metadata in `custom_extensions`. | diff --git a/examples/multi_model_ai_context.yaml b/examples/multi_model_ai_context.yaml new file mode 100644 index 00000000..b7487a52 --- /dev/null +++ b/examples/multi_model_ai_context.yaml @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Document-wide ai_context +# +# This document holds two semantic models, one per data source, each queried in +# its own dialect. The guidance at the root governs both of them: it states the +# publisher's conventions once, instead of being copied into every model where +# the copies drift and a consumer cannot tell them apart from instruction that +# is genuinely specific to one model. +# +# Model-level ai_context remains the place for anything true of only that model +# ("amounts are in USD" below). + +version: "0.2.0.dev0" + +ai_context: + instructions: "The fiscal year starts in July: a date in July 2024 belongs to FY2025. These two models come from separate systems and share no keys, so never join across them; answer questions that span both by querying each and reporting the results side by side." + synonyms: + - "revenue = net_sales" + +semantic_model: + - name: finance + description: General ledger, sourced from the accounting warehouse. + ai_context: + instructions: "Amounts are in USD. Rows with a null posted_at are drafts and must be excluded." + + datasets: + - name: gl_entries + source: finance.public.gl_entries + primary_key: [entry_id] + description: One row per posted general ledger entry. + fields: + - name: entry_id + expression: + dialects: + - dialect: ANSI_SQL + expression: entry_id + datatype: Integer + description: Surrogate key of the ledger entry. + + - name: net_sales + expression: + dialects: + - dialect: ANSI_SQL + expression: net_sales + datatype: Decimal + description: Net sales amount for the entry. + + - name: posted_at + expression: + dialects: + - dialect: ANSI_SQL + expression: posted_at + datatype: DateTime + description: When the entry was posted; null for drafts. + dimension: + is_time: true + + - name: telemetry + description: Product usage events, sourced from the telemetry lakehouse. + + datasets: + - name: events + source: telemetry.public.events + primary_key: [event_id] + description: One row per emitted product usage event. + fields: + - name: event_id + expression: + dialects: + - dialect: DATABRICKS + expression: event_id + datatype: String + description: Unique identifier of the event. + + - name: occurred_at + expression: + dialects: + - dialect: DATABRICKS + expression: occurred_at + datatype: DateTimeTz + description: When the event was emitted. + dimension: + is_time: true diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index e782275c..7d9889af 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -212,6 +212,7 @@ class OssieDocument(BaseModel): version: str = "0.2.0.dev0" dialects: Optional[list[OssieDialect]] = None vendors: Optional[list[OssieVendor]] = None + ai_context: Optional[OssieAIContext] = None semantic_model: list[OssieSemanticModel] def to_ossie_yaml(self, **kwargs: Any) -> str: diff --git a/python/tests/test_models.py b/python/tests/test_models.py index 74d2a320..be2524ae 100644 --- a/python/tests/test_models.py +++ b/python/tests/test_models.py @@ -111,6 +111,64 @@ def test_invalid_datatype_is_rejected() -> None: OssieDocument.model_validate(document) +def test_document_ai_context_matches_core_schema() -> None: + schema_path = Path(__file__).parents[2] / "core-spec" / "ossie-schema.json" + schema = json.loads(schema_path.read_text()) + + assert schema["properties"]["ai_context"]["$ref"] == "#/$defs/AIContext" + + # Document-wide context is optional, so existing documents keep validating + # unchanged, and the root stays closed to anything else. + assert "ai_context" not in schema["required"] + assert schema["additionalProperties"] is False + + +def test_document_ai_context_agrees_with_the_ontology_root() -> None: + """Both document roots resolve ai_context against the same definition. + + The ontology specification already puts ai_context on its root and $refs this + specification's AIContext. If that reference is ever repointed at a different + definition, the two document types would silently disagree about what + document-wide context means. + """ + core = json.loads( + (Path(__file__).parents[2] / "core-spec" / "ossie-schema.json").read_text() + ) + ontology = json.loads( + (Path(__file__).parents[2] / "ontology" / "ontology.json").read_text() + ) + + assert ontology["properties"]["ai_context"]["$ref"].endswith( + "ossie-schema.json#/$defs/AIContext" + ) + assert core["properties"]["ai_context"]["$ref"] == "#/$defs/AIContext" + + +def test_document_ai_context_survives_serialization() -> None: + """Both forms of AIContext round-trip, and an absent one does not serialize.""" + document = _document() + document["ai_context"] = { + "instructions": "Fiscal year starts in July.", + "synonyms": ["revenue = net_sales"], + } + + parsed = OssieDocument.model_validate(document) + assert parsed.ai_context.instructions == "Fiscal year starts in July." + for serialized in ( + json.loads(parsed.to_ossie_json()), + yaml.safe_load(parsed.to_ossie_yaml()), + ): + assert serialized["ai_context"]["instructions"] == "Fiscal year starts in July." + + document["ai_context"] = "Fiscal year starts in July." + assert OssieDocument.model_validate(document).ai_context == "Fiscal year starts in July." + + absent = OssieDocument.model_validate(_document()) + assert absent.ai_context is None + assert "ai_context" not in yaml.safe_load(absent.to_ossie_yaml()) + assert "ai_context" not in json.loads(absent.to_ossie_json()) + + @pytest.mark.parametrize( ("dimension", "datatype", "expected"), [