diff --git a/.github/workflows/validation-ci.yml b/.github/workflows/validation-ci.yml index a5c670a3..6b8c1ddc 100644 --- a/.github/workflows/validation-ci.yml +++ b/.github/workflows/validation-ci.yml @@ -26,6 +26,7 @@ on: - "validation/**" - "core-spec/**" - "examples/tpcds_semantic_model.yaml" + - "examples/multi_model_ai_context.yaml" - ".github/workflows/validation-ci.yml" pull_request: branches: ["main"] @@ -33,6 +34,7 @@ on: - "validation/**" - "core-spec/**" - "examples/tpcds_semantic_model.yaml" + - "examples/multi_model_ai_context.yaml" - ".github/workflows/validation-ci.yml" jobs: @@ -64,3 +66,6 @@ jobs: - name: Validate canonical example run: uv run validation/validate.py examples/tpcds_semantic_model.yaml + + - name: Validate document-wide ai_context example + run: uv run validation/validate.py examples/multi_model_ai_context.yaml diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index c87af723..63984d65 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -10,6 +10,10 @@ "const": "0.2.0.dev0", "description": "Apache Ossie specification version" }, + "ai_context": { + "$ref": "#/$defs/AIContext", + "description": "Document-wide context for AI tools. Applies to every semantic model in this document." + }, "semantic_model": { "type": "array", "description": "Collection of semantic model definitions", diff --git a/core-spec/spec.md b/core-spec/spec.md index 2850edfe..279896c7 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -32,12 +32,13 @@ ## Table of Contents 1. [Enumerations](#enumerations) -2. [Semantic Model](#semantic-model) -3. [Datasets](#datasets) -4. [Relationships](#relationships) -5. [Fields](#fields) -6. [Metrics](#metrics) -7. [Examples](#examples) +2. [Document](#document) +3. [Semantic Model](#semantic-model) +4. [Datasets](#datasets) +5. [Relationships](#relationships) +6. [Fields](#fields) +7. [Metrics](#metrics) +8. [Examples](#examples) --- @@ -80,6 +81,44 @@ ontology specification's built-in value types; `Time`, `DateTimeTz`, and | `DateTimeTz` | Date and time with sufficient offset or timezone context to identify an instant. Preservation of a named timezone identifier is not guaranteed. | | `Opaque` | Known type outside the portable vocabulary; use `custom_extensions` for vendor-specific refinement. Omit `datatype` when the type is unknown or unspecified. | +## Document + +The root object of an Ossie file. A document carries a specification `version` and one or +more semantic models. + +### Schema + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `version` | string | Yes | Apache Ossie specification version | +| `ai_context` | string/object | No | Document-wide context for AI tools | +| `semantic_model` | array | Yes | Collection of semantic model definitions | + +### Document-wide `ai_context` + +A document may hold several semantic models — for example one per data source. Guidance that +governs all of them belongs to the document rather than to any one model, and stating it at +the root avoids copying it into every model, where the copies drift and a consumer cannot +tell them apart from genuinely model-specific instruction. + +```yaml +version: 0.2.0.dev0 +ai_context: + instructions: "Fiscal year starts in July. Never join across the finance and telemetry models." +semantic_model: + - name: finance + ai_context: + instructions: "Amounts are in USD." + datasets: [...] + - name: telemetry + datasets: [...] +``` + +Document-level `ai_context` applies to every semantic model in the document. How it composes +with model-level `ai_context` is not defined by this version of the specification. + +--- + ## Semantic Model The top-level container that represents a complete semantic model, including datasets, relationships, and metrics. diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 4678b004..844fbbfa 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -59,6 +59,24 @@ datatypes: vendor_name: string +--- +# Document root +# Keys allowed on the root object of an Ossie document, alongside the `version` +# declared at the top of this file. + +# Optional: Document-wide context for AI tools. +# Applies to every semantic model in this document. Use this for guidance that +# governs the document as a whole, rather than repeating it in every model. +# Can be a simple string or a structured object; see the authoritative AIContext +# definition in ossie-schema.json, which accepts both forms wherever ai_context +# appears. +ai_context: + instructions: string + synonyms: + - string + examples: + - string + # Top-level semantic model definition semantic_model: # Required: Unique identifier for the semantic model diff --git a/docs/index.md b/docs/index.md index 3836092d..c497156f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -57,7 +57,7 @@ The Ossie core specification (current version: **0.2.0.dev0**, latest released: | **Relationships** | Foreign key connections between datasets, supporting both simple and composite keys. | | **Metrics** | Quantitative measures (sums, averages, ratios, etc.) defined at the model level, capable of spanning multiple datasets. | | **Custom Extensions** | Vendor-specific metadata stored as JSON, allowing platforms to carry additional information without breaking core compatibility. | -| **AI Context** | Optional annotations at every level (model, dataset, field, relationship, metric) to help AI tools understand business meaning — including instructions, synonyms, and example queries. | +| **AI Context** | Optional annotations at every level (document, model, dataset, field, relationship, metric) to help AI tools understand business meaning — including instructions, synonyms, and example queries. | The specification supports multiple SQL dialects (`ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`, `MDX`, `TABLEAU`) so that expressions can be tailored to each platform while maintaining a common model structure. @@ -322,7 +322,7 @@ A practical guide for organizations looking to adopt Ossie. | **Relationship** | A foreign key connection between two datasets, defining how they can be joined. Relationships are always many-to-one (from the referencing dataset to the referenced dataset). | | **Dialect** | A specific SQL or expression language variant (e.g., `ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`). Ossie supports multiple dialects so expressions can be tailored to each platform. | | **Custom Extension** | Vendor-specific metadata attached to any Ossie construct as a JSON string. Extensions allow platforms to carry additional information without modifying the core specification. | -| **AI Context** | Optional annotations on any Ossie construct (model, dataset, field, relationship, metric) that provide additional context for AI tools — including natural language instructions, synonyms, and example queries. | +| **AI Context** | Optional annotations on any Ossie construct (document, model, dataset, field, relationship, metric) that provide additional context for AI tools — including natural language instructions, synonyms, and example queries. | | **Converter** | A tool that translates between the Ossie format and a specific vendor's semantic model format. Converters come in pairs: import (vendor → Ossie) and export (Ossie → vendor). | | **Hub-and-Spoke** | The architectural pattern used by Ossie, where the specification acts as the central format (hub) and vendor converters act as spokes, avoiding the need for point-to-point integrations. | | **Round-Trip Fidelity** | The ability to convert a model from one format to Ossie and back without losing information. Achieved by preserving vendor-specific metadata in `custom_extensions`. | diff --git a/examples/multi_model_ai_context.yaml b/examples/multi_model_ai_context.yaml new file mode 100644 index 00000000..b7487a52 --- /dev/null +++ b/examples/multi_model_ai_context.yaml @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Document-wide ai_context +# +# This document holds two semantic models, one per data source, each queried in +# its own dialect. The guidance at the root governs both of them: it states the +# publisher's conventions once, instead of being copied into every model where +# the copies drift and a consumer cannot tell them apart from instruction that +# is genuinely specific to one model. +# +# Model-level ai_context remains the place for anything true of only that model +# ("amounts are in USD" below). + +version: "0.2.0.dev0" + +ai_context: + instructions: "The fiscal year starts in July: a date in July 2024 belongs to FY2025. These two models come from separate systems and share no keys, so never join across them; answer questions that span both by querying each and reporting the results side by side." + synonyms: + - "revenue = net_sales" + +semantic_model: + - name: finance + description: General ledger, sourced from the accounting warehouse. + ai_context: + instructions: "Amounts are in USD. Rows with a null posted_at are drafts and must be excluded." + + datasets: + - name: gl_entries + source: finance.public.gl_entries + primary_key: [entry_id] + description: One row per posted general ledger entry. + fields: + - name: entry_id + expression: + dialects: + - dialect: ANSI_SQL + expression: entry_id + datatype: Integer + description: Surrogate key of the ledger entry. + + - name: net_sales + expression: + dialects: + - dialect: ANSI_SQL + expression: net_sales + datatype: Decimal + description: Net sales amount for the entry. + + - name: posted_at + expression: + dialects: + - dialect: ANSI_SQL + expression: posted_at + datatype: DateTime + description: When the entry was posted; null for drafts. + dimension: + is_time: true + + - name: telemetry + description: Product usage events, sourced from the telemetry lakehouse. + + datasets: + - name: events + source: telemetry.public.events + primary_key: [event_id] + description: One row per emitted product usage event. + fields: + - name: event_id + expression: + dialects: + - dialect: DATABRICKS + expression: event_id + datatype: String + description: Unique identifier of the event. + + - name: occurred_at + expression: + dialects: + - dialect: DATABRICKS + expression: occurred_at + datatype: DateTimeTz + description: When the event was emitted. + dimension: + is_time: true diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index e782275c..7d9889af 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -212,6 +212,7 @@ class OssieDocument(BaseModel): version: str = "0.2.0.dev0" dialects: Optional[list[OssieDialect]] = None vendors: Optional[list[OssieVendor]] = None + ai_context: Optional[OssieAIContext] = None semantic_model: list[OssieSemanticModel] def to_ossie_yaml(self, **kwargs: Any) -> str: diff --git a/python/tests/test_models.py b/python/tests/test_models.py index 74d2a320..be2524ae 100644 --- a/python/tests/test_models.py +++ b/python/tests/test_models.py @@ -111,6 +111,64 @@ def test_invalid_datatype_is_rejected() -> None: OssieDocument.model_validate(document) +def test_document_ai_context_matches_core_schema() -> None: + schema_path = Path(__file__).parents[2] / "core-spec" / "ossie-schema.json" + schema = json.loads(schema_path.read_text()) + + assert schema["properties"]["ai_context"]["$ref"] == "#/$defs/AIContext" + + # Document-wide context is optional, so existing documents keep validating + # unchanged, and the root stays closed to anything else. + assert "ai_context" not in schema["required"] + assert schema["additionalProperties"] is False + + +def test_document_ai_context_agrees_with_the_ontology_root() -> None: + """Both document roots resolve ai_context against the same definition. + + The ontology specification already puts ai_context on its root and $refs this + specification's AIContext. If that reference is ever repointed at a different + definition, the two document types would silently disagree about what + document-wide context means. + """ + core = json.loads( + (Path(__file__).parents[2] / "core-spec" / "ossie-schema.json").read_text() + ) + ontology = json.loads( + (Path(__file__).parents[2] / "ontology" / "ontology.json").read_text() + ) + + assert ontology["properties"]["ai_context"]["$ref"].endswith( + "ossie-schema.json#/$defs/AIContext" + ) + assert core["properties"]["ai_context"]["$ref"] == "#/$defs/AIContext" + + +def test_document_ai_context_survives_serialization() -> None: + """Both forms of AIContext round-trip, and an absent one does not serialize.""" + document = _document() + document["ai_context"] = { + "instructions": "Fiscal year starts in July.", + "synonyms": ["revenue = net_sales"], + } + + parsed = OssieDocument.model_validate(document) + assert parsed.ai_context.instructions == "Fiscal year starts in July." + for serialized in ( + json.loads(parsed.to_ossie_json()), + yaml.safe_load(parsed.to_ossie_yaml()), + ): + assert serialized["ai_context"]["instructions"] == "Fiscal year starts in July." + + document["ai_context"] = "Fiscal year starts in July." + assert OssieDocument.model_validate(document).ai_context == "Fiscal year starts in July." + + absent = OssieDocument.model_validate(_document()) + assert absent.ai_context is None + assert "ai_context" not in yaml.safe_load(absent.to_ossie_yaml()) + assert "ai_context" not in json.loads(absent.to_ossie_json()) + + @pytest.mark.parametrize( ("dimension", "datatype", "expected"), [