diff --git a/PYDANTIC_GUIDE.md b/AUTHORING.md similarity index 66% rename from PYDANTIC_GUIDE.md rename to AUTHORING.md index 4571b2afe..0c0300281 100644 --- a/PYDANTIC_GUIDE.md +++ b/AUTHORING.md @@ -1,34 +1,251 @@ -# Overture Maps Pydantic Schema Guide - -This guide helps you work with Overture Maps Pydantic schemas - Python models that define geospatial data structures with automatic validation. Whether you're new to Pydantic or migrating from JSON Schema, this guide provides a progressive learning path from basics to advanced patterns. - -## Table of Contents - -- [Quick Start](#quick-start) -- [Basic Concepts](#basic-concepts) - - [Models and Inheritance](#models-and-inheritance) - - [Field Types](#field-types) - - [Field Enhancement](#field-enhancement) - - [Collections and Lists](#collections-and-lists) - - [Enumerations](#enumerations) -- [Advanced Patterns](#advanced-patterns) - - [Relationship Patterns](#relationship-patterns) - - [Discriminated Unions](#discriminated-unions) - - [Pattern Properties (Constrained Key-Value Maps)](#pattern-properties-constrained-key-value-maps) - - [Nested List Validation](#nested-list-validation) - - [Type Aliases for Reusable Patterns](#type-aliases-for-reusable-patterns) -- [Integration Guide](#integration-guide) - - [Project Architecture](#project-architecture) - - [Migrating from JSON Schema](#migrating-from-json-schema) -- [Reference](#reference) - - [Complete Templates](#complete-templates) - - [Quick Reference](#quick-reference) +# Authoring and Extending the Schema + +This page is for people **writing** to the schema: adding feature types, building tools on top of the models, +or authoring new Pydantic models for Overture itself. If you only want to *use* the +schema to validate data, explore models, or generate artifacts, you want +[SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) instead. + +| If you want to | Read | +|---|---| +| Make your own feature types visible to the Overture tooling | [Register your own feature types](#register-your-own-feature-types) | +| Generate an SDK in another language, or build your own CLI | [SCHEMA_GUIDE.md §8](SCHEMA_GUIDE.md#8-building-tools-on-the-models) — that's consumer work | +| Understand entry-point registration and tags | [Registering models and tagging](#registering-models-and-tagging) | +| Author a new Pydantic model for the schema | [Authoring new schema models](#authoring-new-schema-models) | +| Run the tests and checks | [Development workflow](#development-workflow) | +| Copy a working starting point | [Templates and quick reference](#templates-and-quick-reference) | + +This page is the successor to the repo's old `PYDANTIC_GUIDE.md` — the contributor-facing +authoring guide, corrected and re-tested. It was folded into `SCHEMA_GUIDE.md` as Part II +during the docs consolidation and is broken back out here so the contributor material +stands on its own, as the consolidation set out to do. + +Per-package reference — installation, usage, and API for one package — lives in that +package's `README.md` under `packages/`, versioned alongside the code it documents. This +page covers what spans packages. + +See also [CONCEPTS.md](CONCEPTS.md) for why the schema is Pydantic and how the packages +fit together. + +*Every code block on this page has been executed against the repo; +`tests/test_documented_imports.py` keeps the imports honest.* + +--- + +## Extending the schema with your own types + +### Register your own feature types + +Your models become first-class: discovered by the CLI, accepted by `validate()`, +included in generated docs and JSON Schema. Nothing in the tooling special-cases +Overture. + +```python +# mypkg/models.py +from typing import Literal +from overture.schema.common import OvertureFeature +from overture.schema.system.numeric import float32 + + +class Vineyard(OvertureFeature[Literal["agriculture"], Literal["vineyard"]]): + """A cultivated area planted with grapevines.""" + + area_hectares: float32 | None = None +``` + +```toml +# mypkg/pyproject.toml +[project.entry-points."overture.models"] +vineyard = "mypkg.models:Vineyard" +``` + +Install it, and: + +```bash +overture-schema list-types +overture-schema validate vineyard.json +overture-codegen generate --format markdown --output-dir out +``` + +To attach your own tags, register a **tag provider** on `overture.tag_providers`: + +```python +def experimental_provider(types, key, tags): + if any(getattr(t, "__experimental__", False) for t in types): + tags.add("mypkg:experimental") + return tags +``` + +```toml +[project.entry-points."overture.tag_providers"] +experimental = "mypkg.tags:experimental_provider" +``` + +Tag namespaces are reserved: `feature` and `system:` belong to `overture-schema-system`, +`overture:` to `overture-schema-common`. A provider that tries to set a reserved tag from +an unauthorized package gets a logged warning and the tag is discarded. Use your own +namespace. + +You don't have to build on `OvertureFeature` — subclass `system.Feature` directly for a +GeoJSON-serializing model with none of the Overture conventions. + +### Write a new codegen target + +For a format nobody else generates — Arrow schemas, Avro, Go structs, protobuf — add a +renderer to the codegen rather than parsing JSON Schema back out. You get the full +semantic model: NewType names, constraint provenance, discriminated union structure — all +the things JSON Schema flattens away. + +The pipeline is four layers with strictly downward imports: + +``` +Rendering → output formatting, all presentation decisions +Output Layout → what to generate, where it goes, how outputs link +Extraction → FieldShape, FieldSpec, RecordSpec, UnionSpec, EnumSpec +Discovery → discover_models() +``` + +Extraction is target-independent, so a new target is a new renderer, not new extraction +logic. The entry point: + +```python +from overture.schema.codegen.extraction.model_extraction import extract_model +from overture.schema.buildings import Building + +spec = extract_model(Building) +spec.name # 'Building' +spec.description # the class docstring +spec.constraints # model-level constraints + +for f in spec.fields[:6]: + print(f"{f.name:12} required={f.is_required!s:5} {type(f.shape).__name__}") +``` + +``` +id required=True NewTypeShape +bbox required=False Primitive +geometry required=True Primitive +theme required=True LiteralScalar +type required=True LiteralScalar +version required=True NewTypeShape +``` + +`FieldSpec` is `(name, shape, description, is_required, is_optional)`. The `shape` is a +`FieldShape` tree — `NewTypeShape`, `Primitive`, `LiteralScalar`, `ModelRef`, +`UnionRef`, and container variants — with sub-models and sub-unions already resolved. +Constraints carry provenance, so you can tell which NewType contributed which bound: + +```python +from overture.schema.codegen.extraction.type_analyzer import analyze_type +from overture.schema.common.feature import FeatureVersion + +shape, is_nullable, description = analyze_type(FeatureVersion) +# shape → NewTypeShape(name='FeatureVersion', inner=Primitive(base_type='int32', +# constraints=(ConstraintSource(source_name='FeatureVersion', constraint=Ge(ge=0)), ...))) +``` + +`analyze_type` returns a **3-tuple** `(FieldShape, bool, str | None)` — the structural +shape, whether the field accepts `None`, and the first description found while +unwrapping. (The codegen README shows an older `TypeInfo`/`TypeKind` API that no longer +exists.) + +To wire up a new format: add a column to `TypeMapping` in +`extraction/type_registry.py` for type-name resolution, write a pipeline module +consuming `ModelSpec` trees plus a renderer, and register the format in `cli.py`. + +Further reading in the repo: + +- `packages/overture-schema-codegen/docs/design.md` — architecture, data flow, extension points +- `packages/overture-schema-codegen/docs/walkthrough.md` — module-by-module trace of `Segment` through the pipeline + +--- --- -## Quick Start +## Registering models and tagging -### Essential Imports +How feature types make themselves known to the tooling. This is the mechanism that +lets your own models slot in alongside Overture's — see +[Register your own feature types](#register-your-own-feature-types) for a worked example. + +The library is designed to support data producer extensions through multiple patterns. +This extensibility is a core feature that allows organizations to add custom fields and +types while maintaining compatibility with the base Overture schema. We are in the +process of determining how this should work. + +### Model Registration via Entry Points + +Models are registered using [setuptools entry +points](https://setuptools.pypa.io/en/latest/userguide/entry_point.html) in each +package's `pyproject.toml` file. This enables automatic discovery and loading of models +at runtime without requiring explicit imports. + +Registration is done in the `[project.entry-points."overture.models"]` section: + +```toml +[project.entry-points."overture.models"] +building = "overture.schema.buildings:Building" +building_part = "overture.schema.buildings:BuildingPart" +``` + +The discovery system provides programmatic access to registered models: + +```python +from overture.schema.system.discovery import discover_models, get_registered_model + +# Discover all registered models, keyed by ModelKey +all_models = discover_models() + +# Get a specific model by name +building_model = get_registered_model("building") +if building_model: + building = building_model.model_validate(building_data) +``` + +### Tagging + +Each `ModelKey` returned by `discover_models()` carries a `frozenset[str]` of tags +that classify the model orthogonally to its entry-point name -- whether the model +is a `Feature` subclass, which Overture theme it belongs to, which package shipped +it, and so on. Downstream tools (the CLI, codegen, third-party consumers) use tags +to filter the working set without importing every model: + +```python +from overture.schema.system.discovery import ( + TagSelector, + discover_models, + filter_models, +) + +models = discover_models() +# { +# ModelKey(name="building", entry_point="overture.schema.buildings:Building", +# tags=frozenset({"feature", "overture", "overture:theme=buildings"})): Building, +# ModelKey(name="place", entry_point="overture.schema.places:Place", +# tags=frozenset({"feature", "overture", "overture:theme=places"})): Place, +# ... +# } + +buildings = filter_models( + models, + TagSelector(include_any=("overture:theme=buildings",)), +) +``` + +Tags are produced by *tag providers* registered on the `overture.tag_providers` +entry-point group. The `system` and `common` packages ship the built-in providers +(`feature` and `overture:theme=*`); third parties can register their own +to attach custom tags during discovery. See the [`overture-schema-system` +README](packages/overture-schema-system/README.md#tagging) for tag format, +reserved namespaces, and provider authoring. + + +--- + +## Authoring new schema models + +### Quick Start + +#### Essential Imports Copy what you need for most models: @@ -42,7 +259,11 @@ from pydantic import BaseModel, Field # Overture common models from overture.schema.common import OvertureFeature -from overture.schema.system.geometric import Geometry, GeometryType, GeometryTypeConstraint +from overture.schema.system.geometric import ( + Geometry, + GeometryType, + GeometryTypeConstraint, +) # Validation system from overture.schema.system.field_constraint import UniqueItemsConstraint @@ -59,75 +280,35 @@ from overture.schema.system.string import LanguageTag # Numeric types (use these instead of int/float) from overture.schema.system.numeric import ( - int8, int32, int64, - uint8, uint16, uint32, - float32, float64 + int8, + int32, + int64, + uint8, + uint16, + uint32, + float32, + float64, ) ``` -### Basic Model Template +#### Templates -```python -from typing import Annotated -from pydantic import BaseModel, Field -from overture.schema.system.model_constraint import no_extra_fields -from overture.schema.system.numeric import int8, float64 - -@no_extra_fields -class MyCustomType(BaseModel): - """Brief description of what this represents.""" - - # Required fields (no default value) - name: str - category: str - - # Optional fields (with None default) - description: str | None = None - - # Field with constraints and description - priority: Annotated[ - int8 | None, - Field( - ge=1, - le=10, - description="Priority level from 1 (lowest) to 10 (highest)" - ) - ] = None -``` - -### Feature Template - -```python -from typing import Annotated, Literal -from pydantic import Field -from overture.schema.common import OvertureFeature -from overture.schema.system.geometric import Geometry, GeometryType, GeometryTypeConstraint - -class MyFeature(OvertureFeature[Literal["my_theme"], Literal["my_type"]]): - """Description of what this feature represents.""" - - # Geometry with constraints - geometry: Annotated[ - Geometry, - GeometryTypeConstraint(GeometryType.POINT), - Field(description="Location of this feature"), - ] - - # Custom fields - my_field: str | None = None -``` +Copy-paste starting points for the four shapes you'll write most often — a plain +model, a feature, an enum, and a model with validation constraints — live together in +[Templates and quick reference](#templates-and-quick-reference) rather than being +repeated here. --- -## Basic Concepts +### Basic Concepts -### Models and Inheritance +#### Models and Inheritance -#### What are Pydantic models? +##### What are Pydantic models? Pydantic models are Python classes that define data structures and their constraints. Think of them like UML classes with built-in data validation - each model defines what fields are allowed and what types of data they can contain. -#### Model Base Classes and Inheritance +##### Model Base Classes and Inheritance **What is a "base class"?** A base class defines common fields and behaviors that other classes can reuse. Think of it like a slide template - you create one layout, then make specific slides that use that structure. @@ -138,9 +319,11 @@ Pydantic models are Python classes that define data structures and their constra ```python from overture.schema.system.model_constraint import no_extra_fields + @no_extra_fields class Address(BaseModel): """A postal address - no extra fields allowed.""" + street: str city: str postal_code: str | None = None @@ -154,8 +337,10 @@ from typing import Literal from overture.schema.common import OvertureFeature from overture.schema.system.numeric import float64 + class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): """A building feature with strongly-typed theme and type.""" + # Inherits: id, theme, type, geometry, bbox, version, sources height: float64 | None = None ``` @@ -171,7 +356,7 @@ class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): By specifying `OvertureFeature[Literal["buildings"], Literal["building"]]`, you're saying "this is a Feature that must have theme='buildings' and type='building'" - no other values are allowed. This prevents mistakes like accidentally creating a building with theme="places". -#### Inheritance Patterns +##### Inheritance Patterns **Multiple inheritance** combines fields from several base classes: @@ -182,7 +367,10 @@ from overture.schema.common.level import Stacked from overture.schema.common.names import Named from overture.schema.system.numeric import float64 -class Building(OvertureFeature[Literal["buildings"], Literal["building"]], Named, Stacked): + +class Building( + OvertureFeature[Literal["buildings"], Literal["building"]], Named, Stacked +): # Gets fields from Feature: id, theme, type, geometry, etc. # Gets fields from Named: names # Gets fields from Stacked: level @@ -190,7 +378,7 @@ class Building(OvertureFeature[Literal["buildings"], Literal["building"]], Named height: float64 | None = None ``` -#### Field Aliases +##### Field Aliases Sometimes you need a field name that conflicts with Python keywords or conventions (hint: you'll get an error when you try to use it). Use `Field(alias="")` to map between Python-friendly field names and the actual data field names: @@ -198,6 +386,7 @@ Sometimes you need a field name that conflicts with Python keywords or conventio from typing import Annotated from pydantic import Field + class Building(OvertureFeature): # Use class_ in Python code, but "class" in the actual data class_: Annotated[str | None, Field(alias="class")] = None @@ -209,9 +398,9 @@ class Building(OvertureFeature): A common example is `class_` with `Field(alias="class")` since "class" is a Python keyword but a common field name in data schemas. -### Field Types +#### Field Types -#### Required vs Optional Fields +##### Required vs Optional Fields ```python class Building(OvertureFeature): @@ -246,7 +435,7 @@ class Building(OvertureFeature): ```python access_policy: Annotated[ str | None, - Field(description="Access policy for the place. When absent, assume 'open'") + Field(description="Access policy for the place. When absent, assume 'open'"), ] = None ``` @@ -259,33 +448,39 @@ class Building(OvertureFeature): Keep the schema separate from business logic. The schema describes the shape of data, not the business rules about what missing values mean. -#### Numeric Types +##### Numeric Types **Always use specific numeric types instead of Python's generic `int`/`float`:** ```python from overture.schema.system.model_constraint import no_extra_fields from overture.schema.system.numeric import ( - int8, int32, int64, # Signed integers - uint8, uint16, uint32, # Unsigned integers - float32, float64 # Floating point + int8, + int32, + int64, # Signed integers + uint8, + uint16, + uint32, # Unsigned integers + float32, + float64, # Floating point ) + @no_extra_fields class MyModel(BaseModel): # Signed integers with specific ranges - level: int8 | None = None # -128 to 127 - year: int32 | None = None # -2,147,483,648 to 2,147,483,647 - timestamp: int64 | None = None # Full 64-bit range + level: int8 | None = None # -128 to 127 + year: int32 | None = None # -2,147,483,648 to 2,147,483,647 + timestamp: int64 | None = None # Full 64-bit range # Unsigned integers (0 and positive only) - red_value: uint8 | None = None # 0 to 255 (like RGB values) - port: uint16 | None = None # 0 to 65,535 (like network ports) - population: uint32 | None = None # 0 to 4,294,967,295 + red_value: uint8 | None = None # 0 to 255 (like RGB values) + port: uint16 | None = None # 0 to 65,535 (like network ports) + population: uint32 | None = None # 0 to 4,294,967,295 # Floating point numbers - height: float64 | None = None # Double precision (recommended) - ratio: float32 | None = None # Single precision + height: float64 | None = None # Double precision (recommended) + ratio: float32 | None = None # Single precision ``` **When to use each:** @@ -309,13 +504,14 @@ The specific numeric types are crucial for data interchange and storage compatib - **Storage efficiency**: `uint8` uses 1 byte vs `int64` which uses 8 bytes - **Built-in validation**: These types use Pydantic `Field()` constraints to validate ranges (e.g., `Field(ge=0, le=100)` ensures values stay within bounds) -#### Union Types +##### Union Types Union types allow a field to accept multiple different types. The `|` symbol means "or": ```python from typing import Literal + class Building(OvertureFeature): # This field can be either a string OR None (most common union) name: str | None = None @@ -328,7 +524,7 @@ class Building(OvertureFeature): ```python # Optional field (most common union) -height: float64 | None = None # Can be a number or missing +height: float64 | None = None # Can be a number or missing # Specific string values (an alternative to enums where descriptions aren't needed) priority: Literal["low", "medium", "high"] | None = None @@ -347,9 +543,9 @@ is_verified: bool | None = None > [!WARNING] > **Storage compatibility**: Mixed-type unions (combining different basic types like `str | int32`) don't work with Parquet and other storage layers. Use `Literal` values or separate fields instead. -### Field Enhancement +#### Field Enhancement -#### Adding Descriptions and Constraints with Annotated +##### Adding Descriptions and Constraints with Annotated `Annotated` is Python's way to add extra information (metadata) to a type without changing the type itself. Think of it like adding notes or constraints to a field definition. @@ -364,8 +560,8 @@ height: float64 | None = None # With Annotated - type + extra information height: Annotated[ - float64 | None, # The actual type (what kind of data) - Field(description="Height in meters") # Extra metadata + float64 | None, # The actual type (what kind of data) + Field(description="Height in meters"), # Extra metadata ] = None ``` @@ -374,7 +570,7 @@ height: Annotated[ 1. **First argument**: The actual type (`str`, `int32`, `list[str]`, etc.) 2. **Additional arguments**: Metadata like constraints, descriptions, validation rules -#### Field Constraints +##### Field Constraints Use Pydantic's `Field()` function to add constraints and descriptions: @@ -384,17 +580,16 @@ Use Pydantic's `Field()` function to add constraints and descriptions: from typing import Annotated from pydantic import Field + class Building(OvertureFeature): # Range constraints height: Annotated[ - float64 | None, - Field(ge=0, le=1000, description="Height in meters (0-1000m)") + float64 | None, Field(ge=0, le=1000, description="Height in meters (0-1000m)") ] = None # Integer constraints floors: Annotated[ - int32 | None, - Field(gt=0, lt=200, description="Number of floors (1-199)") + int32 | None, Field(gt=0, lt=200, description="Number of floors (1-199)") ] = None ``` @@ -412,13 +607,12 @@ class Place(OvertureFeature): # Length constraints name: Annotated[ str | None, - Field(min_length=1, max_length=100, description="Place name (1-100 chars)") + Field(min_length=1, max_length=100, description="Place name (1-100 chars)"), ] = None # Pattern matching postal_code: Annotated[ - str | None, - Field(pattern=r"^\d{5}(-\d{4})?$", description="US postal code") + str | None, Field(pattern=r"^\d{5}(-\d{4})?$", description="US postal code") ] = None ``` @@ -428,9 +622,9 @@ class Place(OvertureFeature): - **`max_length`**: Maximum string length - **`pattern`**: Regular expression pattern (regex) -### Collections and Lists +#### Collections and Lists -#### Basic List Fields +##### Basic List Fields ```python class Building(Feature): @@ -441,17 +635,18 @@ class Building(Feature): access_rules: list[AccessRule] | None = None ``` -#### List Constraints +##### List Constraints ```python from overture.schema.system.field_constraint import UniqueItemsConstraint + class Building(OvertureFeature): # List with size and uniqueness constraints categories: Annotated[ list[str] | None, Field(min_length=1, max_length=10, description="1-10 categories"), - UniqueItemsConstraint() # Must come AFTER Field() + UniqueItemsConstraint(), # Must come AFTER Field() ] = None ``` @@ -468,11 +663,11 @@ class Building(OvertureFeature): > > **Why**: Pydantic processes annotations in order for JSON Schema generation. `Field()` must come first to set up the field properly. For lists, `Field(min_length=1)` creates a `minItems` constraint in the JSON Schema because the type immediately before it is a list. If `UniqueItemsConstraint()` comes first, Pydantic doesn't see the list type and treats `min_length` as a string constraint (`minLength`). -#### List Behavior +##### List Behavior Lists maintain their **insertion order** (the order data exists in the field), but they are **not automatically sorted**. -### Enumerations +#### Enumerations **What is an enumeration (enum)?** An enumeration is a way to define a fixed set of allowed values for a field. Think of it like a multiple-choice question - you define all the valid answers ahead of time, and users can only pick from those options. @@ -480,13 +675,14 @@ For example, instead of allowing any string for a "status" field (which could le **Enums vs Literal:** You can achieve similar results with `Literal["active", "inactive", "pending"]`, but formal enums are better when you need descriptions, documentation, or want to reuse the same set of values across multiple fields. -#### Creating Enums +##### Creating Enums Enums define a fixed set of allowed values: ```python from enum import Enum + class BuildingClass(str, Enum): """Further delineation of the building's built purpose.""" @@ -495,12 +691,13 @@ class BuildingClass(str, Enum): INDUSTRIAL = "industrial" CIVIC = "civic" + # Usage in a model class Building(OvertureFeature): class_: Annotated[BuildingClass | None, Field(alias="class")] = None ``` -#### Documenting Enum Values +##### Documenting Enum Values Add documentation to describe what the enum and its values mean. In Python, you do this with **docstrings** - text enclosed in triple quotes `"""` that describes what something does: @@ -509,6 +706,7 @@ Use `DocumentedEnum` from `overture.schema.system.doc` when enum members need th ```python from overture.schema.system.doc import DocumentedEnum + class VehicleType(str, DocumentedEnum): """Types of vehicles for transportation.""" @@ -532,17 +730,17 @@ class ConnectionState(str, DocumentedEnum): Use `DocumentedEnum` over plain `str, Enum` when the enum members' semantics aren't obvious from their names and downstream tools (code generators, documentation renderers) need access to member-level descriptions. Use plain `str, Enum` for self-explanatory values. -#### Why str, Enum? +##### Why str, Enum? Inheriting from `str, Enum` makes enum values work as both enums and strings, which is useful for JSON serialization and compatibility. --- -## Advanced Patterns +### Advanced Patterns -### Relationship Patterns +#### Relationship Patterns -#### What are relationships? +##### What are relationships? Relationships represent connections between different features or models. Think of them like links that connect related pieces of information — for example, a building part that is structurally part of a building, or a division area that is administratively nested under a division. @@ -550,11 +748,11 @@ Pydantic provides several ways to express these relationships, each suited to di --- -#### Semantic Relationship Types +##### Semantic Relationship Types Every relationship between two features carries a semantic meaning about coupling strength, lifecycle dependency, and ownership. The schema defines four relationship types, ordered from strongest to weakest coupling. The types describe the *nature* of the link, not which feature is "parent" or "child." Direction is implicit: the feature holding the reference is the source, and the type it references is the destination. -##### `COMPOSITION` — Structural Whole-Part +###### `COMPOSITION` — Structural Whole-Part A structural whole-part relationship with lifecycle dependency. The part has no independent meaning outside the whole. Deleting the whole invalidates the part. @@ -564,7 +762,7 @@ A structural whole-part relationship with lifecycle dependency. The part has no - `BuildingPart` → `Building` — part *is part of* building - `DivisionBoundary` → `Division` — boundary line *defines the boundary of* division -##### `AGGREGATION` — Grouping Without Lifecycle Dependency +###### `AGGREGATION` — Grouping Without Lifecycle Dependency A grouping or collection relationship where both members are independently viable. No lifecycle dependency — the member survives reassignment to another group or orphaning. @@ -574,7 +772,7 @@ A grouping or collection relationship where both members are independently viabl - `Route` → `Segment` — route *groups* segments - `TrailSegment` → `NationalPark` — segment *is grouped by* park -##### `HIERARCHY` — Organizational Nesting +###### `HIERARCHY` — Organizational Nesting An organizational or classificatory nesting relationship. This is not about structural assembly — it's about administrative parentage, taxonomy, or categorization. @@ -584,7 +782,7 @@ An organizational or classificatory nesting relationship. This is not about stru - `DivisionArea` → `Division` — area *is child of* division - `Division` → `Division` — child division nested under parent -##### `ASSOCIATION` — Peer-Level Reference +###### `ASSOCIATION` — Peer-Level Reference A peer-level reference with no ownership, containment, or nesting. Neither feature depends on or contains the other. This is the fallback when none of the stronger types apply. @@ -596,7 +794,7 @@ A peer-level reference with no ownership, containment, or nesting. Neither featu --- -#### Selection Priority +##### Selection Priority When a relationship could fit multiple types, the choice follows a **diamond decision**: start at the top, fork in the middle based on the *kind* of coupling, and fall through to the bottom only when no stronger type applies. @@ -618,7 +816,7 @@ AGGREGATION HIERARCHY --- -#### The `role` Field +##### The `role` Field The `Reference` annotation accepts an optional `role` parameter — a snake_case string that further qualifies the relationship from the source's perspective. It has no effect on schema validation; it is informational metadata for documentation and tooling. @@ -630,17 +828,21 @@ parent_division_id: Annotated[Id, Reference(Relationship.HIERARCHY, Division)] capital_division_ids: Annotated[list[Id], Reference(Relationship.HIERARCHY, Division)] # With role: unambiguous -parent_division_id: Annotated[Id, Reference(Relationship.HIERARCHY, Division, role="child_of")] -capital_division_ids: Annotated[list[Id], Reference(Relationship.HIERARCHY, Division, role="has_as_capital")] +parent_division_id: Annotated[ + Id, Reference(Relationship.HIERARCHY, Division, role="child_of") +] +capital_division_ids: Annotated[ + list[Id], Reference(Relationship.HIERARCHY, Division, role="has_as_capital") +] ``` The `role` must be a non-empty snake_case string (lowercase letters, digits, underscores). It describes the source's role relative to the target using source-perspective phrasing. --- -#### Implementation Patterns +##### Implementation Patterns -##### 1. Direct References (Foreign Keys) +###### 1. Direct References (Foreign Keys) The fundamental pattern is a direct reference where one feature "points to" another using an ID field with type safety and semantic information. @@ -650,34 +852,39 @@ from pydantic import Field from overture.schema.common import OvertureFeature from overture.schema.system.ref import Id, Reference, Relationship + # COMPOSITION — part points to its whole class BuildingPart(OvertureFeature[Literal["buildings"], Literal["building_part"]]): """A structural part of a building.""" + building_id: Annotated[ Id, Reference(Relationship.COMPOSITION, Building, role="part_of"), - Field(description="The building to which this part belongs") + Field(description="The building to which this part belongs"), ] + # HIERARCHY — child points to parent class DivisionArea(OvertureFeature[Literal["divisions"], Literal["division_area"]]): """Area polygon nested under a division.""" + division_id: Annotated[ Id, Reference(Relationship.HIERARCHY, Division, role="child_of"), - Field(description="Division ID of the parent division of this area.") + Field(description="Division ID of the parent division of this area."), ] + # ASSOCIATION — peer reference, no ownership class ConnectorReference(BaseModel): """Reference to a connector feature.""" + connector_id: Annotated[ - Id, - Reference(Relationship.ASSOCIATION, Connector, role="connects_to") + Id, Reference(Relationship.ASSOCIATION, Connector, role="connects_to") ] ``` -##### 2. Association as a Separate Feature (Complex Relationships) +###### 2. Association as a Separate Feature (Complex Relationships) When the relationship itself needs to store information, create a dedicated feature to represent it. This applies regardless of the semantic type — any of the four types can carry metadata. @@ -693,14 +900,8 @@ class AdminCityCenterAssociation( ): """Describes how an administrative area relates to a city center.""" - admin_area_id: Annotated[ - Id, - Reference(Relationship.ASSOCIATION, AdminArea) - ] - city_center_id: Annotated[ - Id, - Reference(Relationship.ASSOCIATION, CityCenter) - ] + admin_area_id: Annotated[Id, Reference(Relationship.ASSOCIATION, AdminArea)] + city_center_id: Annotated[Id, Reference(Relationship.ASSOCIATION, CityCenter)] # Information about the relationship itself relationship_type: Literal["primary_center", "secondary_center"] = "primary_center" @@ -713,22 +914,31 @@ class AdminCityCenterAssociation( - Many-to-many connections exist. - You need to query the relationships independently. -##### 3. Collection References +###### 3. Collection References When a feature needs to reference multiple other features, use a list of references. The semantic type still matters. ```python # COMPOSITION — boundary defines two divisions -class DivisionBoundary(OvertureFeature[Literal["divisions"], Literal["division_boundary"]]): +class DivisionBoundary( + OvertureFeature[Literal["divisions"], Literal["division_boundary"]] +): """A boundary line between two divisions.""" + division_ids: Annotated[ - list[Annotated[Id, Reference(Relationship.COMPOSITION, Division, role="boundary_of")]], + list[ + Annotated[ + Id, Reference(Relationship.COMPOSITION, Division, role="boundary_of") + ] + ], Field(min_length=2, max_length=2, description="Left and right divisions"), ] + # AGGREGATION — route groups segments class Route(OvertureFeature[Literal["transportation"], Literal["route"]]): """A transportation route passing through multiple segments.""" + segment_ids: Annotated[ list[Id], Reference(Relationship.AGGREGATION, TransportationSegment, role="groups"), @@ -739,9 +949,9 @@ class Route(OvertureFeature[Literal["transportation"], Literal["route"]]): --- -#### Best Practices +##### Best Practices -##### Always Use Reference Annotations +###### Always Use Reference Annotations Include `Reference` annotations for semantic clarity and documentation: @@ -750,14 +960,14 @@ Include `Reference` annotations for semantic clarity and documentation: division_id: Annotated[ Id, Reference(Relationship.HIERARCHY, Division, role="child_of"), - Field(description="Division ID of the parent division of this area.") + Field(description="Division ID of the parent division of this area."), ] # Avoid — missing semantic information division_id: Id ``` -##### Choose the Right Semantic Type First, Then the Right Pattern +###### Choose the Right Semantic Type First, Then the Right Pattern 1. **Determine the semantic type** using the selection priority and test questions above. 2. **Then choose the implementation pattern:** @@ -765,7 +975,7 @@ division_id: Id - Relationships with metadata → Separate association features (Pattern 2) - One-to-many references → Collection references (Pattern 3) -### Discriminated Unions +#### Discriminated Unions **What is a discriminated union?** A discriminated union is a type that can be backed by one of several different models, where a specific field (the "discriminator") determines which model it actually is. Think of it like a form that changes its fields based on a category selection. @@ -774,11 +984,15 @@ from typing import Annotated, Literal from pydantic import Field from overture.schema.common import OvertureFeature + # Base class with common fields -class TransportationSegment(OvertureFeature[Literal["transportation"], Literal["segment"]]): +class TransportationSegment( + OvertureFeature[Literal["transportation"], Literal["segment"]] +): subtype: Subtype # This is the discriminator field # ... common fields for all segments + # Specific segment types class RoadSegment(TransportationSegment): subtype: Literal[Subtype.ROAD] # Must be "road" @@ -786,22 +1000,23 @@ class RoadSegment(TransportationSegment): speed_limits: SpeedLimits | None = None # ... road-specific fields + class RailSegment(TransportationSegment): subtype: Literal[Subtype.RAIL] # Must be "rail" class_: Annotated[RailClass, Field(alias="class")] rail_flags: RailFlags | None = None # ... rail-specific fields + # Union type that automatically picks the right model based on subtype Segment = Annotated[ - RoadSegment | RailSegment | WaterSegment, - Field(discriminator="subtype") + RoadSegment | RailSegment | WaterSegment, Field(discriminator="subtype") ] ``` The `discriminator="subtype"` tells Pydantic to look at the `subtype` field to determine which specific model to use. If `subtype` is "road", it uses `RoadSegment`; if "rail", it uses `RailSegment`. -#### Abstract vs Concrete Classes +##### Abstract vs Concrete Classes **What's the difference?** In UML and traditional OOP, abstract classes cannot be instantiated - they serve as templates for concrete classes. In Pydantic, by default, **all classes are concrete** (can be instantiated), but you can make classes abstract when needed. @@ -820,7 +1035,10 @@ from abc import ABC, abstractmethod from typing import Annotated, Literal from pydantic import Field -class TransportationSegment(OvertureFeature[Literal["transportation"], Literal["segment"]], ABC): + +class TransportationSegment( + OvertureFeature[Literal["transportation"], Literal["segment"]], ABC +): """Abstract base - cannot be instantiated directly.""" subtype: Subtype # Discriminator field @@ -830,24 +1048,27 @@ class TransportationSegment(OvertureFeature[Literal["transportation"], Literal[" """Each concrete type must implement this.""" pass + class RoadSegment(TransportationSegment): """Concrete class - can be instantiated.""" + subtype: Literal[Subtype.ROAD] speed_limits: SpeedLimits | None = None def get_speed_limit(self) -> float: return self.speed_limits.max_speed if self.speed_limits else 50.0 + # Now only concrete classes can be instantiated # base_segment = TransportationSegment(...) # TypeError: Can't instantiate abstract class -road_segment = RoadSegment(subtype=Subtype.ROAD, ...) # Valid +road_segment = RoadSegment(subtype=Subtype.ROAD, geometry=...) # Valid ``` **Registration pattern (recommended when working with Overture models):** Instead of making classes abstract, we use **entry point registration** where only specific concrete types are discoverable as map features: -```python +```toml # In packages/overture-schema-theme-transportation/pyproject.toml [project.entry-points."overture.models"] connector = "overture.schema.transportation:Connector" @@ -866,7 +1087,7 @@ segment = "overture.schema.transportation:Segment" 2. The union automatically resolves to the correct concrete type based on the `subtype` field 3. All classes (`TransportationSegment`, `RoadSegment`, etc.) can be reused as base classes for alternate implementations -### Pattern Properties (Constrained Key-Value Maps) +#### Pattern Properties (Constrained Key-Value Maps) **What are pattern properties?** Pattern properties let you create key-value maps where the keys must follow a specific pattern (like language codes) and values have specific types. @@ -874,25 +1095,29 @@ segment = "overture.schema.transportation:Segment" from typing import Annotated from pydantic import BaseModel, Field + @no_extra_fields class Names(BaseModel): primary: str # Keys (strings) must match a language tag pattern, values are strings - common: Annotated[ - dict[ - # The key type - Annotated[ - str, - Field( - pattern=r"^[a-z]{2,3}(-[A-Z]{2})?$", - description="Language tag (e.g., 'en', 'es-MX')" - ) + common: ( + Annotated[ + dict[ + # The key type + Annotated[ + str, + Field( + pattern=r"^[a-z]{2,3}(-[A-Z]{2})?$", + description="Language tag (e.g., 'en', 'es-MX')", + ), + ], + str, # The value type ], - str, # The value type - ], - Field(json_schema_extra={"additionalProperties": False}), - ] | None = None + Field(json_schema_extra={"additionalProperties": False}), + ] + | None + ) = None ``` **Example data:** @@ -910,7 +1135,7 @@ class Names(BaseModel): The `additionalProperties: False` ensures only keys matching the pattern are allowed when generated JSON Schema is used. -### Nested List Validation +#### Nested List Validation **What is nested list validation?** This pattern validates both the outer list and the inner structure of each item, with constraints at multiple levels. @@ -918,22 +1143,24 @@ The `additionalProperties: False` ensures only keys matching the pattern are all from typing import Annotated from pydantic import Field + # Each item has its own field validation @no_extra_fields class HierarchyItem(BaseModel): division_id: str name: str + class Division(OvertureFeature): # Nested list validation: outer list AND inner lists both have length constraints hierarchies: Annotated[ list[ # Outer list Annotated[ list[HierarchyItem], # Inner list - Field(min_length=1) # Inner list must have at least 1 item + Field(min_length=1), # Inner list must have at least 1 item ] ], - Field(min_length=1) # Outer list must have at least 1 hierarchy + Field(min_length=1), # Outer list must have at least 1 hierarchy ] ``` @@ -943,7 +1170,7 @@ This creates validation at three levels: 2. **Inner lists**: Each inner list must have at least 1 item (`min_length=1`) 3. **Outer list**: The `hierarchies` field must have at least 1 inner list (`min_length=1`) -### Type Aliases for Reusable Patterns +#### Type Aliases for Reusable Patterns **What are type aliases?** Type aliases let you create custom names for complex or frequently-used types. Think of them like creating shortcuts or nicknames for long type definitions. @@ -969,10 +1196,11 @@ SegmentId = NewType("SegmentId", str) # IDs are strings, but distinct CountryCode = NewType("CountryCode", str) # Country codes are strings, but distinct # Create aliases for complex field patterns -EmailList = NewType("EmailList", Annotated[ - list[str], - Field(min_length=1, description="List of email addresses") -]) +EmailList = NewType( + "EmailList", + Annotated[list[str], Field(min_length=1, description="List of email addresses")], +) + @no_extra_fields class Contact(BaseModel): @@ -991,82 +1219,115 @@ class Contact(BaseModel): --- -## Integration Guide +### Integration Guide -### Project Architecture +#### Project Architecture -#### File Organization +##### File Organization -Organize code by scope and avoid circular imports: +Organize code by scope, and avoid circular imports. -**Cross-theme shared**: `overture-schema-common` package +**Cross-theme shared**: the `overture-schema-common` package. Definitions more than +one theme needs -- `OvertureFeature`, `Names`, `Sources`, the scoping framework. -- Used by multiple themes (e.g., `OvertureFeature`, `Names`, `Sources`, `Scope`) +**One module per feature type**: at the theme package root, named after the type in +snake_case. -**Theme-level shared**: Theme package root (e.g., `overture-schema-theme-transportation/src/overture/schema/transportation/`) +```text +packages/overture-schema-theme-buildings/src/overture/schema/buildings/ + __init__.py # re-exports the public names, declares __all__ + _common.py # shared by Building and BuildingPart + building.py # Building, BuildingSubtype, BuildingClass + building_part.py # BuildingPart +``` -- Used by multiple types within a theme (e.g., `AccessRules`, `RoadSurface`) +The module owns everything specific to its type: the `Feature` subclass, its enums, +its NewTypes, and its supporting models. `building.py` defines `BuildingSubtype` and +`BuildingClass` next to `Building`, because nothing else uses them. -**Type-specific**: Type subdirectory (e.g., `overture-schema-theme-transportation/src/overture/schema/transportation/segment/`) +**Theme-level shared**: `_common.py` at the theme package root, for definitions two +or more types in the theme need. `buildings/_common.py` holds `Appearance`, +`RoofShape`, and the material enums that both `Building` and `BuildingPart` use. The +leading underscore marks the module private -- the theme's `__init__.py` re-exports +the public names from it. -- Only used by one specific type (e.g., `SegmentType`, `LaneConfiguration`) +**A type large enough to split**: a subpackage named after the type, applying the +same rules one level down. -**File type rules:** +```text +packages/overture-schema-theme-transportation/src/overture/schema/transportation/ + __init__.py # re-exports from connector and segment + connector.py # Connector + segment/ + __init__.py # assembles the Segment discriminated union, re-exports + _common.py # TransportationSegment and what the arms share + road.py # RoadSegment and its supporting types + rail.py # RailSegment and its supporting types + water.py # WaterSegment +``` -- **`models.py`**: Pydantic model classes (and type aliases that reference models) -- **`enums.py`**: Enum classes only, no project imports -- **`types.py`**: Type aliases that don't reference models, no project imports +Every `__init__.py` re-exports its public names and declares `__all__`, so consumers +import from the package rather than reaching into the defining module: +`from overture.schema.buildings import Building`, not +`from overture.schema.buildings.building import Building`. Entry points name the +package root for the same reason -- `building = "overture.schema.buildings:Building"`. -#### Import Organization +There is no `models.py` / `enums.py` / `types.py` split. An enum lives in the module +whose type uses it, and moves up to `_common.py` when a second type needs it. + +##### Import Organization ```python # Standard library imports first -from typing import Annotated, Literal, NewType from enum import Enum +from typing import Annotated, Literal, NewType # Third-party imports from pydantic import BaseModel, ConfigDict, Field -# Cross-theme imports -from overture.schema.common import OvertureFeature +# Cross-theme and system imports +from overture.schema.common.scoping import Heading, Scope, scoped +from overture.schema.system.doc import DocumentedEnum from overture.schema.system.field_constraint import UniqueItemsConstraint from overture.schema.system.model_constraint import no_extra_fields -# Local imports last -from .enums import SegmentType -from .types import SegmentId, LaneWidth # Only non-model type aliases +# Local imports last -- siblings in the theme, then the module's own package +from ..connector import Connector +from ._common import SegmentSubtype, TransportationSegment ``` `uv run ruff format ` will sort your imports in this order automatically. -#### Why Not Use @field_validator or @model_validator? +##### Why Not Use @field_validator or @model_validator? This project uses a custom validation system that generates better JSON Schema output and supports code generation (without additional work, `@field_validator` and `@model_validator` don't make their constraints discoverable). Always use constraints from `overture.schema.system` instead of using Pydantic validation decorators: ```python # Don't do this -@field_validator('categories') +@field_validator("categories") def validate_categories_unique(cls, v): if v and len(v) != len(set(v)): - raise ValueError('Categories must be unique') + raise ValueError("Categories must be unique") return v + # Do this instead from overture.schema.system.field_constraint import UniqueItemsConstraint + class Building(OvertureFeature): categories: Annotated[ list[str] | None, Field(min_length=1, description="Building categories"), - UniqueItemsConstraint() + UniqueItemsConstraint(), ] = None ``` -### Migrating from JSON Schema +#### Migrating from JSON Schema If you're familiar with JSON Schema files (like `schema/schema.yaml`), this section helps translate those patterns to Pydantic models. -#### How $defs and $ref Translate +##### How $defs and $ref Translate **JSON Schema approach:** @@ -1092,9 +1353,11 @@ properties: @no_extra_fields class Address(BaseModel): """A postal address.""" + freeform: str | None = None locality: str | None = None + # In overture-schema-theme-buildings/src/overture/schema/buildings/building.py class Building(OvertureFeature): address: Address | None = None @@ -1106,7 +1369,7 @@ class Building(OvertureFeature): - JSON Schema definitions live in `$defs`; Pydantic models are regular Python classes grouped into modules - JSON Schema allows inline definitions; Pydantic encourages separate model classes -#### How Containers Work +##### How Containers Work **JSON Schema containers** (like `namesContainer`, `shapeContainer`) are reusable property groups: @@ -1131,25 +1394,36 @@ allOf: **Pydantic equivalent** uses **mixin classes**: ```python -# In common/names.py +# namesContainer -> Named, in +# overture-schema-common/src/overture/schema/common/names.py class Named(BaseModel): """Properties defining the names of a feature.""" + names: Names | None = None -# In buildings/models.py -class Shape(BaseModel): - """Properties of the building's shape.""" + +# shapeContainer -> Appearance, in buildings/_common.py, +# shared by Building and BuildingPart +class Appearance(BaseModel): + """Physical and visual properties of a building.""" + height: float64 | None = None num_floors: int32 | None = None + # ... roof and facade fields + -# Usage with multiple inheritance -class Building(Feature, Named, Shape): - pass # "pass" means "do nothing" - Building inherits names, height, num_floors, etc. from its parents +# Usage with multiple inheritance -- this is how Building is actually declared +class Building( + OvertureFeature[Literal["buildings"], Literal["building"]], + Named, + Stacked, + Appearance, +): ... # inherits names, level, height, num_floors, and the rest from its parents ``` JSON Schema containers become **mixin classes** in Pydantic that you inherit from. -#### Common Translation Patterns +##### Common Translation Patterns | JSON Schema | Pydantic | Notes | |-------------|----------|-------| @@ -1164,18 +1438,53 @@ JSON Schema containers become **mixin classes** in Pydantic that you inherit fro --- -## Reference -### Complete Templates +--- + +## Development workflow + -#### Basic Model Template +This project uses [uv](https://docs.astral.sh/uv/) for dependency management: -```python models.py +```bash +# Install dependencies for the entire workspace +uv sync --all-packages + +# Run all tests and type/code quality checks +make check + +# Run tests for a specific package +uv run pytest packages/overture-schema-theme-buildings/ + +# Run tests matching a pattern +uv run pytest -k "buildings" +``` + +Auto-format / fix code to align with project expectations: + +```shell +uv run ruff check --fix +uv run ruff format +uv run docformatter --in-place --recursive packages/ +``` + +--- + +## Templates and quick reference + +### Reference + +#### Complete Templates + +##### Basic Model Template + +```python from typing import Annotated from pydantic import BaseModel, Field from overture.schema.system.model_constraint import no_extra_fields from overture.schema.system.numeric import int8, float64 + @no_extra_fields class MyCustomType(BaseModel): """Brief description of what this represents.""" @@ -1191,20 +1500,23 @@ class MyCustomType(BaseModel): priority: Annotated[ int8 | None, Field( - ge=1, - le=10, - description="Priority level from 1 (lowest) to 10 (highest)" - ) + ge=1, le=10, description="Priority level from 1 (lowest) to 10 (highest)" + ), ] = None ``` -#### Feature Template +##### Feature Template -```python models.py +```python from typing import Annotated, Literal from pydantic import Field from overture.schema.common import OvertureFeature -from overture.schema.system.geometric import Geometry, GeometryType, GeometryTypeConstraint +from overture.schema.system.geometric import ( + Geometry, + GeometryType, + GeometryTypeConstraint, +) + class MyFeature(OvertureFeature[Literal["my_theme"], Literal["my_type"]]): """Description of what this feature represents.""" @@ -1220,11 +1532,12 @@ class MyFeature(OvertureFeature[Literal["my_theme"], Literal["my_type"]]): my_field: str | None = None ``` -#### Enum Template +##### Enum Template -```python enums.py +```python from enum import Enum + class MyEnum(str, Enum): """Description of what this enum represents.""" @@ -1233,14 +1546,15 @@ class MyEnum(str, Enum): VALUE_THREE = "value_three" ``` -#### Model with Validation Constraints +##### Model with Validation Constraints -```python models.py +```python from typing import Annotated from pydantic import BaseModel, Field from overture.schema.system.field_constraint import UniqueItemsConstraint from overture.schema.system.model_constraint import no_extra_fields + @no_extra_fields class Contact(BaseModel): """Contact information with validation constraints.""" @@ -1253,33 +1567,39 @@ class Contact(BaseModel): tags: Annotated[ list[str] | None, Field(min_length=1, description="Contact tags"), - UniqueItemsConstraint() # No duplicate tags + UniqueItemsConstraint(), # No duplicate tags ] = None ``` -#### Association Feature Template +##### Association Feature Template -```python models.py +```python from typing import Annotated, Literal from pydantic import Field from overture.schema.common import OvertureFeature from overture.schema.system.numeric import float64 from overture.schema.system.ref import Id, Reference, Relationship -class MyAssociation(OvertureFeature[Literal["associations"], Literal["my_association"]]): + +class MyAssociation( + OvertureFeature[Literal["associations"], Literal["my_association"]] +): """Represents a relationship between two features with metadata.""" # References to the associated features + # Relationship takes a *kind* (COMPOSITION / AGGREGATION / HIERARCHY / + # ASSOCIATION); what the reference means is carried by `role`. Two + # references to related features need distinct roles to stay unambiguous. feature_a_id: Annotated[ Id, - Reference(Relationship.CONNECTS_TO, FeatureA), - Field(description="First feature in the relationship") + Reference(Relationship.ASSOCIATION, FeatureA, role="connects_from"), + Field(description="First feature in the relationship"), ] feature_b_id: Annotated[ Id, - Reference(Relationship.CONNECTS_TO, FeatureB), - Field(description="Second feature in the relationship") + Reference(Relationship.ASSOCIATION, FeatureB, role="connects_to"), + Field(description="Second feature in the relationship"), ] # Relationship metadata @@ -1290,27 +1610,30 @@ class MyAssociation(OvertureFeature[Literal["associations"], Literal["my_associa notes: str | None = None ``` -### Quick Reference +#### Quick Reference -#### Essential Patterns (Most Common) +##### Essential Patterns (Most Common) ```python # Basic field types -name: str # Required string -name: str | None = None # Optional string -count: int32 # Required integer +name: str # Required string +name: str | None = None # Optional string +count: int32 # Required integer priority: Literal["high", "medium", "low"] | None = None # Constrained values # Validated fields height: Annotated[float64 | None, Field(ge=0, description="Height in meters")] = None tags: Annotated[list[str] | None, Field(min_length=1), UniqueItemsConstraint()] = None -# Association patterns -parent_id: Annotated[Id | None, Reference(Relationship.BELONGS_TO, ParentModel)] = None +# Association patterns -- Relationship is the kind, role is the meaning +parent_id: Annotated[ + Id | None, + Reference(Relationship.HIERARCHY, ParentModel, role="child_of"), +] = None connector_ids: list[Id] # References to multiple related features ``` -#### Model Templates +##### Model Templates ```python # Non-feature model @@ -1319,18 +1642,20 @@ class Address(BaseModel): street: str city: str | None = None + # Feature model class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): geometry: Geometry height: float64 | None = None + # Enum class Status(str, Enum): ACTIVE = "active" INACTIVE = "inactive" ``` -#### Constraint Reference +##### Constraint Reference | Type | Constraint | JSON Schema | Example | |------|------------|-------------|---------| @@ -1339,7 +1664,7 @@ class Status(str, Enum): | **List** | `min_length=1, UniqueItemsConstraint()` | `minItems`, `uniqueItems` | `Field(min_length=1), UniqueItemsConstraint()` | | **Custom** | `LanguageTagConstraint()` | Custom validation | `LanguageTagConstraint()` | -#### Import Cheatsheet +##### Import Cheatsheet ```python # Essential imports for most models @@ -1355,7 +1680,7 @@ from overture.schema.system.numeric import int32, float64 from overture.schema.system.ref import Id, Reference, Relationship ``` -#### Naming Conventions +##### Naming Conventions - **Classes**: `PascalCase` (`Building`, `AccessRule`) - **Fields**: `snake_case` (`construction_year`, `has_parts`) diff --git a/CONCEPTS.md b/CONCEPTS.md new file mode 100644 index 000000000..8f079400e --- /dev/null +++ b/CONCEPTS.md @@ -0,0 +1,839 @@ +# Concepts + +Background on why the Overture schema looks the way it does. **None of this is needed to +get work done** — [SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) is the path through the examples, +and it stands on its own. Read a section here when you hit something whose *why* you want. + +Not to be confused with [GLOSSARY.md](GLOSSARY.md), which defines the same vocabulary in a +sentence or two each. If you want to know what *envelope* or *workspace* or *tag* means, +the glossary is faster. This page is for why they exist. + +| Question | Section | +|---|---| +| What principles is the schema designed under? | [The tenets](#the-tenets) | +| Why have a schema at all? | [Beyond raw data](#beyond-raw-data) | +| Why Pydantic instead of JSON Schema? | [Why Pydantic](#why-pydantic-rather-than-json-schema) | +| Why is the schema split into a dozen packages? | [Many packages](#why-the-schema-is-many-packages) | +| What is an "envelope"? Why do Overture fields sit under `properties`? | [The GeoJSON envelope](#the-geojson-envelope) | +| Why does `type` appear three times in one file? | [Three keys named `type`](#three-keys-named-type) | +| If the schema is Pydantic now, why does it still look like GeoJSON? | [Why there's an envelope at all](#why-theres-an-envelope-at-all) | +| Why can a model be named two ways? | [Two names per model](#two-names-per-model) | +| What decides whether a field is required? | [What makes a field required](#what-makes-a-field-required) | +| What are those `overture:theme=` tags? | [How tags work](#how-tags-work) | +| What is the example file the guide validates? | [The example file](#what-the-example-file-actually-is) | +| Why are the examples YAML? | [Why examples are YAML](#why-examples-are-yaml) | +| Why isn't the generated PySpark code in git? | [Generated code](#why-generated-code-is-gitignored) | + +--- + +## The tenets + +Six principles the schema is designed under, set down by the working group early in the +project under the heading *"These are our tenets unless you know better ones"*. They still +decide arguments, so they are worth knowing before you propose a change. + +1. **Address the core, enable the periphery.** The Overture schema doesn't solve every + problem. It describes fully-formed solutions only for the most fundamental use cases + ("the core") while enabling less common use cases ("the periphery") via extensibility. +2. **Invent across the gap.** Many excellent solutions — published standards, best + practices, open-source tools — already exist and are well understood in the community. + The Overture schema reuses them to maximize compatibility and to focus effort on + unaddressed high-priority pain points. +3. **Backward-compatible is forward-compatible.** No design is future-proof, but good + designs stay relevant by adding features without breaking existing use cases. +4. **The world is neither flat nor still.** The Overture schema links representations of + 2- and 3-dimensional objects in space and time. +5. **Empower, don't dictate.** The schema provides a framework that lets users bring + together the data they need for their own use cases — Overture and non-Overture + sources alike — according to their own viewpoints and perspectives. +6. **Always open, never closed.** The schema and format aim for compatibility with free + and open-source tools, and avoid depending on closed-source or proprietary ones. + +Where they show up in this repository: + +| Tenet | In practice | +|---|---| +| Address the core, enable the periphery | Your own feature types register through entry points and become first-class — nothing in the tooling special-cases Overture. See [AUTHORING.md](AUTHORING.md#register-your-own-feature-types). | +| Invent across the gap | JSON Schema, OGC geometries, GeoJSON, GeoParquet, and Pydantic are all reused rather than reinvented. See [Why Pydantic](#why-pydantic-rather-than-json-schema) and [The GeoJSON envelope](#the-geojson-envelope). | +| Backward-compatible is forward-compatible | Backward-compatible changes land in both the Pydantic models and the deprecated YAML while both are live; major changes wait for `vnext`. See [CONTRIBUTING.md](CONTRIBUTING.md). | +| Always open, never closed | Every published artifact — JSON Schema, PySpark expressions, documentation — is generated by open tooling in this repository. | + +The doctrine the working group built on these tenets in 2023 is recorded in the +[project history](README.md#the-tenets-and-the-doctrine). + +## Why this exists + +This project provides type-safe Python models for validating and working with +[Overture](https://overturemaps.org/) data. Use these schemas to: + +- Validate Overture data +- Build data processing pipelines with type safety +- Extend schemas with custom fields and validation rules + +## Beyond raw data + +This project addresses a fundamental challenge in data consumption: **bridging the +semantic gap between raw data and human understanding** while enabling +machine-actionable workflows. + + +Take a column like `pop_2020`. Is it total population? Population density per square +kilometer? Working-age population? Without a schema, you're left sampling values and +guessing from column names. + +Compare this to OpenStreetMap's approach: features use well-known key/value pairs like +`building=residential` or `addr:housenumber=42` that have semantic meaning and can be +looked up on the OSM wiki. This creates a step toward a schema - shared vocabulary with +documented semantics used across a vast dataset. However, OSM tags remain free-form: +multiple valid ways to express the same concept, no built-in validation, and complex +downstream validation because of undocumented keys that might have meaning to someone, +somewhere. A schema provides the structured alternative: explicit types, clear +validation rules, and semantic meaning that both humans and systems can rely on. + +Data files containing only column names and values aren't fully documented. External +metadata files typically focus on how data was collected and encoded, not on semantic +meaning or validation rules. Data consumers struggle to understand what datasets contain +and which columns they need for their goals. + +## Why Pydantic rather than JSON Schema + +We initially chose JSON Schema because it aligned with our mental model and promised to +solve our problems as we understood them. But JSON Schema surfaced several pain points: + +- **Authoring difficulty**: Hard to write correctly, difficult to verify, limited IDE + support, no refactoring capabilities +- **Tooling gaps**: Generic tools can't tailor output for specific applications like + ours +- **Development friction**: Schema changes required manual coordination across multiple + artifacts + +Pydantic addresses these systematically: author in Python with full IDE support, +generate tailored documentation, and automatically produce the specific artifacts each +workflow needs. Pydantic can also produce JSON Schema, so any application that requires +it can use it while we gain all the Python benefits during authoring. + +## The result + +Instead of spending time deciphering what columns mean and whether data matches +expectations, users can focus on their actual goals: analysis, visualization, +integration. Quality improves because validation happens automatically rather than +through manual inspection. + +The fundamental approach - human-readable authoring that generates machine-actionable +outputs - has broader applications beyond Overture and geospatial data. We hope others +will adapt these patterns for linking with Overture data or modeling their own domains +entirely. + +--- + +## Why the schema is many packages + +> **Why split the schema into so many packages?** Because *what you install determines what +> exists at runtime*. The models register themselves through Python entry points, so +> installing only the buildings theme means the CLI only knows about buildings. That's +> the extension mechanism — see [Using the packages from your own project](SCHEMA_GUIDE.md#7-using-the-packages-from-your-own-project) and +> [Register your own feature types](AUTHORING.md#register-your-own-feature-types). If you just want everything, that's fine too. + +### What "workspace" means + +A **uv workspace** is one repository containing several packages that are developed +together, sharing **one lockfile** and **one virtual environment**. If you've used Cargo +workspaces, npm workspaces, or a monorepo, it's the same idea. + +The root `pyproject.toml` declares it: + +```toml +[project] +name = "overture-schema-workspace" # ← a container, not something you install +version = "0.0.0" + +[tool.uv.workspace] +members = ["packages/*"] # ← every directory under packages/ is a member +``` + +Two things follow from this: + +**1. You never install or import `overture-schema-workspace`.** It's scaffolding. It +exists so `uv` knows which directories are members. There is no `import +overture_schema_workspace`. + +**2. The packages depend on each other *locally*, not through PyPI.** Look at +`packages/overture-schema-cli/pyproject.toml`: + +```toml +[tool.uv.sources] +overture-schema-common = { workspace = true } +overture-schema-system = { workspace = true } +``` + +`workspace = true` means "use the copy in this repo." This is why none of this needs to +be published to PyPI for you to work with it — the packages find each other. + +--- + +## The GeoJSON envelope + +An **envelope** is an outer wrapper that carries a payload plus a little standard +information about it. The term is borrowed from mail: the address and stamp go on the +outside and are the same on every envelope; the letter inside is whatever you wrote. + +GeoJSON works exactly that way. Every GeoJSON Feature has the same four outer keys, fixed +by RFC 7946 — that's the envelope. Everything specific to *your* data goes in one of them, +`properties` — that's the payload. + +```json +{ + "type": "Feature", <- envelope: what kind of object this is + "id": "overture:buildings:building:1234", <- envelope: identity + "geometry": { "type": "Polygon", ... }, <- envelope: where it is + "properties": { <- envelope: the pocket for everything else + "theme": "buildings", payload: Overture's fields + "type": "building", + "height": 21.34, + "num_floors": 4, + "class": "parking" + } +} +``` + +Split them apart for yourself: + +```python +import json, yaml +from overture.schema.buildings import Building + +b = Building.model_validate_json( + json.dumps(yaml.safe_load(open("examples/buildings/building-polygon.yaml")))) +gj = b.model_dump(mode="json", by_alias=True, exclude_none=True) + +print("envelope keys:", sorted(gj)) +print("payload keys :", sorted(gj["properties"])) +``` + +``` +envelope keys: ['geometry', 'id', 'properties', 'type'] +payload keys : ['class', 'ext_bar', 'ext_foo', 'height', 'is_underground', 'level', + 'num_floors', 'num_floors_underground', 'sources', 'subtype', + 'theme', 'type', 'version'] +``` + +Four keys outside, thirteen inside. **A GeoJSON Feature for a road, a lake, or a mailbox +has the same four outer keys** — that's what makes it universally readable. Only the +payload differs. + +So when this guide says "the envelope owns `id` and `geometry`," it means those two are +outer keys, placed there by GeoJSON's rules rather than by Overture's. And "the fields are +nested under the envelope" means the Overture fields sit inside `properties` rather than +at the top. + +One consequence worth carrying forward: **the envelope only exists in the GeoJSON +rendering.** In the Pydantic model, and in Parquet, there is no envelope — `id`, +`geometry`, and `height` are all just fields side by side. + +### Three keys named `type` + +A **key** is a field name — the part left of the colon. This file uses the key `type` +three times, at three nesting levels, meaning three unrelated things: + +| Where | Value | Comes from | Means | +|---|---|---|---| +| top level | `Feature` | GeoJSON spec | "this object is a GeoJSON Feature" | +| inside `geometry` | `Polygon` | GeoJSON spec | "this shape is a polygon" | +| inside `properties` | `building` | Overture | "this feature is a building" | + +List them yourself: + +```python +import yaml +d = yaml.safe_load(open('examples/buildings/building-polygon.yaml')) +print('type =', d['type']) +print('geometry.type =', d['geometry']['type']) +print('properties.type =', d['properties']['type']) +``` + +``` +type = Feature +geometry.type = Polygon +properties.type = building +``` + +Only the third is Overture's. The first two belong to GeoJSON, the envelope Overture data +is wrapped in when written as JSON. Whenever this guide says "the feature's type" it means +`properties.type` — the one holding `building`, `place`, or `segment`. + +That layering is the single most confusing thing about this data, and section 3 is largely +about it. + +**"Validating" means:** the CLI parses the file, reads `theme: buildings` and +`type: building` to decide *which model* to check against, then checks every field +against that model — types, numeric bounds, enum membership, required fields, and +cross-field rules. + +You can watch it pick the model. Delete the `theme:` line and it no longer knows: + +``` +⚠ Ambiguous: Data matches multiple types equally. Consider: + • Specifying --tag or --type to narrow validation + • Adding discriminator fields to clarify intent +``` + +### Why there's an envelope at all + +If the schema is now Pydantic, why does any of this still look like GeoJSON? Because those +two things answer different questions, and only one of them changed. + +| | What it is | Did it change? | +|---|---|---| +| **Pydantic** | How the schema is *authored and enforced*, in Python | **Yes** — it replaced hand-written JSON Schema YAML | +| **GeoJSON** | One way a feature can be *written out as JSON*, per RFC 7946 | **No** — it's an interchange format, not an authoring choice | + +Pydantic replaced JSON Schema as the authoring language. It has nothing to say about how +data is serialized, so GeoJSON was never in scope to replace. + +**But GeoJSON is not how Overture ships bulk data — Parquet is.** The release bucket is +Hive-partitioned Parquet: + +``` +s3a://overturemaps-us-west-2/release/2026-07-22.0/theme=buildings/type=building/ +``` + +That is a columnar table: `id`, `geometry`, `height`, and the rest are *columns*, flat, no +envelope. It's what `overture-validate` reads, what DuckDB attaches to, and what you'd +query for anything at scale. + +So why does GeoJSON appear at all? Because **JSON Schema describes JSON documents**, and +when a geospatial feature is written as a single JSON document, GeoJSON is the format +every GIS tool already reads. Parquet has no JSON representation to describe — it has a +columnar schema instead, which is exactly what +[section 6](SCHEMA_GUIDE.md#6-converting-the-schema-to-other-formats) generates as a Spark `StructType`. + +The honest summary is that there are two serializations and neither is subordinate: + +| Serialization | Shape | Pydantic mode | Where you meet it | +|---|---|---|---| +| **Parquet** | flat / columnar | `python` | the release bucket, Spark, DuckDB — all bulk data | +| **GeoJSON** | nested envelope | `json` | single features, extracts, examples in this repo, web tooling | + +The model supports both deliberately. The JSON Schema you're reading in this subsection +describes the second one, which is the only reason an envelope shows up here at all. + +### Interchange format vs storage format + +An **interchange format** is one whose job is handing data to a program you didn't write. +It's optimized for being *understood by anything*, not for being stored efficiently. A +**storage format** is the opposite: optimized for holding a lot of data and querying it +fast, at the cost of needing specific software to read it at all. + +| | GeoJSON | Parquet | +|---|---|---| +| Optimized for | being read by anything | storing and scanning millions of rows | +| Text or binary | plain text | binary, columnar, compressed | +| Read it with | any JSON parser | a Parquet library | +| Self-describing | yes — the file says what it is | schema in the footer, not human-readable | +| Good at | one feature, an extract, a web map | a whole theme, a whole planet | + +The cost of being universally readable is that GeoJSON repeats every field name on every +feature: + +```python +import json, yaml +from overture.schema.buildings import Building + +b = Building.model_validate_json( + json.dumps(yaml.safe_load(open("examples/buildings/building-polygon.yaml")))) +gj = b.model_dump(mode="json", by_alias=True, exclude_none=True) +flat = b.model_dump(mode="python", by_alias=True, exclude_none=True) +flat["geometry"] = str(flat["geometry"]) + +n = 10_000 +doc = json.dumps({"type": "FeatureCollection", "features": [gj] * n}) +cols = list(flat) +tbl = json.dumps({"columns": cols, "rows": [[flat[k] for k in cols]] * n}) +print(f"{n:,} features as GeoJSON : {len(doc):>10,} bytes") +print(f"{n:,} features, names stored once: {len(tbl):>10,} bytes") +``` + +``` +10,000 features as GeoJSON : 7,040,043 bytes +10,000 features, names stored once: 4,520,199 bytes +``` + +A 1.56x penalty before compression even enters the picture, and that comparison is still +generous to GeoJSON — real Parquet also compresses each column and lets a reader skip +columns it doesn't need. Multiply by a planet's worth of buildings and the reason bulk +data isn't shipped as GeoJSON is obvious. + +"Interchange" is a role, not a ranking. GeoJSON is the right tool for handing one feature +to a web map; Parquet is the right tool for handing a continent to Spark. + +### Is Pydantic wrapped around GeoJSON? + +Short answer: **no.** But the question has three reasonable readings, and one of them is a +qualified yes, so it's worth taking them separately. + +**"Is the model built on top of a GeoJSON structure?"** No. A model is a flat list of +fields, declared one at a time in Python. You can build one and use it without JSON ever +entering the picture: + +```python +from overture.schema.buildings import Building +from overture.schema.system.geometric import Geometry + +b = Building( + id="my-building-1", + geometry=Geometry.from_wkt("POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))"), + theme="buildings", + type="building", + version=1, + height=12.5, +) +print(b.id, b.height) +print(sorted(b.model_dump(mode="python", by_alias=True, exclude_none=True))) +``` + +``` +my-building-1 12.5 +['geometry', 'height', 'id', 'level', 'theme', 'type', 'version'] +``` + +No GeoJSON was parsed, produced, or consulted. If GeoJSON were the substrate, that +wouldn't be possible. + +**"Is there GeoJSON code inside the Pydantic classes?"** Yes — in exactly one of them. +Counting mentions across everything `Building` inherits from: + +``` +Building (building ) 0 +OvertureFeature (feature ) 0 +Identified (id ) 0 +Feature (feature ) 17 <- the base class in overture-schema-system +Named (names ) 0 +Stacked (level ) 0 +Appearance (_common ) 0 +``` + +All of it lives in `Feature`, in one serializer and one validator — the code that reads +GeoJSON in and writes GeoJSON out. Zero mentions in the six classes that actually define +what a building *is*. GeoJSON is an I/O concern parked at the base of the hierarchy, not a +structure the schema is built on. + +**"Is GeoJSON what's really being validated?"** No. What gets validated is the model. A +GeoJSON document is one accepted *input shape* — a flat dict is the other, and both end up +as the same Python object. + +An analogy: a word processor's document isn't "wrapped around `.docx`." It has a document +model, and it can read and write `.docx`. Deleting that import/export code would not +change what a document is. Same here — delete the ten lines below and Overture models +still work; they just stop speaking GeoJSON. + + +The entire GeoJSON transformation is those ten lines, in `Feature` +(`packages/overture-schema-system/src/overture/schema/system/feature.py`): + +```python +@model_serializer(mode="wrap") +def __serialize_with_geo_json_support__(self, serializer, info): + data = serializer(self) # <- the flat dict, produced first + + if info.mode == "json": # <- only in JSON mode + return { + "type": "Feature", + **({"id": data.pop("id")} if "id" in data else {}), + **({"bbox": data.pop("bbox")} if "bbox" in data else {}), + "geometry": data.pop("geometry"), + "properties": data, # <- everything else goes here + } + + return data # <- Python mode: flat, untouched +``` + +Read the first line: Pydantic produces the **flat** dictionary, and only then does this +function move `id`, `bbox`, and `geometry` to the top and sweep the remainder into +`properties`. In `python` mode the flat dict is returned unchanged and none of this runs. + +So the layering is: + +``` + Pydantic model (flat — the actual schema) + | + +----------+----------+ + | | + python mode json mode + | | + flat dict GeoJSON envelope + -> Parquet -> .geojson +``` + +GeoJSON is a costume the model puts on for one specific audience. It isn't the body. + +**The Pydantic model itself has no envelope.** In Python it's completely flat: + +```python +from overture.schema.buildings import Building +print("id in model_fields :", "id" in Building.model_fields) +print("geometry in model_fields :", "geometry" in Building.model_fields) +print("a 'properties' field? :", "properties" in Building.model_fields) +``` + +``` +id in model_fields : True +geometry in model_fields : True +a 'properties' field? : False +``` + +There is no `properties` field on `Building`, and `id` and `geometry` sit alongside +`height` and `num_floors` like any other field. The envelope is **not part of the model**. +It appears only when serializing to JSON, because that is what GeoJSON requires. + +You can watch the same object take both shapes: + +```python +import json, yaml +from overture.schema.buildings import Building + +b = Building.model_validate_json( + json.dumps(yaml.safe_load(open("examples/buildings/building-polygon.yaml")))) + +print("python mode:", sorted(b.model_dump(mode="python", by_alias=True, exclude_none=True))[:8]) +print("json mode :", sorted(b.model_dump(mode="json", by_alias=True, exclude_none=True))) +``` + +``` +python mode: ['class', 'ext_bar', 'ext_foo', 'geometry', 'height', 'id', 'is_underground', 'level'] +json mode : ['geometry', 'id', 'properties', 'type'] +``` + +One model, two renderings — and the flat one is the shape of the data you'd actually +download. Neither is more "real"; the model is what's real, and both are projections of +it. + +**So which answer to "what's required" is correct?** The model's: + +``` +['geometry', 'id', 'theme', 'type', 'version'] +``` + +The JSON Schema's two `required` arrays are that same list, split across the envelope +because that's where those fields land *in that particular output format*. Nobody decided +`geometry` belongs somewhere different from `theme`; GeoJSON did, in 2016. + +This distinction is the single most important thing in this guide, and it returns in force +in [section 3](SCHEMA_GUIDE.md#3-writing-code-against-the-models) — where using the wrong mode for your +data shape is the most common way to get a confusing `ValidationError`. + +--- + +## Two names per model + +Because the registry has to stay correct when packages it has never heard of register +their own models. + +The **canonical key is the entry-point string** — `overture.schema.buildings:Building`. +It includes the module path, so it is globally unique: no two packages can collide. + +The **short name is a derived alias**. It is just the class name after the colon, +snake-cased: + +```python +from overture.schema.system.discovery.entry_point import entry_point_class_alias + +entry_point_class_alias("overture.schema.divisions:DivisionArea") # 'division_area' +entry_point_class_alias("overture.schema.places:Place") # 'place' +``` + +Short names are *not* guaranteed unique. Anyone can +[register their own feature types](AUTHORING.md#register-your-own-feature-types), and nothing +stops a third party from shipping its own `Place`. So the short name can't be the +identity — it's a convenience, because +`validate_model(df, "overture.schema.buildings:Building")` is miserable to type. + +**The nice part is how it degrades.** The alias is offered only while it stays +unambiguous. `model_names()` counts aliases and includes only those appearing once, and +the resolver tries an exact key match first, then the alias: + +```python +from overture.schema.system.discovery.entry_point import resolve_entry_point_key + +registry = {"overture.schema.places:Place": ..., "acme.parks:Place": ...} + +resolve_entry_point_key("place", registry) +# ValueError: Entry-point alias 'place' is ambiguous. +# Specify one of: acme.parks:Place, overture.schema.places:Place + +resolve_entry_point_key( + "acme.parks:Place", registry +) # 'acme.parks:Place' — always works +``` + +Install a package that collides and `place` simply stops being accepted, with an error +naming both candidates — rather than silently validating against the wrong model. The +fully-qualified key never stops working. + +Two functions expose the two views: + +| Function | Returns | Use when | +|---|---|---| +| `model_keys()` | the 15 canonical entry-point keys | you want the authoritative list | +| `model_names()` | all 30 accepted names | you want everything `validate_model` will take | + +> **Skipping this step does not produce an error message.** It produces an empty +> registry. If `validate_model(df, "building")` raises a `KeyError`, or `model_names()` +> is empty, this is why. + +--- + +## What makes a field required + +Nobody maintains a list of required fields. **It's derived** — in Pydantic, a field with +no default is required, and a field with a default is optional: + +```python +from overture.schema.buildings import Building +from pydantic_core import PydanticUndefined + +for n in ["version", "theme", "height", "num_floors"]: + f = Building.model_fields[n] + d = "no default" if f.default is PydanticUndefined else f"default={f.default!r}" + print(f"{n:12} {d:16} -> {'REQUIRED' if f.is_required() else 'optional'}") +``` +``` +version no default -> REQUIRED +theme no default -> REQUIRED +height default=None -> optional +num_floors default=None -> optional +``` + +So "who decided" becomes "where is the field declared." For a building, five fields are +required and four of them come from a shared base class rather than from buildings at all: + +```python +from overture.schema.buildings import Building + +for n, f in Building.model_fields.items(): + if not f.is_required(): + continue + for cls in Building.__mro__: + if n in getattr(cls, "__annotations__", {}): + print(f"{n:10} -> {cls.__name__}") + break +``` +``` +id -> OvertureFeature +geometry -> Building +theme -> OvertureFeature +type -> OvertureFeature +version -> OvertureFeature +``` + +`id`, `theme`, `type`, and `version` are required of *every* Overture feature, declared +once in `OvertureFeature`: + +```python +id: Id = Field(description="A feature ID. ...") +theme: ThemeT +type: TypeT +# Superclass `Feature` provides `geometry` and `bbox`. +version: FeatureVersion +``` + +None carries `= None`, so all four are mandatory. `Building` adds only `geometry`, +narrowing the inherited one to the polygon types a building may have. + +In the generated JSON Schema those same five get split across the GeoJSON envelope — `id` +and `geometry` at the top, `theme`, `type`, and `version` inside `properties` — which is +why that document appears to have two answers to one question. It doesn't; it has one +answer written in the shape GeoJSON demands. + +**The human answer:** the Overture Schema Working Group decides, and changes go through +the process in [CONTRIBUTING.md](CONTRIBUTING.md) — a PR plus a changelog fragment. Making +a field required is a breaking change, so it targets the `vnext` branch and waits for a +major release; making one optional is not, and can go to `main`. + +--- + +## How tags work + +Every feature type carries a handful of **tags**, and `overture-schema list-types` prints +them after the type name: + +``` +building feature overture overture:theme=buildings +``` + +Three tags there. `feature` says this is a map feature rather than some other kind of +model. `overture` says Overture defined it. `overture:theme=buildings` says which theme it +belongs to. + +**Why a tag rather than a field called `theme`?** Because the tooling has to work on +feature types it has never heard of. A field named `theme` would only mean something to +code that already knows Overture has themes; the CLI would have to hardcode that. A tag is +just a label the type declares about itself, and the CLI's job is only to match labels — +so `--tag overture:theme=buildings` and `--tag acme:product=parks` go through exactly the +same code path. + +The `namespace:key=value` shape exists so that two organizations can both tag their types +without colliding. Everything Overture defines is namespaced under `overture:`; if you +register your own feature types, you pick your own namespace and your tags sit alongside +Overture's rather than competing with them. That is the whole extension mechanism — see +[Register your own feature types](AUTHORING.md#register-your-own-feature-types). + +The bare tags (`feature`, `overture`) have no namespace because they are not claims about +a vendor's taxonomy — they are the two facts every Overture feature type shares. + +--- + +## What the example file actually is + +The guide validates `examples/buildings/building-polygon.yaml` as its first real command. +`examples/buildings/building-polygon.yaml` is a file **in the repo you just cloned**. It +is not data you downloaded. The `examples/` tree is the project's own corpus of +hand-written sample features, used as test fixtures and pulled into the documentation +site. This one describes a single building — a parking structure in Washington DC: + +```yaml +id: overture:buildings:building:1234 +type: Feature # ← GeoJSON envelope +geometry: + type: Polygon + coordinates: [[ [-77.036873, 38.897804], ... ]] +properties: + ext_foo: I am a customer user property. # ← custom, non-Overture + theme: buildings # ← which theme + type: building # ← which feature type + version: 1 + height: 21.34 + num_floors: 4 + subtype: transportation + class: parking + sources: + - property: "" + dataset: microsoftMLBuildings +``` + +### See it actually catch something + +A success message proves the command ran, not that it's checking anything. Copy the file +and break it: + +```bash +cp examples/buildings/building-polygon.yaml /tmp/broken.yaml +``` + +Change `class: parking` to `class: skyscraper`: + +``` +class "skyscraper" ← Input should be 'agricultural', 'allotment_house', + 'apartments', 'barn', 'beach_hut', ... +``` + +Change `height: 21.34` to `height: -5`: + +``` +height -5 ← Input should be greater than 0 +``` + +Change `num_floors: 4` to `num_floors: 4.7`: + +``` +num_floors 4.7 ← Input should be a valid integer, got a number with a + fractional part +``` + +Enum membership, numeric bounds, integer-ness — each from the model definition, none of +it written by hand for this file. + +> **What it does not catch:** free-form string fields accept any string. The real +> `building-polygon.yaml` in the repo has a stray trailing comma — +> `dataset: microsoftMLBuildings,` — which YAML reads as part of the value. It parses to +> the string `'microsoftMLBuildings,'` and validates clean, because `dataset` has no +> constraint beyond "is a string." Validation enforces the schema, not your typing. + +--- + +## Why examples are YAML + +**No — real Overture data is GeoJSON or Parquet.** YAML here is purely an authoring +convenience for the example files: it allows comments (`# Custom user properties.`) and +is easier to hand-edit than JSON. + +The CLI accepts **JSON, YAML, and GeoJSON**, and YAML is a superset of JSON, so the +format is irrelevant to the validation. Convert the same file to JSON and you get the +same result: + +```bash +uv run python -c " +import json, yaml +json.dump(yaml.safe_load(open('examples/buildings/building-polygon.yaml')), + open('/tmp/same-building.json','w'), indent=2)" + +uv run overture-schema validate /tmp/same-building.json +``` +``` +✓ Successfully validated /tmp/same-building.json +``` + +Same bytes of meaning, different serialization, identical outcome. Pick whichever is +convenient — you'll mostly hand JSON or GeoJSON to this command in real use. + +--- + +## Why generated code is gitignored + +Not because it's optional, and not because it's for a subset of users. **It's build +output.** The commit that introduced the package says so directly: + +> The generated trees under `expressions/generated/` and `tests/generated/` are +> regenerable output of `make generate-pyspark` and are not tracked in git; `make check` +> and `make test-all` regenerate before running. + +Three reasons that's the right call: + +**One source of truth.** The Pydantic models define the schema; these expressions are a +derivative of them. Committing the derivative creates a second copy that can silently +drift — change a constraint, forget to regenerate, and the committed expressions keep +enforcing the old rule. Deleting them from git makes that failure impossible. + +**Scale.** A full generation is **32 files and roughly 23,000 lines**: + +``` +15 expression modules (one per feature type) +17 test modules (conformance tests, split per union arm) +``` + +Every schema change would produce a mechanical diff of that size, burying the actual +change and guaranteeing merge conflicts. + +**The build regenerates regardless.** `make check` and `make test-all` both depend on +`generate-pyspark`, which begins with `clean-pyspark` (`rm -rf`). The tree is rebuilt +from the current models every time, so a committed copy would never be read. + +It's the same reasoning that keeps `dist/`, `*.o`, and `node_modules/` out of git. + +### "Gitignored" does not mean "not shipped" + +This is the part worth being clear about, since it sounds like these files are somehow +optional for users. **They are not.** Published wheels contain them. + +`.github/workflows/publish-python-packages.yaml` runs `make generate-pyspark` before +`uv build`, then refuses to publish a wheel that lacks them: + +```yaml +- name: Generate PySpark expressions before build + if: matrix.package == 'overture-schema-pyspark' + run: make generate-pyspark +``` + +```bash +if [ "$PACKAGE" = "overture-schema-pyspark" ] && \ + ! unzip -l "$wheel" | grep -q 'expressions/generated/.*\.py'; then + echo " Wheel [$wheel] has no generated expressions -- codegen did not run. Aborting!" + exit 1 +fi +``` + +So the only people who ever run `make generate-pyspark` are people working **from a git +clone** — because a clone is the one place these files don't already exist. Install from +a package index and they arrive with the package, like any other module. + + +--- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cf31d7a4a..f863503b6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,6 +6,14 @@ Thank you for your interest in contributing. > [DevOps tracking issue #490](https://github.com/OvertureMaps/schema/issues/490) > for current status and what is planned next. +## Working with the Python packages + +The schema is authored as Pydantic models. [SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) covers +installing and using the packages; [AUTHORING.md](AUTHORING.md) covers authoring new +schema models and the development workflow (`uv sync`, `make check`, ruff and +docformatter). [TROUBLESHOOTING.md](TROUBLESHOOTING.md) collects the errors that cost +people time. + ## Where to send your change This repository uses a two-branch model. Target the branch that matches your diff --git a/GLOSSARY.md b/GLOSSARY.md index 5778501dd..e060a84b6 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -1,18 +1,173 @@ -# Entity -An entity is a thing in the physical world. It can relate both to physical objects or more abstract concepts that have a spatial presence (e.g. administrative areas are not physical objects as such, but they have physical properties that define their location and extend). An entity can only exist once. +# Glossary -# Feature -A Feature is an abstraction of a specific entity in the map. It exists only in its digital form. +Two vocabularies meet in this repository. The first describes the map: what an entity is, +what a feature is, how feature types are named. The second describes the Python packages +that define the schema: workspaces, entry points, specs. Both are collected here. + +Terms in the second group are cross-referenced to the page and section that covers them +in full — [SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) for using the schema, +[AUTHORING.md](AUTHORING.md) for extending it, and [CONCEPTS.md](CONCEPTS.md) for +background. + +--- + +## Data model + +### Entity + +An entity is a thing in the physical world. It can relate both to physical objects or more +abstract concepts that have a spatial presence (e.g. administrative areas are not physical +objects as such, but they have physical properties that define their location and extent). +An entity can only exist once. + +### Feature + +A Feature is an abstraction of a specific entity in the map. It exists only in its digital +form. + +### Feature Class -# Feature Class Feature Class is a synonym for "Feature Type". The preferred term is "Feature Type". -# Feature Type -A Feature Type is a type of entities with common properties as described in the Overture Schema. +### Feature Type + +A Feature Type is a type of entities with common properties as described in the Overture +Schema. + +### Instance -# Instance Instance is a synonym for "Feature". The preferred term is "Feature". -# Object +### Object + Object is a synonym for "Instance" or "Feature". The preferred term is "Feature". +### Theme + +The top-level grouping a feature type belongs to, and a mandatory property on every +feature. The term is deliberately chosen over "layer" to avoid that word's baggage. There +are six: `addresses`, `base`, `buildings`, `divisions`, `places`, `transportation`. Each +ships as its own Python package, `overture-schema-theme-*`. + +### Type + +The feature type within a theme, and a mandatory property on every feature — for example +`theme=buildings`, `type=building`. Together `theme` and `type` identify the feature type. + +### Subtype + +An optional third property that further refines the feature type — for example +`theme=transportation`, `type=segment`, `subtype=road`. Where a type has subtypes, the +model for that type is usually a [discriminated union](#discriminated-union) with one arm +per subtype. Note the spelling: the field is `subtype`, not `subType`. + +### GERS + +The Global Entity Reference System. A feature's `id` may be a GERS ID if — and only if — +the feature represents an entity that is part of GERS. + +--- + +## Toolchain + +### Alias + +Three unrelated things in this codebase go by this name. Which one is meant is almost +always clear from context, but they are worth separating: + +1. **Field alias** — a mapping from a Python attribute name to the name the data uses, + declared with `Field(alias=...)`. It exists because some data field names are not legal + Python identifiers: `Building.class_` carries `alias="class"`, since `class` is a + reserved word. Dump with `by_alias=True` to get the data name back. +2. **Type alias** — a module-level name bound to a type expression rather than to a class. + `Segment` is one: it is an `Annotated[Union[...], Discriminator(...)]`, *not* a class. + This is why `Segment.model_validate(...)` raises `AttributeError` and you need + `TypeAdapter(Segment).validate_python(...)` instead. See + [Working with Segment and other unions](SCHEMA_GUIDE.md#35-working-with-segment-and-other-unions). +3. **Entry-point name** — the short name a model registers under, which need not match the + class name. `building = "overture.schema.buildings:Building"` registers the class + `Building` under the name `building`. + +### Entry point + +The mechanism by which a package advertises something to the rest of the installed +environment, declared in `pyproject.toml`. Nothing imports these directly; they are +discovered at runtime. The schema uses four groups: `overture.models` (feature types), +`overture.tag_providers` ([tags](#tag)), `project.scripts` (the CLIs), and `pytest11` +(test plugins). Registering your own model is a matter of adding an entry point — see +[Register your own feature types](AUTHORING.md#register-your-own-feature-types). + +### Workspace + +One repository containing several independently-versioned packages that share a single +lockfile and a single virtual environment. Declared by `[tool.uv.workspace]` in the root +`pyproject.toml`. The schema repo is a `uv` workspace of thirteen packages. See +[What "workspace" means](CONCEPTS.md#what-workspace-means). + +### Metapackage + +A package that ships no code of its own and exists only to depend on others. +`overture-schema` is one: installing it pulls in every theme. Its namespace root contains +nothing but a `py.typed` marker, which is why `from overture.schema import Building` +fails. + +### Flat shape + +The form a feature takes as a single record with no nesting of core properties — `id`, +`geometry`, `theme`, `type` and the rest all sitting at the same level. This is the shape +the Pydantic models use, and the shape Overture publishes in Parquet. +`model_validate()` expects it. + +### Envelope + +The GeoJSON form of a feature, where most properties are tucked under a `properties` key +alongside a top-level `type: "Feature"`. Distinct from the [flat shape](#flat-shape), and +the single most common source of confusion when validating: `model_validate()` rejects it, +`model_validate_json()` accepts it. See +[Two representations](SCHEMA_GUIDE.md#32-the-one-thing-to-understand-two-representations). + +### Discriminated union + +A union of models where one field's value decides which arm applies. `Segment` is +discriminated on `subtype`: `road` selects `RoadSegment`, `rail` selects `RailSegment`, +`water` selects `WaterSegment`. Pydantic uses the discriminator to pick an arm without +trying each in turn, which also makes validation errors point at the right model. + +### NewType + +A distinct type wrapping an existing one, used to give a plain value a name and a set of +constraints — `Id`, `CountryCodeAlpha2`, `LanguageTag`. At runtime the value is still a +`str`; the wrapper carries the validation rules and survives into the generated artifacts, +which is why it is preferred over a bare `str` with a `Field` constraint. + +### ModelKey + +The key type returned by `discover_models()`. Carries the model's entry-point `name`, its +`entry_point` string, and its [tags](#tag). Not a plain string, and not a tuple — code +that assumes either will break. + +### Tag + +A string attached to a model during discovery, classifying it orthogonally to its name — +`feature`, or `overture:theme=buildings`. Tags are what the CLI's `--tag` / `--filter` / +`--exclude` options select on. They are produced by *tag providers* registered on the +`overture.tag_providers` entry-point group; third parties can register their own. Eight +tags exist today: `feature`, `overture`, and one `overture:theme=*` per theme. `overture` +marks a model built on Overture's feature model — it subclasses `OvertureFeature` — which +a third party's own type can also be; it is not a claim that the type belongs to the +Overture schema. + +### Spec + +The target-independent description of a model produced by the codegen's extraction layer, +before any output format is chosen — `RecordSpec` for a model, `UnionSpec` for a +discriminated union, `FieldSpec` for a field, `EnumSpec` for an enum. Renderers consume +specs; they never touch Pydantic models directly. This split is what lets a new output +format be a new renderer rather than new extraction logic. See +[Write a new codegen target](AUTHORING.md#write-a-new-codegen-target). + +### Extra + +An optional dependency group declared under `[project.optional-dependencies]` and +installed with bracket syntax (`package[extra]`) or `uv sync --all-extras`. Distinct from +"extra fields", which is what `no_extra_fields` forbids on a model. diff --git a/Makefile b/Makefile index 6a0d92a48..8a50ff6d1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: default uv-sync clean-pyspark generate-pyspark check check-namespace test-all test test-only docformat docformat-only doctest doctest-only mypy mypy-only lint-only update-baselines +.PHONY: default uv-sync clean-pyspark generate-pyspark check check-namespace test-all test test-only docformat docformat-only doctest doctest-only mypy mypy-only lint-only format update-baselines TESTMON ?= --testmon @@ -47,13 +47,13 @@ check-namespace: # the PySpark output first: that tree is no longer tracked in git, so the # generated conformance tests only exist once generation has run. test-all: uv-sync generate-pyspark - @uv run pytest -W error packages/ + @uv run pytest -W error packages/ tests/ test: uv-sync - @uv run pytest -W error $(TESTMON) packages/ -x -q --tb=short + @uv run pytest -W error $(TESTMON) packages/ tests/ -x -q --tb=short test-only: - @uv run pytest -W error $(TESTMON) packages/ -x -q --tb=short + @uv run pytest -W error $(TESTMON) packages/ tests/ -x -q --tb=short coverage: uv-sync @uv run pytest packages/ --cov overture.schema --cov-report=term --cov-report=html && open htmlcov/index.html @@ -90,11 +90,20 @@ mypy-only: | tr - . \ | sed 's|^packages/|-p |' \ | xargs uv run mypy --no-error-summary - @for d in packages/*/tests; do find "$$d" -name "*.py" | sort | xargs uv run mypy --no-error-summary || exit 1; done + @for d in packages/*/tests tests; do find "$$d" -name "*.py" | sort | xargs uv run mypy --no-error-summary || exit 1; done lint-only: - @uv run ruff check -q packages/ - @uv run ruff format --check packages/ + @uv run ruff check -q packages/ tests/ + @uv run ruff format --check packages/ tests/ + @# Python embedded in Markdown -- keeps documented snippets in the + @# same style as the code they describe. + @git ls-files -z '*.md' | xargs -0 --no-run-if-empty uv run ruff format -q --check + +# The fixing counterpart to lint-only: what to run when that gate fires. +format: + @uv run ruff check -q --fix packages/ tests/ + @uv run ruff format -q packages/ tests/ + @git ls-files -z '*.md' | xargs -0 --no-run-if-empty uv run ruff format -q update-baselines: @uv run pytest --update-baselines -m baseline -q packages/ diff --git a/README.md b/README.md index 921a4c6b5..47b7d8b43 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,136 @@ -Overture Maps Schema +Overture Schema === -The Overture Maps schema working group is responsible for designing the Overture Maps Data Schema and the Global Entity Reference System (GERS). +This code in this repository defines the Overture schema. -## Documentation -The contents of this repository are presented in a more human-friendly format at [docs.overturemaps.org](https://docs.overturemaps.org/) + +_ Note: You'll find reference documentation, tutorials, and examples of working with Overture data +at [docs.overturemaps.org](https://docs.overturemaps.org/).__ + +## What's in this repository + +| Path | What it is | +|---|---| +| `packages/` | The schema, authored as [Pydantic](https://docs.pydantic.dev/latest/) models and published as Python packages. | +| `reference/examples/` | Feature instances that are **expected to validate** — one file per case, organized by theme. | +| `reference/counterexamples/` | Feature instances that are **expected to fail**, most carrying the specific error they should raise. | +| `tests/` | Tests that span packages, including the check that keeps the imports in these docs working. | +| `docs/` | Source for the schema pages on docs.overturemaps.org, plus the versioning reference. | +| `schema/` | **Deprecated.** The YAML JSON Schema — see [The YAML schema](#the-yaml-schema-deprecated). | +| `examples/`, `counterexamples/` | **Deprecated.** Fixtures for the YAML schema, not the Pydantic models. | + +Note the two sets of examples. `reference/examples/` and `reference/counterexamples/` are +the current ones, exercised by the Python test suite. The top-level `examples/` and +`counterexamples/` belong to the deprecated YAML schema. They overlap heavily but have +drifted apart; when you add a case, add it under `reference/`. + +## Python packages + +Fourteen packages under `packages/`, versioned and released independently: + +- **`overture-schema`** — the umbrella package. Depends on all themes and support + packages for a coherent set. **This is what consumers pin.** +- **Themes** — `theme-addresses`, `theme-base`, `theme-buildings`, `theme-divisions`, + `theme-places`, `theme-transportation`. One package per theme, each holding its feature + types and the structures they share. +- **Foundation** — `system` (the base types every model builds on) and `common` + (structures shared across themes, like names and sources). +- **Tooling** — `cli` (validate data, generate JSON Schema), `codegen` (generate docs and + code from the models), `validation` (validate against the union of all discovered + models), `pyspark` (validation expressions for Spark). +- **Extensions** — `extensions-operating-hours`, an optional add-on. + +These pages are for people working with the packages in code: + +- [SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) — install the packages, explore the models, validate + data, generate artifacts. **Start here.** +- [AUTHORING.md](AUTHORING.md) — register your own feature types, author new schema + models, build an SDK or CLI on the models. +- [CONCEPTS.md](CONCEPTS.md) — why the schema is Pydantic, why it is many packages, and + what the GeoJSON envelope is. +- [TROUBLESHOOTING.md](TROUBLESHOOTING.md) — symptom-indexed fixes and known gotchas. +- [GLOSSARY.md](GLOSSARY.md) — vocabulary for both the data model (entity, feature type, + theme) and the Python toolchain (entry point, workspace, discriminated union). + +Run the full test and quality suite with: + +```bash +make check +``` + +## The YAML schema (deprecated) + +`schema/` holds the previous definition of the Overture schema as JSON Schema written in +YAML, with `schema/schema.yaml` as its entry point. It is validated by `./test.sh` (which +needs [`jv`](https://github.com/santhosh-tekuri/jsonschema)) against the top-level +`examples/` and `counterexamples/`, wired up in +[test-schema.yaml](.github/workflows/test-schema.yaml). + +**It is deprecated and scheduled for removal in December 2026.** It stays until then +because docs.overturemaps.org still builds from it: the interactive schema blocks come +from `schema/`, and the sample features come from `examples/`. See +[docs/README.md](docs/README.md). + +**Until it is removed, backward-compatible changes belong in both places.** A change to +the published data should land in the Pydantic models *and* in the YAML, so the two +definitions stay in step for as long as both are live. + +## Project history + +The schema working group opened this repository in January 2023, and the artifacts of +every phase of development are still here. + +| | | +|---|---| +| **Jan 2023** | Repository created as `schema-wg`, chartered to design the data schema and GERS. | +| **Mar 2023** | The schema takes shape as JSON Schema written in YAML: `schema/`, `examples/`, `counterexamples/`, and `test.sh` all arrive together. | +| **Aug 2023** | The Schema Task Force writes a *doctrine* on top of the project's tenets, to aim the work at a vision rather than react week to week. | +| **Jul 2024** | Overture data reaches general availability. | +| **Jul 2025** | Pydantic rewrite begins. +| **Nov 2025** | `packages/` are merged to main. The Pydantic schema and the YAML schema are developed in parallel. | +| **Aug 2026** | Repository docs consolidated into the guides listed above. | +| **Dec 2026** | `schema/`, `examples/`, and `counterexamples/` scheduled for removal. | + +1,788 commits from 71 contributors. + +### The tenets and the doctrine + +The working group set down six tenets early on, under the heading *"These are our tenets +unless you know better ones"* — address the core and enable the periphery; invent across +the gap; backward-compatible is forward-compatible; the world is neither flat nor still; +empower, don't dictate; always open, never closed. They still govern the design, and are +recorded with their consequences in [CONCEPTS.md](CONCEPTS.md#the-tenets). + +At the Schema Task Force meeting on 2023-08-30, the group decided to work toward a stated +vision rather than react week to week, and wrote a short doctrine built on those tenets for +the Working Group to ratify. It made five commitments: + +- **The schema optimizes for specific use cases, but allows extension.** Structure targets + selected use cases; user extensions unblock everyone else. +- **The schema gets better over time.** Properties for core use cases keep landing, known + issues keep getting fixed, and the schema advances in a backward-compatible way. +- **The schema is a cohesive whole.** Same problems get same solutions and same things get + same names, within and across themes, so that understanding one theme carries to the rest. +- **The schema is usable for data consumption by humans.** +- **The schema is usable for data consumption by open source tools.** + +The doctrine flagged the last two as an unresolved tension and said the project should lean +toward one. The 2025 move to Pydantic is what settled it: the models are authored for humans +to read and edit, and the artifacts tools need — JSON Schema, PySpark expressions, +documentation — are generated from them. See +[Why Pydantic rather than JSON Schema](CONCEPTS.md#why-pydantic-rather-than-json-schema). + +The specifics the doctrine argued over have all since been resolved. The `admins` theme is +now `divisions`; the camelCase property names it cited (`isoCountryCodeAlpha2`, +`road.roadNames`) became snake_case in February 2024; and the `entityId` property it +proposed as the schema/GERS interface never shipped — features carry a GERS `id` directly. ## Contributing -See [CONTRIBUTING.md](CONTRIBUTING.md) for branching strategy, workflow, and contribution guidelines. -## Feedback -Please provide feedback or ask questions at [Discussions](https://github.com/orgs/OvertureMaps/discussions). +See [CONTRIBUTING.md](CONTRIBUTING.md) for branching strategy, workflow, and contribution +guidelines. +## Feedback +Please provide feedback or ask questions at +[Discussions](https://github.com/orgs/OvertureMaps/discussions). diff --git a/README.pydantic.md b/README.pydantic.md deleted file mode 100644 index 866e39670..000000000 --- a/README.pydantic.md +++ /dev/null @@ -1,233 +0,0 @@ -# Overture Schema - -[Pydantic](https://docs.pydantic.dev/latest/) schemas for [Overture Maps -data](https://docs.overturemaps.org/guides/). - -## Overview - -This project provides type-safe Python models for validating and working with [Overture -Maps Foundation](https://overturemaps.org/) data. Overture Maps is an open geospatial -dataset containing buildings, places, addresses, transportation networks, and -administrative boundaries curated from multiple sources. - -Use these schemas to: - -- Validate Overture Maps data -- Build data processing pipelines with type safety -- Extend schemas with custom fields and validation rules - -## Why Use Pydantic to Define Data Schemas? - -This project addresses a fundamental challenge in data consumption: **bridging the -semantic gap between raw data and human understanding** while enabling -machine-actionable workflows. - -### Why Schema at All: Beyond Raw Data - -Take a column like `pop_2020`. Is it total population? Population density per square -kilometer? Working-age population? Without a schema, you're left sampling values and -guessing from column names. - -Compare this to OpenStreetMap's approach: features use well-known key/value pairs like -`building=residential` or `addr:housenumber=42` that have semantic meaning and can be -looked up on the OSM wiki. This creates a step toward a schema - shared vocabulary with -documented semantics used across a vast dataset. However, OSM tags remain free-form: -multiple valid ways to express the same concept, no built-in validation, and complex -downstream validation because of undocumented keys that might have meaning to someone, -somewhere. A schema provides the structured alternative: explicit types, clear -validation rules, and semantic meaning that both humans and systems can rely on. - -Data files containing only column names and values aren't fully documented. External -metadata files typically focus on how data was collected and encoded, not on semantic -meaning or validation rules. Data consumers struggle to understand what datasets contain -and which columns they need for their goals. - -### Why Pydantic Over JSON Schema: Solving Multiple Problems - -We initially chose JSON Schema because it aligned with our mental model and promised to -solve our problems as we understood them. But JSON Schema surfaced several pain points: - -- **Authoring difficulty**: Hard to write correctly, difficult to verify, limited IDE - support, no refactoring capabilities -- **Tooling gaps**: Generic tools can't tailor output for specific applications like - ours -- **Development friction**: Schema changes required manual coordination across multiple - artifacts - -Pydantic addresses these systematically: author in Python with full IDE support, -generate tailored documentation, and automatically produce the specific artifacts each -workflow needs. Pydantic can also produce JSON Schema, so any application that requires -it can use it while we gain all the Python benefits during authoring. - -### The Result: Faster Understanding, Higher Quality - -Instead of spending time deciphering what columns mean and whether data matches -expectations, users can focus on their actual goals: analysis, visualization, -integration. Quality improves because validation happens automatically rather than -through manual inspection. - -The fundamental approach - human-readable authoring that generates machine-actionable -outputs - has broader applications beyond Overture and geospatial data. We hope others -will adapt these patterns for linking with Overture data or modeling their own domains -entirely. - -## Getting Started - -- Install [Python](https://www.python.org/downloads/) 3.10 or newer -- Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/) -- Clone this repository: `git clone https://github.com/OvertureMaps/schema.git` -- Install dependencies: `uv sync --all-packages` -- Run tests to ensure that everything is configured correctly: `make check` (on Windows, - without `make`: `uv run pytest packages`) - -## Packages - -This workspace contains the following packages: - -### Core Packages - -- **`overture-schema`** - Main entrypoint package that aggregates all types for - convenient usage -- **`overture-schema-common`** - Overture-specific models shared across themes: base - feature class, scoping framework, names, sources, and cartographic hints -- **`overture-schema-system`** - Portable numeric, geometric, and string types, - constraints, and a GeoJSON-aware base model for building Pydantic schemas that - serialize to JSON, Parquet, and Spark - -### Theme Packages - -- **`overture-schema-theme-addresses`** - Address features -- **`overture-schema-theme-base`** - Foundational geographic features (land, water, - infrastructure, bathymetry, land cover, land use) -- **`overture-schema-theme-buildings`** - Building footprints and building parts with - architectural details -- **`overture-schema-theme-divisions`** - Administrative boundaries, division areas, and - political boundaries -- **`overture-schema-theme-places`** - Points of interest, businesses, and named - locations -- **`overture-schema-theme-transportation`** - Road segments and transportation network - connectors - -### Usage (Python) - -Install the main package using `pip` (or your package manager of choice): - -```shell -pip install overture-schema -``` - -```python -from overture.schema.buildings.building import Building -from overture.schema.places.place import Place -import json - -# Validate data - supports both flat/tabular- (Parquet-style) and GeoJSON-formatted -# dicts -building = Building.model_validate(feature_data) -building_geojson = Building.model_validate(geojson_feature) - -# Parse and validate JSON strings -building_from_json = Building.model_validate_json(json_string) - -# Convert to GeoJSON format -geojson_output = building.model_dump(mode="json") -``` - -## Schema Extension - -The library is designed to support data producer extensions through multiple patterns. -This extensibility is a core feature that allows organizations to add custom fields and -types while maintaining compatibility with the base Overture schema. We are in the -process of determining how this should work. - -### Model Registration via Entry Points - -Models are registered using [setuptools entry -points](https://setuptools.pypa.io/en/latest/userguide/entry_point.html) in each -package's `pyproject.toml` file. This enables automatic discovery and loading of models -at runtime without requiring explicit imports. - -Registration is done in the `[project.entry-points."overture.models"]` section: - -```toml -[project.entry-points."overture.models"] -building = "overture.schema.buildings:Building" -building_part = "overture.schema.buildings:BuildingPart" -``` - -The discovery system provides programmatic access to registered models: - -```python -from overture.schema.system.discovery import discover_models, get_registered_model - -# Discover all registered models, keyed by ModelKey -all_models = discover_models() - -# Get a specific model by name -building_model = get_registered_model("building") -if building_model: - building = building_model.model_validate(building_data) -``` - -### Tagging - -Each `ModelKey` returned by `discover_models()` carries a `frozenset[str]` of tags -that classify the model orthogonally to its entry-point name -- whether the model -is a `Feature` subclass, which Overture theme it belongs to, which package shipped -it, and so on. Downstream tools (the CLI, codegen, third-party consumers) use tags -to filter the working set without importing every model: - -```python -from overture.schema.system.discovery import ( - TagSelector, - discover_models, - filter_models, -) - -models = discover_models() -# { -# ModelKey(name="building", entry_point="overture.schema.buildings:Building", -# tags=frozenset({"feature", "overture", "overture:theme=buildings"})): BuildingModel, -# ModelKey(name="place", entry_point="overture.schema.places:Place", -# tags=frozenset({"feature", "overture", "overture:theme=places"})): PlaceModel, -# ... -# } - -buildings = filter_models( - models, - TagSelector(include_any=("overture:theme=buildings",)), -) -``` - -Tags are produced by *tag providers* registered on the `overture.tag_providers` -entry-point group. The `system` and `common` packages ship the built-in providers -(`feature`, `overture`, `overture:theme=*`); third parties can register their own -to attach custom tags during discovery. See the [`overture-schema-system` -README](packages/overture-schema-system/README.md#tagging) for tag format, -reserved namespaces, and provider authoring. - -## Development - -This project uses [uv](https://docs.astral.sh/uv/) for dependency management: - -```bash -# Install dependencies for the entire workspace -uv sync --all-packages - -# Run all tests and type/code quality checks -make check - -# Run tests for a specific package -uv run pytest packages/overture-schema-theme-buildings/ - -# Run tests matching a pattern -uv run pytest -k "buildings" -``` - -Auto-format / fix code to align with project expectations: - -```shell -uv run ruff check --fix -uv run ruff format -uv run docformatter --in-place --recursive packages/ -``` diff --git a/SCHEMA_GUIDE.md b/SCHEMA_GUIDE.md new file mode 100644 index 000000000..d70ade4d7 --- /dev/null +++ b/SCHEMA_GUIDE.md @@ -0,0 +1,1743 @@ +# Overture Schema Guide + +This is a practical guide to installing the Overture schema packages, exploring the models, +writing code against them, validating data, and generating artifacts from the schema. + +Three other pages cover material this guide points to rather than repeats: +[CONCEPTS.md](CONCEPTS.md) for why the schema is built this way, +[TROUBLESHOOTING.md](TROUBLESHOOTING.md) for errors and gotchas, and +[AUTHORING.md](AUTHORING.md) for writing schema models. Reference for a single package +lives in that package's `README.md` under `packages/`. + +*Note: none of these packages are on PyPI yet. Everything below installs from a local +clone. Any `pip install overture-schema` you find in a README is aspirational — it will +not work today.* + +## Contents + +1. [Install](#1-install) +2. [Exploring the models](#2-exploring-the-models) +3. [Writing code against the models](#3-writing-code-against-the-models) +4. [The three CLIs](#4-the-three-clis) +5. [Validating data](#5-validating-data) +6. [Converting the schema to other formats](#6-converting-the-schema-to-other-formats) +7. [Using the packages from your own project](#7-using-the-packages-from-your-own-project) +8. [Building tools on the models](#8-building-tools-on-the-models) + +### Other pages + +| Page | What's on it | +|---|---| +| [AUTHORING.md](AUTHORING.md) | Registering your own feature types, authoring new schema models, building an SDK or CLI, templates | +| [CONCEPTS.md](CONCEPTS.md) | Why Pydantic, why many packages, the GeoJSON envelope, and other *why* questions | +| [TROUBLESHOOTING.md](TROUBLESHOOTING.md) | Symptom-indexed fixes, and the gotchas that cost people time | +| [GLOSSARY.md](GLOSSARY.md) | Data-model and toolchain vocabulary | + +--- + +## 1. Install + +### 1.1 First, what you're installing + +**The schema is not one Python package. It's many Python packages.** + +Open `packages/` and you'll see: + +``` +packages/ +├── overture-schema ← a metapackage: depends on the others, ships no code +├── overture-schema-system ← foundations: numeric types, geometry, discovery +├── overture-schema-common ← Overture conventions: OvertureFeature, names, sources +├── overture-schema-cli ← the `overture-schema` command +├── overture-schema-validation ← validate() / validate_json() +├── overture-schema-codegen ← the `overture-codegen` command +├── overture-schema-pyspark ← the `overture-validate` command +└── overture-schema-theme-* ← six of these: buildings, places, transportation, … +``` + +Each of those directories has its own `pyproject.toml` and its own version number. + +They are separate packages so that **what you install decides what exists at runtime** — +install only the buildings theme and the tooling only knows about buildings. For why that +was worth the complexity, see +[Why the schema is many packages](CONCEPTS.md#why-the-schema-is-many-packages). + +#### What you end up with + +One command installs all the packages into **a single shared virtual environment at the repo root**: + +``` +schema/ +├── .venv/ ← created by uv; all packages live here +│ └── bin/ +│ ├── overture-schema ← the three CLIs land here +│ ├── overture-codegen +│ └── overture-validate +├── packages/ +└── pyproject.toml +``` + + +#### The packages form four layers + +The packages depend on each other in one direction only — each layer below uses the one +above it, and never the reverse. That ordering is what lets the tooling work on feature +types nobody had written when the tooling was built. + +| Layer | Package | Gives you | +|---|---|---| +| Foundation | `system` | `float32`/`uint8`/…, `Geometry`, `BBox`, `CountryCodeAlpha2`, constraint annotations, `Feature` base class, entry-point discovery | +| Overture conventions | `common` | `OvertureFeature` (id/theme/type/version/geometry/sources), `@scoped`, `Names`, `Sources`, cartography hints | +| Feature types | `theme-*` | `Building`, `Place`, `Segment`, `Address`, … plus their enums | +| Tooling | `cli`, `codegen`, `pyspark`, `validation` | commands and functions that consume the above generically | + +The tooling layer never hardcodes feature types. It discovers them. That's why your own models can slot in. + + +### 1.2 Prerequisites + +- **Python 3.10 or newer.** You don't need to install this yourself — `uv` will fetch a + suitable Python if your system one is too old. +- **[`uv`](https://docs.astral.sh/uv/getting-started/installation/).** + +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +You do **not** need Java, Spark, or Homebrew for anything in sections 1 through 8 of this +guide except the PySpark parts. If you already have a Homebrew Spark installed, it may actively break things — see +[`FileNotFoundError` on `spark-submit`](TROUBLESHOOTING.md#filenotfounderror-on-spark-submit). + + +### 1.3 Install + +```bash +git clone https://github.com/OvertureMaps/schema.git +cd schema +uv sync --all-packages +``` + +That's it. `uv sync --all-packages` creates `.venv/` and installs +all thirteen workspace members into it. + +**This is the whole install for most people.** You can now validate data, explore models, +generate JSON Schema, and generate documentation. + +**What about `make install`?** It works too, and nothing about it is risky. It's exactly +`uv sync --all-packages --all-extras` plus the PySpark generation step described in +[Do you need PySpark?](#14-do-you-need-pyspark) — so it just does more than most people +need. (No package in the workspace defines extras today, so `--all-extras` changes +nothing.) Use it if you'd rather run one command and have everything. + + +Run these commands. All should succeed: + +```bash +uv run overture-schema --version +``` + +``` +overture-schema, version 1.17.1 +``` + +```bash +uv run overture-schema list-types +``` + +``` +address feature overture overture:theme=addresses +bathymetry feature overture overture:theme=base +building feature overture overture:theme=buildings +building_part feature overture overture:theme=buildings +connector feature overture overture:theme=transportation +division feature overture overture:theme=divisions +division_area feature overture overture:theme=divisions +division_boundary feature overture overture:theme=divisions +infrastructure feature overture overture:theme=base +land feature overture overture:theme=base +land_cover feature overture overture:theme=base +land_use feature overture overture:theme=base +place feature overture overture:theme=places +segment feature overture overture:theme=transportation +water feature overture overture:theme=base +``` + +```bash +uv run overture-schema validate examples/buildings/building-polygon.yaml +``` +``` +✓ Successfully validated examples/buildings/building-polygon.yaml +``` + +That file is a sample building that ships with the repo. For what's inside it, why it's +YAML, and proof that validation is really checking something, see +[the example file](CONCEPTS.md#what-the-example-file-actually-is). + + +### 1.4 Do you need PySpark? + +Probably not. Answer honestly: + +| I want to… | Need PySpark? | +|---|---| +| Validate files, explore models, write Python against them | **No** | +| Generate JSON Schema or markdown docs | **No** | +| Generate an SDK in another language | **No** | +| Validate millions of rows of Parquet, or data in S3 | **Yes** | +| Get a Spark `StructType` for a feature type | **Yes** | +| Run the full test suite (`make check`) | **Yes** | + +**If no:** you're done. Go to section 2. + +**If yes:** there's one more step, because the PySpark validation expressions are +*generated code that is not committed to git*. They're in `.gitignore`. `uv sync` alone +cannot produce them. + +```bash +make generate-pyspark +``` + +Or `make install`, which is just `uv sync --all-packages --all-extras` followed by +`make generate-pyspark`. The command prints nothing on success. If `model_names()` +later comes back empty, see +[that entry in TROUBLESHOOTING.md](TROUBLESHOOTING.md#model_names-returns--or-keyerror-on-a-feature-type). + +--- + +## 2. Exploring the models + +You can't write code against a model you haven't looked at. This section is about finding +out what feature types exist, what fields they carry, and what values those fields +accept — before writing a line of code against them. + +**Two things can answer your questions, and it's worth keeping them straight:** + +| | What it is | Ask it with | +|---|---|---| +| **The model** | The Pydantic classes themselves. When you validate a file or a DataFrame, this is the code that accepts or rejects it. | Python: `Building.model_fields` | +| **The generated JSON Schema** | A description of the model, written out as a JSON document | `overture-schema json-schema` | + +They always agree, because the second is generated from the first. + +Ask the **model** when you want to understand the schema — it answers in flat Python, and +it is the thing that actually runs. Ask the **JSON Schema** when you need the rules in a +form another program can read: a validator in another language, a code generator, or a +tool that has no idea Python exists. Section 2.4 does the first, 2.5 the second. + +### 2.1 Running Python against the models + +Everything up to now has been shell commands. From here the guide switches to Python, so +first: how do you actually run it? + +**The models are installed in the project's `.venv`, not in whatever `python` your shell +finds.** Starting Python the usual way fails: + +```bash +python +``` +``` +>>> from overture.schema.buildings import Building +Traceback (most recent call last): + File "", line 1, in +ModuleNotFoundError: No module named 'overture' +``` + +`ModuleNotFoundError: No module named 'overture'` always means this: right code, wrong +interpreter. Compare the two: + +```bash +python -c "import sys; print(sys.executable)" +uv run python -c "import sys; print(sys.executable)" +``` + +``` +/Users/you/.pyenv/versions/3.10.15/bin/python +/path/to/schema/.venv/bin/python3 +``` + +The first is your system or `pyenv` Python, which has never heard of these packages. The +second is the project's environment, where `uv sync` installed them. Nothing is broken — +they're simply two different Pythons. + +Prefix with `uv run` and it works — that runs Python inside the project's environment: + +```bash +uv run python -c "from overture.schema.buildings import Building; print(Building.__name__)" +``` +``` +Building +``` + +Three ways to run the Python in this guide, all from the repo root: + +**A one-liner**, for a quick look: + +```bash +uv run python -c "from overture.schema.buildings import Building; print(len(Building.model_fields))" +``` + +**An interactive session**, best for exploring — you can poke at a model, tab-complete, +and try things: + +```bash +uv run python +``` + +``` +Python 3.10.18 +>>> from overture.schema.buildings import Building +>>> len(Building.model_fields) +26 +``` + + +**A script file**, once you're writing more than a couple of lines. Save this as +`explore.py` in the repo root: + +```python +from overture.schema.buildings import Building + +print(f"{len(Building.model_fields)} fields") + +for name, field in Building.model_fields.items(): + if field.is_required(): + print(f" required: {name}") +``` + +and run it: + +```bash +uv run python explore.py +``` + +``` +26 fields + required: id + required: geometry + required: theme + required: type + required: version +``` + +Every Python block from here on is code to run one of these three ways. An interactive session is the best fit for section 2 — +you're looking things up, not building anything yet. + +> You can also activate the environment once (`source .venv/bin/activate`) and then use +> plain `python`. This guide uses `uv run` throughout because it always works, needs no +> setup, and cannot be left half-done. + +### 2.2 Where the models live + +Before you can explore anything, you need to know where it lives. The Python module +path drops the `theme-` prefix: + +| Package | Import from | +|---|---| +| `overture-schema-theme-buildings` | `overture.schema.buildings` | +| `overture-schema-theme-transportation` | `overture.schema.transportation` | +| `overture-schema-theme-places` | `overture.schema.places` | +| `overture-schema-theme-divisions` | `overture.schema.divisions` | +| `overture-schema-theme-addresses` | `overture.schema.addresses` | +| `overture-schema-theme-base` | `overture.schema.base` | +| `overture-schema-common` | `overture.schema.common` | +| `overture-schema-system` | `overture.schema.system` | +| `overture-schema-validation` | `overture.schema.validation` | + +```python +from overture.schema.buildings import Building, BuildingClass +from overture.schema.transportation import Segment, Connector, RoadClass +from overture.schema.places import Place +``` + +### 2.3 From the CLI: what types exist? + +```bash +uv run overture-schema list-types +``` + +``` +address feature overture overture:theme=addresses +bathymetry feature overture overture:theme=base +building feature overture overture:theme=buildings +building_part feature overture overture:theme=buildings +connector feature overture overture:theme=transportation +division feature overture overture:theme=divisions +division_area feature overture overture:theme=divisions +division_boundary feature overture overture:theme=divisions +infrastructure feature overture overture:theme=base +land feature overture overture:theme=base +land_cover feature overture overture:theme=base +land_use feature overture overture:theme=base +place feature overture overture:theme=places +segment feature overture overture:theme=transportation +water feature overture overture:theme=base +``` + +Columns are: **type name**, then its **tags**. Group by a tag key: + +```bash +uv run overture-schema list-types --group-by overture:theme +``` + +``` +overture:theme=addresses (1) +→ address feature overture overture:theme=addresses + +overture:theme=base (6) +... +``` + +Those trailing words — `feature`, `overture`, `overture:theme=addresses` — are **tags**. +Every feature type carries a few, and they are how you select a subset of types without +naming each one. A tag is either a bare word (`feature`), or a key and value joined by +`=` (`overture:theme=buildings`), where the part before the colon says who defined it. + +Three options select by tag, and all three take a tag name and can be repeated. They are +shared by `list-types`, `validate`, and `json-schema`: + +| Option | Keeps a type when… | +|---|---| +| `--tag` | it has **any** of the tags you listed | +| `--filter` | it has **all** of the tags you listed | +| `--exclude` | drops it if it has any of the tags you listed | + +So this lists the buildings types and the places types, and nothing else: + +```bash +uv run overture-schema list-types --tag overture:theme=buildings --tag overture:theme=places +``` + +Tags are also the mechanism your own feature types use to join the set — see +[How tags work](CONCEPTS.md#how-tags-work). + +### 2.4 Ask the model itself (Python) + +This is the primary source. `Building` is an ordinary Python class, and Pydantic gives +every model a `model_fields` dict describing each field — its type, whether it's required, +its documentation, and its constraints. Nothing is generated or rendered here; this *is* +the schema. + +Start a session and look at what you have: + +```bash +uv run python +``` + +```python +>>> from overture.schema.buildings import Building +>>> len(Building.model_fields) +26 +>>> sorted(n for n, f in Building.model_fields.items() if f.is_required()) +['geometry', 'id', 'theme', 'type', 'version'] +``` + +Twenty-six fields, five of them required. Note the shape of that list: `id` and `geometry` +sit alongside `height` and `num_floors`, all at the same level. **A model is flat.** + +That is worth pinning down now, because the data often isn't. If you've seen an Overture +building as GeoJSON, most of its fields were tucked inside a `properties` object, with +only `id`, `geometry`, and `type` outside it. That outer wrapper is called the +**envelope** — the fixed set of keys GeoJSON puts around every feature, the same for a +building as for a lake. The model has no envelope; it appears only when a model is written +out as GeoJSON. [Section 2.5](#25-ask-the-generated-json-schema) meets it again, and +[the GeoJSON envelope](CONCEPTS.md#the-geojson-envelope) explains where it comes from. + +#### Every field at a glance + +```python +from overture.schema.buildings import Building + +print(Building.__doc__) + +for name, f in Building.model_fields.items(): + flag = "required" if f.is_required() else "optional" + print(f"{name:24} {flag:9} {f.annotation}") +``` + +``` +height optional typing.Optional[overture.schema.system.numeric.float64] +is_underground optional bool | None +num_floors optional typing.Optional[overture.schema.system.numeric.int32] +... +id required overture.schema.system.ref.id.Id +geometry required +theme required typing.Literal['buildings'] +type required typing.Literal['building'] +version required overture.schema.common.feature.FeatureVersion +class_ optional overture.schema.buildings.building.BuildingClass | None +``` + +Some of those types look stranger than they are. +`typing.Optional[overture.schema.system.numeric.float64]` is just **"a float, or nothing"** +— `Optional[X]` means the field may be absent, and `float64` is an ordinary Python `float` +that the schema has given a narrower name: + +```python +from overture.schema.system.numeric import float64 + +float64.__supertype__ # +``` + +The schema declares `float64`, `int32`, `uint8` and friends so a field can say how wide it +is on the wire — which Parquet column type it becomes, what range it accepts — while +staying a plain number in Python. Same story for `Id` and `FeatureVersion`: named types +wrapping `str` and `int`. Only `Literal['building']` is different: it means the field must +be exactly that one string. + +#### One field in detail + +Each entry carries the documentation and constraints from the model declaration: + +```python +f = Building.model_fields["height"] +print(f.description) # 'Height of the building or part in meters.\n\n...' +print(f.metadata) # [Gt(gt=0)] +print(f.alias) # None +print(Building.model_fields["class_"].alias) # 'class' +``` + +Note `class_` and its alias `class`. `class` is a Python keyword, so the field is named +`class_` on the model and `class` in the data — a mismatch that bites when serializing. +See [Gotchas](TROUBLESHOOTING.md#model-gotchas). + +The next section reads exactly this information back out of the generated JSON Schema, +where it goes by different names: `f.description` becomes `description`, the `[Gt(gt=0)]` +in `f.metadata` becomes `exclusiveMinimum: 0`, and `f.is_required()` becomes membership in +a list called `required`. Same facts, second rendering. + +### 2.5 Ask the generated JSON Schema + +**Why bother, when 2.4 already answered these questions in Python?** Because the JSON +Schema is the version other programs can read. It's a plain JSON document, so a validator +written in Go, a code generator that emits TypeScript, or a form builder that has never +heard of Pydantic can all consume it. Reach for it when you're feeding a tool rather than +answering a question — and when you want to see the rules in the *GeoJSON* shape, since +that's what it describes. + +Dump it for one type. Save it once rather than re-running the command for every question: + +```bash +uv run overture-schema json-schema --type building > building.schema.json +``` + +That file is a few thousand lines, so the queries below use **`jq`** — a small +command-line tool for querying JSON. You give it a path like `.properties.height` and it +prints what's there. Install it with `brew install jq` or `apt install jq`. Anything here +you'd rather do in Python, you can: `json.load()` and the same paths as dictionary keys. + +#### Finding a field + +Overture's fields are three levels down, and the reason is the GeoJSON envelope: the +document describes a GeoJSON feature, so `id` and `geometry` sit at the top and everything +else is inside `properties`. + +```bash +jq '.properties.properties.properties | keys' building.schema.json +``` +```json +["class","facade_color","facade_material","has_parts","height","is_underground", + "level","min_floor","min_height","names","num_floors","num_floors_underground", + "roof_color","roof_direction","roof_height","roof_material","roof_orientation", + "roof_shape","sources","subtype","theme","type","version"] +``` + +Three `properties` in a row, meaning something different each time: + +| Path | Means | +|---|---| +| `.properties` | "the fields of this document" — a JSON Schema keyword | +| `.properties.properties` | the GeoJSON field actually *named* `properties` | +| `.properties.properties.properties` | "the fields inside that one" — the keyword again | + +`id`, `geometry`, and `bbox` are missing from that list because they're on the envelope, +at `.properties.id` and so on — exactly where they sit in the data. + +**Don't count levels — let `jq` find the path for you.** This works for any field: + +```bash +jq -c 'paths | select(.[-1]=="height")' building.schema.json +``` +```json +["properties","properties","properties","height"] +``` + +Worth knowing because a wrong path returns `null` rather than an error — `jq` treats "no +such key" as an answer, so a `null` usually means you stopped a level too high, not that +the field is missing. + +#### Looking up one field's rules + +```bash +jq '.properties.properties.properties.height' building.schema.json +``` +```json +{ + "description": "Height of the building or part in meters.\n\nThis is the distance from the lowest point to the highest point.", + "exclusiveMinimum": 0, + "title": "Height", + "type": "number" +} +``` + +**That object is not a value of `height` — in the data, `height` is just a number like +`21.34`.** It's the *rules* for `height`: must be a number, must be greater than zero. +Every field gets an object like this even when the field is a bare number, because +there's nowhere else to hang a description and a constraint. + +Nobody wrote that JSON. Each line is rendered from the field's Python declaration in +`overture/schema/buildings/_common.py`: + +| Schema keyword | Comes from | +|---|---| +| `"type": "number"` | the `float64` annotation | +| `"exclusiveMinimum": 0` | `gt=0` — greater than, not greater-or-equal | +| `"description"` | the `description=` argument | +| `"title": "Height"` | generated by Pydantic from the field name | + +That is why the JSON Schema, the validation errors, the PySpark checks, and the generated +docs can't drift apart: they're all renderings of the same declaration. +[Section 6](#6-converting-the-schema-to-other-formats) produces the rest of them. + +#### Listing the valid values for a field + +Enums and shared structures sit at the top level under `$defs`, not inline: + +```bash +jq -r '.["$defs"] | keys[]' building.schema.json +``` +``` +BuildingClass BuildingSubtype FacadeMaterial NameRule NameVariant Names +PerspectiveMode Perspectives RoofMaterial RoofOrientation RoofShape Side SourceItem +``` + +So the full list of values a field accepts, without reading any Python: + +```bash +jq -r '.["$defs"].BuildingClass.enum[]' building.schema.json | head +``` +``` +agricultural +allotment_house +apartments +barn +beach_hut +boathouse +bridge_structure +bungalow +``` + +#### Which fields are required + +The envelope splits this across **two** lists, one per level: + +```bash +jq '.required' building.schema.json +jq '.properties.properties.required' building.schema.json +``` +```json +["type","id","geometry","properties"] +["theme","type","version"] +``` + +Read together they are the same five fields 2.4 gave you — `id` and `geometry` on the +envelope, `theme`, `type`, and `version` inside `properties`. Reading only the second list +and calling it "the required fields of a building" undercounts by exactly the fields the +envelope owns. + +`required` is always relative to the object it sits in; nested structures like `Names` and +`SourceItem` carry their own. **For this particular question the model is the easier +place to ask**, since it has no envelope to split the answer across: + +```python +from overture.schema.buildings import Building + +print(sorted(n for n, f in Building.model_fields.items() if f.is_required())) +``` +``` +['geometry', 'id', 'theme', 'type', 'version'] +``` + +Nobody maintains that list by hand — it falls out of whether a field has a default. For +how that works, and who decides, see +[What makes a field required](CONCEPTS.md#what-makes-a-field-required). + +#### What a subschema doesn't tell you + +The object you get back for `height` is the complete machine-checkable contract for +`height` — but only for `height`, and only in isolation. Three things it does *not* say: + +- **Whether the field may be omitted.** That lives in a sibling `required` array. A + subschema describes the value *if present*. +- **That the number is in meters.** "in meters" is in `description` — prose for humans. + Nothing rejects a value recorded in feet. +- **Anything about other fields.** The schema *can* express cross-field rules — that's + what `@require_any_of` and `@forbid_if` in `overture-schema-system` are for — but no + such rule ties `height` to `min_height`, so a building with a floor above its roof + validates clean: + +```python +import json, yaml +from overture.schema.buildings import Building + +d = yaml.safe_load(open("examples/buildings/building-polygon.yaml")) +d["properties"]["height"] = 5 +d["properties"]["min_height"] = 100 # a floor above the roof +print("accepted:", Building.model_validate_json(json.dumps(d)).height) +``` + +``` +accepted: 5.0 +``` + +Validation enforces the schema, not correctness. + +#### jq or Python? + +Use `jq` when you want the schema exactly as it ships, or when you're feeding it to +another tool — [generating an SDK](#81-generate-an-sdk-from-json-schema-any-language), for +instance. Use Python when you want to understand the model: `Building.model_fields` gives +you the same facts flat, with no envelope to walk, and a wrong field name raises instead +of quietly returning `null`. + +### 2.6 From Python: enumerate everything that's installed + +```python +from overture.schema.system.discovery import discover_models + +for key, model in sorted(discover_models().items(), key=lambda kv: kv[0].name): + print(f"{key.name:20} {key.entry_point:45} {sorted(key.tags)}") +``` + +``` +address overture.schema.addresses:Address ['feature', 'overture:theme=addresses'] +building overture.schema.buildings:Building ['feature', 'overture:theme=buildings'] +segment overture.schema.transportation:Segment ['feature', 'overture:theme=transportation'] +... +``` + +A `ModelKey` carries `.name`, `.entry_point` (`"module:Class"`), and `.tags` +(a `frozenset[str]`). Filter the same way the CLI does: + +```python +from overture.schema.system.discovery import TagSelector, discover_models, filter_models + +models = discover_models() + +buildings = filter_models( + models, TagSelector(include_any=("overture:theme=buildings",)) +) +``` + +`TagSelector` takes `include_any` (OR scope), `require_all` (AND narrowing), and +`exclude_any` (OR-NOT). An empty selector returns the input unchanged. + +### 2.7 Reading enum member documentation + +Enum members carry per-value docstrings, but `member.__doc__` falls back to the *class* +docstring when a member has none — so reading `__doc__` directly gives you misleading +results: + +```python +from overture.schema.buildings import RoofShape + +[(m.value, m.__doc__.strip()[:30]) for m in list(RoofShape)[:2]] +# [('dome', 'The shape of the roof.'), ('flat', 'The shape of the roof.')] +# ^ that's the class docstring repeated, not per-member documentation +``` + +Use the codegen extractor, which does the fallback detection for you: + +```python +from overture.schema.codegen.extraction.enum_extraction import extract_enum +from overture.schema.common.scoping.travel_mode import TravelMode + +spec = extract_enum(TravelMode) +for m in spec.members[:4]: + print(f"{m.value:14} {m.description or '—'}") +``` + +``` +vehicle — +motor_vehicle Includes car, truck and motorcycle +car — +truck — +``` + +`description` is `None` when the member has no documentation of its own. + +### 2.8 Generate browsable reference docs + +For sustained exploration, generate the full markdown reference and read it in your +editor: + +```bash +uv run overture-codegen generate --format markdown --output-dir ./schema-docs +``` + +You get one page per feature type, per enum, and per named type, with field tables, +prose constraint descriptions, cross-page links, and validated examples: + +``` +schema-docs/buildings/building.md +schema-docs/buildings/building_part.md +schema-docs/buildings/types/building_class.md +schema-docs/buildings/types/roof_shape.md +schema-docs/common/names.md +schema-docs/common/sources.md +schema-docs/system/numeric.md +... +``` + +Scope it to one theme with the same tag options: + +```bash +uv run overture-codegen generate --format markdown \ + --tag overture:theme=buildings --output-dir ./schema-docs +``` + +(Supplementary types from `common/` and `system/` are pulled in regardless, since the +feature pages link to them.) + +--- + +## 3. Writing code against the models + +Section 2 was about finding out what exists. This section is about using it: loading +real data into models, reading and changing it, and writing it back out. + +### 3.1 A complete example, start to finish + +Before any of the details, here is the whole job in one piece: load a feature, read it, +change it, write it back out, and confirm it still validates. Everything else in this +section is an explanation of a line in this snippet. + +```python +import json, yaml +from overture.schema.buildings import Building + +# 1. LOAD — the file is GeoJSON, so go through JSON mode +doc = yaml.safe_load(open("examples/buildings/building-polygon.yaml")) +b = Building.model_validate_json(json.dumps(doc)) +print("1. loaded :", b.id) + +# 2. READ — plain Python attributes; enums come back as enum members +print("2. read :", b.height, "m,", b.num_floors, "floors, class", b.class_.value) + +# 3. MODIFY — ordinary assignment +b.height = 25.0 +b.num_floors = 5 +print("3. changed:", b.height, "m,", b.num_floors, "floors") + +# 4. WRITE — by_alias=True so `class_` is written as `class` +out = b.model_dump(mode="json", by_alias=True, exclude_none=True) +print("4. wrote :", json.dumps(out)[:70], "...") + +# 5. CONFIRM — the output is valid input +again = Building.model_validate_json(json.dumps(out)) +print("5. re-read:", again.height, "m — round-trip holds") +``` + +``` +1. loaded : overture:buildings:building:1234 +2. read : 21.34 m, 4 floors, class parking +3. changed: 25.0 m, 5 floors +4. wrote : {"type": "Feature", "id": "overture:buildings:building:1234", "geometr ... +5. re-read: 25.0 m — round-trip holds +``` + +That is the shape of nearly every job: **validate in, work with plain Python objects, +serialize out.** In between, `b` is an ordinary object — attributes, assignment, no +special API. + +Three lines in there are load-bearing, and each gets a subsection below: + +| Line | Why it's written that way | Where | +|---|---|---| +| `model_validate_json(json.dumps(doc))` | the file is GeoJSON, which needs JSON mode | [3.2](#32-the-one-thing-to-understand-two-representations) | +| `b.class_.value` | the field is `class_` in Python, `class` in the data | [3.3](#33-reading-fields) | +| `by_alias=True, exclude_none=True` | without them the output won't re-validate | [3.4](#34-writing-data-back-out) | + +If the snippet above ran, you already know enough to be useful. Read on when one of those +lines bites you, or read straight through if you'd rather know why now. + +### 3.2 The one thing to understand: two representations + +Overture publishes data in one shape — flat and tabular, the column layout of the +Parquet release. The models also read and write GeoJSON, so the schema works with tools +that expect features rather than rows, and because that is the representation the +generated [JSON Schema](#61-json-schema) describes. **Which one you get depends on the +Pydantic mode you use, not on the data you pass.** + +| Shape | Looks like | Pydantic mode | Validate with | Dump with | +|---|---|---|---|---| +| **GeoJSON** | `id`/`geometry` at top level, everything else under `properties` | `json` | `model_validate_json()` | `model_dump(mode="json")`, `model_dump_json()` | +| **Flat / tabular** (Parquet-style) | every field at the top level | `python` | `model_validate()` | `model_dump(mode="python")` | + +This trips people up constantly. Passing a GeoJSON *dict* to `model_validate()` fails, +because `model_validate` is Python mode and Python mode expects the flat shape: + +```python +import json, yaml +from overture.schema.buildings import Building + +doc = yaml.safe_load(open("examples/buildings/building-polygon.yaml")) # GeoJSON-shaped + +Building.model_validate(doc) +# ValidationError: 3 validation errors for building +# theme Field required +# type Input should be 'building' [got 'Feature'] +# version Field required +``` + +The `type: Feature` in the error message is the tell: it read the GeoJSON envelope's +`type` as the feature's `type` field. + +The fix — round-trip through JSON so you're in JSON mode: + +```python +building = Building.model_validate_json(json.dumps(doc)) # works +``` + +Or, if you're already reading from a file or an HTTP response, skip the parse entirely: + +```python +building = Building.model_validate_json(open("building.geojson").read()) +``` + +#### Can't I just tell `model_validate` to use JSON mode? + +No. It's the obvious thing to try, and neither knob does it: + +```python +Building.model_validate(doc, context={"mode": "json"}) # still ValidationError +Building.model_validate(doc, strict=False) # still ValidationError +``` + +The mode isn't a setting you pass — in Pydantic it's determined by *which method you +call*. `model_validate` is Python mode; `model_validate_json` is JSON mode. The base +`Feature` class keys off exactly that: + +```python +@model_validator(mode="wrap") +def __validate_with_geo_json_support__(cls, data, handler, info): + if info.mode == "json": # <- set by the method, not by an argument + ... # unpack the GeoJSON envelope +``` + +So if you have a GeoJSON **dict** in hand, `json.dumps` it and use +`model_validate_json`. The round-trip is slightly wasteful but it is the supported path: + +```python +Building.model_validate_json(json.dumps(doc)) +``` + +Better still, avoid making the dict at all. If the GeoJSON came from a file or an HTTP +response, hand the raw text straight to `model_validate_json` and skip `json.loads` +entirely. The one case where you genuinely can't is YAML — there's no YAML mode, so +`yaml.safe_load` → `json.dumps` → `model_validate_json` is the route, as in the example +above. + +#### Reading the error + +The three errors from a mode mismatch are always the same shape, and worth recognising on +sight: + +``` +theme Field required +type Input should be 'building' [input_value='Feature'] +version Field required +``` + +`theme` and `version` are "missing" because they're really down inside `properties`, where +Python mode isn't looking. And `type` came back as `'Feature'` — the envelope's type, +which is the giveaway. **If you ever see `input_value='Feature'` in a validation error, +you passed GeoJSON to a Python-mode call.** + +For flat data — a Parquet row, a DuckDB result, a dict of columns — `model_validate` is +the right call: + +```python +row = { + "id": "...", + "theme": "buildings", + "type": "building", + "version": 1, + "geometry": ..., + "height": 21.34, + "class": "parking", +} +building = Building.model_validate(row) +``` + +### 3.3 Reading fields + +Model attributes are plain Python. Enums come back as enum members, geometry as a +`Geometry` wrapper around Shapely: + +```python +building.height # 21.34 +building.class_ # +building.class_.value # 'parking' +building.num_floors # 4 +type(building.geometry) # +``` + +### 3.4 Writing data back out + +Some fields are Python keywords, so the model attribute differs from the wire name +(`class_` on the model, `class` in the data). **`model_dump()` uses attribute names by +default**, which produces output that will not validate back: + +```python +d = building.model_dump(mode="json") +sorted(d["properties"]) +# ['class_', 'ext_bar', 'height', ...] ← 'class_' is wrong for the wire + +Building.model_validate(building.model_dump(mode="python")) +# ValidationError: invalid extra field name: class_ +``` + +With `by_alias=True` both round-trips work: + +```python +geojson = building.model_dump(mode="json", by_alias=True, exclude_none=True) +sorted(geojson["properties"]) +# ['class', 'ext_bar', 'height', 'is_underground', 'level', 'num_floors', ...] + +Building.model_validate_json(json.dumps(geojson)) # OK +Building.model_validate( + building.model_dump(mode="python", by_alias=True, exclude_none=True) +) # OK +``` + +Same for `model_dump_json()`: + +```python +'"class":' in building.model_dump_json() # False ← emits "class_" +'"class":' in building.model_dump_json(by_alias=True) # True +``` + +**Rule of thumb: `by_alias=True` on every dump, unless you specifically want Python +attribute names.** `exclude_none=True` is usually what you want too — otherwise you get +every unset optional field as an explicit `null`. + +### 3.5 Working with `Segment` and other unions + +`Segment` is a discriminated union type alias over `RoadSegment`, `RailSegment`, and +`WaterSegment` — not a model class. It has no `model_validate`: + +```python +from overture.schema.transportation import Segment + +type(Segment) # +Segment.model_validate({...}) +# AttributeError: model_validate +``` + +Wrap it in a `TypeAdapter`: + +```python +from pydantic import TypeAdapter +from overture.schema.transportation import Segment + +segments = TypeAdapter(Segment) + +seg = segments.validate_json(raw_geojson_string) # JSON mode → GeoJSON +seg = segments.validate_python(flat_row) # Python mode → flat +type(seg).__name__ # 'RoadSegment' +``` + +Build the `TypeAdapter` once and reuse it; construction is the expensive part. + +The concrete arms *are* ordinary classes if you know which one you want: + +```python +from overture.schema.transportation import RoadSegment +``` + +### 3.6 Validating without knowing the type + +`overture-schema-validation` checks a record against every installed model and returns +whichever matched: + +```python +from overture.schema.validation import validate, validate_json + +feature = validate_json(geojson_string) # JSON mode → GeoJSON shape +type(feature).__name__ # 'Building' + +feature = validate(flat_dict) # Python mode → flat shape +``` + +Both raise `pydantic.ValidationError` when nothing matches. The same mode rule from +[The one thing to understand first](#32-the-one-thing-to-understand-two-representations) applies: `validate()` is +Python mode and wants flat data, `validate_json()` is JSON mode and wants GeoJSON. + +Which models participate is resolved at runtime by entry-point discovery — install more +theme packages and these functions accept more. + +### 3.7 Generating JSON Schema in code + +```python +from overture.schema.system.json_schema import json_schema +from overture.schema.buildings import Building +from overture.schema.places import Place + +schema = json_schema(Building) +schema["title"] # 'building' +sorted(schema) # ['$defs', 'additionalProperties', 'description', +# 'properties', 'required', 'title', 'type'] + +union = json_schema(Building | Place) # unions work too → anyOf +``` + +Use this rather than Pydantic's `model_json_schema()`. The Overture generator treats +`T | None = None` as "omit when unset" instead of Pydantic's "nullable with a null +default", which is what the data actually means. + +--- + +## 4. The three CLIs + +Installing the workspace gives you three commands, from three different packages. + +| Command | Purpose | Full reference | +|---|---|---| +| `overture-schema` | Validate files, emit JSON Schema, list types | [`packages/overture-schema-cli/`](packages/overture-schema-cli/) | +| `overture-codegen` | Generate markdown docs and PySpark expressions | [`overture-schema-codegen/README.md`](packages/overture-schema-codegen/README.md) | +| `overture-validate` | Validate Parquet/S3 data at scale with Spark | [`overture-schema-pyspark/README.md`](packages/overture-schema-pyspark/README.md) | + +Prefix each with `uv run` inside the repo, or activate the venv. + +**This section shows what each command is for and one working invocation of each.** The +complete option lists live in the package READMEs linked above, which are versioned with +the packages they document. + +### 4.1 `overture-schema` + +``` +Usage: overture-schema [OPTIONS] COMMAND [ARGS]... + +Commands: + json-schema Generate JSON schema for Overture Maps types. + list-types List all available types. + validate Validate Overture Maps data against schemas. +``` + +All three subcommands take the shared `--tag` / `--filter` / `--exclude` options from +[From the CLI: what types exist?](#23-from-the-cli-what-types-exist). `validate` and `json-schema` also take +`--type NAME` to target one type directly. + +```bash +# What types do I have? +overture-schema list-types +overture-schema list-types --group-by overture:theme + +# Validate +overture-schema validate data.geojson +overture-schema validate - < data.geojson +overture-schema validate --type building data.json +overture-schema validate --tag overture:theme=buildings data.json +overture-schema validate --show-field id data.json + +# JSON Schema +overture-schema json-schema > all-types.json +overture-schema json-schema --type building > building.json +overture-schema json-schema --tag overture:theme=buildings > buildings.json +``` + +Exit codes: `0` on success, `1` on validation failure — so it drops into CI directly. + +#### Local files or remote? + +It depends which CLI, and the two answer differently. + +| Command | Remote paths | How | +|---|---|---| +| `overture-schema validate` | **no** | pipe through stdin with `-` | +| `overture-validate` (PySpark) | **yes** | `s3a://` natively, anonymous credentials preconfigured | + +`overture-schema validate` takes a filesystem path only. Hand it a URL and it fails — +note that it even mangles the `//`, because the argument is parsed as a path: + +```bash +overture-schema validate https://raw.githubusercontent.com/OvertureMaps/schema/main/examples/buildings/building-polygon.yaml +``` + +``` +Error: 'https:/raw.githubusercontent.com/.../building-polygon.yaml' is not a file. +``` + +The fix is the `-` argument, which reads stdin. Anything that can fetch bytes can feed it: + +```bash +curl -sSf https://raw.githubusercontent.com/OvertureMaps/schema/main/examples/buildings/building-polygon.yaml \ + | overture-schema validate - +``` + +``` +✓ Successfully validated +``` + +That works for anything on stdin — `aws s3 cp ... -`, a database query, a generator +script, another program's output. The only thing you lose is the filename in the output, +which becomes ``. + +`overture-validate` is the opposite: it's built for remote data. `s3a://` paths are +detected automatically and configured with anonymous credentials, so the public Overture +release bucket needs no setup: + +```bash +overture-validate segment s3a://overturemaps-us-west-2/release/2026-07-22.0 +``` + +> **Don't hardcode a release version.** The bucket keeps only the current release, so any +> version written into a script or a doc stops working at the next publish. Ask the bucket +> instead: +> +> ```bash +> curl -sS "https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/?list-type=2&prefix=release/&delimiter=/" \ +> | tr '<' '\n' | grep -oE 'Prefix>release/[^/]+' | sed 's|Prefix>release/||' | sort -r | head -1 +> ``` +> +> ``` +> 2026-07-22.0 +> ``` +> +> Then use it: +> +> ```bash +> RELEASE=$(curl -sS "https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/?list-type=2&prefix=release/&delimiter=/" \ +> | tr '<' '\n' | grep -oE 'Prefix>release/[^/]+' | sed 's|Prefix>release/||' | sort -r | head -1) +> overture-validate segment "s3a://overturemaps-us-west-2/release/$RELEASE" +> ``` +> +> The version shown in the examples here was current when this was written; treat it as a +> placeholder, not a fact. + + +That difference is not arbitrary. `overture-schema validate` is for a file you're looking +at — an example, a fixture, one feature you're debugging. `overture-validate` is for a +release: millions of rows, read in parallel by Spark, where "download it first" isn't an +option. + +### 4.2 `overture-codegen` + +Two commands: `generate` writes code or docs from the discovered models, `list` shows +what it discovered. `generate` takes `--format markdown` or `--format pyspark`, the +same `--tag`/`--filter`/`--exclude` options as `overture-schema`, and an `--output-dir`. + +```bash +# Markdown reference docs +overture-codegen generate --format markdown --output-dir ./schema-docs +overture-codegen generate --format markdown --tag overture:theme=places --output-dir ./out + +# PySpark validation expressions (this is what `make generate-pyspark` runs) +overture-codegen generate --format pyspark \ + --output-dir packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated \ + --test-output-dir packages/overture-schema-pyspark/tests/generated +``` + +### 4.3 `overture-validate` + +Validates real data volumes with Spark. Requires the generated expression tree, so run +`make install` or `make generate-pyspark` first. + +Takes a feature type and a path: + +```bash +overture-validate building local.parquet +overture-validate segment s3a://overturemaps-us-west-2/release/2026-07-22.0 +overture-validate place data.parquet --count-only +overture-validate segment data.parquet --suppress version:bounds -o violations.parquet +``` + +It handles S3A and anonymous credentials for the public Overture bucket automatically, +and expands a release root into the Hive partition path for you. The flags shown above +are the common ones; for the full list — output paths, error-row limits, Spark config, +schema-mismatch and check suppression — see +[`packages/overture-schema-pyspark/README.md`](packages/overture-schema-pyspark/README.md). + +--- + +## 5. Validating data + +Three tiers, pick by data size. + +### 5.1 One file, or a handful — the CLI + +Accepts JSON, YAML, and GeoJSON. A single feature, a JSON array of features, or a +`FeatureCollection` all work: + +```bash +overture-schema validate examples/buildings/building-polygon.yaml +``` + +``` +✓ Successfully validated examples/buildings/building-polygon.yaml +``` + +Failures come back as a rendered table showing the offending value in context: + +```bash +overture-schema validate --show-field id counterexamples/buildings/negative-height.json +``` + +``` + ─ Validation Failed id=foo ──────────────────────────────────────────────────── + ... + id "foo" + version 0 + height -1.23 ← Input should be greater than 0 + ────────────────────────────────────────────────────────────────────────────── +``` + +For a collection, errors are indexed and labeled by the model that best fit: + +``` + ─ [1] (Building) ────────────────────────────────────────────────────────────── + ... + version 0 + height -1.23 ← Input should be greater than 0 + ────────────────────────────────────────────────────────────────────────────── +``` + +Narrow the candidate set to sharpen the error messages — with `--type building` the CLI +stops guessing which model you meant: + +```bash +overture-schema validate --type building data.json +``` + +Every check comes from the model definition — nothing is hand-written per file. Copy an +example, edit a field, and you can watch each kind fire: + +| Edit | What validation says | +|---|---| +| `class: parking` → `class: skyscraper` | `Input should be 'agricultural', 'allotment_house', ...` | +| `num_floors: 4` → `num_floors: 4.7` | `Input should be a valid integer, got a number with a fractional part` | +| delete the `theme:` line | `Ambiguous: Data matches multiple types equally` | + +What it does **not** catch: free-form string fields accept any string, and nothing checks +fields against each other. Validation enforces the schema, not correctness. + +### 5.2 In a Python pipeline + +```python +from pydantic import ValidationError +from overture.schema.validation import validate_json + +ok, bad = 0, [] +for line in open("features.ndjson"): + try: + validate_json(line) + ok += 1 + except ValidationError as e: + bad.append((line[:60], e.errors())) + +print(f"{ok} valid, {len(bad)} invalid") +``` + +`e.errors()` gives you structured dicts with `loc`, `msg`, `type`, and `input` — the +right thing to log or turn into a report. Package reference: +[`packages/overture-schema-validation/README.md`](packages/overture-schema-validation/README.md). + +If you know the type, validate against it directly for better errors and speed: + +```python +from overture.schema.buildings import Building + +Building.model_validate_json(line) +``` + +### 5.3 At scale — PySpark + +```python +from pyspark.sql import SparkSession +from overture.schema.pyspark import validate_model, explain_errors + +spark = SparkSession.builder.getOrCreate() +df = spark.read.parquet("s3a://.../theme=buildings/type=building/") + +result = validate_model(df, "building") +result.evaluated.cache() + +total = result.evaluated.count() +errors = result.error_rows().count() +print(f"{errors} / {total} rows with errors") + +if errors: + violations = explain_errors(result.evaluated, result.checks) + violations.select("id", "field", "check", "message").show(truncate=False) +``` + +`validate_model` accepts either the short name (`"building"`) or the full entry-point key +(`"overture.schema.buildings:Building"`). It looks up the feature type in the registry, +compares the DataFrame schema against the expected one, and evaluates every check in a +single pass — no per-row Python, so it scales. + +| Function | Returns | Purpose | +|---|---|---| +| `validate_model(df, type)` | `ValidationResult` | Registry lookup, schema comparison, check evaluation | +| `result.error_rows()` | `DataFrame` | Rows with at least one violation | +| `explain_errors(evaluated, checks)` | `DataFrame` | One row per violation: `field`, `check`, `message` | +| `model_names()` | `list[str]` | Available type names | + +Tuning, partition handling, and the rest of the PySpark API are documented in +[`packages/overture-schema-pyspark/README.md`](packages/overture-schema-pyspark/README.md). +If `model_names()` comes back empty, see +[Troubleshooting](TROUBLESHOOTING.md#model_names-returns--or-keyerror-on-a-feature-type). + +### 5.4 Validating the schema itself + +If you're changing the models rather than the data: + +```bash +make check +make test +make update-baselines +``` + +The theme packages carry golden-file baseline tests of their generated JSON Schema, so +unintended schema drift fails CI. After an intentional change, run `make +update-baselines` and inspect the `git diff` on the regenerated golden files before +committing. + +--- + +## 6. Converting the schema to other formats + +Three built-in targets, plus everything reachable through JSON Schema. + +| Target | Command | Output | +|---|---|---| +| JSON Schema | `overture-schema json-schema` | A JSON Schema document on stdout | +| Markdown | `overture-codegen generate --format markdown` | Docusaurus-ready reference pages | +| PySpark | `overture-codegen generate --format pyspark` | Python modules of `Check` builders + `StructType` | +| Spark `StructType` | (Python, via the pyspark registry) | A live Spark schema object | + +### 6.1 JSON Schema + +The interop format — this is your bridge to every other ecosystem. + +```bash +# One type +overture-schema json-schema --type building > building.schema.json + +# One theme +overture-schema json-schema --tag overture:theme=transportation > transportation.schema.json + +# Everything (an `anyOf` over all installed types) +overture-schema json-schema > overture.schema.json +``` + +A single type produces a self-contained document with its dependencies inlined under +`$defs`: + +``` +$ jq 'keys' building.schema.json +["$defs", "additionalProperties", "description", "properties", "required", "title", "type"] + +$ jq '.title, (.["$defs"] | length)' building.schema.json +"building" +13 +``` + +All types produce `{"anyOf": [...], "$defs": {...}}`. + +In Python: + +```python +from overture.schema.system.json_schema import json_schema +from overture.schema.buildings import Building + +schema = json_schema(Building) +``` + +### 6.2 Markdown + +Covered in [Generate browsable reference docs](#28-generate-browsable-reference-docs). Output is Docusaurus-flavored +(frontmatter plus `_category_.json` files), but it's plain markdown underneath and reads +fine in any editor or static site generator. + +### 6.3 PySpark expressions and Spark schemas + +```bash +overture-codegen generate --format pyspark --output-dir ./ps --test-output-dir ./ps-tests +``` + +You get one module per feature type, mirroring the Python package layout: + +``` +ps/overture/schema/buildings/building.py +ps/overture/schema/buildings/building_part.py +ps-tests/overture/schema/buildings/test_building.py +``` + +Each module is auto-generated (`# Do not edit`) and contains one builder function per +constraint, returning a `Check` with an unevaluated PySpark `Column`: + +```python +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0), + shape=CheckShape.SCALAR, + root_field="version", + ) +``` + +Plus a `MODEL_VALIDATION` constant pairing the checks with the expected `StructType`. + +**To get a Spark schema for a feature type** — useful for `spark.read.schema(...)`, +Delta table creation, or comparing against your own tables: + +```python +from overture.schema.pyspark._registry import REGISTRY +from overture.schema.pyspark.validate import resolve_entry_point_key + +key = resolve_entry_point_key("building", REGISTRY) +struct = REGISTRY[key].schema + +type(struct).__name__ # 'StructType' +[(f.name, f.dataType.simpleString()) for f in struct.fields][:5] +``` + +``` +[('id', 'string'), + ('bbox', 'struct'), + ('geometry', 'binary'), + ('theme', 'string'), + ('type', 'string')] +``` + +`REGISTRY` is keyed by the full entry-point string +(`"overture.schema.buildings:Building"`), which is why the `resolve_entry_point_key` +step is there — it accepts the short alias too. `ModelValidation` exposes `.schema`, +`.checks`, and `.geometry_types`. + +Since `StructType` has `.json()` and `.jsonValue()`, this is also your route to an +Arrow/Parquet schema. + +For the generator's own architecture and programmatic API — the shape extraction layer +these modules are rendered from — see +[`packages/overture-schema-codegen/README.md`](packages/overture-schema-codegen/README.md). +To write a new output format, see [AUTHORING.md](AUTHORING.md#write-a-new-codegen-target). + +--- + +## 7. Using the packages from your own project + +Everything above assumes you're working *inside* the schema repo. If instead you're +building your own application that depends on these models, you don't use the workspace +at all — you point `uv` at the package directories on disk. + +Depend on the **theme packages you actually need**, not the workspace root: + +```toml +# myapp/pyproject.toml +[project] +name = "myapp" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [ + "overture-schema-theme-buildings", + "overture-schema-cli", +] + +[tool.uv.sources] +overture-schema-theme-buildings = { path = "/path/to/schema/packages/overture-schema-theme-buildings", editable = true } +overture-schema-cli = { path = "/path/to/schema/packages/overture-schema-cli", editable = true } +``` + +```bash +cd myapp +uv sync +uv run python -c "from overture.schema.buildings import Building; print(Building.__name__)" +``` +``` +Building +``` + +`editable = true` means edits in your schema clone take effect immediately in `myapp` — +useful if you're changing both. + +### 7.1 The payoff: install set = runtime set + +In the `myapp` project above, only the buildings theme is installed. So: + +```bash +cd myapp && uv run overture-schema list-types +``` +``` +building feature overture:theme=buildings +building_part feature overture:theme=buildings +``` + +Two types, not fifteen. **The same CLI binary, scoped by what's installed.** Nothing was +configured to make that happen — the models register themselves through entry points, and +the tooling discovers whatever is present. + +This is worth internalizing early, because it's the whole extension story: add your own +package that registers models, and your feature types appear in `list-types`, validate +through the same commands, and show up in generated docs, alongside Overture's. See +[Register your own feature types](AUTHORING.md#register-your-own-feature-types). + +### 7.2 If you'd rather have everything + +`overture-schema` is a metapackage depending on all six themes plus validation and the +CLI: + +```toml +[tool.uv.sources] +overture-schema = { path = "/path/to/schema/packages/overture-schema", editable = true } +``` + +One caveat: `import overture.schema` gives you nothing directly. It's a namespace root +that ships only a `py.typed` marker — no models, no functions. Always import from the +theme packages: + +```python +from overture.schema.buildings import Building # ✓ +from overture.schema import Building # ✗ ImportError +``` + + +--- + +### 7.3 What changes once these packages are published + +Some of this section is scaffolding for the fact that nothing is on a package index yet. +Worth knowing which parts, so you don't over-invest in learning them. + +| Part of this section | After publishing | +|---|---| +| What a workspace is, the shared `.venv`, `uv run` | **Stays** — but becomes reading for contributors only | +| `git clone` + `uv sync --all-packages` | **Stays** — contributors only | +| `make generate-pyspark` | **Gone for consumers.** Published wheels ship the generated expressions already. | +| The `SPARK_HOME` fix | **Stays forever.** It's an environment problem, unrelated to packaging. | +| Empty registry, `exclude-newer` warning | Contributors only | +| The `[tool.uv.sources]` path blocks above | **Deleted entirely.** This is the pure workaround. | +| "Install set = runtime set" | **Stays.** That's entry-point discovery, not packaging. | + +The whole of this subsection collapses to one line: + +```bash +uv add overture-schema-theme-buildings overture-schema-cli +``` + +or, for everything: + +```bash +uv add overture-schema +``` + +**On the PySpark step specifically:** the release pipeline already handles it. +`.github/workflows/publish-python-packages.yaml` runs `make generate-pyspark` before +`uv build`, and aborts the release if the resulting wheel contains no +`expressions/generated/*.py`. Its own comment explains why: + +> the tree must be generated here before the build, or the published wheel ships without +> its `expressions/generated/` modules and `validate_model()` discovers nothing to run + +So consumers of a published wheel never run that step. + +**Which index?** The publish workflow currently pushes to AWS CodeArtifact (the +`overture-pypi` domain), even though its step names say PyPI. `CONTRIBUTING.md` describes +version bumps reaching public PyPI while interim builds stay internal. Either way the +consumer instruction is a one-line install — only the index URL differs. + +Nothing in sections 2 through 7 changes. The behavior you learn there — entry-point +discovery, the two representations, `by_alias`, `TypeAdapter` for `Segment` — is independent +of how the packages get onto your machine. + +--- + +## 8. Building tools on the models + +Sections 1–7 use the schema. This one builds *on* it: generating a client library in +another language, or writing your own command-line tool that stays correct as the +installed packages change. Neither requires touching the schema itself — for that, see +[AUTHORING.md](AUTHORING.md). + +### 8.1 Generate an SDK from JSON Schema (any language) + +Emit JSON Schema, then hand it to a standard generator. Both of these were run against +the output of `overture-schema json-schema --type building` and produced working code. + +**Python (datamodel-code-generator):** + +```bash +uv run overture-schema json-schema --type building > building.schema.json + +uvx --from datamodel-code-generator datamodel-codegen \ + --input building.schema.json \ + --input-file-type jsonschema \ + --output building_models.py +``` + +Produces standalone Pydantic v2 models with no Overture dependency — useful for a +service that shouldn't take the whole workspace as a dependency: + +```python +# generated by datamodel-codegen: +# filename: building.schema.json +from pydantic import BaseModel, ConfigDict, Field, confloat, conint, constr +``` + +**TypeScript (quicktype):** + +```bash +npx -y quicktype --src-lang schema --lang typescript \ + -o Building.ts building.schema.json +``` + +Field descriptions survive as JSDoc: + +```typescript +/** + * Buildings are man-made structures with roofs that exist permanently in one place. + * ... + */ +export interface Building { + bbox?: [number, number, number, number, ...number[]]; + /** The building's footprint or roofprint... */ + geometry: Geometry; +``` + +quicktype also targets Go, Rust, Java, Kotlin, Swift, C#, and others from the same input. +Anything that reads JSON Schema — OpenAPI toolchains, `go-jsonschema`, `schemars`, +`jsonschema2pojo` — works the same way. + +The tradeoff: you get types and structural validation, but not the semantic layer. +Cross-field model constraints (`@require_any_of`, `@radio_group`) do translate into JSON +Schema `if`/`then`/`anyOf` constructs, but domain-specific error messages and the +NewType vocabulary flatten out. + +### 8.2 Build a CLI on discovery and tags + +If you're staying in Python, don't hardcode a type list. Discover, and let the installed +packages decide what exists — same as the built-in CLI. Your tool then automatically +covers new themes and third-party extensions. + +```python +import click +from overture.schema.system.discovery import discover_models, filter_models, TagSelector +from overture.schema.cli.tag_options import tag_selection_options, build_selector + + +@click.command() +@tag_selection_options # gives you --tag / --filter / --exclude for free +def report(tags, filters, excludes): + """Report field counts for the selected feature types.""" + models = filter_models(discover_models(), build_selector(tags, filters, excludes)) + for key, model in sorted(models.items(), key=lambda kv: kv[0].name): + n = len(model.model_fields) if hasattr(model, "model_fields") else "—" + click.echo(f"{key.name:20} {n}") +``` + +`overture.schema.cli` also exports `resolve_types`, `create_union_type_from_models`, +`load_input`, `perform_validation`, `handle_validation_error`, and +`handle_generic_error` — so you can reuse the file-loading and error-rendering behavior +rather than reimplementing it. diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md new file mode 100644 index 000000000..3203ca2ba --- /dev/null +++ b/TROUBLESHOOTING.md @@ -0,0 +1,363 @@ +# Troubleshooting + +This page is meant to help you handle **errors** and navigate **gotchas** when you are installing and working with the schema models. It's written for early testers of the Pydantic schema who are installing locally, prior to the publication of the packages on PyPI and the `v2.0.0` launch of the schema. + + + +| Symptom | Section | +|---|---| +| `ModuleNotFoundError: No module named 'overture'` | [Wrong interpreter](#modulenotfounderror-no-module-named-overture) | +| `command not found: overture-schema` | [Missing `uv run`](#command-not-found-overture-schema) | +| A wall of `exclude-newer` warning text, and a dirty `uv.lock` | [uv is out of date](#uv-warns-about-exclude-newer--and-quietly-rewrites-your-lockfile) | +| `FileNotFoundError: ... spark-submit` | [Stale `SPARK_HOME`](#filenotfounderror-on-spark-submit) | +| `model_names()` returns `[]`, or `KeyError` on a feature type | [PySpark expressions not generated](#model_names-returns--or-keyerror-on-a-feature-type) | +| `wc: #: open: No such file or directory` when pasting | [zsh and `#`](#-is-not-a-comment-in-interactive-zsh) | +| A pasted command turns into something from your shell history | [zsh and `!`](#-runs-a-command-out-of-your-history) | +| `ValidationError` you didn't expect from a model | [Model gotchas](#model-gotchas) | + +--- + +## Install and environment + +### `ModuleNotFoundError: No module named 'overture'` + +You started Python without `uv run`, so you're in your system or `pyenv` interpreter +rather than the project's `.venv`. Use `uv run python` instead of `python`. See +[Running Python against the models](SCHEMA_GUIDE.md#21-running-python-against-the-models). + +### `command not found: overture-schema` + +You're missing the `uv run` prefix, or you're not in the repo root. Every command in this +guide is `uv run overture-schema …`, run from the directory containing `pyproject.toml`. + +### uv warns about `exclude-newer` — and quietly rewrites your lockfile + +``` +warning: Failed to parse `pyproject.toml` during settings discovery: + TOML parse error at line 10, column 17 + | + 10 | exclude-newer = "1 week" + | ^^^^^^^^ + failed to parse year in date "1 week": failed to parse "1 we" as year ... +``` + +> **Do you actually have this problem?** Only if that banner appears when you run a +> command. If your commands print their output cleanly, skip this entire subsection — +> there is nothing to fix, and the four steps below are not maintenance you need to +> perform. Confirm in one line: +> +> ```bash +> uv run overture-schema --version +> ``` +> +> A single line of output means you're fine. A wall of warning text above it means you have a problem to sort out. + +**If you do see it: this is not cosmetic. Fix it before you do anything else.** + +The root `pyproject.toml` writes `exclude-newer` as a relative duration (`"1 week"`), +which caps how new a package `uv` will consider during dependency resolution. Only +reasonably recent `uv` versions parse that form. + +An older `uv` fails to read the whole `[tool.uv]` block, says so, **and carries on +without the cap** — then, because its resolution no longer matches the committed +lockfile, rewrites `uv.lock` in place. You end up with a modified tracked file you never +asked to change: + +```bash +git status --short uv.lock +git diff --stat uv.lock +``` + +On an affected machine: + +``` + M uv.lock + uv.lock | 807 +++++++++++++++++++------------------- + 1 file changed, 464 insertions(+), 343 deletions(-) +``` + +> **Both commands printing nothing is the healthy result.** `git status --short` and +> `git diff --stat` say nothing about a file that hasn't changed. Empty output here means +> your lockfile is untouched and there is nothing to fix. If you'd rather have an explicit +> answer than read silence: +> +> ```bash +> git diff --quiet uv.lock && echo "uv.lock: unmodified" || echo "uv.lock: MODIFIED" +> ``` + +It drops the `[options]` block that records the resolution settings, and pulls in +dependency versions past the cutoff the repo intended. Nothing breaks immediately — the +install works fine — but you're now building against a different dependency set than the +project pinned, and `git status` is dirty. + +**Step 1 — check your version:** + +```bash +uv --version +brew outdated uv +``` + +**Step 2 — upgrade:** + +```bash +brew upgrade uv +``` + +**Step 3 — restore the lockfile if the old `uv` already rewrote it:** + +```bash +git checkout uv.lock +``` + +**Step 4 — confirm:** + +```bash +uv sync --all-packages --locked +``` + +``` +Resolved 64 packages in 12ms +Audited 60 packages in 0.81ms +``` + +`--locked` fails outright if the lockfile isn't authoritative, so a clean pass means your +`uv`, the lockfile, and your `.venv` all agree. Run `git status --short uv.lock` once more +too — it should print nothing, which means the file is unmodified. + +> **If you see `error: The lockfile at uv.lock needs to be updated, but --locked was +> provided`,** you're at step 3, not step 4. A previous `uv sync` under the old `uv` +> already modified the lock. `git checkout uv.lock` and re-run. + +Once you're on a current `uv`, ordinary use leaves the lockfile alone — including the +`uv sync --all-packages --all-extras` that `make install`, `make check`, and +`make generate-pyspark` all run internally. + +--- + +### `FileNotFoundError` on `spark-submit` + +``` +FileNotFoundError: [Errno 2] No such file or directory: +'/opt/homebrew/Cellar/apache-spark/3.5.3/libexec/./bin/spark-submit' +``` + +**This has nothing to do with the generation step in "Do you need PySpark?".** It's a stale `SPARK_HOME` +environment variable, and it will happen whether or not you ran `make generate-pyspark`. + +What's going on: you have a `SPARK_HOME` exported in your shell profile pointing at a +**version-specific Homebrew path**. Homebrew has since upgraded Spark, so that exact +directory no longer exists: + +```bash +echo $SPARK_HOME +ls /opt/homebrew/Cellar/apache-spark/ +``` + +``` +/opt/homebrew/Cellar/apache-spark/3.5.3/libexec +4.0.0 +``` + +The variable names `3.5.3`; the only version present is `4.0.0`. It points at nothing. + +Meanwhile the `pyspark` in your venv (4.2.0) **ships its own copy of Spark** and doesn't +need the Homebrew one at all. But when `SPARK_HOME` is set, PySpark obeys it and looks +for `spark-submit` at that dead path. + +**Confirm that's your problem:** + +```bash +env -u SPARK_HOME uv run python -c " +from pyspark.sql import SparkSession +s = SparkSession.builder.master('local[1]').getOrCreate() +print('SUCCESS — spark', s.version) +s.stop()" +``` + +``` +SUCCESS — spark 4.2.0 +``` + +**Fix it permanently.** The variable is set in *two* files — fixing only one won't help, +because `.zprofile` runs for login shells and `.zshrc` for interactive ones: + +``` +~/.zshrc:8 export SPARK_HOME=/opt/homebrew/Cellar/apache-spark/3.5.3/libexec +~/.zshrc:9 export PATH="$SPARK_HOME/bin/:$PATH" +~/.zprofile:5 export SPARK_HOME=/opt/homebrew/Cellar/apache-spark/3.5.3/libexec +~/.zprofile:6 export PATH="$SPARK_HOME/bin/:$PATH" +``` + +Pick one: + +- **Simplest — delete all four lines.** If you only use Spark through Python projects + like this one, you don't need `SPARK_HOME` at all; each venv's `pyspark` brings its + own. +- **Keep Homebrew Spark for other work — stop hardcoding the version.** Replace the two + `SPARK_HOME` lines with the version-independent symlink Homebrew maintains: + + ```bash + export SPARK_HOME=/opt/homebrew/opt/apache-spark/libexec + ``` + + This survives upgrades. But note it points at Spark **4.0.0** while this project's + `pyspark` is **4.2.0** — mismatched versions cause their own confusing failures, so + prefer the first option while working in this repo. + +Then open a new terminal, or `exec zsh`, and re-run the check above. + +> **Per-shell workaround** if you don't want to touch your profile right now: +> `unset SPARK_HOME` in the terminal you're working in. It lasts until you close it. + +### `model_names()` returns `[]`, or `KeyError` on a feature type + +The PySpark validation expressions are generated code that is **not committed to git** — +they're in `.gitignore`, and `uv sync` alone cannot produce them. Skipping that step +doesn't raise an error; it leaves you with an empty registry, which is why this is easy +to miss. + +```bash +make generate-pyspark +``` + +Then confirm — don't infer it from the output, which is silent by design: + +```bash +uv run python -c "from overture.schema.pyspark import model_names; print(model_names())" +``` + +Before, an empty list. After, 30 entries — 15 feature types, each reachable by two names: + +``` +['address', 'bathymetry', 'building', 'building_part', 'connector', 'division', ...] +``` + +Only people working from a git clone ever need this. Published wheels ship the generated +expressions already; see +[Why generated code is gitignored](CONCEPTS.md#why-generated-code-is-gitignored). + +--- + +## Pasting commands into zsh + +macOS defaults to **zsh**, and two of its interactive behaviors mangle commands copied +out of documentation. Neither affects scripts or non-interactive shells. + +### `#` is not a comment in interactive zsh + +macOS defaults to **zsh**, and interactive zsh does *not* treat `#` as a comment unless +you turn that on. Paste a line like this and zsh hands `#`, `→`, and `15` to `wc` as +filenames: + +``` +find ... | wc -l # → 15 +wc: #: open: No such file or directory +wc: →: open: No such file or directory +wc: 15: open: No such file or directory + 0 total +``` + +A line that *starts* with `#` fails more obviously — `command not found: #`. + +This guide keeps `#` comments only as standalone label lines inside multi-command blocks, +never trailing after a command. To paste those blocks whole, enable comments once: + +```bash +setopt interactive_comments +``` + +Add it to `~/.zshrc` to make it permanent. Otherwise, skip the `#` lines when copying — +they are labels, not commands. Scripts and non-interactive shells are unaffected; this is +purely an interactive-zsh behavior. + +### `!` runs a command out of your history + +This one is worth understanding because it can do real damage. In an interactive shell, +`!` triggers **history expansion**, and it fires *inside double quotes*. `!r` means "the +most recent command starting with `r`" — the shell splices that command's text into your +line before running it. + +So a Python one-liner containing `{value!r}`, pasted into zsh as + +``` +uv run python -c "... f'default={f.default!r}' ..." +``` + +becomes something else entirely. What you get depends on your own shell history: + +``` +SyntaxError: f-string: invalid syntax + (f.defaultrm -rf schema) +``` + +That is a past command of yours, pasted into the middle of a Python f-string. Here it only +produced a syntax error — Python never ran and nothing was deleted. But the same mechanism +can land text somewhere the shell *will* execute. + +**The fix used throughout this guide:** multi-line Python is passed via a heredoc with a +**quoted** delimiter, not `-c "..."`. + +```python +from overture.schema.buildings import Building +print(f"{Building.__name__!r} is safe here") +``` + +Quoting the delimiter (`<<'""" + D + """'` rather than `<<""" + D + """`) disables every +form of expansion in the body — history, variables, command substitution. The text reaches +Python exactly as written. + +Verified rather than assumed: + +``` +double-quoted -c -> !r expanded into a command from history +quoted heredoc -> !r left alone +``` + +Single quotes also block history expansion, but the Python in this guide uses single +quotes internally, so heredocs are the practical choice. If you hit this in your own +one-liners, `{value!r}` can always be written `{repr(value)}` instead — no `!` at all. + +--- + +--- + +## Model gotchas + +Navigating [the two representations](SCHEMA_GUIDE.md#32-the-one-thing-to-understand-two-representations) of the models can be challenging. + +| Gotcha | What happens | Fix | +|---|---|---| +| `model_validate(geojson_dict)` | `ValidationError`: missing `theme`/`version`, `type` is `'Feature'` | Use `model_validate_json()`. Python mode expects flat data, JSON mode expects GeoJSON. | +| `model_dump()` without `by_alias` | Emits `class_`, not `class`; output won't re-validate | Always `by_alias=True` | +| `Segment.model_validate(...)` | `AttributeError` — it's a union alias, not a class | `TypeAdapter(Segment).validate_json(...)` | +| `model_names()` returns `[]` | PySpark expressions are generated, not committed | `make generate-pyspark` (or `make install`) | +| Dumps full of `null` | Unset optionals serialize explicitly | `exclude_none=True` | +| Absent list → `[]` → won't re-validate | An omitted optional list defaults to `[]` on the model, dumps as `[]`, then fails a `min_length` check on the way back in | `exclude_defaults=True`, or drop empty lists before re-validating | + +Note: optional list that is simply *absent* from the input becomes an empty list on the model, and an empty list is not the same as +absent on the way back out. + +```python +segments = TypeAdapter(Segment) +seg = segments.validate_json( + open("road-indoors.yaml-as-json").read() +) # no `connectors` key +seg.connectors # [] ← not None + +flat = seg.model_dump(mode="python", by_alias=True, exclude_none=True) +flat["connectors"] # [] ← exclude_none doesn't drop it +segments.validate_python(flat) +# ValidationError: road.connectors +# List should have at least 2 items after validation, not 0 +``` + +The same document validates fine on the way in (the CLI accepts it) and fails on the way +back. `exclude_defaults=True` avoids it, as does pruning empty lists before re-validating. + +## Docs in the repo that are currently wrong + +- **`pip install overture-schema`** — in every package README. Nothing is on PyPI yet; + see [7.3](SCHEMA_GUIDE.md#73-what-changes-once-these-packages-are-published). Every other item that + stood here has been fixed, and `tests/test_documented_imports.py` now imports every + `overture.*` statement in every tracked Markdown file, so a broken one fails the suite + rather than accumulating here. diff --git a/packages/overture-schema-cli/changelog.d/668.bugfix.md b/packages/overture-schema-cli/changelog.d/668.bugfix.md new file mode 100644 index 000000000..f3dcd1add --- /dev/null +++ b/packages/overture-schema-cli/changelog.d/668.bugfix.md @@ -0,0 +1 @@ +Fixed `--help` example blocks rendering a literal `\b` instead of Click's no-rewrap marker, and replaced the tag-filter examples that cited tags discovery never emitted. diff --git a/packages/overture-schema-cli/pyproject.toml b/packages/overture-schema-cli/pyproject.toml index e5642bc96..783c5d75c 100644 --- a/packages/overture-schema-cli/pyproject.toml +++ b/packages/overture-schema-cli/pyproject.toml @@ -34,7 +34,7 @@ build-backend = "uv_build" [dependency-groups] dev = [ "pytest>=9.0.0", - "ruff>=0.13.0", + "ruff>=0.16.0", "mypy>=1.17.0", ] diff --git a/packages/overture-schema-cli/src/overture/schema/cli/commands.py b/packages/overture-schema-cli/src/overture/schema/cli/commands.py index 24ce225f1..15e9ad727 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/commands.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/commands.py @@ -225,10 +225,17 @@ def get_source_name(filename: Path) -> str: return "" if str(filename) == "-" else str(filename) +# Every `# noqa: D301` below is the same waiver, against `pydocstyle` (see +# the docformat-only target). D301 wants a raw string wherever a docstring +# contains a backslash, but `\b` here is Click's no-rewrap marker: a raw +# string hands Click two literal characters and every example block collapses +# into one paragraph. Any new command with an Examples block needs the waiver +# too. Note the placement is pydocstyle's -- ruff reports D301 at the +# docstring line instead, so selecting ruff's `D` rules would need its own. @click.group() @click.version_option(package_name="overture-schema") -def cli() -> None: - r"""Overture Schema command-line interface. +def cli() -> None: # noqa: D301 + """Overture Schema command-line interface. Provides validation, schema generation, and type discovery for Overture Maps data. @@ -737,8 +744,8 @@ def validate( excludes: tuple[str, ...], types: tuple[str, ...], show_fields: tuple[str, ...], -) -> None: - r"""Validate Overture Maps data against schemas. +) -> None: # noqa: D301 + """Validate Overture Maps data against schemas. Read from FILENAME or stdin if FILENAME is '-'. Supports JSON, YAML, and GeoJSON formats. @@ -757,8 +764,12 @@ def validate( # Validate specific type $ overture-schema validate --type building data.json \b - # Official Overture types only - $ overture-schema validate --tag overture --tag feature data.json + # Two themes at once (repeatable; scope is their union) + $ overture-schema validate --tag overture:theme=buildings \\ + --tag overture:theme=places data.json + \b + # Only types built on the Overture feature model + $ overture-schema validate --tag overture data.json """ # Resolve model type first (errors here are ValueErrors, not ValidationErrors) try: @@ -803,8 +814,8 @@ def json_schema_command( filters: tuple[str, ...], excludes: tuple[str, ...], types: tuple[str, ...], -) -> None: - r"""Generate JSON schema for Overture Maps types. +) -> None: # noqa: D301 + """Generate JSON schema for Overture Maps types. Outputs a JSON Schema document to stdout that can be used for validation or documentation purposes. @@ -820,8 +831,12 @@ def json_schema_command( # Specific types $ overture-schema json-schema --type building \b - # Official Overture types only - $ overture-schema json-schema --tag overture --tag feature + # Two themes at once (repeatable; scope is their union) + $ overture-schema json-schema --tag overture:theme=buildings \\ + --tag overture:theme=places + \b + # Only types built on the Overture feature model + $ overture-schema json-schema --tag overture """ try: model_type = resolve_types( @@ -838,7 +853,8 @@ def json_schema_command( @tag_selection_options @click.option( "--group-by", - help="Group types by a key/value tag's key (e.g. 'overture:theme'). " + help="Group types by a key/value tag's key, as in " + "--group-by overture:theme. " "Plain and namespaced tags have no value to group by and are " "ignored here.", ) @@ -847,8 +863,8 @@ def list_types( filters: tuple[str, ...], excludes: tuple[str, ...], group_by: str | None, -) -> None: - r"""List all available types. +) -> None: # noqa: D301 + """List all available types. Displays all registered models and can be organized by grouping. @@ -856,6 +872,12 @@ def list_types( Examples: # List all types $ overture-schema list-types + \b + # One theme + $ overture-schema list-types --tag overture:theme=buildings + \b + # Group the listing by theme + $ overture-schema list-types --group-by overture:theme """ try: models = discover_models() diff --git a/packages/overture-schema-cli/src/overture/schema/cli/tag_options.py b/packages/overture-schema-cli/src/overture/schema/cli/tag_options.py index 3befb1143..cd1965f09 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/tag_options.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/tag_options.py @@ -9,10 +9,13 @@ F = TypeVar("F", bound=Callable[..., object]) +# Every tag named here must be one discovery actually emits -- a tag in help +# text reads as runnable. The namespaced form has no shipped example, so it is +# described rather than illustrated. _TAG_SYNTAX_NOTE = ( - "Accepts plain tags (e.g. feature), namespaced tags " - "(e.g. overture:approved), or compound key/value tags " - "(e.g. overture:theme=buildings)." + "Accepts plain tags (e.g. feature, overture) and compound key/value tags " + "(e.g. overture:theme=buildings). A namespaced form, namespace:predicate, " + "is also accepted for tags that third-party packages register." ) diff --git a/packages/overture-schema-cli/tests/test_cli_commands.py b/packages/overture-schema-cli/tests/test_cli_commands.py index 9fb088c7d..3a9c50a30 100644 --- a/packages/overture-schema-cli/tests/test_cli_commands.py +++ b/packages/overture-schema-cli/tests/test_cli_commands.py @@ -1,6 +1,7 @@ """Tests for CLI commands (validate, list-types, json-schema).""" import json +import re from io import StringIO import pytest @@ -8,6 +9,32 @@ from conftest import build_feature from overture.schema.cli.commands import cli +from overture.schema.system.discovery import discover_models + +_HELP_INVOCATIONS = [["--help"]] + [[name, "--help"] for name in sorted(cli.commands)] + +# Click renders each option as a line starting with two spaces and the flag, +# its help indented under it. +_OPTION_START = re.compile(r"^ (--[\w-]+)", re.MULTILINE) + +# Options whose `(e.g. ...)` illustrations name something other than a tag. +_NOT_TAGS = {"--type", "--show-field"} + + +def _help_segments(output: str) -> list[tuple[str | None, str]]: + """Split `--help` output into (option, text), so a citation keeps its option. + + The leading segment -- description and examples, before the first option + line -- carries `None`. + """ + bounds = list(_OPTION_START.finditer(output)) + if not bounds: + return [(None, output)] + segments: list[tuple[str | None, str]] = [(None, output[: bounds[0].start()])] + for current, following in zip(bounds, [*bounds[1:], None], strict=True): + end = following.start() if following else len(output) + segments.append((current[1], output[current.start() : end])) + return segments class TestListTypesCommand: @@ -29,6 +56,96 @@ def test_list_types_command_help(self, cli_runner: CliRunner) -> None: assert "list-types" in result.output.lower() +class TestHelpFormatting: + """Tests for `--help` rendering and the tags it cites.""" + + @pytest.mark.parametrize("argv", _HELP_INVOCATIONS, ids=lambda a: " ".join(a)) + def test_help_has_no_literal_escape( + self, cli_runner: CliRunner, argv: list[str] + ) -> None: + """Click's no-rewrap marker is interpreted, not printed.""" + result = cli_runner.invoke(cli, argv) + assert result.exit_code == 0 + assert "\\b" not in result.output + + def test_help_keeps_examples_on_separate_lines(self, cli_runner: CliRunner) -> None: + """Each example command occupies its own line rather than reflowing.""" + result = cli_runner.invoke(cli, ["validate", "--help"]) + assert result.exit_code == 0 + commands = [ + line.strip() + for line in result.output.splitlines() + if line.strip().startswith("$ overture-schema") + ] + assert len(commands) >= 4 + assert "$ overture-schema validate data.json" in commands + + def test_help_cites_only_tags_that_exist(self, cli_runner: CliRunner) -> None: + """Tags named in `--help` work in the option they are named for. + + Two citation forms, because a phantom has hidden in each: an + argument (`--tag X`) and an illustration (`(e.g. X)`). Both are + found wherever they appear in the rendered output, including + across a terminal wrap. + + Each is checked against the set that option accepts, not a global + union: `--group-by` takes the *key* half of a `key=value` tag, so + `--tag overture:theme` selects nothing and `--group-by feature` + groups nothing, and neither may pass. Attribution is by splitting + the Options block per option, so an illustration is judged by the + option whose help it sits in. + + Not checked: a tag named in running prose outside both forms. + `namespace:predicate` in the `--tag` syntax note is deliberately + such a placeholder -- describing the grammar without writing + something that looks runnable is why it is worded that way. Nor are + options in `_NOT_TAGS`, whose illustrations name something else -- + `--type`'s "(e.g., building, segment)" names types. + """ + emitted = {tag for key in discover_models() for tag in key.tags} + keys = {tag.split("=", 1)[0] for tag in emitted if "=" in tag} + # What each tag-taking option accepts. + accepts: dict[str | None, set[str]] = { + "--tag": emitted, + "--filter": emitted, + "--exclude": emitted, + "--group-by": keys, + } + argument_form = re.compile(r"--(tag|filter|exclude|group-by) ([\w.:=-]+)") + illustration_form = re.compile(r"e\.g\. ([^)]+)\)") + + cited: list[tuple[str, str, set[str]]] = [] + for argv in _HELP_INVOCATIONS: + result = cli_runner.invoke(cli, argv) + assert result.exit_code == 0, f"{argv}: --help failed" + where = " ".join(argv) + for option, segment in _help_segments(result.output): + # Rejoin words the terminal wrapper split, so a tag broken + # across lines is still seen whole. + flowed = " ".join(segment.split()) + cited += [ + # A citation ending a sentence picks up its period; the + # tag grammar allows `.` inside a name, never at the end. + (where, value.rstrip(".,"), accepts[f"--{flag}"]) + for flag, value in argument_form.findall(flowed) + if value != "TEXT" # the option signature's metavar + ] + if option in _NOT_TAGS: + continue + # An illustration inside a tag option's help is judged by + # that option; one in the description, the examples, or the + # Commands block belongs to no option, so accept either. + accepted = accepts.get(option, emitted | keys) + cited += [ + (where, token.strip("'\"").rstrip(".,"), accepted) + for group in illustration_form.findall(flowed) + for token in re.split(r",\s*", group.strip()) + ] + assert cited, "no tag cited anywhere in --help -- the sweep found nothing" + for where, tag, accepted in cited: + assert tag in accepted, f"{where}: {tag!r} matches no model" + + class TestJsonSchemaCommand: """Tests for the json-schema command.""" diff --git a/packages/overture-schema-codegen/README.md b/packages/overture-schema-codegen/README.md index 3252dfffd..22384e38a 100644 --- a/packages/overture-schema-codegen/README.md +++ b/packages/overture-schema-codegen/README.md @@ -10,10 +10,10 @@ structure collapses into `anyOf` arrays with duplicated fields. Navigating Python's type annotation machinery -- NewType chains, nested `Annotated` wrappers, union filtering, generic resolution -- is complex. The codegen does it once. -`analyze_type()` unwraps annotations into `TypeInfo`, a flat target-independent -representation. Extractors build specs from `TypeInfo`. Renderers consume specs without -touching the type system. New output targets (Arrow schemas, PySpark expressions) add -renderers, not extraction logic. +`analyze_type()` unwraps an annotation into a `FieldShape`, a tree-shaped +target-independent representation. Extractors build specs from `FieldShape`. Renderers +consume specs without touching the type system. New output targets (Arrow schemas, +PySpark expressions) add renderers, not extraction logic. ## Usage @@ -41,9 +41,9 @@ Rendering Output formatting, all presentation decisions ^ Output Layout What to generate, where it goes, how outputs link ^ -Extraction TypeInfo, FieldSpec, RecordSpec, UnionSpec +Extraction FieldShape, FieldSpec, RecordSpec, UnionSpec ^ -Discovery discover_models() from overture-schema-common +Discovery discover_models() from overture-schema-system ``` **Discovery** loads registered Pydantic models via entry points. The return dict @@ -72,16 +72,38 @@ examples), enum pages, NewType pages, and aggregate numeric/geometry reference p ## Programmatic use +`analyze_type()` returns a 3-tuple: the annotation's `FieldShape`, whether the +field accepts `None`, and the first `Field(description=...)` encountered while +unwrapping. + +```python +from overture.schema.buildings import Building +from overture.schema.codegen.extraction.type_analyzer import analyze_type + +annotation = Building.model_fields["version"].rebuild_annotation() +shape, nullable, description = analyze_type(annotation, owner=Building) + +# NewTypeShape(name='FeatureVersion', ref=..., inner=Primitive(base_type='int32', ...)) +assert shape.name == "FeatureVersion" +assert shape.inner.base_type == "int32" +assert nullable is False +``` + +`FieldShape` is a tree, not a flat record: `NewTypeShape`, `ArrayOf`, and `MapOf` +wrap an inner shape, and the three terminals (`Primitive`, `LiteralScalar`, +`AnyScalar`) sit at the leaves. Nesting order is meaningful -- +`NewTypeShape(inner=ArrayOf(...))` is a NewType over `list[X]`, while +`ArrayOf(element=NewTypeShape(...))` is a list of NewType-wrapped values. + +Constraints attach to the layer they target and carry the NewType that +contributed them: + ```python -from overture.schema.codegen.extraction.type_analyzer import analyze_type, TypeKind - -info = analyze_type(some_annotation) -assert info.kind == TypeKind.PRIMITIVE -assert info.base_type == "int32" -assert info.newtype_name == "FeatureVersion" -# Constraints carry provenance: -for cs in info.constraints: - print(f"{cs.constraint} from {cs.source}") +for source in shape.inner.constraints: + print(f"{source.constraint} from {source.source_name}") +# Ge(ge=0) from FeatureVersion +# Ge(ge=-2147483648) from int32 +# Le(le=2147483647) from int32 ``` ## Fetching sample data diff --git a/packages/overture-schema-codegen/changelog.d/668.bugfix.md b/packages/overture-schema-codegen/changelog.d/668.bugfix.md new file mode 100644 index 000000000..d46f6bef7 --- /dev/null +++ b/packages/overture-schema-codegen/changelog.d/668.bugfix.md @@ -0,0 +1 @@ +Fixed `overture-codegen list` printing a raw `typing.Annotated[...]` expression for discriminated-union entry points such as `Segment`; entries now list by their entry-point class name. diff --git a/packages/overture-schema-codegen/changelog.d/668.docs.md b/packages/overture-schema-codegen/changelog.d/668.docs.md new file mode 100644 index 000000000..88e7d94f0 --- /dev/null +++ b/packages/overture-schema-codegen/changelog.d/668.docs.md @@ -0,0 +1 @@ +Rewrote the README's programmatic-use section against the current `analyze_type()` signature, which returns a `FieldShape` tuple rather than the removed `TypeInfo`/`TypeKind`. diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py index dc5a7da5a..a5687d817 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py @@ -11,6 +11,7 @@ from overture.schema.system.discovery import ( discover_models, filter_models, + split_entry_point, ) from .extraction.specs import ModelSpec, SupplementarySpec, TypeIdentity @@ -59,11 +60,14 @@ def cli() -> None: def list_models() -> None: """List all discovered models.""" models = discover_models() - names = sorted( - model.__name__ if isinstance(model, type) else str(model) - for model in models.values() - ) - for name in names: + # Name every entry from its entry point, not the loaded object: a + # discriminated union loads as an `Annotated[...]` alias with no + # `__name__`, so `str(model)` would print the whole type expression. + names = [] + for key in models: + _, class_name = split_entry_point(key.entry_point) + names.append(class_name) + for name in sorted(names): click.echo(name) diff --git a/packages/overture-schema-codegen/tests/test_cli.py b/packages/overture-schema-codegen/tests/test_cli.py index 18124ed87..dd90c6880 100644 --- a/packages/overture-schema-codegen/tests/test_cli.py +++ b/packages/overture-schema-codegen/tests/test_cli.py @@ -26,6 +26,13 @@ def test_list_shows_discovered_models(self, cli_runner: CliRunner) -> None: assert "Building" in result.output assert "Place" in result.output + def test_list_names_union_alias_entry_points(self, cli_runner: CliRunner) -> None: + """A union alias lists by its entry-point class name, not its repr.""" + result = cli_runner.invoke(cli, ["list"]) + + assert "Segment" in result.output.split() + assert "typing.Annotated" not in result.output + class TestCliGenerate: """Tests for the generate command.""" diff --git a/packages/overture-schema-common/changelog.d/668.feature.md b/packages/overture-schema-common/changelog.d/668.feature.md new file mode 100644 index 000000000..2499fe03d --- /dev/null +++ b/packages/overture-schema-common/changelog.d/668.feature.md @@ -0,0 +1 @@ +Added an `overture` tag provider that marks every entry point built on `OvertureFeature`, so consumers can select those types by tag instead of importing `OvertureFeature`. diff --git a/packages/overture-schema-common/pyproject.toml b/packages/overture-schema-common/pyproject.toml index 7c2448a44..3eca61aff 100644 --- a/packages/overture-schema-common/pyproject.toml +++ b/packages/overture-schema-common/pyproject.toml @@ -37,4 +37,5 @@ dev = [ ] [project.entry-points."overture.tag_providers"] +overture = "overture.schema.common.tag_providers:overture_provider" theme = "overture.schema.common.tag_providers:theme_provider" diff --git a/packages/overture-schema-common/src/overture/schema/common/tag_providers.py b/packages/overture-schema-common/src/overture/schema/common/tag_providers.py index 02b6ed161..57b1462fb 100644 --- a/packages/overture-schema-common/src/overture/schema/common/tag_providers.py +++ b/packages/overture-schema-common/src/overture/schema/common/tag_providers.py @@ -14,6 +14,55 @@ from overture.schema.system.discovery import ModelKey +def overture_provider( + types: Iterable[type[BaseModel]], + key: ModelKey, + tags: set[str], +) -> set[str]: + """Add `"overture"` when the entry point references an `OvertureFeature`. + + The tag says the model is built on Overture's feature model -- that it + carries the theme/type/id/version/sources contract `OvertureFeature` + defines -- as distinct from `feature`, which says only that it is a + `Feature`. It exists so that question is answerable by tag: + `overture-schema list-types --tag overture` selects those types, and a + consumer filtering on them needs no dependency on this package and no + `issubclass` call. + + It is *not* a claim that the type belongs to the Overture schema. Any + package can subclass `OvertureFeature` and register an entry point, and + this provider tags it like any other. Reserving the tag to this package + governs who may *emit* it, not which models receive it. + + One qualifying arm is enough: a discriminated union with any + `OvertureFeature` arm is an Overture feature type. + + This shares its predicate with `theme_provider` below, so `overture` is + also derivable as "carries any `overture:theme=` tag". It is emitted + separately because a selector a user can type is worth more than the + derivation, and because a future non-themed `OvertureFeature` would + break the equivalence. + + Parameters + ---------- + types + Concrete `BaseModel` subclasses for the entry point. For + discriminated-union features this is every arm. + key + Key identifying the model. + tags + Current tags; may be extended. + + Returns + ------- + set[str] + Updated tags, with `"overture"` added if applicable. + """ + if any(issubclass(tp, OvertureFeature) for tp in types): + tags.add("overture") + return tags + + def theme_provider( types: Iterable[type[BaseModel]], key: ModelKey, diff --git a/packages/overture-schema-common/tests/test_common_tag_providers.py b/packages/overture-schema-common/tests/test_common_tag_providers.py index feaa0064b..d323b63b4 100644 --- a/packages/overture-schema-common/tests/test_common_tag_providers.py +++ b/packages/overture-schema-common/tests/test_common_tag_providers.py @@ -7,11 +7,16 @@ from overture.schema.common import OvertureFeature from overture.schema.common.tag_providers import ( + overture_provider, theme_provider, ) from overture.schema.system.discovery import ModelKey from overture.schema.system.discovery.discovery import _generate_tags -from overture.schema.system.discovery.types import TagProviderDict, TagProviderKey +from overture.schema.system.discovery.types import ( + TagProvider, + TagProviderDict, + TagProviderKey, +) @pytest.fixture @@ -34,30 +39,41 @@ def _empty_key(name: str = "x", entry_point: str = "mod:X") -> ModelKey: return ModelKey(name=name, entry_point=entry_point, tags=frozenset()) -def test_theme_provider_plain_class(building: type[OvertureFeature]) -> None: - tags = theme_provider((building,), _empty_key(), set()) - assert tags == {"overture:theme=buildings"} - +@pytest.fixture +def transportation_union() -> object: + """A two-arm transportation union. `_generate_tags` walks it to the arms.""" -def test_theme_provider_discriminated_union() -> None: - # `_generate_tags` is responsible for walking the union to concrete arms. class Road(OvertureFeature[Literal["transportation"], Literal["road"]]): pass class Rail(OvertureFeature[Literal["transportation"], Literal["rail"]]): pass - union = Annotated[ + return Annotated[ Annotated[Road, Tag("road")] | Annotated[Rail, Tag("rail")], Field(discriminator="type"), ] - provider_key = TagProviderKey( - name="theme", - entry_point="common:theme_provider", + + +def _providers(provider: TagProvider) -> TagProviderDict: + """Register one provider under a key this package is allowed to use.""" + key = TagProviderKey( + name=provider.__name__.removesuffix("_provider"), + entry_point=f"common:{provider.__name__}", package_name="overture-schema-common", ) - providers: TagProviderDict = {provider_key: theme_provider} - tags = _generate_tags(union, _empty_key(), providers) + return {key: provider} + + +def test_theme_provider_plain_class(building: type[OvertureFeature]) -> None: + tags = theme_provider((building,), _empty_key(), set()) + assert tags == {"overture:theme=buildings"} + + +def test_theme_provider_discriminated_union(transportation_union: object) -> None: + tags = _generate_tags( + transportation_union, _empty_key(), _providers(theme_provider) + ) assert tags == {"overture:theme=transportation"} @@ -74,3 +90,32 @@ class BadFeature(OvertureFeature): # type: ignore[type-arg] with pytest.raises(TypeError, match="must be annotated Literal"): theme_provider((BadFeature,), _empty_key(), set()) + + +def test_overture_provider_plain_class(building: type[OvertureFeature]) -> None: + tags = overture_provider((building,), _empty_key(), set()) + assert tags == {"overture"} + + +def test_overture_provider_discriminated_union(transportation_union: object) -> None: + tags = _generate_tags( + transportation_union, _empty_key(), _providers(overture_provider) + ) + assert tags == {"overture"} + + +def test_overture_provider_skips_non_overture(not_overture: type[BaseModel]) -> None: + tags = overture_provider((not_overture,), _empty_key(), set()) + assert tags == set() + + +def test_overture_provider_partial_union_still_tags( + not_overture: type[BaseModel], +) -> None: + """One Overture arm is enough; a mixed union still counts as Overture.""" + + class Road(OvertureFeature[Literal["transportation"], Literal["road"]]): + pass + + tags = overture_provider((not_overture, Road), _empty_key(), set()) + assert tags == {"overture"} diff --git a/packages/overture-schema-pyspark/README.md b/packages/overture-schema-pyspark/README.md index 13981ffba..85c85b24a 100644 --- a/packages/overture-schema-pyspark/README.md +++ b/packages/overture-schema-pyspark/README.md @@ -65,7 +65,7 @@ overture-validate segment samples/segment.parquet \ --conf spark.master=local[4] # Continue past schema mismatches (e.g. Float vs Double on bbox) -overture-validate place s3a://overturemaps-us-west-2/release/2026-06-17.0 \ +overture-validate place s3a://overturemaps-us-west-2/release/2026-07-22.0 \ --skip-schema-check # Skip checks for a column absent from the data @@ -106,15 +106,15 @@ structure: | --- | --- | --- | | Hive partition path (contains `/theme=`) | `.../theme=transportation/type=segment/` | Reads directly; derives `basePath` so Spark discovers partition columns. | | Individual file | `segment.parquet` | Reads directly; data already contains `theme`/`type` columns. | -| Release root | `s3a://overturemaps-us-west-2/release/2026-06-17.0` | Appends `theme={theme}/type={type}` using the schema's theme mapping; sets `basePath` to the original path. | +| Release root | `s3a://overturemaps-us-west-2/release/2026-07-22.0` | Appends `theme={theme}/type={type}` using the schema's theme mapping; sets `basePath` to the original path. | This means you can point the CLI at a release root and it constructs the full Hive path automatically: ```bash # These are equivalent: -overture-validate segment s3a://overturemaps-us-west-2/release/2026-06-17.0 -overture-validate segment s3a://overturemaps-us-west-2/release/2026-06-17.0/theme=transportation/type=segment/ +overture-validate segment s3a://overturemaps-us-west-2/release/2026-07-22.0 +overture-validate segment s3a://overturemaps-us-west-2/release/2026-07-22.0/theme=transportation/type=segment/ ``` ### Reading from S3 @@ -126,14 +126,22 @@ the Overture release bucket: ```bash overture-validate segment \ - s3a://overturemaps-us-west-2/release/2026-06-17.0/theme=transportation/type=segment/ + s3a://overturemaps-us-west-2/release/2026-07-22.0/theme=transportation/type=segment/ +``` + +The bucket retains only recent releases -- older identifiers, including ones +named in earlier revisions of this file, have been removed. Read the current +one from the STAC catalog's `latest` field: + +```bash +curl -s https://stac.overturemaps.org/catalog.json | jq -r .latest ``` To use named AWS credentials instead of anonymous access: ```bash overture-validate segment \ - s3a://overturemaps-us-west-2/release/2026-06-17.0/theme=transportation/type=segment/ \ + s3a://overturemaps-us-west-2/release/2026-07-22.0/theme=transportation/type=segment/ \ --conf spark.hadoop.fs.s3a.aws.credentials.provider=software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider ``` diff --git a/packages/overture-schema-pyspark/changelog.d/668.docs.md b/packages/overture-schema-pyspark/changelog.d/668.docs.md new file mode 100644 index 000000000..ce95a6981 --- /dev/null +++ b/packages/overture-schema-pyspark/changelog.d/668.docs.md @@ -0,0 +1 @@ +Updated the README's S3 examples to a release the bucket still retains, and documented how to read the current release identifier from the STAC catalog. diff --git a/packages/overture-schema-system/README.md b/packages/overture-schema-system/README.md index 84ff7b86f..503c7b0ba 100644 --- a/packages/overture-schema-system/README.md +++ b/packages/overture-schema-system/README.md @@ -132,7 +132,7 @@ class Park(Identified): class ParkBench(Identified): - park_id: Annotated[Id, Reference(Relationship.BELONGS_TO, Park)] + park_id: Annotated[Id, Reference(Relationship.COMPOSITION, Park, role="part_of")] ``` ## Discovery @@ -222,6 +222,7 @@ Specific plain tags and namespaces are reserved for designated packages. For exa |---|---| | `feature` (tag) | `overture-schema-system` | | `system:` (namespace) | `overture-schema-system` | +| `overture` (tag) | `overture-schema-common` | | `overture:` (namespace) | `overture-schema-common` | When a provider attempts to set a reserved tag from an unauthorized package, discovery logs a warning and discards the tag. @@ -229,6 +230,7 @@ When a provider attempts to set a reserved tag from an unauthorized package, dis ### Built-in Providers - **`feature`** (in `system`) -- adds `feature` if any concrete arm is a `Feature` subclass. +- **`overture`** (in `common`) -- adds `overture` if any concrete arm is an `OvertureFeature` subclass: the model is built on Overture's feature model, as distinct from `feature`, which says only that it is a `Feature`. Consumers that need to ask that question read the tag rather than importing `OvertureFeature`. The tag does not assert that the type belongs to the Overture schema -- a third-party `OvertureFeature` subclass receives it too, and the reservation governs who may emit the tag, not which models get it. - **`theme`** (in `common`) -- adds `overture:theme={theme}` for each `OvertureFeature` referenced. A discriminated-union feature whose arms span multiple themes contributes one tag per distinct theme. ### Selecting Models by Tag diff --git a/packages/overture-schema-system/changelog.d/668.misc.md b/packages/overture-schema-system/changelog.d/668.misc.md new file mode 100644 index 000000000..b76bcab60 --- /dev/null +++ b/packages/overture-schema-system/changelog.d/668.misc.md @@ -0,0 +1 @@ +Reserved the plain `overture` tag to `overture-schema-common` and documented its provider in the README. diff --git a/packages/overture-schema-system/pyproject.toml b/packages/overture-schema-system/pyproject.toml index 613349cef..952d299ec 100644 --- a/packages/overture-schema-system/pyproject.toml +++ b/packages/overture-schema-system/pyproject.toml @@ -26,7 +26,7 @@ Issues = "https://github.com/OvertureMaps/schema/issues" [dependency-groups] dev = [ "pytest>=9.0.0", - "ruff>=0.13.0", + "ruff>=0.16.0", "mypy>=1.17.0", ] diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py index 2425a7551..fc3537e32 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py @@ -24,6 +24,7 @@ # Tags that are reserved and can only be set by specific packages. _RESERVED_TAGS: dict[str, set[str]] = { "feature": {"overture-schema-system"}, + "overture": {"overture-schema-common"}, } # Namespaces that are reserved and can only be set by specific packages. _RESERVED_NAMESPACES: dict[str, set[str]] = { diff --git a/packages/overture-schema-system/tests/test_tag_providers.py b/packages/overture-schema-system/tests/test_tag_providers.py index 33fd15392..84c1069c8 100644 --- a/packages/overture-schema-system/tests/test_tag_providers.py +++ b/packages/overture-schema-system/tests/test_tag_providers.py @@ -123,6 +123,25 @@ def test_allowed_reserved_tag( assert _generate_tags(any_model, any_key, system_providers) == {"feature"} +def test_reserved_overture_tag( + other_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + providers = {other_tag_provider: fake_provider("overture", "valid")} + result = _generate_tags(any_model, any_key, providers) + assert result == {"valid"} + + +def test_allowed_reserved_overture_tag( + common_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + common_providers = {common_tag_provider: fake_provider("overture")} + assert _generate_tags(any_model, any_key, common_providers) == {"overture"} + + def test_reserved_namespace( other_tag_provider: TagProviderKey, any_key: ModelKey, diff --git a/packages/overture-schema-validation/README.md b/packages/overture-schema-validation/README.md index 41cda9f8b..f6fdb7220 100644 --- a/packages/overture-schema-validation/README.md +++ b/packages/overture-schema-validation/README.md @@ -15,11 +15,13 @@ pip install overture-schema-validation ```python from overture.schema.validation import validate, validate_json -# Validate a Python object (a dict or a model instance) -feature = validate({"type": "segment", "id": "...", "geometry": "..."}) +# A Python object -- the flat, tabular (Parquet-style) shape +feature = validate(feature_row) -# Validate a JSON document -feature = validate_json('{"type": "segment", "id": "...", "geometry": "..."}') +# A JSON document -- GeoJSON +feature = validate_json(geojson_text) ``` +The two entry points are not interchangeable. `validate` runs Pydantic's Python mode, which reads the flat column layout of the Parquet release -- the shape Overture publishes. `validate_json` runs JSON mode, which reads the GeoJSON representation the models support for compatibility with tools that expect features rather than rows. Handing a GeoJSON dict to `validate` reports `theme` and `version` missing and `type` set to `'Feature'`. + Both raise `pydantic.ValidationError` when the input matches no model. Which models participate is resolved at runtime by entry-point discovery, so installing additional Overture theme packages widens what these functions accept. diff --git a/packages/overture-schema-validation/changelog.d/668.docs.md b/packages/overture-schema-validation/changelog.d/668.docs.md new file mode 100644 index 000000000..014c31693 --- /dev/null +++ b/packages/overture-schema-validation/changelog.d/668.docs.md @@ -0,0 +1 @@ +Corrected the README's usage examples, which showed `validate_json` accepting the flat tabular shape rather than GeoJSON. diff --git a/packages/overture-schema/README.md b/packages/overture-schema/README.md index 4a725eda8..d3d55c6b7 100644 --- a/packages/overture-schema/README.md +++ b/packages/overture-schema/README.md @@ -10,76 +10,118 @@ This package provides Pydantic models for validating and working with Overture M pip install overture-schema ``` +`overture-schema` is a metapackage: it pulls in every theme package plus the +validation library and the CLI, and ships no code of its own. `overture.schema` +is a namespace root, so import from the theme and system packages rather than +from `overture.schema` directly. + ## Usage -Import and use schemas: +Import models from the theme package that defines them: ```python -from overture.schema import Building, Place -import json +from overture.schema.buildings import Building +from overture.schema.places import Place +``` + +### Tabular data, and GeoJSON for compatibility + +Overture publishes data in one shape: flat and tabular -- the column layout of the +Parquet release, with `theme`, `type`, and `version` as top-level columns and +geometry as WKT. That is what **Python mode** (`model_validate`) reads. + +The models also accept and emit GeoJSON, through **JSON mode** +(`model_validate_json`), so the schema works with tools that expect features +rather than rows. The generated JSON Schema describes that representation. + +The modes are not interchangeable. Passing a GeoJSON dict to `model_validate` +reports `theme`/`version` missing and `type` set to `'Feature'`, because it is +reading GeoJSON keys as flat columns. -# Validate Overture Maps data (supports both flat/tabular and GeoJSON formats) -building = Building.model_validate(feature_data) -place = Place.model_validate(geojson_feature) +```python +# Flat / tabular (Parquet-shaped) dict +building = Building.model_validate(feature_row) -# Parse and validate JSON strings -building_from_json = Building.model_validate_json(json_string) +# GeoJSON -- JSON mode, from a string or bytes +building = Building.model_validate_json(geojson_text) -# Convert to GeoJSON format for output -geojson_output = building.model_dump(mode="json") +# Serialize back to GeoJSON. by_alias=True is required: without it, +# aliased fields serialize under their Python names (`class_`, not +# `class`) and the output will not re-validate. +geojson_output = building.model_dump(mode="json", by_alias=True, exclude_none=True) ``` ### Available Models +Each model lives in its theme package. The metapackage installs all of them: + ```python -# All models are re-exported from their respective theme packages for convenience -from overture.schema import ( - # Addresses theme - Address, - # Base theme +from overture.schema.addresses import Address +from overture.schema.base import ( Bathymetry, Infrastructure, Land, LandCover, LandUse, Water, - # Buildings theme - Building, - BuildingPart, - # Divisions theme - Division, - DivisionArea, - DivisionBoundary, - # Places theme - Place, - # Transportation theme - Connector, - Segment, ) +from overture.schema.buildings import Building, BuildingPart +from overture.schema.divisions import Division, DivisionArea, DivisionBoundary +from overture.schema.places import Place +from overture.schema.transportation import Connector, Segment +``` + +`Segment` is a discriminated union alias rather than a class, so it validates +through a `TypeAdapter`: + +```python +from pydantic import TypeAdapter + +segments = TypeAdapter(Segment) +segment = segments.validate_json(geojson_text) ``` -### Utility Functions +### Validating without knowing the type -The package also exports several utility functions: +`overture-schema-validation` validates against the union of every installed +model, picking the right one from the data: ```python -from overture.schema import parse, discover_models, json_schema -from overture.schema import Building +from overture.schema.validation import validate, validate_json + +feature = validate(feature_row) # flat / tabular dict +feature = validate_json(geojson_text) # GeoJSON +``` -# Parse any Overture feature (auto-discovers all registered models) -validated_feature = parse(feature_data, mode="json") # Parses GeoJSON format -validated_feature = parse(feature_data, mode="python") # Parses flat format +### Discovering models programmatically + +Discovery lives in `overture-schema-system`. `discover_models()` returns a dict +keyed by `ModelKey` -- entry point `name`, its `entry_point` value, and the set +of tags attached during discovery: + +```python +from overture.schema.system.discovery import discover_models, get_registered_model -# Discover all registered models programmatically all_models = discover_models() -# Returns: # { -# ("buildings", "building"): BuildingModel, -# ("places", "place"): PlaceModel, -# ... +# ModelKey(name="building", entry_point="overture.schema.buildings:Building", +# tags=frozenset({"feature", "overture", "overture:theme=buildings"})): Building, +# ModelKey(name="place", entry_point="overture.schema.places:Place", +# tags=frozenset({"feature", "overture", "overture:theme=places"})): Place, +# ... # } -# Generate JSON Schema for models or unions +building_model = get_registered_model("building") # None if not installed +``` + +### Generating JSON Schema + +```python +from overture.schema.system.json_schema import json_schema + schema = json_schema(Building) -union_schema = json_schema(Building | Place) # Works with unions too +union_schema = json_schema(Building | Place) # emits an anyOf ``` + +See the [`overture-schema-system` README](../overture-schema-system/README.md) +for tag format, tag providers, and the discovery API in full. diff --git a/packages/overture-schema/changelog.d/668.docs.md b/packages/overture-schema/changelog.d/668.docs.md new file mode 100644 index 000000000..d2c7340f6 --- /dev/null +++ b/packages/overture-schema/changelog.d/668.docs.md @@ -0,0 +1 @@ +Rewrote the README against the real import surface: `overture.schema` is a namespace root, so models import from their theme packages and the utility functions from `overture.schema.validation` and `overture.schema.system`. diff --git a/pyproject.toml b/pyproject.toml index bbfe8e01e..b825787d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ select = [ [tool.mypy] disallow_untyped_defs = true explicit_package_bases = true -files = "packages/**/*.py" +files = ["packages/**/*.py", "tests/**/*.py"] # The pyspark test tree is PEP 420 and its tests import `_support` as a # top-level package (mirroring pytest's pythonpath). Put that tests dir on # the mypy path so `_support` resolves the same way for the type checker. @@ -64,7 +64,7 @@ dev = [ "pytest>=9.0.0", "pytest-cov>=7.0.0", "pytest-testmon>=2.2.0", - "ruff>=0.13.0", + "ruff>=0.16.0", "towncrier>=25.8.0", ] diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..bfc4caf6a --- /dev/null +++ b/tests/README.md @@ -0,0 +1,23 @@ +# Workspace tests + +Checks whose subject is the repository or the workspace as a whole, and which +therefore cannot correctly live in any one package. + +A test belongs here only if both hold: + +- **Its subject spans packages, or sits outside them.** `test_documented_imports` + reads every tracked Markdown file, including root-level ones that ship in no + package, and imports across every package's namespace — `overture.schema.codegen` + among them, which no single distribution depends on. +- **It cannot run from one package's install.** Placing such a test under + `packages//tests/` would make that package's suite depend on packages its + `pyproject.toml` does not declare, and it would pass only because the workspace + happens to install everything. + +Anything that can be scoped to a package belongs in that package's `tests/` +instead. This directory is not a home for tests that are merely inconvenient to +place. + +`make check` covers this tree the same way it covers `packages/` — ruff, `ruff +format`, mypy, and pytest. Note that a bare `pytest packages/` does not; use +`make test` or include `tests/` explicitly. diff --git a/tests/test_documented_imports.py b/tests/test_documented_imports.py new file mode 100644 index 000000000..fb3f26131 --- /dev/null +++ b/tests/test_documented_imports.py @@ -0,0 +1,417 @@ +"""Every `overture.*` import written in repo Markdown must resolve. + +Documentation drifts silently: a module moves, a helper is renamed, and the +README keeps confidently describing the old surface. This parses every fenced +Python block in every tracked Markdown file -- including blocks indented inside +a list item -- and imports each `overture.*` statement it finds. + +The subject is the repo's Markdown rather than any one package, so this lives +outside `packages/`. It reaches across the whole workspace -- a README may cite +`overture.schema.codegen`, which no single distribution depends on. +""" + +from __future__ import annotations + +import ast +import hashlib +import importlib +import re +import subprocess +import textwrap +from enum import Enum +from pathlib import Path + +import pytest + +# A fenced ```python block, capturing its body. The indent group matches a +# block nested in a list item; the body is dedented before parsing, since an +# `IndentationError` is a `SyntaxError` and would file the block under +# "expected unparseable" rather than reporting it. +_PYTHON_BLOCK = re.compile( + r"^(?P *)```python[^\n]*\n(?P.*?)^(?P=indent)```", + re.MULTILINE | re.DOTALL, +) + +# Blocks that are deliberately not valid Python: illustrations whose `...` +# placeholders stand in for elided content. Pinned by identity, not count -- a +# bare total stays put when one block breaks as another is fixed. The identity +# is a digest of the block body, so reordering a document does not trip it but +# editing one of these blocks does. +_EXPECTED_UNPARSEABLE: set[str] = set() + +# An import statement, matched textually -- used to police the excuse list, +# where by definition `ast` cannot be applied. +_OVERTURE_IMPORT = re.compile(r"^\s*(?:from|import)\s+overture\b", re.MULTILINE) + +# Docs sometimes have to show what does *not* work. A line carrying this +# marker is such a case, and is exempt from resolving. +COUNTER_EXAMPLE_MARKER = "\u2717" + +# Golden files are generated fixtures, not documentation. +_EXCLUDED = "tests/golden/" + + +def _repo_root() -> Path: + """Locate the checkout root, or skip the module when there isn't one.""" + for candidate in Path(__file__).resolve().parents: + if (candidate / ".git").exists(): + return candidate + pytest.skip("not running from a git checkout", allow_module_level=True) + + +def _markdown_files() -> list[Path]: + root = _ROOT + try: + listed = subprocess.run( + ["git", "ls-files", "-z", "*.md"], + cwd=root, + capture_output=True, + text=True, + check=True, + ).stdout + except (OSError, subprocess.CalledProcessError) as exc: # pragma: no cover + pytest.skip(f"cannot list tracked files: {exc}", allow_module_level=True) + return [root / n for n in listed.split("\0") if n and _EXCLUDED not in n] + + +def python_blocks(text: str) -> list[str]: + """Yield the dedented body of every fenced Python block in `text`. + + Handles a block indented inside a list item. The dedent is load-bearing: + without it `ast.parse` raises `IndentationError`, a `SyntaxError`, and the + block would be filed as "expected unparseable" rather than reported. + """ + return [textwrap.dedent(match["body"]) for match in _PYTHON_BLOCK.finditer(text)] + + +def _documented_enum_references( + paths: list[Path], +) -> list[tuple[Path, str, str, str]]: + """Collect `(file, module, enum, member)` quads across the corpus.""" + found: list[tuple[Path, str, str, str]] = [] + for path in paths: + for body in python_blocks(path.read_text(encoding="utf-8")): + try: + refs = enum_references(body) + except SyntaxError: + continue + found += [(path, module, name, member) for module, name, member in refs] + return found + + +def parse_imports(source: str) -> list[tuple[str, str | None]]: + """Extract `overture.*` imports from one block of Python source. + + Returns `(module, attribute)` pairs; `attribute` is None for a plain + `import overture.x`. Raises `SyntaxError` if the block is not valid Python. + + A REPL transcript is unwrapped first, and an import on a line marked + with `COUNTER_EXAMPLE_MARKER` is skipped -- the docs are showing what fails, and a + counter-example that resolved would be the real bug. + """ + lines = _strip_repl_prompts(source).splitlines() + found: list[tuple[str, str | None]] = [] + for node in ast.walk(ast.parse("\n".join(lines))): + if _is_counter_example(lines, node): + continue + if isinstance(node, ast.ImportFrom): + # A relative import (`from . import x`) has no absolute module. + if node.level or not node.module or not _is_overture(node.module): + continue + found += [ + (node.module, alias.name) for alias in node.names if alias.name != "*" + ] + elif isinstance(node, ast.Import): + found += [ + (alias.name, None) for alias in node.names if _is_overture(alias.name) + ] + return found + + +def _strip_repl_prompts(source: str) -> str: + """Unwrap a `>>>` transcript to the statements it contains. + + Without this a whole REPL block fails to parse and drops out of the + sweep -- silently, which is the failure mode this module exists to + prevent. + """ + if not any(line.startswith(">>> ") for line in source.splitlines()): + return source + return "\n".join( + line[4:] for line in source.splitlines() if line.startswith((">>> ", "... ")) + ) + + +def _is_counter_example(lines: list[str], node: ast.AST) -> bool: + """True if the statement's source carries the counter-example marker.""" + start = getattr(node, "lineno", None) + if start is None: + return False + end = getattr(node, "end_lineno", None) or start + return any(COUNTER_EXAMPLE_MARKER in line for line in lines[start - 1 : end]) + + +def _is_overture(module: str) -> bool: + return module == "overture" or module.startswith("overture.") + + +def enum_references(source: str) -> list[tuple[str, str, str]]: + """Find `Enum.MEMBER` uses of enums imported in the same block. + + Returns `(module, enum_name, member)`. Restricted to enums on purpose: + their members really are class attributes, whereas a Pydantic model's + fields are not, so `Place.addresses` would read as missing. + """ + tree = ast.parse(_strip_repl_prompts(source)) + imported = { + alias.asname or alias.name: node.module + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) + and node.module + and not node.level + and _is_overture(node.module) + for alias in node.names + } + return [ + (imported[node.value.id], node.value.id, node.attr) + for node in ast.walk(tree) + if isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id in imported + and node.attr.isupper() # a member, not a method + ] + + +def _documented_imports( + paths: list[Path], +) -> tuple[list[tuple[Path, str, str | None]], dict[str, str]]: + """Collect `(file, module, attribute)` triples, and identify unparseable blocks.""" + found: list[tuple[Path, str, str | None]] = [] + unparseable: dict[str, str] = {} + for path in paths: + for body in python_blocks(path.read_text(encoding="utf-8")): + try: + imports = parse_imports(body) + except SyntaxError: + digest = hashlib.sha256(body.encode()).hexdigest()[:8] + unparseable[f"{path.relative_to(_ROOT)}:{digest}"] = body + continue + found += [(path, module, attr) for module, attr in imports] + return found, unparseable + + +_ROOT = _repo_root() +_MARKDOWN = _markdown_files() +_DOCUMENTED, _UNPARSEABLE = _documented_imports(_MARKDOWN) +_ENUM_REFERENCES = _documented_enum_references(_MARKDOWN) + + +class TestBlocks: + """`python_blocks` against inline fixtures. + + Same argument as `TestParser` below, one layer up: a fence form the + matcher misses yields no block, and a sweep that collects less still + passes. The corpus cannot pin this. + """ + + def test_flush_block(self) -> None: + text = "text\n\n```python\nx = 1\n```\n\nmore\n" + assert python_blocks(text) == ["x = 1\n"] + + def test_list_item_block(self) -> None: + """A block indented under a list item, dedented to parse.""" + text = "1. Example:\n\n ```python\n x = 1\n ```\n" + assert python_blocks(text) == ["x = 1\n"] + + def test_deeply_indented_block(self) -> None: + text = "- a\n - b\n\n ```python\n x = 1\n ```\n" + assert python_blocks(text) == ["x = 1\n"] + + def test_indented_block_parses_only_after_dedent(self) -> None: + """Without the dedent this is an IndentationError, i.e. excused.""" + text = ( + "- item\n\n ```python\n from overture.schema.places import Place\n ```\n" + ) + (body,) = python_blocks(text) + assert parse_imports(body) == [("overture.schema.places", "Place")] + + def test_relative_indent_inside_a_block_is_preserved(self) -> None: + text = "- item\n\n ```python\n def f():\n return 1\n ```\n" + assert python_blocks(text) == ["def f():\n return 1\n"] + + def test_non_python_fences_are_ignored(self) -> None: + text = "```toml\nx = 1\n```\n\n```\nplain\n```\n" + assert python_blocks(text) == [] + + def test_several_blocks(self) -> None: + text = "```python\na = 1\n```\n\ntext\n\n```python\nb = 2\n```\n" + assert python_blocks(text) == ["a = 1\n", "b = 2\n"] + + +class TestOvertureImportMatcher: + """`_OVERTURE_IMPORT` against fixtures. + + It polices the excuse list, where `ast` cannot be applied by definition. + No excused block imports `overture` today, so the corpus never exercises + it -- the matcher would go blind without anyone noticing. + """ + + def test_matches_from_import(self) -> None: + assert _OVERTURE_IMPORT.search("x = 1\nfrom overture.schema import y\n") + + def test_matches_plain_import(self) -> None: + assert _OVERTURE_IMPORT.search(" import overture.schema.places\n") + + def test_ignores_other_packages(self) -> None: + assert not _OVERTURE_IMPORT.search("import json\nfrom pydantic import X\n") + + def test_ignores_a_longer_name(self) -> None: + assert not _OVERTURE_IMPORT.search("import overtures\n") + + def test_ignores_a_mention_in_prose(self) -> None: + assert not _OVERTURE_IMPORT.search("# import overture is what you'd write\n") + + +class TestParser: + """`parse_imports` against inline fixtures. + + The corpus cannot test the parser: any form it fails to handle simply + yields nothing, and a sweep that collects less still passes. + """ + + def test_single_line(self) -> None: + assert parse_imports("from overture.schema.buildings import Building") == [ + ("overture.schema.buildings", "Building") + ] + + def test_wrapped(self) -> None: + """The form `ruff format` produces once the names outgrow a line.""" + source = ( + "from overture.schema.system.numeric import (\n" + " int8, # signed\n" + " float64,\n" + ")\n" + ) + assert parse_imports(source) == [ + ("overture.schema.system.numeric", "int8"), + ("overture.schema.system.numeric", "float64"), + ] + + def test_alias(self) -> None: + assert parse_imports("from overture.schema.places import Place as P") == [ + ("overture.schema.places", "Place") + ] + + def test_plain_import_with_alias(self) -> None: + assert parse_imports("import overture.schema.buildings as b") == [ + ("overture.schema.buildings", None) + ] + + def test_star_and_relative_are_skipped(self) -> None: + assert parse_imports("from overture.schema.buildings import *") == [] + assert parse_imports("from . import buildings") == [] + + def test_non_overture_is_skipped(self) -> None: + assert parse_imports("import json\nfrom pydantic import BaseModel") == [] + + def test_indented_import(self) -> None: + source = "def f():\n from overture.schema.places import Place\n" + assert parse_imports(source) == [("overture.schema.places", "Place")] + + def test_repl_transcript(self) -> None: + """A `>>>` block would not parse at all without unwrapping.""" + source = ( + ">>> from overture.schema.buildings import Building\n" + ">>> Building.model_fields.keys()\n" + "dict_keys(['id', 'geometry'])\n" + ) + assert parse_imports(source) == [("overture.schema.buildings", "Building")] + + def test_counter_example_is_skipped(self) -> None: + """Docs showing what fails are exempt; the neighbouring line is not.""" + source = ( + "from overture.schema.buildings import Building\n" + f"from overture.schema import Building # {COUNTER_EXAMPLE_MARKER} ImportError\n" + ) + assert parse_imports(source) == [("overture.schema.buildings", "Building")] + + def test_invalid_source_raises(self) -> None: + with pytest.raises(SyntaxError): + parse_imports('{"a": 1, ...}\nthis is not python') + + +def test_corpus_is_non_empty() -> None: + """Guard against a silently empty sweep reporting success. + + The thresholds only have to be low enough to survive ordinary doc churn + and high enough that an empty or near-empty parse fails loudly. + """ + assert len(_MARKDOWN) > 10 + assert len(_DOCUMENTED) > 20 + + +def test_unparseable_blocks_are_the_expected_ones() -> None: + """A block that stops parsing drops out of the sweep silently.""" + assert set(_UNPARSEABLE) == _EXPECTED_UNPARSEABLE + + +def test_no_excused_block_hides_an_import() -> None: + """An excused block may illustrate, but may not cite the API. + + A content-addressed waiver is durable, so a block that both fails to + parse and imports `overture.*` would be exempt from the sweep forever -- + the one outcome this module exists to prevent. Making the block parse is + always available; every case so far took a one-token edit. + """ + for name, body in _UNPARSEABLE.items(): + assert not _OVERTURE_IMPORT.search(body), ( + f"{name} is excused from parsing but imports overture.*; " + "make the block parse instead of excusing it" + ) + + +@pytest.mark.parametrize( + ("path", "module", "enum_name", "member"), + _ENUM_REFERENCES, + ids=[ + f"{path.relative_to(_ROOT)}:{name}.{member}" + for path, _, name, member in _ENUM_REFERENCES + ], +) +def test_documented_enum_member_exists( + path: Path, module: str, enum_name: str, member: str +) -> None: + """An enum member named in the docs exists on the enum. + + The import sweep cannot see this: `from ... import Relationship` + resolves whether or not `Relationship.CONNECTS_TO` does. + """ + enum_class = getattr(importlib.import_module(module), enum_name) + if not (isinstance(enum_class, type) and issubclass(enum_class, Enum)): + return + names = [m.name for m in enum_class] + assert member in names, ( + f"{path}: `{enum_name}` has no member `{member}`; it has {names}" + ) + + +@pytest.mark.parametrize( + ("path", "module", "attribute"), + _DOCUMENTED, + ids=[ + f"{path.relative_to(_ROOT)}:{module}" + (f".{attr}" if attr else "") + for path, module, attr in _DOCUMENTED + ], +) +def test_documented_import_resolves( + path: Path, module: str, attribute: str | None +) -> None: + """An import written in the docs imports cleanly.""" + imported = importlib.import_module(module) + if attribute is None or hasattr(imported, attribute): + return + # `from pkg import submodule` binds only once the submodule is imported. + try: + importlib.import_module(f"{module}.{attribute}") + except ImportError as exc: + pytest.fail(f"{path}: `{module}` has no attribute `{attribute}` ({exc})") diff --git a/uv.lock b/uv.lock index 33b465975..2a2123d90 100644 --- a/uv.lock +++ b/uv.lock @@ -954,7 +954,7 @@ requires-dist = [ dev = [ { name = "mypy", specifier = ">=1.17.0" }, { name = "pytest", specifier = ">=9.0.0" }, - { name = "ruff", specifier = ">=0.13.0" }, + { name = "ruff", specifier = ">=0.16.0" }, ] [[package]] @@ -1087,7 +1087,7 @@ requires-dist = [ dev = [ { name = "mypy", specifier = ">=1.17.0" }, { name = "pytest", specifier = ">=9.0.0" }, - { name = "ruff", specifier = ">=0.13.0" }, + { name = "ruff", specifier = ">=0.16.0" }, ] [[package]] @@ -1248,7 +1248,7 @@ dev = [ { name = "pytest", specifier = ">=9.0.0" }, { name = "pytest-cov", specifier = ">=7.0.0" }, { name = "pytest-testmon", specifier = ">=2.2.0" }, - { name = "ruff", specifier = ">=0.13.0" }, + { name = "ruff", specifier = ">=0.16.0" }, { name = "towncrier", specifier = ">=25.8.0" }, ]