From 5193f867f6dc6dcd65b89b0217dbbfe4d30754b1 Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Wed, 19 Aug 2026 11:53:59 -0700 Subject: [PATCH 1/9] feat(common): tag Overture feature types with `overture` Discovery emitted `feature` and `overture:theme=*` but never the plain `overture` tag, so the filter the CLI's own help suggested matched nothing. Add an `overture_provider` in overture-schema-common that attaches the tag when any concrete arm of an entry point subclasses `OvertureFeature`, and reserve the tag to that package. "Is this built on Overture's feature model?" is now answerable by tag, so a consumer asking it needs neither a dependency on this package nor an `issubclass` call. The tag is not a claim that the type belongs to the Overture schema. Any package can subclass `OvertureFeature` and register an entry point, and this provider tags it like any other; reserving the tag governs who may emit it, not which models receive it. Signed-off-by: Seth Fitzsimmons --- .../changelog.d/668.feature.md | 1 + .../overture-schema-common/pyproject.toml | 1 + .../overture/schema/common/tag_providers.py | 49 +++++++++++++ .../tests/test_common_tag_providers.py | 71 +++++++++++++++---- packages/overture-schema-system/README.md | 4 +- .../changelog.d/668.misc.md | 1 + .../schema/system/discovery/discovery.py | 1 + .../tests/test_tag_providers.py | 19 +++++ 8 files changed, 133 insertions(+), 14 deletions(-) create mode 100644 packages/overture-schema-common/changelog.d/668.feature.md create mode 100644 packages/overture-schema-system/changelog.d/668.misc.md diff --git a/packages/overture-schema-common/changelog.d/668.feature.md b/packages/overture-schema-common/changelog.d/668.feature.md new file mode 100644 index 000000000..2499fe03d --- /dev/null +++ b/packages/overture-schema-common/changelog.d/668.feature.md @@ -0,0 +1 @@ +Added an `overture` tag provider that marks every entry point built on `OvertureFeature`, so consumers can select those types by tag instead of importing `OvertureFeature`. diff --git a/packages/overture-schema-common/pyproject.toml b/packages/overture-schema-common/pyproject.toml index 7c2448a44..3eca61aff 100644 --- a/packages/overture-schema-common/pyproject.toml +++ b/packages/overture-schema-common/pyproject.toml @@ -37,4 +37,5 @@ dev = [ ] [project.entry-points."overture.tag_providers"] +overture = "overture.schema.common.tag_providers:overture_provider" theme = "overture.schema.common.tag_providers:theme_provider" diff --git a/packages/overture-schema-common/src/overture/schema/common/tag_providers.py b/packages/overture-schema-common/src/overture/schema/common/tag_providers.py index 02b6ed161..57b1462fb 100644 --- a/packages/overture-schema-common/src/overture/schema/common/tag_providers.py +++ b/packages/overture-schema-common/src/overture/schema/common/tag_providers.py @@ -14,6 +14,55 @@ from overture.schema.system.discovery import ModelKey +def overture_provider( + types: Iterable[type[BaseModel]], + key: ModelKey, + tags: set[str], +) -> set[str]: + """Add `"overture"` when the entry point references an `OvertureFeature`. + + The tag says the model is built on Overture's feature model -- that it + carries the theme/type/id/version/sources contract `OvertureFeature` + defines -- as distinct from `feature`, which says only that it is a + `Feature`. It exists so that question is answerable by tag: + `overture-schema list-types --tag overture` selects those types, and a + consumer filtering on them needs no dependency on this package and no + `issubclass` call. + + It is *not* a claim that the type belongs to the Overture schema. Any + package can subclass `OvertureFeature` and register an entry point, and + this provider tags it like any other. Reserving the tag to this package + governs who may *emit* it, not which models receive it. + + One qualifying arm is enough: a discriminated union with any + `OvertureFeature` arm is an Overture feature type. + + This shares its predicate with `theme_provider` below, so `overture` is + also derivable as "carries any `overture:theme=` tag". It is emitted + separately because a selector a user can type is worth more than the + derivation, and because a future non-themed `OvertureFeature` would + break the equivalence. + + Parameters + ---------- + types + Concrete `BaseModel` subclasses for the entry point. For + discriminated-union features this is every arm. + key + Key identifying the model. + tags + Current tags; may be extended. + + Returns + ------- + set[str] + Updated tags, with `"overture"` added if applicable. + """ + if any(issubclass(tp, OvertureFeature) for tp in types): + tags.add("overture") + return tags + + def theme_provider( types: Iterable[type[BaseModel]], key: ModelKey, diff --git a/packages/overture-schema-common/tests/test_common_tag_providers.py b/packages/overture-schema-common/tests/test_common_tag_providers.py index feaa0064b..d323b63b4 100644 --- a/packages/overture-schema-common/tests/test_common_tag_providers.py +++ b/packages/overture-schema-common/tests/test_common_tag_providers.py @@ -7,11 +7,16 @@ from overture.schema.common import OvertureFeature from overture.schema.common.tag_providers import ( + overture_provider, theme_provider, ) from overture.schema.system.discovery import ModelKey from overture.schema.system.discovery.discovery import _generate_tags -from overture.schema.system.discovery.types import TagProviderDict, TagProviderKey +from overture.schema.system.discovery.types import ( + TagProvider, + TagProviderDict, + TagProviderKey, +) @pytest.fixture @@ -34,30 +39,41 @@ def _empty_key(name: str = "x", entry_point: str = "mod:X") -> ModelKey: return ModelKey(name=name, entry_point=entry_point, tags=frozenset()) -def test_theme_provider_plain_class(building: type[OvertureFeature]) -> None: - tags = theme_provider((building,), _empty_key(), set()) - assert tags == {"overture:theme=buildings"} - +@pytest.fixture +def transportation_union() -> object: + """A two-arm transportation union. `_generate_tags` walks it to the arms.""" -def test_theme_provider_discriminated_union() -> None: - # `_generate_tags` is responsible for walking the union to concrete arms. class Road(OvertureFeature[Literal["transportation"], Literal["road"]]): pass class Rail(OvertureFeature[Literal["transportation"], Literal["rail"]]): pass - union = Annotated[ + return Annotated[ Annotated[Road, Tag("road")] | Annotated[Rail, Tag("rail")], Field(discriminator="type"), ] - provider_key = TagProviderKey( - name="theme", - entry_point="common:theme_provider", + + +def _providers(provider: TagProvider) -> TagProviderDict: + """Register one provider under a key this package is allowed to use.""" + key = TagProviderKey( + name=provider.__name__.removesuffix("_provider"), + entry_point=f"common:{provider.__name__}", package_name="overture-schema-common", ) - providers: TagProviderDict = {provider_key: theme_provider} - tags = _generate_tags(union, _empty_key(), providers) + return {key: provider} + + +def test_theme_provider_plain_class(building: type[OvertureFeature]) -> None: + tags = theme_provider((building,), _empty_key(), set()) + assert tags == {"overture:theme=buildings"} + + +def test_theme_provider_discriminated_union(transportation_union: object) -> None: + tags = _generate_tags( + transportation_union, _empty_key(), _providers(theme_provider) + ) assert tags == {"overture:theme=transportation"} @@ -74,3 +90,32 @@ class BadFeature(OvertureFeature): # type: ignore[type-arg] with pytest.raises(TypeError, match="must be annotated Literal"): theme_provider((BadFeature,), _empty_key(), set()) + + +def test_overture_provider_plain_class(building: type[OvertureFeature]) -> None: + tags = overture_provider((building,), _empty_key(), set()) + assert tags == {"overture"} + + +def test_overture_provider_discriminated_union(transportation_union: object) -> None: + tags = _generate_tags( + transportation_union, _empty_key(), _providers(overture_provider) + ) + assert tags == {"overture"} + + +def test_overture_provider_skips_non_overture(not_overture: type[BaseModel]) -> None: + tags = overture_provider((not_overture,), _empty_key(), set()) + assert tags == set() + + +def test_overture_provider_partial_union_still_tags( + not_overture: type[BaseModel], +) -> None: + """One Overture arm is enough; a mixed union still counts as Overture.""" + + class Road(OvertureFeature[Literal["transportation"], Literal["road"]]): + pass + + tags = overture_provider((not_overture, Road), _empty_key(), set()) + assert tags == {"overture"} diff --git a/packages/overture-schema-system/README.md b/packages/overture-schema-system/README.md index 84ff7b86f..503c7b0ba 100644 --- a/packages/overture-schema-system/README.md +++ b/packages/overture-schema-system/README.md @@ -132,7 +132,7 @@ class Park(Identified): class ParkBench(Identified): - park_id: Annotated[Id, Reference(Relationship.BELONGS_TO, Park)] + park_id: Annotated[Id, Reference(Relationship.COMPOSITION, Park, role="part_of")] ``` ## Discovery @@ -222,6 +222,7 @@ Specific plain tags and namespaces are reserved for designated packages. For exa |---|---| | `feature` (tag) | `overture-schema-system` | | `system:` (namespace) | `overture-schema-system` | +| `overture` (tag) | `overture-schema-common` | | `overture:` (namespace) | `overture-schema-common` | When a provider attempts to set a reserved tag from an unauthorized package, discovery logs a warning and discards the tag. @@ -229,6 +230,7 @@ When a provider attempts to set a reserved tag from an unauthorized package, dis ### Built-in Providers - **`feature`** (in `system`) -- adds `feature` if any concrete arm is a `Feature` subclass. +- **`overture`** (in `common`) -- adds `overture` if any concrete arm is an `OvertureFeature` subclass: the model is built on Overture's feature model, as distinct from `feature`, which says only that it is a `Feature`. Consumers that need to ask that question read the tag rather than importing `OvertureFeature`. The tag does not assert that the type belongs to the Overture schema -- a third-party `OvertureFeature` subclass receives it too, and the reservation governs who may emit the tag, not which models get it. - **`theme`** (in `common`) -- adds `overture:theme={theme}` for each `OvertureFeature` referenced. A discriminated-union feature whose arms span multiple themes contributes one tag per distinct theme. ### Selecting Models by Tag diff --git a/packages/overture-schema-system/changelog.d/668.misc.md b/packages/overture-schema-system/changelog.d/668.misc.md new file mode 100644 index 000000000..b76bcab60 --- /dev/null +++ b/packages/overture-schema-system/changelog.d/668.misc.md @@ -0,0 +1 @@ +Reserved the plain `overture` tag to `overture-schema-common` and documented its provider in the README. diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py index 2425a7551..fc3537e32 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/discovery.py @@ -24,6 +24,7 @@ # Tags that are reserved and can only be set by specific packages. _RESERVED_TAGS: dict[str, set[str]] = { "feature": {"overture-schema-system"}, + "overture": {"overture-schema-common"}, } # Namespaces that are reserved and can only be set by specific packages. _RESERVED_NAMESPACES: dict[str, set[str]] = { diff --git a/packages/overture-schema-system/tests/test_tag_providers.py b/packages/overture-schema-system/tests/test_tag_providers.py index 33fd15392..84c1069c8 100644 --- a/packages/overture-schema-system/tests/test_tag_providers.py +++ b/packages/overture-schema-system/tests/test_tag_providers.py @@ -123,6 +123,25 @@ def test_allowed_reserved_tag( assert _generate_tags(any_model, any_key, system_providers) == {"feature"} +def test_reserved_overture_tag( + other_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + providers = {other_tag_provider: fake_provider("overture", "valid")} + result = _generate_tags(any_model, any_key, providers) + assert result == {"valid"} + + +def test_allowed_reserved_overture_tag( + common_tag_provider: TagProviderKey, + any_key: ModelKey, + any_model: type[BaseModel], +) -> None: + common_providers = {common_tag_provider: fake_provider("overture")} + assert _generate_tags(any_model, any_key, common_providers) == {"overture"} + + def test_reserved_namespace( other_tag_provider: TagProviderKey, any_key: ModelKey, From 5fd69da5ea6d6b300e5fe2d98f0fc1da3283f6b1 Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Wed, 19 Aug 2026 11:54:13 -0700 Subject: [PATCH 2/9] fix(cli): render `--help` example blocks and cite tags that exist The command docstrings were raw strings, so Click's `\b` no-rewrap marker stayed two literal characters and every example block collapsed into one paragraph. Drop the `r` prefix; pydocstyle's D301 pushes the other way and is a false positive for Click docstrings, so it is waived with the reason recorded once at the module level. The "official Overture types only" examples suggested `--tag overture --tag feature`, wrong twice over: the plain `overture` tag did not exist, and `--tag` is OR, so pairing it with `feature` widens the scope instead of narrowing it. Cite `--tag overture` alone, reworded -- the tag says a type is built on Overture's feature model, not that it is official. Added an example showing what the OR is for, and tag examples to `list-types`, which had none. The `--tag` syntax note illustrated namespaced tags with `overture:approved`, which no provider can register. No shipped model emits a bare namespaced tag, so the note describes the form instead. `test_help_cites_only_tags_that_exist` covers the whole surface: it splits rendered help per option and checks both citation forms -- an argument (`--tag X`) and an illustration (`(e.g. X)`) -- against the set that option accepts, since `--group-by` takes the key half of a `key=value` tag and `--tag` does not. Judged against a global union instead, a phantom passes in whichever option it was not written for. Signed-off-by: Seth Fitzsimmons --- .../changelog.d/668.bugfix.md | 1 + .../src/overture/schema/cli/commands.py | 48 +++++-- .../src/overture/schema/cli/tag_options.py | 9 +- .../tests/test_cli_commands.py | 117 ++++++++++++++++++ 4 files changed, 159 insertions(+), 16 deletions(-) create mode 100644 packages/overture-schema-cli/changelog.d/668.bugfix.md diff --git a/packages/overture-schema-cli/changelog.d/668.bugfix.md b/packages/overture-schema-cli/changelog.d/668.bugfix.md new file mode 100644 index 000000000..f3dcd1add --- /dev/null +++ b/packages/overture-schema-cli/changelog.d/668.bugfix.md @@ -0,0 +1 @@ +Fixed `--help` example blocks rendering a literal `\b` instead of Click's no-rewrap marker, and replaced the tag-filter examples that cited tags discovery never emitted. diff --git a/packages/overture-schema-cli/src/overture/schema/cli/commands.py b/packages/overture-schema-cli/src/overture/schema/cli/commands.py index 24ce225f1..15e9ad727 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/commands.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/commands.py @@ -225,10 +225,17 @@ def get_source_name(filename: Path) -> str: return "" if str(filename) == "-" else str(filename) +# Every `# noqa: D301` below is the same waiver, against `pydocstyle` (see +# the docformat-only target). D301 wants a raw string wherever a docstring +# contains a backslash, but `\b` here is Click's no-rewrap marker: a raw +# string hands Click two literal characters and every example block collapses +# into one paragraph. Any new command with an Examples block needs the waiver +# too. Note the placement is pydocstyle's -- ruff reports D301 at the +# docstring line instead, so selecting ruff's `D` rules would need its own. @click.group() @click.version_option(package_name="overture-schema") -def cli() -> None: - r"""Overture Schema command-line interface. +def cli() -> None: # noqa: D301 + """Overture Schema command-line interface. Provides validation, schema generation, and type discovery for Overture Maps data. @@ -737,8 +744,8 @@ def validate( excludes: tuple[str, ...], types: tuple[str, ...], show_fields: tuple[str, ...], -) -> None: - r"""Validate Overture Maps data against schemas. +) -> None: # noqa: D301 + """Validate Overture Maps data against schemas. Read from FILENAME or stdin if FILENAME is '-'. Supports JSON, YAML, and GeoJSON formats. @@ -757,8 +764,12 @@ def validate( # Validate specific type $ overture-schema validate --type building data.json \b - # Official Overture types only - $ overture-schema validate --tag overture --tag feature data.json + # Two themes at once (repeatable; scope is their union) + $ overture-schema validate --tag overture:theme=buildings \\ + --tag overture:theme=places data.json + \b + # Only types built on the Overture feature model + $ overture-schema validate --tag overture data.json """ # Resolve model type first (errors here are ValueErrors, not ValidationErrors) try: @@ -803,8 +814,8 @@ def json_schema_command( filters: tuple[str, ...], excludes: tuple[str, ...], types: tuple[str, ...], -) -> None: - r"""Generate JSON schema for Overture Maps types. +) -> None: # noqa: D301 + """Generate JSON schema for Overture Maps types. Outputs a JSON Schema document to stdout that can be used for validation or documentation purposes. @@ -820,8 +831,12 @@ def json_schema_command( # Specific types $ overture-schema json-schema --type building \b - # Official Overture types only - $ overture-schema json-schema --tag overture --tag feature + # Two themes at once (repeatable; scope is their union) + $ overture-schema json-schema --tag overture:theme=buildings \\ + --tag overture:theme=places + \b + # Only types built on the Overture feature model + $ overture-schema json-schema --tag overture """ try: model_type = resolve_types( @@ -838,7 +853,8 @@ def json_schema_command( @tag_selection_options @click.option( "--group-by", - help="Group types by a key/value tag's key (e.g. 'overture:theme'). " + help="Group types by a key/value tag's key, as in " + "--group-by overture:theme. " "Plain and namespaced tags have no value to group by and are " "ignored here.", ) @@ -847,8 +863,8 @@ def list_types( filters: tuple[str, ...], excludes: tuple[str, ...], group_by: str | None, -) -> None: - r"""List all available types. +) -> None: # noqa: D301 + """List all available types. Displays all registered models and can be organized by grouping. @@ -856,6 +872,12 @@ def list_types( Examples: # List all types $ overture-schema list-types + \b + # One theme + $ overture-schema list-types --tag overture:theme=buildings + \b + # Group the listing by theme + $ overture-schema list-types --group-by overture:theme """ try: models = discover_models() diff --git a/packages/overture-schema-cli/src/overture/schema/cli/tag_options.py b/packages/overture-schema-cli/src/overture/schema/cli/tag_options.py index 3befb1143..cd1965f09 100644 --- a/packages/overture-schema-cli/src/overture/schema/cli/tag_options.py +++ b/packages/overture-schema-cli/src/overture/schema/cli/tag_options.py @@ -9,10 +9,13 @@ F = TypeVar("F", bound=Callable[..., object]) +# Every tag named here must be one discovery actually emits -- a tag in help +# text reads as runnable. The namespaced form has no shipped example, so it is +# described rather than illustrated. _TAG_SYNTAX_NOTE = ( - "Accepts plain tags (e.g. feature), namespaced tags " - "(e.g. overture:approved), or compound key/value tags " - "(e.g. overture:theme=buildings)." + "Accepts plain tags (e.g. feature, overture) and compound key/value tags " + "(e.g. overture:theme=buildings). A namespaced form, namespace:predicate, " + "is also accepted for tags that third-party packages register." ) diff --git a/packages/overture-schema-cli/tests/test_cli_commands.py b/packages/overture-schema-cli/tests/test_cli_commands.py index 9fb088c7d..3a9c50a30 100644 --- a/packages/overture-schema-cli/tests/test_cli_commands.py +++ b/packages/overture-schema-cli/tests/test_cli_commands.py @@ -1,6 +1,7 @@ """Tests for CLI commands (validate, list-types, json-schema).""" import json +import re from io import StringIO import pytest @@ -8,6 +9,32 @@ from conftest import build_feature from overture.schema.cli.commands import cli +from overture.schema.system.discovery import discover_models + +_HELP_INVOCATIONS = [["--help"]] + [[name, "--help"] for name in sorted(cli.commands)] + +# Click renders each option as a line starting with two spaces and the flag, +# its help indented under it. +_OPTION_START = re.compile(r"^ (--[\w-]+)", re.MULTILINE) + +# Options whose `(e.g. ...)` illustrations name something other than a tag. +_NOT_TAGS = {"--type", "--show-field"} + + +def _help_segments(output: str) -> list[tuple[str | None, str]]: + """Split `--help` output into (option, text), so a citation keeps its option. + + The leading segment -- description and examples, before the first option + line -- carries `None`. + """ + bounds = list(_OPTION_START.finditer(output)) + if not bounds: + return [(None, output)] + segments: list[tuple[str | None, str]] = [(None, output[: bounds[0].start()])] + for current, following in zip(bounds, [*bounds[1:], None], strict=True): + end = following.start() if following else len(output) + segments.append((current[1], output[current.start() : end])) + return segments class TestListTypesCommand: @@ -29,6 +56,96 @@ def test_list_types_command_help(self, cli_runner: CliRunner) -> None: assert "list-types" in result.output.lower() +class TestHelpFormatting: + """Tests for `--help` rendering and the tags it cites.""" + + @pytest.mark.parametrize("argv", _HELP_INVOCATIONS, ids=lambda a: " ".join(a)) + def test_help_has_no_literal_escape( + self, cli_runner: CliRunner, argv: list[str] + ) -> None: + """Click's no-rewrap marker is interpreted, not printed.""" + result = cli_runner.invoke(cli, argv) + assert result.exit_code == 0 + assert "\\b" not in result.output + + def test_help_keeps_examples_on_separate_lines(self, cli_runner: CliRunner) -> None: + """Each example command occupies its own line rather than reflowing.""" + result = cli_runner.invoke(cli, ["validate", "--help"]) + assert result.exit_code == 0 + commands = [ + line.strip() + for line in result.output.splitlines() + if line.strip().startswith("$ overture-schema") + ] + assert len(commands) >= 4 + assert "$ overture-schema validate data.json" in commands + + def test_help_cites_only_tags_that_exist(self, cli_runner: CliRunner) -> None: + """Tags named in `--help` work in the option they are named for. + + Two citation forms, because a phantom has hidden in each: an + argument (`--tag X`) and an illustration (`(e.g. X)`). Both are + found wherever they appear in the rendered output, including + across a terminal wrap. + + Each is checked against the set that option accepts, not a global + union: `--group-by` takes the *key* half of a `key=value` tag, so + `--tag overture:theme` selects nothing and `--group-by feature` + groups nothing, and neither may pass. Attribution is by splitting + the Options block per option, so an illustration is judged by the + option whose help it sits in. + + Not checked: a tag named in running prose outside both forms. + `namespace:predicate` in the `--tag` syntax note is deliberately + such a placeholder -- describing the grammar without writing + something that looks runnable is why it is worded that way. Nor are + options in `_NOT_TAGS`, whose illustrations name something else -- + `--type`'s "(e.g., building, segment)" names types. + """ + emitted = {tag for key in discover_models() for tag in key.tags} + keys = {tag.split("=", 1)[0] for tag in emitted if "=" in tag} + # What each tag-taking option accepts. + accepts: dict[str | None, set[str]] = { + "--tag": emitted, + "--filter": emitted, + "--exclude": emitted, + "--group-by": keys, + } + argument_form = re.compile(r"--(tag|filter|exclude|group-by) ([\w.:=-]+)") + illustration_form = re.compile(r"e\.g\. ([^)]+)\)") + + cited: list[tuple[str, str, set[str]]] = [] + for argv in _HELP_INVOCATIONS: + result = cli_runner.invoke(cli, argv) + assert result.exit_code == 0, f"{argv}: --help failed" + where = " ".join(argv) + for option, segment in _help_segments(result.output): + # Rejoin words the terminal wrapper split, so a tag broken + # across lines is still seen whole. + flowed = " ".join(segment.split()) + cited += [ + # A citation ending a sentence picks up its period; the + # tag grammar allows `.` inside a name, never at the end. + (where, value.rstrip(".,"), accepts[f"--{flag}"]) + for flag, value in argument_form.findall(flowed) + if value != "TEXT" # the option signature's metavar + ] + if option in _NOT_TAGS: + continue + # An illustration inside a tag option's help is judged by + # that option; one in the description, the examples, or the + # Commands block belongs to no option, so accept either. + accepted = accepts.get(option, emitted | keys) + cited += [ + (where, token.strip("'\"").rstrip(".,"), accepted) + for group in illustration_form.findall(flowed) + for token in re.split(r",\s*", group.strip()) + ] + assert cited, "no tag cited anywhere in --help -- the sweep found nothing" + for where, tag, accepted in cited: + assert tag in accepted, f"{where}: {tag!r} matches no model" + + class TestJsonSchemaCommand: """Tests for the json-schema command.""" From 281f6900c9c20fb83d8cedf3615f4e9d71780ff8 Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Wed, 19 Aug 2026 11:54:13 -0700 Subject: [PATCH 3/9] fix(codegen): name union entry points in `overture-codegen list` The listing fell back to `str(model)` for anything without `__name__`, so `Segment` printed its whole `typing.Annotated[...]` type expression -- several hundred characters of discriminator internals in place of a name. Read the name off the entry point instead. Every registered model has one, class or union alias alike, and it is the name the rest of the toolchain already uses. Signed-off-by: Seth Fitzsimmons --- .../changelog.d/668.bugfix.md | 1 + .../src/overture/schema/codegen/cli.py | 14 +++++++++----- packages/overture-schema-codegen/tests/test_cli.py | 7 +++++++ 3 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 packages/overture-schema-codegen/changelog.d/668.bugfix.md diff --git a/packages/overture-schema-codegen/changelog.d/668.bugfix.md b/packages/overture-schema-codegen/changelog.d/668.bugfix.md new file mode 100644 index 000000000..d46f6bef7 --- /dev/null +++ b/packages/overture-schema-codegen/changelog.d/668.bugfix.md @@ -0,0 +1 @@ +Fixed `overture-codegen list` printing a raw `typing.Annotated[...]` expression for discriminated-union entry points such as `Segment`; entries now list by their entry-point class name. diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py index dc5a7da5a..a5687d817 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py @@ -11,6 +11,7 @@ from overture.schema.system.discovery import ( discover_models, filter_models, + split_entry_point, ) from .extraction.specs import ModelSpec, SupplementarySpec, TypeIdentity @@ -59,11 +60,14 @@ def cli() -> None: def list_models() -> None: """List all discovered models.""" models = discover_models() - names = sorted( - model.__name__ if isinstance(model, type) else str(model) - for model in models.values() - ) - for name in names: + # Name every entry from its entry point, not the loaded object: a + # discriminated union loads as an `Annotated[...]` alias with no + # `__name__`, so `str(model)` would print the whole type expression. + names = [] + for key in models: + _, class_name = split_entry_point(key.entry_point) + names.append(class_name) + for name in sorted(names): click.echo(name) diff --git a/packages/overture-schema-codegen/tests/test_cli.py b/packages/overture-schema-codegen/tests/test_cli.py index 18124ed87..dd90c6880 100644 --- a/packages/overture-schema-codegen/tests/test_cli.py +++ b/packages/overture-schema-codegen/tests/test_cli.py @@ -26,6 +26,13 @@ def test_list_shows_discovered_models(self, cli_runner: CliRunner) -> None: assert "Building" in result.output assert "Place" in result.output + def test_list_names_union_alias_entry_points(self, cli_runner: CliRunner) -> None: + """A union alias lists by its entry-point class name, not its repr.""" + result = cli_runner.invoke(cli, ["list"]) + + assert "Segment" in result.output.split() + assert "typing.Annotated" not in result.output + class TestCliGenerate: """Tests for the generate command.""" From 43d82faa8c240215ef374c5722f0fad49bec28d2 Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Wed, 19 Aug 2026 11:55:38 -0700 Subject: [PATCH 4/9] docs: correct the Python package documentation Fixes the inaccuracies catalogued in #604 and #668. `packages/overture-schema/README.md` documented `from overture.schema import Building, parse, discover_models, json_schema`. None of it resolves: `overture.schema` is a namespace root shipping only `py.typed` since #622 moved the validation API out of it, and `parse()` exists nowhere. Rewritten against the real surface -- models from their theme packages, `validate`/`validate_json` from `overture.schema.validation`, discovery and `json_schema` from `overture.schema.system`. Three READMEs claimed `model_validate()` accepts GeoJSON. It does not, and the framing above it was also wrong: Overture publishes one shape, flat and tabular. The models additionally read and write GeoJSON so the schema works with tools that expect features rather than rows, and because that is the representation the generated JSON Schema describes. The two modes are not interchangeable, and the error a GeoJSON dict produces ("theme Field required") does not point at the cause. Round-trip examples now pass `by_alias=True`, without which `class_` is emitted and the output will not re-validate. The codegen README described `analyze_type()` returning a `TypeInfo` with `.kind`/`.base_type` and imported `TypeKind`. Neither exists; it returns a `(FieldShape, bool, str | None)` tuple. Replaced with a worked example whose output is transcribed from a run, and corrected the layer diagram and the package credited with `discover_models`. The pyspark README's S3 examples named release 2026-06-17.0, which the bucket no longer holds. Bumped, with a note on reading the current identifier from the STAC catalog, since a hardcoded release goes stale by design. PYDANTIC_GUIDE.md prescribed a `models.py` / `enums.py` / `types.py` split inside a per-type subdirectory (#604). No theme package has such a file. Rewritten against the settled layout: one module per feature type at the theme root, `_common.py` for what a theme shares, a subpackage only for a type large enough to split. The container-to-mixin example now uses the real `Named` / `Appearance` mixins `Building` inherits. It also named two enum members that do not exist -- `Relationship` has `COMPOSITION`, `AGGREGATION`, `HIERARCHY`, `ASSOCIATION`, and `CONNECTS_TO`/`BELONGS_TO` are *roles*. A third instance was in overture-schema-system's README. And its `$defs` migration example placed an `Address` with `freeform`/`locality` in the addresses theme, claiming `Building` carries one; that model is `places.Address`, and `Place` is what holds it. Removed the Reference section's five "Complete Templates": they prescribed the dead file-name convention, and their content survives in Quick Start, Quick Reference, and the Relationship Patterns section. Python embedded in Markdown is now formatted with `ruff format`, so documented snippets match the style of the code they describe. That needs ruff 0.16, so the dependency floor moves from 0.13. Signed-off-by: Seth Fitzsimmons --- PYDANTIC_GUIDE.md | 447 +++++++++--------- README.pydantic.md | 33 +- packages/overture-schema-cli/pyproject.toml | 2 +- packages/overture-schema-codegen/README.md | 52 +- .../changelog.d/668.docs.md | 1 + packages/overture-schema-pyspark/README.md | 20 +- .../changelog.d/668.docs.md | 1 + .../overture-schema-system/pyproject.toml | 2 +- packages/overture-schema-validation/README.md | 10 +- .../changelog.d/668.docs.md | 1 + packages/overture-schema/README.md | 124 +++-- .../overture-schema/changelog.d/668.docs.md | 1 + pyproject.toml | 2 +- uv.lock | 6 +- 14 files changed, 383 insertions(+), 319 deletions(-) create mode 100644 packages/overture-schema-codegen/changelog.d/668.docs.md create mode 100644 packages/overture-schema-pyspark/changelog.d/668.docs.md create mode 100644 packages/overture-schema-validation/changelog.d/668.docs.md create mode 100644 packages/overture-schema/changelog.d/668.docs.md diff --git a/PYDANTIC_GUIDE.md b/PYDANTIC_GUIDE.md index 4571b2afe..3e61c3191 100644 --- a/PYDANTIC_GUIDE.md +++ b/PYDANTIC_GUIDE.md @@ -21,7 +21,6 @@ This guide helps you work with Overture Maps Pydantic schemas - Python models th - [Project Architecture](#project-architecture) - [Migrating from JSON Schema](#migrating-from-json-schema) - [Reference](#reference) - - [Complete Templates](#complete-templates) - [Quick Reference](#quick-reference) --- @@ -42,7 +41,11 @@ from pydantic import BaseModel, Field # Overture common models from overture.schema.common import OvertureFeature -from overture.schema.system.geometric import Geometry, GeometryType, GeometryTypeConstraint +from overture.schema.system.geometric import ( + Geometry, + GeometryType, + GeometryTypeConstraint, +) # Validation system from overture.schema.system.field_constraint import UniqueItemsConstraint @@ -59,9 +62,14 @@ from overture.schema.system.string import LanguageTag # Numeric types (use these instead of int/float) from overture.schema.system.numeric import ( - int8, int32, int64, - uint8, uint16, uint32, - float32, float64 + int8, + int32, + int64, + uint8, + uint16, + uint32, + float32, + float64, ) ``` @@ -73,6 +81,7 @@ from pydantic import BaseModel, Field from overture.schema.system.model_constraint import no_extra_fields from overture.schema.system.numeric import int8, float64 + @no_extra_fields class MyCustomType(BaseModel): """Brief description of what this represents.""" @@ -88,10 +97,8 @@ class MyCustomType(BaseModel): priority: Annotated[ int8 | None, Field( - ge=1, - le=10, - description="Priority level from 1 (lowest) to 10 (highest)" - ) + ge=1, le=10, description="Priority level from 1 (lowest) to 10 (highest)" + ), ] = None ``` @@ -101,7 +108,12 @@ class MyCustomType(BaseModel): from typing import Annotated, Literal from pydantic import Field from overture.schema.common import OvertureFeature -from overture.schema.system.geometric import Geometry, GeometryType, GeometryTypeConstraint +from overture.schema.system.geometric import ( + Geometry, + GeometryType, + GeometryTypeConstraint, +) + class MyFeature(OvertureFeature[Literal["my_theme"], Literal["my_type"]]): """Description of what this feature represents.""" @@ -138,9 +150,11 @@ Pydantic models are Python classes that define data structures and their constra ```python from overture.schema.system.model_constraint import no_extra_fields + @no_extra_fields class Address(BaseModel): """A postal address - no extra fields allowed.""" + street: str city: str postal_code: str | None = None @@ -154,8 +168,10 @@ from typing import Literal from overture.schema.common import OvertureFeature from overture.schema.system.numeric import float64 + class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): """A building feature with strongly-typed theme and type.""" + # Inherits: id, theme, type, geometry, bbox, version, sources height: float64 | None = None ``` @@ -182,7 +198,10 @@ from overture.schema.common.level import Stacked from overture.schema.common.names import Named from overture.schema.system.numeric import float64 -class Building(OvertureFeature[Literal["buildings"], Literal["building"]], Named, Stacked): + +class Building( + OvertureFeature[Literal["buildings"], Literal["building"]], Named, Stacked +): # Gets fields from Feature: id, theme, type, geometry, etc. # Gets fields from Named: names # Gets fields from Stacked: level @@ -198,6 +217,7 @@ Sometimes you need a field name that conflicts with Python keywords or conventio from typing import Annotated from pydantic import Field + class Building(OvertureFeature): # Use class_ in Python code, but "class" in the actual data class_: Annotated[str | None, Field(alias="class")] = None @@ -246,7 +266,7 @@ class Building(OvertureFeature): ```python access_policy: Annotated[ str | None, - Field(description="Access policy for the place. When absent, assume 'open'") + Field(description="Access policy for the place. When absent, assume 'open'"), ] = None ``` @@ -266,26 +286,32 @@ Keep the schema separate from business logic. The schema describes the shape of ```python from overture.schema.system.model_constraint import no_extra_fields from overture.schema.system.numeric import ( - int8, int32, int64, # Signed integers - uint8, uint16, uint32, # Unsigned integers - float32, float64 # Floating point + int8, + int32, + int64, # Signed integers + uint8, + uint16, + uint32, # Unsigned integers + float32, + float64, # Floating point ) + @no_extra_fields class MyModel(BaseModel): # Signed integers with specific ranges - level: int8 | None = None # -128 to 127 - year: int32 | None = None # -2,147,483,648 to 2,147,483,647 - timestamp: int64 | None = None # Full 64-bit range + level: int8 | None = None # -128 to 127 + year: int32 | None = None # -2,147,483,648 to 2,147,483,647 + timestamp: int64 | None = None # Full 64-bit range # Unsigned integers (0 and positive only) - red_value: uint8 | None = None # 0 to 255 (like RGB values) - port: uint16 | None = None # 0 to 65,535 (like network ports) - population: uint32 | None = None # 0 to 4,294,967,295 + red_value: uint8 | None = None # 0 to 255 (like RGB values) + port: uint16 | None = None # 0 to 65,535 (like network ports) + population: uint32 | None = None # 0 to 4,294,967,295 # Floating point numbers - height: float64 | None = None # Double precision (recommended) - ratio: float32 | None = None # Single precision + height: float64 | None = None # Double precision (recommended) + ratio: float32 | None = None # Single precision ``` **When to use each:** @@ -316,6 +342,7 @@ Union types allow a field to accept multiple different types. The `|` symbol mea ```python from typing import Literal + class Building(OvertureFeature): # This field can be either a string OR None (most common union) name: str | None = None @@ -328,7 +355,7 @@ class Building(OvertureFeature): ```python # Optional field (most common union) -height: float64 | None = None # Can be a number or missing +height: float64 | None = None # Can be a number or missing # Specific string values (an alternative to enums where descriptions aren't needed) priority: Literal["low", "medium", "high"] | None = None @@ -364,8 +391,8 @@ height: float64 | None = None # With Annotated - type + extra information height: Annotated[ - float64 | None, # The actual type (what kind of data) - Field(description="Height in meters") # Extra metadata + float64 | None, # The actual type (what kind of data) + Field(description="Height in meters"), # Extra metadata ] = None ``` @@ -384,17 +411,16 @@ Use Pydantic's `Field()` function to add constraints and descriptions: from typing import Annotated from pydantic import Field + class Building(OvertureFeature): # Range constraints height: Annotated[ - float64 | None, - Field(ge=0, le=1000, description="Height in meters (0-1000m)") + float64 | None, Field(ge=0, le=1000, description="Height in meters (0-1000m)") ] = None # Integer constraints floors: Annotated[ - int32 | None, - Field(gt=0, lt=200, description="Number of floors (1-199)") + int32 | None, Field(gt=0, lt=200, description="Number of floors (1-199)") ] = None ``` @@ -412,13 +438,12 @@ class Place(OvertureFeature): # Length constraints name: Annotated[ str | None, - Field(min_length=1, max_length=100, description="Place name (1-100 chars)") + Field(min_length=1, max_length=100, description="Place name (1-100 chars)"), ] = None # Pattern matching postal_code: Annotated[ - str | None, - Field(pattern=r"^\d{5}(-\d{4})?$", description="US postal code") + str | None, Field(pattern=r"^\d{5}(-\d{4})?$", description="US postal code") ] = None ``` @@ -446,12 +471,13 @@ class Building(Feature): ```python from overture.schema.system.field_constraint import UniqueItemsConstraint + class Building(OvertureFeature): # List with size and uniqueness constraints categories: Annotated[ list[str] | None, Field(min_length=1, max_length=10, description="1-10 categories"), - UniqueItemsConstraint() # Must come AFTER Field() + UniqueItemsConstraint(), # Must come AFTER Field() ] = None ``` @@ -487,6 +513,7 @@ Enums define a fixed set of allowed values: ```python from enum import Enum + class BuildingClass(str, Enum): """Further delineation of the building's built purpose.""" @@ -495,6 +522,7 @@ class BuildingClass(str, Enum): INDUSTRIAL = "industrial" CIVIC = "civic" + # Usage in a model class Building(OvertureFeature): class_: Annotated[BuildingClass | None, Field(alias="class")] = None @@ -509,6 +537,7 @@ Use `DocumentedEnum` from `overture.schema.system.doc` when enum members need th ```python from overture.schema.system.doc import DocumentedEnum + class VehicleType(str, DocumentedEnum): """Types of vehicles for transportation.""" @@ -630,8 +659,12 @@ parent_division_id: Annotated[Id, Reference(Relationship.HIERARCHY, Division)] capital_division_ids: Annotated[list[Id], Reference(Relationship.HIERARCHY, Division)] # With role: unambiguous -parent_division_id: Annotated[Id, Reference(Relationship.HIERARCHY, Division, role="child_of")] -capital_division_ids: Annotated[list[Id], Reference(Relationship.HIERARCHY, Division, role="has_as_capital")] +parent_division_id: Annotated[ + Id, Reference(Relationship.HIERARCHY, Division, role="child_of") +] +capital_division_ids: Annotated[ + list[Id], Reference(Relationship.HIERARCHY, Division, role="has_as_capital") +] ``` The `role` must be a non-empty snake_case string (lowercase letters, digits, underscores). It describes the source's role relative to the target using source-perspective phrasing. @@ -650,30 +683,35 @@ from pydantic import Field from overture.schema.common import OvertureFeature from overture.schema.system.ref import Id, Reference, Relationship + # COMPOSITION — part points to its whole class BuildingPart(OvertureFeature[Literal["buildings"], Literal["building_part"]]): """A structural part of a building.""" + building_id: Annotated[ Id, Reference(Relationship.COMPOSITION, Building, role="part_of"), - Field(description="The building to which this part belongs") + Field(description="The building to which this part belongs"), ] + # HIERARCHY — child points to parent class DivisionArea(OvertureFeature[Literal["divisions"], Literal["division_area"]]): """Area polygon nested under a division.""" + division_id: Annotated[ Id, Reference(Relationship.HIERARCHY, Division, role="child_of"), - Field(description="Division ID of the parent division of this area.") + Field(description="Division ID of the parent division of this area."), ] + # ASSOCIATION — peer reference, no ownership class ConnectorReference(BaseModel): """Reference to a connector feature.""" + connector_id: Annotated[ - Id, - Reference(Relationship.ASSOCIATION, Connector, role="connects_to") + Id, Reference(Relationship.ASSOCIATION, Connector, role="connects_to") ] ``` @@ -693,14 +731,8 @@ class AdminCityCenterAssociation( ): """Describes how an administrative area relates to a city center.""" - admin_area_id: Annotated[ - Id, - Reference(Relationship.ASSOCIATION, AdminArea) - ] - city_center_id: Annotated[ - Id, - Reference(Relationship.ASSOCIATION, CityCenter) - ] + admin_area_id: Annotated[Id, Reference(Relationship.ASSOCIATION, AdminArea)] + city_center_id: Annotated[Id, Reference(Relationship.ASSOCIATION, CityCenter)] # Information about the relationship itself relationship_type: Literal["primary_center", "secondary_center"] = "primary_center" @@ -719,16 +751,25 @@ When a feature needs to reference multiple other features, use a list of referen ```python # COMPOSITION — boundary defines two divisions -class DivisionBoundary(OvertureFeature[Literal["divisions"], Literal["division_boundary"]]): +class DivisionBoundary( + OvertureFeature[Literal["divisions"], Literal["division_boundary"]] +): """A boundary line between two divisions.""" + division_ids: Annotated[ - list[Annotated[Id, Reference(Relationship.COMPOSITION, Division, role="boundary_of")]], + list[ + Annotated[ + Id, Reference(Relationship.COMPOSITION, Division, role="boundary_of") + ] + ], Field(min_length=2, max_length=2, description="Left and right divisions"), ] + # AGGREGATION — route groups segments class Route(OvertureFeature[Literal["transportation"], Literal["route"]]): """A transportation route passing through multiple segments.""" + segment_ids: Annotated[ list[Id], Reference(Relationship.AGGREGATION, TransportationSegment, role="groups"), @@ -750,7 +791,7 @@ Include `Reference` annotations for semantic clarity and documentation: division_id: Annotated[ Id, Reference(Relationship.HIERARCHY, Division, role="child_of"), - Field(description="Division ID of the parent division of this area.") + Field(description="Division ID of the parent division of this area."), ] # Avoid — missing semantic information @@ -774,11 +815,15 @@ from typing import Annotated, Literal from pydantic import Field from overture.schema.common import OvertureFeature + # Base class with common fields -class TransportationSegment(OvertureFeature[Literal["transportation"], Literal["segment"]]): +class TransportationSegment( + OvertureFeature[Literal["transportation"], Literal["segment"]] +): subtype: Subtype # This is the discriminator field # ... common fields for all segments + # Specific segment types class RoadSegment(TransportationSegment): subtype: Literal[Subtype.ROAD] # Must be "road" @@ -786,16 +831,17 @@ class RoadSegment(TransportationSegment): speed_limits: SpeedLimits | None = None # ... road-specific fields + class RailSegment(TransportationSegment): subtype: Literal[Subtype.RAIL] # Must be "rail" class_: Annotated[RailClass, Field(alias="class")] rail_flags: RailFlags | None = None # ... rail-specific fields + # Union type that automatically picks the right model based on subtype Segment = Annotated[ - RoadSegment | RailSegment | WaterSegment, - Field(discriminator="subtype") + RoadSegment | RailSegment | WaterSegment, Field(discriminator="subtype") ] ``` @@ -847,7 +893,7 @@ road_segment = RoadSegment(subtype=Subtype.ROAD, ...) # Valid Instead of making classes abstract, we use **entry point registration** where only specific concrete types are discoverable as map features: -```python +```toml # In packages/overture-schema-theme-transportation/pyproject.toml [project.entry-points."overture.models"] connector = "overture.schema.transportation:Connector" @@ -874,25 +920,29 @@ segment = "overture.schema.transportation:Segment" from typing import Annotated from pydantic import BaseModel, Field + @no_extra_fields class Names(BaseModel): primary: str # Keys (strings) must match a language tag pattern, values are strings - common: Annotated[ - dict[ - # The key type - Annotated[ - str, - Field( - pattern=r"^[a-z]{2,3}(-[A-Z]{2})?$", - description="Language tag (e.g., 'en', 'es-MX')" - ) + common: ( + Annotated[ + dict[ + # The key type + Annotated[ + str, + Field( + pattern=r"^[a-z]{2,3}(-[A-Z]{2})?$", + description="Language tag (e.g., 'en', 'es-MX')", + ), + ], + str, # The value type ], - str, # The value type - ], - Field(json_schema_extra={"additionalProperties": False}), - ] | None = None + Field(json_schema_extra={"additionalProperties": False}), + ] + | None + ) = None ``` **Example data:** @@ -918,22 +968,24 @@ The `additionalProperties: False` ensures only keys matching the pattern are all from typing import Annotated from pydantic import Field + # Each item has its own field validation @no_extra_fields class HierarchyItem(BaseModel): division_id: str name: str + class Division(OvertureFeature): # Nested list validation: outer list AND inner lists both have length constraints hierarchies: Annotated[ list[ # Outer list Annotated[ list[HierarchyItem], # Inner list - Field(min_length=1) # Inner list must have at least 1 item + Field(min_length=1), # Inner list must have at least 1 item ] ], - Field(min_length=1) # Outer list must have at least 1 hierarchy + Field(min_length=1), # Outer list must have at least 1 hierarchy ] ``` @@ -969,10 +1021,11 @@ SegmentId = NewType("SegmentId", str) # IDs are strings, but distinct CountryCode = NewType("CountryCode", str) # Country codes are strings, but distinct # Create aliases for complex field patterns -EmailList = NewType("EmailList", Annotated[ - list[str], - Field(min_length=1, description="List of email addresses") -]) +EmailList = NewType( + "EmailList", + Annotated[list[str], Field(min_length=1, description="List of email addresses")], +) + @no_extra_fields class Contact(BaseModel): @@ -997,44 +1050,75 @@ class Contact(BaseModel): #### File Organization -Organize code by scope and avoid circular imports: +Organize code by scope, and avoid circular imports. -**Cross-theme shared**: `overture-schema-common` package +**Cross-theme shared**: the `overture-schema-common` package. Definitions more than +one theme needs -- `OvertureFeature`, `Names`, `Sources`, the scoping framework. -- Used by multiple themes (e.g., `OvertureFeature`, `Names`, `Sources`, `Scope`) +**One module per feature type**: at the theme package root, named after the type in +snake_case. + +```text +packages/overture-schema-theme-buildings/src/overture/schema/buildings/ + __init__.py # re-exports the public names, declares __all__ + _common.py # shared by Building and BuildingPart + building.py # Building, BuildingSubtype, BuildingClass + building_part.py # BuildingPart +``` -**Theme-level shared**: Theme package root (e.g., `overture-schema-theme-transportation/src/overture/schema/transportation/`) +The module owns everything specific to its type: the `Feature` subclass, its enums, +its NewTypes, and its supporting models. `building.py` defines `BuildingSubtype` and +`BuildingClass` next to `Building`, because nothing else uses them. -- Used by multiple types within a theme (e.g., `AccessRules`, `RoadSurface`) +**Theme-level shared**: `_common.py` at the theme package root, for definitions two +or more types in the theme need. `buildings/_common.py` holds `Appearance`, +`RoofShape`, and the material enums that both `Building` and `BuildingPart` use. The +leading underscore marks the module private -- the theme's `__init__.py` re-exports +the public names from it. -**Type-specific**: Type subdirectory (e.g., `overture-schema-theme-transportation/src/overture/schema/transportation/segment/`) +**A type large enough to split**: a subpackage named after the type, applying the +same rules one level down. -- Only used by one specific type (e.g., `SegmentType`, `LaneConfiguration`) +```text +packages/overture-schema-theme-transportation/src/overture/schema/transportation/ + __init__.py # re-exports from connector and segment + connector.py # Connector + segment/ + __init__.py # assembles the Segment discriminated union, re-exports + _common.py # TransportationSegment and what the arms share + road.py # RoadSegment and its supporting types + rail.py # RailSegment and its supporting types + water.py # WaterSegment +``` -**File type rules:** +Every `__init__.py` re-exports its public names and declares `__all__`, so consumers +import from the package rather than reaching into the defining module: +`from overture.schema.buildings import Building`, not +`from overture.schema.buildings.building import Building`. Entry points name the +package root for the same reason -- `building = "overture.schema.buildings:Building"`. -- **`models.py`**: Pydantic model classes (and type aliases that reference models) -- **`enums.py`**: Enum classes only, no project imports -- **`types.py`**: Type aliases that don't reference models, no project imports +Modules are named for the thing they define, not the kind of thing: an enum lives in +the module whose type uses it, and moves up to `_common.py` once a second type needs it. #### Import Organization ```python # Standard library imports first -from typing import Annotated, Literal, NewType from enum import Enum +from typing import Annotated, Literal, NewType # Third-party imports from pydantic import BaseModel, ConfigDict, Field -# Cross-theme imports -from overture.schema.common import OvertureFeature +# Cross-theme and system imports +from overture.schema.common.scoping import Heading, Scope, scoped +from overture.schema.system.doc import DocumentedEnum from overture.schema.system.field_constraint import UniqueItemsConstraint from overture.schema.system.model_constraint import no_extra_fields -# Local imports last -from .enums import SegmentType -from .types import SegmentId, LaneWidth # Only non-model type aliases +# Local imports last -- siblings in the theme, then the module's own package +from ..connector import Connector +from ._common import SegmentSubtype, TransportationSegment ``` `uv run ruff format ` will sort your imports in this order automatically. @@ -1045,20 +1129,22 @@ This project uses a custom validation system that generates better JSON Schema o ```python # Don't do this -@field_validator('categories') +@field_validator("categories") def validate_categories_unique(cls, v): if v and len(v) != len(set(v)): - raise ValueError('Categories must be unique') + raise ValueError("Categories must be unique") return v + # Do this instead from overture.schema.system.field_constraint import UniqueItemsConstraint + class Building(OvertureFeature): categories: Annotated[ list[str] | None, Field(min_length=1, description="Building categories"), - UniqueItemsConstraint() + UniqueItemsConstraint(), ] = None ``` @@ -1088,16 +1174,18 @@ properties: **Pydantic approach:** ```python -# In overture-schema-theme-addresses/src/overture/schema/addresses/address.py +# In overture-schema-theme-places/src/overture/schema/places/place.py @no_extra_fields class Address(BaseModel): """A postal address.""" + freeform: str | None = None locality: str | None = None -# In overture-schema-theme-buildings/src/overture/schema/buildings/building.py -class Building(OvertureFeature): - address: Address | None = None + +# Same module -- Place is the type that carries one. +class Place(OvertureFeature): + addresses: list[Address] | None = None ``` **Primary differences:** @@ -1131,20 +1219,31 @@ allOf: **Pydantic equivalent** uses **mixin classes**: ```python -# In common/names.py +# namesContainer -> Named, in +# overture-schema-common/src/overture/schema/common/names.py class Named(BaseModel): """Properties defining the names of a feature.""" + names: Names | None = None -# In buildings/models.py -class Shape(BaseModel): - """Properties of the building's shape.""" + +# shapeContainer -> Appearance, in buildings/_common.py, +# shared by Building and BuildingPart +class Appearance(BaseModel): + """Physical and visual properties of a building.""" + height: float64 | None = None num_floors: int32 | None = None + # ... roof and facade fields + -# Usage with multiple inheritance -class Building(Feature, Named, Shape): - pass # "pass" means "do nothing" - Building inherits names, height, num_floors, etc. from its parents +# Usage with multiple inheritance -- this is how Building is actually declared +class Building( + OvertureFeature[Literal["buildings"], Literal["building"]], + Named, + Stacked, + Appearance, +): ... # inherits names, level, height, num_floors, and the rest from its parents ``` JSON Schema containers become **mixin classes** in Pydantic that you inherit from. @@ -1166,147 +1265,25 @@ JSON Schema containers become **mixin classes** in Pydantic that you inherit fro ## Reference -### Complete Templates - -#### Basic Model Template - -```python models.py -from typing import Annotated -from pydantic import BaseModel, Field -from overture.schema.system.model_constraint import no_extra_fields -from overture.schema.system.numeric import int8, float64 - -@no_extra_fields -class MyCustomType(BaseModel): - """Brief description of what this represents.""" - - # Required fields (no default value) - name: str - category: str - - # Optional fields (with default values) - description: str | None = None - - # Field with constraints and description - priority: Annotated[ - int8 | None, - Field( - ge=1, - le=10, - description="Priority level from 1 (lowest) to 10 (highest)" - ) - ] = None -``` - -#### Feature Template - -```python models.py -from typing import Annotated, Literal -from pydantic import Field -from overture.schema.common import OvertureFeature -from overture.schema.system.geometric import Geometry, GeometryType, GeometryTypeConstraint - -class MyFeature(OvertureFeature[Literal["my_theme"], Literal["my_type"]]): - """Description of what this feature represents.""" - - # Geometry with constraints - geometry: Annotated[ - Geometry, - GeometryTypeConstraint(GeometryType.POINT), - Field(description="Location of this feature"), - ] - - # Custom fields - my_field: str | None = None -``` - -#### Enum Template - -```python enums.py -from enum import Enum - -class MyEnum(str, Enum): - """Description of what this enum represents.""" - - VALUE_ONE = "value_one" - VALUE_TWO = "value_two" - VALUE_THREE = "value_three" -``` - -#### Model with Validation Constraints - -```python models.py -from typing import Annotated -from pydantic import BaseModel, Field -from overture.schema.system.field_constraint import UniqueItemsConstraint -from overture.schema.system.model_constraint import no_extra_fields - -@no_extra_fields -class Contact(BaseModel): - """Contact information with validation constraints.""" - - name: str - email: str | None = None - phone: str | None = None - - # List with constraints - tags: Annotated[ - list[str] | None, - Field(min_length=1, description="Contact tags"), - UniqueItemsConstraint() # No duplicate tags - ] = None -``` - -#### Association Feature Template - -```python models.py -from typing import Annotated, Literal -from pydantic import Field -from overture.schema.common import OvertureFeature -from overture.schema.system.numeric import float64 -from overture.schema.system.ref import Id, Reference, Relationship - -class MyAssociation(OvertureFeature[Literal["associations"], Literal["my_association"]]): - """Represents a relationship between two features with metadata.""" - - # References to the associated features - feature_a_id: Annotated[ - Id, - Reference(Relationship.CONNECTS_TO, FeatureA), - Field(description="First feature in the relationship") - ] - - feature_b_id: Annotated[ - Id, - Reference(Relationship.CONNECTS_TO, FeatureB), - Field(description="Second feature in the relationship") - ] - - # Relationship metadata - relationship_type: Literal["primary", "secondary"] = "primary" - confidence: Annotated[float64 | None, Field(ge=0.0, le=1.0)] = None - - # Optional contextual information - notes: str | None = None -``` - ### Quick Reference #### Essential Patterns (Most Common) ```python # Basic field types -name: str # Required string -name: str | None = None # Optional string -count: int32 # Required integer +name: str # Required string +name: str | None = None # Optional string +count: int32 # Required integer priority: Literal["high", "medium", "low"] | None = None # Constrained values # Validated fields height: Annotated[float64 | None, Field(ge=0, description="Height in meters")] = None tags: Annotated[list[str] | None, Field(min_length=1), UniqueItemsConstraint()] = None -# Association patterns -parent_id: Annotated[Id | None, Reference(Relationship.BELONGS_TO, ParentModel)] = None +# Association patterns -- Relationship is the kind, role is the meaning +parent_id: Annotated[ + Id | None, Reference(Relationship.HIERARCHY, ParentModel, role="child_of") +] = None connector_ids: list[Id] # References to multiple related features ``` @@ -1319,11 +1296,13 @@ class Address(BaseModel): street: str city: str | None = None + # Feature model class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): geometry: Geometry height: float64 | None = None + # Enum class Status(str, Enum): ACTIVE = "active" diff --git a/README.pydantic.md b/README.pydantic.md index 866e39670..1b0376f75 100644 --- a/README.pydantic.md +++ b/README.pydantic.md @@ -116,21 +116,28 @@ Install the main package using `pip` (or your package manager of choice): pip install overture-schema ``` +Overture publishes data in one shape: flat and tabular, the column layout of the +Parquet release, which Pydantic's Python mode reads. The models also accept and +emit GeoJSON, through JSON mode, so the schema works with tools that expect +features rather than rows -- and it is the representation the generated JSON +Schema describes. The modes are not interchangeable: a GeoJSON dict passed to +`model_validate` reports `theme` and `version` missing and `type` set to +`'Feature'`. + ```python -from overture.schema.buildings.building import Building -from overture.schema.places.place import Place -import json +from overture.schema.buildings import Building +from overture.schema.places import Place -# Validate data - supports both flat/tabular- (Parquet-style) and GeoJSON-formatted -# dicts -building = Building.model_validate(feature_data) -building_geojson = Building.model_validate(geojson_feature) +# Flat / tabular dict -- Python mode +building = Building.model_validate(feature_row) -# Parse and validate JSON strings -building_from_json = Building.model_validate_json(json_string) +# GeoJSON, as a string or bytes -- JSON mode +building = Building.model_validate_json(geojson_text) -# Convert to GeoJSON format -geojson_output = building.model_dump(mode="json") +# Serialize back to GeoJSON. by_alias=True is required: without it, aliased +# fields serialize under their Python names (`class_`, not `class`) and the +# output will not re-validate. +geojson_output = building.model_dump(mode="json", by_alias=True, exclude_none=True) ``` ## Schema Extension @@ -187,9 +194,9 @@ from overture.schema.system.discovery import ( models = discover_models() # { # ModelKey(name="building", entry_point="overture.schema.buildings:Building", -# tags=frozenset({"feature", "overture", "overture:theme=buildings"})): BuildingModel, +# tags=frozenset({"feature", "overture", "overture:theme=buildings"})): Building, # ModelKey(name="place", entry_point="overture.schema.places:Place", -# tags=frozenset({"feature", "overture", "overture:theme=places"})): PlaceModel, +# tags=frozenset({"feature", "overture", "overture:theme=places"})): Place, # ... # } diff --git a/packages/overture-schema-cli/pyproject.toml b/packages/overture-schema-cli/pyproject.toml index e5642bc96..783c5d75c 100644 --- a/packages/overture-schema-cli/pyproject.toml +++ b/packages/overture-schema-cli/pyproject.toml @@ -34,7 +34,7 @@ build-backend = "uv_build" [dependency-groups] dev = [ "pytest>=9.0.0", - "ruff>=0.13.0", + "ruff>=0.16.0", "mypy>=1.17.0", ] diff --git a/packages/overture-schema-codegen/README.md b/packages/overture-schema-codegen/README.md index 3252dfffd..22384e38a 100644 --- a/packages/overture-schema-codegen/README.md +++ b/packages/overture-schema-codegen/README.md @@ -10,10 +10,10 @@ structure collapses into `anyOf` arrays with duplicated fields. Navigating Python's type annotation machinery -- NewType chains, nested `Annotated` wrappers, union filtering, generic resolution -- is complex. The codegen does it once. -`analyze_type()` unwraps annotations into `TypeInfo`, a flat target-independent -representation. Extractors build specs from `TypeInfo`. Renderers consume specs without -touching the type system. New output targets (Arrow schemas, PySpark expressions) add -renderers, not extraction logic. +`analyze_type()` unwraps an annotation into a `FieldShape`, a tree-shaped +target-independent representation. Extractors build specs from `FieldShape`. Renderers +consume specs without touching the type system. New output targets (Arrow schemas, +PySpark expressions) add renderers, not extraction logic. ## Usage @@ -41,9 +41,9 @@ Rendering Output formatting, all presentation decisions ^ Output Layout What to generate, where it goes, how outputs link ^ -Extraction TypeInfo, FieldSpec, RecordSpec, UnionSpec +Extraction FieldShape, FieldSpec, RecordSpec, UnionSpec ^ -Discovery discover_models() from overture-schema-common +Discovery discover_models() from overture-schema-system ``` **Discovery** loads registered Pydantic models via entry points. The return dict @@ -72,16 +72,38 @@ examples), enum pages, NewType pages, and aggregate numeric/geometry reference p ## Programmatic use +`analyze_type()` returns a 3-tuple: the annotation's `FieldShape`, whether the +field accepts `None`, and the first `Field(description=...)` encountered while +unwrapping. + +```python +from overture.schema.buildings import Building +from overture.schema.codegen.extraction.type_analyzer import analyze_type + +annotation = Building.model_fields["version"].rebuild_annotation() +shape, nullable, description = analyze_type(annotation, owner=Building) + +# NewTypeShape(name='FeatureVersion', ref=..., inner=Primitive(base_type='int32', ...)) +assert shape.name == "FeatureVersion" +assert shape.inner.base_type == "int32" +assert nullable is False +``` + +`FieldShape` is a tree, not a flat record: `NewTypeShape`, `ArrayOf`, and `MapOf` +wrap an inner shape, and the three terminals (`Primitive`, `LiteralScalar`, +`AnyScalar`) sit at the leaves. Nesting order is meaningful -- +`NewTypeShape(inner=ArrayOf(...))` is a NewType over `list[X]`, while +`ArrayOf(element=NewTypeShape(...))` is a list of NewType-wrapped values. + +Constraints attach to the layer they target and carry the NewType that +contributed them: + ```python -from overture.schema.codegen.extraction.type_analyzer import analyze_type, TypeKind - -info = analyze_type(some_annotation) -assert info.kind == TypeKind.PRIMITIVE -assert info.base_type == "int32" -assert info.newtype_name == "FeatureVersion" -# Constraints carry provenance: -for cs in info.constraints: - print(f"{cs.constraint} from {cs.source}") +for source in shape.inner.constraints: + print(f"{source.constraint} from {source.source_name}") +# Ge(ge=0) from FeatureVersion +# Ge(ge=-2147483648) from int32 +# Le(le=2147483647) from int32 ``` ## Fetching sample data diff --git a/packages/overture-schema-codegen/changelog.d/668.docs.md b/packages/overture-schema-codegen/changelog.d/668.docs.md new file mode 100644 index 000000000..88e7d94f0 --- /dev/null +++ b/packages/overture-schema-codegen/changelog.d/668.docs.md @@ -0,0 +1 @@ +Rewrote the README's programmatic-use section against the current `analyze_type()` signature, which returns a `FieldShape` tuple rather than the removed `TypeInfo`/`TypeKind`. diff --git a/packages/overture-schema-pyspark/README.md b/packages/overture-schema-pyspark/README.md index 13981ffba..85c85b24a 100644 --- a/packages/overture-schema-pyspark/README.md +++ b/packages/overture-schema-pyspark/README.md @@ -65,7 +65,7 @@ overture-validate segment samples/segment.parquet \ --conf spark.master=local[4] # Continue past schema mismatches (e.g. Float vs Double on bbox) -overture-validate place s3a://overturemaps-us-west-2/release/2026-06-17.0 \ +overture-validate place s3a://overturemaps-us-west-2/release/2026-07-22.0 \ --skip-schema-check # Skip checks for a column absent from the data @@ -106,15 +106,15 @@ structure: | --- | --- | --- | | Hive partition path (contains `/theme=`) | `.../theme=transportation/type=segment/` | Reads directly; derives `basePath` so Spark discovers partition columns. | | Individual file | `segment.parquet` | Reads directly; data already contains `theme`/`type` columns. | -| Release root | `s3a://overturemaps-us-west-2/release/2026-06-17.0` | Appends `theme={theme}/type={type}` using the schema's theme mapping; sets `basePath` to the original path. | +| Release root | `s3a://overturemaps-us-west-2/release/2026-07-22.0` | Appends `theme={theme}/type={type}` using the schema's theme mapping; sets `basePath` to the original path. | This means you can point the CLI at a release root and it constructs the full Hive path automatically: ```bash # These are equivalent: -overture-validate segment s3a://overturemaps-us-west-2/release/2026-06-17.0 -overture-validate segment s3a://overturemaps-us-west-2/release/2026-06-17.0/theme=transportation/type=segment/ +overture-validate segment s3a://overturemaps-us-west-2/release/2026-07-22.0 +overture-validate segment s3a://overturemaps-us-west-2/release/2026-07-22.0/theme=transportation/type=segment/ ``` ### Reading from S3 @@ -126,14 +126,22 @@ the Overture release bucket: ```bash overture-validate segment \ - s3a://overturemaps-us-west-2/release/2026-06-17.0/theme=transportation/type=segment/ + s3a://overturemaps-us-west-2/release/2026-07-22.0/theme=transportation/type=segment/ +``` + +The bucket retains only recent releases -- older identifiers, including ones +named in earlier revisions of this file, have been removed. Read the current +one from the STAC catalog's `latest` field: + +```bash +curl -s https://stac.overturemaps.org/catalog.json | jq -r .latest ``` To use named AWS credentials instead of anonymous access: ```bash overture-validate segment \ - s3a://overturemaps-us-west-2/release/2026-06-17.0/theme=transportation/type=segment/ \ + s3a://overturemaps-us-west-2/release/2026-07-22.0/theme=transportation/type=segment/ \ --conf spark.hadoop.fs.s3a.aws.credentials.provider=software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider ``` diff --git a/packages/overture-schema-pyspark/changelog.d/668.docs.md b/packages/overture-schema-pyspark/changelog.d/668.docs.md new file mode 100644 index 000000000..ce95a6981 --- /dev/null +++ b/packages/overture-schema-pyspark/changelog.d/668.docs.md @@ -0,0 +1 @@ +Updated the README's S3 examples to a release the bucket still retains, and documented how to read the current release identifier from the STAC catalog. diff --git a/packages/overture-schema-system/pyproject.toml b/packages/overture-schema-system/pyproject.toml index 613349cef..952d299ec 100644 --- a/packages/overture-schema-system/pyproject.toml +++ b/packages/overture-schema-system/pyproject.toml @@ -26,7 +26,7 @@ Issues = "https://github.com/OvertureMaps/schema/issues" [dependency-groups] dev = [ "pytest>=9.0.0", - "ruff>=0.13.0", + "ruff>=0.16.0", "mypy>=1.17.0", ] diff --git a/packages/overture-schema-validation/README.md b/packages/overture-schema-validation/README.md index 41cda9f8b..f6fdb7220 100644 --- a/packages/overture-schema-validation/README.md +++ b/packages/overture-schema-validation/README.md @@ -15,11 +15,13 @@ pip install overture-schema-validation ```python from overture.schema.validation import validate, validate_json -# Validate a Python object (a dict or a model instance) -feature = validate({"type": "segment", "id": "...", "geometry": "..."}) +# A Python object -- the flat, tabular (Parquet-style) shape +feature = validate(feature_row) -# Validate a JSON document -feature = validate_json('{"type": "segment", "id": "...", "geometry": "..."}') +# A JSON document -- GeoJSON +feature = validate_json(geojson_text) ``` +The two entry points are not interchangeable. `validate` runs Pydantic's Python mode, which reads the flat column layout of the Parquet release -- the shape Overture publishes. `validate_json` runs JSON mode, which reads the GeoJSON representation the models support for compatibility with tools that expect features rather than rows. Handing a GeoJSON dict to `validate` reports `theme` and `version` missing and `type` set to `'Feature'`. + Both raise `pydantic.ValidationError` when the input matches no model. Which models participate is resolved at runtime by entry-point discovery, so installing additional Overture theme packages widens what these functions accept. diff --git a/packages/overture-schema-validation/changelog.d/668.docs.md b/packages/overture-schema-validation/changelog.d/668.docs.md new file mode 100644 index 000000000..014c31693 --- /dev/null +++ b/packages/overture-schema-validation/changelog.d/668.docs.md @@ -0,0 +1 @@ +Corrected the README's usage examples, which showed `validate_json` accepting the flat tabular shape rather than GeoJSON. diff --git a/packages/overture-schema/README.md b/packages/overture-schema/README.md index 4a725eda8..d3d55c6b7 100644 --- a/packages/overture-schema/README.md +++ b/packages/overture-schema/README.md @@ -10,76 +10,118 @@ This package provides Pydantic models for validating and working with Overture M pip install overture-schema ``` +`overture-schema` is a metapackage: it pulls in every theme package plus the +validation library and the CLI, and ships no code of its own. `overture.schema` +is a namespace root, so import from the theme and system packages rather than +from `overture.schema` directly. + ## Usage -Import and use schemas: +Import models from the theme package that defines them: ```python -from overture.schema import Building, Place -import json +from overture.schema.buildings import Building +from overture.schema.places import Place +``` + +### Tabular data, and GeoJSON for compatibility + +Overture publishes data in one shape: flat and tabular -- the column layout of the +Parquet release, with `theme`, `type`, and `version` as top-level columns and +geometry as WKT. That is what **Python mode** (`model_validate`) reads. + +The models also accept and emit GeoJSON, through **JSON mode** +(`model_validate_json`), so the schema works with tools that expect features +rather than rows. The generated JSON Schema describes that representation. + +The modes are not interchangeable. Passing a GeoJSON dict to `model_validate` +reports `theme`/`version` missing and `type` set to `'Feature'`, because it is +reading GeoJSON keys as flat columns. -# Validate Overture Maps data (supports both flat/tabular and GeoJSON formats) -building = Building.model_validate(feature_data) -place = Place.model_validate(geojson_feature) +```python +# Flat / tabular (Parquet-shaped) dict +building = Building.model_validate(feature_row) -# Parse and validate JSON strings -building_from_json = Building.model_validate_json(json_string) +# GeoJSON -- JSON mode, from a string or bytes +building = Building.model_validate_json(geojson_text) -# Convert to GeoJSON format for output -geojson_output = building.model_dump(mode="json") +# Serialize back to GeoJSON. by_alias=True is required: without it, +# aliased fields serialize under their Python names (`class_`, not +# `class`) and the output will not re-validate. +geojson_output = building.model_dump(mode="json", by_alias=True, exclude_none=True) ``` ### Available Models +Each model lives in its theme package. The metapackage installs all of them: + ```python -# All models are re-exported from their respective theme packages for convenience -from overture.schema import ( - # Addresses theme - Address, - # Base theme +from overture.schema.addresses import Address +from overture.schema.base import ( Bathymetry, Infrastructure, Land, LandCover, LandUse, Water, - # Buildings theme - Building, - BuildingPart, - # Divisions theme - Division, - DivisionArea, - DivisionBoundary, - # Places theme - Place, - # Transportation theme - Connector, - Segment, ) +from overture.schema.buildings import Building, BuildingPart +from overture.schema.divisions import Division, DivisionArea, DivisionBoundary +from overture.schema.places import Place +from overture.schema.transportation import Connector, Segment +``` + +`Segment` is a discriminated union alias rather than a class, so it validates +through a `TypeAdapter`: + +```python +from pydantic import TypeAdapter + +segments = TypeAdapter(Segment) +segment = segments.validate_json(geojson_text) ``` -### Utility Functions +### Validating without knowing the type -The package also exports several utility functions: +`overture-schema-validation` validates against the union of every installed +model, picking the right one from the data: ```python -from overture.schema import parse, discover_models, json_schema -from overture.schema import Building +from overture.schema.validation import validate, validate_json + +feature = validate(feature_row) # flat / tabular dict +feature = validate_json(geojson_text) # GeoJSON +``` -# Parse any Overture feature (auto-discovers all registered models) -validated_feature = parse(feature_data, mode="json") # Parses GeoJSON format -validated_feature = parse(feature_data, mode="python") # Parses flat format +### Discovering models programmatically + +Discovery lives in `overture-schema-system`. `discover_models()` returns a dict +keyed by `ModelKey` -- entry point `name`, its `entry_point` value, and the set +of tags attached during discovery: + +```python +from overture.schema.system.discovery import discover_models, get_registered_model -# Discover all registered models programmatically all_models = discover_models() -# Returns: # { -# ("buildings", "building"): BuildingModel, -# ("places", "place"): PlaceModel, -# ... +# ModelKey(name="building", entry_point="overture.schema.buildings:Building", +# tags=frozenset({"feature", "overture", "overture:theme=buildings"})): Building, +# ModelKey(name="place", entry_point="overture.schema.places:Place", +# tags=frozenset({"feature", "overture", "overture:theme=places"})): Place, +# ... # } -# Generate JSON Schema for models or unions +building_model = get_registered_model("building") # None if not installed +``` + +### Generating JSON Schema + +```python +from overture.schema.system.json_schema import json_schema + schema = json_schema(Building) -union_schema = json_schema(Building | Place) # Works with unions too +union_schema = json_schema(Building | Place) # emits an anyOf ``` + +See the [`overture-schema-system` README](../overture-schema-system/README.md) +for tag format, tag providers, and the discovery API in full. diff --git a/packages/overture-schema/changelog.d/668.docs.md b/packages/overture-schema/changelog.d/668.docs.md new file mode 100644 index 000000000..d2c7340f6 --- /dev/null +++ b/packages/overture-schema/changelog.d/668.docs.md @@ -0,0 +1 @@ +Rewrote the README against the real import surface: `overture.schema` is a namespace root, so models import from their theme packages and the utility functions from `overture.schema.validation` and `overture.schema.system`. diff --git a/pyproject.toml b/pyproject.toml index bbfe8e01e..b9d6d08bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ dev = [ "pytest>=9.0.0", "pytest-cov>=7.0.0", "pytest-testmon>=2.2.0", - "ruff>=0.13.0", + "ruff>=0.16.0", "towncrier>=25.8.0", ] diff --git a/uv.lock b/uv.lock index 5e0bcfd1a..b9e4e4837 100644 --- a/uv.lock +++ b/uv.lock @@ -954,7 +954,7 @@ requires-dist = [ dev = [ { name = "mypy", specifier = ">=1.17.0" }, { name = "pytest", specifier = ">=9.0.0" }, - { name = "ruff", specifier = ">=0.13.0" }, + { name = "ruff", specifier = ">=0.16.0" }, ] [[package]] @@ -1080,7 +1080,7 @@ requires-dist = [ dev = [ { name = "mypy", specifier = ">=1.17.0" }, { name = "pytest", specifier = ">=9.0.0" }, - { name = "ruff", specifier = ">=0.13.0" }, + { name = "ruff", specifier = ">=0.16.0" }, ] [[package]] @@ -1241,7 +1241,7 @@ dev = [ { name = "pytest", specifier = ">=9.0.0" }, { name = "pytest-cov", specifier = ">=7.0.0" }, { name = "pytest-testmon", specifier = ">=2.2.0" }, - { name = "ruff", specifier = ">=0.13.0" }, + { name = "ruff", specifier = ">=0.16.0" }, { name = "towncrier", specifier = ">=25.8.0" }, ] From ae0d1747777a25dd21293dc7a123370d856ec43b Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Wed, 19 Aug 2026 11:55:38 -0700 Subject: [PATCH 5/9] test: check the imports and enum members the docs name Documentation drifts silently: a module moves, a helper is renamed, and the README keeps confidently describing the old surface. #489, #604 and #668 are all instances. This makes the class fail the suite instead of accumulating. `tests/test_documented_imports.py` parses every fenced Python block in every tracked Markdown file and checks two things: that each `overture.*` import resolves, and that each `Enum.MEMBER` named exists on the enum. The second is not redundant -- `from ... import Relationship` resolves whether or not `Relationship.CONNECTS_TO` does. Executing the blocks outright is not viable: most fail on undefined names, because a documentation fragment legitimately omits its imports and uses placeholder names the prose supplies. Imports and enum members are the granularity that separates a defect from a fragment. Block extraction and import parsing are separate, unit-tested functions. The corpus cannot test either: a form the matcher misses simply yields nothing, and a sweep that collects less still passes -- so the fixtures are inline rather than drawn from repo content. Blocks indented inside list items are dedented before parsing, since an `IndentationError` is a `SyntaxError` and would otherwise be filed as "expected unparseable". Blocks that are deliberately not valid Python are pinned by a digest of their body rather than a count, which would stay put when one breaks as another is fixed; and an excused block may not contain an `overture` import, so the waiver cannot swallow the API citation it excuses. The tests live outside `packages/` because their subject is the repo's Markdown, including root-level files belonging to no package, and they import across every package -- `overture.schema.codegen` among them, which no single distribution depends on. `tests/README.md` records the admission rule. `make check` reaches the tree through ruff, `ruff format`, mypy and pytest; a bare `pytest packages/` does not. `lint-only` also checks Python embedded in Markdown, with `make format` as the counterpart. At the old ruff floor that check silently finds no files, and CI resolves one job at `lowest-direct`. Signed-off-by: Seth Fitzsimmons --- Makefile | 23 +- pyproject.toml | 2 +- tests/README.md | 23 ++ tests/test_documented_imports.py | 368 +++++++++++++++++++++++++++++++ 4 files changed, 408 insertions(+), 8 deletions(-) create mode 100644 tests/README.md create mode 100644 tests/test_documented_imports.py diff --git a/Makefile b/Makefile index 6a0d92a48..8a50ff6d1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: default uv-sync clean-pyspark generate-pyspark check check-namespace test-all test test-only docformat docformat-only doctest doctest-only mypy mypy-only lint-only update-baselines +.PHONY: default uv-sync clean-pyspark generate-pyspark check check-namespace test-all test test-only docformat docformat-only doctest doctest-only mypy mypy-only lint-only format update-baselines TESTMON ?= --testmon @@ -47,13 +47,13 @@ check-namespace: # the PySpark output first: that tree is no longer tracked in git, so the # generated conformance tests only exist once generation has run. test-all: uv-sync generate-pyspark - @uv run pytest -W error packages/ + @uv run pytest -W error packages/ tests/ test: uv-sync - @uv run pytest -W error $(TESTMON) packages/ -x -q --tb=short + @uv run pytest -W error $(TESTMON) packages/ tests/ -x -q --tb=short test-only: - @uv run pytest -W error $(TESTMON) packages/ -x -q --tb=short + @uv run pytest -W error $(TESTMON) packages/ tests/ -x -q --tb=short coverage: uv-sync @uv run pytest packages/ --cov overture.schema --cov-report=term --cov-report=html && open htmlcov/index.html @@ -90,11 +90,20 @@ mypy-only: | tr - . \ | sed 's|^packages/|-p |' \ | xargs uv run mypy --no-error-summary - @for d in packages/*/tests; do find "$$d" -name "*.py" | sort | xargs uv run mypy --no-error-summary || exit 1; done + @for d in packages/*/tests tests; do find "$$d" -name "*.py" | sort | xargs uv run mypy --no-error-summary || exit 1; done lint-only: - @uv run ruff check -q packages/ - @uv run ruff format --check packages/ + @uv run ruff check -q packages/ tests/ + @uv run ruff format --check packages/ tests/ + @# Python embedded in Markdown -- keeps documented snippets in the + @# same style as the code they describe. + @git ls-files -z '*.md' | xargs -0 --no-run-if-empty uv run ruff format -q --check + +# The fixing counterpart to lint-only: what to run when that gate fires. +format: + @uv run ruff check -q --fix packages/ tests/ + @uv run ruff format -q packages/ tests/ + @git ls-files -z '*.md' | xargs -0 --no-run-if-empty uv run ruff format -q update-baselines: @uv run pytest --update-baselines -m baseline -q packages/ diff --git a/pyproject.toml b/pyproject.toml index b9d6d08bf..b825787d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ select = [ [tool.mypy] disallow_untyped_defs = true explicit_package_bases = true -files = "packages/**/*.py" +files = ["packages/**/*.py", "tests/**/*.py"] # The pyspark test tree is PEP 420 and its tests import `_support` as a # top-level package (mirroring pytest's pythonpath). Put that tests dir on # the mypy path so `_support` resolves the same way for the type checker. diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..bfc4caf6a --- /dev/null +++ b/tests/README.md @@ -0,0 +1,23 @@ +# Workspace tests + +Checks whose subject is the repository or the workspace as a whole, and which +therefore cannot correctly live in any one package. + +A test belongs here only if both hold: + +- **Its subject spans packages, or sits outside them.** `test_documented_imports` + reads every tracked Markdown file, including root-level ones that ship in no + package, and imports across every package's namespace — `overture.schema.codegen` + among them, which no single distribution depends on. +- **It cannot run from one package's install.** Placing such a test under + `packages//tests/` would make that package's suite depend on packages its + `pyproject.toml` does not declare, and it would pass only because the workspace + happens to install everything. + +Anything that can be scoped to a package belongs in that package's `tests/` +instead. This directory is not a home for tests that are merely inconvenient to +place. + +`make check` covers this tree the same way it covers `packages/` — ruff, `ruff +format`, mypy, and pytest. Note that a bare `pytest packages/` does not; use +`make test` or include `tests/` explicitly. diff --git a/tests/test_documented_imports.py b/tests/test_documented_imports.py new file mode 100644 index 000000000..783e6af44 --- /dev/null +++ b/tests/test_documented_imports.py @@ -0,0 +1,368 @@ +"""Every `overture.*` import written in repo Markdown must resolve. + +Documentation drifts silently: a module moves, a helper is renamed, and the +README keeps confidently describing the old surface. This parses every fenced +Python block in every tracked Markdown file -- including blocks indented inside +a list item -- and imports each `overture.*` statement it finds. + +The subject is the repo's Markdown rather than any one package, so this lives +outside `packages/`. It reaches across the whole workspace -- a README may cite +`overture.schema.codegen`, which no single distribution depends on. +""" + +from __future__ import annotations + +import ast +import hashlib +import importlib +import re +import subprocess +import textwrap +from enum import Enum +from pathlib import Path + +import pytest + +# A fenced ```python block, capturing its body. The indent group matches a +# block nested in a list item; the body is dedented before parsing, since an +# `IndentationError` is a `SyntaxError` and would file the block under +# "expected unparseable" rather than reporting it. +_PYTHON_BLOCK = re.compile( + r"^(?P *)```python[^\n]*\n(?P.*?)^(?P=indent)```", + re.MULTILINE | re.DOTALL, +) + +# Blocks that are deliberately not valid Python: illustrations whose `...` +# placeholders stand in for elided content. Pinned by identity, not count -- a +# bare total stays put when one block breaks as another is fixed. The identity +# is a digest of the block body, so reordering a document does not trip it but +# editing one of these blocks does. +_EXPECTED_UNPARSEABLE = { + "PYDANTIC_GUIDE.md:43488ef4", +} + +# An import statement, matched textually -- used to police the excuse list, +# where by definition `ast` cannot be applied. +_OVERTURE_IMPORT = re.compile(r"^\s*(?:from|import)\s+overture\b", re.MULTILINE) + +# Golden files are generated fixtures, not documentation. +_EXCLUDED = "tests/golden/" + + +def _repo_root() -> Path: + """Locate the checkout root, or skip the module when there isn't one.""" + for candidate in Path(__file__).resolve().parents: + if (candidate / ".git").exists(): + return candidate + pytest.skip("not running from a git checkout", allow_module_level=True) + + +def _markdown_files() -> list[Path]: + root = _ROOT + try: + listed = subprocess.run( + ["git", "ls-files", "-z", "*.md"], + cwd=root, + capture_output=True, + text=True, + check=True, + ).stdout + except (OSError, subprocess.CalledProcessError) as exc: # pragma: no cover + pytest.skip(f"cannot list tracked files: {exc}", allow_module_level=True) + return [root / n for n in listed.split("\0") if n and _EXCLUDED not in n] + + +def python_blocks(text: str) -> list[str]: + """Yield the dedented body of every fenced Python block in `text`. + + Handles a block indented inside a list item. The dedent is load-bearing: + without it `ast.parse` raises `IndentationError`, a `SyntaxError`, and the + block would be filed as "expected unparseable" rather than reported. + """ + return [textwrap.dedent(match["body"]) for match in _PYTHON_BLOCK.finditer(text)] + + +def _documented_enum_references( + paths: list[Path], +) -> list[tuple[Path, str, str, str]]: + """Collect `(file, module, enum, member)` quads across the corpus.""" + found: list[tuple[Path, str, str, str]] = [] + for path in paths: + for body in python_blocks(path.read_text(encoding="utf-8")): + try: + refs = enum_references(body) + except SyntaxError: + continue + found += [(path, module, name, member) for module, name, member in refs] + return found + + +def parse_imports(source: str) -> list[tuple[str, str | None]]: + """Extract `overture.*` imports from one block of Python source. + + Returns `(module, attribute)` pairs; `attribute` is None for a plain + `import overture.x`. Raises `SyntaxError` if the block is not valid Python. + """ + found: list[tuple[str, str | None]] = [] + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.ImportFrom): + # A relative import (`from . import x`) has no absolute module. + if node.level or not node.module or not _is_overture(node.module): + continue + found += [ + (node.module, alias.name) for alias in node.names if alias.name != "*" + ] + elif isinstance(node, ast.Import): + found += [ + (alias.name, None) for alias in node.names if _is_overture(alias.name) + ] + return found + + +def _is_overture(module: str) -> bool: + return module == "overture" or module.startswith("overture.") + + +def enum_references(source: str) -> list[tuple[str, str, str]]: + """Find `Enum.MEMBER` uses of enums imported in the same block. + + Returns `(module, enum_name, member)`. Restricted to enums on purpose: + their members really are class attributes, whereas a Pydantic model's + fields are not, so `Place.addresses` would read as missing. + """ + tree = ast.parse(source) + imported = { + alias.asname or alias.name: node.module + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) + and node.module + and not node.level + and _is_overture(node.module) + for alias in node.names + } + return [ + (imported[node.value.id], node.value.id, node.attr) + for node in ast.walk(tree) + if isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id in imported + and node.attr.isupper() # a member, not a method + ] + + +def _documented_imports( + paths: list[Path], +) -> tuple[list[tuple[Path, str, str | None]], dict[str, str]]: + """Collect `(file, module, attribute)` triples, and identify unparseable blocks.""" + found: list[tuple[Path, str, str | None]] = [] + unparseable: dict[str, str] = {} + for path in paths: + for body in python_blocks(path.read_text(encoding="utf-8")): + try: + imports = parse_imports(body) + except SyntaxError: + digest = hashlib.sha256(body.encode()).hexdigest()[:8] + unparseable[f"{path.relative_to(_ROOT)}:{digest}"] = body + continue + found += [(path, module, attr) for module, attr in imports] + return found, unparseable + + +_ROOT = _repo_root() +_MARKDOWN = _markdown_files() +_DOCUMENTED, _UNPARSEABLE = _documented_imports(_MARKDOWN) +_ENUM_REFERENCES = _documented_enum_references(_MARKDOWN) + + +class TestBlocks: + """`python_blocks` against inline fixtures. + + Same argument as `TestParser` below, one layer up: a fence form the + matcher misses yields no block, and a sweep that collects less still + passes. The corpus cannot pin this. + """ + + def test_flush_block(self) -> None: + text = "text\n\n```python\nx = 1\n```\n\nmore\n" + assert python_blocks(text) == ["x = 1\n"] + + def test_list_item_block(self) -> None: + """A block indented under a list item, dedented to parse.""" + text = "1. Example:\n\n ```python\n x = 1\n ```\n" + assert python_blocks(text) == ["x = 1\n"] + + def test_deeply_indented_block(self) -> None: + text = "- a\n - b\n\n ```python\n x = 1\n ```\n" + assert python_blocks(text) == ["x = 1\n"] + + def test_indented_block_parses_only_after_dedent(self) -> None: + """Without the dedent this is an IndentationError, i.e. excused.""" + text = ( + "- item\n\n ```python\n from overture.schema.places import Place\n ```\n" + ) + (body,) = python_blocks(text) + assert parse_imports(body) == [("overture.schema.places", "Place")] + + def test_relative_indent_inside_a_block_is_preserved(self) -> None: + text = "- item\n\n ```python\n def f():\n return 1\n ```\n" + assert python_blocks(text) == ["def f():\n return 1\n"] + + def test_non_python_fences_are_ignored(self) -> None: + text = "```toml\nx = 1\n```\n\n```\nplain\n```\n" + assert python_blocks(text) == [] + + def test_several_blocks(self) -> None: + text = "```python\na = 1\n```\n\ntext\n\n```python\nb = 2\n```\n" + assert python_blocks(text) == ["a = 1\n", "b = 2\n"] + + +class TestOvertureImportMatcher: + """`_OVERTURE_IMPORT` against fixtures. + + It polices the excuse list, where `ast` cannot be applied by definition. + No excused block imports `overture` today, so the corpus never exercises + it -- the matcher would go blind without anyone noticing. + """ + + def test_matches_from_import(self) -> None: + assert _OVERTURE_IMPORT.search("x = 1\nfrom overture.schema import y\n") + + def test_matches_plain_import(self) -> None: + assert _OVERTURE_IMPORT.search(" import overture.schema.places\n") + + def test_ignores_other_packages(self) -> None: + assert not _OVERTURE_IMPORT.search("import json\nfrom pydantic import X\n") + + def test_ignores_a_longer_name(self) -> None: + assert not _OVERTURE_IMPORT.search("import overtures\n") + + def test_ignores_a_mention_in_prose(self) -> None: + assert not _OVERTURE_IMPORT.search("# import overture is what you'd write\n") + + +class TestParser: + """`parse_imports` against inline fixtures. + + The corpus cannot test the parser: any form it fails to handle simply + yields nothing, and a sweep that collects less still passes. + """ + + def test_single_line(self) -> None: + assert parse_imports("from overture.schema.buildings import Building") == [ + ("overture.schema.buildings", "Building") + ] + + def test_wrapped(self) -> None: + """The form `ruff format` produces once the names outgrow a line.""" + source = ( + "from overture.schema.system.numeric import (\n" + " int8, # signed\n" + " float64,\n" + ")\n" + ) + assert parse_imports(source) == [ + ("overture.schema.system.numeric", "int8"), + ("overture.schema.system.numeric", "float64"), + ] + + def test_alias(self) -> None: + assert parse_imports("from overture.schema.places import Place as P") == [ + ("overture.schema.places", "Place") + ] + + def test_plain_import_with_alias(self) -> None: + assert parse_imports("import overture.schema.buildings as b") == [ + ("overture.schema.buildings", None) + ] + + def test_star_and_relative_are_skipped(self) -> None: + assert parse_imports("from overture.schema.buildings import *") == [] + assert parse_imports("from . import buildings") == [] + + def test_non_overture_is_skipped(self) -> None: + assert parse_imports("import json\nfrom pydantic import BaseModel") == [] + + def test_indented_import(self) -> None: + source = "def f():\n from overture.schema.places import Place\n" + assert parse_imports(source) == [("overture.schema.places", "Place")] + + def test_invalid_source_raises(self) -> None: + with pytest.raises(SyntaxError): + parse_imports('{"a": 1, ...}\nthis is not python') + + +def test_corpus_is_non_empty() -> None: + """Guard against a silently empty sweep reporting success. + + The thresholds only have to be low enough to survive ordinary doc churn + and high enough that an empty or near-empty parse fails loudly. + """ + assert len(_MARKDOWN) > 10 + assert len(_DOCUMENTED) > 20 + + +def test_unparseable_blocks_are_the_expected_ones() -> None: + """A block that stops parsing drops out of the sweep silently.""" + assert set(_UNPARSEABLE) == _EXPECTED_UNPARSEABLE + + +def test_no_excused_block_hides_an_import() -> None: + """An excused block may illustrate, but may not cite the API. + + A content-addressed waiver is durable, so a block that both fails to + parse and imports `overture.*` would be exempt from the sweep forever -- + the one outcome this module exists to prevent. Making the block parse is + always available; every case so far took a one-token edit. + """ + for name, body in _UNPARSEABLE.items(): + assert not _OVERTURE_IMPORT.search(body), ( + f"{name} is excused from parsing but imports overture.*; " + "make the block parse instead of excusing it" + ) + + +@pytest.mark.parametrize( + ("path", "module", "enum_name", "member"), + _ENUM_REFERENCES, + ids=[ + f"{path.relative_to(_ROOT)}:{name}.{member}" + for path, _, name, member in _ENUM_REFERENCES + ], +) +def test_documented_enum_member_exists( + path: Path, module: str, enum_name: str, member: str +) -> None: + """An enum member named in the docs exists on the enum. + + The import sweep cannot see this: `from ... import Relationship` + resolves whether or not `Relationship.CONNECTS_TO` does. + """ + enum_class = getattr(importlib.import_module(module), enum_name) + if not (isinstance(enum_class, type) and issubclass(enum_class, Enum)): + return + names = [m.name for m in enum_class] + assert member in names, ( + f"{path}: `{enum_name}` has no member `{member}`; it has {names}" + ) + + +@pytest.mark.parametrize( + ("path", "module", "attribute"), + _DOCUMENTED, + ids=[ + f"{path.relative_to(_ROOT)}:{module}" + (f".{attr}" if attr else "") + for path, module, attr in _DOCUMENTED + ], +) +def test_documented_import_resolves( + path: Path, module: str, attribute: str | None +) -> None: + """An import written in the docs imports cleanly.""" + imported = importlib.import_module(module) + if attribute is None or hasattr(imported, attribute): + return + # `from pkg import submodule` binds only once the submodule is imported. + try: + importlib.import_module(f"{module}.{attribute}") + except ImportError as exc: + pytest.fail(f"{path}: `{module}` has no attribute `{attribute}` ({exc})") From 1fc03dbbe94cdd7060d2c5c13e66a2337bcceeff Mon Sep 17 00:00:00 2001 From: Dana Bauer Date: Wed, 19 Aug 2026 09:58:39 -0400 Subject: [PATCH 6/9] update revised and consolidated docs Signed-off-by: Dana Bauer Signed-off-by: Seth Fitzsimmons --- CONTRIBUTING.md | 7 + GLOSSARY.md | 169 +- PYDANTIC_GUIDE.md | 1341 ------------- README.md | 16 + README.pydantic.md | 240 --- SCHEMA_GUIDE.md | 4684 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 4867 insertions(+), 1590 deletions(-) delete mode 100644 PYDANTIC_GUIDE.md delete mode 100644 README.pydantic.md create mode 100644 SCHEMA_GUIDE.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cf31d7a4a..a4579e9df 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,6 +6,13 @@ Thank you for your interest in contributing. > [DevOps tracking issue #490](https://github.com/OvertureMaps/schema/issues/490) > for current status and what is planned next. +## Working with the Python packages + +The schema is authored as Pydantic models. [SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) covers the +whole toolchain: Part I for installing and using the packages, Part II for authoring new +schema models and the development workflow (`uv sync`, `make check`, ruff and +docformatter). + ## Where to send your change This repository uses a two-branch model. Target the branch that matches your diff --git a/GLOSSARY.md b/GLOSSARY.md index 5778501dd..739d2c0f9 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -1,18 +1,169 @@ -# Entity -An entity is a thing in the physical world. It can relate both to physical objects or more abstract concepts that have a spatial presence (e.g. administrative areas are not physical objects as such, but they have physical properties that define their location and extend). An entity can only exist once. +# Glossary -# Feature -A Feature is an abstraction of a specific entity in the map. It exists only in its digital form. +Two vocabularies meet in this repository. The first describes the map: what an entity is, +what a feature is, how feature types are named. The second describes the Python packages +that define the schema: workspaces, entry points, specs. Both are collected here. + +Terms in the second group are cross-referenced to the section of +[SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) that covers them in full. + +--- + +## Data model + +### Entity + +An entity is a thing in the physical world. It can relate both to physical objects or more +abstract concepts that have a spatial presence (e.g. administrative areas are not physical +objects as such, but they have physical properties that define their location and extent). +An entity can only exist once. + +### Feature + +A Feature is an abstraction of a specific entity in the map. It exists only in its digital +form. + +### Feature Class -# Feature Class Feature Class is a synonym for "Feature Type". The preferred term is "Feature Type". -# Feature Type -A Feature Type is a type of entities with common properties as described in the Overture Schema. +### Feature Type + +A Feature Type is a type of entities with common properties as described in the Overture +Schema. + +### Instance -# Instance Instance is a synonym for "Feature". The preferred term is "Feature". -# Object +### Object + Object is a synonym for "Instance" or "Feature". The preferred term is "Feature". +### Theme + +The top-level grouping a feature type belongs to, and a mandatory property on every +feature. The term is deliberately chosen over "layer" to avoid that word's baggage. There +are six: `addresses`, `base`, `buildings`, `divisions`, `places`, `transportation`. Each +ships as its own Python package, `overture-schema-theme-*`. + +### Type + +The feature type within a theme, and a mandatory property on every feature — for example +`theme=buildings`, `type=building`. Together `theme` and `type` identify the feature type. + +### Subtype + +An optional third property that further refines the feature type — for example +`theme=transportation`, `type=segment`, `subtype=road`. Where a type has subtypes, the +model for that type is usually a [discriminated union](#discriminated-union) with one arm +per subtype. Note the spelling: the field is `subtype`, not `subType`. + +### GERS + +The Global Entity Reference System. A feature's `id` may be a GERS ID if — and only if — +the feature represents an entity that is part of GERS. + +--- + +## Toolchain + +### Alias + +Three unrelated things in this codebase go by this name. Which one is meant is almost +always clear from context, but they are worth separating: + +1. **Field alias** — a mapping from a Python attribute name to the name the data uses, + declared with `Field(alias=...)`. It exists because some data field names are not legal + Python identifiers: `Building.class_` carries `alias="class"`, since `class` is a + reserved word. Dump with `by_alias=True` to get the data name back. +2. **Type alias** — a module-level name bound to a type expression rather than to a class. + `Segment` is one: it is an `Annotated[Union[...], Discriminator(...)]`, *not* a class. + This is why `Segment.model_validate(...)` raises `AttributeError` and you need + `TypeAdapter(Segment).validate_python(...)` instead. See + [Working with Segment and other unions](SCHEMA_GUIDE.md#35-working-with-segment-and-other-unions). +3. **Entry-point name** — the short name a model registers under, which need not match the + class name. `building = "overture.schema.buildings:Building"` registers the class + `Building` under the name `building`. + +### Entry point + +The mechanism by which a package advertises something to the rest of the installed +environment, declared in `pyproject.toml`. Nothing imports these directly; they are +discovered at runtime. The schema uses four groups: `overture.models` (feature types), +`overture.tag_providers` ([tags](#tag)), `project.scripts` (the CLIs), and `pytest11` +(test plugins). Registering your own model is a matter of adding an entry point — see +[Register your own feature types](SCHEMA_GUIDE.md#83-register-your-own-feature-types). + +### Workspace + +One repository containing several independently-versioned packages that share a single +lockfile and a single virtual environment. Declared by `[tool.uv.workspace]` in the root +`pyproject.toml`. The schema repo is a `uv` workspace of thirteen packages. See +[What "workspace" means](SCHEMA_GUIDE.md#what-workspace-means). + +### Metapackage + +A package that ships no code of its own and exists only to depend on others. +`overture-schema` is one: installing it pulls in every theme. Its namespace root contains +nothing but a `py.typed` marker, which is why `from overture.schema import Building` +fails. + +### Flat shape + +The form a feature takes as a single record with no nesting of core properties — `id`, +`geometry`, `theme`, `type` and the rest all sitting at the same level. This is the shape +the Pydantic models use, and the shape Overture publishes in Parquet. +`model_validate()` expects it. + +### Envelope + +The GeoJSON form of a feature, where most properties are tucked under a `properties` key +alongside a top-level `type: "Feature"`. Distinct from the [flat shape](#flat-shape), and +the single most common source of confusion when validating: `model_validate()` rejects it, +`model_validate_json()` accepts it. See +[Two data shapes](SCHEMA_GUIDE.md#32-the-one-thing-to-understand-two-data-shapes). + +### Discriminated union + +A union of models where one field's value decides which arm applies. `Segment` is +discriminated on `subtype`: `road` selects `RoadSegment`, `rail` selects `RailSegment`, +`water` selects `WaterSegment`. Pydantic uses the discriminator to pick an arm without +trying each in turn, which also makes validation errors point at the right model. + +### NewType + +A distinct type wrapping an existing one, used to give a plain value a name and a set of +constraints — `Id`, `CountryCodeAlpha2`, `LanguageTag`. At runtime the value is still a +`str`; the wrapper carries the validation rules and survives into the generated artifacts, +which is why it is preferred over a bare `str` with a `Field` constraint. + +### ModelKey + +The key type returned by `discover_models()`. Carries the model's entry-point `name`, its +`entry_point` string, and its [tags](#tag). Not a plain string, and not a tuple — code +that assumes either will break. + +### Tag + +A string attached to a model during discovery, classifying it orthogonally to its name — +`feature`, or `overture:theme=buildings`. Tags are what the CLI's `--tag` / `--filter` / +`--exclude` options select on. They are produced by *tag providers* registered on the +`overture.tag_providers` entry-point group; third parties can register their own. Seven +tags exist today: `feature` plus one `overture:theme=*` per theme. There is no plain +`overture` tag, despite what some help text suggests. + +### Spec + +The target-independent description of a model produced by the codegen's extraction layer, +before any output format is chosen — `RecordSpec` for a model, `UnionSpec` for a +discriminated union, `FieldSpec` for a field, `EnumSpec` for an enum. Renderers consume +specs; they never touch Pydantic models directly. This split is what lets a new output +format be a new renderer rather than new extraction logic. See +[Write a new codegen target](SCHEMA_GUIDE.md#84-write-a-new-codegen-target). + +### Extra + +An optional dependency group declared under `[project.optional-dependencies]` and +installed with bracket syntax (`package[extra]`) or `uv sync --all-extras`. Distinct from +"extra fields", which is what `no_extra_fields` forbids on a model. diff --git a/PYDANTIC_GUIDE.md b/PYDANTIC_GUIDE.md deleted file mode 100644 index 3e61c3191..000000000 --- a/PYDANTIC_GUIDE.md +++ /dev/null @@ -1,1341 +0,0 @@ -# Overture Maps Pydantic Schema Guide - -This guide helps you work with Overture Maps Pydantic schemas - Python models that define geospatial data structures with automatic validation. Whether you're new to Pydantic or migrating from JSON Schema, this guide provides a progressive learning path from basics to advanced patterns. - -## Table of Contents - -- [Quick Start](#quick-start) -- [Basic Concepts](#basic-concepts) - - [Models and Inheritance](#models-and-inheritance) - - [Field Types](#field-types) - - [Field Enhancement](#field-enhancement) - - [Collections and Lists](#collections-and-lists) - - [Enumerations](#enumerations) -- [Advanced Patterns](#advanced-patterns) - - [Relationship Patterns](#relationship-patterns) - - [Discriminated Unions](#discriminated-unions) - - [Pattern Properties (Constrained Key-Value Maps)](#pattern-properties-constrained-key-value-maps) - - [Nested List Validation](#nested-list-validation) - - [Type Aliases for Reusable Patterns](#type-aliases-for-reusable-patterns) -- [Integration Guide](#integration-guide) - - [Project Architecture](#project-architecture) - - [Migrating from JSON Schema](#migrating-from-json-schema) -- [Reference](#reference) - - [Quick Reference](#quick-reference) - ---- - -## Quick Start - -### Essential Imports - -Copy what you need for most models: - -```python -# Basic Python types -from typing import Annotated, Literal -from enum import Enum - -# Pydantic essentials -from pydantic import BaseModel, Field - -# Overture common models -from overture.schema.common import OvertureFeature -from overture.schema.system.geometric import ( - Geometry, - GeometryType, - GeometryTypeConstraint, -) - -# Validation system -from overture.schema.system.field_constraint import UniqueItemsConstraint -from overture.schema.system.model_constraint import no_extra_fields - -# Common types -from overture.schema.system.string import ( - CountryCodeAlpha2, - NoWhitespaceString, - StrippedString, -) -from overture.schema.common.confidence import ConfidenceScore -from overture.schema.system.string import LanguageTag - -# Numeric types (use these instead of int/float) -from overture.schema.system.numeric import ( - int8, - int32, - int64, - uint8, - uint16, - uint32, - float32, - float64, -) -``` - -### Basic Model Template - -```python -from typing import Annotated -from pydantic import BaseModel, Field -from overture.schema.system.model_constraint import no_extra_fields -from overture.schema.system.numeric import int8, float64 - - -@no_extra_fields -class MyCustomType(BaseModel): - """Brief description of what this represents.""" - - # Required fields (no default value) - name: str - category: str - - # Optional fields (with None default) - description: str | None = None - - # Field with constraints and description - priority: Annotated[ - int8 | None, - Field( - ge=1, le=10, description="Priority level from 1 (lowest) to 10 (highest)" - ), - ] = None -``` - -### Feature Template - -```python -from typing import Annotated, Literal -from pydantic import Field -from overture.schema.common import OvertureFeature -from overture.schema.system.geometric import ( - Geometry, - GeometryType, - GeometryTypeConstraint, -) - - -class MyFeature(OvertureFeature[Literal["my_theme"], Literal["my_type"]]): - """Description of what this feature represents.""" - - # Geometry with constraints - geometry: Annotated[ - Geometry, - GeometryTypeConstraint(GeometryType.POINT), - Field(description="Location of this feature"), - ] - - # Custom fields - my_field: str | None = None -``` - ---- - -## Basic Concepts - -### Models and Inheritance - -#### What are Pydantic models? - -Pydantic models are Python classes that define data structures and their constraints. Think of them like UML classes with built-in data validation - each model defines what fields are allowed and what types of data they can contain. - -#### Model Base Classes and Inheritance - -**What is a "base class"?** A base class defines common fields and behaviors that other classes can reuse. Think of it like a slide template - you create one layout, then make specific slides that use that structure. - -**What is "inheritance"?** Inheritance means one class automatically gets all the fields and behaviors from another class. If Building inherits from OvertureFeature, it automatically gets all of Feature's fields (like `id`, `geometry`) plus any new fields you add to Building (like `height`). When multiple parent classes have the same field name, Python uses a [specific order](https://docs.python.org/3/tutorial/classes.html#multiple-inheritance) to determine which one takes precedence. - -**@no_extra_fields** - Use for structured data components that should reject unknown fields: - -```python -from overture.schema.system.model_constraint import no_extra_fields - - -@no_extra_fields -class Address(BaseModel): - """A postal address - no extra fields allowed.""" - - street: str - city: str - postal_code: str | None = None - # Any field not defined here will cause validation to fail -``` - -**OvertureFeature[ThemeT, TypeT]** - A generic base class for all geospatial features with typed theme and type parameters: - -```python -from typing import Literal -from overture.schema.common import OvertureFeature -from overture.schema.system.numeric import float64 - - -class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): - """A building feature with strongly-typed theme and type.""" - - # Inherits: id, theme, type, geometry, bbox, version, sources - height: float64 | None = None -``` - -**What does "generic" mean?** The `OvertureFeature[ThemeT, TypeT]` syntax makes OvertureFeature a "generic" class - think of it like a template that can be customized with specific values. The square brackets `[]` contain "type parameters" that specify exactly what theme and type this feature represents. - -**What are ThemeT and TypeT?** These are placeholders for specific text values: - -- **ThemeT**: The data theme (like "buildings", "places", "transportation") -- **TypeT**: The specific feature type within that theme (like "building", "place", "segment") - -**What is `Literal`?** `Literal` means the field must be exactly one of the specified values - nothing else is allowed. So `Literal["buildings"]` means this theme can only be "buildings", not any other string. - -By specifying `OvertureFeature[Literal["buildings"], Literal["building"]]`, you're saying "this is a Feature that must have theme='buildings' and type='building'" - no other values are allowed. This prevents mistakes like accidentally creating a building with theme="places". - -#### Inheritance Patterns - -**Multiple inheritance** combines fields from several base classes: - -```python -from typing import Literal -from overture.schema.common import OvertureFeature -from overture.schema.common.level import Stacked -from overture.schema.common.names import Named -from overture.schema.system.numeric import float64 - - -class Building( - OvertureFeature[Literal["buildings"], Literal["building"]], Named, Stacked -): - # Gets fields from Feature: id, theme, type, geometry, etc. - # Gets fields from Named: names - # Gets fields from Stacked: level - # Plus its own fields: - height: float64 | None = None -``` - -#### Field Aliases - -Sometimes you need a field name that conflicts with Python keywords or conventions (hint: you'll get an error when you try to use it). Use `Field(alias="")` to map between Python-friendly field names and the actual data field names: - -```python -from typing import Annotated -from pydantic import Field - - -class Building(OvertureFeature): - # Use class_ in Python code, but "class" in the actual data - class_: Annotated[str | None, Field(alias="class")] = None - - # Other common cases might include: - type_: Annotated[str | None, Field(alias="type")] = None # if type conflicts - from_: Annotated[str | None, Field(alias="from")] = None # from is a keyword -``` - -A common example is `class_` with `Field(alias="class")` since "class" is a Python keyword but a common field name in data schemas. - -### Field Types - -#### Required vs Optional Fields - -```python -class Building(OvertureFeature): - # Required field (no default value) - geometry: Geometry - - # Optional field (has default value of None) - height: float64 | None = None -``` - -- **Required fields**: Must be provided when creating an instance -- **Optional fields**: Can be omitted; they have a default value (usually `None`) - -> [!WARNING] -> **Always use `None` defaults** for optional fields. Non-`None` defaults create ambiguity between schema defaults and actual data values. - -**Why do non-`None` defaults cause problems?** - -1. **Data transformation ambiguity**: Pydantic adds default values that weren't in the input, making it impossible to distinguish between original data and schema defaults. - -2. **Schema vs. data confusion**: Schemas serve multiple purposes: - - **Validation only**: Check if existing data is valid (shouldn't transform it) - - **Data processing**: Parse and potentially transform data with Pydantic - - **Documentation**: Show developers what fields exist and what they mean - -3. **Implicit semantic meaning**: Default values encode business logic into the schema, which should be in business logic instead. - -**Better approaches:** - -1. **Use `None` and document semantics:** - - ```python - access_policy: Annotated[ - str | None, - Field(description="Access policy for the place. When absent, assume 'open'"), - ] = None - ``` - -2. **Always populate in data pipeline:** - - ```python - # In your data processing pipeline, always set the value - place.access_policy = place.access_policy or "open" - ``` - -Keep the schema separate from business logic. The schema describes the shape of data, not the business rules about what missing values mean. - -#### Numeric Types - -**Always use specific numeric types instead of Python's generic `int`/`float`:** - -```python -from overture.schema.system.model_constraint import no_extra_fields -from overture.schema.system.numeric import ( - int8, - int32, - int64, # Signed integers - uint8, - uint16, - uint32, # Unsigned integers - float32, - float64, # Floating point -) - - -@no_extra_fields -class MyModel(BaseModel): - # Signed integers with specific ranges - level: int8 | None = None # -128 to 127 - year: int32 | None = None # -2,147,483,648 to 2,147,483,647 - timestamp: int64 | None = None # Full 64-bit range - - # Unsigned integers (0 and positive only) - red_value: uint8 | None = None # 0 to 255 (like RGB values) - port: uint16 | None = None # 0 to 65,535 (like network ports) - population: uint32 | None = None # 0 to 4,294,967,295 - - # Floating point numbers - height: float64 | None = None # Double precision (recommended) - ratio: float32 | None = None # Single precision -``` - -**When to use each:** - -- **`int32`**: Most integer fields (years, counts, IDs) -- **`uint8`**: Small positive values (0-255), like color components, confidence percentages -- **`uint16`**: Medium positive values (0-65K), like ports, small counts -- **`uint32`**: Large positive values, like population, large IDs -- **`float64`**: Most decimal numbers (heights, coordinates, measurements) - **this is the default choice** -- **`float32`**: When space is critical and precision isn't - -When in doubt, use `int32` (equivalent to `int`), `int64` (equivalent to `long`, although [not safely representable in JSON](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER)), or `float64` (equivalent to `double`). - -**Why specific numeric types matter:** - -The specific numeric types are crucial for data interchange and storage compatibility: - -- **Cross-platform consistency**: Ensures the same data types across Python, Arrow, Parquet, and other geospatial tools -- **Round-trip compatibility**: Data round-trips cleanly between Parquet files, databases (PostgreSQL, Trino), Shapefiles, and JSON Schema -- **Value range validation**: Prevents invalid values (e.g., negative heights, RGB values > 255) -- **Storage efficiency**: `uint8` uses 1 byte vs `int64` which uses 8 bytes -- **Built-in validation**: These types use Pydantic `Field()` constraints to validate ranges (e.g., `Field(ge=0, le=100)` ensures values stay within bounds) - -#### Union Types - -Union types allow a field to accept multiple different types. The `|` symbol means "or": - -```python -from typing import Literal - - -class Building(OvertureFeature): - # This field can be either a string OR None (most common union) - name: str | None = None - - # This field can be one of specific string values OR None - status: Literal["active", "inactive", "pending"] | None = None -``` - -**Common union patterns:** - -```python -# Optional field (most common union) -height: float64 | None = None # Can be a number or missing - -# Specific string values (an alternative to enums where descriptions aren't needed) -priority: Literal["low", "medium", "high"] | None = None - -# Boolean or None -is_verified: bool | None = None -``` - -**Union best practices:** - -- Keep unions simple - avoid more than 2-3 types when possible -- Optional fields will include `None` to become optional -- Use `Literal` values for specific string choices rather than mixing basic types -- **Avoid mixed-type unions** like `str | int32` - these don't work well with many storage layers - -> [!WARNING] -> **Storage compatibility**: Mixed-type unions (combining different basic types like `str | int32`) don't work with Parquet and other storage layers. Use `Literal` values or separate fields instead. - -### Field Enhancement - -#### Adding Descriptions and Constraints with Annotated - -`Annotated` is Python's way to add extra information (metadata) to a type without changing the type itself. Think of it like adding notes or constraints to a field definition. - -**Basic concept:** - -```python -from typing import Annotated -from pydantic import Field - -# Without Annotated - just the type -height: float64 | None = None - -# With Annotated - type + extra information -height: Annotated[ - float64 | None, # The actual type (what kind of data) - Field(description="Height in meters"), # Extra metadata -] = None -``` - -**What goes inside `Annotated`:** - -1. **First argument**: The actual type (`str`, `int32`, `list[str]`, etc.) -2. **Additional arguments**: Metadata like constraints, descriptions, validation rules - -#### Field Constraints - -Use Pydantic's `Field()` function to add constraints and descriptions: - -**Numeric constraints:** - -```python -from typing import Annotated -from pydantic import Field - - -class Building(OvertureFeature): - # Range constraints - height: Annotated[ - float64 | None, Field(ge=0, le=1000, description="Height in meters (0-1000m)") - ] = None - - # Integer constraints - floors: Annotated[ - int32 | None, Field(gt=0, lt=200, description="Number of floors (1-199)") - ] = None -``` - -**Numeric constraint options:** - -- **`ge`**: Greater than or equal to (≥) -- **`gt`**: Greater than (>) -- **`le`**: Less than or equal to (≤) -- **`lt`**: Less than (<) - -**String constraints:** - -```python -class Place(OvertureFeature): - # Length constraints - name: Annotated[ - str | None, - Field(min_length=1, max_length=100, description="Place name (1-100 chars)"), - ] = None - - # Pattern matching - postal_code: Annotated[ - str | None, Field(pattern=r"^\d{5}(-\d{4})?$", description="US postal code") - ] = None -``` - -**String constraint options:** - -- **`min_length`**: Minimum string length -- **`max_length`**: Maximum string length -- **`pattern`**: Regular expression pattern (regex) - -### Collections and Lists - -#### Basic List Fields - -```python -class Building(Feature): - # Simple list of strings - tags: list[str] | None = None - - # List of complex objects - access_rules: list[AccessRule] | None = None -``` - -#### List Constraints - -```python -from overture.schema.system.field_constraint import UniqueItemsConstraint - - -class Building(OvertureFeature): - # List with size and uniqueness constraints - categories: Annotated[ - list[str] | None, - Field(min_length=1, max_length=10, description="1-10 categories"), - UniqueItemsConstraint(), # Must come AFTER Field() - ] = None -``` - -**List constraint options:** - -- **`min_length`**: Minimum number of items -- **`max_length`**: Maximum number of items -- **`UniqueItemsConstraint()`**: No duplicate items (custom validation) - -**Important**: `UniqueItemsConstraint()` must come AFTER `Field()` for proper JSON Schema generation. - -> [!CAUTION] -> **Constraint order matters**: Always put `Field()` before `UniqueItemsConstraint()` or JSON Schema generation will create `minLength` (string constraint) instead of `minItems` (array constraint). -> -> **Why**: Pydantic processes annotations in order for JSON Schema generation. `Field()` must come first to set up the field properly. For lists, `Field(min_length=1)` creates a `minItems` constraint in the JSON Schema because the type immediately before it is a list. If `UniqueItemsConstraint()` comes first, Pydantic doesn't see the list type and treats `min_length` as a string constraint (`minLength`). - -#### List Behavior - -Lists maintain their **insertion order** (the order data exists in the field), but they are **not automatically sorted**. - -### Enumerations - -**What is an enumeration (enum)?** An enumeration is a way to define a fixed set of allowed values for a field. Think of it like a multiple-choice question - you define all the valid answers ahead of time, and users can only pick from those options. - -For example, instead of allowing any string for a "status" field (which could lead to typos like "activ" or "Active"), you create an enum with exactly "active", "inactive", and "pending" as the only allowed values. - -**Enums vs Literal:** You can achieve similar results with `Literal["active", "inactive", "pending"]`, but formal enums are better when you need descriptions, documentation, or want to reuse the same set of values across multiple fields. - -#### Creating Enums - -Enums define a fixed set of allowed values: - -```python -from enum import Enum - - -class BuildingClass(str, Enum): - """Further delineation of the building's built purpose.""" - - RESIDENTIAL = "residential" - COMMERCIAL = "commercial" - INDUSTRIAL = "industrial" - CIVIC = "civic" - - -# Usage in a model -class Building(OvertureFeature): - class_: Annotated[BuildingClass | None, Field(alias="class")] = None -``` - -#### Documenting Enum Values - -Add documentation to describe what the enum and its values mean. In Python, you do this with **docstrings** - text enclosed in triple quotes `"""` that describes what something does: - -Use `DocumentedEnum` from `overture.schema.system.doc` when enum members need their own descriptions for code generation and documentation tooling. Each member takes a `(value, description)` tuple: - -```python -from overture.schema.system.doc import DocumentedEnum - - -class VehicleType(str, DocumentedEnum): - """Types of vehicles for transportation.""" - - CAR = ("car", "Standard passenger vehicle") - TRUCK = ("truck", "Commercial freight vehicle") - BICYCLE = ("bicycle", "Human-powered two-wheeler") - MOTORCYCLE = ("motorcycle", "Motorized two-wheeler") -``` - -Members without descriptions use the plain value form -- documentation is optional per-member: - -```python -class ConnectionState(str, DocumentedEnum): - CONNECTED = "connected" - DISCONNECTED = "disconnected" - QUIESCING = ( - "quiescing", - "Gracefully shutting down, rejecting new requests but completing existing ones", - ) -``` - -Use `DocumentedEnum` over plain `str, Enum` when the enum members' semantics aren't obvious from their names and downstream tools (code generators, documentation renderers) need access to member-level descriptions. Use plain `str, Enum` for self-explanatory values. - -#### Why str, Enum? - -Inheriting from `str, Enum` makes enum values work as both enums and strings, which is useful for JSON serialization and compatibility. - ---- - -## Advanced Patterns - -### Relationship Patterns - -#### What are relationships? - -Relationships represent connections between different features or models. Think of them like links that connect related pieces of information — for example, a building part that is structurally part of a building, or a division area that is administratively nested under a division. - -Pydantic provides several ways to express these relationships, each suited to different use cases and complexity levels. Before choosing a pattern, it's important to understand the **semantic type** of the relationship you're modeling. - ---- - -#### Semantic Relationship Types - -Every relationship between two features carries a semantic meaning about coupling strength, lifecycle dependency, and ownership. The schema defines four relationship types, ordered from strongest to weakest coupling. The types describe the *nature* of the link, not which feature is "parent" or "child." Direction is implicit: the feature holding the reference is the source, and the type it references is the destination. - -##### `COMPOSITION` — Structural Whole-Part - -A structural whole-part relationship with lifecycle dependency. The part has no independent meaning outside the whole. Deleting the whole invalidates the part. - -**Test question:** *"If I delete the whole, does keeping the part orphaned make any sense at all?"* If the answer is no, it's `COMPOSITION`. - -**Examples:** -- `BuildingPart` → `Building` — part *is part of* building -- `DivisionBoundary` → `Division` — boundary line *defines the boundary of* division - -##### `AGGREGATION` — Grouping Without Lifecycle Dependency - -A grouping or collection relationship where both members are independently viable. No lifecycle dependency — the member survives reassignment to another group or orphaning. - -**Test question:** *"Can both sides belong to something else or nothing and still be a valid map feature?"* If yes, they form an `AGGREGATION`. - -**Examples:** -- `Route` → `Segment` — route *groups* segments -- `TrailSegment` → `NationalPark` — segment *is grouped by* park - -##### `HIERARCHY` — Organizational Nesting - -An organizational or classificatory nesting relationship. This is not about structural assembly — it's about administrative parentage, taxonomy, or categorization. - -**Test question:** *"Is this about organizational subordination rather than structural assembly?"* If yes, it's `HIERARCHY`. - -**Examples:** -- `DivisionArea` → `Division` — area *is child of* division -- `Division` → `Division` — child division nested under parent - -##### `ASSOCIATION` — Peer-Level Reference - -A peer-level reference with no ownership, containment, or nesting. Neither feature depends on or contains the other. This is the fallback when none of the stronger types apply. - -**Test question:** *"Are these just peers that know about each other?"* If yes, it's `ASSOCIATION`. - -**Examples:** -- `Segment` → `Connector` — segment references its start/end connector -- `Building` → `Address` — a building references its address, neither owns the other - ---- - -#### Selection Priority - -When a relationship could fit multiple types, the choice follows a **diamond decision**: start at the top, fork in the middle based on the *kind* of coupling, and fall through to the bottom only when no stronger type applies. - -```text - COMPOSITION - / \ -AGGREGATION HIERARCHY - \ / - ASSOCIATION -``` - -| If the relationship implies... | Use | -|---------------------------------------------------------|----------------| -| Structural whole-part with lifecycle dependency | `COMPOSITION` | -| Geometric boundary definition (lifecycle dependent) | `COMPOSITION` | -| Grouping/collection without lifecycle dependency | `AGGREGATION` | -| Organizational nesting or classification tree | `HIERARCHY` | -| Peer-level reference, no ownership or nesting | `ASSOCIATION` | - ---- - -#### The `role` Field - -The `Reference` annotation accepts an optional `role` parameter — a snake_case string that further qualifies the relationship from the source's perspective. It has no effect on schema validation; it is informational metadata for documentation and tooling. - -Use `role` when the semantic type alone is ambiguous. For example, multiple `HIERARCHY` references on the same model can be disambiguated: - -```python -# Without role: two HIERARCHY references to Division — which is which? -parent_division_id: Annotated[Id, Reference(Relationship.HIERARCHY, Division)] -capital_division_ids: Annotated[list[Id], Reference(Relationship.HIERARCHY, Division)] - -# With role: unambiguous -parent_division_id: Annotated[ - Id, Reference(Relationship.HIERARCHY, Division, role="child_of") -] -capital_division_ids: Annotated[ - list[Id], Reference(Relationship.HIERARCHY, Division, role="has_as_capital") -] -``` - -The `role` must be a non-empty snake_case string (lowercase letters, digits, underscores). It describes the source's role relative to the target using source-perspective phrasing. - ---- - -#### Implementation Patterns - -##### 1. Direct References (Foreign Keys) - -The fundamental pattern is a direct reference where one feature "points to" another using an ID field with type safety and semantic information. - -```python -from typing import Annotated, Literal -from pydantic import Field -from overture.schema.common import OvertureFeature -from overture.schema.system.ref import Id, Reference, Relationship - - -# COMPOSITION — part points to its whole -class BuildingPart(OvertureFeature[Literal["buildings"], Literal["building_part"]]): - """A structural part of a building.""" - - building_id: Annotated[ - Id, - Reference(Relationship.COMPOSITION, Building, role="part_of"), - Field(description="The building to which this part belongs"), - ] - - -# HIERARCHY — child points to parent -class DivisionArea(OvertureFeature[Literal["divisions"], Literal["division_area"]]): - """Area polygon nested under a division.""" - - division_id: Annotated[ - Id, - Reference(Relationship.HIERARCHY, Division, role="child_of"), - Field(description="Division ID of the parent division of this area."), - ] - - -# ASSOCIATION — peer reference, no ownership -class ConnectorReference(BaseModel): - """Reference to a connector feature.""" - - connector_id: Annotated[ - Id, Reference(Relationship.ASSOCIATION, Connector, role="connects_to") - ] -``` - -##### 2. Association as a Separate Feature (Complex Relationships) - -When the relationship itself needs to store information, create a dedicated feature to represent it. This applies regardless of the semantic type — any of the four types can carry metadata. - -**Simple relationship (use Pattern 1):** -- "Building Part A is part of Building B" — just needs an ID reference. - -**Complex relationship (use Pattern 2):** -- "Admin Area X has City Center Y as its primary center since 2010 with 85% confidence" — the relationship has properties. - -```python -class AdminCityCenterAssociation( - OvertureFeature[Literal["associations"], Literal["admin_city_center"]] -): - """Describes how an administrative area relates to a city center.""" - - admin_area_id: Annotated[Id, Reference(Relationship.ASSOCIATION, AdminArea)] - city_center_id: Annotated[Id, Reference(Relationship.ASSOCIATION, CityCenter)] - - # Information about the relationship itself - relationship_type: Literal["primary_center", "secondary_center"] = "primary_center" - established_date: str | None = None - confidence_score: Annotated[float64, Field(ge=0.0, le=1.0)] | None = None -``` - -**When to use separate association features:** -- The relationship has properties (confidence scores, dates, types, notes). -- Many-to-many connections exist. -- You need to query the relationships independently. - -##### 3. Collection References - -When a feature needs to reference multiple other features, use a list of references. The semantic type still matters. - -```python -# COMPOSITION — boundary defines two divisions -class DivisionBoundary( - OvertureFeature[Literal["divisions"], Literal["division_boundary"]] -): - """A boundary line between two divisions.""" - - division_ids: Annotated[ - list[ - Annotated[ - Id, Reference(Relationship.COMPOSITION, Division, role="boundary_of") - ] - ], - Field(min_length=2, max_length=2, description="Left and right divisions"), - ] - - -# AGGREGATION — route groups segments -class Route(OvertureFeature[Literal["transportation"], Literal["route"]]): - """A transportation route passing through multiple segments.""" - - segment_ids: Annotated[ - list[Id], - Reference(Relationship.AGGREGATION, TransportationSegment, role="groups"), - Field(min_length=1, description="Ordered segments in this route"), - UniqueItemsConstraint(), - ] -``` - ---- - -#### Best Practices - -##### Always Use Reference Annotations - -Include `Reference` annotations for semantic clarity and documentation: - -```python -# Good — complete relationship information with semantic type and role -division_id: Annotated[ - Id, - Reference(Relationship.HIERARCHY, Division, role="child_of"), - Field(description="Division ID of the parent division of this area."), -] - -# Avoid — missing semantic information -division_id: Id -``` - -##### Choose the Right Semantic Type First, Then the Right Pattern - -1. **Determine the semantic type** using the selection priority and test questions above. -2. **Then choose the implementation pattern:** - - Simple relationships → Direct references (Pattern 1) - - Relationships with metadata → Separate association features (Pattern 2) - - One-to-many references → Collection references (Pattern 3) - -### Discriminated Unions - -**What is a discriminated union?** A discriminated union is a type that can be backed by one of several different models, where a specific field (the "discriminator") determines which model it actually is. Think of it like a form that changes its fields based on a category selection. - -```python -from typing import Annotated, Literal -from pydantic import Field -from overture.schema.common import OvertureFeature - - -# Base class with common fields -class TransportationSegment( - OvertureFeature[Literal["transportation"], Literal["segment"]] -): - subtype: Subtype # This is the discriminator field - # ... common fields for all segments - - -# Specific segment types -class RoadSegment(TransportationSegment): - subtype: Literal[Subtype.ROAD] # Must be "road" - class_: Annotated[RoadClass, Field(alias="class")] - speed_limits: SpeedLimits | None = None - # ... road-specific fields - - -class RailSegment(TransportationSegment): - subtype: Literal[Subtype.RAIL] # Must be "rail" - class_: Annotated[RailClass, Field(alias="class")] - rail_flags: RailFlags | None = None - # ... rail-specific fields - - -# Union type that automatically picks the right model based on subtype -Segment = Annotated[ - RoadSegment | RailSegment | WaterSegment, Field(discriminator="subtype") -] -``` - -The `discriminator="subtype"` tells Pydantic to look at the `subtype` field to determine which specific model to use. If `subtype` is "road", it uses `RoadSegment`; if "rail", it uses `RailSegment`. - -#### Abstract vs Concrete Classes - -**What's the difference?** In UML and traditional OOP, abstract classes cannot be instantiated - they serve as templates for concrete classes. In Pydantic, by default, **all classes are concrete** (can be instantiated), but you can make classes abstract when needed. - -**Current pattern (all concrete):** - -```python -# Both can be instantiated as map features -base_segment = TransportationSegment(subtype=Subtype.ROAD, geometry=...) # Valid -road_segment = RoadSegment(subtype=Subtype.ROAD, geometry=..., class_=...) # Valid -``` - -**Making the base class abstract:** - -```python -from abc import ABC, abstractmethod -from typing import Annotated, Literal -from pydantic import Field - -class TransportationSegment(OvertureFeature[Literal["transportation"], Literal["segment"]], ABC): - """Abstract base - cannot be instantiated directly.""" - - subtype: Subtype # Discriminator field - - @abstractmethod - def get_speed_limit(self) -> float: - """Each concrete type must implement this.""" - pass - -class RoadSegment(TransportationSegment): - """Concrete class - can be instantiated.""" - subtype: Literal[Subtype.ROAD] - speed_limits: SpeedLimits | None = None - - def get_speed_limit(self) -> float: - return self.speed_limits.max_speed if self.speed_limits else 50.0 - -# Now only concrete classes can be instantiated -# base_segment = TransportationSegment(...) # TypeError: Can't instantiate abstract class -road_segment = RoadSegment(subtype=Subtype.ROAD, ...) # Valid -``` - -**Registration pattern (recommended when working with Overture models):** - -Instead of making classes abstract, we use **entry point registration** where only specific concrete types are discoverable as map features: - -```toml -# In packages/overture-schema-theme-transportation/pyproject.toml -[project.entry-points."overture.models"] -connector = "overture.schema.transportation:Connector" -segment = "overture.schema.transportation:Segment" -``` - -**Real example:** See [`packages/overture-schema-theme-transportation/src/overture/schema/transportation/segment/__init__.py`](packages/overture-schema-theme-transportation/src/overture/schema/transportation/segment/__init__.py) where: - -- **`Segment`** is a discriminated union: `RoadSegment | RailSegment | WaterSegment` -- **`TransportationSegment`** is the concrete base class that all segment types inherit from -- **Individual segment types** (`RoadSegment`, `RailSegment`, `WaterSegment`) are NOT directly registered - -**This registration pattern means:** - -1. Only **`Segment`** (the union type) is discoverable as an official map feature -2. The union automatically resolves to the correct concrete type based on the `subtype` field -3. All classes (`TransportationSegment`, `RoadSegment`, etc.) can be reused as base classes for alternate implementations - -### Pattern Properties (Constrained Key-Value Maps) - -**What are pattern properties?** Pattern properties let you create key-value maps where the keys must follow a specific pattern (like language codes) and values have specific types. - -```python -from typing import Annotated -from pydantic import BaseModel, Field - - -@no_extra_fields -class Names(BaseModel): - primary: str - - # Keys (strings) must match a language tag pattern, values are strings - common: ( - Annotated[ - dict[ - # The key type - Annotated[ - str, - Field( - pattern=r"^[a-z]{2,3}(-[A-Z]{2})?$", - description="Language tag (e.g., 'en', 'es-MX')", - ), - ], - str, # The value type - ], - Field(json_schema_extra={"additionalProperties": False}), - ] - | None - ) = None -``` - -**Example data:** - -```json -{ - "primary": "New York City", - "common": { - "es": "Ciudad de Nueva York", - "fr": "New York", - "zh-CN": "纽约市" - } -} -``` - -The `additionalProperties: False` ensures only keys matching the pattern are allowed when generated JSON Schema is used. - -### Nested List Validation - -**What is nested list validation?** This pattern validates both the outer list and the inner structure of each item, with constraints at multiple levels. - -```python -from typing import Annotated -from pydantic import Field - - -# Each item has its own field validation -@no_extra_fields -class HierarchyItem(BaseModel): - division_id: str - name: str - - -class Division(OvertureFeature): - # Nested list validation: outer list AND inner lists both have length constraints - hierarchies: Annotated[ - list[ # Outer list - Annotated[ - list[HierarchyItem], # Inner list - Field(min_length=1), # Inner list must have at least 1 item - ] - ], - Field(min_length=1), # Outer list must have at least 1 hierarchy - ] -``` - -This creates validation at three levels: - -1. **Individual items**: Each `HierarchyItem` validates its own fields (`division_id`, `name`) -2. **Inner lists**: Each inner list must have at least 1 item (`min_length=1`) -3. **Outer list**: The `hierarchies` field must have at least 1 inner list (`min_length=1`) - -### Type Aliases for Reusable Patterns - -**What are type aliases?** Type aliases let you create custom names for complex or frequently-used types. Think of them like creating shortcuts or nicknames for long type definitions. - -**What is `NewType`?** `NewType` creates a distinct type that's based on an existing type but is treated as different for type checking purposes. This helps prevent mistakes like using an email address where you need a country code, or using a person's name where you need an ID - they're all strings, but they have different meanings and shouldn't be interchangeable. - -**Note**: `NewType` is primarily useful when working with Pydantic models in Python code (development, testing, certain data processing tasks). It doesn't affect data validation or JSON Schema generation - it's a development tool to catch mistakes before they happen. - -> [!WARNING] -> **Naming conflicts**: Never use the same name for a model class and type alias in the same module - this creates circular references and confusing code. - -**Guidelines:** - -- **Model classes**: Use noun names (`SourceItem`, `AccessRule`, `GeometricScope`) -- **Type aliases**: Use plural or descriptive names (`Sources`, `AccessRules`, `ConnectivityData`) -- **Avoid using the same names**: Use different names for models and type aliases, even if they're related - -```python -from typing import NewType, Annotated -from pydantic import BaseModel, Field - -# Create distinct types for different kinds of strings -SegmentId = NewType("SegmentId", str) # IDs are strings, but distinct -CountryCode = NewType("CountryCode", str) # Country codes are strings, but distinct - -# Create aliases for complex field patterns -EmailList = NewType( - "EmailList", - Annotated[list[str], Field(min_length=1, description="List of email addresses")], -) - - -@no_extra_fields -class Contact(BaseModel): - # Clear, self-documenting field types - id: SegmentId # Can't accidentally use a CountryCode here - country: CountryCode # Can't accidentally use a SegmentId here - emails: EmailList # Reusable validation pattern -``` - -**Why use type aliases?** - -1. **Prevent mistakes**: `SegmentId` and `CountryCode` are both strings, but you can't mix them up when they're created using `NewType` -2. **Reusable patterns**: Define complex field validation once, use it many times -3. **Self-documenting code**: `EmailList` is clearer than `list[str]` -4. **Consistency**: Everyone uses the same validation rules for the same concept - ---- - -## Integration Guide - -### Project Architecture - -#### File Organization - -Organize code by scope, and avoid circular imports. - -**Cross-theme shared**: the `overture-schema-common` package. Definitions more than -one theme needs -- `OvertureFeature`, `Names`, `Sources`, the scoping framework. - -**One module per feature type**: at the theme package root, named after the type in -snake_case. - -```text -packages/overture-schema-theme-buildings/src/overture/schema/buildings/ - __init__.py # re-exports the public names, declares __all__ - _common.py # shared by Building and BuildingPart - building.py # Building, BuildingSubtype, BuildingClass - building_part.py # BuildingPart -``` - -The module owns everything specific to its type: the `Feature` subclass, its enums, -its NewTypes, and its supporting models. `building.py` defines `BuildingSubtype` and -`BuildingClass` next to `Building`, because nothing else uses them. - -**Theme-level shared**: `_common.py` at the theme package root, for definitions two -or more types in the theme need. `buildings/_common.py` holds `Appearance`, -`RoofShape`, and the material enums that both `Building` and `BuildingPart` use. The -leading underscore marks the module private -- the theme's `__init__.py` re-exports -the public names from it. - -**A type large enough to split**: a subpackage named after the type, applying the -same rules one level down. - -```text -packages/overture-schema-theme-transportation/src/overture/schema/transportation/ - __init__.py # re-exports from connector and segment - connector.py # Connector - segment/ - __init__.py # assembles the Segment discriminated union, re-exports - _common.py # TransportationSegment and what the arms share - road.py # RoadSegment and its supporting types - rail.py # RailSegment and its supporting types - water.py # WaterSegment -``` - -Every `__init__.py` re-exports its public names and declares `__all__`, so consumers -import from the package rather than reaching into the defining module: -`from overture.schema.buildings import Building`, not -`from overture.schema.buildings.building import Building`. Entry points name the -package root for the same reason -- `building = "overture.schema.buildings:Building"`. - -Modules are named for the thing they define, not the kind of thing: an enum lives in -the module whose type uses it, and moves up to `_common.py` once a second type needs it. - -#### Import Organization - -```python -# Standard library imports first -from enum import Enum -from typing import Annotated, Literal, NewType - -# Third-party imports -from pydantic import BaseModel, ConfigDict, Field - -# Cross-theme and system imports -from overture.schema.common.scoping import Heading, Scope, scoped -from overture.schema.system.doc import DocumentedEnum -from overture.schema.system.field_constraint import UniqueItemsConstraint -from overture.schema.system.model_constraint import no_extra_fields - -# Local imports last -- siblings in the theme, then the module's own package -from ..connector import Connector -from ._common import SegmentSubtype, TransportationSegment -``` - -`uv run ruff format ` will sort your imports in this order automatically. - -#### Why Not Use @field_validator or @model_validator? - -This project uses a custom validation system that generates better JSON Schema output and supports code generation (without additional work, `@field_validator` and `@model_validator` don't make their constraints discoverable). Always use constraints from `overture.schema.system` instead of using Pydantic validation decorators: - -```python -# Don't do this -@field_validator("categories") -def validate_categories_unique(cls, v): - if v and len(v) != len(set(v)): - raise ValueError("Categories must be unique") - return v - - -# Do this instead -from overture.schema.system.field_constraint import UniqueItemsConstraint - - -class Building(OvertureFeature): - categories: Annotated[ - list[str] | None, - Field(min_length=1, description="Building categories"), - UniqueItemsConstraint(), - ] = None -``` - -### Migrating from JSON Schema - -If you're familiar with JSON Schema files (like `schema/schema.yaml`), this section helps translate those patterns to Pydantic models. - -#### How $defs and $ref Translate - -**JSON Schema approach:** - -```yaml -# In defs.yaml -"$defs": - propertyDefinitions: - address: - type: object - properties: - freeform: { type: string } - locality: { type: string } - -# In building.yaml -properties: - address: { "$ref": "../defs.yaml#/$defs/propertyDefinitions/address" } -``` - -**Pydantic approach:** - -```python -# In overture-schema-theme-places/src/overture/schema/places/place.py -@no_extra_fields -class Address(BaseModel): - """A postal address.""" - - freeform: str | None = None - locality: str | None = None - - -# Same module -- Place is the type that carries one. -class Place(OvertureFeature): - addresses: list[Address] | None = None -``` - -**Primary differences:** - -- JSON Schema uses `$ref` to reference definitions; Pydantic uses direct Python imports -- JSON Schema definitions live in `$defs`; Pydantic models are regular Python classes grouped into modules -- JSON Schema allows inline definitions; Pydantic encourages separate model classes - -#### How Containers Work - -**JSON Schema containers** (like `namesContainer`, `shapeContainer`) are reusable property groups: - -```yaml -# In defs.yaml -propertyContainers: - namesContainer: - properties: - names: { "$ref": "#/$defs/propertyDefinitions/allNames" } - - shapeContainer: - properties: - height: { type: number } - num_floors: { type: integer } - -# In building.yaml -allOf: - - "$ref": ../defs.yaml#/$defs/propertyContainers/namesContainer - - "$ref": ./defs.yaml#/$defs/propertyContainers/shapeContainer -``` - -**Pydantic equivalent** uses **mixin classes**: - -```python -# namesContainer -> Named, in -# overture-schema-common/src/overture/schema/common/names.py -class Named(BaseModel): - """Properties defining the names of a feature.""" - - names: Names | None = None - - -# shapeContainer -> Appearance, in buildings/_common.py, -# shared by Building and BuildingPart -class Appearance(BaseModel): - """Physical and visual properties of a building.""" - - height: float64 | None = None - num_floors: int32 | None = None - # ... roof and facade fields - - -# Usage with multiple inheritance -- this is how Building is actually declared -class Building( - OvertureFeature[Literal["buildings"], Literal["building"]], - Named, - Stacked, - Appearance, -): ... # inherits names, level, height, num_floors, and the rest from its parents -``` - -JSON Schema containers become **mixin classes** in Pydantic that you inherit from. - -#### Common Translation Patterns - -| JSON Schema | Pydantic | Notes | -|-------------|----------|-------| -| `"$ref": "other.yaml#/path"` | `from other import Model` | Direct Python imports | -| `allOf: [ref1, ref2]` | `class Model(Base1, Base2)` | Multiple inheritance | -| `minLength: 1` | `Field(min_length=1)` | Field constraints | -| `minimum: 0, maximum: 100` | `Field(ge=0, le=100)` | Numeric ranges | -| `uniqueItems: true` | `UniqueItemsConstraint()` | Custom constraint | -| `enum: [a, b, c]` | `class E(str, Enum): A="a"` | Enum class | -| `type: ["string", "null"]` | `str \| None = None` | Optional types | -| `if/then` conditional | Custom validation constraints | Model constraints | - ---- - -## Reference - -### Quick Reference - -#### Essential Patterns (Most Common) - -```python -# Basic field types -name: str # Required string -name: str | None = None # Optional string -count: int32 # Required integer -priority: Literal["high", "medium", "low"] | None = None # Constrained values - -# Validated fields -height: Annotated[float64 | None, Field(ge=0, description="Height in meters")] = None -tags: Annotated[list[str] | None, Field(min_length=1), UniqueItemsConstraint()] = None - -# Association patterns -- Relationship is the kind, role is the meaning -parent_id: Annotated[ - Id | None, Reference(Relationship.HIERARCHY, ParentModel, role="child_of") -] = None -connector_ids: list[Id] # References to multiple related features -``` - -#### Model Templates - -```python -# Non-feature model -@no_extra_fields -class Address(BaseModel): - street: str - city: str | None = None - - -# Feature model -class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): - geometry: Geometry - height: float64 | None = None - - -# Enum -class Status(str, Enum): - ACTIVE = "active" - INACTIVE = "inactive" -``` - -#### Constraint Reference - -| Type | Constraint | JSON Schema | Example | -|------|------------|-------------|---------| -| **Numeric** | `ge=0, le=100` | `minimum`, `maximum` | `Field(ge=0, le=100)` | -| **String** | `min_length=1, pattern=r"..."` | `minLength`, `pattern` | `Field(min_length=1, pattern=r"^[A-Z]+$")` | -| **List** | `min_length=1, UniqueItemsConstraint()` | `minItems`, `uniqueItems` | `Field(min_length=1), UniqueItemsConstraint()` | -| **Custom** | `LanguageTagConstraint()` | Custom validation | `LanguageTagConstraint()` | - -#### Import Cheatsheet - -```python -# Essential imports for most models -from typing import Annotated, Literal -from enum import Enum -from pydantic import Field -from overture.schema.common import OvertureFeature -from overture.schema.system.field_constraint import UniqueItemsConstraint -from overture.schema.system.model_constraint import no_extra_fields -from overture.schema.system.numeric import int32, float64 - -# For associations and references -from overture.schema.system.ref import Id, Reference, Relationship -``` - -#### Naming Conventions - -- **Classes**: `PascalCase` (`Building`, `AccessRule`) -- **Fields**: `snake_case` (`construction_year`, `has_parts`) -- **Enums**: `UPPER_SNAKE_CASE = "value"` (`ACTIVE = "active"`) diff --git a/README.md b/README.md index 921a4c6b5..365c577c3 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,22 @@ The Overture Maps schema working group is responsible for designing the Overture ## Documentation The contents of this repository are presented in a more human-friendly format at [docs.overturemaps.org](https://docs.overturemaps.org/) +## Python packages +The schema is authored as [Pydantic](https://docs.pydantic.dev/latest/) models, published +as a set of Python packages under `packages/`. See +[SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) for installing them, validating data, generating +artifacts, and authoring new schema models. + +## Schema reference + +- [GLOSSARY.md](GLOSSARY.md) — vocabulary for both the data model (entity, feature type, + theme) and the Python toolchain (entry point, workspace, discriminated union). +- [SCHEMA_CONVENTIONS.md](SCHEMA_CONVENTIONS.md) — naming and modelling conventions. + **Out of date:** it predates the Pydantic packages and still describes JSON Schema as + the way the schema is defined, spells `subtype` as `subType`, and leaves the extensions + section unfinished. Useful for the conventions themselves; check anything structural + against [SCHEMA_GUIDE.md](SCHEMA_GUIDE.md). + ## Contributing See [CONTRIBUTING.md](CONTRIBUTING.md) for branching strategy, workflow, and contribution guidelines. diff --git a/README.pydantic.md b/README.pydantic.md deleted file mode 100644 index 1b0376f75..000000000 --- a/README.pydantic.md +++ /dev/null @@ -1,240 +0,0 @@ -# Overture Schema - -[Pydantic](https://docs.pydantic.dev/latest/) schemas for [Overture Maps -data](https://docs.overturemaps.org/guides/). - -## Overview - -This project provides type-safe Python models for validating and working with [Overture -Maps Foundation](https://overturemaps.org/) data. Overture Maps is an open geospatial -dataset containing buildings, places, addresses, transportation networks, and -administrative boundaries curated from multiple sources. - -Use these schemas to: - -- Validate Overture Maps data -- Build data processing pipelines with type safety -- Extend schemas with custom fields and validation rules - -## Why Use Pydantic to Define Data Schemas? - -This project addresses a fundamental challenge in data consumption: **bridging the -semantic gap between raw data and human understanding** while enabling -machine-actionable workflows. - -### Why Schema at All: Beyond Raw Data - -Take a column like `pop_2020`. Is it total population? Population density per square -kilometer? Working-age population? Without a schema, you're left sampling values and -guessing from column names. - -Compare this to OpenStreetMap's approach: features use well-known key/value pairs like -`building=residential` or `addr:housenumber=42` that have semantic meaning and can be -looked up on the OSM wiki. This creates a step toward a schema - shared vocabulary with -documented semantics used across a vast dataset. However, OSM tags remain free-form: -multiple valid ways to express the same concept, no built-in validation, and complex -downstream validation because of undocumented keys that might have meaning to someone, -somewhere. A schema provides the structured alternative: explicit types, clear -validation rules, and semantic meaning that both humans and systems can rely on. - -Data files containing only column names and values aren't fully documented. External -metadata files typically focus on how data was collected and encoded, not on semantic -meaning or validation rules. Data consumers struggle to understand what datasets contain -and which columns they need for their goals. - -### Why Pydantic Over JSON Schema: Solving Multiple Problems - -We initially chose JSON Schema because it aligned with our mental model and promised to -solve our problems as we understood them. But JSON Schema surfaced several pain points: - -- **Authoring difficulty**: Hard to write correctly, difficult to verify, limited IDE - support, no refactoring capabilities -- **Tooling gaps**: Generic tools can't tailor output for specific applications like - ours -- **Development friction**: Schema changes required manual coordination across multiple - artifacts - -Pydantic addresses these systematically: author in Python with full IDE support, -generate tailored documentation, and automatically produce the specific artifacts each -workflow needs. Pydantic can also produce JSON Schema, so any application that requires -it can use it while we gain all the Python benefits during authoring. - -### The Result: Faster Understanding, Higher Quality - -Instead of spending time deciphering what columns mean and whether data matches -expectations, users can focus on their actual goals: analysis, visualization, -integration. Quality improves because validation happens automatically rather than -through manual inspection. - -The fundamental approach - human-readable authoring that generates machine-actionable -outputs - has broader applications beyond Overture and geospatial data. We hope others -will adapt these patterns for linking with Overture data or modeling their own domains -entirely. - -## Getting Started - -- Install [Python](https://www.python.org/downloads/) 3.10 or newer -- Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/) -- Clone this repository: `git clone https://github.com/OvertureMaps/schema.git` -- Install dependencies: `uv sync --all-packages` -- Run tests to ensure that everything is configured correctly: `make check` (on Windows, - without `make`: `uv run pytest packages`) - -## Packages - -This workspace contains the following packages: - -### Core Packages - -- **`overture-schema`** - Main entrypoint package that aggregates all types for - convenient usage -- **`overture-schema-common`** - Overture-specific models shared across themes: base - feature class, scoping framework, names, sources, and cartographic hints -- **`overture-schema-system`** - Portable numeric, geometric, and string types, - constraints, and a GeoJSON-aware base model for building Pydantic schemas that - serialize to JSON, Parquet, and Spark - -### Theme Packages - -- **`overture-schema-theme-addresses`** - Address features -- **`overture-schema-theme-base`** - Foundational geographic features (land, water, - infrastructure, bathymetry, land cover, land use) -- **`overture-schema-theme-buildings`** - Building footprints and building parts with - architectural details -- **`overture-schema-theme-divisions`** - Administrative boundaries, division areas, and - political boundaries -- **`overture-schema-theme-places`** - Points of interest, businesses, and named - locations -- **`overture-schema-theme-transportation`** - Road segments and transportation network - connectors - -### Usage (Python) - -Install the main package using `pip` (or your package manager of choice): - -```shell -pip install overture-schema -``` - -Overture publishes data in one shape: flat and tabular, the column layout of the -Parquet release, which Pydantic's Python mode reads. The models also accept and -emit GeoJSON, through JSON mode, so the schema works with tools that expect -features rather than rows -- and it is the representation the generated JSON -Schema describes. The modes are not interchangeable: a GeoJSON dict passed to -`model_validate` reports `theme` and `version` missing and `type` set to -`'Feature'`. - -```python -from overture.schema.buildings import Building -from overture.schema.places import Place - -# Flat / tabular dict -- Python mode -building = Building.model_validate(feature_row) - -# GeoJSON, as a string or bytes -- JSON mode -building = Building.model_validate_json(geojson_text) - -# Serialize back to GeoJSON. by_alias=True is required: without it, aliased -# fields serialize under their Python names (`class_`, not `class`) and the -# output will not re-validate. -geojson_output = building.model_dump(mode="json", by_alias=True, exclude_none=True) -``` - -## Schema Extension - -The library is designed to support data producer extensions through multiple patterns. -This extensibility is a core feature that allows organizations to add custom fields and -types while maintaining compatibility with the base Overture schema. We are in the -process of determining how this should work. - -### Model Registration via Entry Points - -Models are registered using [setuptools entry -points](https://setuptools.pypa.io/en/latest/userguide/entry_point.html) in each -package's `pyproject.toml` file. This enables automatic discovery and loading of models -at runtime without requiring explicit imports. - -Registration is done in the `[project.entry-points."overture.models"]` section: - -```toml -[project.entry-points."overture.models"] -building = "overture.schema.buildings:Building" -building_part = "overture.schema.buildings:BuildingPart" -``` - -The discovery system provides programmatic access to registered models: - -```python -from overture.schema.system.discovery import discover_models, get_registered_model - -# Discover all registered models, keyed by ModelKey -all_models = discover_models() - -# Get a specific model by name -building_model = get_registered_model("building") -if building_model: - building = building_model.model_validate(building_data) -``` - -### Tagging - -Each `ModelKey` returned by `discover_models()` carries a `frozenset[str]` of tags -that classify the model orthogonally to its entry-point name -- whether the model -is a `Feature` subclass, which Overture theme it belongs to, which package shipped -it, and so on. Downstream tools (the CLI, codegen, third-party consumers) use tags -to filter the working set without importing every model: - -```python -from overture.schema.system.discovery import ( - TagSelector, - discover_models, - filter_models, -) - -models = discover_models() -# { -# ModelKey(name="building", entry_point="overture.schema.buildings:Building", -# tags=frozenset({"feature", "overture", "overture:theme=buildings"})): Building, -# ModelKey(name="place", entry_point="overture.schema.places:Place", -# tags=frozenset({"feature", "overture", "overture:theme=places"})): Place, -# ... -# } - -buildings = filter_models( - models, - TagSelector(include_any=("overture:theme=buildings",)), -) -``` - -Tags are produced by *tag providers* registered on the `overture.tag_providers` -entry-point group. The `system` and `common` packages ship the built-in providers -(`feature`, `overture`, `overture:theme=*`); third parties can register their own -to attach custom tags during discovery. See the [`overture-schema-system` -README](packages/overture-schema-system/README.md#tagging) for tag format, -reserved namespaces, and provider authoring. - -## Development - -This project uses [uv](https://docs.astral.sh/uv/) for dependency management: - -```bash -# Install dependencies for the entire workspace -uv sync --all-packages - -# Run all tests and type/code quality checks -make check - -# Run tests for a specific package -uv run pytest packages/overture-schema-theme-buildings/ - -# Run tests matching a pattern -uv run pytest -k "buildings" -``` - -Auto-format / fix code to align with project expectations: - -```shell -uv run ruff check --fix -uv run ruff format -uv run docformatter --in-place --recursive packages/ -``` diff --git a/SCHEMA_GUIDE.md b/SCHEMA_GUIDE.md new file mode 100644 index 000000000..3929620a4 --- /dev/null +++ b/SCHEMA_GUIDE.md @@ -0,0 +1,4684 @@ +# Overture Schema Guide + +A practical guide to installing, exploring, building on, and authoring the Overture Maps +schema packages. + +**Audience:** everyone who touches these packages in Python. The guide is in parts, and +you probably want one of them rather than all of them: + +| If you want to | Read | +|---|---| +| Understand why the schema is Pydantic at all | Part 0 | +| Validate data, write code against the models, generate artifacts | Part I | +| Register your own feature types, or author new schema models | Part II | +| Look something up | Part III | +| Look up a term | [Glossary](GLOSSARY.md) | + +**Status:** none of these packages are on PyPI yet. Everything below installs from a +local clone with `uv`. Any `pip install overture-schema` you find in a README is +aspirational — it will not work today. + +**Verification:** every command and code block in this guide was checked against the +schema repo at commit `2a6170c4` on Python 3.10 — 87 Python blocks parsed, every +`overture.*` import resolved against the installed packages, and every runnable snippet +executed. Blocks that are deliberately wrong (marked ✗) or written as `>>>` transcripts +are excluded, as are template fragments with placeholder names. + +--- + +## Table of contents + +**Part 0 — [Why Pydantic](#part-0--why-pydantic)** + +**Part I — [Using the schema](#part-i--using-the-schema)** + +1. [Install](#1-install) +2. [Exploring the models](#2-exploring-the-models) +3. [Writing code against the models](#3-writing-code-against-the-models) +4. [The three CLIs](#4-the-three-clis) +5. [Validating data](#5-validating-data) +6. [Converting the schema to other formats](#6-converting-the-schema-to-other-formats) +7. [Using the packages from your own project](#7-using-the-packages-from-your-own-project) + +**Part II — [Extending and authoring the schema](#part-ii--extending-and-authoring-the-schema)** + +8. [Building your own SDK or CLI](#8-building-your-own-sdk-or-cli) +9. [Registering models and tagging](#9-registering-models-and-tagging) +10. [Authoring new schema models](#10-authoring-new-schema-models) +11. [Development workflow](#11-development-workflow) + +**Part III — [Reference](#part-iii--reference)** + +12. [Gotchas](#12-gotchas) +13. [Templates and quick reference](#13-templates-and-quick-reference) + +**[Glossary](GLOSSARY.md)** — data-model and toolchain vocabulary, cross-linked back into this guide. + +--- + +# Part 0 — Why Pydantic + +### Why this exists + +This project provides type-safe Python models for validating and working with [Overture +Maps Foundation](https://overturemaps.org/) data. Overture Maps is an open geospatial +dataset containing buildings, places, addresses, transportation networks, and +administrative boundaries curated from multiple sources. + +Use these schemas to: + +- Validate Overture Maps data +- Build data processing pipelines with type safety +- Extend schemas with custom fields and validation rules + +### Why Pydantic rather than JSON Schema? + +This project addresses a fundamental challenge in data consumption: **bridging the +semantic gap between raw data and human understanding** while enabling +machine-actionable workflows. + +### Why Schema at All: Beyond Raw Data + +Take a column like `pop_2020`. Is it total population? Population density per square +kilometer? Working-age population? Without a schema, you're left sampling values and +guessing from column names. + +Compare this to OpenStreetMap's approach: features use well-known key/value pairs like +`building=residential` or `addr:housenumber=42` that have semantic meaning and can be +looked up on the OSM wiki. This creates a step toward a schema - shared vocabulary with +documented semantics used across a vast dataset. However, OSM tags remain free-form: +multiple valid ways to express the same concept, no built-in validation, and complex +downstream validation because of undocumented keys that might have meaning to someone, +somewhere. A schema provides the structured alternative: explicit types, clear +validation rules, and semantic meaning that both humans and systems can rely on. + +Data files containing only column names and values aren't fully documented. External +metadata files typically focus on how data was collected and encoded, not on semantic +meaning or validation rules. Data consumers struggle to understand what datasets contain +and which columns they need for their goals. + +### Why Pydantic Over JSON Schema: Solving Multiple Problems + +We initially chose JSON Schema because it aligned with our mental model and promised to +solve our problems as we understood them. But JSON Schema surfaced several pain points: + +- **Authoring difficulty**: Hard to write correctly, difficult to verify, limited IDE + support, no refactoring capabilities +- **Tooling gaps**: Generic tools can't tailor output for specific applications like + ours +- **Development friction**: Schema changes required manual coordination across multiple + artifacts + +Pydantic addresses these systematically: author in Python with full IDE support, +generate tailored documentation, and automatically produce the specific artifacts each +workflow needs. Pydantic can also produce JSON Schema, so any application that requires +it can use it while we gain all the Python benefits during authoring. + +### The Result: Faster Understanding, Higher Quality + +Instead of spending time deciphering what columns mean and whether data matches +expectations, users can focus on their actual goals: analysis, visualization, +integration. Quality improves because validation happens automatically rather than +through manual inspection. + +The fundamental approach - human-readable authoring that generates machine-actionable +outputs - has broader applications beyond Overture and geospatial data. We hope others +will adapt these patterns for linking with Overture data or modeling their own domains +entirely. + + +--- + +# Part I — Using the schema + +## 1. Install + +### 1.1 First, what you're installing + +**The schema repo is not one Python package. It's thirteen.** + +That's the thing that makes this confusing at the start, so it's worth a minute before +you run anything. + +Open `packages/` and you'll see: + +``` +packages/ +├── overture-schema ← a metapackage: depends on the others, ships no code +├── overture-schema-system ← foundations: numeric types, geometry, discovery +├── overture-schema-common ← Overture conventions: OvertureFeature, names, sources +├── overture-schema-cli ← the `overture-schema` command +├── overture-schema-validation ← validate() / validate_json() +├── overture-schema-codegen ← the `overture-codegen` command +├── overture-schema-pyspark ← the `overture-validate` command +└── overture-schema-theme-* ← six of these: buildings, places, transportation, … +``` + +Each of those directories has its own `pyproject.toml` and its own version number. They +release independently. + +#### What "workspace" means + +A **uv workspace** is one repository containing several packages that are developed +together, sharing **one lockfile** and **one virtual environment**. If you've used Cargo +workspaces, npm workspaces, or a monorepo, it's the same idea. + +The root `pyproject.toml` declares it: + +```toml +[project] +name = "overture-schema-workspace" # ← a container, not something you install +version = "0.0.0" + +[tool.uv.workspace] +members = ["packages/*"] # ← every directory under packages/ is a member +``` + +Two things follow from this, and they're the ones that trip people up: + +**1. You never install or import `overture-schema-workspace`.** It's scaffolding. It +exists so `uv` knows which directories are members. There is no `import +overture_schema_workspace`. + +**2. The packages depend on each other *locally*, not through PyPI.** Look at +`packages/overture-schema-cli/pyproject.toml`: + +```toml +[tool.uv.sources] +overture-schema-common = { workspace = true } +overture-schema-system = { workspace = true } +``` + +`workspace = true` means "use the copy in this repo." This is why none of this needs to +be published to PyPI for you to work with it — the packages find each other on disk. + +#### What you end up with + +One command installs all thirteen into **a single shared virtualenv at the repo root**: + +``` +schema/ +├── .venv/ ← created by uv; all 13 packages live here +│ └── bin/ +│ ├── overture-schema ← the three CLIs land here +│ ├── overture-codegen +│ └── overture-validate +├── packages/ +└── pyproject.toml +``` + +You don't activate it. `uv run ` runs `` inside that venv for you. +That's why every command in this guide starts with `uv run`. + +> **Why split into thirteen packages at all?** Because *what you install determines what +> exists at runtime*. The models register themselves through Python entry points, so +> installing only the buildings theme means the CLI only knows about buildings. That's +> the extension mechanism — see [Using the packages from your own project](#7-using-the-packages-from-your-own-project) and +> [Building your own SDK or CLI](#8-building-your-own-sdk-or-cli). If you just want everything, that's fine too. + +--- + +**Layering, bottom up:** + +| Layer | Package | Gives you | +|---|---|---| +| Foundation | `system` | `float32`/`uint8`/…, `Geometry`, `BBox`, `CountryCodeAlpha2`, constraint annotations, `Feature` base class, entry-point discovery | +| Overture conventions | `common` | `OvertureFeature` (id/theme/type/version/geometry/sources), `@scoped`, `Names`, `Sources`, cartography hints | +| Feature types | `theme-*` | `Building`, `Place`, `Segment`, `Address`, … plus their enums | +| Tooling | `cli`, `codegen`, `pyspark`, `validation` | commands and functions that consume the above generically | + +The tooling layer never hardcodes feature types. It discovers them. That's why your own +models can slot in. + +--- + +### 1.2 Prerequisites + +- **Python 3.10 or newer.** You don't need to install this yourself — `uv` will fetch a + suitable Python if your system one is too old. +- **[`uv`](https://docs.astral.sh/uv/getting-started/installation/).** + +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +You do **not** need Java, Spark, or Homebrew for anything in sections 1 through 8 of this +guide except the PySpark parts. If you already have a Homebrew Spark installed, skip +ahead to [When something goes wrong](#16-when-something-goes-wrong) — it may actively break things. + +--- + +### 1.2a Two zsh gotchas when pasting commands + +macOS defaults to **zsh**, and interactive zsh does *not* treat `#` as a comment unless +you turn that on. Paste a line like this and zsh hands `#`, `→`, and `15` to `wc` as +filenames: + +``` +find ... | wc -l # → 15 +wc: #: open: No such file or directory +wc: →: open: No such file or directory +wc: 15: open: No such file or directory + 0 total +``` + +A line that *starts* with `#` fails more obviously — `command not found: #`. + +This guide keeps `#` comments only as standalone label lines inside multi-command blocks, +never trailing after a command. To paste those blocks whole, enable comments once: + +```bash +setopt interactive_comments +``` + +Add it to `~/.zshrc` to make it permanent. Otherwise, skip the `#` lines when copying — +they are labels, not commands. Scripts and non-interactive shells are unaffected; this is +purely an interactive-zsh behavior. + +#### 2. `!` runs a command out of your history + +This one is worth understanding because it can do real damage. In an interactive shell, +`!` triggers **history expansion**, and it fires *inside double quotes*. `!r` means "the +most recent command starting with `r`" — the shell splices that command's text into your +line before running it. + +So a Python one-liner containing `{value!r}`, pasted into zsh as + +``` +uv run python -c "... f'default={f.default!r}' ..." +``` + +becomes something else entirely. What you get depends on your own shell history: + +``` +SyntaxError: f-string: invalid syntax + (f.defaultrm -rf schema) +``` + +That is a past command of yours, pasted into the middle of a Python f-string. Here it only +produced a syntax error — Python never ran and nothing was deleted. But the same mechanism +can land text somewhere the shell *will* execute. + +**The fix used throughout this guide:** multi-line Python is passed via a heredoc with a +**quoted** delimiter, not `-c "..."`. + +```bash +uv run python <<'PY' +from overture.schema.buildings import Building +print(f"{Building.__name__!r} is safe here") +PY +``` + +Quoting the delimiter (`<<'""" + D + """'` rather than `<<""" + D + """`) disables every +form of expansion in the body — history, variables, command substitution. The text reaches +Python exactly as written. + +Verified rather than assumed: + +``` +double-quoted -c -> !r expanded into a command from history +quoted heredoc -> !r left alone +``` + +Single quotes also block history expansion, but the Python in this guide uses single +quotes internally, so heredocs are the practical choice. If you hit this in your own +one-liners, `{value!r}` can always be written `{repr(value)}` instead — no `!` at all. + +--- + +### 1.3 Install + +```bash +git clone https://github.com/OvertureMaps/schema.git +cd schema +uv sync --all-packages +``` + +That's it. `uv sync --all-packages` reads the lockfile, creates `.venv/`, and installs +all thirteen workspace members into it. + +**This is the whole install for most people.** You can now validate data, explore models, +generate JSON Schema, and generate documentation. + +**What about `make install`?** It works too, and nothing about it is risky. It's exactly +`uv sync --all-packages --all-extras` plus the PySpark generation step described in +[Do you need PySpark?](#15-do-you-need-pyspark) — so it just does more than most people +need. (No package in the workspace defines extras today, so `--all-extras` changes +nothing.) Use it if you'd rather run one command and have everything. + +--- + +### 1.4 Check it worked + +Run these three. All should succeed: + +```bash +uv run overture-schema --version +``` +``` +overture-schema, version 1.17.1 +``` + +> **Seeing a warning banner above that line?** Something like +> `warning: Failed to parse pyproject.toml ... exclude-newer = "1 week"`. The command +> still worked — but your `uv` is too old for this repo, and it is quietly rewriting +> `uv.lock` behind your back. Stop and fix it before going further: +> [uv warns about `exclude-newer`](#uv-warns-about-exclude-newer--and-quietly-rewrites-your-lockfile). +> Every output shown in this guide assumes a correctly configured `uv` and omits that +> banner. + +```bash +uv run overture-schema list-types +``` +``` +address feature overture:theme=addresses +bathymetry feature overture:theme=base +building feature overture:theme=buildings +building_part feature overture:theme=buildings +connector feature overture:theme=transportation +division feature overture:theme=divisions +division_area feature overture:theme=divisions +division_boundary feature overture:theme=divisions +infrastructure feature overture:theme=base +land feature overture:theme=base +land_cover feature overture:theme=base +land_use feature overture:theme=base +place feature overture:theme=places +segment feature overture:theme=transportation +water feature overture:theme=base +``` + +```bash +uv run overture-schema validate examples/buildings/building-polygon.yaml +``` +``` +✓ Successfully validated examples/buildings/building-polygon.yaml +``` + +If all three worked, you're installed. **Skip to section 2** unless you need PySpark. + +#### What did that third command actually validate? + +Fair question — it's the first command that does real work, and the filename explains +nothing. + +`examples/buildings/building-polygon.yaml` is a file **in the repo you just cloned**. It +is not data you downloaded. The `examples/` tree is the project's own corpus of +hand-written sample features, used as test fixtures and pulled into the documentation +site. This one describes a single building — a parking structure in Washington DC: + +```yaml +id: overture:buildings:building:1234 +type: Feature # ← GeoJSON envelope +geometry: + type: Polygon + coordinates: [[ [-77.036873, 38.897804], ... ]] +properties: + ext_foo: I am a customer user property. # ← custom, non-Overture + theme: buildings # ← which theme + type: building # ← which feature type + version: 1 + height: 21.34 + num_floors: 4 + subtype: transportation + class: parking + sources: + - property: "" + dataset: microsoftMLBuildings +``` + +#### "Envelope" — the word this guide keeps using + +An **envelope** is an outer wrapper that carries a payload plus a little standard +information about it. The term is borrowed from mail: the address and stamp go on the +outside and are the same on every envelope; the letter inside is whatever you wrote. + +GeoJSON works exactly that way. Every GeoJSON Feature has the same four outer keys, fixed +by RFC 7946 — that's the envelope. Everything specific to *your* data goes in one of them, +`properties` — that's the payload. + +```json +{ + "type": "Feature", <- envelope: what kind of object this is + "id": "overture:buildings:building:1234", <- envelope: identity + "geometry": { "type": "Polygon", ... }, <- envelope: where it is + "properties": { <- envelope: the pocket for everything else + "theme": "buildings", payload: Overture's fields + "type": "building", + "height": 21.34, + "num_floors": 4, + "class": "parking" + } +} +``` + +Split them apart for yourself: + +```bash +uv run python <<'PY' +import json, yaml +from overture.schema.buildings import Building + +b = Building.model_validate_json( + json.dumps(yaml.safe_load(open("examples/buildings/building-polygon.yaml")))) +gj = b.model_dump(mode="json", by_alias=True, exclude_none=True) + +print("envelope keys:", sorted(gj)) +print("payload keys :", sorted(gj["properties"])) +PY +``` + +``` +envelope keys: ['geometry', 'id', 'properties', 'type'] +payload keys : ['class', 'ext_bar', 'ext_foo', 'height', 'is_underground', 'level', + 'num_floors', 'num_floors_underground', 'sources', 'subtype', + 'theme', 'type', 'version'] +``` + +Four keys outside, thirteen inside. **A GeoJSON Feature for a road, a lake, or a mailbox +has the same four outer keys** — that's what makes it universally readable. Only the +payload differs. + +So when this guide says "the envelope owns `id` and `geometry`," it means those two are +outer keys, placed there by GeoJSON's rules rather than by Overture's. And "the fields are +nested under the envelope" means the Overture fields sit inside `properties` rather than +at the top. + +One consequence worth carrying forward: **the envelope only exists in the GeoJSON +rendering.** In the Pydantic model, and in Parquet, there is no envelope — `id`, +`geometry`, and `height` are all just fields side by side. + +#### Three keys named `type` + +A **key** is a field name — the part left of the colon. This file uses the key `type` +three times, at three nesting levels, meaning three unrelated things: + +| Where | Value | Comes from | Means | +|---|---|---|---| +| top level | `Feature` | GeoJSON spec | "this object is a GeoJSON Feature" | +| inside `geometry` | `Polygon` | GeoJSON spec | "this shape is a polygon" | +| inside `properties` | `building` | Overture | "this feature is a building" | + +List them yourself: + +```bash +uv run python -c " +import yaml +d = yaml.safe_load(open('examples/buildings/building-polygon.yaml')) +print('type =', d['type']) +print('geometry.type =', d['geometry']['type']) +print('properties.type =', d['properties']['type'])" +``` + +``` +type = Feature +geometry.type = Polygon +properties.type = building +``` + +Only the third is Overture's. The first two belong to GeoJSON, the envelope Overture data +is wrapped in when written as JSON. Whenever this guide says "the feature's type" it means +`properties.type` — the one holding `building`, `place`, or `segment`. + +That layering is the single most confusing thing about this data, and section 3 is largely +about it. + +**"Validating" means:** the CLI parses the file, reads `theme: buildings` and +`type: building` to decide *which model* to check against, then checks every field +against that model — types, numeric bounds, enum membership, required fields, and +cross-field rules. + +You can watch it pick the model. Delete the `theme:` line and it no longer knows: + +``` +⚠ Ambiguous: Data matches multiple types equally. Consider: + • Specifying --tag or --type to narrow validation + • Adding discriminator fields to clarify intent +``` + +#### Why is it YAML? Is Overture data YAML? + +**No — real Overture data is GeoJSON or Parquet.** YAML here is purely an authoring +convenience for the example files: it allows comments (`# Custom user properties.`) and +is easier to hand-edit than JSON. + +The CLI accepts **JSON, YAML, and GeoJSON**, and YAML is a superset of JSON, so the +format is irrelevant to the validation. Convert the same file to JSON and you get the +same result: + +```bash +uv run python -c " +import json, yaml +json.dump(yaml.safe_load(open('examples/buildings/building-polygon.yaml')), + open('/tmp/same-building.json','w'), indent=2)" + +uv run overture-schema validate /tmp/same-building.json +``` +``` +✓ Successfully validated /tmp/same-building.json +``` + +Same bytes of meaning, different serialization, identical outcome. Pick whichever is +convenient — you'll mostly hand JSON or GeoJSON to this command in real use. + +#### See it actually catch something + +A success message proves the command ran, not that it's checking anything. Copy the file +and break it: + +```bash +cp examples/buildings/building-polygon.yaml /tmp/broken.yaml +``` + +Change `class: parking` to `class: skyscraper`: + +``` +class "skyscraper" ← Input should be 'agricultural', 'allotment_house', + 'apartments', 'barn', 'beach_hut', ... +``` + +Change `height: 21.34` to `height: -5`: + +``` +height -5 ← Input should be greater than 0 +``` + +Change `num_floors: 4` to `num_floors: 4.7`: + +``` +num_floors 4.7 ← Input should be a valid integer, got a number with a + fractional part +``` + +Enum membership, numeric bounds, integer-ness — each from the model definition, none of +it written by hand for this file. + +> **What it does not catch:** free-form string fields accept any string. The real +> `building-polygon.yaml` in the repo has a stray trailing comma — +> `dataset: microsoftMLBuildings,` — which YAML reads as part of the value. It parses to +> the string `'microsoftMLBuildings,'` and validates clean, because `dataset` has no +> constraint beyond "is a string." Validation enforces the schema, not your typing. + +--- + +### 1.5 Do you need PySpark? + +Probably not. Answer honestly: + +| I want to… | Need PySpark? | +|---|---| +| Validate files, explore models, write Python against them | **No** | +| Generate JSON Schema or markdown docs | **No** | +| Generate an SDK in another language | **No** | +| Validate millions of rows of Parquet, or data in S3 | **Yes** | +| Get a Spark `StructType` for a feature type | **Yes** | +| Run the full test suite (`make check`) | **Yes** | + +**If no:** you're done. Go to section 2. + +**If yes:** there's one more step, because the PySpark validation expressions are +*generated code that is not committed to git*. They're in `.gitignore`. `uv sync` alone +cannot produce them. + +```bash +make generate-pyspark +``` + +Or `make install`, which is just `uv sync --all-packages --all-extras` followed by +`make generate-pyspark`. + +#### What a successful run looks like + +**Nothing.** No progress, no file list, no "done" message — the command returns you +straight to your prompt: + +``` +$ make generate-pyspark +$ +``` + +That is success. It looks identical to nothing having happened, which is why the next +subsection exists. + +#### What the command actually does + +`make generate-pyspark` isn't a program — it's a *target* in the repo's `Makefile`, a +named recipe of shell commands. Here it is in full: + +```make +generate-pyspark: uv-sync clean-pyspark + @uv run overture-codegen generate --format pyspark \ + --output-dir $(PYSPARK_EXPRESSIONS) \ + --test-output-dir $(PYSPARK_GENERATED_TESTS) + @uv run ruff check --fix --quiet $(PYSPARK_EXPRESSIONS) $(PYSPARK_GENERATED_TESTS) + @uv run ruff format --quiet $(PYSPARK_EXPRESSIONS) $(PYSPARK_GENERATED_TESTS) +``` + +Reading that: + +- **`generate-pyspark:`** is the target name — what you typed after `make`. +- **`uv-sync clean-pyspark`** on the same line are *prerequisites*: other targets that + must run first, in that order. +- The tab-indented lines below are the *recipe* — the shell commands, run in order. +- The leading **`@`** tells make not to echo the command before running it. Without it, + make prints each command as it goes. This is why you see no output. +- **`$(PYSPARK_EXPRESSIONS)`** and **`$(PYSPARK_GENERATED_TESTS)`** are variables defined + higher up in the `Makefile`; they expand to the two output directories. + +So typing one command runs five steps: + +| # | Step | What it does | Visible? | +|---|---|---|---| +| 1 | `uv-sync` | `uv sync --all-packages --all-extras` — makes sure dependencies are installed | No: the target captures its output and prints it only on failure | +| 2 | `clean-pyspark` | `rm -rf` both output directories, so generation starts from empty | No: nothing to say | +| 3 | `overture-codegen generate --format pyspark` | **The actual work.** Reads the Pydantic models and writes ~23,000 lines of Python: 15 expression modules and 17 test modules | No: prints nothing on success | +| 4 | `ruff check --fix` | Lints the generated code and auto-fixes what it can, e.g. unused imports | No: `--quiet` | +| 5 | `ruff format` | Reformats the generated code to the project's style | No: `--quiet` | + +Steps 4 and 5 exist because generated code is still code that has to pass the repo's own +lint and format checks — `make check` runs `ruff` over everything, generated files +included. + +Every step is silent by design, which is why a successful run prints nothing at all. + +> **On an out-of-date `uv`** you'll instead see the `exclude-newer` banner three times — +> once each for steps 3, 4, and 5, the three visible `uv run` calls — and nothing else. That is still a successful run, but fix +> the `uv` problem before continuing: +> [uv warns about `exclude-newer`](#uv-warns-about-exclude-newer--and-quietly-rewrites-your-lockfile). + +#### Confirm it worked + +Don't infer it from the output — check the result: + +```bash +uv run python -c "from overture.schema.pyspark import model_names; print(model_names())" +``` + +Before — an empty list, no error, which is why this is easy to miss: + +``` +[] +``` + +After — 30 entries, which is 15 feature types each reachable by two names: + +``` +['address', 'bathymetry', 'building', 'building_part', 'connector', 'division', ...] +``` + +Or count the files directly: + +```bash +find packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated -name '*.py' | wc -l +find packages/overture-schema-pyspark/tests/generated -name '*.py' | wc -l +``` + +``` + 15 + 17 +``` + +#### Why two names per model? + +Because the registry has to stay correct when packages it has never heard of register +their own models. + +The **canonical key is the entry-point string** — `overture.schema.buildings:Building`. +It includes the module path, so it is globally unique: no two packages can collide. + +The **short name is a derived alias**. It is just the class name after the colon, +snake-cased: + +```python +from overture.schema.system.discovery.entry_point import entry_point_class_alias + +entry_point_class_alias("overture.schema.divisions:DivisionArea") # 'division_area' +entry_point_class_alias("overture.schema.places:Place") # 'place' +``` + +Short names are *not* guaranteed unique. Anyone can +[register their own feature types](#83-register-your-own-feature-types), and nothing +stops a third party from shipping its own `Place`. So the short name can't be the +identity — it's a convenience, because +`validate_model(df, "overture.schema.buildings:Building")` is miserable to type. + +**The nice part is how it degrades.** The alias is offered only while it stays +unambiguous. `model_names()` counts aliases and includes only those appearing once, and +the resolver tries an exact key match first, then the alias: + +```python +from overture.schema.system.discovery.entry_point import resolve_entry_point_key + +registry = {"overture.schema.places:Place": ..., "acme.parks:Place": ...} + +resolve_entry_point_key("place", registry) +# ValueError: Entry-point alias 'place' is ambiguous. +# Specify one of: acme.parks:Place, overture.schema.places:Place + +resolve_entry_point_key("acme.parks:Place", registry) # 'acme.parks:Place' — always works +``` + +Install a package that collides and `place` simply stops being accepted, with an error +naming both candidates — rather than silently validating against the wrong model. The +fully-qualified key never stops working. + +Two functions expose the two views: + +| Function | Returns | Use when | +|---|---|---| +| `model_keys()` | the 15 canonical entry-point keys | you want the authoritative list | +| `model_names()` | all 30 accepted names | you want everything `validate_model` will take | + +> **Skipping this step does not produce an error message.** It produces an empty +> registry. If `validate_model(df, "building")` raises a `KeyError`, or `model_names()` +> is empty, this is why. + +#### Why is generated code in `.gitignore`? + +Not because it's optional, and not because it's for a subset of users. **It's build +output.** The commit that introduced the package says so directly: + +> The generated trees under `expressions/generated/` and `tests/generated/` are +> regenerable output of `make generate-pyspark` and are not tracked in git; `make check` +> and `make test-all` regenerate before running. + +Three reasons that's the right call: + +**One source of truth.** The Pydantic models define the schema; these expressions are a +derivative of them. Committing the derivative creates a second copy that can silently +drift — change a constraint, forget to regenerate, and the committed expressions keep +enforcing the old rule. Deleting them from git makes that failure impossible. + +**Scale.** A full generation is **32 files and roughly 23,000 lines**: + +``` +15 expression modules (one per feature type) +17 test modules (conformance tests, split per union arm) +``` + +Every schema change would produce a mechanical diff of that size, burying the actual +change and guaranteeing merge conflicts. + +**The build regenerates regardless.** `make check` and `make test-all` both depend on +`generate-pyspark`, which begins with `clean-pyspark` (`rm -rf`). The tree is rebuilt +from the current models every time, so a committed copy would never be read. + +It's the same reasoning that keeps `dist/`, `*.o`, and `node_modules/` out of git. + +#### "Gitignored" does not mean "not shipped" + +This is the part worth being clear about, since it sounds like these files are somehow +optional for users. **They are not.** Published wheels contain them. + +`.github/workflows/publish-python-packages.yaml` runs `make generate-pyspark` before +`uv build`, then refuses to publish a wheel that lacks them: + +```yaml +- name: Generate PySpark expressions before build + if: matrix.package == 'overture-schema-pyspark' + run: make generate-pyspark +``` + +```bash +if [ "$PACKAGE" = "overture-schema-pyspark" ] && \ + ! unzip -l "$wheel" | grep -q 'expressions/generated/.*\.py'; then + echo " Wheel [$wheel] has no generated expressions -- codegen did not run. Aborting!" + exit 1 +fi +``` + +So the only people who ever run `make generate-pyspark` are people working **from a git +clone** — because a clone is the one place these files don't already exist. Install from +a package index and they arrive with the package, like any other module. + +--- + +### 1.6 When something goes wrong + +**This section is a reference, not a checklist.** Nothing here is setup you need to +perform. Each entry starts with a symptom — read the one matching an error you actually +saw, and skip the rest. If section 1.4 gave you clean output and +[Confirm it worked](#confirm-it-worked) checked out, you can skip the whole section and +go on to section 2. + +#### `FileNotFoundError: ... /apache-spark/3.5.3/libexec/./bin/spark-submit` + +``` +FileNotFoundError: [Errno 2] No such file or directory: +'/opt/homebrew/Cellar/apache-spark/3.5.3/libexec/./bin/spark-submit' +``` + +**This has nothing to do with the generation step in "Do you need PySpark?".** It's a stale `SPARK_HOME` +environment variable, and it will happen whether or not you ran `make generate-pyspark`. + +What's going on: you have a `SPARK_HOME` exported in your shell profile pointing at a +**version-specific Homebrew path**. Homebrew has since upgraded Spark, so that exact +directory no longer exists: + +```bash +echo $SPARK_HOME +ls /opt/homebrew/Cellar/apache-spark/ +``` + +``` +/opt/homebrew/Cellar/apache-spark/3.5.3/libexec +4.0.0 +``` + +The variable names `3.5.3`; the only version present is `4.0.0`. It points at nothing. + +Meanwhile the `pyspark` in your venv (4.2.0) **ships its own copy of Spark** and doesn't +need the Homebrew one at all. But when `SPARK_HOME` is set, PySpark obeys it and looks +for `spark-submit` at that dead path. + +**Confirm that's your problem:** + +```bash +env -u SPARK_HOME uv run python -c " +from pyspark.sql import SparkSession +s = SparkSession.builder.master('local[1]').getOrCreate() +print('SUCCESS — spark', s.version) +s.stop()" +``` + +``` +SUCCESS — spark 4.2.0 +``` + +**Fix it permanently.** The variable is set in *two* files — fixing only one won't help, +because `.zprofile` runs for login shells and `.zshrc` for interactive ones: + +``` +~/.zshrc:8 export SPARK_HOME=/opt/homebrew/Cellar/apache-spark/3.5.3/libexec +~/.zshrc:9 export PATH="$SPARK_HOME/bin/:$PATH" +~/.zprofile:5 export SPARK_HOME=/opt/homebrew/Cellar/apache-spark/3.5.3/libexec +~/.zprofile:6 export PATH="$SPARK_HOME/bin/:$PATH" +``` + +Pick one: + +- **Simplest — delete all four lines.** If you only use Spark through Python projects + like this one, you don't need `SPARK_HOME` at all; each venv's `pyspark` brings its + own. +- **Keep Homebrew Spark for other work — stop hardcoding the version.** Replace the two + `SPARK_HOME` lines with the version-independent symlink Homebrew maintains: + + ```bash + export SPARK_HOME=/opt/homebrew/opt/apache-spark/libexec + ``` + + This survives upgrades. But note it points at Spark **4.0.0** while this project's + `pyspark` is **4.2.0** — mismatched versions cause their own confusing failures, so + prefer the first option while working in this repo. + +Then open a new terminal, or `exec zsh`, and re-run the check above. + +> **Per-shell workaround** if you don't want to touch your profile right now: +> `unset SPARK_HOME` in the terminal you're working in. It lasts until you close it. + +#### `model_names()` returns `[]`, or `KeyError` on a feature type + +You skipped [Do you need PySpark?](#15-do-you-need-pyspark). Run `make generate-pyspark`. + +#### `ModuleNotFoundError: No module named 'overture'` + +You started Python without `uv run`, so you're in your system or `pyenv` interpreter +rather than the project's `.venv`. Use `uv run python` instead of `python`. See +[Running Python against the models](#21-running-python-against-the-models). + +#### `command not found: overture-schema` + +You're missing the `uv run` prefix, or you're not in the repo root. Every command in this +guide is `uv run overture-schema …`, run from the directory containing `pyproject.toml`. + +#### uv warns about `exclude-newer` — and quietly rewrites your lockfile + +``` +warning: Failed to parse `pyproject.toml` during settings discovery: + TOML parse error at line 10, column 17 + | + 10 | exclude-newer = "1 week" + | ^^^^^^^^ + failed to parse year in date "1 week": failed to parse "1 we" as year ... +``` + +> **Do you actually have this problem?** Only if that banner appears when you run a +> command. If your commands print their output cleanly, skip this entire subsection — +> there is nothing to fix, and the four steps below are not maintenance you need to +> perform. Confirm in one line: +> +> ```bash +> uv run overture-schema --version +> ``` +> +> A single line of output means you're fine. A wall of warning text above it means read on. + +**If you do see it: this is not cosmetic. Fix it before you do anything else.** + +The root `pyproject.toml` writes `exclude-newer` as a relative duration (`"1 week"`), +which caps how new a package `uv` will consider during dependency resolution. Only +reasonably recent `uv` versions parse that form. + +An older `uv` fails to read the whole `[tool.uv]` block, says so, **and carries on +without the cap** — then, because its resolution no longer matches the committed +lockfile, rewrites `uv.lock` in place. You end up with a modified tracked file you never +asked to change: + +```bash +git status --short uv.lock +git diff --stat uv.lock +``` + +On an affected machine: + +``` + M uv.lock + uv.lock | 807 +++++++++++++++++++------------------- + 1 file changed, 464 insertions(+), 343 deletions(-) +``` + +> **Both commands printing nothing is the healthy result.** `git status --short` and +> `git diff --stat` say nothing about a file that hasn't changed. Empty output here means +> your lockfile is untouched and there is nothing to fix. If you'd rather have an explicit +> answer than read silence: +> +> ```bash +> git diff --quiet uv.lock && echo "uv.lock: unmodified" || echo "uv.lock: MODIFIED" +> ``` + +It drops the `[options]` block that records the resolution settings, and pulls in +dependency versions past the cutoff the repo intended. Nothing breaks immediately — the +install works fine — but you're now building against a different dependency set than the +project pinned, and `git status` is dirty. + +**Step 1 — check your version:** + +```bash +uv --version +brew outdated uv +``` + +**Step 2 — upgrade:** + +```bash +brew upgrade uv +``` + +**Step 3 — restore the lockfile if the old `uv` already rewrote it:** + +```bash +git checkout uv.lock +``` + +**Step 4 — confirm:** + +```bash +uv sync --all-packages --locked +``` + +``` +Resolved 64 packages in 12ms +Audited 60 packages in 0.81ms +``` + +`--locked` fails outright if the lockfile isn't authoritative, so a clean pass means your +`uv`, the lockfile, and your `.venv` all agree. Run `git status --short uv.lock` once more +too — it should print nothing, which means the file is unmodified. + +> **If you see `error: The lockfile at uv.lock needs to be updated, but --locked was +> provided`,** you're at step 3, not step 4. A previous `uv sync` under the old `uv` +> already modified the lock. `git checkout uv.lock` and re-run. + +Once you're on a current `uv`, ordinary use leaves the lockfile alone — including the +`uv sync --all-packages --all-extras` that `make install`, `make check`, and +`make generate-pyspark` all run internally. + +--- + +## 2. Exploring the models + +You can't write code against a model you haven't looked at. This section is about finding +out what feature types exist, what fields they carry, and what values those fields +accept — before writing a line of code against them. + +**Two things can answer your questions, and it's worth keeping them straight:** + +| | What it is | Ask it with | +|---|---|---| +| **The model** | The Pydantic classes themselves — the source of truth, and what actually validates your data | Python: `Building.model_fields` | +| **The generated JSON Schema** | An artifact *rendered from* the model, in the GeoJSON shape | `overture-schema json-schema` + `jq` | + +They agree, because one is generated from the other. The model is flatter and easier to +interrogate; the JSON Schema is the published contract and the thing other tools consume. +This section asks the model first, then the schema. + +### 2.1 Running Python against the models + +Everything up to now has been shell commands. From here the guide switches to Python, so +first: how do you actually run it? + +**The models are installed in the project's `.venv`, not in whatever `python` your shell +finds.** Starting Python the usual way fails: + +```bash +python +``` +``` +>>> from overture.schema.buildings import Building +Traceback (most recent call last): + File "", line 1, in +ModuleNotFoundError: No module named 'overture' +``` + +`ModuleNotFoundError: No module named 'overture'` always means this: right code, wrong +interpreter. Compare the two: + +```bash +python -c "import sys; print(sys.executable)" +uv run python -c "import sys; print(sys.executable)" +``` + +``` +/Users/you/.pyenv/versions/3.10.15/bin/python +/path/to/schema/.venv/bin/python3 +``` + +The first is your system or `pyenv` Python, which has never heard of these packages. The +second is the project's environment, where `uv sync` installed them. Nothing is broken — +they're simply two different Pythons. + +Prefix with `uv run` and it works — that runs Python inside the project's environment: + +```bash +uv run python -c "from overture.schema.buildings import Building; print(Building.__name__)" +``` +``` +Building +``` + +Three ways to run the Python in this guide, all from the repo root: + +**A one-liner**, for a quick look: + +```bash +uv run python -c "from overture.schema.buildings import Building; print(len(Building.model_fields))" +``` + +**An interactive session**, best for exploring — you can poke at a model, tab-complete, +and try things: + +```bash +uv run python +``` + +``` +Python 3.10.18 +>>> from overture.schema.buildings import Building +>>> len(Building.model_fields) +26 +``` + +Press Ctrl-D to exit. + +**A script file**, once you're writing more than a couple of lines: + +```bash +uv run python explore.py +``` + +Unless a snippet is explicitly marked as shell, every Python block from here on is code +to run one of these three ways. An interactive session is the best fit for section 2 — +you're looking things up, not building anything yet. + +> You can also activate the environment once (`source .venv/bin/activate`) and then use +> plain `python`. This guide uses `uv run` throughout because it always works, needs no +> setup, and cannot be left half-done. + +### 2.2 Where the models live + +Before you can explore anything, you need to know where it lives. The Python module +path drops the `theme-` prefix: + +| Package | Import from | +|---|---| +| `overture-schema-theme-buildings` | `overture.schema.buildings` | +| `overture-schema-theme-transportation` | `overture.schema.transportation` | +| `overture-schema-theme-places` | `overture.schema.places` | +| `overture-schema-theme-divisions` | `overture.schema.divisions` | +| `overture-schema-theme-addresses` | `overture.schema.addresses` | +| `overture-schema-theme-base` | `overture.schema.base` | +| `overture-schema-common` | `overture.schema.common` | +| `overture-schema-system` | `overture.schema.system` | +| `overture-schema-validation` | `overture.schema.validation` | + +```python +from overture.schema.buildings import Building, BuildingClass +from overture.schema.transportation import Segment, Connector, RoadClass +from overture.schema.places import Place +``` + +### 2.3 From the CLI: what types exist? + +```bash +uv run overture-schema list-types +``` + +``` +address feature overture:theme=addresses +bathymetry feature overture:theme=base +building feature overture:theme=buildings +building_part feature overture:theme=buildings +connector feature overture:theme=transportation +division feature overture:theme=divisions +division_area feature overture:theme=divisions +division_boundary feature overture:theme=divisions +infrastructure feature overture:theme=base +land feature overture:theme=base +land_cover feature overture:theme=base +land_use feature overture:theme=base +place feature overture:theme=places +segment feature overture:theme=transportation +water feature overture:theme=base +``` + +Columns are: **type name**, then its **tags**. Group by a tag key: + +```bash +uv run overture-schema list-types --group-by overture:theme +``` + +``` +overture:theme=buildings (2) +→ building feature overture:theme=buildings +→ building_part feature overture:theme=buildings +... +``` + +Filter with the tag options, which are shared by `list-types`, `validate`, and +`json-schema`: + +| Option | Semantics | +|---|---| +| `--tag T` | OR — defines scope. Repeatable. | +| `--filter T` | AND — every listed tag must be present. Repeatable. | +| `--exclude T` | OR-NOT — any match drops the type. Repeatable. | + +Tag format is `[namespace:]predicate[=value]`: + +- plain — `feature` +- namespaced — `system:extension` +- key/value — `overture:theme=buildings` + +```bash +uv run overture-schema list-types --tag overture:theme=buildings --tag overture:theme=places +``` + +### 2.4 Ask the model itself (Python) + +This is the primary source. `Building` is an ordinary Python class, and Pydantic gives +every model a `model_fields` dict describing each field — its type, whether it's required, +its documentation, and its constraints. Nothing is generated or rendered here; this *is* +the schema. + +Start a session and look at what you have: + +```bash +uv run python +``` + +```python +>>> from overture.schema.buildings import Building +>>> len(Building.model_fields) +26 +>>> sorted(n for n, f in Building.model_fields.items() if f.is_required()) +['geometry', 'id', 'theme', 'type', 'version'] +``` + +Twenty-six fields, five of them required. Note the shape of that list: `id` and `geometry` +sit alongside the rest, with **no nesting and no envelope**. A model is flat. (If you've +seen Overture data as GeoJSON with things tucked under `properties`, that's a +serialization format, not the model — [2.5](#25-ask-the-generated-json-schema-cli--jq) +covers why they differ.) + +#### Every field at a glance + +```python +from overture.schema.buildings import Building + +print(Building.__doc__) + +for name, f in Building.model_fields.items(): + flag = "required" if f.is_required() else "optional" + print(f"{name:24} {flag:9} {f.annotation}") +``` + +``` +height optional typing.Optional[overture.schema.system.numeric.float64] +is_underground optional bool | None +num_floors optional typing.Optional[overture.schema.system.numeric.int32] +... +id required overture.schema.system.ref.id.Id +geometry required +theme required typing.Literal['buildings'] +type required typing.Literal['building'] +version required overture.schema.common.feature.FeatureVersion +class_ optional overture.schema.buildings.building.BuildingClass | None +``` + +#### One field in detail + +Each entry carries the documentation and constraints from the model declaration: + +```python +f = Building.model_fields["height"] +print(f.description) # 'Height of the building or part in meters.\n\n...' +print(f.metadata) # [Gt(gt=0)] +print(f.alias) # None +print(Building.model_fields["class_"].alias) # 'class' +``` + +Note `class_` and its alias `class`. `class` is a Python keyword, so the field is named +`class_` on the model and `class` in the data — a mismatch that bites when serializing. +See [Gotchas](#12-gotchas). + +This is the same information [2.5](#25-ask-the-generated-json-schema-cli--jq) reads out +of the JSON Schema, minus the envelope — `f.description` is the `description` keyword, +`f.metadata` of `[Gt(gt=0)]` is `exclusiveMinimum: 0`, and `f.is_required()` is +membership in a `required` array. + +### 2.5 Ask the generated JSON Schema (CLI + jq) + +Dump the JSON Schema for one type. Save it once rather than re-running the command for +every question: + +```bash +uv run overture-schema json-schema --type building > building.schema.json +``` + +#### Two kinds of key + +Before walking the nesting, one distinction that makes the rest obvious. Everything in a +JSON Schema document is a JSON object, so `jq keys` will happily list any level — but +what it lists alternates between two completely different kinds of name. + +**Schema keywords** are vocabulary defined by the JSON Schema specification. They are +instructions to a validator, and the set is fixed — you could not invent a new one. Their +values describe the document: + +```bash +jq 'keys' building.schema.json +``` +```json +["$defs","additionalProperties","description","properties","required","title","type"] +``` + +``` +.type = "object" +.title = "building" +.description = "Buildings are man-made structures with roofs..." +.required = ["type","id","geometry","properties"] +.additionalProperties = false +``` + +Read that as prose: *this is an object, called `building`, described like so, these four +fields are mandatory, and no others are allowed.* + +**Field names** are names that appear in actual data. They are not vocabulary — they come +from Overture and GeoJSON, and every one of them maps to a *subschema*: another little +JSON Schema object describing that one field. + +```bash +jq '.properties | keys' building.schema.json +``` +```json +["bbox","geometry","id","properties","type"] +``` + +``` +.properties.type = {"const":"Feature","type":"string"} +.properties.id = {"description":"A feature ID...", ...} +.properties.geometry = {"description":"The building's footprint...", ...} +``` + +Those values aren't descriptions of the document — they're rules for one field each. +`.properties.type` says: *the data field named `type` must be the string `Feature`.* + +So "keys" is technically accurate for both lists and useless for telling them apart. The +first list is **keywords**; the second is **field names**. The keyword `properties` is the +gate between them: everything under it is data field names, and each of those opens into +a subschema made of keywords again. + +You can see both roles inside a single value: + +```json +{"const": "Feature", "type": "string"} +``` + +The key `type` that got you here was a *field name*. The `type` inside the value is a +*keyword* meaning "JSON string." Same word, opposite roles, one level apart. + +#### Mind the nesting + +With that distinction, the path to the Overture fields reads cleanly. `.properties` is a +keyword, so its contents are field names — and they're GeoJSON's, the same envelope from +[Three keys named `type`](#three-keys-named-type): + +```bash +jq '.properties | keys' building.schema.json +``` +```json +["bbox","geometry","id","properties","type"] +``` + +One of those field names is itself `properties` — the GeoJSON member holding the Overture +fields. Descend into it, then through *its* `properties` keyword: + +```bash +jq '.properties.properties.properties | keys' building.schema.json +``` +```json +["class","facade_color","facade_material","has_parts","height","is_underground", + "level","min_floor","min_height","names","num_floors","num_floors_underground", + "roof_color","roof_direction","roof_height","roof_material","roof_orientation", + "roof_shape","sources","subtype","theme","type","version"] +``` + +Three `properties` in a row, alternating role each time: + +| Path segment | Kind | Meaning | +|---|---|---| +| `.properties` | keyword | "the fields of the GeoJSON object" | +| `.properties.properties` | field name | the GeoJSON field *named* `properties` | +| `.properties.properties.properties` | keyword | "the fields inside that one" | + +Note `id`, `geometry`, and `bbox` are absent from the final list. They live on the +envelope, at `.properties.id` and so on — exactly as they do in the data. + +#### That JSON object is not the field's value + +A reasonable reading of this is "height is an object": + +```json +{ + "description": "Height of the building or part in meters.\n\nThis is the distance from the lowest point to the highest point.", + "exclusiveMinimum": 0, + "title": "Height", + "type": "number" +} +``` + +It isn't. **In the data, `height` is a plain number.** What you're looking at is the +*subschema* — the rules for `height`, not a value of `height`. And `"type": "number"` is +the keyword saying so. + +The actual data looks like this: + +```json +{"height": 21.34} +``` + +Every field's subschema is a JSON object, no matter how simple the field is, because +that's the only place to hang a description and a constraint. You can't attach +"must be greater than zero" to a bare `"number"`. + +Each keyword traces back to one thing in the Python model — +`packages/overture-schema-theme-buildings/src/overture/schema/buildings/_common.py`: + +```python +height: Annotated[ + float64 | None, + Field( + gt=0, + description=textwrap.dedent(""" + Height of the building or part in meters. + + This is the distance from the lowest point to the highest point. + """).strip(), + ), +] = None +``` + +| Schema keyword | Comes from | +|---|---| +| `"type": "number"` | the `float64` annotation | +| `"exclusiveMinimum": 0` | `gt=0` — greater than, not greater-or-equal | +| `"description"` | the `description=` argument | +| `"title": "Height"` | generated by Pydantic from the field name | +| *absent from `required`* | the `= None` default | + +Nobody hand-wrote that JSON. It's a rendering of the Python declaration, which is why the +JSON Schema, the validation errors, the PySpark checks, and the generated documentation +can't drift from each other — [section 6](#6-converting-the-schema-to-other-formats) is +all the other renderings of the same source. + +#### Why a wrong path gives you `null`, not an error + +Ask `jq` for a key that isn't there and it returns `null`. That is an answer, not a +failure — `null` means "no such key at this path": + +```bash +jq '.properties.height' building.schema.json +jq '.properties.banana' building.schema.json +``` +``` +null +null +``` + +`height` isn't missing from the schema; it just isn't on the *envelope*, which only has +`bbox`, `geometry`, `id`, `properties`, and `type`. A `null` here almost always means you +stopped one level too high. + +Two ways to make that louder. `jq -e` sets the exit status — `1` for `null`, `0` for a +real result, useful in scripts: + +```bash +jq -e '.properties.height' building.schema.json > /dev/null; echo $? +jq -e '.properties.properties.properties.height' building.schema.json > /dev/null; echo $? +``` +``` +1 +0 +``` + +Or stop guessing at the nesting and let `jq` find the field for you: + +```bash +jq -c 'paths | select(.[-1]=="height")' building.schema.json +``` +```json +["properties","properties","properties","height"] +``` + +That prints the exact path to any field, which you can then read directly. Handy whenever +a lookup comes back `null` and you're not sure how deep the thing actually lives. + +#### Asking useful questions + +Which fields are mandatory — and here the envelope bites. There is no single `required` +list. There are **two**, one per level: + +```bash +jq '.required' building.schema.json +jq '.properties.properties.required' building.schema.json +``` +```json +["type","id","geometry","properties"] +["theme","type","version"] +``` + +Read them together: + +| Required | Where | What it is | +|---|---|---| +| `type` | envelope | GeoJSON's own `"type": "Feature"` | +| `id` | envelope | the feature's ID | +| `geometry` | envelope | the shape | +| `properties` | envelope | the container itself must be present | +| `theme` | in `properties` | `buildings` | +| `type` | in `properties` | `building` | +| `version` | in `properties` | the feature version | + +So **`id` and `geometry` are absolutely required** — they're just not in the array you'd +find by looking only under `properties`, because in GeoJSON they don't live under +`properties`. Asking `.properties.properties.required` and reading it as "the required +fields of a building" undercounts by exactly the fields the envelope owns. + +Prove it by deleting them: + +```bash +uv run python <<'PY' +import json, yaml +from overture.schema.buildings import Building + +for drop in ["geometry", "id"]: + d = yaml.safe_load(open("examples/buildings/building-polygon.yaml")) + del d[drop] + try: + Building.model_validate_json(json.dumps(d)) + print(f"dropping {drop}: ACCEPTED") + except Exception as e: + print(f"dropping {drop}:", str(e).splitlines()[2].strip()[:30]) +PY +``` +``` +dropping geometry: Field required +dropping id: Field required +``` + +**The model is the easier place to ask this question**, because it has no envelope — one +flat answer, all five: + +```bash +uv run python -c " +from overture.schema.buildings import Building +print(sorted(n for n, f in Building.model_fields.items() if f.is_required()))" +``` +``` +['geometry', 'id', 'theme', 'type', 'version'] +``` + +There are other `required` arrays deeper in the schema too — every nested structure has +its own. `Names`, `SourceItem`, and the two `geometry` variants each carry one: + +```bash +jq -c 'paths | select(.[-1]=="required")' building.schema.json +``` +```json +["$defs","NameRule","required"] +["$defs","Names","required"] +["$defs","Perspectives","required"] +["$defs","SourceItem","required"] +["properties","geometry","oneOf",0,"required"] +["properties","geometry","oneOf",1,"required"] +["properties","properties","not","required"] +["properties","properties","required"] +["required"] +``` + +`required` is always relative to the object it sits in — never a global list for the +feature. + +#### Who decides what's required? + +Nobody maintains that list. **It's derived** — in Pydantic, a field with no default is +required, and a field with a default is optional: + +```bash +uv run python <<'PY' +from overture.schema.buildings import Building +from pydantic_core import PydanticUndefined + +for n in ["version", "theme", "height", "num_floors"]: + f = Building.model_fields[n] + d = "no default" if f.default is PydanticUndefined else f"default={f.default!r}" + print(f"{n:12} {d:16} -> {'REQUIRED' if f.is_required() else 'optional'}") +PY +``` +``` +version no default -> REQUIRED +theme no default -> REQUIRED +height default=None -> optional +num_floors default=None -> optional +``` + +So "who decided" is answered by finding where the field is declared. For a building, five +fields are required, and four of them come from the shared base class rather than from +buildings at all: + +```bash +uv run python <<'PY' +from overture.schema.buildings import Building + +for n, f in Building.model_fields.items(): + if not f.is_required(): + continue + for cls in Building.__mro__: + if n in getattr(cls, "__annotations__", {}): + print(f"{n:10} -> {cls.__name__}") + break +PY +``` +``` +id -> OvertureFeature +geometry -> Building +theme -> OvertureFeature +type -> OvertureFeature +version -> OvertureFeature +``` + +`id`, `theme`, `type`, and `version` are required of *every* Overture feature — they're +declared once in `OvertureFeature` +(`packages/overture-schema-common/src/overture/schema/common/feature.py`): + +```python +id: Id = Field(description="A feature ID. ...") +theme: ThemeT +type: TypeT +# Superclass `Feature` provides `geometry` and `bbox`. +version: FeatureVersion +``` + +No `= None`, so all four are mandatory. `Building` adds only `geometry`, narrowing the +inherited one to the polygon types a building may have. + +The two `required` arrays in the JSON Schema split those five across the GeoJSON envelope +— `id` and `geometry` sit at `.required`, while `theme`, `type`, and `version` sit at +`.properties.properties.required`, since that's where they live in the data. + +**As for the human answer:** the Overture Schema Working Group decides, and changes go +through the process in `CONTRIBUTING.md` — a PR plus a changelog fragment. Making a field +required is a breaking change, so it targets the `vnext` branch and waits for a major +release; making one optional is not, and can go to `main`. + +One field's type, constraints, and documentation: + +```bash +jq '.properties.properties.properties.height' building.schema.json +``` +```json +{ + "description": "Height of the building or part in meters.\n\nThis is the distance from the lowest point to the highest point.", + "exclusiveMinimum": 0, + "title": "Height", + "type": "number" +} +``` + +By **contract** I mean the set of rules a validator will enforce — what a producer must +satisfy and what a consumer may therefore rely on. For `height` that's: it must be a +number, and it must be strictly greater than zero. + +But "the contract for `height`" is narrower than "everything true about `height`", in three +ways worth knowing before you rely on it. + +**Optionality isn't in there.** Whether `height` may be omitted is recorded in a sibling +`required` array, not in the field's own subschema. A subschema describes the value *if +present*. + +**Units are documentation, not a rule.** "in meters" appears in `description` — prose for +humans. Nothing rejects a value recorded in feet. The machine-checkable part is only +`type: number` and `exclusiveMinimum: 0`. + +**Relationships between fields are mostly absent.** The schema *can* express cross-field +rules, and elsewhere it does — `@require_any_of`, `@forbid_if` and friends from +the `@require_any_of` / `@forbid_if` decorators in `overture-schema-system`. But no such rule ties `height` to +`min_height`, so this passes: + +```bash +uv run python <<'PY' +import json, yaml +from overture.schema.buildings import Building + +d = yaml.safe_load(open("examples/buildings/building-polygon.yaml")) +d["properties"]["height"] = 5 +d["properties"]["min_height"] = 100 # a floor above the roof +print("accepted:", Building.model_validate_json(json.dumps(d)).height) +PY +``` + +``` +accepted: 5.0 +``` + +A building whose lowest point is 100m and whose highest is 5m is physically impossible and +schema-valid. Both fields satisfy their own contracts; nothing checks them against each +other. + +So: a subschema is the complete machine-checkable contract for **one field in isolation**. +It is not a guarantee that the data makes sense. Same lesson as the stray comma in +[See it actually catch something](#see-it-actually-catch-something) — validation enforces +the schema, not correctness. + +Enums and shared structures live at the top level under `$defs`, not nested: + +```bash +jq -r '.["$defs"] | keys[]' building.schema.json +``` +``` +BuildingClass BuildingSubtype FacadeMaterial NameRule NameVariant Names +PerspectiveMode Perspectives RoofMaterial RoofOrientation RoofShape Side SourceItem +``` + +So the full list of valid values for a field, without reading Python source: + +```bash +jq -r '.["$defs"].BuildingClass.enum[]' building.schema.json | head +``` +``` +agricultural +allotment_house +apartments +barn +beach_hut +boathouse +bridge_structure +bungalow +``` + +> **If the nesting is annoying, skip to Python.** `Building.model_fields` in +> [2.4](#24-ask-the-model-itself-python) gives you the same field list flat, with no +> envelope to walk through. The JSON Schema route is most useful when you want the exact +> published contract — or when you're feeding it to another tool, as in +> [section 8](#8-building-your-own-sdk-or-cli). + +#### Wait — why is there a GeoJSON envelope at all? + +If the schema is now Pydantic, why does any of this still look like GeoJSON? Because those +two things answer different questions, and only one of them changed. + +| | What it is | Did it change? | +|---|---|---| +| **Pydantic** | How the schema is *authored and enforced*, in Python | **Yes** — it replaced hand-written JSON Schema YAML | +| **GeoJSON** | One way a feature can be *written out as JSON*, per RFC 7946 | **No** — it's an interchange format, not an authoring choice | + +Pydantic replaced JSON Schema as the authoring language. It has nothing to say about how +data is serialized, so GeoJSON was never in scope to replace. + +**But GeoJSON is not how Overture ships bulk data — Parquet is.** The release bucket is +Hive-partitioned Parquet: + +``` +s3a://overturemaps-us-west-2/release/2026-07-22.0/theme=buildings/type=building/ +``` + +That is a columnar table: `id`, `geometry`, `height`, and the rest are *columns*, flat, no +envelope. It's what `overture-validate` reads, what DuckDB attaches to, and what you'd +query for anything at scale. + +So why does GeoJSON appear at all? Because **JSON Schema describes JSON documents**, and +when a geospatial feature is written as a single JSON document, GeoJSON is the format +every GIS tool already reads. Parquet has no JSON representation to describe — it has a +columnar schema instead, which is exactly what +[section 6](#6-converting-the-schema-to-other-formats) generates as a Spark `StructType`. + +The honest summary is that there are two serializations and neither is subordinate: + +| Serialization | Shape | Pydantic mode | Where you meet it | +|---|---|---|---| +| **Parquet** | flat / columnar | `python` | the release bucket, Spark, DuckDB — all bulk data | +| **GeoJSON** | nested envelope | `json` | single features, extracts, examples in this repo, web tooling | + +The model supports both deliberately. The JSON Schema you're reading in this subsection +describes the second one, which is the only reason an envelope shows up here at all. + +#### "Interchange format" — what that actually means + +An **interchange format** is one whose job is handing data to a program you didn't write. +It's optimized for being *understood by anything*, not for being stored efficiently. A +**storage format** is the opposite: optimized for holding a lot of data and querying it +fast, at the cost of needing specific software to read it at all. + +| | GeoJSON | Parquet | +|---|---|---| +| Optimized for | being read by anything | storing and scanning millions of rows | +| Text or binary | plain text | binary, columnar, compressed | +| Read it with | any JSON parser | a Parquet library | +| Self-describing | yes — the file says what it is | schema in the footer, not human-readable | +| Good at | one feature, an extract, a web map | a whole theme, a whole planet | + +The cost of being universally readable is that GeoJSON repeats every field name on every +feature: + +```bash +uv run python <<'PY' +import json, yaml +from overture.schema.buildings import Building + +b = Building.model_validate_json( + json.dumps(yaml.safe_load(open("examples/buildings/building-polygon.yaml")))) +gj = b.model_dump(mode="json", by_alias=True, exclude_none=True) +flat = b.model_dump(mode="python", by_alias=True, exclude_none=True) +flat["geometry"] = str(flat["geometry"]) + +n = 10_000 +doc = json.dumps({"type": "FeatureCollection", "features": [gj] * n}) +cols = list(flat) +tbl = json.dumps({"columns": cols, "rows": [[flat[k] for k in cols]] * n}) +print(f"{n:,} features as GeoJSON : {len(doc):>10,} bytes") +print(f"{n:,} features, names stored once: {len(tbl):>10,} bytes") +PY +``` + +``` +10,000 features as GeoJSON : 7,040,043 bytes +10,000 features, names stored once: 4,520,199 bytes +``` + +A 1.56x penalty before compression even enters the picture, and that comparison is still +generous to GeoJSON — real Parquet also compresses each column and lets a reader skip +columns it doesn't need. Multiply by a planet's worth of buildings and the reason bulk +data isn't shipped as GeoJSON is obvious. + +"Interchange" is a role, not a ranking. GeoJSON is the right tool for handing one feature +to a web map; Parquet is the right tool for handing a continent to Spark. + +#### Is Pydantic wrapped around GeoJSON? + +Short answer: **no.** But the question has three reasonable readings, and one of them is a +qualified yes, so it's worth taking them separately. + +**"Is the model built on top of a GeoJSON structure?"** No. A model is a flat list of +fields, declared one at a time in Python. You can build one and use it without JSON ever +entering the picture: + +```bash +uv run python <<'PY' +from overture.schema.buildings import Building +from overture.schema.system.geometric import Geometry + +b = Building( + id="my-building-1", + geometry=Geometry.from_wkt("POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))"), + theme="buildings", + type="building", + version=1, + height=12.5, +) +print(b.id, b.height) +print(sorted(b.model_dump(mode="python", by_alias=True, exclude_none=True))) +PY +``` + +``` +my-building-1 12.5 +['geometry', 'height', 'id', 'level', 'theme', 'type', 'version'] +``` + +No GeoJSON was parsed, produced, or consulted. If GeoJSON were the substrate, that +wouldn't be possible. + +**"Is there GeoJSON code inside the Pydantic classes?"** Yes — in exactly one of them. +Counting mentions across everything `Building` inherits from: + +``` +Building (building ) 0 +OvertureFeature (feature ) 0 +Identified (id ) 0 +Feature (feature ) 17 <- the base class in overture-schema-system +Named (names ) 0 +Stacked (level ) 0 +Appearance (_common ) 0 +``` + +All of it lives in `Feature`, in one serializer and one validator — the code that reads +GeoJSON in and writes GeoJSON out. Zero mentions in the six classes that actually define +what a building *is*. GeoJSON is an I/O concern parked at the base of the hierarchy, not a +structure the schema is built on. + +**"Is GeoJSON what's really being validated?"** No. What gets validated is the model. A +GeoJSON document is one accepted *input shape* — a flat dict is the other, and both end up +as the same Python object. + +An analogy: a word processor's document isn't "wrapped around `.docx`." It has a document +model, and it can read and write `.docx`. Deleting that import/export code would not +change what a document is. Same here — delete the ten lines below and Overture models +still work; they just stop speaking GeoJSON. + + +The entire GeoJSON transformation is those ten lines, in `Feature` +(`packages/overture-schema-system/src/overture/schema/system/feature.py`): + +```python +@model_serializer(mode="wrap") +def __serialize_with_geo_json_support__(self, serializer, info): + data = serializer(self) # <- the flat dict, produced first + + if info.mode == "json": # <- only in JSON mode + return { + "type": "Feature", + **({"id": data.pop("id")} if "id" in data else {}), + **({"bbox": data.pop("bbox")} if "bbox" in data else {}), + "geometry": data.pop("geometry"), + "properties": data, # <- everything else goes here + } + + return data # <- Python mode: flat, untouched +``` + +Read the first line: Pydantic produces the **flat** dictionary, and only then does this +function move `id`, `bbox`, and `geometry` to the top and sweep the remainder into +`properties`. In `python` mode the flat dict is returned unchanged and none of this runs. + +So the layering is: + +``` + Pydantic model (flat — the actual schema) + | + +----------+----------+ + | | + python mode json mode + | | + flat dict GeoJSON envelope + -> Parquet -> .geojson +``` + +GeoJSON is a costume the model puts on for one specific audience. It isn't the body. + +**The Pydantic model itself has no envelope.** In Python it's completely flat: + +```bash +uv run python <<'PY' +from overture.schema.buildings import Building +print("id in model_fields :", "id" in Building.model_fields) +print("geometry in model_fields :", "geometry" in Building.model_fields) +print("a 'properties' field? :", "properties" in Building.model_fields) +PY +``` + +``` +id in model_fields : True +geometry in model_fields : True +a 'properties' field? : False +``` + +There is no `properties` field on `Building`, and `id` and `geometry` sit alongside +`height` and `num_floors` like any other field. The envelope is **not part of the model**. +It appears only when serializing to JSON, because that is what GeoJSON requires. + +You can watch the same object take both shapes: + +```bash +uv run python <<'PY' +import json, yaml +from overture.schema.buildings import Building + +b = Building.model_validate_json( + json.dumps(yaml.safe_load(open("examples/buildings/building-polygon.yaml")))) + +print("python mode:", sorted(b.model_dump(mode="python", by_alias=True, exclude_none=True))[:8]) +print("json mode :", sorted(b.model_dump(mode="json", by_alias=True, exclude_none=True))) +PY +``` + +``` +python mode: ['class', 'ext_bar', 'ext_foo', 'geometry', 'height', 'id', 'is_underground', 'level'] +json mode : ['geometry', 'id', 'properties', 'type'] +``` + +One model, two renderings — and the flat one is the shape of the data you'd actually +download. Neither is more "real"; the model is what's real, and both are projections of +it. + +**So which answer to "what's required" is correct?** The model's: + +``` +['geometry', 'id', 'theme', 'type', 'version'] +``` + +The JSON Schema's two `required` arrays are that same list, split across the envelope +because that's where those fields land *in that particular output format*. Nobody decided +`geometry` belongs somewhere different from `theme`; GeoJSON did, in 2016. + +This distinction is the single most important thing in this guide, and it returns in force +in [section 3](#3-writing-code-against-the-models) — where using the wrong mode for your +data shape is the most common way to get a confusing `ValidationError`. + +#### Why `jq` for this + +`jq` isn't required — you could parse the schema in Python. It suits *poking around* +specifically, for four reasons: + +**Paths mirror the structure.** JSON is a tree and `jq` is a query language for trees, so +`.properties.properties.properties.height` reads exactly like the nesting it walks. You +compose a path left to right instead of writing traversal code. + +**It answers "what is this?", not just "give me X".** Most JSON tools retrieve a value you +already know the name of. `jq` has structure-discovery built in: + +```bash +jq 'keys' building.schema.json +jq -c 'paths | select(.[-1]=="height")' building.schema.json +``` +```json +["$defs","additionalProperties","description","properties","required","title","type"] +["properties","properties","properties","height"] +``` + +That second one finds a field wherever it lives — you don't have to know the shape first. +That is the difference between a query tool and an exploration tool. + +**It's a filter, so it composes.** Data in, data out — pipe it, redirect it, chain it with +`head`, feed one query's output to another. And because the output is JSON, `jq` composes +with itself. + +**Zero setup.** No file to create, no imports, no session to keep alive. One line in the +shell you're already in. + +It also reshapes, which is useful for scanning a lot at once — every field with its type, +as a table: + +```bash +jq -r '.properties.properties.properties + | to_entries[] + | "\(.key)\t\(.value.type // .value["$ref"] // "?")"' building.schema.json +``` +``` +class #/$defs/BuildingClass +facade_color string +facade_material #/$defs/FacadeMaterial +has_parts boolean +height number +is_underground boolean +``` + +**Where it's the wrong tool.** `jq` is a whole separate language, its error messages are +terse, and — as the `null` above shows — it answers a wrong path with a shrug rather than +a complaint. For *this* schema, Python introspection is usually easier: the field list +comes out flat, with no GeoJSON envelope to walk. Use `jq` when you want the published +contract exactly as it ships, or when you're piping it into another tool. Use Python when +you want to understand the model. That's the next subsection. + +### 2.6 From Python: enumerate everything that's installed + +```python +from overture.schema.system.discovery import discover_models + +for key, model in sorted(discover_models().items(), key=lambda kv: kv[0].name): + print(f"{key.name:20} {key.entry_point:45} {sorted(key.tags)}") +``` + +``` +address overture.schema.addresses:Address ['feature', 'overture:theme=addresses'] +building overture.schema.buildings:Building ['feature', 'overture:theme=buildings'] +segment overture.schema.transportation:Segment ['feature', 'overture:theme=transportation'] +... +``` + +A `ModelKey` carries `.name`, `.entry_point` (`"module:Class"`), and `.tags` +(a `frozenset[str]`). Filter the same way the CLI does: + +```python +from overture.schema.system.discovery import TagSelector, discover_models, filter_models + +models = discover_models() + +buildings = filter_models(models, TagSelector(include_any=("overture:theme=buildings",))) +``` + +`TagSelector` takes `include_any` (OR scope), `require_all` (AND narrowing), and +`exclude_any` (OR-NOT). An empty selector returns the input unchanged. + +### 2.7 Reading enum member documentation + +Enum members carry per-value docstrings, but `member.__doc__` falls back to the *class* +docstring when a member has none — so reading `__doc__` directly gives you misleading +results: + +```python +from overture.schema.buildings import RoofShape +[ (m.value, m.__doc__.strip()[:30]) for m in list(RoofShape)[:2] ] +# [('dome', 'The shape of the roof.'), ('flat', 'The shape of the roof.')] +# ^ that's the class docstring repeated, not per-member documentation +``` + +Use the codegen extractor, which does the fallback detection for you: + +```python +from overture.schema.codegen.extraction.enum_extraction import extract_enum +from overture.schema.common.scoping.travel_mode import TravelMode + +spec = extract_enum(TravelMode) +for m in spec.members[:4]: + print(f"{m.value:14} {m.description or '—'}") +``` + +``` +vehicle — +motor_vehicle Includes car, truck and motorcycle +car — +truck — +``` + +`description` is `None` when the member has no documentation of its own. + +### 2.8 Generate browsable reference docs + +For sustained exploration, generate the full markdown reference and read it in your +editor: + +```bash +uv run overture-codegen generate --format markdown --output-dir ./schema-docs +``` + +You get one page per feature type, per enum, and per named type, with field tables, +prose constraint descriptions, cross-page links, and validated examples: + +``` +schema-docs/buildings/building.md +schema-docs/buildings/building_part.md +schema-docs/buildings/types/building_class.md +schema-docs/buildings/types/roof_shape.md +schema-docs/common/names.md +schema-docs/common/sources.md +schema-docs/system/numeric.md +... +``` + +Scope it to one theme with the same tag options: + +```bash +uv run overture-codegen generate --format markdown \ + --tag overture:theme=buildings --output-dir ./schema-docs +``` + +(Supplementary types from `common/` and `system/` are pulled in regardless, since the +feature pages link to them.) + +--- + +--- + +## 3. Writing code against the models + +Section 2 was about finding out what exists. This section is about using it: loading +real data into models, reading and changing it, and writing it back out. + +### 3.1 A complete example, start to finish + +Before any of the details, here is the whole job in one piece: load a feature, read it, +change it, write it back out, and confirm it still validates. Everything else in this +section is an explanation of a line in this snippet. + +```bash +uv run python <<'PY' +import json, yaml +from overture.schema.buildings import Building + +# 1. LOAD — the file is GeoJSON, so go through JSON mode +doc = yaml.safe_load(open("examples/buildings/building-polygon.yaml")) +b = Building.model_validate_json(json.dumps(doc)) +print("1. loaded :", b.id) + +# 2. READ — plain Python attributes; enums come back as enum members +print("2. read :", b.height, "m,", b.num_floors, "floors, class", b.class_.value) + +# 3. MODIFY — ordinary assignment +b.height = 25.0 +b.num_floors = 5 +print("3. changed:", b.height, "m,", b.num_floors, "floors") + +# 4. WRITE — by_alias=True so `class_` is written as `class` +out = b.model_dump(mode="json", by_alias=True, exclude_none=True) +print("4. wrote :", json.dumps(out)[:70], "...") + +# 5. CONFIRM — the output is valid input +again = Building.model_validate_json(json.dumps(out)) +print("5. re-read:", again.height, "m — round-trip holds") +PY +``` + +``` +1. loaded : overture:buildings:building:1234 +2. read : 21.34 m, 4 floors, class parking +3. changed: 25.0 m, 5 floors +4. wrote : {"type": "Feature", "id": "overture:buildings:building:1234", "geometr ... +5. re-read: 25.0 m — round-trip holds +``` + +That is the shape of nearly every job: **validate in, work with plain Python objects, +serialize out.** In between, `b` is an ordinary object — attributes, assignment, no +special API. + +Three lines in there are load-bearing, and each gets a subsection below: + +| Line | Why it's written that way | Where | +|---|---|---| +| `model_validate_json(json.dumps(doc))` | the file is GeoJSON, which needs JSON mode | [3.2](#32-the-one-thing-to-understand-two-data-shapes) | +| `b.class_.value` | the field is `class_` in Python, `class` in the data | [3.3](#33-reading-fields) | +| `by_alias=True, exclude_none=True` | without them the output won't re-validate | [3.4](#34-writing-data-back-out) | + +If the snippet above ran, you already know enough to be useful. Read on when one of those +lines bites you, or read straight through if you'd rather know why now. + +### 3.2 The one thing to understand: two data shapes + +Overture data shows up in two shapes, and the models handle both — but **which one you +get depends on the Pydantic mode you use, not on the data you pass**. + +| Shape | Looks like | Pydantic mode | Validate with | Dump with | +|---|---|---|---|---| +| **GeoJSON** | `id`/`geometry` at top level, everything else under `properties` | `json` | `model_validate_json()` | `model_dump(mode="json")`, `model_dump_json()` | +| **Flat / tabular** (Parquet-style) | every field at the top level | `python` | `model_validate()` | `model_dump(mode="python")` | + +This trips people up constantly. Passing a GeoJSON *dict* to `model_validate()` fails, +because `model_validate` is Python mode and Python mode expects the flat shape: + +```python +import json, yaml +from overture.schema.buildings import Building + +doc = yaml.safe_load(open("examples/buildings/building-polygon.yaml")) # GeoJSON-shaped + +Building.model_validate(doc) +# ValidationError: 3 validation errors for building +# theme Field required +# type Input should be 'building' [got 'Feature'] +# version Field required +``` + +The `type: Feature` in the error message is the tell: it read the GeoJSON envelope's +`type` as the feature's `type` field. + +The fix — round-trip through JSON so you're in JSON mode: + +```python +building = Building.model_validate_json(json.dumps(doc)) # works +``` + +Or, if you're already reading from a file or an HTTP response, skip the parse entirely: + +```python +building = Building.model_validate_json(open("building.geojson").read()) +``` + +#### Can't I just tell `model_validate` to use JSON mode? + +No. It's the obvious thing to try, and neither knob does it: + +```python +Building.model_validate(doc, context={"mode": "json"}) # still ValidationError +Building.model_validate(doc, strict=False) # still ValidationError +``` + +The mode isn't a setting you pass — in Pydantic it's determined by *which method you +call*. `model_validate` is Python mode; `model_validate_json` is JSON mode. The base +`Feature` class keys off exactly that: + +```python +@model_validator(mode="wrap") +def __validate_with_geo_json_support__(cls, data, handler, info): + if info.mode == "json": # <- set by the method, not by an argument + ... # unpack the GeoJSON envelope +``` + +So if you have a GeoJSON **dict** in hand, `json.dumps` it and use +`model_validate_json`. The round-trip is slightly wasteful but it is the supported path: + +```python +Building.model_validate_json(json.dumps(doc)) +``` + +Better still, avoid making the dict at all. If the GeoJSON came from a file or an HTTP +response, hand the raw text straight to `model_validate_json` and skip `json.loads` +entirely. The one case where you genuinely can't is YAML — there's no YAML mode, so +`yaml.safe_load` → `json.dumps` → `model_validate_json` is the route, as in the example +above. + +#### Reading the error + +The three errors from a mode mismatch are always the same shape, and worth recognising on +sight: + +``` +theme Field required +type Input should be 'building' [input_value='Feature'] +version Field required +``` + +`theme` and `version` are "missing" because they're really down inside `properties`, where +Python mode isn't looking. And `type` came back as `'Feature'` — the envelope's type, +which is the giveaway. **If you ever see `input_value='Feature'` in a validation error, +you passed GeoJSON to a Python-mode call.** + +For flat data — a Parquet row, a DuckDB result, a dict of columns — `model_validate` is +the right call: + +```python +row = {"id": "...", "theme": "buildings", "type": "building", "version": 1, + "geometry": ..., "height": 21.34, "class": "parking"} +building = Building.model_validate(row) +``` + +### 3.3 Reading fields + +Model attributes are plain Python. Enums come back as enum members, geometry as a +`Geometry` wrapper around Shapely: + +```python +building.height # 21.34 +building.class_ # +building.class_.value # 'parking' +building.num_floors # 4 +type(building.geometry) # +``` + +### 3.4 Writing data back out + +Some fields are Python keywords, so the model attribute differs from the wire name +(`class_` on the model, `class` in the data). **`model_dump()` uses attribute names by +default**, which produces output that will not validate back: + +```python +d = building.model_dump(mode="json") +sorted(d["properties"]) +# ['class_', 'ext_bar', 'height', ...] ← 'class_' is wrong for the wire + +Building.model_validate(building.model_dump(mode="python")) +# ValidationError: invalid extra field name: class_ +``` + +With `by_alias=True` both round-trips work: + +```python +geojson = building.model_dump(mode="json", by_alias=True, exclude_none=True) +sorted(geojson["properties"]) +# ['class', 'ext_bar', 'height', 'is_underground', 'level', 'num_floors', ...] + +Building.model_validate_json(json.dumps(geojson)) # OK +Building.model_validate(building.model_dump(mode="python", by_alias=True, exclude_none=True)) # OK +``` + +Same for `model_dump_json()`: + +```python +'"class":' in building.model_dump_json() # False ← emits "class_" +'"class":' in building.model_dump_json(by_alias=True) # True +``` + +**Rule of thumb: `by_alias=True` on every dump, unless you specifically want Python +attribute names.** `exclude_none=True` is usually what you want too — otherwise you get +every unset optional field as an explicit `null`. + +### 3.5 Working with `Segment` and other unions + +`Segment` is a discriminated union type alias over `RoadSegment`, `RailSegment`, and +`WaterSegment` — not a model class. It has no `model_validate`: + +```python +from overture.schema.transportation import Segment + +type(Segment) # +Segment.model_validate({...}) +# AttributeError: model_validate +``` + +Wrap it in a `TypeAdapter`: + +```python +from pydantic import TypeAdapter +from overture.schema.transportation import Segment + +segments = TypeAdapter(Segment) + +seg = segments.validate_json(raw_geojson_string) # JSON mode → GeoJSON +seg = segments.validate_python(flat_row) # Python mode → flat +type(seg).__name__ # 'RoadSegment' +``` + +Build the `TypeAdapter` once and reuse it; construction is the expensive part. + +The concrete arms *are* ordinary classes if you know which one you want: + +```python +from overture.schema.transportation import RoadSegment +``` + +### 3.6 Validating without knowing the type + +`overture-schema-validation` checks a record against every installed model and returns +whichever matched: + +```python +from overture.schema.validation import validate, validate_json + +feature = validate_json(geojson_string) # JSON mode → GeoJSON shape +type(feature).__name__ # 'Building' + +feature = validate(flat_dict) # Python mode → flat shape +``` + +Both raise `pydantic.ValidationError` when nothing matches. The same mode rule from +[The one thing to understand first](#32-the-one-thing-to-understand-two-data-shapes) applies: `validate()` is +Python mode and wants flat data, `validate_json()` is JSON mode and wants GeoJSON. + +Which models participate is resolved at runtime by entry-point discovery — install more +theme packages and these functions accept more. + +### 3.7 Generating JSON Schema in code + +```python +from overture.schema.system.json_schema import json_schema +from overture.schema.buildings import Building +from overture.schema.places import Place + +schema = json_schema(Building) +schema["title"] # 'building' +sorted(schema) # ['$defs', 'additionalProperties', 'description', + # 'properties', 'required', 'title', 'type'] + +union = json_schema(Building | Place) # unions work too → anyOf +``` + +Use this rather than Pydantic's `model_json_schema()`. The Overture generator treats +`T | None = None` as "omit when unset" instead of Pydantic's "nullable with a null +default", which is what the data actually means. + +--- + +--- + +## 4. The three CLIs + +Installing the workspace gives you three commands, from three different packages. + +| Command | Package | Purpose | +|---|---|---| +| `overture-schema` | `overture-schema-cli` | Validate files, emit JSON Schema, list types | +| `overture-codegen` | `overture-schema-codegen` | Generate markdown docs and PySpark expressions | +| `overture-validate` | `overture-schema-pyspark` | Validate Parquet/S3 data at scale with Spark | + +Prefix each with `uv run` inside the repo, or activate the venv. + +### 4.1 `overture-schema` + +``` +Usage: overture-schema [OPTIONS] COMMAND [ARGS]... + +Commands: + json-schema Generate JSON schema for Overture Maps types. + list-types List all available types. + validate Validate Overture Maps data against schemas. +``` + +All three subcommands take the shared `--tag` / `--filter` / `--exclude` options from +[From the CLI: what types exist?](#23-from-the-cli-what-types-exist). `validate` and `json-schema` also take +`--type NAME` to target one type directly. + +```bash +# What types do I have? +overture-schema list-types +overture-schema list-types --group-by overture:theme + +# Validate +overture-schema validate data.geojson +overture-schema validate - < data.geojson +overture-schema validate --type building data.json +overture-schema validate --tag overture:theme=buildings data.json +overture-schema validate --show-field id data.json + +# JSON Schema +overture-schema json-schema > all-types.json +overture-schema json-schema --type building > building.json +overture-schema json-schema --tag overture:theme=buildings > buildings.json +``` + +Exit codes: `0` on success, `1` on validation failure — so it drops into CI directly. + +#### Local files or remote? + +It depends which CLI, and the two answer differently. + +| Command | Remote paths | How | +|---|---|---| +| `overture-schema validate` | **no** | pipe through stdin with `-` | +| `overture-validate` (PySpark) | **yes** | `s3a://` natively, anonymous credentials preconfigured | + +`overture-schema validate` takes a filesystem path only. Hand it a URL and it fails — +note that it even mangles the `//`, because the argument is parsed as a path: + +```bash +overture-schema validate https://raw.githubusercontent.com/OvertureMaps/schema/main/examples/buildings/building-polygon.yaml +``` + +``` +Error: 'https:/raw.githubusercontent.com/.../building-polygon.yaml' is not a file. +``` + +The fix is the `-` argument, which reads stdin. Anything that can fetch bytes can feed it: + +```bash +curl -sSf https://raw.githubusercontent.com/OvertureMaps/schema/main/examples/buildings/building-polygon.yaml \ + | overture-schema validate - +``` + +``` +✓ Successfully validated +``` + +That works for anything on stdin — `aws s3 cp ... -`, a database query, a generator +script, another program's output. The only thing you lose is the filename in the output, +which becomes ``. + +`overture-validate` is the opposite: it's built for remote data. `s3a://` paths are +detected automatically and configured with anonymous credentials, so the public Overture +release bucket needs no setup: + +```bash +overture-validate segment s3a://overturemaps-us-west-2/release/2026-07-22.0 +``` + +> **Don't hardcode a release version.** The bucket keeps only the current release, so any +> version written into a script or a doc stops working at the next publish. Ask the bucket +> instead: +> +> ```bash +> curl -sS "https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/?list-type=2&prefix=release/&delimiter=/" \ +> | tr '<' '\n' | grep -oE 'Prefix>release/[^/]+' | sed 's|Prefix>release/||' | sort -r | head -1 +> ``` +> +> ``` +> 2026-07-22.0 +> ``` +> +> Then use it: +> +> ```bash +> RELEASE=$(curl -sS "https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/?list-type=2&prefix=release/&delimiter=/" \ +> | tr '<' '\n' | grep -oE 'Prefix>release/[^/]+' | sed 's|Prefix>release/||' | sort -r | head -1) +> overture-validate segment "s3a://overturemaps-us-west-2/release/$RELEASE" +> ``` +> +> The version shown in the examples here was current when this was written; treat it as a +> placeholder, not a fact. + + +That difference is not arbitrary. `overture-schema validate` is for a file you're looking +at — an example, a fixture, one feature you're debugging. `overture-validate` is for a +release: millions of rows, read in parallel by Spark, where "download it first" isn't an +option. + +### 4.2 `overture-codegen` + +``` +Usage: overture-codegen [OPTIONS] COMMAND [ARGS]... + +Commands: + generate Generate code/docs from discovered models. + list List all discovered models. +``` + +``` +Usage: overture-codegen generate [OPTIONS] + + --format [markdown|pyspark] Output format [required] + --tag / --filter / --exclude TEXT + --output-dir PATH Default: stdout + --test-output-dir PATH Write generated conformance tests (pyspark only) +``` + +```bash +# Markdown reference docs +overture-codegen generate --format markdown --output-dir ./schema-docs +overture-codegen generate --format markdown --tag overture:theme=places --output-dir ./out + +# PySpark validation expressions (this is what `make generate-pyspark` runs) +overture-codegen generate --format pyspark \ + --output-dir packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated \ + --test-output-dir packages/overture-schema-pyspark/tests/generated +``` + +### 4.3 `overture-validate` + +Validates real data volumes with Spark. Requires the generated expression tree, so run +`make install` or `make generate-pyspark` first. + +``` +Usage: overture-validate [OPTIONS] FEATURE_TYPE PATH + + -o, --output TEXT Output path for validated Parquet. + --head INTEGER Error rows to display. [default: 20] + --conf TEXT Spark config key=value pairs. + --count-only Report error count only. + --skip-schema-check Warn on schema mismatches instead of aborting. + --skip-columns TEXT Columns declared absent from data. + --ignore-extra-columns TEXT Extra data columns to ignore in schema comparison. + --suppress TEXT Suppress checks: FIELD or FIELD:CHECK. +``` + +```bash +overture-validate building local.parquet +overture-validate segment s3a://overturemaps-us-west-2/release/2026-07-22.0 +overture-validate place data.parquet --count-only +overture-validate segment data.parquet --suppress version:bounds -o violations.parquet +``` + +It handles S3A and anonymous credentials for the public Overture bucket automatically, +and expands a release root into the Hive partition path for you. Full option and +path-resolution reference: `packages/overture-schema-pyspark/README.md`. + +--- + +--- + +## 5. Validating data + +Three tiers, pick by data size. + +### 5.1 One file, or a handful — the CLI + +Accepts JSON, YAML, and GeoJSON. A single feature, a JSON array of features, or a +`FeatureCollection` all work: + +```bash +overture-schema validate examples/buildings/building-polygon.yaml +``` + +``` +✓ Successfully validated examples/buildings/building-polygon.yaml +``` + +Failures come back as a rendered table showing the offending value in context: + +```bash +overture-schema validate --show-field id counterexamples/buildings/negative-height.json +``` + +``` + ─ Validation Failed id=foo ──────────────────────────────────────────────────── + ... + id "foo" + version 0 + height -1.23 ← Input should be greater than 0 + ────────────────────────────────────────────────────────────────────────────── +``` + +For a collection, errors are indexed and labeled by the model that best fit: + +``` + ─ [1] (Building) ────────────────────────────────────────────────────────────── + ... + version 0 + height -1.23 ← Input should be greater than 0 + ────────────────────────────────────────────────────────────────────────────── +``` + +Narrow the candidate set to sharpen the error messages — with `--type building` the CLI +stops guessing which model you meant: + +```bash +overture-schema validate --type building data.json +``` + +### 5.2 In a Python pipeline + +```python +from pydantic import ValidationError +from overture.schema.validation import validate_json + +ok, bad = 0, [] +for line in open("features.ndjson"): + try: + validate_json(line) + ok += 1 + except ValidationError as e: + bad.append((line[:60], e.errors())) + +print(f"{ok} valid, {len(bad)} invalid") +``` + +`e.errors()` gives you structured dicts with `loc`, `msg`, `type`, and `input` — the +right thing to log or turn into a report. + +If you know the type, validate against it directly for better errors and speed: + +```python +from overture.schema.buildings import Building +Building.model_validate_json(line) +``` + +### 5.3 At scale — PySpark + +```python +from pyspark.sql import SparkSession +from overture.schema.pyspark import validate_model, explain_errors + +spark = SparkSession.builder.getOrCreate() +df = spark.read.parquet("s3a://.../theme=buildings/type=building/") + +result = validate_model(df, "building") +result.evaluated.cache() + +total = result.evaluated.count() +errors = result.error_rows().count() +print(f"{errors} / {total} rows with errors") + +if errors: + violations = explain_errors(result.evaluated, result.checks) + violations.select("id", "field", "check", "message").show(truncate=False) +``` + +`validate_model` accepts either the short name (`"building"`) or the full entry-point key +(`"overture.schema.buildings:Building"`). It looks up the feature type in the registry, +compares the DataFrame schema against the expected one, and evaluates every check in a +single pass — no per-row Python, so it scales. + +| Function | Returns | Purpose | +|---|---|---| +| `validate_model(df, type)` | `ValidationResult` | Registry lookup, schema comparison, check evaluation | +| `result.error_rows()` | `DataFrame` | Rows with at least one violation | +| `explain_errors(evaluated, checks)` | `DataFrame` | One row per violation: `field`, `check`, `message` | +| `model_names()` | `list[str]` | Available type names | + +### 5.4 Validating the schema itself + +If you're changing the models rather than the data: + +```bash +make check +make test +make update-baselines +``` + +The theme packages carry golden-file baseline tests of their generated JSON Schema, so +unintended schema drift fails CI. After an intentional change, run `make +update-baselines` and inspect the `git diff` on the regenerated golden files before +committing. + +--- + +--- + +## 6. Converting the schema to other formats + +Three built-in targets, plus everything reachable through JSON Schema. + +| Target | Command | Output | +|---|---|---| +| JSON Schema | `overture-schema json-schema` | A JSON Schema document on stdout | +| Markdown | `overture-codegen generate --format markdown` | Docusaurus-ready reference pages | +| PySpark | `overture-codegen generate --format pyspark` | Python modules of `Check` builders + `StructType` | +| Spark `StructType` | (Python, via the pyspark registry) | A live Spark schema object | + +### 6.1 JSON Schema + +The interop format — this is your bridge to every other ecosystem. + +```bash +# One type +overture-schema json-schema --type building > building.schema.json + +# One theme +overture-schema json-schema --tag overture:theme=transportation > transportation.schema.json + +# Everything (an `anyOf` over all installed types) +overture-schema json-schema > overture.schema.json +``` + +A single type produces a self-contained document with its dependencies inlined under +`$defs`: + +``` +$ jq 'keys' building.schema.json +["$defs", "additionalProperties", "description", "properties", "required", "title", "type"] + +$ jq '.title, (.["$defs"] | length)' building.schema.json +"building" +13 +``` + +All types produce `{"anyOf": [...], "$defs": {...}}`. + +In Python: + +```python +from overture.schema.system.json_schema import json_schema +from overture.schema.buildings import Building + +schema = json_schema(Building) +``` + +### 6.2 Markdown + +Covered in [Generate browsable reference docs](#28-generate-browsable-reference-docs). Output is Docusaurus-flavored +(frontmatter plus `_category_.json` files), but it's plain markdown underneath and reads +fine in any editor or static site generator. + +### 6.3 PySpark expressions and Spark schemas + +```bash +overture-codegen generate --format pyspark --output-dir ./ps --test-output-dir ./ps-tests +``` + +You get one module per feature type, mirroring the Python package layout: + +``` +ps/overture/schema/buildings/building.py +ps/overture/schema/buildings/building_part.py +ps-tests/overture/schema/buildings/test_building.py +``` + +Each module is auto-generated (`# Do not edit`) and contains one builder function per +constraint, returning a `Check` with an unevaluated PySpark `Column`: + +```python +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0), + shape=CheckShape.SCALAR, + root_field="version", + ) +``` + +Plus a `MODEL_VALIDATION` constant pairing the checks with the expected `StructType`. + +**To get a Spark schema for a feature type** — useful for `spark.read.schema(...)`, +Delta table creation, or comparing against your own tables: + +```python +from overture.schema.pyspark._registry import REGISTRY +from overture.schema.pyspark.validate import resolve_entry_point_key + +key = resolve_entry_point_key("building", REGISTRY) +struct = REGISTRY[key].schema + +type(struct).__name__ # 'StructType' +[(f.name, f.dataType.simpleString()) for f in struct.fields][:5] +``` + +``` +[('id', 'string'), + ('bbox', 'struct'), + ('geometry', 'binary'), + ('theme', 'string'), + ('type', 'string')] +``` + +`REGISTRY` is keyed by the full entry-point string +(`"overture.schema.buildings:Building"`), which is why the `resolve_entry_point_key` +step is there — it accepts the short alias too. `ModelValidation` exposes `.schema`, +`.checks`, and `.geometry_types`. + +Since `StructType` has `.json()` and `.jsonValue()`, this is also your route to an +Arrow/Parquet schema. + +--- + +--- + +## 7. Using the packages from your own project + +Everything above assumes you're working *inside* the schema repo. If instead you're +building your own application that depends on these models, you don't use the workspace +at all — you point `uv` at the package directories on disk. + +Depend on the **theme packages you actually need**, not the workspace root: + +```toml +# myapp/pyproject.toml +[project] +name = "myapp" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [ + "overture-schema-theme-buildings", + "overture-schema-cli", +] + +[tool.uv.sources] +overture-schema-theme-buildings = { path = "/path/to/schema/packages/overture-schema-theme-buildings", editable = true } +overture-schema-cli = { path = "/path/to/schema/packages/overture-schema-cli", editable = true } +``` + +```bash +cd myapp +uv sync +uv run python -c "from overture.schema.buildings import Building; print(Building.__name__)" +``` +``` +Building +``` + +`editable = true` means edits in your schema clone take effect immediately in `myapp` — +useful if you're changing both. + +### 7.1 The payoff: install set = runtime set + +In the `myapp` project above, only the buildings theme is installed. So: + +```bash +cd myapp && uv run overture-schema list-types +``` +``` +building feature overture:theme=buildings +building_part feature overture:theme=buildings +``` + +Two types, not fifteen. **The same CLI binary, scoped by what's installed.** Nothing was +configured to make that happen — the models register themselves through entry points, and +the tooling discovers whatever is present. + +This is worth internalizing early, because it's the whole extension story: add your own +package that registers models, and your feature types appear in `list-types`, validate +through the same commands, and show up in generated docs, alongside Overture's. See +[Register your own feature types](#83-register-your-own-feature-types). + +### 7.2 If you'd rather have everything + +`overture-schema` is a metapackage depending on all six themes plus validation and the +CLI: + +```toml +[tool.uv.sources] +overture-schema = { path = "/path/to/schema/packages/overture-schema", editable = true } +``` + +One caveat: `import overture.schema` gives you nothing directly. It's a namespace root +that ships only a `py.typed` marker — no models, no functions. Always import from the +theme packages: + +```python +from overture.schema.buildings import Building # ✓ +from overture.schema import Building # ✗ ImportError +``` + + +--- + +### 7.3 What changes once these packages are published + +Some of this section is scaffolding for the fact that nothing is on a package index yet. +Worth knowing which parts, so you don't over-invest in learning them. + +| Part of this section | After publishing | +|---|---| +| What a workspace is, the shared `.venv`, `uv run` | **Stays** — but becomes reading for contributors only | +| `git clone` + `uv sync --all-packages` | **Stays** — contributors only | +| `make generate-pyspark` | **Gone for consumers.** Published wheels ship the generated expressions already. | +| The `SPARK_HOME` fix | **Stays forever.** It's an environment problem, unrelated to packaging. | +| Empty registry, `exclude-newer` warning | Contributors only | +| The `[tool.uv.sources]` path blocks above | **Deleted entirely.** This is the pure workaround. | +| "Install set = runtime set" | **Stays.** That's entry-point discovery, not packaging. | + +The whole of this subsection collapses to one line: + +```bash +uv add overture-schema-theme-buildings overture-schema-cli +``` + +or, for everything: + +```bash +uv add overture-schema +``` + +**On the PySpark step specifically:** the release pipeline already handles it. +`.github/workflows/publish-python-packages.yaml` runs `make generate-pyspark` before +`uv build`, and aborts the release if the resulting wheel contains no +`expressions/generated/*.py`. Its own comment explains why: + +> the tree must be generated here before the build, or the published wheel ships without +> its `expressions/generated/` modules and `validate_model()` discovers nothing to run + +So consumers of a published wheel never run that step. + +**Which index?** The publish workflow currently pushes to AWS CodeArtifact (the +`overture-pypi` domain), even though its step names say PyPI. `CONTRIBUTING.md` describes +version bumps reaching public PyPI while interim builds stay internal. Either way the +consumer instruction is a one-line install — only the index URL differs. + +Nothing in sections 2 through 9 changes. The behavior you learn there — entry-point +discovery, the two data shapes, `by_alias`, `TypeAdapter` for `Segment` — is independent +of how the packages get onto your machine. + +--- + +--- + +# Part II — Extending and authoring the schema + +## 8. Building your own SDK or CLI + +Four approaches, cheapest first. + +### 8.1 Generate an SDK from JSON Schema (any language) + +Emit JSON Schema, then hand it to a standard generator. Both of these were run against +the output of `overture-schema json-schema --type building` and produced working code. + +**Python (datamodel-code-generator):** + +```bash +uv run overture-schema json-schema --type building > building.schema.json + +uvx --from datamodel-code-generator datamodel-codegen \ + --input building.schema.json \ + --input-file-type jsonschema \ + --output building_models.py +``` + +Produces standalone Pydantic v2 models with no Overture dependency — useful for a +service that shouldn't take the whole workspace as a dependency: + +```python +# generated by datamodel-codegen: +# filename: building.schema.json +from pydantic import BaseModel, ConfigDict, Field, confloat, conint, constr +``` + +**TypeScript (quicktype):** + +```bash +npx -y quicktype --src-lang schema --lang typescript \ + -o Building.ts building.schema.json +``` + +Field descriptions survive as JSDoc: + +```typescript +/** + * Buildings are man-made structures with roofs that exist permanently in one place. + * ... + */ +export interface Building { + bbox?: [number, number, number, number, ...number[]]; + /** The building's footprint or roofprint... */ + geometry: Geometry; +``` + +quicktype also targets Go, Rust, Java, Kotlin, Swift, C#, and others from the same input. +Anything that reads JSON Schema — OpenAPI toolchains, `go-jsonschema`, `schemars`, +`jsonschema2pojo` — works the same way. + +The tradeoff: you get types and structural validation, but not the semantic layer. +Cross-field model constraints (`@require_any_of`, `@radio_group`) do translate into JSON +Schema `if`/`then`/`anyOf` constructs, but domain-specific error messages and the +NewType vocabulary flatten out. + +### 8.2 Build a CLI on discovery and tags + +If you're staying in Python, don't hardcode a type list. Discover, and let the installed +packages decide what exists — same as the built-in CLI. Your tool then automatically +covers new themes and third-party extensions. + +```python +import click +from overture.schema.system.discovery import discover_models, filter_models, TagSelector +from overture.schema.cli.tag_options import tag_selection_options, build_selector + +@click.command() +@tag_selection_options # gives you --tag / --filter / --exclude for free +def report(tags, filters, excludes): + """Report field counts for the selected feature types.""" + models = filter_models(discover_models(), build_selector(tags, filters, excludes)) + for key, model in sorted(models.items(), key=lambda kv: kv[0].name): + n = len(model.model_fields) if hasattr(model, "model_fields") else "—" + click.echo(f"{key.name:20} {n}") +``` + +`overture.schema.cli` also exports `resolve_types`, `create_union_type_from_models`, +`load_input`, `perform_validation`, `handle_validation_error`, and +`handle_generic_error` — so you can reuse the file-loading and error-rendering behavior +rather than reimplementing it. + +### 8.3 Register your own feature types + +Your models become first-class: discovered by the CLI, accepted by `validate()`, +included in generated docs and JSON Schema. Nothing in the tooling special-cases +Overture. + +```python +# mypkg/models.py +from typing import Literal +from overture.schema.common import OvertureFeature +from overture.schema.system.numeric import float32 + +class Vineyard(OvertureFeature[Literal["agriculture"], Literal["vineyard"]]): + """A cultivated area planted with grapevines.""" + area_hectares: float32 | None = None +``` + +```toml +# mypkg/pyproject.toml +[project.entry-points."overture.models"] +vineyard = "mypkg.models:Vineyard" +``` + +Install it, and: + +```bash +overture-schema list-types +overture-schema validate vineyard.json +overture-codegen generate --format markdown --output-dir out +``` + +To attach your own tags, register a **tag provider** on `overture.tag_providers`: + +```python +def experimental_provider(types, key, tags): + if any(getattr(t, "__experimental__", False) for t in types): + tags.add("mypkg:experimental") + return tags +``` + +```toml +[project.entry-points."overture.tag_providers"] +experimental = "mypkg.tags:experimental_provider" +``` + +Tag namespaces are reserved: `feature` and `system:` belong to `overture-schema-system`, +`overture:` to `overture-schema-common`. A provider that tries to set a reserved tag from +an unauthorized package gets a logged warning and the tag is discarded. Use your own +namespace. + +You don't have to build on `OvertureFeature` — subclass `system.Feature` directly for a +GeoJSON-serializing model with none of the Overture conventions. + +### 8.4 Write a new codegen target + +For a format nobody else generates — Arrow schemas, Avro, Go structs, protobuf — add a +renderer to the codegen rather than parsing JSON Schema back out. You get the full +semantic model: NewType names, constraint provenance, discriminated union structure — all +the things JSON Schema flattens away. + +The pipeline is four layers with strictly downward imports: + +``` +Rendering → output formatting, all presentation decisions +Output Layout → what to generate, where it goes, how outputs link +Extraction → FieldShape, FieldSpec, RecordSpec, UnionSpec, EnumSpec +Discovery → discover_models() +``` + +Extraction is target-independent, so a new target is a new renderer, not new extraction +logic. The entry point: + +```python +from overture.schema.codegen.extraction.model_extraction import extract_model +from overture.schema.buildings import Building + +spec = extract_model(Building) +spec.name # 'Building' +spec.description # the class docstring +spec.constraints # model-level constraints + +for f in spec.fields[:6]: + print(f"{f.name:12} required={f.is_required!s:5} {type(f.shape).__name__}") +``` + +``` +id required=True NewTypeShape +bbox required=False Primitive +geometry required=True Primitive +theme required=True LiteralScalar +type required=True LiteralScalar +version required=True NewTypeShape +``` + +`FieldSpec` is `(name, shape, description, is_required, is_optional)`. The `shape` is a +`FieldShape` tree — `NewTypeShape`, `Primitive`, `LiteralScalar`, `ModelRef`, +`UnionRef`, and container variants — with sub-models and sub-unions already resolved. +Constraints carry provenance, so you can tell which NewType contributed which bound: + +```python +from overture.schema.codegen.extraction.type_analyzer import analyze_type +from overture.schema.common.feature import FeatureVersion + +shape, is_nullable, description = analyze_type(FeatureVersion) +# shape → NewTypeShape(name='FeatureVersion', inner=Primitive(base_type='int32', +# constraints=(ConstraintSource(source_name='FeatureVersion', constraint=Ge(ge=0)), ...))) +``` + +`analyze_type` returns a **3-tuple** `(FieldShape, bool, str | None)` — the structural +shape, whether the field accepts `None`, and the first description found while +unwrapping. (The codegen README shows an older `TypeInfo`/`TypeKind` API that no longer +exists.) + +To wire up a new format: add a column to `TypeMapping` in +`extraction/type_registry.py` for type-name resolution, write a pipeline module +consuming `ModelSpec` trees plus a renderer, and register the format in `cli.py`. + +Further reading in the repo: + +- `packages/overture-schema-codegen/docs/design.md` — architecture, data flow, extension points +- `packages/overture-schema-codegen/docs/walkthrough.md` — module-by-module trace of `Segment` through the pipeline + +--- + +--- + +## 9. Registering models and tagging + +How feature types make themselves known to the tooling. This is the mechanism that +lets your own models slot in alongside Overture's — see +[Register your own feature types](#83-register-your-own-feature-types) for a worked example. + +The library is designed to support data producer extensions through multiple patterns. +This extensibility is a core feature that allows organizations to add custom fields and +types while maintaining compatibility with the base Overture schema. We are in the +process of determining how this should work. + +### Model Registration via Entry Points + +Models are registered using [setuptools entry +points](https://setuptools.pypa.io/en/latest/userguide/entry_point.html) in each +package's `pyproject.toml` file. This enables automatic discovery and loading of models +at runtime without requiring explicit imports. + +Registration is done in the `[project.entry-points."overture.models"]` section: + +```toml +[project.entry-points."overture.models"] +building = "overture.schema.buildings:Building" +building_part = "overture.schema.buildings:BuildingPart" +``` + +The discovery system provides programmatic access to registered models: + +```python +from overture.schema.system.discovery import discover_models, get_registered_model + +# Discover all registered models, keyed by ModelKey +all_models = discover_models() + +# Get a specific model by name +building_model = get_registered_model("building") +if building_model: + building = building_model.model_validate(building_data) +``` + +### Tagging + +Each `ModelKey` returned by `discover_models()` carries a `frozenset[str]` of tags +that classify the model orthogonally to its entry-point name -- whether the model +is a `Feature` subclass, which Overture theme it belongs to, which package shipped +it, and so on. Downstream tools (the CLI, codegen, third-party consumers) use tags +to filter the working set without importing every model: + +```python +from overture.schema.system.discovery import ( + TagSelector, + discover_models, + filter_models, +) + +models = discover_models() +# { +# ModelKey(name="building", entry_point="overture.schema.buildings:Building", +# tags=frozenset({"feature", "overture:theme=buildings"})): BuildingModel, +# ModelKey(name="place", entry_point="overture.schema.places:Place", +# tags=frozenset({"feature", "overture:theme=places"})): PlaceModel, +# ... +# } + +buildings = filter_models( + models, + TagSelector(include_any=("overture:theme=buildings",)), +) +``` + +Tags are produced by *tag providers* registered on the `overture.tag_providers` +entry-point group. The `system` and `common` packages ship the built-in providers +(`feature` and `overture:theme=*`); third parties can register their own +to attach custom tags during discovery. See the [`overture-schema-system` +README](packages/overture-schema-system/README.md#tagging) for tag format, +reserved namespaces, and provider authoring. + + +--- + +## 10. Authoring new schema models + +### Quick Start + +#### Essential Imports + +Copy what you need for most models: + +```python +# Basic Python types +from typing import Annotated, Literal +from enum import Enum + +# Pydantic essentials +from pydantic import BaseModel, Field + +# Overture common models +from overture.schema.common import OvertureFeature +from overture.schema.system.geometric import Geometry, GeometryType, GeometryTypeConstraint + +# Validation system +from overture.schema.system.field_constraint import UniqueItemsConstraint +from overture.schema.system.model_constraint import no_extra_fields + +# Common types +from overture.schema.system.string import ( + CountryCodeAlpha2, + NoWhitespaceString, + StrippedString, +) +from overture.schema.common.confidence import ConfidenceScore +from overture.schema.system.string import LanguageTag + +# Numeric types (use these instead of int/float) +from overture.schema.system.numeric import ( + int8, int32, int64, + uint8, uint16, uint32, + float32, float64 +) +``` + +#### Templates + +Copy-paste starting points for the four shapes you'll write most often — a plain +model, a feature, an enum, and a model with validation constraints — live together in +[Templates and quick reference](#13-templates-and-quick-reference) rather than being +repeated here. + +--- + +### Basic Concepts + +#### Models and Inheritance + +##### What are Pydantic models? + +Pydantic models are Python classes that define data structures and their constraints. Think of them like UML classes with built-in data validation - each model defines what fields are allowed and what types of data they can contain. + +##### Model Base Classes and Inheritance + +**What is a "base class"?** A base class defines common fields and behaviors that other classes can reuse. Think of it like a slide template - you create one layout, then make specific slides that use that structure. + +**What is "inheritance"?** Inheritance means one class automatically gets all the fields and behaviors from another class. If Building inherits from OvertureFeature, it automatically gets all of Feature's fields (like `id`, `geometry`) plus any new fields you add to Building (like `height`). When multiple parent classes have the same field name, Python uses a [specific order](https://docs.python.org/3/tutorial/classes.html#multiple-inheritance) to determine which one takes precedence. + +**@no_extra_fields** - Use for structured data components that should reject unknown fields: + +```python +from overture.schema.system.model_constraint import no_extra_fields + +@no_extra_fields +class Address(BaseModel): + """A postal address - no extra fields allowed.""" + street: str + city: str + postal_code: str | None = None + # Any field not defined here will cause validation to fail +``` + +**OvertureFeature[ThemeT, TypeT]** - A generic base class for all geospatial features with typed theme and type parameters: + +```python +from typing import Literal +from overture.schema.common import OvertureFeature +from overture.schema.system.numeric import float64 + +class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): + """A building feature with strongly-typed theme and type.""" + # Inherits: id, theme, type, geometry, bbox, version, sources + height: float64 | None = None +``` + +**What does "generic" mean?** The `OvertureFeature[ThemeT, TypeT]` syntax makes OvertureFeature a "generic" class - think of it like a template that can be customized with specific values. The square brackets `[]` contain "type parameters" that specify exactly what theme and type this feature represents. + +**What are ThemeT and TypeT?** These are placeholders for specific text values: + +- **ThemeT**: The data theme (like "buildings", "places", "transportation") +- **TypeT**: The specific feature type within that theme (like "building", "place", "segment") + +**What is `Literal`?** `Literal` means the field must be exactly one of the specified values - nothing else is allowed. So `Literal["buildings"]` means this theme can only be "buildings", not any other string. + +By specifying `OvertureFeature[Literal["buildings"], Literal["building"]]`, you're saying "this is a Feature that must have theme='buildings' and type='building'" - no other values are allowed. This prevents mistakes like accidentally creating a building with theme="places". + +##### Inheritance Patterns + +**Multiple inheritance** combines fields from several base classes: + +```python +from typing import Literal +from overture.schema.common import OvertureFeature +from overture.schema.common.level import Stacked +from overture.schema.common.names import Named +from overture.schema.system.numeric import float64 + +class Building(OvertureFeature[Literal["buildings"], Literal["building"]], Named, Stacked): + # Gets fields from Feature: id, theme, type, geometry, etc. + # Gets fields from Named: names + # Gets fields from Stacked: level + # Plus its own fields: + height: float64 | None = None +``` + +##### Field Aliases + +Sometimes you need a field name that conflicts with Python keywords or conventions (hint: you'll get an error when you try to use it). Use `Field(alias="")` to map between Python-friendly field names and the actual data field names: + +```python +from typing import Annotated +from pydantic import Field + +class Building(OvertureFeature): + # Use class_ in Python code, but "class" in the actual data + class_: Annotated[str | None, Field(alias="class")] = None + + # Other common cases might include: + type_: Annotated[str | None, Field(alias="type")] = None # if type conflicts + from_: Annotated[str | None, Field(alias="from")] = None # from is a keyword +``` + +A common example is `class_` with `Field(alias="class")` since "class" is a Python keyword but a common field name in data schemas. + +#### Field Types + +##### Required vs Optional Fields + +```python +class Building(OvertureFeature): + # Required field (no default value) + geometry: Geometry + + # Optional field (has default value of None) + height: float64 | None = None +``` + +- **Required fields**: Must be provided when creating an instance +- **Optional fields**: Can be omitted; they have a default value (usually `None`) + +> [!WARNING] +> **Always use `None` defaults** for optional fields. Non-`None` defaults create ambiguity between schema defaults and actual data values. + +**Why do non-`None` defaults cause problems?** + +1. **Data transformation ambiguity**: Pydantic adds default values that weren't in the input, making it impossible to distinguish between original data and schema defaults. + +2. **Schema vs. data confusion**: Schemas serve multiple purposes: + - **Validation only**: Check if existing data is valid (shouldn't transform it) + - **Data processing**: Parse and potentially transform data with Pydantic + - **Documentation**: Show developers what fields exist and what they mean + +3. **Implicit semantic meaning**: Default values encode business logic into the schema, which should be in business logic instead. + +**Better approaches:** + +1. **Use `None` and document semantics:** + + ```python + access_policy: Annotated[ + str | None, + Field(description="Access policy for the place. When absent, assume 'open'") + ] = None + ``` + +2. **Always populate in data pipeline:** + + ```python + # In your data processing pipeline, always set the value + place.access_policy = place.access_policy or "open" + ``` + +Keep the schema separate from business logic. The schema describes the shape of data, not the business rules about what missing values mean. + +##### Numeric Types + +**Always use specific numeric types instead of Python's generic `int`/`float`:** + +```python +from overture.schema.system.model_constraint import no_extra_fields +from overture.schema.system.numeric import ( + int8, int32, int64, # Signed integers + uint8, uint16, uint32, # Unsigned integers + float32, float64 # Floating point +) + +@no_extra_fields +class MyModel(BaseModel): + # Signed integers with specific ranges + level: int8 | None = None # -128 to 127 + year: int32 | None = None # -2,147,483,648 to 2,147,483,647 + timestamp: int64 | None = None # Full 64-bit range + + # Unsigned integers (0 and positive only) + red_value: uint8 | None = None # 0 to 255 (like RGB values) + port: uint16 | None = None # 0 to 65,535 (like network ports) + population: uint32 | None = None # 0 to 4,294,967,295 + + # Floating point numbers + height: float64 | None = None # Double precision (recommended) + ratio: float32 | None = None # Single precision +``` + +**When to use each:** + +- **`int32`**: Most integer fields (years, counts, IDs) +- **`uint8`**: Small positive values (0-255), like color components, confidence percentages +- **`uint16`**: Medium positive values (0-65K), like ports, small counts +- **`uint32`**: Large positive values, like population, large IDs +- **`float64`**: Most decimal numbers (heights, coordinates, measurements) - **this is the default choice** +- **`float32`**: When space is critical and precision isn't + +When in doubt, use `int32` (equivalent to `int`), `int64` (equivalent to `long`, although [not safely representable in JSON](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER)), or `float64` (equivalent to `double`). + +**Why specific numeric types matter:** + +The specific numeric types are crucial for data interchange and storage compatibility: + +- **Cross-platform consistency**: Ensures the same data types across Python, Arrow, Parquet, and other geospatial tools +- **Round-trip compatibility**: Data round-trips cleanly between Parquet files, databases (PostgreSQL, Trino), Shapefiles, and JSON Schema +- **Value range validation**: Prevents invalid values (e.g., negative heights, RGB values > 255) +- **Storage efficiency**: `uint8` uses 1 byte vs `int64` which uses 8 bytes +- **Built-in validation**: These types use Pydantic `Field()` constraints to validate ranges (e.g., `Field(ge=0, le=100)` ensures values stay within bounds) + +##### Union Types + +Union types allow a field to accept multiple different types. The `|` symbol means "or": + +```python +from typing import Literal + +class Building(OvertureFeature): + # This field can be either a string OR None (most common union) + name: str | None = None + + # This field can be one of specific string values OR None + status: Literal["active", "inactive", "pending"] | None = None +``` + +**Common union patterns:** + +```python +# Optional field (most common union) +height: float64 | None = None # Can be a number or missing + +# Specific string values (an alternative to enums where descriptions aren't needed) +priority: Literal["low", "medium", "high"] | None = None + +# Boolean or None +is_verified: bool | None = None +``` + +**Union best practices:** + +- Keep unions simple - avoid more than 2-3 types when possible +- Optional fields will include `None` to become optional +- Use `Literal` values for specific string choices rather than mixing basic types +- **Avoid mixed-type unions** like `str | int32` - these don't work well with many storage layers + +> [!WARNING] +> **Storage compatibility**: Mixed-type unions (combining different basic types like `str | int32`) don't work with Parquet and other storage layers. Use `Literal` values or separate fields instead. + +#### Field Enhancement + +##### Adding Descriptions and Constraints with Annotated + +`Annotated` is Python's way to add extra information (metadata) to a type without changing the type itself. Think of it like adding notes or constraints to a field definition. + +**Basic concept:** + +```python +from typing import Annotated +from pydantic import Field + +# Without Annotated - just the type +height: float64 | None = None + +# With Annotated - type + extra information +height: Annotated[ + float64 | None, # The actual type (what kind of data) + Field(description="Height in meters") # Extra metadata +] = None +``` + +**What goes inside `Annotated`:** + +1. **First argument**: The actual type (`str`, `int32`, `list[str]`, etc.) +2. **Additional arguments**: Metadata like constraints, descriptions, validation rules + +##### Field Constraints + +Use Pydantic's `Field()` function to add constraints and descriptions: + +**Numeric constraints:** + +```python +from typing import Annotated +from pydantic import Field + +class Building(OvertureFeature): + # Range constraints + height: Annotated[ + float64 | None, + Field(ge=0, le=1000, description="Height in meters (0-1000m)") + ] = None + + # Integer constraints + floors: Annotated[ + int32 | None, + Field(gt=0, lt=200, description="Number of floors (1-199)") + ] = None +``` + +**Numeric constraint options:** + +- **`ge`**: Greater than or equal to (≥) +- **`gt`**: Greater than (>) +- **`le`**: Less than or equal to (≤) +- **`lt`**: Less than (<) + +**String constraints:** + +```python +class Place(OvertureFeature): + # Length constraints + name: Annotated[ + str | None, + Field(min_length=1, max_length=100, description="Place name (1-100 chars)") + ] = None + + # Pattern matching + postal_code: Annotated[ + str | None, + Field(pattern=r"^\d{5}(-\d{4})?$", description="US postal code") + ] = None +``` + +**String constraint options:** + +- **`min_length`**: Minimum string length +- **`max_length`**: Maximum string length +- **`pattern`**: Regular expression pattern (regex) + +#### Collections and Lists + +##### Basic List Fields + +```python +class Building(Feature): + # Simple list of strings + tags: list[str] | None = None + + # List of complex objects + access_rules: list[AccessRule] | None = None +``` + +##### List Constraints + +```python +from overture.schema.system.field_constraint import UniqueItemsConstraint + +class Building(OvertureFeature): + # List with size and uniqueness constraints + categories: Annotated[ + list[str] | None, + Field(min_length=1, max_length=10, description="1-10 categories"), + UniqueItemsConstraint() # Must come AFTER Field() + ] = None +``` + +**List constraint options:** + +- **`min_length`**: Minimum number of items +- **`max_length`**: Maximum number of items +- **`UniqueItemsConstraint()`**: No duplicate items (custom validation) + +**Important**: `UniqueItemsConstraint()` must come AFTER `Field()` for proper JSON Schema generation. + +> [!CAUTION] +> **Constraint order matters**: Always put `Field()` before `UniqueItemsConstraint()` or JSON Schema generation will create `minLength` (string constraint) instead of `minItems` (array constraint). +> +> **Why**: Pydantic processes annotations in order for JSON Schema generation. `Field()` must come first to set up the field properly. For lists, `Field(min_length=1)` creates a `minItems` constraint in the JSON Schema because the type immediately before it is a list. If `UniqueItemsConstraint()` comes first, Pydantic doesn't see the list type and treats `min_length` as a string constraint (`minLength`). + +##### List Behavior + +Lists maintain their **insertion order** (the order data exists in the field), but they are **not automatically sorted**. + +#### Enumerations + +**What is an enumeration (enum)?** An enumeration is a way to define a fixed set of allowed values for a field. Think of it like a multiple-choice question - you define all the valid answers ahead of time, and users can only pick from those options. + +For example, instead of allowing any string for a "status" field (which could lead to typos like "activ" or "Active"), you create an enum with exactly "active", "inactive", and "pending" as the only allowed values. + +**Enums vs Literal:** You can achieve similar results with `Literal["active", "inactive", "pending"]`, but formal enums are better when you need descriptions, documentation, or want to reuse the same set of values across multiple fields. + +##### Creating Enums + +Enums define a fixed set of allowed values: + +```python +from enum import Enum + +class BuildingClass(str, Enum): + """Further delineation of the building's built purpose.""" + + RESIDENTIAL = "residential" + COMMERCIAL = "commercial" + INDUSTRIAL = "industrial" + CIVIC = "civic" + +# Usage in a model +class Building(OvertureFeature): + class_: Annotated[BuildingClass | None, Field(alias="class")] = None +``` + +##### Documenting Enum Values + +Add documentation to describe what the enum and its values mean. In Python, you do this with **docstrings** - text enclosed in triple quotes `"""` that describes what something does: + +Use `DocumentedEnum` from `overture.schema.system.doc` when enum members need their own descriptions for code generation and documentation tooling. Each member takes a `(value, description)` tuple: + +```python +from overture.schema.system.doc import DocumentedEnum + +class VehicleType(str, DocumentedEnum): + """Types of vehicles for transportation.""" + + CAR = ("car", "Standard passenger vehicle") + TRUCK = ("truck", "Commercial freight vehicle") + BICYCLE = ("bicycle", "Human-powered two-wheeler") + MOTORCYCLE = ("motorcycle", "Motorized two-wheeler") +``` + +Members without descriptions use the plain value form -- documentation is optional per-member: + +```python +class ConnectionState(str, DocumentedEnum): + CONNECTED = "connected" + DISCONNECTED = "disconnected" + QUIESCING = ( + "quiescing", + "Gracefully shutting down, rejecting new requests but completing existing ones", + ) +``` + +Use `DocumentedEnum` over plain `str, Enum` when the enum members' semantics aren't obvious from their names and downstream tools (code generators, documentation renderers) need access to member-level descriptions. Use plain `str, Enum` for self-explanatory values. + +##### Why str, Enum? + +Inheriting from `str, Enum` makes enum values work as both enums and strings, which is useful for JSON serialization and compatibility. + +--- + +### Advanced Patterns + +#### Relationship Patterns + +##### What are relationships? + +Relationships represent connections between different features or models. Think of them like links that connect related pieces of information — for example, a building part that is structurally part of a building, or a division area that is administratively nested under a division. + +Pydantic provides several ways to express these relationships, each suited to different use cases and complexity levels. Before choosing a pattern, it's important to understand the **semantic type** of the relationship you're modeling. + +--- + +##### Semantic Relationship Types + +Every relationship between two features carries a semantic meaning about coupling strength, lifecycle dependency, and ownership. The schema defines four relationship types, ordered from strongest to weakest coupling. The types describe the *nature* of the link, not which feature is "parent" or "child." Direction is implicit: the feature holding the reference is the source, and the type it references is the destination. + +###### `COMPOSITION` — Structural Whole-Part + +A structural whole-part relationship with lifecycle dependency. The part has no independent meaning outside the whole. Deleting the whole invalidates the part. + +**Test question:** *"If I delete the whole, does keeping the part orphaned make any sense at all?"* If the answer is no, it's `COMPOSITION`. + +**Examples:** +- `BuildingPart` → `Building` — part *is part of* building +- `DivisionBoundary` → `Division` — boundary line *defines the boundary of* division + +###### `AGGREGATION` — Grouping Without Lifecycle Dependency + +A grouping or collection relationship where both members are independently viable. No lifecycle dependency — the member survives reassignment to another group or orphaning. + +**Test question:** *"Can both sides belong to something else or nothing and still be a valid map feature?"* If yes, they form an `AGGREGATION`. + +**Examples:** +- `Route` → `Segment` — route *groups* segments +- `TrailSegment` → `NationalPark` — segment *is grouped by* park + +###### `HIERARCHY` — Organizational Nesting + +An organizational or classificatory nesting relationship. This is not about structural assembly — it's about administrative parentage, taxonomy, or categorization. + +**Test question:** *"Is this about organizational subordination rather than structural assembly?"* If yes, it's `HIERARCHY`. + +**Examples:** +- `DivisionArea` → `Division` — area *is child of* division +- `Division` → `Division` — child division nested under parent + +###### `ASSOCIATION` — Peer-Level Reference + +A peer-level reference with no ownership, containment, or nesting. Neither feature depends on or contains the other. This is the fallback when none of the stronger types apply. + +**Test question:** *"Are these just peers that know about each other?"* If yes, it's `ASSOCIATION`. + +**Examples:** +- `Segment` → `Connector` — segment references its start/end connector +- `Building` → `Address` — a building references its address, neither owns the other + +--- + +##### Selection Priority + +When a relationship could fit multiple types, the choice follows a **diamond decision**: start at the top, fork in the middle based on the *kind* of coupling, and fall through to the bottom only when no stronger type applies. + +```text + COMPOSITION + / \ +AGGREGATION HIERARCHY + \ / + ASSOCIATION +``` + +| If the relationship implies... | Use | +|---------------------------------------------------------|----------------| +| Structural whole-part with lifecycle dependency | `COMPOSITION` | +| Geometric boundary definition (lifecycle dependent) | `COMPOSITION` | +| Grouping/collection without lifecycle dependency | `AGGREGATION` | +| Organizational nesting or classification tree | `HIERARCHY` | +| Peer-level reference, no ownership or nesting | `ASSOCIATION` | + +--- + +##### The `role` Field + +The `Reference` annotation accepts an optional `role` parameter — a snake_case string that further qualifies the relationship from the source's perspective. It has no effect on schema validation; it is informational metadata for documentation and tooling. + +Use `role` when the semantic type alone is ambiguous. For example, multiple `HIERARCHY` references on the same model can be disambiguated: + +```python +# Without role: two HIERARCHY references to Division — which is which? +parent_division_id: Annotated[Id, Reference(Relationship.HIERARCHY, Division)] +capital_division_ids: Annotated[list[Id], Reference(Relationship.HIERARCHY, Division)] + +# With role: unambiguous +parent_division_id: Annotated[Id, Reference(Relationship.HIERARCHY, Division, role="child_of")] +capital_division_ids: Annotated[list[Id], Reference(Relationship.HIERARCHY, Division, role="has_as_capital")] +``` + +The `role` must be a non-empty snake_case string (lowercase letters, digits, underscores). It describes the source's role relative to the target using source-perspective phrasing. + +--- + +##### Implementation Patterns + +###### 1. Direct References (Foreign Keys) + +The fundamental pattern is a direct reference where one feature "points to" another using an ID field with type safety and semantic information. + +```python +from typing import Annotated, Literal +from pydantic import Field +from overture.schema.common import OvertureFeature +from overture.schema.system.ref import Id, Reference, Relationship + +# COMPOSITION — part points to its whole +class BuildingPart(OvertureFeature[Literal["buildings"], Literal["building_part"]]): + """A structural part of a building.""" + building_id: Annotated[ + Id, + Reference(Relationship.COMPOSITION, Building, role="part_of"), + Field(description="The building to which this part belongs") + ] + +# HIERARCHY — child points to parent +class DivisionArea(OvertureFeature[Literal["divisions"], Literal["division_area"]]): + """Area polygon nested under a division.""" + division_id: Annotated[ + Id, + Reference(Relationship.HIERARCHY, Division, role="child_of"), + Field(description="Division ID of the parent division of this area.") + ] + +# ASSOCIATION — peer reference, no ownership +class ConnectorReference(BaseModel): + """Reference to a connector feature.""" + connector_id: Annotated[ + Id, + Reference(Relationship.ASSOCIATION, Connector, role="connects_to") + ] +``` + +###### 2. Association as a Separate Feature (Complex Relationships) + +When the relationship itself needs to store information, create a dedicated feature to represent it. This applies regardless of the semantic type — any of the four types can carry metadata. + +**Simple relationship (use Pattern 1):** +- "Building Part A is part of Building B" — just needs an ID reference. + +**Complex relationship (use Pattern 2):** +- "Admin Area X has City Center Y as its primary center since 2010 with 85% confidence" — the relationship has properties. + +```python +class AdminCityCenterAssociation( + OvertureFeature[Literal["associations"], Literal["admin_city_center"]] +): + """Describes how an administrative area relates to a city center.""" + + admin_area_id: Annotated[ + Id, + Reference(Relationship.ASSOCIATION, AdminArea) + ] + city_center_id: Annotated[ + Id, + Reference(Relationship.ASSOCIATION, CityCenter) + ] + + # Information about the relationship itself + relationship_type: Literal["primary_center", "secondary_center"] = "primary_center" + established_date: str | None = None + confidence_score: Annotated[float64, Field(ge=0.0, le=1.0)] | None = None +``` + +**When to use separate association features:** +- The relationship has properties (confidence scores, dates, types, notes). +- Many-to-many connections exist. +- You need to query the relationships independently. + +###### 3. Collection References + +When a feature needs to reference multiple other features, use a list of references. The semantic type still matters. + +```python +# COMPOSITION — boundary defines two divisions +class DivisionBoundary(OvertureFeature[Literal["divisions"], Literal["division_boundary"]]): + """A boundary line between two divisions.""" + division_ids: Annotated[ + list[Annotated[Id, Reference(Relationship.COMPOSITION, Division, role="boundary_of")]], + Field(min_length=2, max_length=2, description="Left and right divisions"), + ] + +# AGGREGATION — route groups segments +class Route(OvertureFeature[Literal["transportation"], Literal["route"]]): + """A transportation route passing through multiple segments.""" + segment_ids: Annotated[ + list[Id], + Reference(Relationship.AGGREGATION, TransportationSegment, role="groups"), + Field(min_length=1, description="Ordered segments in this route"), + UniqueItemsConstraint(), + ] +``` + +--- + +##### Best Practices + +###### Always Use Reference Annotations + +Include `Reference` annotations for semantic clarity and documentation: + +```python +# Good — complete relationship information with semantic type and role +division_id: Annotated[ + Id, + Reference(Relationship.HIERARCHY, Division, role="child_of"), + Field(description="Division ID of the parent division of this area.") +] + +# Avoid — missing semantic information +division_id: Id +``` + +###### Choose the Right Semantic Type First, Then the Right Pattern + +1. **Determine the semantic type** using the selection priority and test questions above. +2. **Then choose the implementation pattern:** + - Simple relationships → Direct references (Pattern 1) + - Relationships with metadata → Separate association features (Pattern 2) + - One-to-many references → Collection references (Pattern 3) + +#### Discriminated Unions + +**What is a discriminated union?** A discriminated union is a type that can be backed by one of several different models, where a specific field (the "discriminator") determines which model it actually is. Think of it like a form that changes its fields based on a category selection. + +```python +from typing import Annotated, Literal +from pydantic import Field +from overture.schema.common import OvertureFeature + +# Base class with common fields +class TransportationSegment(OvertureFeature[Literal["transportation"], Literal["segment"]]): + subtype: Subtype # This is the discriminator field + # ... common fields for all segments + +# Specific segment types +class RoadSegment(TransportationSegment): + subtype: Literal[Subtype.ROAD] # Must be "road" + class_: Annotated[RoadClass, Field(alias="class")] + speed_limits: SpeedLimits | None = None + # ... road-specific fields + +class RailSegment(TransportationSegment): + subtype: Literal[Subtype.RAIL] # Must be "rail" + class_: Annotated[RailClass, Field(alias="class")] + rail_flags: RailFlags | None = None + # ... rail-specific fields + +# Union type that automatically picks the right model based on subtype +Segment = Annotated[ + RoadSegment | RailSegment | WaterSegment, + Field(discriminator="subtype") +] +``` + +The `discriminator="subtype"` tells Pydantic to look at the `subtype` field to determine which specific model to use. If `subtype` is "road", it uses `RoadSegment`; if "rail", it uses `RailSegment`. + +##### Abstract vs Concrete Classes + +**What's the difference?** In UML and traditional OOP, abstract classes cannot be instantiated - they serve as templates for concrete classes. In Pydantic, by default, **all classes are concrete** (can be instantiated), but you can make classes abstract when needed. + +**Current pattern (all concrete):** + +```python +# Both can be instantiated as map features +base_segment = TransportationSegment(subtype=Subtype.ROAD, geometry=...) # Valid +road_segment = RoadSegment(subtype=Subtype.ROAD, geometry=..., class_=...) # Valid +``` + +**Making the base class abstract:** + +```python +from abc import ABC, abstractmethod +from typing import Annotated, Literal +from pydantic import Field + +class TransportationSegment(OvertureFeature[Literal["transportation"], Literal["segment"]], ABC): + """Abstract base - cannot be instantiated directly.""" + + subtype: Subtype # Discriminator field + + @abstractmethod + def get_speed_limit(self) -> float: + """Each concrete type must implement this.""" + pass + +class RoadSegment(TransportationSegment): + """Concrete class - can be instantiated.""" + subtype: Literal[Subtype.ROAD] + speed_limits: SpeedLimits | None = None + + def get_speed_limit(self) -> float: + return self.speed_limits.max_speed if self.speed_limits else 50.0 + +# Now only concrete classes can be instantiated +# base_segment = TransportationSegment(...) # TypeError: Can't instantiate abstract class +road_segment = RoadSegment(subtype=Subtype.ROAD, geometry=...) # Valid +``` + +**Registration pattern (recommended when working with Overture models):** + +Instead of making classes abstract, we use **entry point registration** where only specific concrete types are discoverable as map features: + +```toml +# In packages/overture-schema-theme-transportation/pyproject.toml +[project.entry-points."overture.models"] +connector = "overture.schema.transportation:Connector" +segment = "overture.schema.transportation:Segment" +``` + +**Real example:** See [`packages/overture-schema-theme-transportation/src/overture/schema/transportation/segment/__init__.py`](packages/overture-schema-theme-transportation/src/overture/schema/transportation/segment/__init__.py) where: + +- **`Segment`** is a discriminated union: `RoadSegment | RailSegment | WaterSegment` +- **`TransportationSegment`** is the concrete base class that all segment types inherit from +- **Individual segment types** (`RoadSegment`, `RailSegment`, `WaterSegment`) are NOT directly registered + +**This registration pattern means:** + +1. Only **`Segment`** (the union type) is discoverable as an official map feature +2. The union automatically resolves to the correct concrete type based on the `subtype` field +3. All classes (`TransportationSegment`, `RoadSegment`, etc.) can be reused as base classes for alternate implementations + +#### Pattern Properties (Constrained Key-Value Maps) + +**What are pattern properties?** Pattern properties let you create key-value maps where the keys must follow a specific pattern (like language codes) and values have specific types. + +```python +from typing import Annotated +from pydantic import BaseModel, Field + +@no_extra_fields +class Names(BaseModel): + primary: str + + # Keys (strings) must match a language tag pattern, values are strings + common: Annotated[ + dict[ + # The key type + Annotated[ + str, + Field( + pattern=r"^[a-z]{2,3}(-[A-Z]{2})?$", + description="Language tag (e.g., 'en', 'es-MX')" + ) + ], + str, # The value type + ], + Field(json_schema_extra={"additionalProperties": False}), + ] | None = None +``` + +**Example data:** + +```json +{ + "primary": "New York City", + "common": { + "es": "Ciudad de Nueva York", + "fr": "New York", + "zh-CN": "纽约市" + } +} +``` + +The `additionalProperties: False` ensures only keys matching the pattern are allowed when generated JSON Schema is used. + +#### Nested List Validation + +**What is nested list validation?** This pattern validates both the outer list and the inner structure of each item, with constraints at multiple levels. + +```python +from typing import Annotated +from pydantic import Field + +# Each item has its own field validation +@no_extra_fields +class HierarchyItem(BaseModel): + division_id: str + name: str + +class Division(OvertureFeature): + # Nested list validation: outer list AND inner lists both have length constraints + hierarchies: Annotated[ + list[ # Outer list + Annotated[ + list[HierarchyItem], # Inner list + Field(min_length=1) # Inner list must have at least 1 item + ] + ], + Field(min_length=1) # Outer list must have at least 1 hierarchy + ] +``` + +This creates validation at three levels: + +1. **Individual items**: Each `HierarchyItem` validates its own fields (`division_id`, `name`) +2. **Inner lists**: Each inner list must have at least 1 item (`min_length=1`) +3. **Outer list**: The `hierarchies` field must have at least 1 inner list (`min_length=1`) + +#### Type Aliases for Reusable Patterns + +**What are type aliases?** Type aliases let you create custom names for complex or frequently-used types. Think of them like creating shortcuts or nicknames for long type definitions. + +**What is `NewType`?** `NewType` creates a distinct type that's based on an existing type but is treated as different for type checking purposes. This helps prevent mistakes like using an email address where you need a country code, or using a person's name where you need an ID - they're all strings, but they have different meanings and shouldn't be interchangeable. + +**Note**: `NewType` is primarily useful when working with Pydantic models in Python code (development, testing, certain data processing tasks). It doesn't affect data validation or JSON Schema generation - it's a development tool to catch mistakes before they happen. + +> [!WARNING] +> **Naming conflicts**: Never use the same name for a model class and type alias in the same module - this creates circular references and confusing code. + +**Guidelines:** + +- **Model classes**: Use noun names (`SourceItem`, `AccessRule`, `GeometricScope`) +- **Type aliases**: Use plural or descriptive names (`Sources`, `AccessRules`, `ConnectivityData`) +- **Avoid using the same names**: Use different names for models and type aliases, even if they're related + +```python +from typing import NewType, Annotated +from pydantic import BaseModel, Field + +# Create distinct types for different kinds of strings +SegmentId = NewType("SegmentId", str) # IDs are strings, but distinct +CountryCode = NewType("CountryCode", str) # Country codes are strings, but distinct + +# Create aliases for complex field patterns +EmailList = NewType("EmailList", Annotated[ + list[str], + Field(min_length=1, description="List of email addresses") +]) + +@no_extra_fields +class Contact(BaseModel): + # Clear, self-documenting field types + id: SegmentId # Can't accidentally use a CountryCode here + country: CountryCode # Can't accidentally use a SegmentId here + emails: EmailList # Reusable validation pattern +``` + +**Why use type aliases?** + +1. **Prevent mistakes**: `SegmentId` and `CountryCode` are both strings, but you can't mix them up when they're created using `NewType` +2. **Reusable patterns**: Define complex field validation once, use it many times +3. **Self-documenting code**: `EmailList` is clearer than `list[str]` +4. **Consistency**: Everyone uses the same validation rules for the same concept + +--- + +### Integration Guide + +#### Project Architecture + +##### File Organization + +Organize code by scope and avoid circular imports: + +**Cross-theme shared**: `overture-schema-common` package + +- Used by multiple themes (e.g., `OvertureFeature`, `Names`, `Sources`, `Scope`) + +**Theme-level shared**: Theme package root (e.g., `overture-schema-theme-transportation/src/overture/schema/transportation/`) + +- Used by multiple types within a theme (e.g., `AccessRules`, `RoadSurface`) + +**Type-specific**: Type subdirectory (e.g., `overture-schema-theme-transportation/src/overture/schema/transportation/segment/`) + +- Only used by one specific type (e.g., `SegmentType`, `LaneConfiguration`) + +**File type rules:** + +- **`models.py`**: Pydantic model classes (and type aliases that reference models) +- **`enums.py`**: Enum classes only, no project imports +- **`types.py`**: Type aliases that don't reference models, no project imports + +##### Import Organization + +```python +# Standard library imports first +from typing import Annotated, Literal, NewType +from enum import Enum + +# Third-party imports +from pydantic import BaseModel, ConfigDict, Field + +# Cross-theme imports +from overture.schema.common import OvertureFeature +from overture.schema.system.field_constraint import UniqueItemsConstraint +from overture.schema.system.model_constraint import no_extra_fields + +# Local imports last +from .enums import SegmentType +from .types import SegmentId, LaneWidth # Only non-model type aliases +``` + +`uv run ruff format ` will sort your imports in this order automatically. + +##### Why Not Use @field_validator or @model_validator? + +This project uses a custom validation system that generates better JSON Schema output and supports code generation (without additional work, `@field_validator` and `@model_validator` don't make their constraints discoverable). Always use constraints from `overture.schema.system` instead of using Pydantic validation decorators: + +```python +# Don't do this +@field_validator('categories') +def validate_categories_unique(cls, v): + if v and len(v) != len(set(v)): + raise ValueError('Categories must be unique') + return v + +# Do this instead +from overture.schema.system.field_constraint import UniqueItemsConstraint + +class Building(OvertureFeature): + categories: Annotated[ + list[str] | None, + Field(min_length=1, description="Building categories"), + UniqueItemsConstraint() + ] = None +``` + +#### Migrating from JSON Schema + +If you're familiar with JSON Schema files (like `schema/schema.yaml`), this section helps translate those patterns to Pydantic models. + +##### How $defs and $ref Translate + +**JSON Schema approach:** + +```yaml +# In defs.yaml +"$defs": + propertyDefinitions: + address: + type: object + properties: + freeform: { type: string } + locality: { type: string } + +# In building.yaml +properties: + address: { "$ref": "../defs.yaml#/$defs/propertyDefinitions/address" } +``` + +**Pydantic approach:** + +```python +# In overture-schema-theme-addresses/src/overture/schema/addresses/address.py +@no_extra_fields +class Address(BaseModel): + """A postal address.""" + freeform: str | None = None + locality: str | None = None + +# In overture-schema-theme-buildings/src/overture/schema/buildings/building.py +class Building(OvertureFeature): + address: Address | None = None +``` + +**Primary differences:** + +- JSON Schema uses `$ref` to reference definitions; Pydantic uses direct Python imports +- JSON Schema definitions live in `$defs`; Pydantic models are regular Python classes grouped into modules +- JSON Schema allows inline definitions; Pydantic encourages separate model classes + +##### How Containers Work + +**JSON Schema containers** (like `namesContainer`, `shapeContainer`) are reusable property groups: + +```yaml +# In defs.yaml +propertyContainers: + namesContainer: + properties: + names: { "$ref": "#/$defs/propertyDefinitions/allNames" } + + shapeContainer: + properties: + height: { type: number } + num_floors: { type: integer } + +# In building.yaml +allOf: + - "$ref": ../defs.yaml#/$defs/propertyContainers/namesContainer + - "$ref": ./defs.yaml#/$defs/propertyContainers/shapeContainer +``` + +**Pydantic equivalent** uses **mixin classes**: + +```python +# In common/names.py +class Named(BaseModel): + """Properties defining the names of a feature.""" + names: Names | None = None + +# In buildings/models.py +class Shape(BaseModel): + """Properties of the building's shape.""" + height: float64 | None = None + num_floors: int32 | None = None + +# Usage with multiple inheritance +class Building(Feature, Named, Shape): + pass # "pass" means "do nothing" - Building inherits names, height, num_floors, etc. from its parents +``` + +JSON Schema containers become **mixin classes** in Pydantic that you inherit from. + +##### Common Translation Patterns + +| JSON Schema | Pydantic | Notes | +|-------------|----------|-------| +| `"$ref": "other.yaml#/path"` | `from other import Model` | Direct Python imports | +| `allOf: [ref1, ref2]` | `class Model(Base1, Base2)` | Multiple inheritance | +| `minLength: 1` | `Field(min_length=1)` | Field constraints | +| `minimum: 0, maximum: 100` | `Field(ge=0, le=100)` | Numeric ranges | +| `uniqueItems: true` | `UniqueItemsConstraint()` | Custom constraint | +| `enum: [a, b, c]` | `class E(str, Enum): A="a"` | Enum class | +| `type: ["string", "null"]` | `str \| None = None` | Optional types | +| `if/then` conditional | Custom validation constraints | Model constraints | + +--- + + +--- + +## 11. Development workflow + + +This project uses [uv](https://docs.astral.sh/uv/) for dependency management: + +```bash +# Install dependencies for the entire workspace +uv sync --all-packages + +# Run all tests and type/code quality checks +make check + +# Run tests for a specific package +uv run pytest packages/overture-schema-theme-buildings/ + +# Run tests matching a pattern +uv run pytest -k "buildings" +``` + +Auto-format / fix code to align with project expectations: + +```shell +uv run ruff check --fix +uv run ruff format +uv run docformatter --in-place --recursive packages/ +``` + +--- + +# Part III — Reference + +## 12. Gotchas + +Things that cost time if you don't know them. + +| Gotcha | What happens | Fix | +|---|---|---| +| `model_validate(geojson_dict)` | `ValidationError`: missing `theme`/`version`, `type` is `'Feature'` | Use `model_validate_json()`. Python mode expects flat data, JSON mode expects GeoJSON. | +| `model_dump()` without `by_alias` | Emits `class_`, not `class`; output won't re-validate | Always `by_alias=True` | +| `Segment.model_validate(...)` | `AttributeError` — it's a union alias, not a class | `TypeAdapter(Segment).validate_json(...)` | +| `model_names()` returns `[]` | PySpark expressions are generated, not committed | `make generate-pyspark` (or `make install`) | +| Dumps full of `null` | Unset optionals serialize explicitly | `exclude_none=True` | +| Absent list → `[]` → won't re-validate | An omitted optional list defaults to `[]` on the model, dumps as `[]`, then fails a `min_length` check on the way back in | `exclude_defaults=True`, or drop empty lists before re-validating | + +Asymmetric round-trip, worth knowing about: an optional list that is simply *absent* +from the input becomes an empty list on the model, and an empty list is not the same as +absent on the way back out. + +```python +segments = TypeAdapter(Segment) +seg = segments.validate_json(open("road-indoors.yaml-as-json").read()) # no `connectors` key +seg.connectors # [] ← not None + +flat = seg.model_dump(mode="python", by_alias=True, exclude_none=True) +flat["connectors"] # [] ← exclude_none doesn't drop it +segments.validate_python(flat) +# ValidationError: road.connectors +# List should have at least 2 items after validation, not 0 +``` + +The same document validates fine on the way in (the CLI accepts it) and fails on the way +back. `exclude_defaults=True` avoids it, as does pruning empty lists before re-validating. + +### Docs in the repo that are currently wrong + +Worth knowing so you don't follow them into a wall: + +- **`pip install overture-schema`** — in every package README. Nothing is on PyPI yet. +- **`from overture.schema import Building, Place`** — in + `packages/overture-schema/README.md`. `overture.schema` is a bare namespace root that + ships only `py.typed`. Import from `overture.schema.buildings` etc. +- **`from overture.schema import parse, discover_models, json_schema`** — `parse()` does + not exist anywhere in the codebase. `discover_models` is in + `overture.schema.system.discovery`; `json_schema` is in + `overture.schema.system.json_schema`. +- **`Building.model_validate(geojson_feature)`** — documented as supported in + `packages/overture-schema/README.md`. It isn't; see the table above. +- **The plain `overture` tag** — `overture-schema --help` suggests + `--tag overture --tag feature` for "official Overture types only". Discovery actually + emits only `feature` and `overture:theme=*` (seven tags in all), so that filter matches + nothing. +- **`analyze_type()` returning `TypeInfo` with `.kind`/`.base_type`** — in the codegen + README. It returns a 3-tuple now, and `TypeKind` no longer exists. +- **`overture-codegen list`** prints the raw `typing.Annotated[...]` repr for `Segment` + instead of a name, because the union alias has no `__name__`. +- **CLI `--help` examples render literal `\b`** — the docstrings use raw strings + (`r"""`), so Click's escape sequence isn't interpreted and the example formatting + collapses. + +--- + +## 13. Templates and quick reference + +### Reference + +#### Complete Templates + +##### Basic Model Template + +```python models.py +from typing import Annotated +from pydantic import BaseModel, Field +from overture.schema.system.model_constraint import no_extra_fields +from overture.schema.system.numeric import int8, float64 + +@no_extra_fields +class MyCustomType(BaseModel): + """Brief description of what this represents.""" + + # Required fields (no default value) + name: str + category: str + + # Optional fields (with default values) + description: str | None = None + + # Field with constraints and description + priority: Annotated[ + int8 | None, + Field( + ge=1, + le=10, + description="Priority level from 1 (lowest) to 10 (highest)" + ) + ] = None +``` + +##### Feature Template + +```python models.py +from typing import Annotated, Literal +from pydantic import Field +from overture.schema.common import OvertureFeature +from overture.schema.system.geometric import Geometry, GeometryType, GeometryTypeConstraint + +class MyFeature(OvertureFeature[Literal["my_theme"], Literal["my_type"]]): + """Description of what this feature represents.""" + + # Geometry with constraints + geometry: Annotated[ + Geometry, + GeometryTypeConstraint(GeometryType.POINT), + Field(description="Location of this feature"), + ] + + # Custom fields + my_field: str | None = None +``` + +##### Enum Template + +```python enums.py +from enum import Enum + +class MyEnum(str, Enum): + """Description of what this enum represents.""" + + VALUE_ONE = "value_one" + VALUE_TWO = "value_two" + VALUE_THREE = "value_three" +``` + +##### Model with Validation Constraints + +```python models.py +from typing import Annotated +from pydantic import BaseModel, Field +from overture.schema.system.field_constraint import UniqueItemsConstraint +from overture.schema.system.model_constraint import no_extra_fields + +@no_extra_fields +class Contact(BaseModel): + """Contact information with validation constraints.""" + + name: str + email: str | None = None + phone: str | None = None + + # List with constraints + tags: Annotated[ + list[str] | None, + Field(min_length=1, description="Contact tags"), + UniqueItemsConstraint() # No duplicate tags + ] = None +``` + +##### Association Feature Template + +```python models.py +from typing import Annotated, Literal +from pydantic import Field +from overture.schema.common import OvertureFeature +from overture.schema.system.numeric import float64 +from overture.schema.system.ref import Id, Reference, Relationship + +class MyAssociation(OvertureFeature[Literal["associations"], Literal["my_association"]]): + """Represents a relationship between two features with metadata.""" + + # References to the associated features + # Relationship takes a *kind* (COMPOSITION / AGGREGATION / HIERARCHY / + # ASSOCIATION); what the reference means is carried by `role`. Two + # references to related features need distinct roles to stay unambiguous. + feature_a_id: Annotated[ + Id, + Reference(Relationship.ASSOCIATION, FeatureA, role="connects_from"), + Field(description="First feature in the relationship") + ] + + feature_b_id: Annotated[ + Id, + Reference(Relationship.ASSOCIATION, FeatureB, role="connects_to"), + Field(description="Second feature in the relationship") + ] + + # Relationship metadata + relationship_type: Literal["primary", "secondary"] = "primary" + confidence: Annotated[float64 | None, Field(ge=0.0, le=1.0)] = None + + # Optional contextual information + notes: str | None = None +``` + +#### Quick Reference + +##### Essential Patterns (Most Common) + +```python +# Basic field types +name: str # Required string +name: str | None = None # Optional string +count: int32 # Required integer +priority: Literal["high", "medium", "low"] | None = None # Constrained values + +# Validated fields +height: Annotated[float64 | None, Field(ge=0, description="Height in meters")] = None +tags: Annotated[list[str] | None, Field(min_length=1), UniqueItemsConstraint()] = None + +# Association patterns -- Relationship is the kind, role is the meaning +parent_id: Annotated[ + Id | None, + Reference(Relationship.HIERARCHY, ParentModel, role="child_of"), +] = None +connector_ids: list[Id] # References to multiple related features +``` + +##### Model Templates + +```python +# Non-feature model +@no_extra_fields +class Address(BaseModel): + street: str + city: str | None = None + +# Feature model +class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): + geometry: Geometry + height: float64 | None = None + +# Enum +class Status(str, Enum): + ACTIVE = "active" + INACTIVE = "inactive" +``` + +##### Constraint Reference + +| Type | Constraint | JSON Schema | Example | +|------|------------|-------------|---------| +| **Numeric** | `ge=0, le=100` | `minimum`, `maximum` | `Field(ge=0, le=100)` | +| **String** | `min_length=1, pattern=r"..."` | `minLength`, `pattern` | `Field(min_length=1, pattern=r"^[A-Z]+$")` | +| **List** | `min_length=1, UniqueItemsConstraint()` | `minItems`, `uniqueItems` | `Field(min_length=1), UniqueItemsConstraint()` | +| **Custom** | `LanguageTagConstraint()` | Custom validation | `LanguageTagConstraint()` | + +##### Import Cheatsheet + +```python +# Essential imports for most models +from typing import Annotated, Literal +from enum import Enum +from pydantic import Field +from overture.schema.common import OvertureFeature +from overture.schema.system.field_constraint import UniqueItemsConstraint +from overture.schema.system.model_constraint import no_extra_fields +from overture.schema.system.numeric import int32, float64 + +# For associations and references +from overture.schema.system.ref import Id, Reference, Relationship +``` + +##### Naming Conventions + +- **Classes**: `PascalCase` (`Building`, `AccessRule`) +- **Fields**: `snake_case` (`construction_year`, `has_parts`) +- **Enums**: `UPPER_SNAKE_CASE = "value"` (`ACTIVE = "active"`) From 2126c3ddbf774512aee784331b38c52e55c7fb42 Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Wed, 19 Aug 2026 11:49:40 -0700 Subject: [PATCH 7/9] docs: carry the corrections into SCHEMA_GUIDE.md The consolidation was written against main before the doc fixes landed, so it brought the stale content forward and catalogued the rest. Ported from the corrected PYDANTIC_GUIDE.md, extracted from that file rather than retyped: the module-organization section, the import-organization example, and the container-to-mixin example built on the real `Named` / `Appearance` mixins. Section 3.2 is reframed too -- Overture publishes one shape, and GeoJSON is the representation the models support for compatibility and that the generated JSON Schema describes; the renamed heading's anchor is chased through GLOSSARY.md and two in-guide links. "Docs in the repo that are currently wrong" listed eight items; seven are fixed, so only `pip install` survives, pointing at the section that explains it. GLOSSARY.md's Tag entry said there is no plain `overture` tag, which was true when written and is what prompted adding one; it now reads eight tags, with a note on what the tag does and does not assert. The transcribed `list-types` output was regenerated by running the command. The imported guide also exposed two holes in the drift detector. A `>>>` REPL block failed to parse and dropped out of the sweep whole, taking its imports with it. And the guide deliberately shows `from overture.schema import Building` as a counter-example, which the detector read as a defect; a line carrying the counter-example glyph is exempt. Both have unit fixtures and a mutation that fails them. With those handled and the validation README's block fixed, no block in the repo is excused. Signed-off-by: Seth Fitzsimmons --- GLOSSARY.md | 10 +- SCHEMA_GUIDE.md | 571 +++++++++++++++++++------------ tests/test_documented_imports.py | 59 +++- 3 files changed, 405 insertions(+), 235 deletions(-) diff --git a/GLOSSARY.md b/GLOSSARY.md index 739d2c0f9..341fc8566 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -122,7 +122,7 @@ The GeoJSON form of a feature, where most properties are tucked under a `propert alongside a top-level `type: "Feature"`. Distinct from the [flat shape](#flat-shape), and the single most common source of confusion when validating: `model_validate()` rejects it, `model_validate_json()` accepts it. See -[Two data shapes](SCHEMA_GUIDE.md#32-the-one-thing-to-understand-two-data-shapes). +[Two representations](SCHEMA_GUIDE.md#32-the-one-thing-to-understand-two-representations). ### Discriminated union @@ -149,9 +149,11 @@ that assumes either will break. A string attached to a model during discovery, classifying it orthogonally to its name — `feature`, or `overture:theme=buildings`. Tags are what the CLI's `--tag` / `--filter` / `--exclude` options select on. They are produced by *tag providers* registered on the -`overture.tag_providers` entry-point group; third parties can register their own. Seven -tags exist today: `feature` plus one `overture:theme=*` per theme. There is no plain -`overture` tag, despite what some help text suggests. +`overture.tag_providers` entry-point group; third parties can register their own. Eight +tags exist today: `feature`, `overture`, and one `overture:theme=*` per theme. `overture` +marks a model built on Overture's feature model — it subclasses `OvertureFeature` — which +a third party's own type can also be; it is not a claim that the type belongs to the +Overture schema. ### Spec diff --git a/SCHEMA_GUIDE.md b/SCHEMA_GUIDE.md index 3929620a4..db97b2c27 100644 --- a/SCHEMA_GUIDE.md +++ b/SCHEMA_GUIDE.md @@ -371,21 +371,21 @@ overture-schema, version 1.17.1 uv run overture-schema list-types ``` ``` -address feature overture:theme=addresses -bathymetry feature overture:theme=base -building feature overture:theme=buildings -building_part feature overture:theme=buildings -connector feature overture:theme=transportation -division feature overture:theme=divisions -division_area feature overture:theme=divisions -division_boundary feature overture:theme=divisions -infrastructure feature overture:theme=base -land feature overture:theme=base -land_cover feature overture:theme=base -land_use feature overture:theme=base -place feature overture:theme=places -segment feature overture:theme=transportation -water feature overture:theme=base +address feature overture overture:theme=addresses +bathymetry feature overture overture:theme=base +building feature overture overture:theme=buildings +building_part feature overture overture:theme=buildings +connector feature overture overture:theme=transportation +division feature overture overture:theme=divisions +division_area feature overture overture:theme=divisions +division_boundary feature overture overture:theme=divisions +infrastructure feature overture overture:theme=base +land feature overture overture:theme=base +land_cover feature overture overture:theme=base +land_use feature overture overture:theme=base +place feature overture overture:theme=places +segment feature overture overture:theme=transportation +water feature overture overture:theme=base ``` ```bash @@ -732,8 +732,8 @@ snake-cased: ```python from overture.schema.system.discovery.entry_point import entry_point_class_alias -entry_point_class_alias("overture.schema.divisions:DivisionArea") # 'division_area' -entry_point_class_alias("overture.schema.places:Place") # 'place' +entry_point_class_alias("overture.schema.divisions:DivisionArea") # 'division_area' +entry_point_class_alias("overture.schema.places:Place") # 'place' ``` Short names are *not* guaranteed unique. Anyone can @@ -755,7 +755,9 @@ resolve_entry_point_key("place", registry) # ValueError: Entry-point alias 'place' is ambiguous. # Specify one of: acme.parks:Place, overture.schema.places:Place -resolve_entry_point_key("acme.parks:Place", registry) # 'acme.parks:Place' — always works +resolve_entry_point_key( + "acme.parks:Place", registry +) # 'acme.parks:Place' — always works ``` Install a package that collides and `place` simply stops being accepted, with an error @@ -1164,21 +1166,21 @@ uv run overture-schema list-types ``` ``` -address feature overture:theme=addresses -bathymetry feature overture:theme=base -building feature overture:theme=buildings -building_part feature overture:theme=buildings -connector feature overture:theme=transportation -division feature overture:theme=divisions -division_area feature overture:theme=divisions -division_boundary feature overture:theme=divisions -infrastructure feature overture:theme=base -land feature overture:theme=base -land_cover feature overture:theme=base -land_use feature overture:theme=base -place feature overture:theme=places -segment feature overture:theme=transportation -water feature overture:theme=base +address feature overture overture:theme=addresses +bathymetry feature overture overture:theme=base +building feature overture overture:theme=buildings +building_part feature overture overture:theme=buildings +connector feature overture overture:theme=transportation +division feature overture overture:theme=divisions +division_area feature overture overture:theme=divisions +division_boundary feature overture overture:theme=divisions +infrastructure feature overture overture:theme=base +land feature overture overture:theme=base +land_cover feature overture overture:theme=base +land_use feature overture overture:theme=base +place feature overture overture:theme=places +segment feature overture overture:theme=transportation +water feature overture overture:theme=base ``` Columns are: **type name**, then its **tags**. Group by a tag key: @@ -1188,9 +1190,10 @@ uv run overture-schema list-types --group-by overture:theme ``` ``` -overture:theme=buildings (2) -→ building feature overture:theme=buildings -→ building_part feature overture:theme=buildings +overture:theme=addresses (1) +→ address feature overture overture:theme=addresses + +overture:theme=base (6) ... ``` @@ -1271,10 +1274,10 @@ Each entry carries the documentation and constraints from the model declaration: ```python f = Building.model_fields["height"] -print(f.description) # 'Height of the building or part in meters.\n\n...' -print(f.metadata) # [Gt(gt=0)] -print(f.alias) # None -print(Building.model_fields["class_"].alias) # 'class' +print(f.description) # 'Height of the building or part in meters.\n\n...' +print(f.metadata) # [Gt(gt=0)] +print(f.alias) # None +print(Building.model_fields["class_"].alias) # 'class' ``` Note `class_` and its alias `class`. `class` is a Python keyword, so the field is named @@ -1902,18 +1905,18 @@ The entire GeoJSON transformation is those ten lines, in `Feature` ```python @model_serializer(mode="wrap") def __serialize_with_geo_json_support__(self, serializer, info): - data = serializer(self) # <- the flat dict, produced first + data = serializer(self) # <- the flat dict, produced first - if info.mode == "json": # <- only in JSON mode + if info.mode == "json": # <- only in JSON mode return { "type": "Feature", **({"id": data.pop("id")} if "id" in data else {}), **({"bbox": data.pop("bbox")} if "bbox" in data else {}), "geometry": data.pop("geometry"), - "properties": data, # <- everything else goes here + "properties": data, # <- everything else goes here } - return data # <- Python mode: flat, untouched + return data # <- Python mode: flat, untouched ``` Read the first line: Pydantic produces the **flat** dictionary, and only then does this @@ -2073,7 +2076,9 @@ from overture.schema.system.discovery import TagSelector, discover_models, filte models = discover_models() -buildings = filter_models(models, TagSelector(include_any=("overture:theme=buildings",))) +buildings = filter_models( + models, TagSelector(include_any=("overture:theme=buildings",)) +) ``` `TagSelector` takes `include_any` (OR scope), `require_all` (AND narrowing), and @@ -2087,7 +2092,8 @@ results: ```python from overture.schema.buildings import RoofShape -[ (m.value, m.__doc__.strip()[:30]) for m in list(RoofShape)[:2] ] + +[(m.value, m.__doc__.strip()[:30]) for m in list(RoofShape)[:2]] # [('dome', 'The shape of the roof.'), ('flat', 'The shape of the roof.')] # ^ that's the class docstring repeated, not per-member documentation ``` @@ -2204,17 +2210,20 @@ Three lines in there are load-bearing, and each gets a subsection below: | Line | Why it's written that way | Where | |---|---|---| -| `model_validate_json(json.dumps(doc))` | the file is GeoJSON, which needs JSON mode | [3.2](#32-the-one-thing-to-understand-two-data-shapes) | +| `model_validate_json(json.dumps(doc))` | the file is GeoJSON, which needs JSON mode | [3.2](#32-the-one-thing-to-understand-two-representations) | | `b.class_.value` | the field is `class_` in Python, `class` in the data | [3.3](#33-reading-fields) | | `by_alias=True, exclude_none=True` | without them the output won't re-validate | [3.4](#34-writing-data-back-out) | If the snippet above ran, you already know enough to be useful. Read on when one of those lines bites you, or read straight through if you'd rather know why now. -### 3.2 The one thing to understand: two data shapes +### 3.2 The one thing to understand: two representations -Overture data shows up in two shapes, and the models handle both — but **which one you -get depends on the Pydantic mode you use, not on the data you pass**. +Overture publishes data in one shape — flat and tabular, the column layout of the +Parquet release. The models also read and write GeoJSON, so the schema works with tools +that expect features rather than rows, and because that is the representation the +generated [JSON Schema](#61-json-schema) describes. **Which one you get depends on the +Pydantic mode you use, not on the data you pass.** | Shape | Looks like | Pydantic mode | Validate with | Dump with | |---|---|---|---|---| @@ -2243,7 +2252,7 @@ The `type: Feature` in the error message is the tell: it read the GeoJSON envelo The fix — round-trip through JSON so you're in JSON mode: ```python -building = Building.model_validate_json(json.dumps(doc)) # works +building = Building.model_validate_json(json.dumps(doc)) # works ``` Or, if you're already reading from a file or an HTTP response, skip the parse entirely: @@ -2257,8 +2266,8 @@ building = Building.model_validate_json(open("building.geojson").read()) No. It's the obvious thing to try, and neither knob does it: ```python -Building.model_validate(doc, context={"mode": "json"}) # still ValidationError -Building.model_validate(doc, strict=False) # still ValidationError +Building.model_validate(doc, context={"mode": "json"}) # still ValidationError +Building.model_validate(doc, strict=False) # still ValidationError ``` The mode isn't a setting you pass — in Pydantic it's determined by *which method you @@ -2268,8 +2277,8 @@ call*. `model_validate` is Python mode; `model_validate_json` is JSON mode. The ```python @model_validator(mode="wrap") def __validate_with_geo_json_support__(cls, data, handler, info): - if info.mode == "json": # <- set by the method, not by an argument - ... # unpack the GeoJSON envelope + if info.mode == "json": # <- set by the method, not by an argument + ... # unpack the GeoJSON envelope ``` So if you have a GeoJSON **dict** in hand, `json.dumps` it and use @@ -2305,8 +2314,15 @@ For flat data — a Parquet row, a DuckDB result, a dict of columns — `model_v the right call: ```python -row = {"id": "...", "theme": "buildings", "type": "building", "version": 1, - "geometry": ..., "height": 21.34, "class": "parking"} +row = { + "id": "...", + "theme": "buildings", + "type": "building", + "version": 1, + "geometry": ..., + "height": 21.34, + "class": "parking", +} building = Building.model_validate(row) ``` @@ -2316,10 +2332,10 @@ Model attributes are plain Python. Enums come back as enum members, geometry as `Geometry` wrapper around Shapely: ```python -building.height # 21.34 -building.class_ # -building.class_.value # 'parking' -building.num_floors # 4 +building.height # 21.34 +building.class_ # +building.class_.value # 'parking' +building.num_floors # 4 type(building.geometry) # ``` @@ -2345,15 +2361,17 @@ geojson = building.model_dump(mode="json", by_alias=True, exclude_none=True) sorted(geojson["properties"]) # ['class', 'ext_bar', 'height', 'is_underground', 'level', 'num_floors', ...] -Building.model_validate_json(json.dumps(geojson)) # OK -Building.model_validate(building.model_dump(mode="python", by_alias=True, exclude_none=True)) # OK +Building.model_validate_json(json.dumps(geojson)) # OK +Building.model_validate( + building.model_dump(mode="python", by_alias=True, exclude_none=True) +) # OK ``` Same for `model_dump_json()`: ```python -'"class":' in building.model_dump_json() # False ← emits "class_" -'"class":' in building.model_dump_json(by_alias=True) # True +'"class":' in building.model_dump_json() # False ← emits "class_" +'"class":' in building.model_dump_json(by_alias=True) # True ``` **Rule of thumb: `by_alias=True` on every dump, unless you specifically want Python @@ -2368,7 +2386,7 @@ every unset optional field as an explicit `null`. ```python from overture.schema.transportation import Segment -type(Segment) # +type(Segment) # Segment.model_validate({...}) # AttributeError: model_validate ``` @@ -2381,9 +2399,9 @@ from overture.schema.transportation import Segment segments = TypeAdapter(Segment) -seg = segments.validate_json(raw_geojson_string) # JSON mode → GeoJSON -seg = segments.validate_python(flat_row) # Python mode → flat -type(seg).__name__ # 'RoadSegment' +seg = segments.validate_json(raw_geojson_string) # JSON mode → GeoJSON +seg = segments.validate_python(flat_row) # Python mode → flat +type(seg).__name__ # 'RoadSegment' ``` Build the `TypeAdapter` once and reuse it; construction is the expensive part. @@ -2402,14 +2420,14 @@ whichever matched: ```python from overture.schema.validation import validate, validate_json -feature = validate_json(geojson_string) # JSON mode → GeoJSON shape -type(feature).__name__ # 'Building' +feature = validate_json(geojson_string) # JSON mode → GeoJSON shape +type(feature).__name__ # 'Building' -feature = validate(flat_dict) # Python mode → flat shape +feature = validate(flat_dict) # Python mode → flat shape ``` Both raise `pydantic.ValidationError` when nothing matches. The same mode rule from -[The one thing to understand first](#32-the-one-thing-to-understand-two-data-shapes) applies: `validate()` is +[The one thing to understand first](#32-the-one-thing-to-understand-two-representations) applies: `validate()` is Python mode and wants flat data, `validate_json()` is JSON mode and wants GeoJSON. Which models participate is resolved at runtime by entry-point discovery — install more @@ -2423,11 +2441,11 @@ from overture.schema.buildings import Building from overture.schema.places import Place schema = json_schema(Building) -schema["title"] # 'building' -sorted(schema) # ['$defs', 'additionalProperties', 'description', - # 'properties', 'required', 'title', 'type'] +schema["title"] # 'building' +sorted(schema) # ['$defs', 'additionalProperties', 'description', +# 'properties', 'required', 'title', 'type'] -union = json_schema(Building | Place) # unions work too → anyOf +union = json_schema(Building | Place) # unions work too → anyOf ``` Use this rather than Pydantic's `model_json_schema()`. The Overture generator treats @@ -2694,6 +2712,7 @@ If you know the type, validate against it directly for better errors and speed: ```python from overture.schema.buildings import Building + Building.model_validate_json(line) ``` @@ -2844,7 +2863,7 @@ from overture.schema.pyspark.validate import resolve_entry_point_key key = resolve_entry_point_key("building", REGISTRY) struct = REGISTRY[key].schema -type(struct).__name__ # 'StructType' +type(struct).__name__ # 'StructType' [(f.name, f.dataType.simpleString()) for f in struct.fields][:5] ``` @@ -2940,8 +2959,8 @@ that ships only a `py.typed` marker — no models, no functions. Always import f theme packages: ```python -from overture.schema.buildings import Building # ✓ -from overture.schema import Building # ✗ ImportError +from overture.schema.buildings import Building # ✓ +from overture.schema import Building # ✗ ImportError ``` @@ -2990,7 +3009,7 @@ version bumps reaching public PyPI while interim builds stay internal. Either wa consumer instruction is a one-line install — only the index URL differs. Nothing in sections 2 through 9 changes. The behavior you learn there — entry-point -discovery, the two data shapes, `by_alias`, `TypeAdapter` for `Segment` — is independent +discovery, the two representations, `by_alias`, `TypeAdapter` for `Segment` — is independent of how the packages get onto your machine. --- @@ -3068,8 +3087,9 @@ import click from overture.schema.system.discovery import discover_models, filter_models, TagSelector from overture.schema.cli.tag_options import tag_selection_options, build_selector + @click.command() -@tag_selection_options # gives you --tag / --filter / --exclude for free +@tag_selection_options # gives you --tag / --filter / --exclude for free def report(tags, filters, excludes): """Report field counts for the selected feature types.""" models = filter_models(discover_models(), build_selector(tags, filters, excludes)) @@ -3095,8 +3115,10 @@ from typing import Literal from overture.schema.common import OvertureFeature from overture.schema.system.numeric import float32 + class Vineyard(OvertureFeature[Literal["agriculture"], Literal["vineyard"]]): """A cultivated area planted with grapevines.""" + area_hectares: float32 | None = None ``` @@ -3160,9 +3182,9 @@ from overture.schema.codegen.extraction.model_extraction import extract_model from overture.schema.buildings import Building spec = extract_model(Building) -spec.name # 'Building' -spec.description # the class docstring -spec.constraints # model-level constraints +spec.name # 'Building' +spec.description # the class docstring +spec.constraints # model-level constraints for f in spec.fields[:6]: print(f"{f.name:12} required={f.is_required!s:5} {type(f.shape).__name__}") @@ -3267,9 +3289,9 @@ from overture.schema.system.discovery import ( models = discover_models() # { # ModelKey(name="building", entry_point="overture.schema.buildings:Building", -# tags=frozenset({"feature", "overture:theme=buildings"})): BuildingModel, +# tags=frozenset({"feature", "overture", "overture:theme=buildings"})): Building, # ModelKey(name="place", entry_point="overture.schema.places:Place", -# tags=frozenset({"feature", "overture:theme=places"})): PlaceModel, +# tags=frozenset({"feature", "overture", "overture:theme=places"})): Place, # ... # } @@ -3307,7 +3329,11 @@ from pydantic import BaseModel, Field # Overture common models from overture.schema.common import OvertureFeature -from overture.schema.system.geometric import Geometry, GeometryType, GeometryTypeConstraint +from overture.schema.system.geometric import ( + Geometry, + GeometryType, + GeometryTypeConstraint, +) # Validation system from overture.schema.system.field_constraint import UniqueItemsConstraint @@ -3324,9 +3350,14 @@ from overture.schema.system.string import LanguageTag # Numeric types (use these instead of int/float) from overture.schema.system.numeric import ( - int8, int32, int64, - uint8, uint16, uint32, - float32, float64 + int8, + int32, + int64, + uint8, + uint16, + uint32, + float32, + float64, ) ``` @@ -3358,9 +3389,11 @@ Pydantic models are Python classes that define data structures and their constra ```python from overture.schema.system.model_constraint import no_extra_fields + @no_extra_fields class Address(BaseModel): """A postal address - no extra fields allowed.""" + street: str city: str postal_code: str | None = None @@ -3374,8 +3407,10 @@ from typing import Literal from overture.schema.common import OvertureFeature from overture.schema.system.numeric import float64 + class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): """A building feature with strongly-typed theme and type.""" + # Inherits: id, theme, type, geometry, bbox, version, sources height: float64 | None = None ``` @@ -3402,7 +3437,10 @@ from overture.schema.common.level import Stacked from overture.schema.common.names import Named from overture.schema.system.numeric import float64 -class Building(OvertureFeature[Literal["buildings"], Literal["building"]], Named, Stacked): + +class Building( + OvertureFeature[Literal["buildings"], Literal["building"]], Named, Stacked +): # Gets fields from Feature: id, theme, type, geometry, etc. # Gets fields from Named: names # Gets fields from Stacked: level @@ -3418,6 +3456,7 @@ Sometimes you need a field name that conflicts with Python keywords or conventio from typing import Annotated from pydantic import Field + class Building(OvertureFeature): # Use class_ in Python code, but "class" in the actual data class_: Annotated[str | None, Field(alias="class")] = None @@ -3466,7 +3505,7 @@ class Building(OvertureFeature): ```python access_policy: Annotated[ str | None, - Field(description="Access policy for the place. When absent, assume 'open'") + Field(description="Access policy for the place. When absent, assume 'open'"), ] = None ``` @@ -3486,26 +3525,32 @@ Keep the schema separate from business logic. The schema describes the shape of ```python from overture.schema.system.model_constraint import no_extra_fields from overture.schema.system.numeric import ( - int8, int32, int64, # Signed integers - uint8, uint16, uint32, # Unsigned integers - float32, float64 # Floating point + int8, + int32, + int64, # Signed integers + uint8, + uint16, + uint32, # Unsigned integers + float32, + float64, # Floating point ) + @no_extra_fields class MyModel(BaseModel): # Signed integers with specific ranges - level: int8 | None = None # -128 to 127 - year: int32 | None = None # -2,147,483,648 to 2,147,483,647 - timestamp: int64 | None = None # Full 64-bit range + level: int8 | None = None # -128 to 127 + year: int32 | None = None # -2,147,483,648 to 2,147,483,647 + timestamp: int64 | None = None # Full 64-bit range # Unsigned integers (0 and positive only) - red_value: uint8 | None = None # 0 to 255 (like RGB values) - port: uint16 | None = None # 0 to 65,535 (like network ports) - population: uint32 | None = None # 0 to 4,294,967,295 + red_value: uint8 | None = None # 0 to 255 (like RGB values) + port: uint16 | None = None # 0 to 65,535 (like network ports) + population: uint32 | None = None # 0 to 4,294,967,295 # Floating point numbers - height: float64 | None = None # Double precision (recommended) - ratio: float32 | None = None # Single precision + height: float64 | None = None # Double precision (recommended) + ratio: float32 | None = None # Single precision ``` **When to use each:** @@ -3536,6 +3581,7 @@ Union types allow a field to accept multiple different types. The `|` symbol mea ```python from typing import Literal + class Building(OvertureFeature): # This field can be either a string OR None (most common union) name: str | None = None @@ -3548,7 +3594,7 @@ class Building(OvertureFeature): ```python # Optional field (most common union) -height: float64 | None = None # Can be a number or missing +height: float64 | None = None # Can be a number or missing # Specific string values (an alternative to enums where descriptions aren't needed) priority: Literal["low", "medium", "high"] | None = None @@ -3584,8 +3630,8 @@ height: float64 | None = None # With Annotated - type + extra information height: Annotated[ - float64 | None, # The actual type (what kind of data) - Field(description="Height in meters") # Extra metadata + float64 | None, # The actual type (what kind of data) + Field(description="Height in meters"), # Extra metadata ] = None ``` @@ -3604,17 +3650,16 @@ Use Pydantic's `Field()` function to add constraints and descriptions: from typing import Annotated from pydantic import Field + class Building(OvertureFeature): # Range constraints height: Annotated[ - float64 | None, - Field(ge=0, le=1000, description="Height in meters (0-1000m)") + float64 | None, Field(ge=0, le=1000, description="Height in meters (0-1000m)") ] = None # Integer constraints floors: Annotated[ - int32 | None, - Field(gt=0, lt=200, description="Number of floors (1-199)") + int32 | None, Field(gt=0, lt=200, description="Number of floors (1-199)") ] = None ``` @@ -3632,13 +3677,12 @@ class Place(OvertureFeature): # Length constraints name: Annotated[ str | None, - Field(min_length=1, max_length=100, description="Place name (1-100 chars)") + Field(min_length=1, max_length=100, description="Place name (1-100 chars)"), ] = None # Pattern matching postal_code: Annotated[ - str | None, - Field(pattern=r"^\d{5}(-\d{4})?$", description="US postal code") + str | None, Field(pattern=r"^\d{5}(-\d{4})?$", description="US postal code") ] = None ``` @@ -3666,12 +3710,13 @@ class Building(Feature): ```python from overture.schema.system.field_constraint import UniqueItemsConstraint + class Building(OvertureFeature): # List with size and uniqueness constraints categories: Annotated[ list[str] | None, Field(min_length=1, max_length=10, description="1-10 categories"), - UniqueItemsConstraint() # Must come AFTER Field() + UniqueItemsConstraint(), # Must come AFTER Field() ] = None ``` @@ -3707,6 +3752,7 @@ Enums define a fixed set of allowed values: ```python from enum import Enum + class BuildingClass(str, Enum): """Further delineation of the building's built purpose.""" @@ -3715,6 +3761,7 @@ class BuildingClass(str, Enum): INDUSTRIAL = "industrial" CIVIC = "civic" + # Usage in a model class Building(OvertureFeature): class_: Annotated[BuildingClass | None, Field(alias="class")] = None @@ -3729,6 +3776,7 @@ Use `DocumentedEnum` from `overture.schema.system.doc` when enum members need th ```python from overture.schema.system.doc import DocumentedEnum + class VehicleType(str, DocumentedEnum): """Types of vehicles for transportation.""" @@ -3850,8 +3898,12 @@ parent_division_id: Annotated[Id, Reference(Relationship.HIERARCHY, Division)] capital_division_ids: Annotated[list[Id], Reference(Relationship.HIERARCHY, Division)] # With role: unambiguous -parent_division_id: Annotated[Id, Reference(Relationship.HIERARCHY, Division, role="child_of")] -capital_division_ids: Annotated[list[Id], Reference(Relationship.HIERARCHY, Division, role="has_as_capital")] +parent_division_id: Annotated[ + Id, Reference(Relationship.HIERARCHY, Division, role="child_of") +] +capital_division_ids: Annotated[ + list[Id], Reference(Relationship.HIERARCHY, Division, role="has_as_capital") +] ``` The `role` must be a non-empty snake_case string (lowercase letters, digits, underscores). It describes the source's role relative to the target using source-perspective phrasing. @@ -3870,30 +3922,35 @@ from pydantic import Field from overture.schema.common import OvertureFeature from overture.schema.system.ref import Id, Reference, Relationship + # COMPOSITION — part points to its whole class BuildingPart(OvertureFeature[Literal["buildings"], Literal["building_part"]]): """A structural part of a building.""" + building_id: Annotated[ Id, Reference(Relationship.COMPOSITION, Building, role="part_of"), - Field(description="The building to which this part belongs") + Field(description="The building to which this part belongs"), ] + # HIERARCHY — child points to parent class DivisionArea(OvertureFeature[Literal["divisions"], Literal["division_area"]]): """Area polygon nested under a division.""" + division_id: Annotated[ Id, Reference(Relationship.HIERARCHY, Division, role="child_of"), - Field(description="Division ID of the parent division of this area.") + Field(description="Division ID of the parent division of this area."), ] + # ASSOCIATION — peer reference, no ownership class ConnectorReference(BaseModel): """Reference to a connector feature.""" + connector_id: Annotated[ - Id, - Reference(Relationship.ASSOCIATION, Connector, role="connects_to") + Id, Reference(Relationship.ASSOCIATION, Connector, role="connects_to") ] ``` @@ -3913,14 +3970,8 @@ class AdminCityCenterAssociation( ): """Describes how an administrative area relates to a city center.""" - admin_area_id: Annotated[ - Id, - Reference(Relationship.ASSOCIATION, AdminArea) - ] - city_center_id: Annotated[ - Id, - Reference(Relationship.ASSOCIATION, CityCenter) - ] + admin_area_id: Annotated[Id, Reference(Relationship.ASSOCIATION, AdminArea)] + city_center_id: Annotated[Id, Reference(Relationship.ASSOCIATION, CityCenter)] # Information about the relationship itself relationship_type: Literal["primary_center", "secondary_center"] = "primary_center" @@ -3939,16 +3990,25 @@ When a feature needs to reference multiple other features, use a list of referen ```python # COMPOSITION — boundary defines two divisions -class DivisionBoundary(OvertureFeature[Literal["divisions"], Literal["division_boundary"]]): +class DivisionBoundary( + OvertureFeature[Literal["divisions"], Literal["division_boundary"]] +): """A boundary line between two divisions.""" + division_ids: Annotated[ - list[Annotated[Id, Reference(Relationship.COMPOSITION, Division, role="boundary_of")]], + list[ + Annotated[ + Id, Reference(Relationship.COMPOSITION, Division, role="boundary_of") + ] + ], Field(min_length=2, max_length=2, description="Left and right divisions"), ] + # AGGREGATION — route groups segments class Route(OvertureFeature[Literal["transportation"], Literal["route"]]): """A transportation route passing through multiple segments.""" + segment_ids: Annotated[ list[Id], Reference(Relationship.AGGREGATION, TransportationSegment, role="groups"), @@ -3970,7 +4030,7 @@ Include `Reference` annotations for semantic clarity and documentation: division_id: Annotated[ Id, Reference(Relationship.HIERARCHY, Division, role="child_of"), - Field(description="Division ID of the parent division of this area.") + Field(description="Division ID of the parent division of this area."), ] # Avoid — missing semantic information @@ -3994,11 +4054,15 @@ from typing import Annotated, Literal from pydantic import Field from overture.schema.common import OvertureFeature + # Base class with common fields -class TransportationSegment(OvertureFeature[Literal["transportation"], Literal["segment"]]): +class TransportationSegment( + OvertureFeature[Literal["transportation"], Literal["segment"]] +): subtype: Subtype # This is the discriminator field # ... common fields for all segments + # Specific segment types class RoadSegment(TransportationSegment): subtype: Literal[Subtype.ROAD] # Must be "road" @@ -4006,16 +4070,17 @@ class RoadSegment(TransportationSegment): speed_limits: SpeedLimits | None = None # ... road-specific fields + class RailSegment(TransportationSegment): subtype: Literal[Subtype.RAIL] # Must be "rail" class_: Annotated[RailClass, Field(alias="class")] rail_flags: RailFlags | None = None # ... rail-specific fields + # Union type that automatically picks the right model based on subtype Segment = Annotated[ - RoadSegment | RailSegment | WaterSegment, - Field(discriminator="subtype") + RoadSegment | RailSegment | WaterSegment, Field(discriminator="subtype") ] ``` @@ -4040,7 +4105,10 @@ from abc import ABC, abstractmethod from typing import Annotated, Literal from pydantic import Field -class TransportationSegment(OvertureFeature[Literal["transportation"], Literal["segment"]], ABC): + +class TransportationSegment( + OvertureFeature[Literal["transportation"], Literal["segment"]], ABC +): """Abstract base - cannot be instantiated directly.""" subtype: Subtype # Discriminator field @@ -4050,14 +4118,17 @@ class TransportationSegment(OvertureFeature[Literal["transportation"], Literal[" """Each concrete type must implement this.""" pass + class RoadSegment(TransportationSegment): """Concrete class - can be instantiated.""" + subtype: Literal[Subtype.ROAD] speed_limits: SpeedLimits | None = None def get_speed_limit(self) -> float: return self.speed_limits.max_speed if self.speed_limits else 50.0 + # Now only concrete classes can be instantiated # base_segment = TransportationSegment(...) # TypeError: Can't instantiate abstract class road_segment = RoadSegment(subtype=Subtype.ROAD, geometry=...) # Valid @@ -4094,25 +4165,29 @@ segment = "overture.schema.transportation:Segment" from typing import Annotated from pydantic import BaseModel, Field + @no_extra_fields class Names(BaseModel): primary: str # Keys (strings) must match a language tag pattern, values are strings - common: Annotated[ - dict[ - # The key type - Annotated[ - str, - Field( - pattern=r"^[a-z]{2,3}(-[A-Z]{2})?$", - description="Language tag (e.g., 'en', 'es-MX')" - ) + common: ( + Annotated[ + dict[ + # The key type + Annotated[ + str, + Field( + pattern=r"^[a-z]{2,3}(-[A-Z]{2})?$", + description="Language tag (e.g., 'en', 'es-MX')", + ), + ], + str, # The value type ], - str, # The value type - ], - Field(json_schema_extra={"additionalProperties": False}), - ] | None = None + Field(json_schema_extra={"additionalProperties": False}), + ] + | None + ) = None ``` **Example data:** @@ -4138,22 +4213,24 @@ The `additionalProperties: False` ensures only keys matching the pattern are all from typing import Annotated from pydantic import Field + # Each item has its own field validation @no_extra_fields class HierarchyItem(BaseModel): division_id: str name: str + class Division(OvertureFeature): # Nested list validation: outer list AND inner lists both have length constraints hierarchies: Annotated[ list[ # Outer list Annotated[ list[HierarchyItem], # Inner list - Field(min_length=1) # Inner list must have at least 1 item + Field(min_length=1), # Inner list must have at least 1 item ] ], - Field(min_length=1) # Outer list must have at least 1 hierarchy + Field(min_length=1), # Outer list must have at least 1 hierarchy ] ``` @@ -4189,10 +4266,11 @@ SegmentId = NewType("SegmentId", str) # IDs are strings, but distinct CountryCode = NewType("CountryCode", str) # Country codes are strings, but distinct # Create aliases for complex field patterns -EmailList = NewType("EmailList", Annotated[ - list[str], - Field(min_length=1, description="List of email addresses") -]) +EmailList = NewType( + "EmailList", + Annotated[list[str], Field(min_length=1, description="List of email addresses")], +) + @no_extra_fields class Contact(BaseModel): @@ -4217,44 +4295,75 @@ class Contact(BaseModel): ##### File Organization -Organize code by scope and avoid circular imports: +Organize code by scope, and avoid circular imports. -**Cross-theme shared**: `overture-schema-common` package +**Cross-theme shared**: the `overture-schema-common` package. Definitions more than +one theme needs -- `OvertureFeature`, `Names`, `Sources`, the scoping framework. -- Used by multiple themes (e.g., `OvertureFeature`, `Names`, `Sources`, `Scope`) +**One module per feature type**: at the theme package root, named after the type in +snake_case. -**Theme-level shared**: Theme package root (e.g., `overture-schema-theme-transportation/src/overture/schema/transportation/`) - -- Used by multiple types within a theme (e.g., `AccessRules`, `RoadSurface`) +```text +packages/overture-schema-theme-buildings/src/overture/schema/buildings/ + __init__.py # re-exports the public names, declares __all__ + _common.py # shared by Building and BuildingPart + building.py # Building, BuildingSubtype, BuildingClass + building_part.py # BuildingPart +``` -**Type-specific**: Type subdirectory (e.g., `overture-schema-theme-transportation/src/overture/schema/transportation/segment/`) +The module owns everything specific to its type: the `Feature` subclass, its enums, +its NewTypes, and its supporting models. `building.py` defines `BuildingSubtype` and +`BuildingClass` next to `Building`, because nothing else uses them. -- Only used by one specific type (e.g., `SegmentType`, `LaneConfiguration`) +**Theme-level shared**: `_common.py` at the theme package root, for definitions two +or more types in the theme need. `buildings/_common.py` holds `Appearance`, +`RoofShape`, and the material enums that both `Building` and `BuildingPart` use. The +leading underscore marks the module private -- the theme's `__init__.py` re-exports +the public names from it. -**File type rules:** +**A type large enough to split**: a subpackage named after the type, applying the +same rules one level down. -- **`models.py`**: Pydantic model classes (and type aliases that reference models) -- **`enums.py`**: Enum classes only, no project imports -- **`types.py`**: Type aliases that don't reference models, no project imports +```text +packages/overture-schema-theme-transportation/src/overture/schema/transportation/ + __init__.py # re-exports from connector and segment + connector.py # Connector + segment/ + __init__.py # assembles the Segment discriminated union, re-exports + _common.py # TransportationSegment and what the arms share + road.py # RoadSegment and its supporting types + rail.py # RailSegment and its supporting types + water.py # WaterSegment +``` + +Every `__init__.py` re-exports its public names and declares `__all__`, so consumers +import from the package rather than reaching into the defining module: +`from overture.schema.buildings import Building`, not +`from overture.schema.buildings.building import Building`. Entry points name the +package root for the same reason -- `building = "overture.schema.buildings:Building"`. + +There is no `models.py` / `enums.py` / `types.py` split. An enum lives in the module +whose type uses it, and moves up to `_common.py` when a second type needs it. ##### Import Organization ```python # Standard library imports first -from typing import Annotated, Literal, NewType from enum import Enum +from typing import Annotated, Literal, NewType # Third-party imports from pydantic import BaseModel, ConfigDict, Field -# Cross-theme imports -from overture.schema.common import OvertureFeature +# Cross-theme and system imports +from overture.schema.common.scoping import Heading, Scope, scoped +from overture.schema.system.doc import DocumentedEnum from overture.schema.system.field_constraint import UniqueItemsConstraint from overture.schema.system.model_constraint import no_extra_fields -# Local imports last -from .enums import SegmentType -from .types import SegmentId, LaneWidth # Only non-model type aliases +# Local imports last -- siblings in the theme, then the module's own package +from ..connector import Connector +from ._common import SegmentSubtype, TransportationSegment ``` `uv run ruff format ` will sort your imports in this order automatically. @@ -4265,20 +4374,22 @@ This project uses a custom validation system that generates better JSON Schema o ```python # Don't do this -@field_validator('categories') +@field_validator("categories") def validate_categories_unique(cls, v): if v and len(v) != len(set(v)): - raise ValueError('Categories must be unique') + raise ValueError("Categories must be unique") return v + # Do this instead from overture.schema.system.field_constraint import UniqueItemsConstraint + class Building(OvertureFeature): categories: Annotated[ list[str] | None, Field(min_length=1, description="Building categories"), - UniqueItemsConstraint() + UniqueItemsConstraint(), ] = None ``` @@ -4312,9 +4423,11 @@ properties: @no_extra_fields class Address(BaseModel): """A postal address.""" + freeform: str | None = None locality: str | None = None + # In overture-schema-theme-buildings/src/overture/schema/buildings/building.py class Building(OvertureFeature): address: Address | None = None @@ -4351,20 +4464,31 @@ allOf: **Pydantic equivalent** uses **mixin classes**: ```python -# In common/names.py +# namesContainer -> Named, in +# overture-schema-common/src/overture/schema/common/names.py class Named(BaseModel): """Properties defining the names of a feature.""" + names: Names | None = None -# In buildings/models.py -class Shape(BaseModel): - """Properties of the building's shape.""" + +# shapeContainer -> Appearance, in buildings/_common.py, +# shared by Building and BuildingPart +class Appearance(BaseModel): + """Physical and visual properties of a building.""" + height: float64 | None = None num_floors: int32 | None = None + # ... roof and facade fields + -# Usage with multiple inheritance -class Building(Feature, Named, Shape): - pass # "pass" means "do nothing" - Building inherits names, height, num_floors, etc. from its parents +# Usage with multiple inheritance -- this is how Building is actually declared +class Building( + OvertureFeature[Literal["buildings"], Literal["building"]], + Named, + Stacked, + Appearance, +): ... # inherits names, level, height, num_floors, and the rest from its parents ``` JSON Schema containers become **mixin classes** in Pydantic that you inherit from. @@ -4437,11 +4561,13 @@ absent on the way back out. ```python segments = TypeAdapter(Segment) -seg = segments.validate_json(open("road-indoors.yaml-as-json").read()) # no `connectors` key -seg.connectors # [] ← not None +seg = segments.validate_json( + open("road-indoors.yaml-as-json").read() +) # no `connectors` key +seg.connectors # [] ← not None flat = seg.model_dump(mode="python", by_alias=True, exclude_none=True) -flat["connectors"] # [] ← exclude_none doesn't drop it +flat["connectors"] # [] ← exclude_none doesn't drop it segments.validate_python(flat) # ValidationError: road.connectors # List should have at least 2 items after validation, not 0 @@ -4452,29 +4578,11 @@ back. `exclude_defaults=True` avoids it, as does pruning empty lists before re-v ### Docs in the repo that are currently wrong -Worth knowing so you don't follow them into a wall: - -- **`pip install overture-schema`** — in every package README. Nothing is on PyPI yet. -- **`from overture.schema import Building, Place`** — in - `packages/overture-schema/README.md`. `overture.schema` is a bare namespace root that - ships only `py.typed`. Import from `overture.schema.buildings` etc. -- **`from overture.schema import parse, discover_models, json_schema`** — `parse()` does - not exist anywhere in the codebase. `discover_models` is in - `overture.schema.system.discovery`; `json_schema` is in - `overture.schema.system.json_schema`. -- **`Building.model_validate(geojson_feature)`** — documented as supported in - `packages/overture-schema/README.md`. It isn't; see the table above. -- **The plain `overture` tag** — `overture-schema --help` suggests - `--tag overture --tag feature` for "official Overture types only". Discovery actually - emits only `feature` and `overture:theme=*` (seven tags in all), so that filter matches - nothing. -- **`analyze_type()` returning `TypeInfo` with `.kind`/`.base_type`** — in the codegen - README. It returns a 3-tuple now, and `TypeKind` no longer exists. -- **`overture-codegen list`** prints the raw `typing.Annotated[...]` repr for `Segment` - instead of a name, because the union alias has no `__name__`. -- **CLI `--help` examples render literal `\b`** — the docstrings use raw strings - (`r"""`), so Click's escape sequence isn't interpreted and the example formatting - collapses. +- **`pip install overture-schema`** — in every package README. Nothing is on PyPI yet; + see [7.3](#73-what-changes-once-these-packages-are-published). Every other item that + stood here has been fixed, and `tests/test_documented_imports.py` now imports every + `overture.*` statement in every tracked Markdown file, so a broken one fails the suite + rather than accumulating here. --- @@ -4486,12 +4594,13 @@ Worth knowing so you don't follow them into a wall: ##### Basic Model Template -```python models.py +```python from typing import Annotated from pydantic import BaseModel, Field from overture.schema.system.model_constraint import no_extra_fields from overture.schema.system.numeric import int8, float64 + @no_extra_fields class MyCustomType(BaseModel): """Brief description of what this represents.""" @@ -4507,20 +4616,23 @@ class MyCustomType(BaseModel): priority: Annotated[ int8 | None, Field( - ge=1, - le=10, - description="Priority level from 1 (lowest) to 10 (highest)" - ) + ge=1, le=10, description="Priority level from 1 (lowest) to 10 (highest)" + ), ] = None ``` ##### Feature Template -```python models.py +```python from typing import Annotated, Literal from pydantic import Field from overture.schema.common import OvertureFeature -from overture.schema.system.geometric import Geometry, GeometryType, GeometryTypeConstraint +from overture.schema.system.geometric import ( + Geometry, + GeometryType, + GeometryTypeConstraint, +) + class MyFeature(OvertureFeature[Literal["my_theme"], Literal["my_type"]]): """Description of what this feature represents.""" @@ -4538,9 +4650,10 @@ class MyFeature(OvertureFeature[Literal["my_theme"], Literal["my_type"]]): ##### Enum Template -```python enums.py +```python from enum import Enum + class MyEnum(str, Enum): """Description of what this enum represents.""" @@ -4551,12 +4664,13 @@ class MyEnum(str, Enum): ##### Model with Validation Constraints -```python models.py +```python from typing import Annotated from pydantic import BaseModel, Field from overture.schema.system.field_constraint import UniqueItemsConstraint from overture.schema.system.model_constraint import no_extra_fields + @no_extra_fields class Contact(BaseModel): """Contact information with validation constraints.""" @@ -4569,20 +4683,23 @@ class Contact(BaseModel): tags: Annotated[ list[str] | None, Field(min_length=1, description="Contact tags"), - UniqueItemsConstraint() # No duplicate tags + UniqueItemsConstraint(), # No duplicate tags ] = None ``` ##### Association Feature Template -```python models.py +```python from typing import Annotated, Literal from pydantic import Field from overture.schema.common import OvertureFeature from overture.schema.system.numeric import float64 from overture.schema.system.ref import Id, Reference, Relationship -class MyAssociation(OvertureFeature[Literal["associations"], Literal["my_association"]]): + +class MyAssociation( + OvertureFeature[Literal["associations"], Literal["my_association"]] +): """Represents a relationship between two features with metadata.""" # References to the associated features @@ -4592,13 +4709,13 @@ class MyAssociation(OvertureFeature[Literal["associations"], Literal["my_associa feature_a_id: Annotated[ Id, Reference(Relationship.ASSOCIATION, FeatureA, role="connects_from"), - Field(description="First feature in the relationship") + Field(description="First feature in the relationship"), ] feature_b_id: Annotated[ Id, Reference(Relationship.ASSOCIATION, FeatureB, role="connects_to"), - Field(description="Second feature in the relationship") + Field(description="Second feature in the relationship"), ] # Relationship metadata @@ -4615,9 +4732,9 @@ class MyAssociation(OvertureFeature[Literal["associations"], Literal["my_associa ```python # Basic field types -name: str # Required string -name: str | None = None # Optional string -count: int32 # Required integer +name: str # Required string +name: str | None = None # Optional string +count: int32 # Required integer priority: Literal["high", "medium", "low"] | None = None # Constrained values # Validated fields @@ -4641,11 +4758,13 @@ class Address(BaseModel): street: str city: str | None = None + # Feature model class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): geometry: Geometry height: float64 | None = None + # Enum class Status(str, Enum): ACTIVE = "active" diff --git a/tests/test_documented_imports.py b/tests/test_documented_imports.py index 783e6af44..fb3f26131 100644 --- a/tests/test_documented_imports.py +++ b/tests/test_documented_imports.py @@ -37,14 +37,16 @@ # bare total stays put when one block breaks as another is fixed. The identity # is a digest of the block body, so reordering a document does not trip it but # editing one of these blocks does. -_EXPECTED_UNPARSEABLE = { - "PYDANTIC_GUIDE.md:43488ef4", -} +_EXPECTED_UNPARSEABLE: set[str] = set() # An import statement, matched textually -- used to police the excuse list, # where by definition `ast` cannot be applied. _OVERTURE_IMPORT = re.compile(r"^\s*(?:from|import)\s+overture\b", re.MULTILINE) +# Docs sometimes have to show what does *not* work. A line carrying this +# marker is such a case, and is exempt from resolving. +COUNTER_EXAMPLE_MARKER = "\u2717" + # Golden files are generated fixtures, not documentation. _EXCLUDED = "tests/golden/" @@ -102,9 +104,16 @@ def parse_imports(source: str) -> list[tuple[str, str | None]]: Returns `(module, attribute)` pairs; `attribute` is None for a plain `import overture.x`. Raises `SyntaxError` if the block is not valid Python. + + A REPL transcript is unwrapped first, and an import on a line marked + with `COUNTER_EXAMPLE_MARKER` is skipped -- the docs are showing what fails, and a + counter-example that resolved would be the real bug. """ + lines = _strip_repl_prompts(source).splitlines() found: list[tuple[str, str | None]] = [] - for node in ast.walk(ast.parse(source)): + for node in ast.walk(ast.parse("\n".join(lines))): + if _is_counter_example(lines, node): + continue if isinstance(node, ast.ImportFrom): # A relative import (`from . import x`) has no absolute module. if node.level or not node.module or not _is_overture(node.module): @@ -119,6 +128,29 @@ def parse_imports(source: str) -> list[tuple[str, str | None]]: return found +def _strip_repl_prompts(source: str) -> str: + """Unwrap a `>>>` transcript to the statements it contains. + + Without this a whole REPL block fails to parse and drops out of the + sweep -- silently, which is the failure mode this module exists to + prevent. + """ + if not any(line.startswith(">>> ") for line in source.splitlines()): + return source + return "\n".join( + line[4:] for line in source.splitlines() if line.startswith((">>> ", "... ")) + ) + + +def _is_counter_example(lines: list[str], node: ast.AST) -> bool: + """True if the statement's source carries the counter-example marker.""" + start = getattr(node, "lineno", None) + if start is None: + return False + end = getattr(node, "end_lineno", None) or start + return any(COUNTER_EXAMPLE_MARKER in line for line in lines[start - 1 : end]) + + def _is_overture(module: str) -> bool: return module == "overture" or module.startswith("overture.") @@ -130,7 +162,7 @@ def enum_references(source: str) -> list[tuple[str, str, str]]: their members really are class attributes, whereas a Pydantic model's fields are not, so `Place.addresses` would read as missing. """ - tree = ast.parse(source) + tree = ast.parse(_strip_repl_prompts(source)) imported = { alias.asname or alias.name: node.module for node in ast.walk(tree) @@ -286,6 +318,23 @@ def test_indented_import(self) -> None: source = "def f():\n from overture.schema.places import Place\n" assert parse_imports(source) == [("overture.schema.places", "Place")] + def test_repl_transcript(self) -> None: + """A `>>>` block would not parse at all without unwrapping.""" + source = ( + ">>> from overture.schema.buildings import Building\n" + ">>> Building.model_fields.keys()\n" + "dict_keys(['id', 'geometry'])\n" + ) + assert parse_imports(source) == [("overture.schema.buildings", "Building")] + + def test_counter_example_is_skipped(self) -> None: + """Docs showing what fails are exempt; the neighbouring line is not.""" + source = ( + "from overture.schema.buildings import Building\n" + f"from overture.schema import Building # {COUNTER_EXAMPLE_MARKER} ImportError\n" + ) + assert parse_imports(source) == [("overture.schema.buildings", "Building")] + def test_invalid_source_raises(self) -> None: with pytest.raises(SyntaxError): parse_imports('{"a": 1, ...}\nthis is not python') From 4518abd717f2729abbab1fb8885629095491ef6f Mon Sep 17 00:00:00 2001 From: Dana Bauer Date: Thu, 20 Aug 2026 01:21:48 -0400 Subject: [PATCH 8/9] docs: split SCHEMA_GUIDE.md into four audience-scoped pages The guide had grown to 4,803 lines across four parts, mixing rationale, consumer instructions, contributor reference, and troubleshooting. The consolidation issue asked for consumer and contributor material to be separated rather than interleaved; this promotes that separation from parts within one file to pages. SCHEMA_GUIDE.md 1,743 using the schema (was Part I) AUTHORING.md 1,687 extending and authoring it (was Part II + templates) CONCEPTS.md 803 why the schema is built this way (was Part 0 + digressions) TROUBLESHOOTING.md 370 symptom-indexed errors and model gotchas (was section 12) Guide sections 4-6 now delegate per-command reference to the package READMEs under packages/, which are versioned with the code they document, rather than restating it. SDK and CLI construction moved from Part II into the guide as section 8 -- that is consumer work, not authoring. Section 2.5 rewritten around the queries people run rather than an explanation of JSON Schema: 454 lines to 194. The required-field derivation, the tag model, and the example-file walkthrough moved to CONCEPTS.md. All `uv run python <<'PY'` heredocs converted to plain python blocks, which also brings them under tests/test_documented_imports.py -- that raised the checked-block count from 227 to 241. REVIEW STATE: guide sections 1 through 2.6 have had a full pass. Sections 2.6-8, CONCEPTS.md, and AUTHORING.md are not yet reviewed as prose; AUTHORING.md is a near-verbatim move of Part II. All code blocks are parsed, import-checked, and enum-checked by the test suite, and all internal links and anchors resolve. Co-Authored-By: Claude Opus 5 --- AUTHORING.md | 1687 ++++++++++++++ CONCEPTS.md | 803 +++++++ CONTRIBUTING.md | 7 +- GLOSSARY.md | 12 +- README.md | 18 +- SCHEMA_GUIDE.md | 5314 ++++++++++---------------------------------- TROUBLESHOOTING.md | 370 +++ 7 files changed, 4012 insertions(+), 4199 deletions(-) create mode 100644 AUTHORING.md create mode 100644 CONCEPTS.md create mode 100644 TROUBLESHOOTING.md diff --git a/AUTHORING.md b/AUTHORING.md new file mode 100644 index 000000000..10a82c6ce --- /dev/null +++ b/AUTHORING.md @@ -0,0 +1,1687 @@ +# Authoring and Extending the Schema + +For people **writing** schema — adding feature types, building tools on top of the models, +or authoring new Pydantic models for Overture itself. If you only want to *use* the +schema — validate data, explore models, generate artifacts — you want +[SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) instead. + +| If you want to | Read | +|---|---| +| Make your own feature types visible to the Overture tooling | [Register your own feature types](#register-your-own-feature-types) | +| Generate an SDK in another language, or build your own CLI | [SCHEMA_GUIDE.md §8](SCHEMA_GUIDE.md#8-building-tools-on-the-models) — that's consumer work | +| Understand entry-point registration and tags | [Registering models and tagging](#registering-models-and-tagging) | +| Author a new Pydantic model for the schema | [Authoring new schema models](#authoring-new-schema-models) | +| Run the tests and checks | [Development workflow](#development-workflow) | +| Copy a working starting point | [Templates and quick reference](#templates-and-quick-reference) | + +This page is the successor to the repo's old `PYDANTIC_GUIDE.md` — the contributor-facing +authoring guide, corrected and re-tested. It was folded into `SCHEMA_GUIDE.md` as Part II +during the docs consolidation and is broken back out here so the contributor material +stands on its own, as the consolidation set out to do. + +Per-package reference — installation, usage, and API for one package — lives in that +package's `README.md` under `packages/`, versioned alongside the code it documents. This +page covers what spans packages. + +See also [CONCEPTS.md](CONCEPTS.md) for why the schema is Pydantic and how the packages +fit together, and [SCHEMA_CONVENTIONS.md](SCHEMA_CONVENTIONS.md) for naming rules. + +*Every code block on this page has been executed against the repo; +`tests/test_documented_imports.py` keeps the imports honest.* + +--- + +## Extending the schema with your own types + +### Register your own feature types + +Your models become first-class: discovered by the CLI, accepted by `validate()`, +included in generated docs and JSON Schema. Nothing in the tooling special-cases +Overture. + +```python +# mypkg/models.py +from typing import Literal +from overture.schema.common import OvertureFeature +from overture.schema.system.numeric import float32 + + +class Vineyard(OvertureFeature[Literal["agriculture"], Literal["vineyard"]]): + """A cultivated area planted with grapevines.""" + + area_hectares: float32 | None = None +``` + +```toml +# mypkg/pyproject.toml +[project.entry-points."overture.models"] +vineyard = "mypkg.models:Vineyard" +``` + +Install it, and: + +```bash +overture-schema list-types +overture-schema validate vineyard.json +overture-codegen generate --format markdown --output-dir out +``` + +To attach your own tags, register a **tag provider** on `overture.tag_providers`: + +```python +def experimental_provider(types, key, tags): + if any(getattr(t, "__experimental__", False) for t in types): + tags.add("mypkg:experimental") + return tags +``` + +```toml +[project.entry-points."overture.tag_providers"] +experimental = "mypkg.tags:experimental_provider" +``` + +Tag namespaces are reserved: `feature` and `system:` belong to `overture-schema-system`, +`overture:` to `overture-schema-common`. A provider that tries to set a reserved tag from +an unauthorized package gets a logged warning and the tag is discarded. Use your own +namespace. + +You don't have to build on `OvertureFeature` — subclass `system.Feature` directly for a +GeoJSON-serializing model with none of the Overture conventions. + +### Write a new codegen target + +For a format nobody else generates — Arrow schemas, Avro, Go structs, protobuf — add a +renderer to the codegen rather than parsing JSON Schema back out. You get the full +semantic model: NewType names, constraint provenance, discriminated union structure — all +the things JSON Schema flattens away. + +The pipeline is four layers with strictly downward imports: + +``` +Rendering → output formatting, all presentation decisions +Output Layout → what to generate, where it goes, how outputs link +Extraction → FieldShape, FieldSpec, RecordSpec, UnionSpec, EnumSpec +Discovery → discover_models() +``` + +Extraction is target-independent, so a new target is a new renderer, not new extraction +logic. The entry point: + +```python +from overture.schema.codegen.extraction.model_extraction import extract_model +from overture.schema.buildings import Building + +spec = extract_model(Building) +spec.name # 'Building' +spec.description # the class docstring +spec.constraints # model-level constraints + +for f in spec.fields[:6]: + print(f"{f.name:12} required={f.is_required!s:5} {type(f.shape).__name__}") +``` + +``` +id required=True NewTypeShape +bbox required=False Primitive +geometry required=True Primitive +theme required=True LiteralScalar +type required=True LiteralScalar +version required=True NewTypeShape +``` + +`FieldSpec` is `(name, shape, description, is_required, is_optional)`. The `shape` is a +`FieldShape` tree — `NewTypeShape`, `Primitive`, `LiteralScalar`, `ModelRef`, +`UnionRef`, and container variants — with sub-models and sub-unions already resolved. +Constraints carry provenance, so you can tell which NewType contributed which bound: + +```python +from overture.schema.codegen.extraction.type_analyzer import analyze_type +from overture.schema.common.feature import FeatureVersion + +shape, is_nullable, description = analyze_type(FeatureVersion) +# shape → NewTypeShape(name='FeatureVersion', inner=Primitive(base_type='int32', +# constraints=(ConstraintSource(source_name='FeatureVersion', constraint=Ge(ge=0)), ...))) +``` + +`analyze_type` returns a **3-tuple** `(FieldShape, bool, str | None)` — the structural +shape, whether the field accepts `None`, and the first description found while +unwrapping. (The codegen README shows an older `TypeInfo`/`TypeKind` API that no longer +exists.) + +To wire up a new format: add a column to `TypeMapping` in +`extraction/type_registry.py` for type-name resolution, write a pipeline module +consuming `ModelSpec` trees plus a renderer, and register the format in `cli.py`. + +Further reading in the repo: + +- `packages/overture-schema-codegen/docs/design.md` — architecture, data flow, extension points +- `packages/overture-schema-codegen/docs/walkthrough.md` — module-by-module trace of `Segment` through the pipeline + +--- + +--- + +## Registering models and tagging + +How feature types make themselves known to the tooling. This is the mechanism that +lets your own models slot in alongside Overture's — see +[Register your own feature types](#register-your-own-feature-types) for a worked example. + +The library is designed to support data producer extensions through multiple patterns. +This extensibility is a core feature that allows organizations to add custom fields and +types while maintaining compatibility with the base Overture schema. We are in the +process of determining how this should work. + +### Model Registration via Entry Points + +Models are registered using [setuptools entry +points](https://setuptools.pypa.io/en/latest/userguide/entry_point.html) in each +package's `pyproject.toml` file. This enables automatic discovery and loading of models +at runtime without requiring explicit imports. + +Registration is done in the `[project.entry-points."overture.models"]` section: + +```toml +[project.entry-points."overture.models"] +building = "overture.schema.buildings:Building" +building_part = "overture.schema.buildings:BuildingPart" +``` + +The discovery system provides programmatic access to registered models: + +```python +from overture.schema.system.discovery import discover_models, get_registered_model + +# Discover all registered models, keyed by ModelKey +all_models = discover_models() + +# Get a specific model by name +building_model = get_registered_model("building") +if building_model: + building = building_model.model_validate(building_data) +``` + +### Tagging + +Each `ModelKey` returned by `discover_models()` carries a `frozenset[str]` of tags +that classify the model orthogonally to its entry-point name -- whether the model +is a `Feature` subclass, which Overture theme it belongs to, which package shipped +it, and so on. Downstream tools (the CLI, codegen, third-party consumers) use tags +to filter the working set without importing every model: + +```python +from overture.schema.system.discovery import ( + TagSelector, + discover_models, + filter_models, +) + +models = discover_models() +# { +# ModelKey(name="building", entry_point="overture.schema.buildings:Building", +# tags=frozenset({"feature", "overture", "overture:theme=buildings"})): Building, +# ModelKey(name="place", entry_point="overture.schema.places:Place", +# tags=frozenset({"feature", "overture", "overture:theme=places"})): Place, +# ... +# } + +buildings = filter_models( + models, + TagSelector(include_any=("overture:theme=buildings",)), +) +``` + +Tags are produced by *tag providers* registered on the `overture.tag_providers` +entry-point group. The `system` and `common` packages ship the built-in providers +(`feature` and `overture:theme=*`); third parties can register their own +to attach custom tags during discovery. See the [`overture-schema-system` +README](packages/overture-schema-system/README.md#tagging) for tag format, +reserved namespaces, and provider authoring. + + +--- + +## Authoring new schema models + +### Quick Start + +#### Essential Imports + +Copy what you need for most models: + +```python +# Basic Python types +from typing import Annotated, Literal +from enum import Enum + +# Pydantic essentials +from pydantic import BaseModel, Field + +# Overture common models +from overture.schema.common import OvertureFeature +from overture.schema.system.geometric import ( + Geometry, + GeometryType, + GeometryTypeConstraint, +) + +# Validation system +from overture.schema.system.field_constraint import UniqueItemsConstraint +from overture.schema.system.model_constraint import no_extra_fields + +# Common types +from overture.schema.system.string import ( + CountryCodeAlpha2, + NoWhitespaceString, + StrippedString, +) +from overture.schema.common.confidence import ConfidenceScore +from overture.schema.system.string import LanguageTag + +# Numeric types (use these instead of int/float) +from overture.schema.system.numeric import ( + int8, + int32, + int64, + uint8, + uint16, + uint32, + float32, + float64, +) +``` + +#### Templates + +Copy-paste starting points for the four shapes you'll write most often — a plain +model, a feature, an enum, and a model with validation constraints — live together in +[Templates and quick reference](#templates-and-quick-reference) rather than being +repeated here. + +--- + +### Basic Concepts + +#### Models and Inheritance + +##### What are Pydantic models? + +Pydantic models are Python classes that define data structures and their constraints. Think of them like UML classes with built-in data validation - each model defines what fields are allowed and what types of data they can contain. + +##### Model Base Classes and Inheritance + +**What is a "base class"?** A base class defines common fields and behaviors that other classes can reuse. Think of it like a slide template - you create one layout, then make specific slides that use that structure. + +**What is "inheritance"?** Inheritance means one class automatically gets all the fields and behaviors from another class. If Building inherits from OvertureFeature, it automatically gets all of Feature's fields (like `id`, `geometry`) plus any new fields you add to Building (like `height`). When multiple parent classes have the same field name, Python uses a [specific order](https://docs.python.org/3/tutorial/classes.html#multiple-inheritance) to determine which one takes precedence. + +**@no_extra_fields** - Use for structured data components that should reject unknown fields: + +```python +from overture.schema.system.model_constraint import no_extra_fields + + +@no_extra_fields +class Address(BaseModel): + """A postal address - no extra fields allowed.""" + + street: str + city: str + postal_code: str | None = None + # Any field not defined here will cause validation to fail +``` + +**OvertureFeature[ThemeT, TypeT]** - A generic base class for all geospatial features with typed theme and type parameters: + +```python +from typing import Literal +from overture.schema.common import OvertureFeature +from overture.schema.system.numeric import float64 + + +class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): + """A building feature with strongly-typed theme and type.""" + + # Inherits: id, theme, type, geometry, bbox, version, sources + height: float64 | None = None +``` + +**What does "generic" mean?** The `OvertureFeature[ThemeT, TypeT]` syntax makes OvertureFeature a "generic" class - think of it like a template that can be customized with specific values. The square brackets `[]` contain "type parameters" that specify exactly what theme and type this feature represents. + +**What are ThemeT and TypeT?** These are placeholders for specific text values: + +- **ThemeT**: The data theme (like "buildings", "places", "transportation") +- **TypeT**: The specific feature type within that theme (like "building", "place", "segment") + +**What is `Literal`?** `Literal` means the field must be exactly one of the specified values - nothing else is allowed. So `Literal["buildings"]` means this theme can only be "buildings", not any other string. + +By specifying `OvertureFeature[Literal["buildings"], Literal["building"]]`, you're saying "this is a Feature that must have theme='buildings' and type='building'" - no other values are allowed. This prevents mistakes like accidentally creating a building with theme="places". + +##### Inheritance Patterns + +**Multiple inheritance** combines fields from several base classes: + +```python +from typing import Literal +from overture.schema.common import OvertureFeature +from overture.schema.common.level import Stacked +from overture.schema.common.names import Named +from overture.schema.system.numeric import float64 + + +class Building( + OvertureFeature[Literal["buildings"], Literal["building"]], Named, Stacked +): + # Gets fields from Feature: id, theme, type, geometry, etc. + # Gets fields from Named: names + # Gets fields from Stacked: level + # Plus its own fields: + height: float64 | None = None +``` + +##### Field Aliases + +Sometimes you need a field name that conflicts with Python keywords or conventions (hint: you'll get an error when you try to use it). Use `Field(alias="")` to map between Python-friendly field names and the actual data field names: + +```python +from typing import Annotated +from pydantic import Field + + +class Building(OvertureFeature): + # Use class_ in Python code, but "class" in the actual data + class_: Annotated[str | None, Field(alias="class")] = None + + # Other common cases might include: + type_: Annotated[str | None, Field(alias="type")] = None # if type conflicts + from_: Annotated[str | None, Field(alias="from")] = None # from is a keyword +``` + +A common example is `class_` with `Field(alias="class")` since "class" is a Python keyword but a common field name in data schemas. + +#### Field Types + +##### Required vs Optional Fields + +```python +class Building(OvertureFeature): + # Required field (no default value) + geometry: Geometry + + # Optional field (has default value of None) + height: float64 | None = None +``` + +- **Required fields**: Must be provided when creating an instance +- **Optional fields**: Can be omitted; they have a default value (usually `None`) + +> [!WARNING] +> **Always use `None` defaults** for optional fields. Non-`None` defaults create ambiguity between schema defaults and actual data values. + +**Why do non-`None` defaults cause problems?** + +1. **Data transformation ambiguity**: Pydantic adds default values that weren't in the input, making it impossible to distinguish between original data and schema defaults. + +2. **Schema vs. data confusion**: Schemas serve multiple purposes: + - **Validation only**: Check if existing data is valid (shouldn't transform it) + - **Data processing**: Parse and potentially transform data with Pydantic + - **Documentation**: Show developers what fields exist and what they mean + +3. **Implicit semantic meaning**: Default values encode business logic into the schema, which should be in business logic instead. + +**Better approaches:** + +1. **Use `None` and document semantics:** + + ```python + access_policy: Annotated[ + str | None, + Field(description="Access policy for the place. When absent, assume 'open'"), + ] = None + ``` + +2. **Always populate in data pipeline:** + + ```python + # In your data processing pipeline, always set the value + place.access_policy = place.access_policy or "open" + ``` + +Keep the schema separate from business logic. The schema describes the shape of data, not the business rules about what missing values mean. + +##### Numeric Types + +**Always use specific numeric types instead of Python's generic `int`/`float`:** + +```python +from overture.schema.system.model_constraint import no_extra_fields +from overture.schema.system.numeric import ( + int8, + int32, + int64, # Signed integers + uint8, + uint16, + uint32, # Unsigned integers + float32, + float64, # Floating point +) + + +@no_extra_fields +class MyModel(BaseModel): + # Signed integers with specific ranges + level: int8 | None = None # -128 to 127 + year: int32 | None = None # -2,147,483,648 to 2,147,483,647 + timestamp: int64 | None = None # Full 64-bit range + + # Unsigned integers (0 and positive only) + red_value: uint8 | None = None # 0 to 255 (like RGB values) + port: uint16 | None = None # 0 to 65,535 (like network ports) + population: uint32 | None = None # 0 to 4,294,967,295 + + # Floating point numbers + height: float64 | None = None # Double precision (recommended) + ratio: float32 | None = None # Single precision +``` + +**When to use each:** + +- **`int32`**: Most integer fields (years, counts, IDs) +- **`uint8`**: Small positive values (0-255), like color components, confidence percentages +- **`uint16`**: Medium positive values (0-65K), like ports, small counts +- **`uint32`**: Large positive values, like population, large IDs +- **`float64`**: Most decimal numbers (heights, coordinates, measurements) - **this is the default choice** +- **`float32`**: When space is critical and precision isn't + +When in doubt, use `int32` (equivalent to `int`), `int64` (equivalent to `long`, although [not safely representable in JSON](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER)), or `float64` (equivalent to `double`). + +**Why specific numeric types matter:** + +The specific numeric types are crucial for data interchange and storage compatibility: + +- **Cross-platform consistency**: Ensures the same data types across Python, Arrow, Parquet, and other geospatial tools +- **Round-trip compatibility**: Data round-trips cleanly between Parquet files, databases (PostgreSQL, Trino), Shapefiles, and JSON Schema +- **Value range validation**: Prevents invalid values (e.g., negative heights, RGB values > 255) +- **Storage efficiency**: `uint8` uses 1 byte vs `int64` which uses 8 bytes +- **Built-in validation**: These types use Pydantic `Field()` constraints to validate ranges (e.g., `Field(ge=0, le=100)` ensures values stay within bounds) + +##### Union Types + +Union types allow a field to accept multiple different types. The `|` symbol means "or": + +```python +from typing import Literal + + +class Building(OvertureFeature): + # This field can be either a string OR None (most common union) + name: str | None = None + + # This field can be one of specific string values OR None + status: Literal["active", "inactive", "pending"] | None = None +``` + +**Common union patterns:** + +```python +# Optional field (most common union) +height: float64 | None = None # Can be a number or missing + +# Specific string values (an alternative to enums where descriptions aren't needed) +priority: Literal["low", "medium", "high"] | None = None + +# Boolean or None +is_verified: bool | None = None +``` + +**Union best practices:** + +- Keep unions simple - avoid more than 2-3 types when possible +- Optional fields will include `None` to become optional +- Use `Literal` values for specific string choices rather than mixing basic types +- **Avoid mixed-type unions** like `str | int32` - these don't work well with many storage layers + +> [!WARNING] +> **Storage compatibility**: Mixed-type unions (combining different basic types like `str | int32`) don't work with Parquet and other storage layers. Use `Literal` values or separate fields instead. + +#### Field Enhancement + +##### Adding Descriptions and Constraints with Annotated + +`Annotated` is Python's way to add extra information (metadata) to a type without changing the type itself. Think of it like adding notes or constraints to a field definition. + +**Basic concept:** + +```python +from typing import Annotated +from pydantic import Field + +# Without Annotated - just the type +height: float64 | None = None + +# With Annotated - type + extra information +height: Annotated[ + float64 | None, # The actual type (what kind of data) + Field(description="Height in meters"), # Extra metadata +] = None +``` + +**What goes inside `Annotated`:** + +1. **First argument**: The actual type (`str`, `int32`, `list[str]`, etc.) +2. **Additional arguments**: Metadata like constraints, descriptions, validation rules + +##### Field Constraints + +Use Pydantic's `Field()` function to add constraints and descriptions: + +**Numeric constraints:** + +```python +from typing import Annotated +from pydantic import Field + + +class Building(OvertureFeature): + # Range constraints + height: Annotated[ + float64 | None, Field(ge=0, le=1000, description="Height in meters (0-1000m)") + ] = None + + # Integer constraints + floors: Annotated[ + int32 | None, Field(gt=0, lt=200, description="Number of floors (1-199)") + ] = None +``` + +**Numeric constraint options:** + +- **`ge`**: Greater than or equal to (≥) +- **`gt`**: Greater than (>) +- **`le`**: Less than or equal to (≤) +- **`lt`**: Less than (<) + +**String constraints:** + +```python +class Place(OvertureFeature): + # Length constraints + name: Annotated[ + str | None, + Field(min_length=1, max_length=100, description="Place name (1-100 chars)"), + ] = None + + # Pattern matching + postal_code: Annotated[ + str | None, Field(pattern=r"^\d{5}(-\d{4})?$", description="US postal code") + ] = None +``` + +**String constraint options:** + +- **`min_length`**: Minimum string length +- **`max_length`**: Maximum string length +- **`pattern`**: Regular expression pattern (regex) + +#### Collections and Lists + +##### Basic List Fields + +```python +class Building(Feature): + # Simple list of strings + tags: list[str] | None = None + + # List of complex objects + access_rules: list[AccessRule] | None = None +``` + +##### List Constraints + +```python +from overture.schema.system.field_constraint import UniqueItemsConstraint + + +class Building(OvertureFeature): + # List with size and uniqueness constraints + categories: Annotated[ + list[str] | None, + Field(min_length=1, max_length=10, description="1-10 categories"), + UniqueItemsConstraint(), # Must come AFTER Field() + ] = None +``` + +**List constraint options:** + +- **`min_length`**: Minimum number of items +- **`max_length`**: Maximum number of items +- **`UniqueItemsConstraint()`**: No duplicate items (custom validation) + +**Important**: `UniqueItemsConstraint()` must come AFTER `Field()` for proper JSON Schema generation. + +> [!CAUTION] +> **Constraint order matters**: Always put `Field()` before `UniqueItemsConstraint()` or JSON Schema generation will create `minLength` (string constraint) instead of `minItems` (array constraint). +> +> **Why**: Pydantic processes annotations in order for JSON Schema generation. `Field()` must come first to set up the field properly. For lists, `Field(min_length=1)` creates a `minItems` constraint in the JSON Schema because the type immediately before it is a list. If `UniqueItemsConstraint()` comes first, Pydantic doesn't see the list type and treats `min_length` as a string constraint (`minLength`). + +##### List Behavior + +Lists maintain their **insertion order** (the order data exists in the field), but they are **not automatically sorted**. + +#### Enumerations + +**What is an enumeration (enum)?** An enumeration is a way to define a fixed set of allowed values for a field. Think of it like a multiple-choice question - you define all the valid answers ahead of time, and users can only pick from those options. + +For example, instead of allowing any string for a "status" field (which could lead to typos like "activ" or "Active"), you create an enum with exactly "active", "inactive", and "pending" as the only allowed values. + +**Enums vs Literal:** You can achieve similar results with `Literal["active", "inactive", "pending"]`, but formal enums are better when you need descriptions, documentation, or want to reuse the same set of values across multiple fields. + +##### Creating Enums + +Enums define a fixed set of allowed values: + +```python +from enum import Enum + + +class BuildingClass(str, Enum): + """Further delineation of the building's built purpose.""" + + RESIDENTIAL = "residential" + COMMERCIAL = "commercial" + INDUSTRIAL = "industrial" + CIVIC = "civic" + + +# Usage in a model +class Building(OvertureFeature): + class_: Annotated[BuildingClass | None, Field(alias="class")] = None +``` + +##### Documenting Enum Values + +Add documentation to describe what the enum and its values mean. In Python, you do this with **docstrings** - text enclosed in triple quotes `"""` that describes what something does: + +Use `DocumentedEnum` from `overture.schema.system.doc` when enum members need their own descriptions for code generation and documentation tooling. Each member takes a `(value, description)` tuple: + +```python +from overture.schema.system.doc import DocumentedEnum + + +class VehicleType(str, DocumentedEnum): + """Types of vehicles for transportation.""" + + CAR = ("car", "Standard passenger vehicle") + TRUCK = ("truck", "Commercial freight vehicle") + BICYCLE = ("bicycle", "Human-powered two-wheeler") + MOTORCYCLE = ("motorcycle", "Motorized two-wheeler") +``` + +Members without descriptions use the plain value form -- documentation is optional per-member: + +```python +class ConnectionState(str, DocumentedEnum): + CONNECTED = "connected" + DISCONNECTED = "disconnected" + QUIESCING = ( + "quiescing", + "Gracefully shutting down, rejecting new requests but completing existing ones", + ) +``` + +Use `DocumentedEnum` over plain `str, Enum` when the enum members' semantics aren't obvious from their names and downstream tools (code generators, documentation renderers) need access to member-level descriptions. Use plain `str, Enum` for self-explanatory values. + +##### Why str, Enum? + +Inheriting from `str, Enum` makes enum values work as both enums and strings, which is useful for JSON serialization and compatibility. + +--- + +### Advanced Patterns + +#### Relationship Patterns + +##### What are relationships? + +Relationships represent connections between different features or models. Think of them like links that connect related pieces of information — for example, a building part that is structurally part of a building, or a division area that is administratively nested under a division. + +Pydantic provides several ways to express these relationships, each suited to different use cases and complexity levels. Before choosing a pattern, it's important to understand the **semantic type** of the relationship you're modeling. + +--- + +##### Semantic Relationship Types + +Every relationship between two features carries a semantic meaning about coupling strength, lifecycle dependency, and ownership. The schema defines four relationship types, ordered from strongest to weakest coupling. The types describe the *nature* of the link, not which feature is "parent" or "child." Direction is implicit: the feature holding the reference is the source, and the type it references is the destination. + +###### `COMPOSITION` — Structural Whole-Part + +A structural whole-part relationship with lifecycle dependency. The part has no independent meaning outside the whole. Deleting the whole invalidates the part. + +**Test question:** *"If I delete the whole, does keeping the part orphaned make any sense at all?"* If the answer is no, it's `COMPOSITION`. + +**Examples:** +- `BuildingPart` → `Building` — part *is part of* building +- `DivisionBoundary` → `Division` — boundary line *defines the boundary of* division + +###### `AGGREGATION` — Grouping Without Lifecycle Dependency + +A grouping or collection relationship where both members are independently viable. No lifecycle dependency — the member survives reassignment to another group or orphaning. + +**Test question:** *"Can both sides belong to something else or nothing and still be a valid map feature?"* If yes, they form an `AGGREGATION`. + +**Examples:** +- `Route` → `Segment` — route *groups* segments +- `TrailSegment` → `NationalPark` — segment *is grouped by* park + +###### `HIERARCHY` — Organizational Nesting + +An organizational or classificatory nesting relationship. This is not about structural assembly — it's about administrative parentage, taxonomy, or categorization. + +**Test question:** *"Is this about organizational subordination rather than structural assembly?"* If yes, it's `HIERARCHY`. + +**Examples:** +- `DivisionArea` → `Division` — area *is child of* division +- `Division` → `Division` — child division nested under parent + +###### `ASSOCIATION` — Peer-Level Reference + +A peer-level reference with no ownership, containment, or nesting. Neither feature depends on or contains the other. This is the fallback when none of the stronger types apply. + +**Test question:** *"Are these just peers that know about each other?"* If yes, it's `ASSOCIATION`. + +**Examples:** +- `Segment` → `Connector` — segment references its start/end connector +- `Building` → `Address` — a building references its address, neither owns the other + +--- + +##### Selection Priority + +When a relationship could fit multiple types, the choice follows a **diamond decision**: start at the top, fork in the middle based on the *kind* of coupling, and fall through to the bottom only when no stronger type applies. + +```text + COMPOSITION + / \ +AGGREGATION HIERARCHY + \ / + ASSOCIATION +``` + +| If the relationship implies... | Use | +|---------------------------------------------------------|----------------| +| Structural whole-part with lifecycle dependency | `COMPOSITION` | +| Geometric boundary definition (lifecycle dependent) | `COMPOSITION` | +| Grouping/collection without lifecycle dependency | `AGGREGATION` | +| Organizational nesting or classification tree | `HIERARCHY` | +| Peer-level reference, no ownership or nesting | `ASSOCIATION` | + +--- + +##### The `role` Field + +The `Reference` annotation accepts an optional `role` parameter — a snake_case string that further qualifies the relationship from the source's perspective. It has no effect on schema validation; it is informational metadata for documentation and tooling. + +Use `role` when the semantic type alone is ambiguous. For example, multiple `HIERARCHY` references on the same model can be disambiguated: + +```python +# Without role: two HIERARCHY references to Division — which is which? +parent_division_id: Annotated[Id, Reference(Relationship.HIERARCHY, Division)] +capital_division_ids: Annotated[list[Id], Reference(Relationship.HIERARCHY, Division)] + +# With role: unambiguous +parent_division_id: Annotated[ + Id, Reference(Relationship.HIERARCHY, Division, role="child_of") +] +capital_division_ids: Annotated[ + list[Id], Reference(Relationship.HIERARCHY, Division, role="has_as_capital") +] +``` + +The `role` must be a non-empty snake_case string (lowercase letters, digits, underscores). It describes the source's role relative to the target using source-perspective phrasing. + +--- + +##### Implementation Patterns + +###### 1. Direct References (Foreign Keys) + +The fundamental pattern is a direct reference where one feature "points to" another using an ID field with type safety and semantic information. + +```python +from typing import Annotated, Literal +from pydantic import Field +from overture.schema.common import OvertureFeature +from overture.schema.system.ref import Id, Reference, Relationship + + +# COMPOSITION — part points to its whole +class BuildingPart(OvertureFeature[Literal["buildings"], Literal["building_part"]]): + """A structural part of a building.""" + + building_id: Annotated[ + Id, + Reference(Relationship.COMPOSITION, Building, role="part_of"), + Field(description="The building to which this part belongs"), + ] + + +# HIERARCHY — child points to parent +class DivisionArea(OvertureFeature[Literal["divisions"], Literal["division_area"]]): + """Area polygon nested under a division.""" + + division_id: Annotated[ + Id, + Reference(Relationship.HIERARCHY, Division, role="child_of"), + Field(description="Division ID of the parent division of this area."), + ] + + +# ASSOCIATION — peer reference, no ownership +class ConnectorReference(BaseModel): + """Reference to a connector feature.""" + + connector_id: Annotated[ + Id, Reference(Relationship.ASSOCIATION, Connector, role="connects_to") + ] +``` + +###### 2. Association as a Separate Feature (Complex Relationships) + +When the relationship itself needs to store information, create a dedicated feature to represent it. This applies regardless of the semantic type — any of the four types can carry metadata. + +**Simple relationship (use Pattern 1):** +- "Building Part A is part of Building B" — just needs an ID reference. + +**Complex relationship (use Pattern 2):** +- "Admin Area X has City Center Y as its primary center since 2010 with 85% confidence" — the relationship has properties. + +```python +class AdminCityCenterAssociation( + OvertureFeature[Literal["associations"], Literal["admin_city_center"]] +): + """Describes how an administrative area relates to a city center.""" + + admin_area_id: Annotated[Id, Reference(Relationship.ASSOCIATION, AdminArea)] + city_center_id: Annotated[Id, Reference(Relationship.ASSOCIATION, CityCenter)] + + # Information about the relationship itself + relationship_type: Literal["primary_center", "secondary_center"] = "primary_center" + established_date: str | None = None + confidence_score: Annotated[float64, Field(ge=0.0, le=1.0)] | None = None +``` + +**When to use separate association features:** +- The relationship has properties (confidence scores, dates, types, notes). +- Many-to-many connections exist. +- You need to query the relationships independently. + +###### 3. Collection References + +When a feature needs to reference multiple other features, use a list of references. The semantic type still matters. + +```python +# COMPOSITION — boundary defines two divisions +class DivisionBoundary( + OvertureFeature[Literal["divisions"], Literal["division_boundary"]] +): + """A boundary line between two divisions.""" + + division_ids: Annotated[ + list[ + Annotated[ + Id, Reference(Relationship.COMPOSITION, Division, role="boundary_of") + ] + ], + Field(min_length=2, max_length=2, description="Left and right divisions"), + ] + + +# AGGREGATION — route groups segments +class Route(OvertureFeature[Literal["transportation"], Literal["route"]]): + """A transportation route passing through multiple segments.""" + + segment_ids: Annotated[ + list[Id], + Reference(Relationship.AGGREGATION, TransportationSegment, role="groups"), + Field(min_length=1, description="Ordered segments in this route"), + UniqueItemsConstraint(), + ] +``` + +--- + +##### Best Practices + +###### Always Use Reference Annotations + +Include `Reference` annotations for semantic clarity and documentation: + +```python +# Good — complete relationship information with semantic type and role +division_id: Annotated[ + Id, + Reference(Relationship.HIERARCHY, Division, role="child_of"), + Field(description="Division ID of the parent division of this area."), +] + +# Avoid — missing semantic information +division_id: Id +``` + +###### Choose the Right Semantic Type First, Then the Right Pattern + +1. **Determine the semantic type** using the selection priority and test questions above. +2. **Then choose the implementation pattern:** + - Simple relationships → Direct references (Pattern 1) + - Relationships with metadata → Separate association features (Pattern 2) + - One-to-many references → Collection references (Pattern 3) + +#### Discriminated Unions + +**What is a discriminated union?** A discriminated union is a type that can be backed by one of several different models, where a specific field (the "discriminator") determines which model it actually is. Think of it like a form that changes its fields based on a category selection. + +```python +from typing import Annotated, Literal +from pydantic import Field +from overture.schema.common import OvertureFeature + + +# Base class with common fields +class TransportationSegment( + OvertureFeature[Literal["transportation"], Literal["segment"]] +): + subtype: Subtype # This is the discriminator field + # ... common fields for all segments + + +# Specific segment types +class RoadSegment(TransportationSegment): + subtype: Literal[Subtype.ROAD] # Must be "road" + class_: Annotated[RoadClass, Field(alias="class")] + speed_limits: SpeedLimits | None = None + # ... road-specific fields + + +class RailSegment(TransportationSegment): + subtype: Literal[Subtype.RAIL] # Must be "rail" + class_: Annotated[RailClass, Field(alias="class")] + rail_flags: RailFlags | None = None + # ... rail-specific fields + + +# Union type that automatically picks the right model based on subtype +Segment = Annotated[ + RoadSegment | RailSegment | WaterSegment, Field(discriminator="subtype") +] +``` + +The `discriminator="subtype"` tells Pydantic to look at the `subtype` field to determine which specific model to use. If `subtype` is "road", it uses `RoadSegment`; if "rail", it uses `RailSegment`. + +##### Abstract vs Concrete Classes + +**What's the difference?** In UML and traditional OOP, abstract classes cannot be instantiated - they serve as templates for concrete classes. In Pydantic, by default, **all classes are concrete** (can be instantiated), but you can make classes abstract when needed. + +**Current pattern (all concrete):** + +```python +# Both can be instantiated as map features +base_segment = TransportationSegment(subtype=Subtype.ROAD, geometry=...) # Valid +road_segment = RoadSegment(subtype=Subtype.ROAD, geometry=..., class_=...) # Valid +``` + +**Making the base class abstract:** + +```python +from abc import ABC, abstractmethod +from typing import Annotated, Literal +from pydantic import Field + + +class TransportationSegment( + OvertureFeature[Literal["transportation"], Literal["segment"]], ABC +): + """Abstract base - cannot be instantiated directly.""" + + subtype: Subtype # Discriminator field + + @abstractmethod + def get_speed_limit(self) -> float: + """Each concrete type must implement this.""" + pass + + +class RoadSegment(TransportationSegment): + """Concrete class - can be instantiated.""" + + subtype: Literal[Subtype.ROAD] + speed_limits: SpeedLimits | None = None + + def get_speed_limit(self) -> float: + return self.speed_limits.max_speed if self.speed_limits else 50.0 + + +# Now only concrete classes can be instantiated +# base_segment = TransportationSegment(...) # TypeError: Can't instantiate abstract class +road_segment = RoadSegment(subtype=Subtype.ROAD, geometry=...) # Valid +``` + +**Registration pattern (recommended when working with Overture models):** + +Instead of making classes abstract, we use **entry point registration** where only specific concrete types are discoverable as map features: + +```toml +# In packages/overture-schema-theme-transportation/pyproject.toml +[project.entry-points."overture.models"] +connector = "overture.schema.transportation:Connector" +segment = "overture.schema.transportation:Segment" +``` + +**Real example:** See [`packages/overture-schema-theme-transportation/src/overture/schema/transportation/segment/__init__.py`](packages/overture-schema-theme-transportation/src/overture/schema/transportation/segment/__init__.py) where: + +- **`Segment`** is a discriminated union: `RoadSegment | RailSegment | WaterSegment` +- **`TransportationSegment`** is the concrete base class that all segment types inherit from +- **Individual segment types** (`RoadSegment`, `RailSegment`, `WaterSegment`) are NOT directly registered + +**This registration pattern means:** + +1. Only **`Segment`** (the union type) is discoverable as an official map feature +2. The union automatically resolves to the correct concrete type based on the `subtype` field +3. All classes (`TransportationSegment`, `RoadSegment`, etc.) can be reused as base classes for alternate implementations + +#### Pattern Properties (Constrained Key-Value Maps) + +**What are pattern properties?** Pattern properties let you create key-value maps where the keys must follow a specific pattern (like language codes) and values have specific types. + +```python +from typing import Annotated +from pydantic import BaseModel, Field + + +@no_extra_fields +class Names(BaseModel): + primary: str + + # Keys (strings) must match a language tag pattern, values are strings + common: ( + Annotated[ + dict[ + # The key type + Annotated[ + str, + Field( + pattern=r"^[a-z]{2,3}(-[A-Z]{2})?$", + description="Language tag (e.g., 'en', 'es-MX')", + ), + ], + str, # The value type + ], + Field(json_schema_extra={"additionalProperties": False}), + ] + | None + ) = None +``` + +**Example data:** + +```json +{ + "primary": "New York City", + "common": { + "es": "Ciudad de Nueva York", + "fr": "New York", + "zh-CN": "纽约市" + } +} +``` + +The `additionalProperties: False` ensures only keys matching the pattern are allowed when generated JSON Schema is used. + +#### Nested List Validation + +**What is nested list validation?** This pattern validates both the outer list and the inner structure of each item, with constraints at multiple levels. + +```python +from typing import Annotated +from pydantic import Field + + +# Each item has its own field validation +@no_extra_fields +class HierarchyItem(BaseModel): + division_id: str + name: str + + +class Division(OvertureFeature): + # Nested list validation: outer list AND inner lists both have length constraints + hierarchies: Annotated[ + list[ # Outer list + Annotated[ + list[HierarchyItem], # Inner list + Field(min_length=1), # Inner list must have at least 1 item + ] + ], + Field(min_length=1), # Outer list must have at least 1 hierarchy + ] +``` + +This creates validation at three levels: + +1. **Individual items**: Each `HierarchyItem` validates its own fields (`division_id`, `name`) +2. **Inner lists**: Each inner list must have at least 1 item (`min_length=1`) +3. **Outer list**: The `hierarchies` field must have at least 1 inner list (`min_length=1`) + +#### Type Aliases for Reusable Patterns + +**What are type aliases?** Type aliases let you create custom names for complex or frequently-used types. Think of them like creating shortcuts or nicknames for long type definitions. + +**What is `NewType`?** `NewType` creates a distinct type that's based on an existing type but is treated as different for type checking purposes. This helps prevent mistakes like using an email address where you need a country code, or using a person's name where you need an ID - they're all strings, but they have different meanings and shouldn't be interchangeable. + +**Note**: `NewType` is primarily useful when working with Pydantic models in Python code (development, testing, certain data processing tasks). It doesn't affect data validation or JSON Schema generation - it's a development tool to catch mistakes before they happen. + +> [!WARNING] +> **Naming conflicts**: Never use the same name for a model class and type alias in the same module - this creates circular references and confusing code. + +**Guidelines:** + +- **Model classes**: Use noun names (`SourceItem`, `AccessRule`, `GeometricScope`) +- **Type aliases**: Use plural or descriptive names (`Sources`, `AccessRules`, `ConnectivityData`) +- **Avoid using the same names**: Use different names for models and type aliases, even if they're related + +```python +from typing import NewType, Annotated +from pydantic import BaseModel, Field + +# Create distinct types for different kinds of strings +SegmentId = NewType("SegmentId", str) # IDs are strings, but distinct +CountryCode = NewType("CountryCode", str) # Country codes are strings, but distinct + +# Create aliases for complex field patterns +EmailList = NewType( + "EmailList", + Annotated[list[str], Field(min_length=1, description="List of email addresses")], +) + + +@no_extra_fields +class Contact(BaseModel): + # Clear, self-documenting field types + id: SegmentId # Can't accidentally use a CountryCode here + country: CountryCode # Can't accidentally use a SegmentId here + emails: EmailList # Reusable validation pattern +``` + +**Why use type aliases?** + +1. **Prevent mistakes**: `SegmentId` and `CountryCode` are both strings, but you can't mix them up when they're created using `NewType` +2. **Reusable patterns**: Define complex field validation once, use it many times +3. **Self-documenting code**: `EmailList` is clearer than `list[str]` +4. **Consistency**: Everyone uses the same validation rules for the same concept + +--- + +### Integration Guide + +#### Project Architecture + +##### File Organization + +Organize code by scope, and avoid circular imports. + +**Cross-theme shared**: the `overture-schema-common` package. Definitions more than +one theme needs -- `OvertureFeature`, `Names`, `Sources`, the scoping framework. + +**One module per feature type**: at the theme package root, named after the type in +snake_case. + +```text +packages/overture-schema-theme-buildings/src/overture/schema/buildings/ + __init__.py # re-exports the public names, declares __all__ + _common.py # shared by Building and BuildingPart + building.py # Building, BuildingSubtype, BuildingClass + building_part.py # BuildingPart +``` + +The module owns everything specific to its type: the `Feature` subclass, its enums, +its NewTypes, and its supporting models. `building.py` defines `BuildingSubtype` and +`BuildingClass` next to `Building`, because nothing else uses them. + +**Theme-level shared**: `_common.py` at the theme package root, for definitions two +or more types in the theme need. `buildings/_common.py` holds `Appearance`, +`RoofShape`, and the material enums that both `Building` and `BuildingPart` use. The +leading underscore marks the module private -- the theme's `__init__.py` re-exports +the public names from it. + +**A type large enough to split**: a subpackage named after the type, applying the +same rules one level down. + +```text +packages/overture-schema-theme-transportation/src/overture/schema/transportation/ + __init__.py # re-exports from connector and segment + connector.py # Connector + segment/ + __init__.py # assembles the Segment discriminated union, re-exports + _common.py # TransportationSegment and what the arms share + road.py # RoadSegment and its supporting types + rail.py # RailSegment and its supporting types + water.py # WaterSegment +``` + +Every `__init__.py` re-exports its public names and declares `__all__`, so consumers +import from the package rather than reaching into the defining module: +`from overture.schema.buildings import Building`, not +`from overture.schema.buildings.building import Building`. Entry points name the +package root for the same reason -- `building = "overture.schema.buildings:Building"`. + +There is no `models.py` / `enums.py` / `types.py` split. An enum lives in the module +whose type uses it, and moves up to `_common.py` when a second type needs it. + +##### Import Organization + +```python +# Standard library imports first +from enum import Enum +from typing import Annotated, Literal, NewType + +# Third-party imports +from pydantic import BaseModel, ConfigDict, Field + +# Cross-theme and system imports +from overture.schema.common.scoping import Heading, Scope, scoped +from overture.schema.system.doc import DocumentedEnum +from overture.schema.system.field_constraint import UniqueItemsConstraint +from overture.schema.system.model_constraint import no_extra_fields + +# Local imports last -- siblings in the theme, then the module's own package +from ..connector import Connector +from ._common import SegmentSubtype, TransportationSegment +``` + +`uv run ruff format ` will sort your imports in this order automatically. + +##### Why Not Use @field_validator or @model_validator? + +This project uses a custom validation system that generates better JSON Schema output and supports code generation (without additional work, `@field_validator` and `@model_validator` don't make their constraints discoverable). Always use constraints from `overture.schema.system` instead of using Pydantic validation decorators: + +```python +# Don't do this +@field_validator("categories") +def validate_categories_unique(cls, v): + if v and len(v) != len(set(v)): + raise ValueError("Categories must be unique") + return v + + +# Do this instead +from overture.schema.system.field_constraint import UniqueItemsConstraint + + +class Building(OvertureFeature): + categories: Annotated[ + list[str] | None, + Field(min_length=1, description="Building categories"), + UniqueItemsConstraint(), + ] = None +``` + +#### Migrating from JSON Schema + +If you're familiar with JSON Schema files (like `schema/schema.yaml`), this section helps translate those patterns to Pydantic models. + +##### How $defs and $ref Translate + +**JSON Schema approach:** + +```yaml +# In defs.yaml +"$defs": + propertyDefinitions: + address: + type: object + properties: + freeform: { type: string } + locality: { type: string } + +# In building.yaml +properties: + address: { "$ref": "../defs.yaml#/$defs/propertyDefinitions/address" } +``` + +**Pydantic approach:** + +```python +# In overture-schema-theme-addresses/src/overture/schema/addresses/address.py +@no_extra_fields +class Address(BaseModel): + """A postal address.""" + + freeform: str | None = None + locality: str | None = None + + +# In overture-schema-theme-buildings/src/overture/schema/buildings/building.py +class Building(OvertureFeature): + address: Address | None = None +``` + +**Primary differences:** + +- JSON Schema uses `$ref` to reference definitions; Pydantic uses direct Python imports +- JSON Schema definitions live in `$defs`; Pydantic models are regular Python classes grouped into modules +- JSON Schema allows inline definitions; Pydantic encourages separate model classes + +##### How Containers Work + +**JSON Schema containers** (like `namesContainer`, `shapeContainer`) are reusable property groups: + +```yaml +# In defs.yaml +propertyContainers: + namesContainer: + properties: + names: { "$ref": "#/$defs/propertyDefinitions/allNames" } + + shapeContainer: + properties: + height: { type: number } + num_floors: { type: integer } + +# In building.yaml +allOf: + - "$ref": ../defs.yaml#/$defs/propertyContainers/namesContainer + - "$ref": ./defs.yaml#/$defs/propertyContainers/shapeContainer +``` + +**Pydantic equivalent** uses **mixin classes**: + +```python +# namesContainer -> Named, in +# overture-schema-common/src/overture/schema/common/names.py +class Named(BaseModel): + """Properties defining the names of a feature.""" + + names: Names | None = None + + +# shapeContainer -> Appearance, in buildings/_common.py, +# shared by Building and BuildingPart +class Appearance(BaseModel): + """Physical and visual properties of a building.""" + + height: float64 | None = None + num_floors: int32 | None = None + # ... roof and facade fields + + +# Usage with multiple inheritance -- this is how Building is actually declared +class Building( + OvertureFeature[Literal["buildings"], Literal["building"]], + Named, + Stacked, + Appearance, +): ... # inherits names, level, height, num_floors, and the rest from its parents +``` + +JSON Schema containers become **mixin classes** in Pydantic that you inherit from. + +##### Common Translation Patterns + +| JSON Schema | Pydantic | Notes | +|-------------|----------|-------| +| `"$ref": "other.yaml#/path"` | `from other import Model` | Direct Python imports | +| `allOf: [ref1, ref2]` | `class Model(Base1, Base2)` | Multiple inheritance | +| `minLength: 1` | `Field(min_length=1)` | Field constraints | +| `minimum: 0, maximum: 100` | `Field(ge=0, le=100)` | Numeric ranges | +| `uniqueItems: true` | `UniqueItemsConstraint()` | Custom constraint | +| `enum: [a, b, c]` | `class E(str, Enum): A="a"` | Enum class | +| `type: ["string", "null"]` | `str \| None = None` | Optional types | +| `if/then` conditional | Custom validation constraints | Model constraints | + +--- + + +--- + +## Development workflow + + +This project uses [uv](https://docs.astral.sh/uv/) for dependency management: + +```bash +# Install dependencies for the entire workspace +uv sync --all-packages + +# Run all tests and type/code quality checks +make check + +# Run tests for a specific package +uv run pytest packages/overture-schema-theme-buildings/ + +# Run tests matching a pattern +uv run pytest -k "buildings" +``` + +Auto-format / fix code to align with project expectations: + +```shell +uv run ruff check --fix +uv run ruff format +uv run docformatter --in-place --recursive packages/ +``` + +--- + +## Templates and quick reference + +### Reference + +#### Complete Templates + +##### Basic Model Template + +```python +from typing import Annotated +from pydantic import BaseModel, Field +from overture.schema.system.model_constraint import no_extra_fields +from overture.schema.system.numeric import int8, float64 + + +@no_extra_fields +class MyCustomType(BaseModel): + """Brief description of what this represents.""" + + # Required fields (no default value) + name: str + category: str + + # Optional fields (with default values) + description: str | None = None + + # Field with constraints and description + priority: Annotated[ + int8 | None, + Field( + ge=1, le=10, description="Priority level from 1 (lowest) to 10 (highest)" + ), + ] = None +``` + +##### Feature Template + +```python +from typing import Annotated, Literal +from pydantic import Field +from overture.schema.common import OvertureFeature +from overture.schema.system.geometric import ( + Geometry, + GeometryType, + GeometryTypeConstraint, +) + + +class MyFeature(OvertureFeature[Literal["my_theme"], Literal["my_type"]]): + """Description of what this feature represents.""" + + # Geometry with constraints + geometry: Annotated[ + Geometry, + GeometryTypeConstraint(GeometryType.POINT), + Field(description="Location of this feature"), + ] + + # Custom fields + my_field: str | None = None +``` + +##### Enum Template + +```python +from enum import Enum + + +class MyEnum(str, Enum): + """Description of what this enum represents.""" + + VALUE_ONE = "value_one" + VALUE_TWO = "value_two" + VALUE_THREE = "value_three" +``` + +##### Model with Validation Constraints + +```python +from typing import Annotated +from pydantic import BaseModel, Field +from overture.schema.system.field_constraint import UniqueItemsConstraint +from overture.schema.system.model_constraint import no_extra_fields + + +@no_extra_fields +class Contact(BaseModel): + """Contact information with validation constraints.""" + + name: str + email: str | None = None + phone: str | None = None + + # List with constraints + tags: Annotated[ + list[str] | None, + Field(min_length=1, description="Contact tags"), + UniqueItemsConstraint(), # No duplicate tags + ] = None +``` + +##### Association Feature Template + +```python +from typing import Annotated, Literal +from pydantic import Field +from overture.schema.common import OvertureFeature +from overture.schema.system.numeric import float64 +from overture.schema.system.ref import Id, Reference, Relationship + + +class MyAssociation( + OvertureFeature[Literal["associations"], Literal["my_association"]] +): + """Represents a relationship between two features with metadata.""" + + # References to the associated features + # Relationship takes a *kind* (COMPOSITION / AGGREGATION / HIERARCHY / + # ASSOCIATION); what the reference means is carried by `role`. Two + # references to related features need distinct roles to stay unambiguous. + feature_a_id: Annotated[ + Id, + Reference(Relationship.ASSOCIATION, FeatureA, role="connects_from"), + Field(description="First feature in the relationship"), + ] + + feature_b_id: Annotated[ + Id, + Reference(Relationship.ASSOCIATION, FeatureB, role="connects_to"), + Field(description="Second feature in the relationship"), + ] + + # Relationship metadata + relationship_type: Literal["primary", "secondary"] = "primary" + confidence: Annotated[float64 | None, Field(ge=0.0, le=1.0)] = None + + # Optional contextual information + notes: str | None = None +``` + +#### Quick Reference + +##### Essential Patterns (Most Common) + +```python +# Basic field types +name: str # Required string +name: str | None = None # Optional string +count: int32 # Required integer +priority: Literal["high", "medium", "low"] | None = None # Constrained values + +# Validated fields +height: Annotated[float64 | None, Field(ge=0, description="Height in meters")] = None +tags: Annotated[list[str] | None, Field(min_length=1), UniqueItemsConstraint()] = None + +# Association patterns -- Relationship is the kind, role is the meaning +parent_id: Annotated[ + Id | None, + Reference(Relationship.HIERARCHY, ParentModel, role="child_of"), +] = None +connector_ids: list[Id] # References to multiple related features +``` + +##### Model Templates + +```python +# Non-feature model +@no_extra_fields +class Address(BaseModel): + street: str + city: str | None = None + + +# Feature model +class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): + geometry: Geometry + height: float64 | None = None + + +# Enum +class Status(str, Enum): + ACTIVE = "active" + INACTIVE = "inactive" +``` + +##### Constraint Reference + +| Type | Constraint | JSON Schema | Example | +|------|------------|-------------|---------| +| **Numeric** | `ge=0, le=100` | `minimum`, `maximum` | `Field(ge=0, le=100)` | +| **String** | `min_length=1, pattern=r"..."` | `minLength`, `pattern` | `Field(min_length=1, pattern=r"^[A-Z]+$")` | +| **List** | `min_length=1, UniqueItemsConstraint()` | `minItems`, `uniqueItems` | `Field(min_length=1), UniqueItemsConstraint()` | +| **Custom** | `LanguageTagConstraint()` | Custom validation | `LanguageTagConstraint()` | + +##### Import Cheatsheet + +```python +# Essential imports for most models +from typing import Annotated, Literal +from enum import Enum +from pydantic import Field +from overture.schema.common import OvertureFeature +from overture.schema.system.field_constraint import UniqueItemsConstraint +from overture.schema.system.model_constraint import no_extra_fields +from overture.schema.system.numeric import int32, float64 + +# For associations and references +from overture.schema.system.ref import Id, Reference, Relationship +``` + +##### Naming Conventions + +- **Classes**: `PascalCase` (`Building`, `AccessRule`) +- **Fields**: `snake_case` (`construction_year`, `has_parts`) +- **Enums**: `UPPER_SNAKE_CASE = "value"` (`ACTIVE = "active"`) diff --git a/CONCEPTS.md b/CONCEPTS.md new file mode 100644 index 000000000..366748a17 --- /dev/null +++ b/CONCEPTS.md @@ -0,0 +1,803 @@ +# Concepts + +Background on why the Overture schema looks the way it does. **None of this is needed to +get work done** — [SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) is the path through the examples, +and it stands on its own. Read a section here when you hit something whose *why* you want. + +Not to be confused with [GLOSSARY.md](GLOSSARY.md), which defines the same vocabulary in a +sentence or two each. If you want to know what *envelope* or *workspace* or *tag* means, +the glossary is faster. This page is for why they exist. + +| Question | Section | +|---|---| +| Why have a schema at all? | [Beyond raw data](#beyond-raw-data) | +| Why Pydantic instead of JSON Schema? | [Why Pydantic](#why-pydantic-rather-than-json-schema) | +| Why is the schema split into a dozen packages? | [Many packages](#why-the-schema-is-many-packages) | +| What is an "envelope"? Why do Overture fields sit under `properties`? | [The GeoJSON envelope](#the-geojson-envelope) | +| Why does `type` appear three times in one file? | [Three keys named `type`](#three-keys-named-type) | +| If the schema is Pydantic now, why does it still look like GeoJSON? | [Why there's an envelope at all](#why-theres-an-envelope-at-all) | +| Why can a model be named two ways? | [Two names per model](#two-names-per-model) | +| What decides whether a field is required? | [What makes a field required](#what-makes-a-field-required) | +| What are those `overture:theme=` tags? | [How tags work](#how-tags-work) | +| What is the example file the guide validates? | [The example file](#what-the-example-file-actually-is) | +| Why are the examples YAML? | [Why examples are YAML](#why-examples-are-yaml) | +| Why isn't the generated PySpark code in git? | [Generated code](#why-generated-code-is-gitignored) | + +--- + +## Why this exists + +This project provides type-safe Python models for validating and working with +[Overture](https://overturemaps.org/) data. Use these schemas to: + +- Validate Overture data +- Build data processing pipelines with type safety +- Extend schemas with custom fields and validation rules + +## Beyond raw data + +This project addresses a fundamental challenge in data consumption: **bridging the +semantic gap between raw data and human understanding** while enabling +machine-actionable workflows. + + +Take a column like `pop_2020`. Is it total population? Population density per square +kilometer? Working-age population? Without a schema, you're left sampling values and +guessing from column names. + +Compare this to OpenStreetMap's approach: features use well-known key/value pairs like +`building=residential` or `addr:housenumber=42` that have semantic meaning and can be +looked up on the OSM wiki. This creates a step toward a schema - shared vocabulary with +documented semantics used across a vast dataset. However, OSM tags remain free-form: +multiple valid ways to express the same concept, no built-in validation, and complex +downstream validation because of undocumented keys that might have meaning to someone, +somewhere. A schema provides the structured alternative: explicit types, clear +validation rules, and semantic meaning that both humans and systems can rely on. + +Data files containing only column names and values aren't fully documented. External +metadata files typically focus on how data was collected and encoded, not on semantic +meaning or validation rules. Data consumers struggle to understand what datasets contain +and which columns they need for their goals. + +## Why Pydantic rather than JSON Schema + +We initially chose JSON Schema because it aligned with our mental model and promised to +solve our problems as we understood them. But JSON Schema surfaced several pain points: + +- **Authoring difficulty**: Hard to write correctly, difficult to verify, limited IDE + support, no refactoring capabilities +- **Tooling gaps**: Generic tools can't tailor output for specific applications like + ours +- **Development friction**: Schema changes required manual coordination across multiple + artifacts + +Pydantic addresses these systematically: author in Python with full IDE support, +generate tailored documentation, and automatically produce the specific artifacts each +workflow needs. Pydantic can also produce JSON Schema, so any application that requires +it can use it while we gain all the Python benefits during authoring. + +## The result + +Instead of spending time deciphering what columns mean and whether data matches +expectations, users can focus on their actual goals: analysis, visualization, +integration. Quality improves because validation happens automatically rather than +through manual inspection. + +The fundamental approach - human-readable authoring that generates machine-actionable +outputs - has broader applications beyond Overture and geospatial data. We hope others +will adapt these patterns for linking with Overture data or modeling their own domains +entirely. + +--- + +## Why the schema is many packages + +> **Why split the schema into so many packages?** Because *what you install determines what +> exists at runtime*. The models register themselves through Python entry points, so +> installing only the buildings theme means the CLI only knows about buildings. That's +> the extension mechanism — see [Using the packages from your own project](SCHEMA_GUIDE.md#7-using-the-packages-from-your-own-project) and +> [Register your own feature types](AUTHORING.md#register-your-own-feature-types). If you just want everything, that's fine too. + +### What "workspace" means + +A **uv workspace** is one repository containing several packages that are developed +together, sharing **one lockfile** and **one virtual environment**. If you've used Cargo +workspaces, npm workspaces, or a monorepo, it's the same idea. + +The root `pyproject.toml` declares it: + +```toml +[project] +name = "overture-schema-workspace" # ← a container, not something you install +version = "0.0.0" + +[tool.uv.workspace] +members = ["packages/*"] # ← every directory under packages/ is a member +``` + +Two things follow from this: + +**1. You never install or import `overture-schema-workspace`.** It's scaffolding. It +exists so `uv` knows which directories are members. There is no `import +overture_schema_workspace`. + +**2. The packages depend on each other *locally*, not through PyPI.** Look at +`packages/overture-schema-cli/pyproject.toml`: + +```toml +[tool.uv.sources] +overture-schema-common = { workspace = true } +overture-schema-system = { workspace = true } +``` + +`workspace = true` means "use the copy in this repo." This is why none of this needs to +be published to PyPI for you to work with it — the packages find each other. + +--- + +## The GeoJSON envelope + +An **envelope** is an outer wrapper that carries a payload plus a little standard +information about it. The term is borrowed from mail: the address and stamp go on the +outside and are the same on every envelope; the letter inside is whatever you wrote. + +GeoJSON works exactly that way. Every GeoJSON Feature has the same four outer keys, fixed +by RFC 7946 — that's the envelope. Everything specific to *your* data goes in one of them, +`properties` — that's the payload. + +```json +{ + "type": "Feature", <- envelope: what kind of object this is + "id": "overture:buildings:building:1234", <- envelope: identity + "geometry": { "type": "Polygon", ... }, <- envelope: where it is + "properties": { <- envelope: the pocket for everything else + "theme": "buildings", payload: Overture's fields + "type": "building", + "height": 21.34, + "num_floors": 4, + "class": "parking" + } +} +``` + +Split them apart for yourself: + +```python +import json, yaml +from overture.schema.buildings import Building + +b = Building.model_validate_json( + json.dumps(yaml.safe_load(open("examples/buildings/building-polygon.yaml")))) +gj = b.model_dump(mode="json", by_alias=True, exclude_none=True) + +print("envelope keys:", sorted(gj)) +print("payload keys :", sorted(gj["properties"])) +``` + +``` +envelope keys: ['geometry', 'id', 'properties', 'type'] +payload keys : ['class', 'ext_bar', 'ext_foo', 'height', 'is_underground', 'level', + 'num_floors', 'num_floors_underground', 'sources', 'subtype', + 'theme', 'type', 'version'] +``` + +Four keys outside, thirteen inside. **A GeoJSON Feature for a road, a lake, or a mailbox +has the same four outer keys** — that's what makes it universally readable. Only the +payload differs. + +So when this guide says "the envelope owns `id` and `geometry`," it means those two are +outer keys, placed there by GeoJSON's rules rather than by Overture's. And "the fields are +nested under the envelope" means the Overture fields sit inside `properties` rather than +at the top. + +One consequence worth carrying forward: **the envelope only exists in the GeoJSON +rendering.** In the Pydantic model, and in Parquet, there is no envelope — `id`, +`geometry`, and `height` are all just fields side by side. + +### Three keys named `type` + +A **key** is a field name — the part left of the colon. This file uses the key `type` +three times, at three nesting levels, meaning three unrelated things: + +| Where | Value | Comes from | Means | +|---|---|---|---| +| top level | `Feature` | GeoJSON spec | "this object is a GeoJSON Feature" | +| inside `geometry` | `Polygon` | GeoJSON spec | "this shape is a polygon" | +| inside `properties` | `building` | Overture | "this feature is a building" | + +List them yourself: + +```python +import yaml +d = yaml.safe_load(open('examples/buildings/building-polygon.yaml')) +print('type =', d['type']) +print('geometry.type =', d['geometry']['type']) +print('properties.type =', d['properties']['type']) +``` + +``` +type = Feature +geometry.type = Polygon +properties.type = building +``` + +Only the third is Overture's. The first two belong to GeoJSON, the envelope Overture data +is wrapped in when written as JSON. Whenever this guide says "the feature's type" it means +`properties.type` — the one holding `building`, `place`, or `segment`. + +That layering is the single most confusing thing about this data, and section 3 is largely +about it. + +**"Validating" means:** the CLI parses the file, reads `theme: buildings` and +`type: building` to decide *which model* to check against, then checks every field +against that model — types, numeric bounds, enum membership, required fields, and +cross-field rules. + +You can watch it pick the model. Delete the `theme:` line and it no longer knows: + +``` +⚠ Ambiguous: Data matches multiple types equally. Consider: + • Specifying --tag or --type to narrow validation + • Adding discriminator fields to clarify intent +``` + +### Why there's an envelope at all + +If the schema is now Pydantic, why does any of this still look like GeoJSON? Because those +two things answer different questions, and only one of them changed. + +| | What it is | Did it change? | +|---|---|---| +| **Pydantic** | How the schema is *authored and enforced*, in Python | **Yes** — it replaced hand-written JSON Schema YAML | +| **GeoJSON** | One way a feature can be *written out as JSON*, per RFC 7946 | **No** — it's an interchange format, not an authoring choice | + +Pydantic replaced JSON Schema as the authoring language. It has nothing to say about how +data is serialized, so GeoJSON was never in scope to replace. + +**But GeoJSON is not how Overture ships bulk data — Parquet is.** The release bucket is +Hive-partitioned Parquet: + +``` +s3a://overturemaps-us-west-2/release/2026-07-22.0/theme=buildings/type=building/ +``` + +That is a columnar table: `id`, `geometry`, `height`, and the rest are *columns*, flat, no +envelope. It's what `overture-validate` reads, what DuckDB attaches to, and what you'd +query for anything at scale. + +So why does GeoJSON appear at all? Because **JSON Schema describes JSON documents**, and +when a geospatial feature is written as a single JSON document, GeoJSON is the format +every GIS tool already reads. Parquet has no JSON representation to describe — it has a +columnar schema instead, which is exactly what +[section 6](SCHEMA_GUIDE.md#6-converting-the-schema-to-other-formats) generates as a Spark `StructType`. + +The honest summary is that there are two serializations and neither is subordinate: + +| Serialization | Shape | Pydantic mode | Where you meet it | +|---|---|---|---| +| **Parquet** | flat / columnar | `python` | the release bucket, Spark, DuckDB — all bulk data | +| **GeoJSON** | nested envelope | `json` | single features, extracts, examples in this repo, web tooling | + +The model supports both deliberately. The JSON Schema you're reading in this subsection +describes the second one, which is the only reason an envelope shows up here at all. + +### Interchange format vs storage format + +An **interchange format** is one whose job is handing data to a program you didn't write. +It's optimized for being *understood by anything*, not for being stored efficiently. A +**storage format** is the opposite: optimized for holding a lot of data and querying it +fast, at the cost of needing specific software to read it at all. + +| | GeoJSON | Parquet | +|---|---|---| +| Optimized for | being read by anything | storing and scanning millions of rows | +| Text or binary | plain text | binary, columnar, compressed | +| Read it with | any JSON parser | a Parquet library | +| Self-describing | yes — the file says what it is | schema in the footer, not human-readable | +| Good at | one feature, an extract, a web map | a whole theme, a whole planet | + +The cost of being universally readable is that GeoJSON repeats every field name on every +feature: + +```python +import json, yaml +from overture.schema.buildings import Building + +b = Building.model_validate_json( + json.dumps(yaml.safe_load(open("examples/buildings/building-polygon.yaml")))) +gj = b.model_dump(mode="json", by_alias=True, exclude_none=True) +flat = b.model_dump(mode="python", by_alias=True, exclude_none=True) +flat["geometry"] = str(flat["geometry"]) + +n = 10_000 +doc = json.dumps({"type": "FeatureCollection", "features": [gj] * n}) +cols = list(flat) +tbl = json.dumps({"columns": cols, "rows": [[flat[k] for k in cols]] * n}) +print(f"{n:,} features as GeoJSON : {len(doc):>10,} bytes") +print(f"{n:,} features, names stored once: {len(tbl):>10,} bytes") +``` + +``` +10,000 features as GeoJSON : 7,040,043 bytes +10,000 features, names stored once: 4,520,199 bytes +``` + +A 1.56x penalty before compression even enters the picture, and that comparison is still +generous to GeoJSON — real Parquet also compresses each column and lets a reader skip +columns it doesn't need. Multiply by a planet's worth of buildings and the reason bulk +data isn't shipped as GeoJSON is obvious. + +"Interchange" is a role, not a ranking. GeoJSON is the right tool for handing one feature +to a web map; Parquet is the right tool for handing a continent to Spark. + +### Is Pydantic wrapped around GeoJSON? + +Short answer: **no.** But the question has three reasonable readings, and one of them is a +qualified yes, so it's worth taking them separately. + +**"Is the model built on top of a GeoJSON structure?"** No. A model is a flat list of +fields, declared one at a time in Python. You can build one and use it without JSON ever +entering the picture: + +```python +from overture.schema.buildings import Building +from overture.schema.system.geometric import Geometry + +b = Building( + id="my-building-1", + geometry=Geometry.from_wkt("POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))"), + theme="buildings", + type="building", + version=1, + height=12.5, +) +print(b.id, b.height) +print(sorted(b.model_dump(mode="python", by_alias=True, exclude_none=True))) +``` + +``` +my-building-1 12.5 +['geometry', 'height', 'id', 'level', 'theme', 'type', 'version'] +``` + +No GeoJSON was parsed, produced, or consulted. If GeoJSON were the substrate, that +wouldn't be possible. + +**"Is there GeoJSON code inside the Pydantic classes?"** Yes — in exactly one of them. +Counting mentions across everything `Building` inherits from: + +``` +Building (building ) 0 +OvertureFeature (feature ) 0 +Identified (id ) 0 +Feature (feature ) 17 <- the base class in overture-schema-system +Named (names ) 0 +Stacked (level ) 0 +Appearance (_common ) 0 +``` + +All of it lives in `Feature`, in one serializer and one validator — the code that reads +GeoJSON in and writes GeoJSON out. Zero mentions in the six classes that actually define +what a building *is*. GeoJSON is an I/O concern parked at the base of the hierarchy, not a +structure the schema is built on. + +**"Is GeoJSON what's really being validated?"** No. What gets validated is the model. A +GeoJSON document is one accepted *input shape* — a flat dict is the other, and both end up +as the same Python object. + +An analogy: a word processor's document isn't "wrapped around `.docx`." It has a document +model, and it can read and write `.docx`. Deleting that import/export code would not +change what a document is. Same here — delete the ten lines below and Overture models +still work; they just stop speaking GeoJSON. + + +The entire GeoJSON transformation is those ten lines, in `Feature` +(`packages/overture-schema-system/src/overture/schema/system/feature.py`): + +```python +@model_serializer(mode="wrap") +def __serialize_with_geo_json_support__(self, serializer, info): + data = serializer(self) # <- the flat dict, produced first + + if info.mode == "json": # <- only in JSON mode + return { + "type": "Feature", + **({"id": data.pop("id")} if "id" in data else {}), + **({"bbox": data.pop("bbox")} if "bbox" in data else {}), + "geometry": data.pop("geometry"), + "properties": data, # <- everything else goes here + } + + return data # <- Python mode: flat, untouched +``` + +Read the first line: Pydantic produces the **flat** dictionary, and only then does this +function move `id`, `bbox`, and `geometry` to the top and sweep the remainder into +`properties`. In `python` mode the flat dict is returned unchanged and none of this runs. + +So the layering is: + +``` + Pydantic model (flat — the actual schema) + | + +----------+----------+ + | | + python mode json mode + | | + flat dict GeoJSON envelope + -> Parquet -> .geojson +``` + +GeoJSON is a costume the model puts on for one specific audience. It isn't the body. + +**The Pydantic model itself has no envelope.** In Python it's completely flat: + +```python +from overture.schema.buildings import Building +print("id in model_fields :", "id" in Building.model_fields) +print("geometry in model_fields :", "geometry" in Building.model_fields) +print("a 'properties' field? :", "properties" in Building.model_fields) +``` + +``` +id in model_fields : True +geometry in model_fields : True +a 'properties' field? : False +``` + +There is no `properties` field on `Building`, and `id` and `geometry` sit alongside +`height` and `num_floors` like any other field. The envelope is **not part of the model**. +It appears only when serializing to JSON, because that is what GeoJSON requires. + +You can watch the same object take both shapes: + +```python +import json, yaml +from overture.schema.buildings import Building + +b = Building.model_validate_json( + json.dumps(yaml.safe_load(open("examples/buildings/building-polygon.yaml")))) + +print("python mode:", sorted(b.model_dump(mode="python", by_alias=True, exclude_none=True))[:8]) +print("json mode :", sorted(b.model_dump(mode="json", by_alias=True, exclude_none=True))) +``` + +``` +python mode: ['class', 'ext_bar', 'ext_foo', 'geometry', 'height', 'id', 'is_underground', 'level'] +json mode : ['geometry', 'id', 'properties', 'type'] +``` + +One model, two renderings — and the flat one is the shape of the data you'd actually +download. Neither is more "real"; the model is what's real, and both are projections of +it. + +**So which answer to "what's required" is correct?** The model's: + +``` +['geometry', 'id', 'theme', 'type', 'version'] +``` + +The JSON Schema's two `required` arrays are that same list, split across the envelope +because that's where those fields land *in that particular output format*. Nobody decided +`geometry` belongs somewhere different from `theme`; GeoJSON did, in 2016. + +This distinction is the single most important thing in this guide, and it returns in force +in [section 3](SCHEMA_GUIDE.md#3-writing-code-against-the-models) — where using the wrong mode for your +data shape is the most common way to get a confusing `ValidationError`. + +--- + +## Two names per model + +Because the registry has to stay correct when packages it has never heard of register +their own models. + +The **canonical key is the entry-point string** — `overture.schema.buildings:Building`. +It includes the module path, so it is globally unique: no two packages can collide. + +The **short name is a derived alias**. It is just the class name after the colon, +snake-cased: + +```python +from overture.schema.system.discovery.entry_point import entry_point_class_alias + +entry_point_class_alias("overture.schema.divisions:DivisionArea") # 'division_area' +entry_point_class_alias("overture.schema.places:Place") # 'place' +``` + +Short names are *not* guaranteed unique. Anyone can +[register their own feature types](AUTHORING.md#register-your-own-feature-types), and nothing +stops a third party from shipping its own `Place`. So the short name can't be the +identity — it's a convenience, because +`validate_model(df, "overture.schema.buildings:Building")` is miserable to type. + +**The nice part is how it degrades.** The alias is offered only while it stays +unambiguous. `model_names()` counts aliases and includes only those appearing once, and +the resolver tries an exact key match first, then the alias: + +```python +from overture.schema.system.discovery.entry_point import resolve_entry_point_key + +registry = {"overture.schema.places:Place": ..., "acme.parks:Place": ...} + +resolve_entry_point_key("place", registry) +# ValueError: Entry-point alias 'place' is ambiguous. +# Specify one of: acme.parks:Place, overture.schema.places:Place + +resolve_entry_point_key( + "acme.parks:Place", registry +) # 'acme.parks:Place' — always works +``` + +Install a package that collides and `place` simply stops being accepted, with an error +naming both candidates — rather than silently validating against the wrong model. The +fully-qualified key never stops working. + +Two functions expose the two views: + +| Function | Returns | Use when | +|---|---|---| +| `model_keys()` | the 15 canonical entry-point keys | you want the authoritative list | +| `model_names()` | all 30 accepted names | you want everything `validate_model` will take | + +> **Skipping this step does not produce an error message.** It produces an empty +> registry. If `validate_model(df, "building")` raises a `KeyError`, or `model_names()` +> is empty, this is why. + +--- + +## What makes a field required + +Nobody maintains a list of required fields. **It's derived** — in Pydantic, a field with +no default is required, and a field with a default is optional: + +```python +from overture.schema.buildings import Building +from pydantic_core import PydanticUndefined + +for n in ["version", "theme", "height", "num_floors"]: + f = Building.model_fields[n] + d = "no default" if f.default is PydanticUndefined else f"default={f.default!r}" + print(f"{n:12} {d:16} -> {'REQUIRED' if f.is_required() else 'optional'}") +``` +``` +version no default -> REQUIRED +theme no default -> REQUIRED +height default=None -> optional +num_floors default=None -> optional +``` + +So "who decided" becomes "where is the field declared." For a building, five fields are +required and four of them come from a shared base class rather than from buildings at all: + +```python +from overture.schema.buildings import Building + +for n, f in Building.model_fields.items(): + if not f.is_required(): + continue + for cls in Building.__mro__: + if n in getattr(cls, "__annotations__", {}): + print(f"{n:10} -> {cls.__name__}") + break +``` +``` +id -> OvertureFeature +geometry -> Building +theme -> OvertureFeature +type -> OvertureFeature +version -> OvertureFeature +``` + +`id`, `theme`, `type`, and `version` are required of *every* Overture feature, declared +once in `OvertureFeature`: + +```python +id: Id = Field(description="A feature ID. ...") +theme: ThemeT +type: TypeT +# Superclass `Feature` provides `geometry` and `bbox`. +version: FeatureVersion +``` + +None carries `= None`, so all four are mandatory. `Building` adds only `geometry`, +narrowing the inherited one to the polygon types a building may have. + +In the generated JSON Schema those same five get split across the GeoJSON envelope — `id` +and `geometry` at the top, `theme`, `type`, and `version` inside `properties` — which is +why that document appears to have two answers to one question. It doesn't; it has one +answer written in the shape GeoJSON demands. + +**The human answer:** the Overture Schema Working Group decides, and changes go through +the process in [CONTRIBUTING.md](CONTRIBUTING.md) — a PR plus a changelog fragment. Making +a field required is a breaking change, so it targets the `vnext` branch and waits for a +major release; making one optional is not, and can go to `main`. + +--- + +## How tags work + +Every feature type carries a handful of **tags**, and `overture-schema list-types` prints +them after the type name: + +``` +building feature overture overture:theme=buildings +``` + +Three tags there. `feature` says this is a map feature rather than some other kind of +model. `overture` says Overture defined it. `overture:theme=buildings` says which theme it +belongs to. + +**Why a tag rather than a field called `theme`?** Because the tooling has to work on +feature types it has never heard of. A field named `theme` would only mean something to +code that already knows Overture has themes; the CLI would have to hardcode that. A tag is +just a label the type declares about itself, and the CLI's job is only to match labels — +so `--tag overture:theme=buildings` and `--tag acme:product=parks` go through exactly the +same code path. + +The `namespace:key=value` shape exists so that two organizations can both tag their types +without colliding. Everything Overture defines is namespaced under `overture:`; if you +register your own feature types, you pick your own namespace and your tags sit alongside +Overture's rather than competing with them. That is the whole extension mechanism — see +[Register your own feature types](AUTHORING.md#register-your-own-feature-types). + +The bare tags (`feature`, `overture`) have no namespace because they are not claims about +a vendor's taxonomy — they are the two facts every Overture feature type shares. + +--- + +## What the example file actually is + +The guide validates `examples/buildings/building-polygon.yaml` as its first real command. +`examples/buildings/building-polygon.yaml` is a file **in the repo you just cloned**. It +is not data you downloaded. The `examples/` tree is the project's own corpus of +hand-written sample features, used as test fixtures and pulled into the documentation +site. This one describes a single building — a parking structure in Washington DC: + +```yaml +id: overture:buildings:building:1234 +type: Feature # ← GeoJSON envelope +geometry: + type: Polygon + coordinates: [[ [-77.036873, 38.897804], ... ]] +properties: + ext_foo: I am a customer user property. # ← custom, non-Overture + theme: buildings # ← which theme + type: building # ← which feature type + version: 1 + height: 21.34 + num_floors: 4 + subtype: transportation + class: parking + sources: + - property: "" + dataset: microsoftMLBuildings +``` + +### See it actually catch something + +A success message proves the command ran, not that it's checking anything. Copy the file +and break it: + +```bash +cp examples/buildings/building-polygon.yaml /tmp/broken.yaml +``` + +Change `class: parking` to `class: skyscraper`: + +``` +class "skyscraper" ← Input should be 'agricultural', 'allotment_house', + 'apartments', 'barn', 'beach_hut', ... +``` + +Change `height: 21.34` to `height: -5`: + +``` +height -5 ← Input should be greater than 0 +``` + +Change `num_floors: 4` to `num_floors: 4.7`: + +``` +num_floors 4.7 ← Input should be a valid integer, got a number with a + fractional part +``` + +Enum membership, numeric bounds, integer-ness — each from the model definition, none of +it written by hand for this file. + +> **What it does not catch:** free-form string fields accept any string. The real +> `building-polygon.yaml` in the repo has a stray trailing comma — +> `dataset: microsoftMLBuildings,` — which YAML reads as part of the value. It parses to +> the string `'microsoftMLBuildings,'` and validates clean, because `dataset` has no +> constraint beyond "is a string." Validation enforces the schema, not your typing. + +--- + +## Why examples are YAML + +**No — real Overture data is GeoJSON or Parquet.** YAML here is purely an authoring +convenience for the example files: it allows comments (`# Custom user properties.`) and +is easier to hand-edit than JSON. + +The CLI accepts **JSON, YAML, and GeoJSON**, and YAML is a superset of JSON, so the +format is irrelevant to the validation. Convert the same file to JSON and you get the +same result: + +```bash +uv run python -c " +import json, yaml +json.dump(yaml.safe_load(open('examples/buildings/building-polygon.yaml')), + open('/tmp/same-building.json','w'), indent=2)" + +uv run overture-schema validate /tmp/same-building.json +``` +``` +✓ Successfully validated /tmp/same-building.json +``` + +Same bytes of meaning, different serialization, identical outcome. Pick whichever is +convenient — you'll mostly hand JSON or GeoJSON to this command in real use. + +--- + +## Why generated code is gitignored + +Not because it's optional, and not because it's for a subset of users. **It's build +output.** The commit that introduced the package says so directly: + +> The generated trees under `expressions/generated/` and `tests/generated/` are +> regenerable output of `make generate-pyspark` and are not tracked in git; `make check` +> and `make test-all` regenerate before running. + +Three reasons that's the right call: + +**One source of truth.** The Pydantic models define the schema; these expressions are a +derivative of them. Committing the derivative creates a second copy that can silently +drift — change a constraint, forget to regenerate, and the committed expressions keep +enforcing the old rule. Deleting them from git makes that failure impossible. + +**Scale.** A full generation is **32 files and roughly 23,000 lines**: + +``` +15 expression modules (one per feature type) +17 test modules (conformance tests, split per union arm) +``` + +Every schema change would produce a mechanical diff of that size, burying the actual +change and guaranteeing merge conflicts. + +**The build regenerates regardless.** `make check` and `make test-all` both depend on +`generate-pyspark`, which begins with `clean-pyspark` (`rm -rf`). The tree is rebuilt +from the current models every time, so a committed copy would never be read. + +It's the same reasoning that keeps `dist/`, `*.o`, and `node_modules/` out of git. + +### "Gitignored" does not mean "not shipped" + +This is the part worth being clear about, since it sounds like these files are somehow +optional for users. **They are not.** Published wheels contain them. + +`.github/workflows/publish-python-packages.yaml` runs `make generate-pyspark` before +`uv build`, then refuses to publish a wheel that lacks them: + +```yaml +- name: Generate PySpark expressions before build + if: matrix.package == 'overture-schema-pyspark' + run: make generate-pyspark +``` + +```bash +if [ "$PACKAGE" = "overture-schema-pyspark" ] && \ + ! unzip -l "$wheel" | grep -q 'expressions/generated/.*\.py'; then + echo " Wheel [$wheel] has no generated expressions -- codegen did not run. Aborting!" + exit 1 +fi +``` + +So the only people who ever run `make generate-pyspark` are people working **from a git +clone** — because a clone is the one place these files don't already exist. Install from +a package index and they arrive with the package, like any other module. + + +--- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a4579e9df..f863503b6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,10 +8,11 @@ Thank you for your interest in contributing. ## Working with the Python packages -The schema is authored as Pydantic models. [SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) covers the -whole toolchain: Part I for installing and using the packages, Part II for authoring new +The schema is authored as Pydantic models. [SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) covers +installing and using the packages; [AUTHORING.md](AUTHORING.md) covers authoring new schema models and the development workflow (`uv sync`, `make check`, ruff and -docformatter). +docformatter). [TROUBLESHOOTING.md](TROUBLESHOOTING.md) collects the errors that cost +people time. ## Where to send your change diff --git a/GLOSSARY.md b/GLOSSARY.md index 341fc8566..e060a84b6 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -4,8 +4,10 @@ Two vocabularies meet in this repository. The first describes the map: what an e what a feature is, how feature types are named. The second describes the Python packages that define the schema: workspaces, entry points, specs. Both are collected here. -Terms in the second group are cross-referenced to the section of -[SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) that covers them in full. +Terms in the second group are cross-referenced to the page and section that covers them +in full — [SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) for using the schema, +[AUTHORING.md](AUTHORING.md) for extending it, and [CONCEPTS.md](CONCEPTS.md) for +background. --- @@ -93,14 +95,14 @@ environment, declared in `pyproject.toml`. Nothing imports these directly; they discovered at runtime. The schema uses four groups: `overture.models` (feature types), `overture.tag_providers` ([tags](#tag)), `project.scripts` (the CLIs), and `pytest11` (test plugins). Registering your own model is a matter of adding an entry point — see -[Register your own feature types](SCHEMA_GUIDE.md#83-register-your-own-feature-types). +[Register your own feature types](AUTHORING.md#register-your-own-feature-types). ### Workspace One repository containing several independently-versioned packages that share a single lockfile and a single virtual environment. Declared by `[tool.uv.workspace]` in the root `pyproject.toml`. The schema repo is a `uv` workspace of thirteen packages. See -[What "workspace" means](SCHEMA_GUIDE.md#what-workspace-means). +[What "workspace" means](CONCEPTS.md#what-workspace-means). ### Metapackage @@ -162,7 +164,7 @@ before any output format is chosen — `RecordSpec` for a model, `UnionSpec` for discriminated union, `FieldSpec` for a field, `EnumSpec` for an enum. Renderers consume specs; they never touch Pydantic models directly. This split is what lets a new output format be a new renderer rather than new extraction logic. See -[Write a new codegen target](SCHEMA_GUIDE.md#84-write-a-new-codegen-target). +[Write a new codegen target](AUTHORING.md#write-a-new-codegen-target). ### Extra diff --git a/README.md b/README.md index 365c577c3..4969c2726 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,19 @@ The contents of this repository are presented in a more human-friendly format at ## Python packages The schema is authored as [Pydantic](https://docs.pydantic.dev/latest/) models, published -as a set of Python packages under `packages/`. See -[SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) for installing them, validating data, generating -artifacts, and authoring new schema models. +as a set of Python packages under `packages/`. + +These pages are for people working with the packages in code. **To read the schema itself +— what feature types exist, what fields they carry, what values are valid — use +[docs.overturemaps.org](https://docs.overturemaps.org/).** + +- [SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) — install the packages, explore the models, validate + data, generate artifacts. **Start here.** +- [AUTHORING.md](AUTHORING.md) — register your own feature types, author new schema + models, build an SDK or CLI on the models. +- [CONCEPTS.md](CONCEPTS.md) — why the schema is Pydantic, why it is many packages, and + what the GeoJSON envelope is. +- [TROUBLESHOOTING.md](TROUBLESHOOTING.md) — symptom-indexed fixes and known gotchas. ## Schema reference @@ -20,7 +30,7 @@ artifacts, and authoring new schema models. **Out of date:** it predates the Pydantic packages and still describes JSON Schema as the way the schema is defined, spells `subtype` as `subType`, and leaves the extensions section unfinished. Useful for the conventions themselves; check anything structural - against [SCHEMA_GUIDE.md](SCHEMA_GUIDE.md). + against [SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) and [AUTHORING.md](AUTHORING.md). ## Contributing See [CONTRIBUTING.md](CONTRIBUTING.md) for branching strategy, workflow, and contribution guidelines. diff --git a/SCHEMA_GUIDE.md b/SCHEMA_GUIDE.md index db97b2c27..943a699d9 100644 --- a/SCHEMA_GUIDE.md +++ b/SCHEMA_GUIDE.md @@ -1,36 +1,19 @@ # Overture Schema Guide -A practical guide to installing, exploring, building on, and authoring the Overture Maps -schema packages. +A practical guide to installing the Overture Maps schema packages, exploring the models, +writing code against them, validating data, and generating artifacts from the schema. -**Audience:** everyone who touches these packages in Python. The guide is in parts, and -you probably want one of them rather than all of them: +Three other pages cover material this guide points to rather than repeats: +[CONCEPTS.md](CONCEPTS.md) for why the schema is built this way, +[TROUBLESHOOTING.md](TROUBLESHOOTING.md) for errors and gotchas, and +[AUTHORING.md](AUTHORING.md) for writing schema models. Reference for a single package +lives in that package's `README.md` under `packages/`. -| If you want to | Read | -|---|---| -| Understand why the schema is Pydantic at all | Part 0 | -| Validate data, write code against the models, generate artifacts | Part I | -| Register your own feature types, or author new schema models | Part II | -| Look something up | Part III | -| Look up a term | [Glossary](GLOSSARY.md) | - -**Status:** none of these packages are on PyPI yet. Everything below installs from a -local clone with `uv`. Any `pip install overture-schema` you find in a README is -aspirational — it will not work today. - -**Verification:** every command and code block in this guide was checked against the -schema repo at commit `2a6170c4` on Python 3.10 — 87 Python blocks parsed, every -`overture.*` import resolved against the installed packages, and every runnable snippet -executed. Blocks that are deliberately wrong (marked ✗) or written as `>>>` transcripts -are excluded, as are template fragments with placeholder names. - ---- - -## Table of contents +*Note: none of these packages are on PyPI yet. Everything below installs from a local +clone. Any `pip install overture-schema` you find in a README is aspirational — it will +not work today.* -**Part 0 — [Why Pydantic](#part-0--why-pydantic)** - -**Part I — [Using the schema](#part-i--using-the-schema)** +## Contents 1. [Install](#1-install) 2. [Exploring the models](#2-exploring-the-models) @@ -39,106 +22,24 @@ are excluded, as are template fragments with placeholder names. 5. [Validating data](#5-validating-data) 6. [Converting the schema to other formats](#6-converting-the-schema-to-other-formats) 7. [Using the packages from your own project](#7-using-the-packages-from-your-own-project) +8. [Building tools on the models](#8-building-tools-on-the-models) -**Part II — [Extending and authoring the schema](#part-ii--extending-and-authoring-the-schema)** - -8. [Building your own SDK or CLI](#8-building-your-own-sdk-or-cli) -9. [Registering models and tagging](#9-registering-models-and-tagging) -10. [Authoring new schema models](#10-authoring-new-schema-models) -11. [Development workflow](#11-development-workflow) - -**Part III — [Reference](#part-iii--reference)** - -12. [Gotchas](#12-gotchas) -13. [Templates and quick reference](#13-templates-and-quick-reference) - -**[Glossary](GLOSSARY.md)** — data-model and toolchain vocabulary, cross-linked back into this guide. - ---- - -# Part 0 — Why Pydantic - -### Why this exists - -This project provides type-safe Python models for validating and working with [Overture -Maps Foundation](https://overturemaps.org/) data. Overture Maps is an open geospatial -dataset containing buildings, places, addresses, transportation networks, and -administrative boundaries curated from multiple sources. - -Use these schemas to: - -- Validate Overture Maps data -- Build data processing pipelines with type safety -- Extend schemas with custom fields and validation rules - -### Why Pydantic rather than JSON Schema? - -This project addresses a fundamental challenge in data consumption: **bridging the -semantic gap between raw data and human understanding** while enabling -machine-actionable workflows. - -### Why Schema at All: Beyond Raw Data - -Take a column like `pop_2020`. Is it total population? Population density per square -kilometer? Working-age population? Without a schema, you're left sampling values and -guessing from column names. - -Compare this to OpenStreetMap's approach: features use well-known key/value pairs like -`building=residential` or `addr:housenumber=42` that have semantic meaning and can be -looked up on the OSM wiki. This creates a step toward a schema - shared vocabulary with -documented semantics used across a vast dataset. However, OSM tags remain free-form: -multiple valid ways to express the same concept, no built-in validation, and complex -downstream validation because of undocumented keys that might have meaning to someone, -somewhere. A schema provides the structured alternative: explicit types, clear -validation rules, and semantic meaning that both humans and systems can rely on. - -Data files containing only column names and values aren't fully documented. External -metadata files typically focus on how data was collected and encoded, not on semantic -meaning or validation rules. Data consumers struggle to understand what datasets contain -and which columns they need for their goals. - -### Why Pydantic Over JSON Schema: Solving Multiple Problems - -We initially chose JSON Schema because it aligned with our mental model and promised to -solve our problems as we understood them. But JSON Schema surfaced several pain points: - -- **Authoring difficulty**: Hard to write correctly, difficult to verify, limited IDE - support, no refactoring capabilities -- **Tooling gaps**: Generic tools can't tailor output for specific applications like - ours -- **Development friction**: Schema changes required manual coordination across multiple - artifacts - -Pydantic addresses these systematically: author in Python with full IDE support, -generate tailored documentation, and automatically produce the specific artifacts each -workflow needs. Pydantic can also produce JSON Schema, so any application that requires -it can use it while we gain all the Python benefits during authoring. - -### The Result: Faster Understanding, Higher Quality - -Instead of spending time deciphering what columns mean and whether data matches -expectations, users can focus on their actual goals: analysis, visualization, -integration. Quality improves because validation happens automatically rather than -through manual inspection. - -The fundamental approach - human-readable authoring that generates machine-actionable -outputs - has broader applications beyond Overture and geospatial data. We hope others -will adapt these patterns for linking with Overture data or modeling their own domains -entirely. +### Other pages +| Page | What's on it | +|---|---| +| [AUTHORING.md](AUTHORING.md) | Registering your own feature types, authoring new schema models, building an SDK or CLI, templates | +| [CONCEPTS.md](CONCEPTS.md) | Why Pydantic, why many packages, the GeoJSON envelope, and other *why* questions | +| [TROUBLESHOOTING.md](TROUBLESHOOTING.md) | Symptom-indexed fixes, and the gotchas that cost people time | +| [GLOSSARY.md](GLOSSARY.md) | Data-model and toolchain vocabulary | --- -# Part I — Using the schema - ## 1. Install ### 1.1 First, what you're installing -**The schema repo is not one Python package. It's thirteen.** - -That's the thing that makes this confusing at the start, so it's worth a minute before -you run anything. +**The schema is not one Python package. It's many Python packages.** Open `packages/` and you'll see: @@ -154,51 +55,20 @@ packages/ └── overture-schema-theme-* ← six of these: buildings, places, transportation, … ``` -Each of those directories has its own `pyproject.toml` and its own version number. They -release independently. +Each of those directories has its own `pyproject.toml` and its own version number. -#### What "workspace" means - -A **uv workspace** is one repository containing several packages that are developed -together, sharing **one lockfile** and **one virtual environment**. If you've used Cargo -workspaces, npm workspaces, or a monorepo, it's the same idea. - -The root `pyproject.toml` declares it: - -```toml -[project] -name = "overture-schema-workspace" # ← a container, not something you install -version = "0.0.0" - -[tool.uv.workspace] -members = ["packages/*"] # ← every directory under packages/ is a member -``` - -Two things follow from this, and they're the ones that trip people up: - -**1. You never install or import `overture-schema-workspace`.** It's scaffolding. It -exists so `uv` knows which directories are members. There is no `import -overture_schema_workspace`. - -**2. The packages depend on each other *locally*, not through PyPI.** Look at -`packages/overture-schema-cli/pyproject.toml`: - -```toml -[tool.uv.sources] -overture-schema-common = { workspace = true } -overture-schema-system = { workspace = true } -``` - -`workspace = true` means "use the copy in this repo." This is why none of this needs to -be published to PyPI for you to work with it — the packages find each other on disk. +They are separate packages so that **what you install decides what exists at runtime** — +install only the buildings theme and the tooling only knows about buildings. For why that +was worth the complexity, see +[Why the schema is many packages](CONCEPTS.md#why-the-schema-is-many-packages). #### What you end up with -One command installs all thirteen into **a single shared virtualenv at the repo root**: +One command installs all the packages into **a single shared virtual environment at the repo root**: ``` schema/ -├── .venv/ ← created by uv; all 13 packages live here +├── .venv/ ← created by uv; all packages live here │ └── bin/ │ ├── overture-schema ← the three CLIs land here │ ├── overture-codegen @@ -207,18 +77,12 @@ schema/ └── pyproject.toml ``` -You don't activate it. `uv run ` runs `` inside that venv for you. -That's why every command in this guide starts with `uv run`. - -> **Why split into thirteen packages at all?** Because *what you install determines what -> exists at runtime*. The models register themselves through Python entry points, so -> installing only the buildings theme means the CLI only knows about buildings. That's -> the extension mechanism — see [Using the packages from your own project](#7-using-the-packages-from-your-own-project) and -> [Building your own SDK or CLI](#8-building-your-own-sdk-or-cli). If you just want everything, that's fine too. ---- +#### The packages form four layers -**Layering, bottom up:** +The packages depend on each other in one direction only — each layer below uses the one +above it, and never the reverse. That ordering is what lets the tooling work on feature +types nobody had written when the tooling was built. | Layer | Package | Gives you | |---|---|---| @@ -227,10 +91,8 @@ That's why every command in this guide starts with `uv run`. | Feature types | `theme-*` | `Building`, `Place`, `Segment`, `Address`, … plus their enums | | Tooling | `cli`, `codegen`, `pyspark`, `validation` | commands and functions that consume the above generically | -The tooling layer never hardcodes feature types. It discovers them. That's why your own -models can slot in. +The tooling layer never hardcodes feature types. It discovers them. That's why your own models can slot in. ---- ### 1.2 Prerequisites @@ -243,88 +105,9 @@ curl -LsSf https://astral.sh/uv/install.sh | sh ``` You do **not** need Java, Spark, or Homebrew for anything in sections 1 through 8 of this -guide except the PySpark parts. If you already have a Homebrew Spark installed, skip -ahead to [When something goes wrong](#16-when-something-goes-wrong) — it may actively break things. - ---- - -### 1.2a Two zsh gotchas when pasting commands - -macOS defaults to **zsh**, and interactive zsh does *not* treat `#` as a comment unless -you turn that on. Paste a line like this and zsh hands `#`, `→`, and `15` to `wc` as -filenames: - -``` -find ... | wc -l # → 15 -wc: #: open: No such file or directory -wc: →: open: No such file or directory -wc: 15: open: No such file or directory - 0 total -``` - -A line that *starts* with `#` fails more obviously — `command not found: #`. - -This guide keeps `#` comments only as standalone label lines inside multi-command blocks, -never trailing after a command. To paste those blocks whole, enable comments once: - -```bash -setopt interactive_comments -``` - -Add it to `~/.zshrc` to make it permanent. Otherwise, skip the `#` lines when copying — -they are labels, not commands. Scripts and non-interactive shells are unaffected; this is -purely an interactive-zsh behavior. - -#### 2. `!` runs a command out of your history +guide except the PySpark parts. If you already have a Homebrew Spark installed, it may actively break things — see +[`FileNotFoundError` on `spark-submit`](TROUBLESHOOTING.md#filenotfounderror-on-spark-submit). -This one is worth understanding because it can do real damage. In an interactive shell, -`!` triggers **history expansion**, and it fires *inside double quotes*. `!r` means "the -most recent command starting with `r`" — the shell splices that command's text into your -line before running it. - -So a Python one-liner containing `{value!r}`, pasted into zsh as - -``` -uv run python -c "... f'default={f.default!r}' ..." -``` - -becomes something else entirely. What you get depends on your own shell history: - -``` -SyntaxError: f-string: invalid syntax - (f.defaultrm -rf schema) -``` - -That is a past command of yours, pasted into the middle of a Python f-string. Here it only -produced a syntax error — Python never ran and nothing was deleted. But the same mechanism -can land text somewhere the shell *will* execute. - -**The fix used throughout this guide:** multi-line Python is passed via a heredoc with a -**quoted** delimiter, not `-c "..."`. - -```bash -uv run python <<'PY' -from overture.schema.buildings import Building -print(f"{Building.__name__!r} is safe here") -PY -``` - -Quoting the delimiter (`<<'""" + D + """'` rather than `<<""" + D + """`) disables every -form of expansion in the body — history, variables, command substitution. The text reaches -Python exactly as written. - -Verified rather than assumed: - -``` -double-quoted -c -> !r expanded into a command from history -quoted heredoc -> !r left alone -``` - -Single quotes also block history expansion, but the Python in this guide uses single -quotes internally, so heredocs are the practical choice. If you hit this in your own -one-liners, `{value!r}` can always be written `{repr(value)}` instead — no `!` at all. - ---- ### 1.3 Install @@ -334,7 +117,7 @@ cd schema uv sync --all-packages ``` -That's it. `uv sync --all-packages` reads the lockfile, creates `.venv/`, and installs +That's it. `uv sync --all-packages` creates `.venv/` and installs all thirteen workspace members into it. **This is the whole install for most people.** You can now validate data, explore models, @@ -342,34 +125,25 @@ generate JSON Schema, and generate documentation. **What about `make install`?** It works too, and nothing about it is risky. It's exactly `uv sync --all-packages --all-extras` plus the PySpark generation step described in -[Do you need PySpark?](#15-do-you-need-pyspark) — so it just does more than most people +[Do you need PySpark?](#14-do-you-need-pyspark) — so it just does more than most people need. (No package in the workspace defines extras today, so `--all-extras` changes nothing.) Use it if you'd rather run one command and have everything. ---- - -### 1.4 Check it worked -Run these three. All should succeed: +Run these commands. All should succeed: ```bash uv run overture-schema --version ``` + ``` overture-schema, version 1.17.1 ``` -> **Seeing a warning banner above that line?** Something like -> `warning: Failed to parse pyproject.toml ... exclude-newer = "1 week"`. The command -> still worked — but your `uv` is too old for this repo, and it is quietly rewriting -> `uv.lock` behind your back. Stop and fix it before going further: -> [uv warns about `exclude-newer`](#uv-warns-about-exclude-newer--and-quietly-rewrites-your-lockfile). -> Every output shown in this guide assumes a correctly configured `uv` and omits that -> banner. - ```bash uv run overture-schema list-types ``` + ``` address feature overture overture:theme=addresses bathymetry feature overture overture:theme=base @@ -395,4409 +169,1575 @@ uv run overture-schema validate examples/buildings/building-polygon.yaml ✓ Successfully validated examples/buildings/building-polygon.yaml ``` -If all three worked, you're installed. **Skip to section 2** unless you need PySpark. - -#### What did that third command actually validate? +That file is a sample building that ships with the repo. For what's inside it, why it's +YAML, and proof that validation is really checking something, see +[the example file](CONCEPTS.md#what-the-example-file-actually-is). -Fair question — it's the first command that does real work, and the filename explains -nothing. -`examples/buildings/building-polygon.yaml` is a file **in the repo you just cloned**. It -is not data you downloaded. The `examples/` tree is the project's own corpus of -hand-written sample features, used as test fixtures and pulled into the documentation -site. This one describes a single building — a parking structure in Washington DC: +### 1.4 Do you need PySpark? -```yaml -id: overture:buildings:building:1234 -type: Feature # ← GeoJSON envelope -geometry: - type: Polygon - coordinates: [[ [-77.036873, 38.897804], ... ]] -properties: - ext_foo: I am a customer user property. # ← custom, non-Overture - theme: buildings # ← which theme - type: building # ← which feature type - version: 1 - height: 21.34 - num_floors: 4 - subtype: transportation - class: parking - sources: - - property: "" - dataset: microsoftMLBuildings -``` +Probably not. Answer honestly: -#### "Envelope" — the word this guide keeps using +| I want to… | Need PySpark? | +|---|---| +| Validate files, explore models, write Python against them | **No** | +| Generate JSON Schema or markdown docs | **No** | +| Generate an SDK in another language | **No** | +| Validate millions of rows of Parquet, or data in S3 | **Yes** | +| Get a Spark `StructType` for a feature type | **Yes** | +| Run the full test suite (`make check`) | **Yes** | -An **envelope** is an outer wrapper that carries a payload plus a little standard -information about it. The term is borrowed from mail: the address and stamp go on the -outside and are the same on every envelope; the letter inside is whatever you wrote. +**If no:** you're done. Go to section 2. -GeoJSON works exactly that way. Every GeoJSON Feature has the same four outer keys, fixed -by RFC 7946 — that's the envelope. Everything specific to *your* data goes in one of them, -`properties` — that's the payload. +**If yes:** there's one more step, because the PySpark validation expressions are +*generated code that is not committed to git*. They're in `.gitignore`. `uv sync` alone +cannot produce them. -```json -{ - "type": "Feature", <- envelope: what kind of object this is - "id": "overture:buildings:building:1234", <- envelope: identity - "geometry": { "type": "Polygon", ... }, <- envelope: where it is - "properties": { <- envelope: the pocket for everything else - "theme": "buildings", payload: Overture's fields - "type": "building", - "height": 21.34, - "num_floors": 4, - "class": "parking" - } -} +```bash +make generate-pyspark ``` -Split them apart for yourself: - -```bash -uv run python <<'PY' -import json, yaml -from overture.schema.buildings import Building +Or `make install`, which is just `uv sync --all-packages --all-extras` followed by +`make generate-pyspark`. The command prints nothing on success. If `model_names()` +later comes back empty, see +[that entry in TROUBLESHOOTING.md](TROUBLESHOOTING.md#model_names-returns--or-keyerror-on-a-feature-type). -b = Building.model_validate_json( - json.dumps(yaml.safe_load(open("examples/buildings/building-polygon.yaml")))) -gj = b.model_dump(mode="json", by_alias=True, exclude_none=True) +--- -print("envelope keys:", sorted(gj)) -print("payload keys :", sorted(gj["properties"])) -PY -``` +## 2. Exploring the models -``` -envelope keys: ['geometry', 'id', 'properties', 'type'] -payload keys : ['class', 'ext_bar', 'ext_foo', 'height', 'is_underground', 'level', - 'num_floors', 'num_floors_underground', 'sources', 'subtype', - 'theme', 'type', 'version'] -``` +You can't write code against a model you haven't looked at. This section is about finding +out what feature types exist, what fields they carry, and what values those fields +accept — before writing a line of code against them. -Four keys outside, thirteen inside. **A GeoJSON Feature for a road, a lake, or a mailbox -has the same four outer keys** — that's what makes it universally readable. Only the -payload differs. +**Two things can answer your questions, and it's worth keeping them straight:** -So when this guide says "the envelope owns `id` and `geometry`," it means those two are -outer keys, placed there by GeoJSON's rules rather than by Overture's. And "the fields are -nested under the envelope" means the Overture fields sit inside `properties` rather than -at the top. +| | What it is | Ask it with | +|---|---|---| +| **The model** | The Pydantic classes themselves. When you validate a file or a DataFrame, this is the code that accepts or rejects it. | Python: `Building.model_fields` | +| **The generated JSON Schema** | A description of the model, written out as a JSON document | `overture-schema json-schema` | -One consequence worth carrying forward: **the envelope only exists in the GeoJSON -rendering.** In the Pydantic model, and in Parquet, there is no envelope — `id`, -`geometry`, and `height` are all just fields side by side. +They always agree, because the second is generated from the first. -#### Three keys named `type` +Ask the **model** when you want to understand the schema — it answers in flat Python, and +it is the thing that actually runs. Ask the **JSON Schema** when you need the rules in a +form another program can read: a validator in another language, a code generator, or a +tool that has no idea Python exists. Section 2.4 does the first, 2.5 the second. -A **key** is a field name — the part left of the colon. This file uses the key `type` -three times, at three nesting levels, meaning three unrelated things: +### 2.1 Running Python against the models -| Where | Value | Comes from | Means | -|---|---|---|---| -| top level | `Feature` | GeoJSON spec | "this object is a GeoJSON Feature" | -| inside `geometry` | `Polygon` | GeoJSON spec | "this shape is a polygon" | -| inside `properties` | `building` | Overture | "this feature is a building" | +Everything up to now has been shell commands. From here the guide switches to Python, so +first: how do you actually run it? -List them yourself: +**The models are installed in the project's `.venv`, not in whatever `python` your shell +finds.** Starting Python the usual way fails: ```bash -uv run python -c " -import yaml -d = yaml.safe_load(open('examples/buildings/building-polygon.yaml')) -print('type =', d['type']) -print('geometry.type =', d['geometry']['type']) -print('properties.type =', d['properties']['type'])" +python ``` - ``` -type = Feature -geometry.type = Polygon -properties.type = building +>>> from overture.schema.buildings import Building +Traceback (most recent call last): + File "", line 1, in +ModuleNotFoundError: No module named 'overture' ``` -Only the third is Overture's. The first two belong to GeoJSON, the envelope Overture data -is wrapped in when written as JSON. Whenever this guide says "the feature's type" it means -`properties.type` — the one holding `building`, `place`, or `segment`. - -That layering is the single most confusing thing about this data, and section 3 is largely -about it. - -**"Validating" means:** the CLI parses the file, reads `theme: buildings` and -`type: building` to decide *which model* to check against, then checks every field -against that model — types, numeric bounds, enum membership, required fields, and -cross-field rules. +`ModuleNotFoundError: No module named 'overture'` always means this: right code, wrong +interpreter. Compare the two: -You can watch it pick the model. Delete the `theme:` line and it no longer knows: +```bash +python -c "import sys; print(sys.executable)" +uv run python -c "import sys; print(sys.executable)" +``` ``` -⚠ Ambiguous: Data matches multiple types equally. Consider: - • Specifying --tag or --type to narrow validation - • Adding discriminator fields to clarify intent +/Users/you/.pyenv/versions/3.10.15/bin/python +/path/to/schema/.venv/bin/python3 ``` -#### Why is it YAML? Is Overture data YAML? - -**No — real Overture data is GeoJSON or Parquet.** YAML here is purely an authoring -convenience for the example files: it allows comments (`# Custom user properties.`) and -is easier to hand-edit than JSON. +The first is your system or `pyenv` Python, which has never heard of these packages. The +second is the project's environment, where `uv sync` installed them. Nothing is broken — +they're simply two different Pythons. -The CLI accepts **JSON, YAML, and GeoJSON**, and YAML is a superset of JSON, so the -format is irrelevant to the validation. Convert the same file to JSON and you get the -same result: +Prefix with `uv run` and it works — that runs Python inside the project's environment: ```bash -uv run python -c " -import json, yaml -json.dump(yaml.safe_load(open('examples/buildings/building-polygon.yaml')), - open('/tmp/same-building.json','w'), indent=2)" - -uv run overture-schema validate /tmp/same-building.json +uv run python -c "from overture.schema.buildings import Building; print(Building.__name__)" ``` ``` -✓ Successfully validated /tmp/same-building.json +Building ``` -Same bytes of meaning, different serialization, identical outcome. Pick whichever is -convenient — you'll mostly hand JSON or GeoJSON to this command in real use. - -#### See it actually catch something +Three ways to run the Python in this guide, all from the repo root: -A success message proves the command ran, not that it's checking anything. Copy the file -and break it: +**A one-liner**, for a quick look: ```bash -cp examples/buildings/building-polygon.yaml /tmp/broken.yaml +uv run python -c "from overture.schema.buildings import Building; print(len(Building.model_fields))" ``` -Change `class: parking` to `class: skyscraper`: +**An interactive session**, best for exploring — you can poke at a model, tab-complete, +and try things: -``` -class "skyscraper" ← Input should be 'agricultural', 'allotment_house', - 'apartments', 'barn', 'beach_hut', ... +```bash +uv run python ``` -Change `height: 21.34` to `height: -5`: - ``` -height -5 ← Input should be greater than 0 +Python 3.10.18 +>>> from overture.schema.buildings import Building +>>> len(Building.model_fields) +26 ``` -Change `num_floors: 4` to `num_floors: 4.7`: +**A script file**, once you're writing more than a couple of lines. Save this as +`explore.py` in the repo root: + +```python +from overture.schema.buildings import Building + +print(f"{len(Building.model_fields)} fields") + +for name, field in Building.model_fields.items(): + if field.is_required(): + print(f" required: {name}") ``` -num_floors 4.7 ← Input should be a valid integer, got a number with a - fractional part + +and run it: + +```bash +uv run python explore.py ``` -Enum membership, numeric bounds, integer-ness — each from the model definition, none of -it written by hand for this file. +``` +26 fields + required: id + required: geometry + required: theme + required: type + required: version +``` -> **What it does not catch:** free-form string fields accept any string. The real -> `building-polygon.yaml` in the repo has a stray trailing comma — -> `dataset: microsoftMLBuildings,` — which YAML reads as part of the value. It parses to -> the string `'microsoftMLBuildings,'` and validates clean, because `dataset` has no -> constraint beyond "is a string." Validation enforces the schema, not your typing. +Every Python block from here on is code to run one of these three ways. An interactive session is the best fit for section 2 — +you're looking things up, not building anything yet. ---- +> You can also activate the environment once (`source .venv/bin/activate`) and then use +> plain `python`. This guide uses `uv run` throughout because it always works, needs no +> setup, and cannot be left half-done. -### 1.5 Do you need PySpark? +### 2.2 Where the models live -Probably not. Answer honestly: +Before you can explore anything, you need to know where it lives. The Python module +path drops the `theme-` prefix: -| I want to… | Need PySpark? | +| Package | Import from | |---|---| -| Validate files, explore models, write Python against them | **No** | -| Generate JSON Schema or markdown docs | **No** | -| Generate an SDK in another language | **No** | -| Validate millions of rows of Parquet, or data in S3 | **Yes** | -| Get a Spark `StructType` for a feature type | **Yes** | -| Run the full test suite (`make check`) | **Yes** | +| `overture-schema-theme-buildings` | `overture.schema.buildings` | +| `overture-schema-theme-transportation` | `overture.schema.transportation` | +| `overture-schema-theme-places` | `overture.schema.places` | +| `overture-schema-theme-divisions` | `overture.schema.divisions` | +| `overture-schema-theme-addresses` | `overture.schema.addresses` | +| `overture-schema-theme-base` | `overture.schema.base` | +| `overture-schema-common` | `overture.schema.common` | +| `overture-schema-system` | `overture.schema.system` | +| `overture-schema-validation` | `overture.schema.validation` | -**If no:** you're done. Go to section 2. +```python +from overture.schema.buildings import Building, BuildingClass +from overture.schema.transportation import Segment, Connector, RoadClass +from overture.schema.places import Place +``` -**If yes:** there's one more step, because the PySpark validation expressions are -*generated code that is not committed to git*. They're in `.gitignore`. `uv sync` alone -cannot produce them. +### 2.3 From the CLI: what types exist? ```bash -make generate-pyspark +uv run overture-schema list-types ``` -Or `make install`, which is just `uv sync --all-packages --all-extras` followed by -`make generate-pyspark`. - -#### What a successful run looks like - -**Nothing.** No progress, no file list, no "done" message — the command returns you -straight to your prompt: - ``` -$ make generate-pyspark -$ +address feature overture overture:theme=addresses +bathymetry feature overture overture:theme=base +building feature overture overture:theme=buildings +building_part feature overture overture:theme=buildings +connector feature overture overture:theme=transportation +division feature overture overture:theme=divisions +division_area feature overture overture:theme=divisions +division_boundary feature overture overture:theme=divisions +infrastructure feature overture overture:theme=base +land feature overture overture:theme=base +land_cover feature overture overture:theme=base +land_use feature overture overture:theme=base +place feature overture overture:theme=places +segment feature overture overture:theme=transportation +water feature overture overture:theme=base ``` -That is success. It looks identical to nothing having happened, which is why the next -subsection exists. - -#### What the command actually does - -`make generate-pyspark` isn't a program — it's a *target* in the repo's `Makefile`, a -named recipe of shell commands. Here it is in full: +Columns are: **type name**, then its **tags**. Group by a tag key: -```make -generate-pyspark: uv-sync clean-pyspark - @uv run overture-codegen generate --format pyspark \ - --output-dir $(PYSPARK_EXPRESSIONS) \ - --test-output-dir $(PYSPARK_GENERATED_TESTS) - @uv run ruff check --fix --quiet $(PYSPARK_EXPRESSIONS) $(PYSPARK_GENERATED_TESTS) - @uv run ruff format --quiet $(PYSPARK_EXPRESSIONS) $(PYSPARK_GENERATED_TESTS) +```bash +uv run overture-schema list-types --group-by overture:theme ``` -Reading that: - -- **`generate-pyspark:`** is the target name — what you typed after `make`. -- **`uv-sync clean-pyspark`** on the same line are *prerequisites*: other targets that - must run first, in that order. -- The tab-indented lines below are the *recipe* — the shell commands, run in order. -- The leading **`@`** tells make not to echo the command before running it. Without it, - make prints each command as it goes. This is why you see no output. -- **`$(PYSPARK_EXPRESSIONS)`** and **`$(PYSPARK_GENERATED_TESTS)`** are variables defined - higher up in the `Makefile`; they expand to the two output directories. - -So typing one command runs five steps: - -| # | Step | What it does | Visible? | -|---|---|---|---| -| 1 | `uv-sync` | `uv sync --all-packages --all-extras` — makes sure dependencies are installed | No: the target captures its output and prints it only on failure | -| 2 | `clean-pyspark` | `rm -rf` both output directories, so generation starts from empty | No: nothing to say | -| 3 | `overture-codegen generate --format pyspark` | **The actual work.** Reads the Pydantic models and writes ~23,000 lines of Python: 15 expression modules and 17 test modules | No: prints nothing on success | -| 4 | `ruff check --fix` | Lints the generated code and auto-fixes what it can, e.g. unused imports | No: `--quiet` | -| 5 | `ruff format` | Reformats the generated code to the project's style | No: `--quiet` | +``` +overture:theme=addresses (1) +→ address feature overture overture:theme=addresses -Steps 4 and 5 exist because generated code is still code that has to pass the repo's own -lint and format checks — `make check` runs `ruff` over everything, generated files -included. +overture:theme=base (6) +... +``` -Every step is silent by design, which is why a successful run prints nothing at all. +Those trailing words — `feature`, `overture`, `overture:theme=addresses` — are **tags**. +Every feature type carries a few, and they are how you select a subset of types without +naming each one. A tag is either a bare word (`feature`), or a key and value joined by +`=` (`overture:theme=buildings`), where the part before the colon says who defined it. -> **On an out-of-date `uv`** you'll instead see the `exclude-newer` banner three times — -> once each for steps 3, 4, and 5, the three visible `uv run` calls — and nothing else. That is still a successful run, but fix -> the `uv` problem before continuing: -> [uv warns about `exclude-newer`](#uv-warns-about-exclude-newer--and-quietly-rewrites-your-lockfile). +Three options select by tag, and all three take a tag name and can be repeated. They are +shared by `list-types`, `validate`, and `json-schema`: -#### Confirm it worked +| Option | Keeps a type when… | +|---|---| +| `--tag` | it has **any** of the tags you listed | +| `--filter` | it has **all** of the tags you listed | +| `--exclude` | drops it if it has any of the tags you listed | -Don't infer it from the output — check the result: +So this lists the buildings types and the places types, and nothing else: ```bash -uv run python -c "from overture.schema.pyspark import model_names; print(model_names())" +uv run overture-schema list-types --tag overture:theme=buildings --tag overture:theme=places ``` -Before — an empty list, no error, which is why this is easy to miss: +Tags are also the mechanism your own feature types use to join the set — see +[How tags work](CONCEPTS.md#how-tags-work). -``` -[] -``` - -After — 30 entries, which is 15 feature types each reachable by two names: +### 2.4 Ask the model itself (Python) -``` -['address', 'bathymetry', 'building', 'building_part', 'connector', 'division', ...] -``` +This is the primary source. `Building` is an ordinary Python class, and Pydantic gives +every model a `model_fields` dict describing each field — its type, whether it's required, +its documentation, and its constraints. Nothing is generated or rendered here; this *is* +the schema. -Or count the files directly: +Start a session and look at what you have: ```bash -find packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated -name '*.py' | wc -l -find packages/overture-schema-pyspark/tests/generated -name '*.py' | wc -l +uv run python ``` -``` - 15 - 17 +```python +>>> from overture.schema.buildings import Building +>>> len(Building.model_fields) +26 +>>> sorted(n for n, f in Building.model_fields.items() if f.is_required()) +['geometry', 'id', 'theme', 'type', 'version'] ``` -#### Why two names per model? - -Because the registry has to stay correct when packages it has never heard of register -their own models. +Twenty-six fields, five of them required. Note the shape of that list: `id` and `geometry` +sit alongside `height` and `num_floors`, all at the same level. **A model is flat.** -The **canonical key is the entry-point string** — `overture.schema.buildings:Building`. -It includes the module path, so it is globally unique: no two packages can collide. +That is worth pinning down now, because the data often isn't. If you've seen an Overture +building as GeoJSON, most of its fields were tucked inside a `properties` object, with +only `id`, `geometry`, and `type` outside it. That outer wrapper is called the +**envelope** — the fixed set of keys GeoJSON puts around every feature, the same for a +building as for a lake. The model has no envelope; it appears only when a model is written +out as GeoJSON. [Section 2.5](#25-ask-the-generated-json-schema) meets it again, and +[the GeoJSON envelope](CONCEPTS.md#the-geojson-envelope) explains where it comes from. -The **short name is a derived alias**. It is just the class name after the colon, -snake-cased: +#### Every field at a glance ```python -from overture.schema.system.discovery.entry_point import entry_point_class_alias +from overture.schema.buildings import Building + +print(Building.__doc__) -entry_point_class_alias("overture.schema.divisions:DivisionArea") # 'division_area' -entry_point_class_alias("overture.schema.places:Place") # 'place' +for name, f in Building.model_fields.items(): + flag = "required" if f.is_required() else "optional" + print(f"{name:24} {flag:9} {f.annotation}") ``` -Short names are *not* guaranteed unique. Anyone can -[register their own feature types](#83-register-your-own-feature-types), and nothing -stops a third party from shipping its own `Place`. So the short name can't be the -identity — it's a convenience, because -`validate_model(df, "overture.schema.buildings:Building")` is miserable to type. +``` +height optional typing.Optional[overture.schema.system.numeric.float64] +is_underground optional bool | None +num_floors optional typing.Optional[overture.schema.system.numeric.int32] +... +id required overture.schema.system.ref.id.Id +geometry required +theme required typing.Literal['buildings'] +type required typing.Literal['building'] +version required overture.schema.common.feature.FeatureVersion +class_ optional overture.schema.buildings.building.BuildingClass | None +``` -**The nice part is how it degrades.** The alias is offered only while it stays -unambiguous. `model_names()` counts aliases and includes only those appearing once, and -the resolver tries an exact key match first, then the alias: +Some of those types look stranger than they are. +`typing.Optional[overture.schema.system.numeric.float64]` is just **"a float, or nothing"** +— `Optional[X]` means the field may be absent, and `float64` is an ordinary Python `float` +that the schema has given a narrower name: ```python -from overture.schema.system.discovery.entry_point import resolve_entry_point_key +from overture.schema.system.numeric import float64 -registry = {"overture.schema.places:Place": ..., "acme.parks:Place": ...} +float64.__supertype__ # +``` -resolve_entry_point_key("place", registry) -# ValueError: Entry-point alias 'place' is ambiguous. -# Specify one of: acme.parks:Place, overture.schema.places:Place +The schema declares `float64`, `int32`, `uint8` and friends so a field can say how wide it +is on the wire — which Parquet column type it becomes, what range it accepts — while +staying a plain number in Python. Same story for `Id` and `FeatureVersion`: named types +wrapping `str` and `int`. Only `Literal['building']` is different: it means the field must +be exactly that one string. -resolve_entry_point_key( - "acme.parks:Place", registry -) # 'acme.parks:Place' — always works -``` +#### One field in detail + +Each entry carries the documentation and constraints from the model declaration: -Install a package that collides and `place` simply stops being accepted, with an error -naming both candidates — rather than silently validating against the wrong model. The -fully-qualified key never stops working. +```python +f = Building.model_fields["height"] +print(f.description) # 'Height of the building or part in meters.\n\n...' +print(f.metadata) # [Gt(gt=0)] +print(f.alias) # None +print(Building.model_fields["class_"].alias) # 'class' +``` -Two functions expose the two views: +Note `class_` and its alias `class`. `class` is a Python keyword, so the field is named +`class_` on the model and `class` in the data — a mismatch that bites when serializing. +See [Gotchas](TROUBLESHOOTING.md#model-gotchas). -| Function | Returns | Use when | -|---|---|---| -| `model_keys()` | the 15 canonical entry-point keys | you want the authoritative list | -| `model_names()` | all 30 accepted names | you want everything `validate_model` will take | +The next section reads exactly this information back out of the generated JSON Schema, +where it goes by different names: `f.description` becomes `description`, the `[Gt(gt=0)]` +in `f.metadata` becomes `exclusiveMinimum: 0`, and `f.is_required()` becomes membership in +a list called `required`. Same facts, second rendering. -> **Skipping this step does not produce an error message.** It produces an empty -> registry. If `validate_model(df, "building")` raises a `KeyError`, or `model_names()` -> is empty, this is why. +### 2.5 Ask the generated JSON Schema -#### Why is generated code in `.gitignore`? +**Why bother, when 2.4 already answered these questions in Python?** Because the JSON +Schema is the version other programs can read. It's a plain JSON document, so a validator +written in Go, a code generator that emits TypeScript, or a form builder that has never +heard of Pydantic can all consume it. Reach for it when you're feeding a tool rather than +answering a question — and when you want to see the rules in the *GeoJSON* shape, since +that's what it describes. -Not because it's optional, and not because it's for a subset of users. **It's build -output.** The commit that introduced the package says so directly: +Dump it for one type. Save it once rather than re-running the command for every question: -> The generated trees under `expressions/generated/` and `tests/generated/` are -> regenerable output of `make generate-pyspark` and are not tracked in git; `make check` -> and `make test-all` regenerate before running. +```bash +uv run overture-schema json-schema --type building > building.schema.json +``` -Three reasons that's the right call: +That file is a few thousand lines, so the queries below use **`jq`** — a small +command-line tool for querying JSON. You give it a path like `.properties.height` and it +prints what's there. Install it with `brew install jq` or `apt install jq`. Anything here +you'd rather do in Python, you can: `json.load()` and the same paths as dictionary keys. -**One source of truth.** The Pydantic models define the schema; these expressions are a -derivative of them. Committing the derivative creates a second copy that can silently -drift — change a constraint, forget to regenerate, and the committed expressions keep -enforcing the old rule. Deleting them from git makes that failure impossible. +#### Finding a field -**Scale.** A full generation is **32 files and roughly 23,000 lines**: +Overture's fields are three levels down, and the reason is the GeoJSON envelope: the +document describes a GeoJSON feature, so `id` and `geometry` sit at the top and everything +else is inside `properties`. +```bash +jq '.properties.properties.properties | keys' building.schema.json ``` -15 expression modules (one per feature type) -17 test modules (conformance tests, split per union arm) +```json +["class","facade_color","facade_material","has_parts","height","is_underground", + "level","min_floor","min_height","names","num_floors","num_floors_underground", + "roof_color","roof_direction","roof_height","roof_material","roof_orientation", + "roof_shape","sources","subtype","theme","type","version"] ``` -Every schema change would produce a mechanical diff of that size, burying the actual -change and guaranteeing merge conflicts. +Three `properties` in a row, meaning something different each time: -**The build regenerates regardless.** `make check` and `make test-all` both depend on -`generate-pyspark`, which begins with `clean-pyspark` (`rm -rf`). The tree is rebuilt -from the current models every time, so a committed copy would never be read. +| Path | Means | +|---|---| +| `.properties` | "the fields of this document" — a JSON Schema keyword | +| `.properties.properties` | the GeoJSON field actually *named* `properties` | +| `.properties.properties.properties` | "the fields inside that one" — the keyword again | -It's the same reasoning that keeps `dist/`, `*.o`, and `node_modules/` out of git. +`id`, `geometry`, and `bbox` are missing from that list because they're on the envelope, +at `.properties.id` and so on — exactly where they sit in the data. -#### "Gitignored" does not mean "not shipped" +**Don't count levels — let `jq` find the path for you.** This works for any field: -This is the part worth being clear about, since it sounds like these files are somehow -optional for users. **They are not.** Published wheels contain them. +```bash +jq -c 'paths | select(.[-1]=="height")' building.schema.json +``` +```json +["properties","properties","properties","height"] +``` -`.github/workflows/publish-python-packages.yaml` runs `make generate-pyspark` before -`uv build`, then refuses to publish a wheel that lacks them: +Worth knowing because a wrong path returns `null` rather than an error — `jq` treats "no +such key" as an answer, so a `null` usually means you stopped a level too high, not that +the field is missing. -```yaml -- name: Generate PySpark expressions before build - if: matrix.package == 'overture-schema-pyspark' - run: make generate-pyspark -``` +#### Looking up one field's rules ```bash -if [ "$PACKAGE" = "overture-schema-pyspark" ] && \ - ! unzip -l "$wheel" | grep -q 'expressions/generated/.*\.py'; then - echo " Wheel [$wheel] has no generated expressions -- codegen did not run. Aborting!" - exit 1 -fi +jq '.properties.properties.properties.height' building.schema.json +``` +```json +{ + "description": "Height of the building or part in meters.\n\nThis is the distance from the lowest point to the highest point.", + "exclusiveMinimum": 0, + "title": "Height", + "type": "number" +} ``` -So the only people who ever run `make generate-pyspark` are people working **from a git -clone** — because a clone is the one place these files don't already exist. Install from -a package index and they arrive with the package, like any other module. +**That object is not a value of `height` — in the data, `height` is just a number like +`21.34`.** It's the *rules* for `height`: must be a number, must be greater than zero. +Every field gets an object like this even when the field is a bare number, because +there's nowhere else to hang a description and a constraint. ---- +Nobody wrote that JSON. Each line is rendered from the field's Python declaration in +`overture/schema/buildings/_common.py`: + +| Schema keyword | Comes from | +|---|---| +| `"type": "number"` | the `float64` annotation | +| `"exclusiveMinimum": 0` | `gt=0` — greater than, not greater-or-equal | +| `"description"` | the `description=` argument | +| `"title": "Height"` | generated by Pydantic from the field name | -### 1.6 When something goes wrong +That is why the JSON Schema, the validation errors, the PySpark checks, and the generated +docs can't drift apart: they're all renderings of the same declaration. +[Section 6](#6-converting-the-schema-to-other-formats) produces the rest of them. -**This section is a reference, not a checklist.** Nothing here is setup you need to -perform. Each entry starts with a symptom — read the one matching an error you actually -saw, and skip the rest. If section 1.4 gave you clean output and -[Confirm it worked](#confirm-it-worked) checked out, you can skip the whole section and -go on to section 2. +#### Listing the valid values for a field -#### `FileNotFoundError: ... /apache-spark/3.5.3/libexec/./bin/spark-submit` +Enums and shared structures sit at the top level under `$defs`, not inline: +```bash +jq -r '.["$defs"] | keys[]' building.schema.json ``` -FileNotFoundError: [Errno 2] No such file or directory: -'/opt/homebrew/Cellar/apache-spark/3.5.3/libexec/./bin/spark-submit' +``` +BuildingClass BuildingSubtype FacadeMaterial NameRule NameVariant Names +PerspectiveMode Perspectives RoofMaterial RoofOrientation RoofShape Side SourceItem ``` -**This has nothing to do with the generation step in "Do you need PySpark?".** It's a stale `SPARK_HOME` -environment variable, and it will happen whether or not you ran `make generate-pyspark`. - -What's going on: you have a `SPARK_HOME` exported in your shell profile pointing at a -**version-specific Homebrew path**. Homebrew has since upgraded Spark, so that exact -directory no longer exists: +So the full list of values a field accepts, without reading any Python: ```bash -echo $SPARK_HOME -ls /opt/homebrew/Cellar/apache-spark/ +jq -r '.["$defs"].BuildingClass.enum[]' building.schema.json | head ``` - ``` -/opt/homebrew/Cellar/apache-spark/3.5.3/libexec -4.0.0 +agricultural +allotment_house +apartments +barn +beach_hut +boathouse +bridge_structure +bungalow ``` -The variable names `3.5.3`; the only version present is `4.0.0`. It points at nothing. - -Meanwhile the `pyspark` in your venv (4.2.0) **ships its own copy of Spark** and doesn't -need the Homebrew one at all. But when `SPARK_HOME` is set, PySpark obeys it and looks -for `spark-submit` at that dead path. +#### Which fields are required -**Confirm that's your problem:** +The envelope splits this across **two** lists, one per level: ```bash -env -u SPARK_HOME uv run python -c " -from pyspark.sql import SparkSession -s = SparkSession.builder.master('local[1]').getOrCreate() -print('SUCCESS — spark', s.version) -s.stop()" -``` - +jq '.required' building.schema.json +jq '.properties.properties.required' building.schema.json ``` -SUCCESS — spark 4.2.0 +```json +["type","id","geometry","properties"] +["theme","type","version"] ``` -**Fix it permanently.** The variable is set in *two* files — fixing only one won't help, -because `.zprofile` runs for login shells and `.zshrc` for interactive ones: +Read together they are the same five fields 2.4 gave you — `id` and `geometry` on the +envelope, `theme`, `type`, and `version` inside `properties`. Reading only the second list +and calling it "the required fields of a building" undercounts by exactly the fields the +envelope owns. + +`required` is always relative to the object it sits in; nested structures like `Names` and +`SourceItem` carry their own. **For this particular question the model is the easier +place to ask**, since it has no envelope to split the answer across: + +```python +from overture.schema.buildings import Building +print(sorted(n for n, f in Building.model_fields.items() if f.is_required())) ``` -~/.zshrc:8 export SPARK_HOME=/opt/homebrew/Cellar/apache-spark/3.5.3/libexec -~/.zshrc:9 export PATH="$SPARK_HOME/bin/:$PATH" -~/.zprofile:5 export SPARK_HOME=/opt/homebrew/Cellar/apache-spark/3.5.3/libexec -~/.zprofile:6 export PATH="$SPARK_HOME/bin/:$PATH" +``` +['geometry', 'id', 'theme', 'type', 'version'] ``` -Pick one: +Nobody maintains that list by hand — it falls out of whether a field has a default. For +how that works, and who decides, see +[What makes a field required](CONCEPTS.md#what-makes-a-field-required). -- **Simplest — delete all four lines.** If you only use Spark through Python projects - like this one, you don't need `SPARK_HOME` at all; each venv's `pyspark` brings its - own. -- **Keep Homebrew Spark for other work — stop hardcoding the version.** Replace the two - `SPARK_HOME` lines with the version-independent symlink Homebrew maintains: +#### What a subschema doesn't tell you - ```bash - export SPARK_HOME=/opt/homebrew/opt/apache-spark/libexec - ``` +The object you get back for `height` is the complete machine-checkable contract for +`height` — but only for `height`, and only in isolation. Three things it does *not* say: - This survives upgrades. But note it points at Spark **4.0.0** while this project's - `pyspark` is **4.2.0** — mismatched versions cause their own confusing failures, so - prefer the first option while working in this repo. +- **Whether the field may be omitted.** That lives in a sibling `required` array. A + subschema describes the value *if present*. +- **That the number is in meters.** "in meters" is in `description` — prose for humans. + Nothing rejects a value recorded in feet. +- **Anything about other fields.** The schema *can* express cross-field rules — that's + what `@require_any_of` and `@forbid_if` in `overture-schema-system` are for — but no + such rule ties `height` to `min_height`, so a building with a floor above its roof + validates clean: -Then open a new terminal, or `exec zsh`, and re-run the check above. +```python +import json, yaml +from overture.schema.buildings import Building -> **Per-shell workaround** if you don't want to touch your profile right now: -> `unset SPARK_HOME` in the terminal you're working in. It lasts until you close it. +d = yaml.safe_load(open("examples/buildings/building-polygon.yaml")) +d["properties"]["height"] = 5 +d["properties"]["min_height"] = 100 # a floor above the roof +print("accepted:", Building.model_validate_json(json.dumps(d)).height) +``` -#### `model_names()` returns `[]`, or `KeyError` on a feature type +``` +accepted: 5.0 +``` -You skipped [Do you need PySpark?](#15-do-you-need-pyspark). Run `make generate-pyspark`. +Validation enforces the schema, not correctness. -#### `ModuleNotFoundError: No module named 'overture'` +#### jq or Python? -You started Python without `uv run`, so you're in your system or `pyenv` interpreter -rather than the project's `.venv`. Use `uv run python` instead of `python`. See -[Running Python against the models](#21-running-python-against-the-models). +Use `jq` when you want the schema exactly as it ships, or when you're feeding it to +another tool — [generating an SDK](#81-generate-an-sdk-from-json-schema-any-language), for +instance. Use Python when you want to understand the model: `Building.model_fields` gives +you the same facts flat, with no envelope to walk, and a wrong field name raises instead +of quietly returning `null`. -#### `command not found: overture-schema` +### 2.6 From Python: enumerate everything that's installed -You're missing the `uv run` prefix, or you're not in the repo root. Every command in this -guide is `uv run overture-schema …`, run from the directory containing `pyproject.toml`. +```python +from overture.schema.system.discovery import discover_models -#### uv warns about `exclude-newer` — and quietly rewrites your lockfile +for key, model in sorted(discover_models().items(), key=lambda kv: kv[0].name): + print(f"{key.name:20} {key.entry_point:45} {sorted(key.tags)}") +``` ``` -warning: Failed to parse `pyproject.toml` during settings discovery: - TOML parse error at line 10, column 17 - | - 10 | exclude-newer = "1 week" - | ^^^^^^^^ - failed to parse year in date "1 week": failed to parse "1 we" as year ... +address overture.schema.addresses:Address ['feature', 'overture:theme=addresses'] +building overture.schema.buildings:Building ['feature', 'overture:theme=buildings'] +segment overture.schema.transportation:Segment ['feature', 'overture:theme=transportation'] +... ``` -> **Do you actually have this problem?** Only if that banner appears when you run a -> command. If your commands print their output cleanly, skip this entire subsection — -> there is nothing to fix, and the four steps below are not maintenance you need to -> perform. Confirm in one line: -> -> ```bash -> uv run overture-schema --version -> ``` -> -> A single line of output means you're fine. A wall of warning text above it means read on. - -**If you do see it: this is not cosmetic. Fix it before you do anything else.** +A `ModelKey` carries `.name`, `.entry_point` (`"module:Class"`), and `.tags` +(a `frozenset[str]`). Filter the same way the CLI does: -The root `pyproject.toml` writes `exclude-newer` as a relative duration (`"1 week"`), -which caps how new a package `uv` will consider during dependency resolution. Only -reasonably recent `uv` versions parse that form. +```python +from overture.schema.system.discovery import TagSelector, discover_models, filter_models -An older `uv` fails to read the whole `[tool.uv]` block, says so, **and carries on -without the cap** — then, because its resolution no longer matches the committed -lockfile, rewrites `uv.lock` in place. You end up with a modified tracked file you never -asked to change: +models = discover_models() -```bash -git status --short uv.lock -git diff --stat uv.lock +buildings = filter_models( + models, TagSelector(include_any=("overture:theme=buildings",)) +) ``` -On an affected machine: - -``` - M uv.lock - uv.lock | 807 +++++++++++++++++++------------------- - 1 file changed, 464 insertions(+), 343 deletions(-) -``` +`TagSelector` takes `include_any` (OR scope), `require_all` (AND narrowing), and +`exclude_any` (OR-NOT). An empty selector returns the input unchanged. -> **Both commands printing nothing is the healthy result.** `git status --short` and -> `git diff --stat` say nothing about a file that hasn't changed. Empty output here means -> your lockfile is untouched and there is nothing to fix. If you'd rather have an explicit -> answer than read silence: -> -> ```bash -> git diff --quiet uv.lock && echo "uv.lock: unmodified" || echo "uv.lock: MODIFIED" -> ``` +### 2.7 Reading enum member documentation -It drops the `[options]` block that records the resolution settings, and pulls in -dependency versions past the cutoff the repo intended. Nothing breaks immediately — the -install works fine — but you're now building against a different dependency set than the -project pinned, and `git status` is dirty. +Enum members carry per-value docstrings, but `member.__doc__` falls back to the *class* +docstring when a member has none — so reading `__doc__` directly gives you misleading +results: -**Step 1 — check your version:** +```python +from overture.schema.buildings import RoofShape -```bash -uv --version -brew outdated uv +[(m.value, m.__doc__.strip()[:30]) for m in list(RoofShape)[:2]] +# [('dome', 'The shape of the roof.'), ('flat', 'The shape of the roof.')] +# ^ that's the class docstring repeated, not per-member documentation ``` -**Step 2 — upgrade:** +Use the codegen extractor, which does the fallback detection for you: + +```python +from overture.schema.codegen.extraction.enum_extraction import extract_enum +from overture.schema.common.scoping.travel_mode import TravelMode -```bash -brew upgrade uv +spec = extract_enum(TravelMode) +for m in spec.members[:4]: + print(f"{m.value:14} {m.description or '—'}") ``` -**Step 3 — restore the lockfile if the old `uv` already rewrote it:** - -```bash -git checkout uv.lock ``` +vehicle — +motor_vehicle Includes car, truck and motorcycle +car — +truck — +``` + +`description` is `None` when the member has no documentation of its own. + +### 2.8 Generate browsable reference docs -**Step 4 — confirm:** +For sustained exploration, generate the full markdown reference and read it in your +editor: ```bash -uv sync --all-packages --locked +uv run overture-codegen generate --format markdown --output-dir ./schema-docs ``` +You get one page per feature type, per enum, and per named type, with field tables, +prose constraint descriptions, cross-page links, and validated examples: + ``` -Resolved 64 packages in 12ms -Audited 60 packages in 0.81ms +schema-docs/buildings/building.md +schema-docs/buildings/building_part.md +schema-docs/buildings/types/building_class.md +schema-docs/buildings/types/roof_shape.md +schema-docs/common/names.md +schema-docs/common/sources.md +schema-docs/system/numeric.md +... ``` -`--locked` fails outright if the lockfile isn't authoritative, so a clean pass means your -`uv`, the lockfile, and your `.venv` all agree. Run `git status --short uv.lock` once more -too — it should print nothing, which means the file is unmodified. +Scope it to one theme with the same tag options: -> **If you see `error: The lockfile at uv.lock needs to be updated, but --locked was -> provided`,** you're at step 3, not step 4. A previous `uv sync` under the old `uv` -> already modified the lock. `git checkout uv.lock` and re-run. +```bash +uv run overture-codegen generate --format markdown \ + --tag overture:theme=buildings --output-dir ./schema-docs +``` -Once you're on a current `uv`, ordinary use leaves the lockfile alone — including the -`uv sync --all-packages --all-extras` that `make install`, `make check`, and -`make generate-pyspark` all run internally. +(Supplementary types from `common/` and `system/` are pulled in regardless, since the +feature pages link to them.) --- -## 2. Exploring the models +## 3. Writing code against the models -You can't write code against a model you haven't looked at. This section is about finding -out what feature types exist, what fields they carry, and what values those fields -accept — before writing a line of code against them. +Section 2 was about finding out what exists. This section is about using it: loading +real data into models, reading and changing it, and writing it back out. -**Two things can answer your questions, and it's worth keeping them straight:** +### 3.1 A complete example, start to finish -| | What it is | Ask it with | -|---|---|---| -| **The model** | The Pydantic classes themselves — the source of truth, and what actually validates your data | Python: `Building.model_fields` | -| **The generated JSON Schema** | An artifact *rendered from* the model, in the GeoJSON shape | `overture-schema json-schema` + `jq` | +Before any of the details, here is the whole job in one piece: load a feature, read it, +change it, write it back out, and confirm it still validates. Everything else in this +section is an explanation of a line in this snippet. -They agree, because one is generated from the other. The model is flatter and easier to -interrogate; the JSON Schema is the published contract and the thing other tools consume. -This section asks the model first, then the schema. +```python +import json, yaml +from overture.schema.buildings import Building -### 2.1 Running Python against the models +# 1. LOAD — the file is GeoJSON, so go through JSON mode +doc = yaml.safe_load(open("examples/buildings/building-polygon.yaml")) +b = Building.model_validate_json(json.dumps(doc)) +print("1. loaded :", b.id) -Everything up to now has been shell commands. From here the guide switches to Python, so -first: how do you actually run it? +# 2. READ — plain Python attributes; enums come back as enum members +print("2. read :", b.height, "m,", b.num_floors, "floors, class", b.class_.value) -**The models are installed in the project's `.venv`, not in whatever `python` your shell -finds.** Starting Python the usual way fails: +# 3. MODIFY — ordinary assignment +b.height = 25.0 +b.num_floors = 5 +print("3. changed:", b.height, "m,", b.num_floors, "floors") -```bash -python -``` -``` ->>> from overture.schema.buildings import Building -Traceback (most recent call last): - File "", line 1, in -ModuleNotFoundError: No module named 'overture' -``` - -`ModuleNotFoundError: No module named 'overture'` always means this: right code, wrong -interpreter. Compare the two: - -```bash -python -c "import sys; print(sys.executable)" -uv run python -c "import sys; print(sys.executable)" -``` - -``` -/Users/you/.pyenv/versions/3.10.15/bin/python -/path/to/schema/.venv/bin/python3 -``` - -The first is your system or `pyenv` Python, which has never heard of these packages. The -second is the project's environment, where `uv sync` installed them. Nothing is broken — -they're simply two different Pythons. - -Prefix with `uv run` and it works — that runs Python inside the project's environment: - -```bash -uv run python -c "from overture.schema.buildings import Building; print(Building.__name__)" -``` -``` -Building -``` - -Three ways to run the Python in this guide, all from the repo root: - -**A one-liner**, for a quick look: - -```bash -uv run python -c "from overture.schema.buildings import Building; print(len(Building.model_fields))" -``` - -**An interactive session**, best for exploring — you can poke at a model, tab-complete, -and try things: - -```bash -uv run python -``` - -``` -Python 3.10.18 ->>> from overture.schema.buildings import Building ->>> len(Building.model_fields) -26 -``` - -Press Ctrl-D to exit. - -**A script file**, once you're writing more than a couple of lines: - -```bash -uv run python explore.py -``` - -Unless a snippet is explicitly marked as shell, every Python block from here on is code -to run one of these three ways. An interactive session is the best fit for section 2 — -you're looking things up, not building anything yet. - -> You can also activate the environment once (`source .venv/bin/activate`) and then use -> plain `python`. This guide uses `uv run` throughout because it always works, needs no -> setup, and cannot be left half-done. - -### 2.2 Where the models live - -Before you can explore anything, you need to know where it lives. The Python module -path drops the `theme-` prefix: - -| Package | Import from | -|---|---| -| `overture-schema-theme-buildings` | `overture.schema.buildings` | -| `overture-schema-theme-transportation` | `overture.schema.transportation` | -| `overture-schema-theme-places` | `overture.schema.places` | -| `overture-schema-theme-divisions` | `overture.schema.divisions` | -| `overture-schema-theme-addresses` | `overture.schema.addresses` | -| `overture-schema-theme-base` | `overture.schema.base` | -| `overture-schema-common` | `overture.schema.common` | -| `overture-schema-system` | `overture.schema.system` | -| `overture-schema-validation` | `overture.schema.validation` | - -```python -from overture.schema.buildings import Building, BuildingClass -from overture.schema.transportation import Segment, Connector, RoadClass -from overture.schema.places import Place -``` - -### 2.3 From the CLI: what types exist? - -```bash -uv run overture-schema list-types -``` - -``` -address feature overture overture:theme=addresses -bathymetry feature overture overture:theme=base -building feature overture overture:theme=buildings -building_part feature overture overture:theme=buildings -connector feature overture overture:theme=transportation -division feature overture overture:theme=divisions -division_area feature overture overture:theme=divisions -division_boundary feature overture overture:theme=divisions -infrastructure feature overture overture:theme=base -land feature overture overture:theme=base -land_cover feature overture overture:theme=base -land_use feature overture overture:theme=base -place feature overture overture:theme=places -segment feature overture overture:theme=transportation -water feature overture overture:theme=base -``` - -Columns are: **type name**, then its **tags**. Group by a tag key: - -```bash -uv run overture-schema list-types --group-by overture:theme -``` - -``` -overture:theme=addresses (1) -→ address feature overture overture:theme=addresses - -overture:theme=base (6) -... -``` - -Filter with the tag options, which are shared by `list-types`, `validate`, and -`json-schema`: - -| Option | Semantics | -|---|---| -| `--tag T` | OR — defines scope. Repeatable. | -| `--filter T` | AND — every listed tag must be present. Repeatable. | -| `--exclude T` | OR-NOT — any match drops the type. Repeatable. | - -Tag format is `[namespace:]predicate[=value]`: - -- plain — `feature` -- namespaced — `system:extension` -- key/value — `overture:theme=buildings` - -```bash -uv run overture-schema list-types --tag overture:theme=buildings --tag overture:theme=places -``` - -### 2.4 Ask the model itself (Python) - -This is the primary source. `Building` is an ordinary Python class, and Pydantic gives -every model a `model_fields` dict describing each field — its type, whether it's required, -its documentation, and its constraints. Nothing is generated or rendered here; this *is* -the schema. - -Start a session and look at what you have: - -```bash -uv run python -``` - -```python ->>> from overture.schema.buildings import Building ->>> len(Building.model_fields) -26 ->>> sorted(n for n, f in Building.model_fields.items() if f.is_required()) -['geometry', 'id', 'theme', 'type', 'version'] -``` - -Twenty-six fields, five of them required. Note the shape of that list: `id` and `geometry` -sit alongside the rest, with **no nesting and no envelope**. A model is flat. (If you've -seen Overture data as GeoJSON with things tucked under `properties`, that's a -serialization format, not the model — [2.5](#25-ask-the-generated-json-schema-cli--jq) -covers why they differ.) - -#### Every field at a glance - -```python -from overture.schema.buildings import Building - -print(Building.__doc__) - -for name, f in Building.model_fields.items(): - flag = "required" if f.is_required() else "optional" - print(f"{name:24} {flag:9} {f.annotation}") -``` - -``` -height optional typing.Optional[overture.schema.system.numeric.float64] -is_underground optional bool | None -num_floors optional typing.Optional[overture.schema.system.numeric.int32] -... -id required overture.schema.system.ref.id.Id -geometry required -theme required typing.Literal['buildings'] -type required typing.Literal['building'] -version required overture.schema.common.feature.FeatureVersion -class_ optional overture.schema.buildings.building.BuildingClass | None -``` - -#### One field in detail - -Each entry carries the documentation and constraints from the model declaration: - -```python -f = Building.model_fields["height"] -print(f.description) # 'Height of the building or part in meters.\n\n...' -print(f.metadata) # [Gt(gt=0)] -print(f.alias) # None -print(Building.model_fields["class_"].alias) # 'class' -``` - -Note `class_` and its alias `class`. `class` is a Python keyword, so the field is named -`class_` on the model and `class` in the data — a mismatch that bites when serializing. -See [Gotchas](#12-gotchas). - -This is the same information [2.5](#25-ask-the-generated-json-schema-cli--jq) reads out -of the JSON Schema, minus the envelope — `f.description` is the `description` keyword, -`f.metadata` of `[Gt(gt=0)]` is `exclusiveMinimum: 0`, and `f.is_required()` is -membership in a `required` array. - -### 2.5 Ask the generated JSON Schema (CLI + jq) - -Dump the JSON Schema for one type. Save it once rather than re-running the command for -every question: - -```bash -uv run overture-schema json-schema --type building > building.schema.json -``` - -#### Two kinds of key - -Before walking the nesting, one distinction that makes the rest obvious. Everything in a -JSON Schema document is a JSON object, so `jq keys` will happily list any level — but -what it lists alternates between two completely different kinds of name. - -**Schema keywords** are vocabulary defined by the JSON Schema specification. They are -instructions to a validator, and the set is fixed — you could not invent a new one. Their -values describe the document: - -```bash -jq 'keys' building.schema.json -``` -```json -["$defs","additionalProperties","description","properties","required","title","type"] -``` - -``` -.type = "object" -.title = "building" -.description = "Buildings are man-made structures with roofs..." -.required = ["type","id","geometry","properties"] -.additionalProperties = false -``` - -Read that as prose: *this is an object, called `building`, described like so, these four -fields are mandatory, and no others are allowed.* - -**Field names** are names that appear in actual data. They are not vocabulary — they come -from Overture and GeoJSON, and every one of them maps to a *subschema*: another little -JSON Schema object describing that one field. - -```bash -jq '.properties | keys' building.schema.json -``` -```json -["bbox","geometry","id","properties","type"] -``` - -``` -.properties.type = {"const":"Feature","type":"string"} -.properties.id = {"description":"A feature ID...", ...} -.properties.geometry = {"description":"The building's footprint...", ...} -``` - -Those values aren't descriptions of the document — they're rules for one field each. -`.properties.type` says: *the data field named `type` must be the string `Feature`.* - -So "keys" is technically accurate for both lists and useless for telling them apart. The -first list is **keywords**; the second is **field names**. The keyword `properties` is the -gate between them: everything under it is data field names, and each of those opens into -a subschema made of keywords again. - -You can see both roles inside a single value: - -```json -{"const": "Feature", "type": "string"} -``` - -The key `type` that got you here was a *field name*. The `type` inside the value is a -*keyword* meaning "JSON string." Same word, opposite roles, one level apart. - -#### Mind the nesting - -With that distinction, the path to the Overture fields reads cleanly. `.properties` is a -keyword, so its contents are field names — and they're GeoJSON's, the same envelope from -[Three keys named `type`](#three-keys-named-type): - -```bash -jq '.properties | keys' building.schema.json -``` -```json -["bbox","geometry","id","properties","type"] -``` - -One of those field names is itself `properties` — the GeoJSON member holding the Overture -fields. Descend into it, then through *its* `properties` keyword: - -```bash -jq '.properties.properties.properties | keys' building.schema.json -``` -```json -["class","facade_color","facade_material","has_parts","height","is_underground", - "level","min_floor","min_height","names","num_floors","num_floors_underground", - "roof_color","roof_direction","roof_height","roof_material","roof_orientation", - "roof_shape","sources","subtype","theme","type","version"] -``` - -Three `properties` in a row, alternating role each time: - -| Path segment | Kind | Meaning | -|---|---|---| -| `.properties` | keyword | "the fields of the GeoJSON object" | -| `.properties.properties` | field name | the GeoJSON field *named* `properties` | -| `.properties.properties.properties` | keyword | "the fields inside that one" | - -Note `id`, `geometry`, and `bbox` are absent from the final list. They live on the -envelope, at `.properties.id` and so on — exactly as they do in the data. - -#### That JSON object is not the field's value - -A reasonable reading of this is "height is an object": - -```json -{ - "description": "Height of the building or part in meters.\n\nThis is the distance from the lowest point to the highest point.", - "exclusiveMinimum": 0, - "title": "Height", - "type": "number" -} -``` - -It isn't. **In the data, `height` is a plain number.** What you're looking at is the -*subschema* — the rules for `height`, not a value of `height`. And `"type": "number"` is -the keyword saying so. - -The actual data looks like this: - -```json -{"height": 21.34} -``` - -Every field's subschema is a JSON object, no matter how simple the field is, because -that's the only place to hang a description and a constraint. You can't attach -"must be greater than zero" to a bare `"number"`. - -Each keyword traces back to one thing in the Python model — -`packages/overture-schema-theme-buildings/src/overture/schema/buildings/_common.py`: - -```python -height: Annotated[ - float64 | None, - Field( - gt=0, - description=textwrap.dedent(""" - Height of the building or part in meters. - - This is the distance from the lowest point to the highest point. - """).strip(), - ), -] = None -``` - -| Schema keyword | Comes from | -|---|---| -| `"type": "number"` | the `float64` annotation | -| `"exclusiveMinimum": 0` | `gt=0` — greater than, not greater-or-equal | -| `"description"` | the `description=` argument | -| `"title": "Height"` | generated by Pydantic from the field name | -| *absent from `required`* | the `= None` default | - -Nobody hand-wrote that JSON. It's a rendering of the Python declaration, which is why the -JSON Schema, the validation errors, the PySpark checks, and the generated documentation -can't drift from each other — [section 6](#6-converting-the-schema-to-other-formats) is -all the other renderings of the same source. - -#### Why a wrong path gives you `null`, not an error - -Ask `jq` for a key that isn't there and it returns `null`. That is an answer, not a -failure — `null` means "no such key at this path": - -```bash -jq '.properties.height' building.schema.json -jq '.properties.banana' building.schema.json -``` -``` -null -null -``` - -`height` isn't missing from the schema; it just isn't on the *envelope*, which only has -`bbox`, `geometry`, `id`, `properties`, and `type`. A `null` here almost always means you -stopped one level too high. - -Two ways to make that louder. `jq -e` sets the exit status — `1` for `null`, `0` for a -real result, useful in scripts: - -```bash -jq -e '.properties.height' building.schema.json > /dev/null; echo $? -jq -e '.properties.properties.properties.height' building.schema.json > /dev/null; echo $? -``` -``` -1 -0 -``` - -Or stop guessing at the nesting and let `jq` find the field for you: - -```bash -jq -c 'paths | select(.[-1]=="height")' building.schema.json -``` -```json -["properties","properties","properties","height"] -``` - -That prints the exact path to any field, which you can then read directly. Handy whenever -a lookup comes back `null` and you're not sure how deep the thing actually lives. - -#### Asking useful questions - -Which fields are mandatory — and here the envelope bites. There is no single `required` -list. There are **two**, one per level: - -```bash -jq '.required' building.schema.json -jq '.properties.properties.required' building.schema.json -``` -```json -["type","id","geometry","properties"] -["theme","type","version"] -``` - -Read them together: - -| Required | Where | What it is | -|---|---|---| -| `type` | envelope | GeoJSON's own `"type": "Feature"` | -| `id` | envelope | the feature's ID | -| `geometry` | envelope | the shape | -| `properties` | envelope | the container itself must be present | -| `theme` | in `properties` | `buildings` | -| `type` | in `properties` | `building` | -| `version` | in `properties` | the feature version | - -So **`id` and `geometry` are absolutely required** — they're just not in the array you'd -find by looking only under `properties`, because in GeoJSON they don't live under -`properties`. Asking `.properties.properties.required` and reading it as "the required -fields of a building" undercounts by exactly the fields the envelope owns. - -Prove it by deleting them: - -```bash -uv run python <<'PY' -import json, yaml -from overture.schema.buildings import Building - -for drop in ["geometry", "id"]: - d = yaml.safe_load(open("examples/buildings/building-polygon.yaml")) - del d[drop] - try: - Building.model_validate_json(json.dumps(d)) - print(f"dropping {drop}: ACCEPTED") - except Exception as e: - print(f"dropping {drop}:", str(e).splitlines()[2].strip()[:30]) -PY -``` -``` -dropping geometry: Field required -dropping id: Field required -``` - -**The model is the easier place to ask this question**, because it has no envelope — one -flat answer, all five: - -```bash -uv run python -c " -from overture.schema.buildings import Building -print(sorted(n for n, f in Building.model_fields.items() if f.is_required()))" -``` -``` -['geometry', 'id', 'theme', 'type', 'version'] -``` - -There are other `required` arrays deeper in the schema too — every nested structure has -its own. `Names`, `SourceItem`, and the two `geometry` variants each carry one: - -```bash -jq -c 'paths | select(.[-1]=="required")' building.schema.json -``` -```json -["$defs","NameRule","required"] -["$defs","Names","required"] -["$defs","Perspectives","required"] -["$defs","SourceItem","required"] -["properties","geometry","oneOf",0,"required"] -["properties","geometry","oneOf",1,"required"] -["properties","properties","not","required"] -["properties","properties","required"] -["required"] -``` - -`required` is always relative to the object it sits in — never a global list for the -feature. - -#### Who decides what's required? - -Nobody maintains that list. **It's derived** — in Pydantic, a field with no default is -required, and a field with a default is optional: - -```bash -uv run python <<'PY' -from overture.schema.buildings import Building -from pydantic_core import PydanticUndefined - -for n in ["version", "theme", "height", "num_floors"]: - f = Building.model_fields[n] - d = "no default" if f.default is PydanticUndefined else f"default={f.default!r}" - print(f"{n:12} {d:16} -> {'REQUIRED' if f.is_required() else 'optional'}") -PY -``` -``` -version no default -> REQUIRED -theme no default -> REQUIRED -height default=None -> optional -num_floors default=None -> optional -``` - -So "who decided" is answered by finding where the field is declared. For a building, five -fields are required, and four of them come from the shared base class rather than from -buildings at all: - -```bash -uv run python <<'PY' -from overture.schema.buildings import Building - -for n, f in Building.model_fields.items(): - if not f.is_required(): - continue - for cls in Building.__mro__: - if n in getattr(cls, "__annotations__", {}): - print(f"{n:10} -> {cls.__name__}") - break -PY -``` -``` -id -> OvertureFeature -geometry -> Building -theme -> OvertureFeature -type -> OvertureFeature -version -> OvertureFeature -``` - -`id`, `theme`, `type`, and `version` are required of *every* Overture feature — they're -declared once in `OvertureFeature` -(`packages/overture-schema-common/src/overture/schema/common/feature.py`): - -```python -id: Id = Field(description="A feature ID. ...") -theme: ThemeT -type: TypeT -# Superclass `Feature` provides `geometry` and `bbox`. -version: FeatureVersion -``` - -No `= None`, so all four are mandatory. `Building` adds only `geometry`, narrowing the -inherited one to the polygon types a building may have. - -The two `required` arrays in the JSON Schema split those five across the GeoJSON envelope -— `id` and `geometry` sit at `.required`, while `theme`, `type`, and `version` sit at -`.properties.properties.required`, since that's where they live in the data. - -**As for the human answer:** the Overture Schema Working Group decides, and changes go -through the process in `CONTRIBUTING.md` — a PR plus a changelog fragment. Making a field -required is a breaking change, so it targets the `vnext` branch and waits for a major -release; making one optional is not, and can go to `main`. - -One field's type, constraints, and documentation: - -```bash -jq '.properties.properties.properties.height' building.schema.json -``` -```json -{ - "description": "Height of the building or part in meters.\n\nThis is the distance from the lowest point to the highest point.", - "exclusiveMinimum": 0, - "title": "Height", - "type": "number" -} -``` - -By **contract** I mean the set of rules a validator will enforce — what a producer must -satisfy and what a consumer may therefore rely on. For `height` that's: it must be a -number, and it must be strictly greater than zero. - -But "the contract for `height`" is narrower than "everything true about `height`", in three -ways worth knowing before you rely on it. - -**Optionality isn't in there.** Whether `height` may be omitted is recorded in a sibling -`required` array, not in the field's own subschema. A subschema describes the value *if -present*. - -**Units are documentation, not a rule.** "in meters" appears in `description` — prose for -humans. Nothing rejects a value recorded in feet. The machine-checkable part is only -`type: number` and `exclusiveMinimum: 0`. - -**Relationships between fields are mostly absent.** The schema *can* express cross-field -rules, and elsewhere it does — `@require_any_of`, `@forbid_if` and friends from -the `@require_any_of` / `@forbid_if` decorators in `overture-schema-system`. But no such rule ties `height` to -`min_height`, so this passes: - -```bash -uv run python <<'PY' -import json, yaml -from overture.schema.buildings import Building - -d = yaml.safe_load(open("examples/buildings/building-polygon.yaml")) -d["properties"]["height"] = 5 -d["properties"]["min_height"] = 100 # a floor above the roof -print("accepted:", Building.model_validate_json(json.dumps(d)).height) -PY -``` - -``` -accepted: 5.0 -``` - -A building whose lowest point is 100m and whose highest is 5m is physically impossible and -schema-valid. Both fields satisfy their own contracts; nothing checks them against each -other. - -So: a subschema is the complete machine-checkable contract for **one field in isolation**. -It is not a guarantee that the data makes sense. Same lesson as the stray comma in -[See it actually catch something](#see-it-actually-catch-something) — validation enforces -the schema, not correctness. - -Enums and shared structures live at the top level under `$defs`, not nested: - -```bash -jq -r '.["$defs"] | keys[]' building.schema.json -``` -``` -BuildingClass BuildingSubtype FacadeMaterial NameRule NameVariant Names -PerspectiveMode Perspectives RoofMaterial RoofOrientation RoofShape Side SourceItem -``` - -So the full list of valid values for a field, without reading Python source: - -```bash -jq -r '.["$defs"].BuildingClass.enum[]' building.schema.json | head -``` -``` -agricultural -allotment_house -apartments -barn -beach_hut -boathouse -bridge_structure -bungalow -``` - -> **If the nesting is annoying, skip to Python.** `Building.model_fields` in -> [2.4](#24-ask-the-model-itself-python) gives you the same field list flat, with no -> envelope to walk through. The JSON Schema route is most useful when you want the exact -> published contract — or when you're feeding it to another tool, as in -> [section 8](#8-building-your-own-sdk-or-cli). - -#### Wait — why is there a GeoJSON envelope at all? - -If the schema is now Pydantic, why does any of this still look like GeoJSON? Because those -two things answer different questions, and only one of them changed. - -| | What it is | Did it change? | -|---|---|---| -| **Pydantic** | How the schema is *authored and enforced*, in Python | **Yes** — it replaced hand-written JSON Schema YAML | -| **GeoJSON** | One way a feature can be *written out as JSON*, per RFC 7946 | **No** — it's an interchange format, not an authoring choice | - -Pydantic replaced JSON Schema as the authoring language. It has nothing to say about how -data is serialized, so GeoJSON was never in scope to replace. - -**But GeoJSON is not how Overture ships bulk data — Parquet is.** The release bucket is -Hive-partitioned Parquet: - -``` -s3a://overturemaps-us-west-2/release/2026-07-22.0/theme=buildings/type=building/ -``` - -That is a columnar table: `id`, `geometry`, `height`, and the rest are *columns*, flat, no -envelope. It's what `overture-validate` reads, what DuckDB attaches to, and what you'd -query for anything at scale. - -So why does GeoJSON appear at all? Because **JSON Schema describes JSON documents**, and -when a geospatial feature is written as a single JSON document, GeoJSON is the format -every GIS tool already reads. Parquet has no JSON representation to describe — it has a -columnar schema instead, which is exactly what -[section 6](#6-converting-the-schema-to-other-formats) generates as a Spark `StructType`. - -The honest summary is that there are two serializations and neither is subordinate: - -| Serialization | Shape | Pydantic mode | Where you meet it | -|---|---|---|---| -| **Parquet** | flat / columnar | `python` | the release bucket, Spark, DuckDB — all bulk data | -| **GeoJSON** | nested envelope | `json` | single features, extracts, examples in this repo, web tooling | - -The model supports both deliberately. The JSON Schema you're reading in this subsection -describes the second one, which is the only reason an envelope shows up here at all. - -#### "Interchange format" — what that actually means - -An **interchange format** is one whose job is handing data to a program you didn't write. -It's optimized for being *understood by anything*, not for being stored efficiently. A -**storage format** is the opposite: optimized for holding a lot of data and querying it -fast, at the cost of needing specific software to read it at all. - -| | GeoJSON | Parquet | -|---|---|---| -| Optimized for | being read by anything | storing and scanning millions of rows | -| Text or binary | plain text | binary, columnar, compressed | -| Read it with | any JSON parser | a Parquet library | -| Self-describing | yes — the file says what it is | schema in the footer, not human-readable | -| Good at | one feature, an extract, a web map | a whole theme, a whole planet | - -The cost of being universally readable is that GeoJSON repeats every field name on every -feature: - -```bash -uv run python <<'PY' -import json, yaml -from overture.schema.buildings import Building - -b = Building.model_validate_json( - json.dumps(yaml.safe_load(open("examples/buildings/building-polygon.yaml")))) -gj = b.model_dump(mode="json", by_alias=True, exclude_none=True) -flat = b.model_dump(mode="python", by_alias=True, exclude_none=True) -flat["geometry"] = str(flat["geometry"]) - -n = 10_000 -doc = json.dumps({"type": "FeatureCollection", "features": [gj] * n}) -cols = list(flat) -tbl = json.dumps({"columns": cols, "rows": [[flat[k] for k in cols]] * n}) -print(f"{n:,} features as GeoJSON : {len(doc):>10,} bytes") -print(f"{n:,} features, names stored once: {len(tbl):>10,} bytes") -PY -``` - -``` -10,000 features as GeoJSON : 7,040,043 bytes -10,000 features, names stored once: 4,520,199 bytes -``` - -A 1.56x penalty before compression even enters the picture, and that comparison is still -generous to GeoJSON — real Parquet also compresses each column and lets a reader skip -columns it doesn't need. Multiply by a planet's worth of buildings and the reason bulk -data isn't shipped as GeoJSON is obvious. - -"Interchange" is a role, not a ranking. GeoJSON is the right tool for handing one feature -to a web map; Parquet is the right tool for handing a continent to Spark. - -#### Is Pydantic wrapped around GeoJSON? - -Short answer: **no.** But the question has three reasonable readings, and one of them is a -qualified yes, so it's worth taking them separately. - -**"Is the model built on top of a GeoJSON structure?"** No. A model is a flat list of -fields, declared one at a time in Python. You can build one and use it without JSON ever -entering the picture: - -```bash -uv run python <<'PY' -from overture.schema.buildings import Building -from overture.schema.system.geometric import Geometry - -b = Building( - id="my-building-1", - geometry=Geometry.from_wkt("POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))"), - theme="buildings", - type="building", - version=1, - height=12.5, -) -print(b.id, b.height) -print(sorted(b.model_dump(mode="python", by_alias=True, exclude_none=True))) -PY -``` - -``` -my-building-1 12.5 -['geometry', 'height', 'id', 'level', 'theme', 'type', 'version'] -``` - -No GeoJSON was parsed, produced, or consulted. If GeoJSON were the substrate, that -wouldn't be possible. - -**"Is there GeoJSON code inside the Pydantic classes?"** Yes — in exactly one of them. -Counting mentions across everything `Building` inherits from: - -``` -Building (building ) 0 -OvertureFeature (feature ) 0 -Identified (id ) 0 -Feature (feature ) 17 <- the base class in overture-schema-system -Named (names ) 0 -Stacked (level ) 0 -Appearance (_common ) 0 -``` - -All of it lives in `Feature`, in one serializer and one validator — the code that reads -GeoJSON in and writes GeoJSON out. Zero mentions in the six classes that actually define -what a building *is*. GeoJSON is an I/O concern parked at the base of the hierarchy, not a -structure the schema is built on. - -**"Is GeoJSON what's really being validated?"** No. What gets validated is the model. A -GeoJSON document is one accepted *input shape* — a flat dict is the other, and both end up -as the same Python object. - -An analogy: a word processor's document isn't "wrapped around `.docx`." It has a document -model, and it can read and write `.docx`. Deleting that import/export code would not -change what a document is. Same here — delete the ten lines below and Overture models -still work; they just stop speaking GeoJSON. - - -The entire GeoJSON transformation is those ten lines, in `Feature` -(`packages/overture-schema-system/src/overture/schema/system/feature.py`): - -```python -@model_serializer(mode="wrap") -def __serialize_with_geo_json_support__(self, serializer, info): - data = serializer(self) # <- the flat dict, produced first - - if info.mode == "json": # <- only in JSON mode - return { - "type": "Feature", - **({"id": data.pop("id")} if "id" in data else {}), - **({"bbox": data.pop("bbox")} if "bbox" in data else {}), - "geometry": data.pop("geometry"), - "properties": data, # <- everything else goes here - } - - return data # <- Python mode: flat, untouched -``` - -Read the first line: Pydantic produces the **flat** dictionary, and only then does this -function move `id`, `bbox`, and `geometry` to the top and sweep the remainder into -`properties`. In `python` mode the flat dict is returned unchanged and none of this runs. - -So the layering is: - -``` - Pydantic model (flat — the actual schema) - | - +----------+----------+ - | | - python mode json mode - | | - flat dict GeoJSON envelope - -> Parquet -> .geojson -``` - -GeoJSON is a costume the model puts on for one specific audience. It isn't the body. - -**The Pydantic model itself has no envelope.** In Python it's completely flat: - -```bash -uv run python <<'PY' -from overture.schema.buildings import Building -print("id in model_fields :", "id" in Building.model_fields) -print("geometry in model_fields :", "geometry" in Building.model_fields) -print("a 'properties' field? :", "properties" in Building.model_fields) -PY -``` - -``` -id in model_fields : True -geometry in model_fields : True -a 'properties' field? : False -``` - -There is no `properties` field on `Building`, and `id` and `geometry` sit alongside -`height` and `num_floors` like any other field. The envelope is **not part of the model**. -It appears only when serializing to JSON, because that is what GeoJSON requires. - -You can watch the same object take both shapes: - -```bash -uv run python <<'PY' -import json, yaml -from overture.schema.buildings import Building - -b = Building.model_validate_json( - json.dumps(yaml.safe_load(open("examples/buildings/building-polygon.yaml")))) - -print("python mode:", sorted(b.model_dump(mode="python", by_alias=True, exclude_none=True))[:8]) -print("json mode :", sorted(b.model_dump(mode="json", by_alias=True, exclude_none=True))) -PY -``` - -``` -python mode: ['class', 'ext_bar', 'ext_foo', 'geometry', 'height', 'id', 'is_underground', 'level'] -json mode : ['geometry', 'id', 'properties', 'type'] -``` - -One model, two renderings — and the flat one is the shape of the data you'd actually -download. Neither is more "real"; the model is what's real, and both are projections of -it. - -**So which answer to "what's required" is correct?** The model's: - -``` -['geometry', 'id', 'theme', 'type', 'version'] -``` - -The JSON Schema's two `required` arrays are that same list, split across the envelope -because that's where those fields land *in that particular output format*. Nobody decided -`geometry` belongs somewhere different from `theme`; GeoJSON did, in 2016. - -This distinction is the single most important thing in this guide, and it returns in force -in [section 3](#3-writing-code-against-the-models) — where using the wrong mode for your -data shape is the most common way to get a confusing `ValidationError`. - -#### Why `jq` for this - -`jq` isn't required — you could parse the schema in Python. It suits *poking around* -specifically, for four reasons: - -**Paths mirror the structure.** JSON is a tree and `jq` is a query language for trees, so -`.properties.properties.properties.height` reads exactly like the nesting it walks. You -compose a path left to right instead of writing traversal code. - -**It answers "what is this?", not just "give me X".** Most JSON tools retrieve a value you -already know the name of. `jq` has structure-discovery built in: - -```bash -jq 'keys' building.schema.json -jq -c 'paths | select(.[-1]=="height")' building.schema.json -``` -```json -["$defs","additionalProperties","description","properties","required","title","type"] -["properties","properties","properties","height"] -``` - -That second one finds a field wherever it lives — you don't have to know the shape first. -That is the difference between a query tool and an exploration tool. - -**It's a filter, so it composes.** Data in, data out — pipe it, redirect it, chain it with -`head`, feed one query's output to another. And because the output is JSON, `jq` composes -with itself. - -**Zero setup.** No file to create, no imports, no session to keep alive. One line in the -shell you're already in. - -It also reshapes, which is useful for scanning a lot at once — every field with its type, -as a table: - -```bash -jq -r '.properties.properties.properties - | to_entries[] - | "\(.key)\t\(.value.type // .value["$ref"] // "?")"' building.schema.json -``` -``` -class #/$defs/BuildingClass -facade_color string -facade_material #/$defs/FacadeMaterial -has_parts boolean -height number -is_underground boolean -``` - -**Where it's the wrong tool.** `jq` is a whole separate language, its error messages are -terse, and — as the `null` above shows — it answers a wrong path with a shrug rather than -a complaint. For *this* schema, Python introspection is usually easier: the field list -comes out flat, with no GeoJSON envelope to walk. Use `jq` when you want the published -contract exactly as it ships, or when you're piping it into another tool. Use Python when -you want to understand the model. That's the next subsection. - -### 2.6 From Python: enumerate everything that's installed - -```python -from overture.schema.system.discovery import discover_models - -for key, model in sorted(discover_models().items(), key=lambda kv: kv[0].name): - print(f"{key.name:20} {key.entry_point:45} {sorted(key.tags)}") -``` - -``` -address overture.schema.addresses:Address ['feature', 'overture:theme=addresses'] -building overture.schema.buildings:Building ['feature', 'overture:theme=buildings'] -segment overture.schema.transportation:Segment ['feature', 'overture:theme=transportation'] -... -``` - -A `ModelKey` carries `.name`, `.entry_point` (`"module:Class"`), and `.tags` -(a `frozenset[str]`). Filter the same way the CLI does: - -```python -from overture.schema.system.discovery import TagSelector, discover_models, filter_models - -models = discover_models() - -buildings = filter_models( - models, TagSelector(include_any=("overture:theme=buildings",)) -) -``` - -`TagSelector` takes `include_any` (OR scope), `require_all` (AND narrowing), and -`exclude_any` (OR-NOT). An empty selector returns the input unchanged. - -### 2.7 Reading enum member documentation - -Enum members carry per-value docstrings, but `member.__doc__` falls back to the *class* -docstring when a member has none — so reading `__doc__` directly gives you misleading -results: - -```python -from overture.schema.buildings import RoofShape - -[(m.value, m.__doc__.strip()[:30]) for m in list(RoofShape)[:2]] -# [('dome', 'The shape of the roof.'), ('flat', 'The shape of the roof.')] -# ^ that's the class docstring repeated, not per-member documentation -``` - -Use the codegen extractor, which does the fallback detection for you: - -```python -from overture.schema.codegen.extraction.enum_extraction import extract_enum -from overture.schema.common.scoping.travel_mode import TravelMode - -spec = extract_enum(TravelMode) -for m in spec.members[:4]: - print(f"{m.value:14} {m.description or '—'}") -``` - -``` -vehicle — -motor_vehicle Includes car, truck and motorcycle -car — -truck — -``` - -`description` is `None` when the member has no documentation of its own. - -### 2.8 Generate browsable reference docs - -For sustained exploration, generate the full markdown reference and read it in your -editor: - -```bash -uv run overture-codegen generate --format markdown --output-dir ./schema-docs -``` - -You get one page per feature type, per enum, and per named type, with field tables, -prose constraint descriptions, cross-page links, and validated examples: - -``` -schema-docs/buildings/building.md -schema-docs/buildings/building_part.md -schema-docs/buildings/types/building_class.md -schema-docs/buildings/types/roof_shape.md -schema-docs/common/names.md -schema-docs/common/sources.md -schema-docs/system/numeric.md -... -``` - -Scope it to one theme with the same tag options: - -```bash -uv run overture-codegen generate --format markdown \ - --tag overture:theme=buildings --output-dir ./schema-docs -``` - -(Supplementary types from `common/` and `system/` are pulled in regardless, since the -feature pages link to them.) - ---- - ---- - -## 3. Writing code against the models - -Section 2 was about finding out what exists. This section is about using it: loading -real data into models, reading and changing it, and writing it back out. - -### 3.1 A complete example, start to finish - -Before any of the details, here is the whole job in one piece: load a feature, read it, -change it, write it back out, and confirm it still validates. Everything else in this -section is an explanation of a line in this snippet. - -```bash -uv run python <<'PY' -import json, yaml -from overture.schema.buildings import Building - -# 1. LOAD — the file is GeoJSON, so go through JSON mode -doc = yaml.safe_load(open("examples/buildings/building-polygon.yaml")) -b = Building.model_validate_json(json.dumps(doc)) -print("1. loaded :", b.id) - -# 2. READ — plain Python attributes; enums come back as enum members -print("2. read :", b.height, "m,", b.num_floors, "floors, class", b.class_.value) - -# 3. MODIFY — ordinary assignment -b.height = 25.0 -b.num_floors = 5 -print("3. changed:", b.height, "m,", b.num_floors, "floors") - -# 4. WRITE — by_alias=True so `class_` is written as `class` -out = b.model_dump(mode="json", by_alias=True, exclude_none=True) -print("4. wrote :", json.dumps(out)[:70], "...") - -# 5. CONFIRM — the output is valid input -again = Building.model_validate_json(json.dumps(out)) -print("5. re-read:", again.height, "m — round-trip holds") -PY -``` - -``` -1. loaded : overture:buildings:building:1234 -2. read : 21.34 m, 4 floors, class parking -3. changed: 25.0 m, 5 floors -4. wrote : {"type": "Feature", "id": "overture:buildings:building:1234", "geometr ... -5. re-read: 25.0 m — round-trip holds -``` - -That is the shape of nearly every job: **validate in, work with plain Python objects, -serialize out.** In between, `b` is an ordinary object — attributes, assignment, no -special API. - -Three lines in there are load-bearing, and each gets a subsection below: - -| Line | Why it's written that way | Where | -|---|---|---| -| `model_validate_json(json.dumps(doc))` | the file is GeoJSON, which needs JSON mode | [3.2](#32-the-one-thing-to-understand-two-representations) | -| `b.class_.value` | the field is `class_` in Python, `class` in the data | [3.3](#33-reading-fields) | -| `by_alias=True, exclude_none=True` | without them the output won't re-validate | [3.4](#34-writing-data-back-out) | - -If the snippet above ran, you already know enough to be useful. Read on when one of those -lines bites you, or read straight through if you'd rather know why now. - -### 3.2 The one thing to understand: two representations - -Overture publishes data in one shape — flat and tabular, the column layout of the -Parquet release. The models also read and write GeoJSON, so the schema works with tools -that expect features rather than rows, and because that is the representation the -generated [JSON Schema](#61-json-schema) describes. **Which one you get depends on the -Pydantic mode you use, not on the data you pass.** - -| Shape | Looks like | Pydantic mode | Validate with | Dump with | -|---|---|---|---|---| -| **GeoJSON** | `id`/`geometry` at top level, everything else under `properties` | `json` | `model_validate_json()` | `model_dump(mode="json")`, `model_dump_json()` | -| **Flat / tabular** (Parquet-style) | every field at the top level | `python` | `model_validate()` | `model_dump(mode="python")` | - -This trips people up constantly. Passing a GeoJSON *dict* to `model_validate()` fails, -because `model_validate` is Python mode and Python mode expects the flat shape: - -```python -import json, yaml -from overture.schema.buildings import Building - -doc = yaml.safe_load(open("examples/buildings/building-polygon.yaml")) # GeoJSON-shaped - -Building.model_validate(doc) -# ValidationError: 3 validation errors for building -# theme Field required -# type Input should be 'building' [got 'Feature'] -# version Field required -``` - -The `type: Feature` in the error message is the tell: it read the GeoJSON envelope's -`type` as the feature's `type` field. - -The fix — round-trip through JSON so you're in JSON mode: - -```python -building = Building.model_validate_json(json.dumps(doc)) # works -``` - -Or, if you're already reading from a file or an HTTP response, skip the parse entirely: - -```python -building = Building.model_validate_json(open("building.geojson").read()) -``` - -#### Can't I just tell `model_validate` to use JSON mode? - -No. It's the obvious thing to try, and neither knob does it: - -```python -Building.model_validate(doc, context={"mode": "json"}) # still ValidationError -Building.model_validate(doc, strict=False) # still ValidationError -``` - -The mode isn't a setting you pass — in Pydantic it's determined by *which method you -call*. `model_validate` is Python mode; `model_validate_json` is JSON mode. The base -`Feature` class keys off exactly that: - -```python -@model_validator(mode="wrap") -def __validate_with_geo_json_support__(cls, data, handler, info): - if info.mode == "json": # <- set by the method, not by an argument - ... # unpack the GeoJSON envelope -``` - -So if you have a GeoJSON **dict** in hand, `json.dumps` it and use -`model_validate_json`. The round-trip is slightly wasteful but it is the supported path: - -```python -Building.model_validate_json(json.dumps(doc)) -``` - -Better still, avoid making the dict at all. If the GeoJSON came from a file or an HTTP -response, hand the raw text straight to `model_validate_json` and skip `json.loads` -entirely. The one case where you genuinely can't is YAML — there's no YAML mode, so -`yaml.safe_load` → `json.dumps` → `model_validate_json` is the route, as in the example -above. - -#### Reading the error - -The three errors from a mode mismatch are always the same shape, and worth recognising on -sight: - -``` -theme Field required -type Input should be 'building' [input_value='Feature'] -version Field required -``` - -`theme` and `version` are "missing" because they're really down inside `properties`, where -Python mode isn't looking. And `type` came back as `'Feature'` — the envelope's type, -which is the giveaway. **If you ever see `input_value='Feature'` in a validation error, -you passed GeoJSON to a Python-mode call.** - -For flat data — a Parquet row, a DuckDB result, a dict of columns — `model_validate` is -the right call: - -```python -row = { - "id": "...", - "theme": "buildings", - "type": "building", - "version": 1, - "geometry": ..., - "height": 21.34, - "class": "parking", -} -building = Building.model_validate(row) -``` - -### 3.3 Reading fields - -Model attributes are plain Python. Enums come back as enum members, geometry as a -`Geometry` wrapper around Shapely: - -```python -building.height # 21.34 -building.class_ # -building.class_.value # 'parking' -building.num_floors # 4 -type(building.geometry) # -``` - -### 3.4 Writing data back out - -Some fields are Python keywords, so the model attribute differs from the wire name -(`class_` on the model, `class` in the data). **`model_dump()` uses attribute names by -default**, which produces output that will not validate back: - -```python -d = building.model_dump(mode="json") -sorted(d["properties"]) -# ['class_', 'ext_bar', 'height', ...] ← 'class_' is wrong for the wire - -Building.model_validate(building.model_dump(mode="python")) -# ValidationError: invalid extra field name: class_ -``` - -With `by_alias=True` both round-trips work: - -```python -geojson = building.model_dump(mode="json", by_alias=True, exclude_none=True) -sorted(geojson["properties"]) -# ['class', 'ext_bar', 'height', 'is_underground', 'level', 'num_floors', ...] - -Building.model_validate_json(json.dumps(geojson)) # OK -Building.model_validate( - building.model_dump(mode="python", by_alias=True, exclude_none=True) -) # OK -``` - -Same for `model_dump_json()`: - -```python -'"class":' in building.model_dump_json() # False ← emits "class_" -'"class":' in building.model_dump_json(by_alias=True) # True -``` - -**Rule of thumb: `by_alias=True` on every dump, unless you specifically want Python -attribute names.** `exclude_none=True` is usually what you want too — otherwise you get -every unset optional field as an explicit `null`. - -### 3.5 Working with `Segment` and other unions - -`Segment` is a discriminated union type alias over `RoadSegment`, `RailSegment`, and -`WaterSegment` — not a model class. It has no `model_validate`: - -```python -from overture.schema.transportation import Segment - -type(Segment) # -Segment.model_validate({...}) -# AttributeError: model_validate -``` - -Wrap it in a `TypeAdapter`: - -```python -from pydantic import TypeAdapter -from overture.schema.transportation import Segment - -segments = TypeAdapter(Segment) - -seg = segments.validate_json(raw_geojson_string) # JSON mode → GeoJSON -seg = segments.validate_python(flat_row) # Python mode → flat -type(seg).__name__ # 'RoadSegment' -``` - -Build the `TypeAdapter` once and reuse it; construction is the expensive part. - -The concrete arms *are* ordinary classes if you know which one you want: - -```python -from overture.schema.transportation import RoadSegment -``` - -### 3.6 Validating without knowing the type - -`overture-schema-validation` checks a record against every installed model and returns -whichever matched: - -```python -from overture.schema.validation import validate, validate_json - -feature = validate_json(geojson_string) # JSON mode → GeoJSON shape -type(feature).__name__ # 'Building' - -feature = validate(flat_dict) # Python mode → flat shape -``` - -Both raise `pydantic.ValidationError` when nothing matches. The same mode rule from -[The one thing to understand first](#32-the-one-thing-to-understand-two-representations) applies: `validate()` is -Python mode and wants flat data, `validate_json()` is JSON mode and wants GeoJSON. - -Which models participate is resolved at runtime by entry-point discovery — install more -theme packages and these functions accept more. - -### 3.7 Generating JSON Schema in code - -```python -from overture.schema.system.json_schema import json_schema -from overture.schema.buildings import Building -from overture.schema.places import Place - -schema = json_schema(Building) -schema["title"] # 'building' -sorted(schema) # ['$defs', 'additionalProperties', 'description', -# 'properties', 'required', 'title', 'type'] - -union = json_schema(Building | Place) # unions work too → anyOf -``` - -Use this rather than Pydantic's `model_json_schema()`. The Overture generator treats -`T | None = None` as "omit when unset" instead of Pydantic's "nullable with a null -default", which is what the data actually means. - ---- - ---- - -## 4. The three CLIs - -Installing the workspace gives you three commands, from three different packages. - -| Command | Package | Purpose | -|---|---|---| -| `overture-schema` | `overture-schema-cli` | Validate files, emit JSON Schema, list types | -| `overture-codegen` | `overture-schema-codegen` | Generate markdown docs and PySpark expressions | -| `overture-validate` | `overture-schema-pyspark` | Validate Parquet/S3 data at scale with Spark | - -Prefix each with `uv run` inside the repo, or activate the venv. - -### 4.1 `overture-schema` - -``` -Usage: overture-schema [OPTIONS] COMMAND [ARGS]... - -Commands: - json-schema Generate JSON schema for Overture Maps types. - list-types List all available types. - validate Validate Overture Maps data against schemas. -``` - -All three subcommands take the shared `--tag` / `--filter` / `--exclude` options from -[From the CLI: what types exist?](#23-from-the-cli-what-types-exist). `validate` and `json-schema` also take -`--type NAME` to target one type directly. - -```bash -# What types do I have? -overture-schema list-types -overture-schema list-types --group-by overture:theme - -# Validate -overture-schema validate data.geojson -overture-schema validate - < data.geojson -overture-schema validate --type building data.json -overture-schema validate --tag overture:theme=buildings data.json -overture-schema validate --show-field id data.json - -# JSON Schema -overture-schema json-schema > all-types.json -overture-schema json-schema --type building > building.json -overture-schema json-schema --tag overture:theme=buildings > buildings.json -``` - -Exit codes: `0` on success, `1` on validation failure — so it drops into CI directly. - -#### Local files or remote? - -It depends which CLI, and the two answer differently. - -| Command | Remote paths | How | -|---|---|---| -| `overture-schema validate` | **no** | pipe through stdin with `-` | -| `overture-validate` (PySpark) | **yes** | `s3a://` natively, anonymous credentials preconfigured | - -`overture-schema validate` takes a filesystem path only. Hand it a URL and it fails — -note that it even mangles the `//`, because the argument is parsed as a path: - -```bash -overture-schema validate https://raw.githubusercontent.com/OvertureMaps/schema/main/examples/buildings/building-polygon.yaml -``` - -``` -Error: 'https:/raw.githubusercontent.com/.../building-polygon.yaml' is not a file. -``` - -The fix is the `-` argument, which reads stdin. Anything that can fetch bytes can feed it: - -```bash -curl -sSf https://raw.githubusercontent.com/OvertureMaps/schema/main/examples/buildings/building-polygon.yaml \ - | overture-schema validate - -``` - -``` -✓ Successfully validated -``` - -That works for anything on stdin — `aws s3 cp ... -`, a database query, a generator -script, another program's output. The only thing you lose is the filename in the output, -which becomes ``. - -`overture-validate` is the opposite: it's built for remote data. `s3a://` paths are -detected automatically and configured with anonymous credentials, so the public Overture -release bucket needs no setup: - -```bash -overture-validate segment s3a://overturemaps-us-west-2/release/2026-07-22.0 -``` - -> **Don't hardcode a release version.** The bucket keeps only the current release, so any -> version written into a script or a doc stops working at the next publish. Ask the bucket -> instead: -> -> ```bash -> curl -sS "https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/?list-type=2&prefix=release/&delimiter=/" \ -> | tr '<' '\n' | grep -oE 'Prefix>release/[^/]+' | sed 's|Prefix>release/||' | sort -r | head -1 -> ``` -> -> ``` -> 2026-07-22.0 -> ``` -> -> Then use it: -> -> ```bash -> RELEASE=$(curl -sS "https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/?list-type=2&prefix=release/&delimiter=/" \ -> | tr '<' '\n' | grep -oE 'Prefix>release/[^/]+' | sed 's|Prefix>release/||' | sort -r | head -1) -> overture-validate segment "s3a://overturemaps-us-west-2/release/$RELEASE" -> ``` -> -> The version shown in the examples here was current when this was written; treat it as a -> placeholder, not a fact. - - -That difference is not arbitrary. `overture-schema validate` is for a file you're looking -at — an example, a fixture, one feature you're debugging. `overture-validate` is for a -release: millions of rows, read in parallel by Spark, where "download it first" isn't an -option. - -### 4.2 `overture-codegen` - -``` -Usage: overture-codegen [OPTIONS] COMMAND [ARGS]... - -Commands: - generate Generate code/docs from discovered models. - list List all discovered models. -``` - -``` -Usage: overture-codegen generate [OPTIONS] - - --format [markdown|pyspark] Output format [required] - --tag / --filter / --exclude TEXT - --output-dir PATH Default: stdout - --test-output-dir PATH Write generated conformance tests (pyspark only) -``` - -```bash -# Markdown reference docs -overture-codegen generate --format markdown --output-dir ./schema-docs -overture-codegen generate --format markdown --tag overture:theme=places --output-dir ./out - -# PySpark validation expressions (this is what `make generate-pyspark` runs) -overture-codegen generate --format pyspark \ - --output-dir packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated \ - --test-output-dir packages/overture-schema-pyspark/tests/generated -``` - -### 4.3 `overture-validate` - -Validates real data volumes with Spark. Requires the generated expression tree, so run -`make install` or `make generate-pyspark` first. - -``` -Usage: overture-validate [OPTIONS] FEATURE_TYPE PATH - - -o, --output TEXT Output path for validated Parquet. - --head INTEGER Error rows to display. [default: 20] - --conf TEXT Spark config key=value pairs. - --count-only Report error count only. - --skip-schema-check Warn on schema mismatches instead of aborting. - --skip-columns TEXT Columns declared absent from data. - --ignore-extra-columns TEXT Extra data columns to ignore in schema comparison. - --suppress TEXT Suppress checks: FIELD or FIELD:CHECK. -``` - -```bash -overture-validate building local.parquet -overture-validate segment s3a://overturemaps-us-west-2/release/2026-07-22.0 -overture-validate place data.parquet --count-only -overture-validate segment data.parquet --suppress version:bounds -o violations.parquet -``` - -It handles S3A and anonymous credentials for the public Overture bucket automatically, -and expands a release root into the Hive partition path for you. Full option and -path-resolution reference: `packages/overture-schema-pyspark/README.md`. - ---- - ---- - -## 5. Validating data - -Three tiers, pick by data size. - -### 5.1 One file, or a handful — the CLI - -Accepts JSON, YAML, and GeoJSON. A single feature, a JSON array of features, or a -`FeatureCollection` all work: - -```bash -overture-schema validate examples/buildings/building-polygon.yaml -``` - -``` -✓ Successfully validated examples/buildings/building-polygon.yaml -``` - -Failures come back as a rendered table showing the offending value in context: - -```bash -overture-schema validate --show-field id counterexamples/buildings/negative-height.json -``` - -``` - ─ Validation Failed id=foo ──────────────────────────────────────────────────── - ... - id "foo" - version 0 - height -1.23 ← Input should be greater than 0 - ────────────────────────────────────────────────────────────────────────────── -``` - -For a collection, errors are indexed and labeled by the model that best fit: - -``` - ─ [1] (Building) ────────────────────────────────────────────────────────────── - ... - version 0 - height -1.23 ← Input should be greater than 0 - ────────────────────────────────────────────────────────────────────────────── -``` - -Narrow the candidate set to sharpen the error messages — with `--type building` the CLI -stops guessing which model you meant: - -```bash -overture-schema validate --type building data.json -``` - -### 5.2 In a Python pipeline - -```python -from pydantic import ValidationError -from overture.schema.validation import validate_json - -ok, bad = 0, [] -for line in open("features.ndjson"): - try: - validate_json(line) - ok += 1 - except ValidationError as e: - bad.append((line[:60], e.errors())) - -print(f"{ok} valid, {len(bad)} invalid") -``` - -`e.errors()` gives you structured dicts with `loc`, `msg`, `type`, and `input` — the -right thing to log or turn into a report. - -If you know the type, validate against it directly for better errors and speed: - -```python -from overture.schema.buildings import Building - -Building.model_validate_json(line) -``` - -### 5.3 At scale — PySpark - -```python -from pyspark.sql import SparkSession -from overture.schema.pyspark import validate_model, explain_errors - -spark = SparkSession.builder.getOrCreate() -df = spark.read.parquet("s3a://.../theme=buildings/type=building/") - -result = validate_model(df, "building") -result.evaluated.cache() - -total = result.evaluated.count() -errors = result.error_rows().count() -print(f"{errors} / {total} rows with errors") - -if errors: - violations = explain_errors(result.evaluated, result.checks) - violations.select("id", "field", "check", "message").show(truncate=False) -``` - -`validate_model` accepts either the short name (`"building"`) or the full entry-point key -(`"overture.schema.buildings:Building"`). It looks up the feature type in the registry, -compares the DataFrame schema against the expected one, and evaluates every check in a -single pass — no per-row Python, so it scales. - -| Function | Returns | Purpose | -|---|---|---| -| `validate_model(df, type)` | `ValidationResult` | Registry lookup, schema comparison, check evaluation | -| `result.error_rows()` | `DataFrame` | Rows with at least one violation | -| `explain_errors(evaluated, checks)` | `DataFrame` | One row per violation: `field`, `check`, `message` | -| `model_names()` | `list[str]` | Available type names | - -### 5.4 Validating the schema itself - -If you're changing the models rather than the data: - -```bash -make check -make test -make update-baselines -``` - -The theme packages carry golden-file baseline tests of their generated JSON Schema, so -unintended schema drift fails CI. After an intentional change, run `make -update-baselines` and inspect the `git diff` on the regenerated golden files before -committing. - ---- - ---- - -## 6. Converting the schema to other formats - -Three built-in targets, plus everything reachable through JSON Schema. - -| Target | Command | Output | -|---|---|---| -| JSON Schema | `overture-schema json-schema` | A JSON Schema document on stdout | -| Markdown | `overture-codegen generate --format markdown` | Docusaurus-ready reference pages | -| PySpark | `overture-codegen generate --format pyspark` | Python modules of `Check` builders + `StructType` | -| Spark `StructType` | (Python, via the pyspark registry) | A live Spark schema object | - -### 6.1 JSON Schema - -The interop format — this is your bridge to every other ecosystem. - -```bash -# One type -overture-schema json-schema --type building > building.schema.json - -# One theme -overture-schema json-schema --tag overture:theme=transportation > transportation.schema.json - -# Everything (an `anyOf` over all installed types) -overture-schema json-schema > overture.schema.json -``` - -A single type produces a self-contained document with its dependencies inlined under -`$defs`: - -``` -$ jq 'keys' building.schema.json -["$defs", "additionalProperties", "description", "properties", "required", "title", "type"] - -$ jq '.title, (.["$defs"] | length)' building.schema.json -"building" -13 -``` - -All types produce `{"anyOf": [...], "$defs": {...}}`. - -In Python: - -```python -from overture.schema.system.json_schema import json_schema -from overture.schema.buildings import Building - -schema = json_schema(Building) -``` - -### 6.2 Markdown - -Covered in [Generate browsable reference docs](#28-generate-browsable-reference-docs). Output is Docusaurus-flavored -(frontmatter plus `_category_.json` files), but it's plain markdown underneath and reads -fine in any editor or static site generator. - -### 6.3 PySpark expressions and Spark schemas - -```bash -overture-codegen generate --format pyspark --output-dir ./ps --test-output-dir ./ps-tests -``` - -You get one module per feature type, mirroring the Python package layout: - -``` -ps/overture/schema/buildings/building.py -ps/overture/schema/buildings/building_part.py -ps-tests/overture/schema/buildings/test_building.py -``` - -Each module is auto-generated (`# Do not edit`) and contains one builder function per -constraint, returning a `Check` with an unevaluated PySpark `Column`: - -```python -def _version_bounds_check() -> Check: - return Check( - field="version", - name="bounds", - expr=check_bounds(F.col("version"), ge=0), - shape=CheckShape.SCALAR, - root_field="version", - ) -``` - -Plus a `MODEL_VALIDATION` constant pairing the checks with the expected `StructType`. - -**To get a Spark schema for a feature type** — useful for `spark.read.schema(...)`, -Delta table creation, or comparing against your own tables: - -```python -from overture.schema.pyspark._registry import REGISTRY -from overture.schema.pyspark.validate import resolve_entry_point_key - -key = resolve_entry_point_key("building", REGISTRY) -struct = REGISTRY[key].schema - -type(struct).__name__ # 'StructType' -[(f.name, f.dataType.simpleString()) for f in struct.fields][:5] -``` - -``` -[('id', 'string'), - ('bbox', 'struct'), - ('geometry', 'binary'), - ('theme', 'string'), - ('type', 'string')] -``` - -`REGISTRY` is keyed by the full entry-point string -(`"overture.schema.buildings:Building"`), which is why the `resolve_entry_point_key` -step is there — it accepts the short alias too. `ModelValidation` exposes `.schema`, -`.checks`, and `.geometry_types`. - -Since `StructType` has `.json()` and `.jsonValue()`, this is also your route to an -Arrow/Parquet schema. - ---- - ---- - -## 7. Using the packages from your own project - -Everything above assumes you're working *inside* the schema repo. If instead you're -building your own application that depends on these models, you don't use the workspace -at all — you point `uv` at the package directories on disk. - -Depend on the **theme packages you actually need**, not the workspace root: - -```toml -# myapp/pyproject.toml -[project] -name = "myapp" -version = "0.1.0" -requires-python = ">=3.10" -dependencies = [ - "overture-schema-theme-buildings", - "overture-schema-cli", -] - -[tool.uv.sources] -overture-schema-theme-buildings = { path = "/path/to/schema/packages/overture-schema-theme-buildings", editable = true } -overture-schema-cli = { path = "/path/to/schema/packages/overture-schema-cli", editable = true } -``` - -```bash -cd myapp -uv sync -uv run python -c "from overture.schema.buildings import Building; print(Building.__name__)" -``` -``` -Building -``` - -`editable = true` means edits in your schema clone take effect immediately in `myapp` — -useful if you're changing both. - -### 7.1 The payoff: install set = runtime set - -In the `myapp` project above, only the buildings theme is installed. So: - -```bash -cd myapp && uv run overture-schema list-types -``` -``` -building feature overture:theme=buildings -building_part feature overture:theme=buildings -``` - -Two types, not fifteen. **The same CLI binary, scoped by what's installed.** Nothing was -configured to make that happen — the models register themselves through entry points, and -the tooling discovers whatever is present. - -This is worth internalizing early, because it's the whole extension story: add your own -package that registers models, and your feature types appear in `list-types`, validate -through the same commands, and show up in generated docs, alongside Overture's. See -[Register your own feature types](#83-register-your-own-feature-types). - -### 7.2 If you'd rather have everything - -`overture-schema` is a metapackage depending on all six themes plus validation and the -CLI: - -```toml -[tool.uv.sources] -overture-schema = { path = "/path/to/schema/packages/overture-schema", editable = true } -``` - -One caveat: `import overture.schema` gives you nothing directly. It's a namespace root -that ships only a `py.typed` marker — no models, no functions. Always import from the -theme packages: - -```python -from overture.schema.buildings import Building # ✓ -from overture.schema import Building # ✗ ImportError -``` - - ---- - -### 7.3 What changes once these packages are published - -Some of this section is scaffolding for the fact that nothing is on a package index yet. -Worth knowing which parts, so you don't over-invest in learning them. - -| Part of this section | After publishing | -|---|---| -| What a workspace is, the shared `.venv`, `uv run` | **Stays** — but becomes reading for contributors only | -| `git clone` + `uv sync --all-packages` | **Stays** — contributors only | -| `make generate-pyspark` | **Gone for consumers.** Published wheels ship the generated expressions already. | -| The `SPARK_HOME` fix | **Stays forever.** It's an environment problem, unrelated to packaging. | -| Empty registry, `exclude-newer` warning | Contributors only | -| The `[tool.uv.sources]` path blocks above | **Deleted entirely.** This is the pure workaround. | -| "Install set = runtime set" | **Stays.** That's entry-point discovery, not packaging. | - -The whole of this subsection collapses to one line: - -```bash -uv add overture-schema-theme-buildings overture-schema-cli -``` - -or, for everything: - -```bash -uv add overture-schema -``` - -**On the PySpark step specifically:** the release pipeline already handles it. -`.github/workflows/publish-python-packages.yaml` runs `make generate-pyspark` before -`uv build`, and aborts the release if the resulting wheel contains no -`expressions/generated/*.py`. Its own comment explains why: - -> the tree must be generated here before the build, or the published wheel ships without -> its `expressions/generated/` modules and `validate_model()` discovers nothing to run - -So consumers of a published wheel never run that step. - -**Which index?** The publish workflow currently pushes to AWS CodeArtifact (the -`overture-pypi` domain), even though its step names say PyPI. `CONTRIBUTING.md` describes -version bumps reaching public PyPI while interim builds stay internal. Either way the -consumer instruction is a one-line install — only the index URL differs. - -Nothing in sections 2 through 9 changes. The behavior you learn there — entry-point -discovery, the two representations, `by_alias`, `TypeAdapter` for `Segment` — is independent -of how the packages get onto your machine. - ---- - ---- - -# Part II — Extending and authoring the schema - -## 8. Building your own SDK or CLI - -Four approaches, cheapest first. - -### 8.1 Generate an SDK from JSON Schema (any language) - -Emit JSON Schema, then hand it to a standard generator. Both of these were run against -the output of `overture-schema json-schema --type building` and produced working code. - -**Python (datamodel-code-generator):** - -```bash -uv run overture-schema json-schema --type building > building.schema.json - -uvx --from datamodel-code-generator datamodel-codegen \ - --input building.schema.json \ - --input-file-type jsonschema \ - --output building_models.py -``` - -Produces standalone Pydantic v2 models with no Overture dependency — useful for a -service that shouldn't take the whole workspace as a dependency: - -```python -# generated by datamodel-codegen: -# filename: building.schema.json -from pydantic import BaseModel, ConfigDict, Field, confloat, conint, constr -``` - -**TypeScript (quicktype):** - -```bash -npx -y quicktype --src-lang schema --lang typescript \ - -o Building.ts building.schema.json -``` - -Field descriptions survive as JSDoc: - -```typescript -/** - * Buildings are man-made structures with roofs that exist permanently in one place. - * ... - */ -export interface Building { - bbox?: [number, number, number, number, ...number[]]; - /** The building's footprint or roofprint... */ - geometry: Geometry; -``` - -quicktype also targets Go, Rust, Java, Kotlin, Swift, C#, and others from the same input. -Anything that reads JSON Schema — OpenAPI toolchains, `go-jsonschema`, `schemars`, -`jsonschema2pojo` — works the same way. - -The tradeoff: you get types and structural validation, but not the semantic layer. -Cross-field model constraints (`@require_any_of`, `@radio_group`) do translate into JSON -Schema `if`/`then`/`anyOf` constructs, but domain-specific error messages and the -NewType vocabulary flatten out. - -### 8.2 Build a CLI on discovery and tags - -If you're staying in Python, don't hardcode a type list. Discover, and let the installed -packages decide what exists — same as the built-in CLI. Your tool then automatically -covers new themes and third-party extensions. - -```python -import click -from overture.schema.system.discovery import discover_models, filter_models, TagSelector -from overture.schema.cli.tag_options import tag_selection_options, build_selector - - -@click.command() -@tag_selection_options # gives you --tag / --filter / --exclude for free -def report(tags, filters, excludes): - """Report field counts for the selected feature types.""" - models = filter_models(discover_models(), build_selector(tags, filters, excludes)) - for key, model in sorted(models.items(), key=lambda kv: kv[0].name): - n = len(model.model_fields) if hasattr(model, "model_fields") else "—" - click.echo(f"{key.name:20} {n}") -``` - -`overture.schema.cli` also exports `resolve_types`, `create_union_type_from_models`, -`load_input`, `perform_validation`, `handle_validation_error`, and -`handle_generic_error` — so you can reuse the file-loading and error-rendering behavior -rather than reimplementing it. - -### 8.3 Register your own feature types - -Your models become first-class: discovered by the CLI, accepted by `validate()`, -included in generated docs and JSON Schema. Nothing in the tooling special-cases -Overture. - -```python -# mypkg/models.py -from typing import Literal -from overture.schema.common import OvertureFeature -from overture.schema.system.numeric import float32 - - -class Vineyard(OvertureFeature[Literal["agriculture"], Literal["vineyard"]]): - """A cultivated area planted with grapevines.""" - - area_hectares: float32 | None = None -``` - -```toml -# mypkg/pyproject.toml -[project.entry-points."overture.models"] -vineyard = "mypkg.models:Vineyard" -``` - -Install it, and: - -```bash -overture-schema list-types -overture-schema validate vineyard.json -overture-codegen generate --format markdown --output-dir out -``` - -To attach your own tags, register a **tag provider** on `overture.tag_providers`: - -```python -def experimental_provider(types, key, tags): - if any(getattr(t, "__experimental__", False) for t in types): - tags.add("mypkg:experimental") - return tags -``` - -```toml -[project.entry-points."overture.tag_providers"] -experimental = "mypkg.tags:experimental_provider" -``` - -Tag namespaces are reserved: `feature` and `system:` belong to `overture-schema-system`, -`overture:` to `overture-schema-common`. A provider that tries to set a reserved tag from -an unauthorized package gets a logged warning and the tag is discarded. Use your own -namespace. - -You don't have to build on `OvertureFeature` — subclass `system.Feature` directly for a -GeoJSON-serializing model with none of the Overture conventions. - -### 8.4 Write a new codegen target - -For a format nobody else generates — Arrow schemas, Avro, Go structs, protobuf — add a -renderer to the codegen rather than parsing JSON Schema back out. You get the full -semantic model: NewType names, constraint provenance, discriminated union structure — all -the things JSON Schema flattens away. - -The pipeline is four layers with strictly downward imports: - -``` -Rendering → output formatting, all presentation decisions -Output Layout → what to generate, where it goes, how outputs link -Extraction → FieldShape, FieldSpec, RecordSpec, UnionSpec, EnumSpec -Discovery → discover_models() -``` - -Extraction is target-independent, so a new target is a new renderer, not new extraction -logic. The entry point: - -```python -from overture.schema.codegen.extraction.model_extraction import extract_model -from overture.schema.buildings import Building - -spec = extract_model(Building) -spec.name # 'Building' -spec.description # the class docstring -spec.constraints # model-level constraints - -for f in spec.fields[:6]: - print(f"{f.name:12} required={f.is_required!s:5} {type(f.shape).__name__}") -``` - -``` -id required=True NewTypeShape -bbox required=False Primitive -geometry required=True Primitive -theme required=True LiteralScalar -type required=True LiteralScalar -version required=True NewTypeShape -``` - -`FieldSpec` is `(name, shape, description, is_required, is_optional)`. The `shape` is a -`FieldShape` tree — `NewTypeShape`, `Primitive`, `LiteralScalar`, `ModelRef`, -`UnionRef`, and container variants — with sub-models and sub-unions already resolved. -Constraints carry provenance, so you can tell which NewType contributed which bound: - -```python -from overture.schema.codegen.extraction.type_analyzer import analyze_type -from overture.schema.common.feature import FeatureVersion - -shape, is_nullable, description = analyze_type(FeatureVersion) -# shape → NewTypeShape(name='FeatureVersion', inner=Primitive(base_type='int32', -# constraints=(ConstraintSource(source_name='FeatureVersion', constraint=Ge(ge=0)), ...))) -``` - -`analyze_type` returns a **3-tuple** `(FieldShape, bool, str | None)` — the structural -shape, whether the field accepts `None`, and the first description found while -unwrapping. (The codegen README shows an older `TypeInfo`/`TypeKind` API that no longer -exists.) - -To wire up a new format: add a column to `TypeMapping` in -`extraction/type_registry.py` for type-name resolution, write a pipeline module -consuming `ModelSpec` trees plus a renderer, and register the format in `cli.py`. - -Further reading in the repo: - -- `packages/overture-schema-codegen/docs/design.md` — architecture, data flow, extension points -- `packages/overture-schema-codegen/docs/walkthrough.md` — module-by-module trace of `Segment` through the pipeline - ---- - ---- - -## 9. Registering models and tagging - -How feature types make themselves known to the tooling. This is the mechanism that -lets your own models slot in alongside Overture's — see -[Register your own feature types](#83-register-your-own-feature-types) for a worked example. - -The library is designed to support data producer extensions through multiple patterns. -This extensibility is a core feature that allows organizations to add custom fields and -types while maintaining compatibility with the base Overture schema. We are in the -process of determining how this should work. - -### Model Registration via Entry Points - -Models are registered using [setuptools entry -points](https://setuptools.pypa.io/en/latest/userguide/entry_point.html) in each -package's `pyproject.toml` file. This enables automatic discovery and loading of models -at runtime without requiring explicit imports. - -Registration is done in the `[project.entry-points."overture.models"]` section: - -```toml -[project.entry-points."overture.models"] -building = "overture.schema.buildings:Building" -building_part = "overture.schema.buildings:BuildingPart" -``` - -The discovery system provides programmatic access to registered models: - -```python -from overture.schema.system.discovery import discover_models, get_registered_model - -# Discover all registered models, keyed by ModelKey -all_models = discover_models() - -# Get a specific model by name -building_model = get_registered_model("building") -if building_model: - building = building_model.model_validate(building_data) -``` - -### Tagging - -Each `ModelKey` returned by `discover_models()` carries a `frozenset[str]` of tags -that classify the model orthogonally to its entry-point name -- whether the model -is a `Feature` subclass, which Overture theme it belongs to, which package shipped -it, and so on. Downstream tools (the CLI, codegen, third-party consumers) use tags -to filter the working set without importing every model: - -```python -from overture.schema.system.discovery import ( - TagSelector, - discover_models, - filter_models, -) - -models = discover_models() -# { -# ModelKey(name="building", entry_point="overture.schema.buildings:Building", -# tags=frozenset({"feature", "overture", "overture:theme=buildings"})): Building, -# ModelKey(name="place", entry_point="overture.schema.places:Place", -# tags=frozenset({"feature", "overture", "overture:theme=places"})): Place, -# ... -# } - -buildings = filter_models( - models, - TagSelector(include_any=("overture:theme=buildings",)), -) -``` - -Tags are produced by *tag providers* registered on the `overture.tag_providers` -entry-point group. The `system` and `common` packages ship the built-in providers -(`feature` and `overture:theme=*`); third parties can register their own -to attach custom tags during discovery. See the [`overture-schema-system` -README](packages/overture-schema-system/README.md#tagging) for tag format, -reserved namespaces, and provider authoring. - - ---- - -## 10. Authoring new schema models - -### Quick Start - -#### Essential Imports - -Copy what you need for most models: - -```python -# Basic Python types -from typing import Annotated, Literal -from enum import Enum - -# Pydantic essentials -from pydantic import BaseModel, Field - -# Overture common models -from overture.schema.common import OvertureFeature -from overture.schema.system.geometric import ( - Geometry, - GeometryType, - GeometryTypeConstraint, -) - -# Validation system -from overture.schema.system.field_constraint import UniqueItemsConstraint -from overture.schema.system.model_constraint import no_extra_fields - -# Common types -from overture.schema.system.string import ( - CountryCodeAlpha2, - NoWhitespaceString, - StrippedString, -) -from overture.schema.common.confidence import ConfidenceScore -from overture.schema.system.string import LanguageTag - -# Numeric types (use these instead of int/float) -from overture.schema.system.numeric import ( - int8, - int32, - int64, - uint8, - uint16, - uint32, - float32, - float64, -) -``` - -#### Templates - -Copy-paste starting points for the four shapes you'll write most often — a plain -model, a feature, an enum, and a model with validation constraints — live together in -[Templates and quick reference](#13-templates-and-quick-reference) rather than being -repeated here. - ---- - -### Basic Concepts - -#### Models and Inheritance - -##### What are Pydantic models? - -Pydantic models are Python classes that define data structures and their constraints. Think of them like UML classes with built-in data validation - each model defines what fields are allowed and what types of data they can contain. - -##### Model Base Classes and Inheritance - -**What is a "base class"?** A base class defines common fields and behaviors that other classes can reuse. Think of it like a slide template - you create one layout, then make specific slides that use that structure. - -**What is "inheritance"?** Inheritance means one class automatically gets all the fields and behaviors from another class. If Building inherits from OvertureFeature, it automatically gets all of Feature's fields (like `id`, `geometry`) plus any new fields you add to Building (like `height`). When multiple parent classes have the same field name, Python uses a [specific order](https://docs.python.org/3/tutorial/classes.html#multiple-inheritance) to determine which one takes precedence. - -**@no_extra_fields** - Use for structured data components that should reject unknown fields: - -```python -from overture.schema.system.model_constraint import no_extra_fields - - -@no_extra_fields -class Address(BaseModel): - """A postal address - no extra fields allowed.""" - - street: str - city: str - postal_code: str | None = None - # Any field not defined here will cause validation to fail -``` - -**OvertureFeature[ThemeT, TypeT]** - A generic base class for all geospatial features with typed theme and type parameters: - -```python -from typing import Literal -from overture.schema.common import OvertureFeature -from overture.schema.system.numeric import float64 - - -class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): - """A building feature with strongly-typed theme and type.""" - - # Inherits: id, theme, type, geometry, bbox, version, sources - height: float64 | None = None -``` - -**What does "generic" mean?** The `OvertureFeature[ThemeT, TypeT]` syntax makes OvertureFeature a "generic" class - think of it like a template that can be customized with specific values. The square brackets `[]` contain "type parameters" that specify exactly what theme and type this feature represents. - -**What are ThemeT and TypeT?** These are placeholders for specific text values: - -- **ThemeT**: The data theme (like "buildings", "places", "transportation") -- **TypeT**: The specific feature type within that theme (like "building", "place", "segment") - -**What is `Literal`?** `Literal` means the field must be exactly one of the specified values - nothing else is allowed. So `Literal["buildings"]` means this theme can only be "buildings", not any other string. - -By specifying `OvertureFeature[Literal["buildings"], Literal["building"]]`, you're saying "this is a Feature that must have theme='buildings' and type='building'" - no other values are allowed. This prevents mistakes like accidentally creating a building with theme="places". - -##### Inheritance Patterns - -**Multiple inheritance** combines fields from several base classes: - -```python -from typing import Literal -from overture.schema.common import OvertureFeature -from overture.schema.common.level import Stacked -from overture.schema.common.names import Named -from overture.schema.system.numeric import float64 - - -class Building( - OvertureFeature[Literal["buildings"], Literal["building"]], Named, Stacked -): - # Gets fields from Feature: id, theme, type, geometry, etc. - # Gets fields from Named: names - # Gets fields from Stacked: level - # Plus its own fields: - height: float64 | None = None -``` - -##### Field Aliases - -Sometimes you need a field name that conflicts with Python keywords or conventions (hint: you'll get an error when you try to use it). Use `Field(alias="")` to map between Python-friendly field names and the actual data field names: - -```python -from typing import Annotated -from pydantic import Field - - -class Building(OvertureFeature): - # Use class_ in Python code, but "class" in the actual data - class_: Annotated[str | None, Field(alias="class")] = None - - # Other common cases might include: - type_: Annotated[str | None, Field(alias="type")] = None # if type conflicts - from_: Annotated[str | None, Field(alias="from")] = None # from is a keyword -``` - -A common example is `class_` with `Field(alias="class")` since "class" is a Python keyword but a common field name in data schemas. - -#### Field Types - -##### Required vs Optional Fields - -```python -class Building(OvertureFeature): - # Required field (no default value) - geometry: Geometry - - # Optional field (has default value of None) - height: float64 | None = None -``` - -- **Required fields**: Must be provided when creating an instance -- **Optional fields**: Can be omitted; they have a default value (usually `None`) - -> [!WARNING] -> **Always use `None` defaults** for optional fields. Non-`None` defaults create ambiguity between schema defaults and actual data values. - -**Why do non-`None` defaults cause problems?** - -1. **Data transformation ambiguity**: Pydantic adds default values that weren't in the input, making it impossible to distinguish between original data and schema defaults. - -2. **Schema vs. data confusion**: Schemas serve multiple purposes: - - **Validation only**: Check if existing data is valid (shouldn't transform it) - - **Data processing**: Parse and potentially transform data with Pydantic - - **Documentation**: Show developers what fields exist and what they mean - -3. **Implicit semantic meaning**: Default values encode business logic into the schema, which should be in business logic instead. - -**Better approaches:** - -1. **Use `None` and document semantics:** - - ```python - access_policy: Annotated[ - str | None, - Field(description="Access policy for the place. When absent, assume 'open'"), - ] = None - ``` - -2. **Always populate in data pipeline:** - - ```python - # In your data processing pipeline, always set the value - place.access_policy = place.access_policy or "open" - ``` - -Keep the schema separate from business logic. The schema describes the shape of data, not the business rules about what missing values mean. - -##### Numeric Types - -**Always use specific numeric types instead of Python's generic `int`/`float`:** - -```python -from overture.schema.system.model_constraint import no_extra_fields -from overture.schema.system.numeric import ( - int8, - int32, - int64, # Signed integers - uint8, - uint16, - uint32, # Unsigned integers - float32, - float64, # Floating point -) - - -@no_extra_fields -class MyModel(BaseModel): - # Signed integers with specific ranges - level: int8 | None = None # -128 to 127 - year: int32 | None = None # -2,147,483,648 to 2,147,483,647 - timestamp: int64 | None = None # Full 64-bit range - - # Unsigned integers (0 and positive only) - red_value: uint8 | None = None # 0 to 255 (like RGB values) - port: uint16 | None = None # 0 to 65,535 (like network ports) - population: uint32 | None = None # 0 to 4,294,967,295 +# 4. WRITE — by_alias=True so `class_` is written as `class` +out = b.model_dump(mode="json", by_alias=True, exclude_none=True) +print("4. wrote :", json.dumps(out)[:70], "...") - # Floating point numbers - height: float64 | None = None # Double precision (recommended) - ratio: float32 | None = None # Single precision +# 5. CONFIRM — the output is valid input +again = Building.model_validate_json(json.dumps(out)) +print("5. re-read:", again.height, "m — round-trip holds") ``` -**When to use each:** - -- **`int32`**: Most integer fields (years, counts, IDs) -- **`uint8`**: Small positive values (0-255), like color components, confidence percentages -- **`uint16`**: Medium positive values (0-65K), like ports, small counts -- **`uint32`**: Large positive values, like population, large IDs -- **`float64`**: Most decimal numbers (heights, coordinates, measurements) - **this is the default choice** -- **`float32`**: When space is critical and precision isn't - -When in doubt, use `int32` (equivalent to `int`), `int64` (equivalent to `long`, although [not safely representable in JSON](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER)), or `float64` (equivalent to `double`). - -**Why specific numeric types matter:** - -The specific numeric types are crucial for data interchange and storage compatibility: - -- **Cross-platform consistency**: Ensures the same data types across Python, Arrow, Parquet, and other geospatial tools -- **Round-trip compatibility**: Data round-trips cleanly between Parquet files, databases (PostgreSQL, Trino), Shapefiles, and JSON Schema -- **Value range validation**: Prevents invalid values (e.g., negative heights, RGB values > 255) -- **Storage efficiency**: `uint8` uses 1 byte vs `int64` which uses 8 bytes -- **Built-in validation**: These types use Pydantic `Field()` constraints to validate ranges (e.g., `Field(ge=0, le=100)` ensures values stay within bounds) - -##### Union Types - -Union types allow a field to accept multiple different types. The `|` symbol means "or": - -```python -from typing import Literal - - -class Building(OvertureFeature): - # This field can be either a string OR None (most common union) - name: str | None = None - - # This field can be one of specific string values OR None - status: Literal["active", "inactive", "pending"] | None = None ``` - -**Common union patterns:** - -```python -# Optional field (most common union) -height: float64 | None = None # Can be a number or missing - -# Specific string values (an alternative to enums where descriptions aren't needed) -priority: Literal["low", "medium", "high"] | None = None - -# Boolean or None -is_verified: bool | None = None +1. loaded : overture:buildings:building:1234 +2. read : 21.34 m, 4 floors, class parking +3. changed: 25.0 m, 5 floors +4. wrote : {"type": "Feature", "id": "overture:buildings:building:1234", "geometr ... +5. re-read: 25.0 m — round-trip holds ``` -**Union best practices:** +That is the shape of nearly every job: **validate in, work with plain Python objects, +serialize out.** In between, `b` is an ordinary object — attributes, assignment, no +special API. + +Three lines in there are load-bearing, and each gets a subsection below: -- Keep unions simple - avoid more than 2-3 types when possible -- Optional fields will include `None` to become optional -- Use `Literal` values for specific string choices rather than mixing basic types -- **Avoid mixed-type unions** like `str | int32` - these don't work well with many storage layers +| Line | Why it's written that way | Where | +|---|---|---| +| `model_validate_json(json.dumps(doc))` | the file is GeoJSON, which needs JSON mode | [3.2](#32-the-one-thing-to-understand-two-representations) | +| `b.class_.value` | the field is `class_` in Python, `class` in the data | [3.3](#33-reading-fields) | +| `by_alias=True, exclude_none=True` | without them the output won't re-validate | [3.4](#34-writing-data-back-out) | -> [!WARNING] -> **Storage compatibility**: Mixed-type unions (combining different basic types like `str | int32`) don't work with Parquet and other storage layers. Use `Literal` values or separate fields instead. +If the snippet above ran, you already know enough to be useful. Read on when one of those +lines bites you, or read straight through if you'd rather know why now. -#### Field Enhancement +### 3.2 The one thing to understand: two representations -##### Adding Descriptions and Constraints with Annotated +Overture publishes data in one shape — flat and tabular, the column layout of the +Parquet release. The models also read and write GeoJSON, so the schema works with tools +that expect features rather than rows, and because that is the representation the +generated [JSON Schema](#61-json-schema) describes. **Which one you get depends on the +Pydantic mode you use, not on the data you pass.** -`Annotated` is Python's way to add extra information (metadata) to a type without changing the type itself. Think of it like adding notes or constraints to a field definition. +| Shape | Looks like | Pydantic mode | Validate with | Dump with | +|---|---|---|---|---| +| **GeoJSON** | `id`/`geometry` at top level, everything else under `properties` | `json` | `model_validate_json()` | `model_dump(mode="json")`, `model_dump_json()` | +| **Flat / tabular** (Parquet-style) | every field at the top level | `python` | `model_validate()` | `model_dump(mode="python")` | -**Basic concept:** +This trips people up constantly. Passing a GeoJSON *dict* to `model_validate()` fails, +because `model_validate` is Python mode and Python mode expects the flat shape: ```python -from typing import Annotated -from pydantic import Field +import json, yaml +from overture.schema.buildings import Building -# Without Annotated - just the type -height: float64 | None = None +doc = yaml.safe_load(open("examples/buildings/building-polygon.yaml")) # GeoJSON-shaped -# With Annotated - type + extra information -height: Annotated[ - float64 | None, # The actual type (what kind of data) - Field(description="Height in meters"), # Extra metadata -] = None +Building.model_validate(doc) +# ValidationError: 3 validation errors for building +# theme Field required +# type Input should be 'building' [got 'Feature'] +# version Field required ``` -**What goes inside `Annotated`:** - -1. **First argument**: The actual type (`str`, `int32`, `list[str]`, etc.) -2. **Additional arguments**: Metadata like constraints, descriptions, validation rules - -##### Field Constraints - -Use Pydantic's `Field()` function to add constraints and descriptions: +The `type: Feature` in the error message is the tell: it read the GeoJSON envelope's +`type` as the feature's `type` field. -**Numeric constraints:** +The fix — round-trip through JSON so you're in JSON mode: ```python -from typing import Annotated -from pydantic import Field - - -class Building(OvertureFeature): - # Range constraints - height: Annotated[ - float64 | None, Field(ge=0, le=1000, description="Height in meters (0-1000m)") - ] = None - - # Integer constraints - floors: Annotated[ - int32 | None, Field(gt=0, lt=200, description="Number of floors (1-199)") - ] = None +building = Building.model_validate_json(json.dumps(doc)) # works ``` -**Numeric constraint options:** - -- **`ge`**: Greater than or equal to (≥) -- **`gt`**: Greater than (>) -- **`le`**: Less than or equal to (≤) -- **`lt`**: Less than (<) - -**String constraints:** +Or, if you're already reading from a file or an HTTP response, skip the parse entirely: ```python -class Place(OvertureFeature): - # Length constraints - name: Annotated[ - str | None, - Field(min_length=1, max_length=100, description="Place name (1-100 chars)"), - ] = None - - # Pattern matching - postal_code: Annotated[ - str | None, Field(pattern=r"^\d{5}(-\d{4})?$", description="US postal code") - ] = None +building = Building.model_validate_json(open("building.geojson").read()) ``` -**String constraint options:** - -- **`min_length`**: Minimum string length -- **`max_length`**: Maximum string length -- **`pattern`**: Regular expression pattern (regex) - -#### Collections and Lists +#### Can't I just tell `model_validate` to use JSON mode? -##### Basic List Fields +No. It's the obvious thing to try, and neither knob does it: ```python -class Building(Feature): - # Simple list of strings - tags: list[str] | None = None - - # List of complex objects - access_rules: list[AccessRule] | None = None +Building.model_validate(doc, context={"mode": "json"}) # still ValidationError +Building.model_validate(doc, strict=False) # still ValidationError ``` -##### List Constraints +The mode isn't a setting you pass — in Pydantic it's determined by *which method you +call*. `model_validate` is Python mode; `model_validate_json` is JSON mode. The base +`Feature` class keys off exactly that: ```python -from overture.schema.system.field_constraint import UniqueItemsConstraint - - -class Building(OvertureFeature): - # List with size and uniqueness constraints - categories: Annotated[ - list[str] | None, - Field(min_length=1, max_length=10, description="1-10 categories"), - UniqueItemsConstraint(), # Must come AFTER Field() - ] = None +@model_validator(mode="wrap") +def __validate_with_geo_json_support__(cls, data, handler, info): + if info.mode == "json": # <- set by the method, not by an argument + ... # unpack the GeoJSON envelope ``` -**List constraint options:** - -- **`min_length`**: Minimum number of items -- **`max_length`**: Maximum number of items -- **`UniqueItemsConstraint()`**: No duplicate items (custom validation) - -**Important**: `UniqueItemsConstraint()` must come AFTER `Field()` for proper JSON Schema generation. - -> [!CAUTION] -> **Constraint order matters**: Always put `Field()` before `UniqueItemsConstraint()` or JSON Schema generation will create `minLength` (string constraint) instead of `minItems` (array constraint). -> -> **Why**: Pydantic processes annotations in order for JSON Schema generation. `Field()` must come first to set up the field properly. For lists, `Field(min_length=1)` creates a `minItems` constraint in the JSON Schema because the type immediately before it is a list. If `UniqueItemsConstraint()` comes first, Pydantic doesn't see the list type and treats `min_length` as a string constraint (`minLength`). - -##### List Behavior - -Lists maintain their **insertion order** (the order data exists in the field), but they are **not automatically sorted**. - -#### Enumerations - -**What is an enumeration (enum)?** An enumeration is a way to define a fixed set of allowed values for a field. Think of it like a multiple-choice question - you define all the valid answers ahead of time, and users can only pick from those options. - -For example, instead of allowing any string for a "status" field (which could lead to typos like "activ" or "Active"), you create an enum with exactly "active", "inactive", and "pending" as the only allowed values. - -**Enums vs Literal:** You can achieve similar results with `Literal["active", "inactive", "pending"]`, but formal enums are better when you need descriptions, documentation, or want to reuse the same set of values across multiple fields. - -##### Creating Enums - -Enums define a fixed set of allowed values: +So if you have a GeoJSON **dict** in hand, `json.dumps` it and use +`model_validate_json`. The round-trip is slightly wasteful but it is the supported path: ```python -from enum import Enum - - -class BuildingClass(str, Enum): - """Further delineation of the building's built purpose.""" - - RESIDENTIAL = "residential" - COMMERCIAL = "commercial" - INDUSTRIAL = "industrial" - CIVIC = "civic" - - -# Usage in a model -class Building(OvertureFeature): - class_: Annotated[BuildingClass | None, Field(alias="class")] = None +Building.model_validate_json(json.dumps(doc)) ``` -##### Documenting Enum Values - -Add documentation to describe what the enum and its values mean. In Python, you do this with **docstrings** - text enclosed in triple quotes `"""` that describes what something does: - -Use `DocumentedEnum` from `overture.schema.system.doc` when enum members need their own descriptions for code generation and documentation tooling. Each member takes a `(value, description)` tuple: - -```python -from overture.schema.system.doc import DocumentedEnum +Better still, avoid making the dict at all. If the GeoJSON came from a file or an HTTP +response, hand the raw text straight to `model_validate_json` and skip `json.loads` +entirely. The one case where you genuinely can't is YAML — there's no YAML mode, so +`yaml.safe_load` → `json.dumps` → `model_validate_json` is the route, as in the example +above. +#### Reading the error -class VehicleType(str, DocumentedEnum): - """Types of vehicles for transportation.""" +The three errors from a mode mismatch are always the same shape, and worth recognising on +sight: - CAR = ("car", "Standard passenger vehicle") - TRUCK = ("truck", "Commercial freight vehicle") - BICYCLE = ("bicycle", "Human-powered two-wheeler") - MOTORCYCLE = ("motorcycle", "Motorized two-wheeler") ``` - -Members without descriptions use the plain value form -- documentation is optional per-member: - -```python -class ConnectionState(str, DocumentedEnum): - CONNECTED = "connected" - DISCONNECTED = "disconnected" - QUIESCING = ( - "quiescing", - "Gracefully shutting down, rejecting new requests but completing existing ones", - ) +theme Field required +type Input should be 'building' [input_value='Feature'] +version Field required ``` -Use `DocumentedEnum` over plain `str, Enum` when the enum members' semantics aren't obvious from their names and downstream tools (code generators, documentation renderers) need access to member-level descriptions. Use plain `str, Enum` for self-explanatory values. - -##### Why str, Enum? - -Inheriting from `str, Enum` makes enum values work as both enums and strings, which is useful for JSON serialization and compatibility. - ---- - -### Advanced Patterns - -#### Relationship Patterns - -##### What are relationships? - -Relationships represent connections between different features or models. Think of them like links that connect related pieces of information — for example, a building part that is structurally part of a building, or a division area that is administratively nested under a division. - -Pydantic provides several ways to express these relationships, each suited to different use cases and complexity levels. Before choosing a pattern, it's important to understand the **semantic type** of the relationship you're modeling. - ---- - -##### Semantic Relationship Types - -Every relationship between two features carries a semantic meaning about coupling strength, lifecycle dependency, and ownership. The schema defines four relationship types, ordered from strongest to weakest coupling. The types describe the *nature* of the link, not which feature is "parent" or "child." Direction is implicit: the feature holding the reference is the source, and the type it references is the destination. - -###### `COMPOSITION` — Structural Whole-Part - -A structural whole-part relationship with lifecycle dependency. The part has no independent meaning outside the whole. Deleting the whole invalidates the part. - -**Test question:** *"If I delete the whole, does keeping the part orphaned make any sense at all?"* If the answer is no, it's `COMPOSITION`. - -**Examples:** -- `BuildingPart` → `Building` — part *is part of* building -- `DivisionBoundary` → `Division` — boundary line *defines the boundary of* division - -###### `AGGREGATION` — Grouping Without Lifecycle Dependency - -A grouping or collection relationship where both members are independently viable. No lifecycle dependency — the member survives reassignment to another group or orphaning. - -**Test question:** *"Can both sides belong to something else or nothing and still be a valid map feature?"* If yes, they form an `AGGREGATION`. - -**Examples:** -- `Route` → `Segment` — route *groups* segments -- `TrailSegment` → `NationalPark` — segment *is grouped by* park - -###### `HIERARCHY` — Organizational Nesting - -An organizational or classificatory nesting relationship. This is not about structural assembly — it's about administrative parentage, taxonomy, or categorization. - -**Test question:** *"Is this about organizational subordination rather than structural assembly?"* If yes, it's `HIERARCHY`. - -**Examples:** -- `DivisionArea` → `Division` — area *is child of* division -- `Division` → `Division` — child division nested under parent - -###### `ASSOCIATION` — Peer-Level Reference - -A peer-level reference with no ownership, containment, or nesting. Neither feature depends on or contains the other. This is the fallback when none of the stronger types apply. - -**Test question:** *"Are these just peers that know about each other?"* If yes, it's `ASSOCIATION`. - -**Examples:** -- `Segment` → `Connector` — segment references its start/end connector -- `Building` → `Address` — a building references its address, neither owns the other - ---- - -##### Selection Priority +`theme` and `version` are "missing" because they're really down inside `properties`, where +Python mode isn't looking. And `type` came back as `'Feature'` — the envelope's type, +which is the giveaway. **If you ever see `input_value='Feature'` in a validation error, +you passed GeoJSON to a Python-mode call.** -When a relationship could fit multiple types, the choice follows a **diamond decision**: start at the top, fork in the middle based on the *kind* of coupling, and fall through to the bottom only when no stronger type applies. +For flat data — a Parquet row, a DuckDB result, a dict of columns — `model_validate` is +the right call: -```text - COMPOSITION - / \ -AGGREGATION HIERARCHY - \ / - ASSOCIATION +```python +row = { + "id": "...", + "theme": "buildings", + "type": "building", + "version": 1, + "geometry": ..., + "height": 21.34, + "class": "parking", +} +building = Building.model_validate(row) ``` -| If the relationship implies... | Use | -|---------------------------------------------------------|----------------| -| Structural whole-part with lifecycle dependency | `COMPOSITION` | -| Geometric boundary definition (lifecycle dependent) | `COMPOSITION` | -| Grouping/collection without lifecycle dependency | `AGGREGATION` | -| Organizational nesting or classification tree | `HIERARCHY` | -| Peer-level reference, no ownership or nesting | `ASSOCIATION` | - ---- - -##### The `role` Field - -The `Reference` annotation accepts an optional `role` parameter — a snake_case string that further qualifies the relationship from the source's perspective. It has no effect on schema validation; it is informational metadata for documentation and tooling. +### 3.3 Reading fields -Use `role` when the semantic type alone is ambiguous. For example, multiple `HIERARCHY` references on the same model can be disambiguated: +Model attributes are plain Python. Enums come back as enum members, geometry as a +`Geometry` wrapper around Shapely: ```python -# Without role: two HIERARCHY references to Division — which is which? -parent_division_id: Annotated[Id, Reference(Relationship.HIERARCHY, Division)] -capital_division_ids: Annotated[list[Id], Reference(Relationship.HIERARCHY, Division)] - -# With role: unambiguous -parent_division_id: Annotated[ - Id, Reference(Relationship.HIERARCHY, Division, role="child_of") -] -capital_division_ids: Annotated[ - list[Id], Reference(Relationship.HIERARCHY, Division, role="has_as_capital") -] +building.height # 21.34 +building.class_ # +building.class_.value # 'parking' +building.num_floors # 4 +type(building.geometry) # ``` -The `role` must be a non-empty snake_case string (lowercase letters, digits, underscores). It describes the source's role relative to the target using source-perspective phrasing. - ---- - -##### Implementation Patterns - -###### 1. Direct References (Foreign Keys) +### 3.4 Writing data back out -The fundamental pattern is a direct reference where one feature "points to" another using an ID field with type safety and semantic information. +Some fields are Python keywords, so the model attribute differs from the wire name +(`class_` on the model, `class` in the data). **`model_dump()` uses attribute names by +default**, which produces output that will not validate back: ```python -from typing import Annotated, Literal -from pydantic import Field -from overture.schema.common import OvertureFeature -from overture.schema.system.ref import Id, Reference, Relationship - - -# COMPOSITION — part points to its whole -class BuildingPart(OvertureFeature[Literal["buildings"], Literal["building_part"]]): - """A structural part of a building.""" - - building_id: Annotated[ - Id, - Reference(Relationship.COMPOSITION, Building, role="part_of"), - Field(description="The building to which this part belongs"), - ] - - -# HIERARCHY — child points to parent -class DivisionArea(OvertureFeature[Literal["divisions"], Literal["division_area"]]): - """Area polygon nested under a division.""" - - division_id: Annotated[ - Id, - Reference(Relationship.HIERARCHY, Division, role="child_of"), - Field(description="Division ID of the parent division of this area."), - ] - - -# ASSOCIATION — peer reference, no ownership -class ConnectorReference(BaseModel): - """Reference to a connector feature.""" +d = building.model_dump(mode="json") +sorted(d["properties"]) +# ['class_', 'ext_bar', 'height', ...] ← 'class_' is wrong for the wire - connector_id: Annotated[ - Id, Reference(Relationship.ASSOCIATION, Connector, role="connects_to") - ] +Building.model_validate(building.model_dump(mode="python")) +# ValidationError: invalid extra field name: class_ ``` -###### 2. Association as a Separate Feature (Complex Relationships) - -When the relationship itself needs to store information, create a dedicated feature to represent it. This applies regardless of the semantic type — any of the four types can carry metadata. - -**Simple relationship (use Pattern 1):** -- "Building Part A is part of Building B" — just needs an ID reference. - -**Complex relationship (use Pattern 2):** -- "Admin Area X has City Center Y as its primary center since 2010 with 85% confidence" — the relationship has properties. +With `by_alias=True` both round-trips work: ```python -class AdminCityCenterAssociation( - OvertureFeature[Literal["associations"], Literal["admin_city_center"]] -): - """Describes how an administrative area relates to a city center.""" - - admin_area_id: Annotated[Id, Reference(Relationship.ASSOCIATION, AdminArea)] - city_center_id: Annotated[Id, Reference(Relationship.ASSOCIATION, CityCenter)] +geojson = building.model_dump(mode="json", by_alias=True, exclude_none=True) +sorted(geojson["properties"]) +# ['class', 'ext_bar', 'height', 'is_underground', 'level', 'num_floors', ...] - # Information about the relationship itself - relationship_type: Literal["primary_center", "secondary_center"] = "primary_center" - established_date: str | None = None - confidence_score: Annotated[float64, Field(ge=0.0, le=1.0)] | None = None +Building.model_validate_json(json.dumps(geojson)) # OK +Building.model_validate( + building.model_dump(mode="python", by_alias=True, exclude_none=True) +) # OK ``` -**When to use separate association features:** -- The relationship has properties (confidence scores, dates, types, notes). -- Many-to-many connections exist. -- You need to query the relationships independently. - -###### 3. Collection References - -When a feature needs to reference multiple other features, use a list of references. The semantic type still matters. +Same for `model_dump_json()`: ```python -# COMPOSITION — boundary defines two divisions -class DivisionBoundary( - OvertureFeature[Literal["divisions"], Literal["division_boundary"]] -): - """A boundary line between two divisions.""" - - division_ids: Annotated[ - list[ - Annotated[ - Id, Reference(Relationship.COMPOSITION, Division, role="boundary_of") - ] - ], - Field(min_length=2, max_length=2, description="Left and right divisions"), - ] - - -# AGGREGATION — route groups segments -class Route(OvertureFeature[Literal["transportation"], Literal["route"]]): - """A transportation route passing through multiple segments.""" - - segment_ids: Annotated[ - list[Id], - Reference(Relationship.AGGREGATION, TransportationSegment, role="groups"), - Field(min_length=1, description="Ordered segments in this route"), - UniqueItemsConstraint(), - ] +'"class":' in building.model_dump_json() # False ← emits "class_" +'"class":' in building.model_dump_json(by_alias=True) # True ``` ---- - -##### Best Practices +**Rule of thumb: `by_alias=True` on every dump, unless you specifically want Python +attribute names.** `exclude_none=True` is usually what you want too — otherwise you get +every unset optional field as an explicit `null`. -###### Always Use Reference Annotations +### 3.5 Working with `Segment` and other unions -Include `Reference` annotations for semantic clarity and documentation: +`Segment` is a discriminated union type alias over `RoadSegment`, `RailSegment`, and +`WaterSegment` — not a model class. It has no `model_validate`: ```python -# Good — complete relationship information with semantic type and role -division_id: Annotated[ - Id, - Reference(Relationship.HIERARCHY, Division, role="child_of"), - Field(description="Division ID of the parent division of this area."), -] +from overture.schema.transportation import Segment -# Avoid — missing semantic information -division_id: Id +type(Segment) # +Segment.model_validate({...}) +# AttributeError: model_validate ``` -###### Choose the Right Semantic Type First, Then the Right Pattern - -1. **Determine the semantic type** using the selection priority and test questions above. -2. **Then choose the implementation pattern:** - - Simple relationships → Direct references (Pattern 1) - - Relationships with metadata → Separate association features (Pattern 2) - - One-to-many references → Collection references (Pattern 3) - -#### Discriminated Unions - -**What is a discriminated union?** A discriminated union is a type that can be backed by one of several different models, where a specific field (the "discriminator") determines which model it actually is. Think of it like a form that changes its fields based on a category selection. +Wrap it in a `TypeAdapter`: ```python -from typing import Annotated, Literal -from pydantic import Field -from overture.schema.common import OvertureFeature - - -# Base class with common fields -class TransportationSegment( - OvertureFeature[Literal["transportation"], Literal["segment"]] -): - subtype: Subtype # This is the discriminator field - # ... common fields for all segments - +from pydantic import TypeAdapter +from overture.schema.transportation import Segment -# Specific segment types -class RoadSegment(TransportationSegment): - subtype: Literal[Subtype.ROAD] # Must be "road" - class_: Annotated[RoadClass, Field(alias="class")] - speed_limits: SpeedLimits | None = None - # ... road-specific fields +segments = TypeAdapter(Segment) +seg = segments.validate_json(raw_geojson_string) # JSON mode → GeoJSON +seg = segments.validate_python(flat_row) # Python mode → flat +type(seg).__name__ # 'RoadSegment' +``` -class RailSegment(TransportationSegment): - subtype: Literal[Subtype.RAIL] # Must be "rail" - class_: Annotated[RailClass, Field(alias="class")] - rail_flags: RailFlags | None = None - # ... rail-specific fields +Build the `TypeAdapter` once and reuse it; construction is the expensive part. +The concrete arms *are* ordinary classes if you know which one you want: -# Union type that automatically picks the right model based on subtype -Segment = Annotated[ - RoadSegment | RailSegment | WaterSegment, Field(discriminator="subtype") -] +```python +from overture.schema.transportation import RoadSegment ``` -The `discriminator="subtype"` tells Pydantic to look at the `subtype` field to determine which specific model to use. If `subtype` is "road", it uses `RoadSegment`; if "rail", it uses `RailSegment`. +### 3.6 Validating without knowing the type -##### Abstract vs Concrete Classes +`overture-schema-validation` checks a record against every installed model and returns +whichever matched: -**What's the difference?** In UML and traditional OOP, abstract classes cannot be instantiated - they serve as templates for concrete classes. In Pydantic, by default, **all classes are concrete** (can be instantiated), but you can make classes abstract when needed. +```python +from overture.schema.validation import validate, validate_json -**Current pattern (all concrete):** +feature = validate_json(geojson_string) # JSON mode → GeoJSON shape +type(feature).__name__ # 'Building' -```python -# Both can be instantiated as map features -base_segment = TransportationSegment(subtype=Subtype.ROAD, geometry=...) # Valid -road_segment = RoadSegment(subtype=Subtype.ROAD, geometry=..., class_=...) # Valid +feature = validate(flat_dict) # Python mode → flat shape ``` -**Making the base class abstract:** +Both raise `pydantic.ValidationError` when nothing matches. The same mode rule from +[The one thing to understand first](#32-the-one-thing-to-understand-two-representations) applies: `validate()` is +Python mode and wants flat data, `validate_json()` is JSON mode and wants GeoJSON. -```python -from abc import ABC, abstractmethod -from typing import Annotated, Literal -from pydantic import Field +Which models participate is resolved at runtime by entry-point discovery — install more +theme packages and these functions accept more. +### 3.7 Generating JSON Schema in code -class TransportationSegment( - OvertureFeature[Literal["transportation"], Literal["segment"]], ABC -): - """Abstract base - cannot be instantiated directly.""" +```python +from overture.schema.system.json_schema import json_schema +from overture.schema.buildings import Building +from overture.schema.places import Place - subtype: Subtype # Discriminator field +schema = json_schema(Building) +schema["title"] # 'building' +sorted(schema) # ['$defs', 'additionalProperties', 'description', +# 'properties', 'required', 'title', 'type'] - @abstractmethod - def get_speed_limit(self) -> float: - """Each concrete type must implement this.""" - pass +union = json_schema(Building | Place) # unions work too → anyOf +``` +Use this rather than Pydantic's `model_json_schema()`. The Overture generator treats +`T | None = None` as "omit when unset" instead of Pydantic's "nullable with a null +default", which is what the data actually means. -class RoadSegment(TransportationSegment): - """Concrete class - can be instantiated.""" +--- - subtype: Literal[Subtype.ROAD] - speed_limits: SpeedLimits | None = None +## 4. The three CLIs - def get_speed_limit(self) -> float: - return self.speed_limits.max_speed if self.speed_limits else 50.0 +Installing the workspace gives you three commands, from three different packages. +| Command | Purpose | Full reference | +|---|---|---| +| `overture-schema` | Validate files, emit JSON Schema, list types | [`packages/overture-schema-cli/`](packages/overture-schema-cli/) | +| `overture-codegen` | Generate markdown docs and PySpark expressions | [`overture-schema-codegen/README.md`](packages/overture-schema-codegen/README.md) | +| `overture-validate` | Validate Parquet/S3 data at scale with Spark | [`overture-schema-pyspark/README.md`](packages/overture-schema-pyspark/README.md) | -# Now only concrete classes can be instantiated -# base_segment = TransportationSegment(...) # TypeError: Can't instantiate abstract class -road_segment = RoadSegment(subtype=Subtype.ROAD, geometry=...) # Valid -``` +Prefix each with `uv run` inside the repo, or activate the venv. -**Registration pattern (recommended when working with Overture models):** +**This section shows what each command is for and one working invocation of each.** The +complete option lists live in the package READMEs linked above, which are versioned with +the packages they document. -Instead of making classes abstract, we use **entry point registration** where only specific concrete types are discoverable as map features: +### 4.1 `overture-schema` -```toml -# In packages/overture-schema-theme-transportation/pyproject.toml -[project.entry-points."overture.models"] -connector = "overture.schema.transportation:Connector" -segment = "overture.schema.transportation:Segment" ``` +Usage: overture-schema [OPTIONS] COMMAND [ARGS]... -**Real example:** See [`packages/overture-schema-theme-transportation/src/overture/schema/transportation/segment/__init__.py`](packages/overture-schema-theme-transportation/src/overture/schema/transportation/segment/__init__.py) where: - -- **`Segment`** is a discriminated union: `RoadSegment | RailSegment | WaterSegment` -- **`TransportationSegment`** is the concrete base class that all segment types inherit from -- **Individual segment types** (`RoadSegment`, `RailSegment`, `WaterSegment`) are NOT directly registered +Commands: + json-schema Generate JSON schema for Overture Maps types. + list-types List all available types. + validate Validate Overture Maps data against schemas. +``` -**This registration pattern means:** +All three subcommands take the shared `--tag` / `--filter` / `--exclude` options from +[From the CLI: what types exist?](#23-from-the-cli-what-types-exist). `validate` and `json-schema` also take +`--type NAME` to target one type directly. -1. Only **`Segment`** (the union type) is discoverable as an official map feature -2. The union automatically resolves to the correct concrete type based on the `subtype` field -3. All classes (`TransportationSegment`, `RoadSegment`, etc.) can be reused as base classes for alternate implementations +```bash +# What types do I have? +overture-schema list-types +overture-schema list-types --group-by overture:theme -#### Pattern Properties (Constrained Key-Value Maps) +# Validate +overture-schema validate data.geojson +overture-schema validate - < data.geojson +overture-schema validate --type building data.json +overture-schema validate --tag overture:theme=buildings data.json +overture-schema validate --show-field id data.json -**What are pattern properties?** Pattern properties let you create key-value maps where the keys must follow a specific pattern (like language codes) and values have specific types. +# JSON Schema +overture-schema json-schema > all-types.json +overture-schema json-schema --type building > building.json +overture-schema json-schema --tag overture:theme=buildings > buildings.json +``` -```python -from typing import Annotated -from pydantic import BaseModel, Field - - -@no_extra_fields -class Names(BaseModel): - primary: str - - # Keys (strings) must match a language tag pattern, values are strings - common: ( - Annotated[ - dict[ - # The key type - Annotated[ - str, - Field( - pattern=r"^[a-z]{2,3}(-[A-Z]{2})?$", - description="Language tag (e.g., 'en', 'es-MX')", - ), - ], - str, # The value type - ], - Field(json_schema_extra={"additionalProperties": False}), - ] - | None - ) = None -``` - -**Example data:** +Exit codes: `0` on success, `1` on validation failure — so it drops into CI directly. -```json -{ - "primary": "New York City", - "common": { - "es": "Ciudad de Nueva York", - "fr": "New York", - "zh-CN": "纽约市" - } -} -``` +#### Local files or remote? -The `additionalProperties: False` ensures only keys matching the pattern are allowed when generated JSON Schema is used. +It depends which CLI, and the two answer differently. -#### Nested List Validation +| Command | Remote paths | How | +|---|---|---| +| `overture-schema validate` | **no** | pipe through stdin with `-` | +| `overture-validate` (PySpark) | **yes** | `s3a://` natively, anonymous credentials preconfigured | -**What is nested list validation?** This pattern validates both the outer list and the inner structure of each item, with constraints at multiple levels. +`overture-schema validate` takes a filesystem path only. Hand it a URL and it fails — +note that it even mangles the `//`, because the argument is parsed as a path: -```python -from typing import Annotated -from pydantic import Field +```bash +overture-schema validate https://raw.githubusercontent.com/OvertureMaps/schema/main/examples/buildings/building-polygon.yaml +``` +``` +Error: 'https:/raw.githubusercontent.com/.../building-polygon.yaml' is not a file. +``` -# Each item has its own field validation -@no_extra_fields -class HierarchyItem(BaseModel): - division_id: str - name: str +The fix is the `-` argument, which reads stdin. Anything that can fetch bytes can feed it: +```bash +curl -sSf https://raw.githubusercontent.com/OvertureMaps/schema/main/examples/buildings/building-polygon.yaml \ + | overture-schema validate - +``` -class Division(OvertureFeature): - # Nested list validation: outer list AND inner lists both have length constraints - hierarchies: Annotated[ - list[ # Outer list - Annotated[ - list[HierarchyItem], # Inner list - Field(min_length=1), # Inner list must have at least 1 item - ] - ], - Field(min_length=1), # Outer list must have at least 1 hierarchy - ] +``` +✓ Successfully validated ``` -This creates validation at three levels: +That works for anything on stdin — `aws s3 cp ... -`, a database query, a generator +script, another program's output. The only thing you lose is the filename in the output, +which becomes ``. -1. **Individual items**: Each `HierarchyItem` validates its own fields (`division_id`, `name`) -2. **Inner lists**: Each inner list must have at least 1 item (`min_length=1`) -3. **Outer list**: The `hierarchies` field must have at least 1 inner list (`min_length=1`) +`overture-validate` is the opposite: it's built for remote data. `s3a://` paths are +detected automatically and configured with anonymous credentials, so the public Overture +release bucket needs no setup: -#### Type Aliases for Reusable Patterns +```bash +overture-validate segment s3a://overturemaps-us-west-2/release/2026-07-22.0 +``` -**What are type aliases?** Type aliases let you create custom names for complex or frequently-used types. Think of them like creating shortcuts or nicknames for long type definitions. +> **Don't hardcode a release version.** The bucket keeps only the current release, so any +> version written into a script or a doc stops working at the next publish. Ask the bucket +> instead: +> +> ```bash +> curl -sS "https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/?list-type=2&prefix=release/&delimiter=/" \ +> | tr '<' '\n' | grep -oE 'Prefix>release/[^/]+' | sed 's|Prefix>release/||' | sort -r | head -1 +> ``` +> +> ``` +> 2026-07-22.0 +> ``` +> +> Then use it: +> +> ```bash +> RELEASE=$(curl -sS "https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/?list-type=2&prefix=release/&delimiter=/" \ +> | tr '<' '\n' | grep -oE 'Prefix>release/[^/]+' | sed 's|Prefix>release/||' | sort -r | head -1) +> overture-validate segment "s3a://overturemaps-us-west-2/release/$RELEASE" +> ``` +> +> The version shown in the examples here was current when this was written; treat it as a +> placeholder, not a fact. -**What is `NewType`?** `NewType` creates a distinct type that's based on an existing type but is treated as different for type checking purposes. This helps prevent mistakes like using an email address where you need a country code, or using a person's name where you need an ID - they're all strings, but they have different meanings and shouldn't be interchangeable. -**Note**: `NewType` is primarily useful when working with Pydantic models in Python code (development, testing, certain data processing tasks). It doesn't affect data validation or JSON Schema generation - it's a development tool to catch mistakes before they happen. +That difference is not arbitrary. `overture-schema validate` is for a file you're looking +at — an example, a fixture, one feature you're debugging. `overture-validate` is for a +release: millions of rows, read in parallel by Spark, where "download it first" isn't an +option. -> [!WARNING] -> **Naming conflicts**: Never use the same name for a model class and type alias in the same module - this creates circular references and confusing code. +### 4.2 `overture-codegen` -**Guidelines:** +Two commands: `generate` writes code or docs from the discovered models, `list` shows +what it discovered. `generate` takes `--format markdown` or `--format pyspark`, the +same `--tag`/`--filter`/`--exclude` options as `overture-schema`, and an `--output-dir`. -- **Model classes**: Use noun names (`SourceItem`, `AccessRule`, `GeometricScope`) -- **Type aliases**: Use plural or descriptive names (`Sources`, `AccessRules`, `ConnectivityData`) -- **Avoid using the same names**: Use different names for models and type aliases, even if they're related +```bash +# Markdown reference docs +overture-codegen generate --format markdown --output-dir ./schema-docs +overture-codegen generate --format markdown --tag overture:theme=places --output-dir ./out -```python -from typing import NewType, Annotated -from pydantic import BaseModel, Field +# PySpark validation expressions (this is what `make generate-pyspark` runs) +overture-codegen generate --format pyspark \ + --output-dir packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated \ + --test-output-dir packages/overture-schema-pyspark/tests/generated +``` -# Create distinct types for different kinds of strings -SegmentId = NewType("SegmentId", str) # IDs are strings, but distinct -CountryCode = NewType("CountryCode", str) # Country codes are strings, but distinct +### 4.3 `overture-validate` -# Create aliases for complex field patterns -EmailList = NewType( - "EmailList", - Annotated[list[str], Field(min_length=1, description="List of email addresses")], -) +Validates real data volumes with Spark. Requires the generated expression tree, so run +`make install` or `make generate-pyspark` first. +Takes a feature type and a path: -@no_extra_fields -class Contact(BaseModel): - # Clear, self-documenting field types - id: SegmentId # Can't accidentally use a CountryCode here - country: CountryCode # Can't accidentally use a SegmentId here - emails: EmailList # Reusable validation pattern +```bash +overture-validate building local.parquet +overture-validate segment s3a://overturemaps-us-west-2/release/2026-07-22.0 +overture-validate place data.parquet --count-only +overture-validate segment data.parquet --suppress version:bounds -o violations.parquet ``` -**Why use type aliases?** - -1. **Prevent mistakes**: `SegmentId` and `CountryCode` are both strings, but you can't mix them up when they're created using `NewType` -2. **Reusable patterns**: Define complex field validation once, use it many times -3. **Self-documenting code**: `EmailList` is clearer than `list[str]` -4. **Consistency**: Everyone uses the same validation rules for the same concept +It handles S3A and anonymous credentials for the public Overture bucket automatically, +and expands a release root into the Hive partition path for you. The flags shown above +are the common ones; for the full list — output paths, error-row limits, Spark config, +schema-mismatch and check suppression — see +[`packages/overture-schema-pyspark/README.md`](packages/overture-schema-pyspark/README.md). --- -### Integration Guide - -#### Project Architecture - -##### File Organization +## 5. Validating data -Organize code by scope, and avoid circular imports. +Three tiers, pick by data size. -**Cross-theme shared**: the `overture-schema-common` package. Definitions more than -one theme needs -- `OvertureFeature`, `Names`, `Sources`, the scoping framework. +### 5.1 One file, or a handful — the CLI -**One module per feature type**: at the theme package root, named after the type in -snake_case. +Accepts JSON, YAML, and GeoJSON. A single feature, a JSON array of features, or a +`FeatureCollection` all work: -```text -packages/overture-schema-theme-buildings/src/overture/schema/buildings/ - __init__.py # re-exports the public names, declares __all__ - _common.py # shared by Building and BuildingPart - building.py # Building, BuildingSubtype, BuildingClass - building_part.py # BuildingPart +```bash +overture-schema validate examples/buildings/building-polygon.yaml ``` -The module owns everything specific to its type: the `Feature` subclass, its enums, -its NewTypes, and its supporting models. `building.py` defines `BuildingSubtype` and -`BuildingClass` next to `Building`, because nothing else uses them. - -**Theme-level shared**: `_common.py` at the theme package root, for definitions two -or more types in the theme need. `buildings/_common.py` holds `Appearance`, -`RoofShape`, and the material enums that both `Building` and `BuildingPart` use. The -leading underscore marks the module private -- the theme's `__init__.py` re-exports -the public names from it. - -**A type large enough to split**: a subpackage named after the type, applying the -same rules one level down. - -```text -packages/overture-schema-theme-transportation/src/overture/schema/transportation/ - __init__.py # re-exports from connector and segment - connector.py # Connector - segment/ - __init__.py # assembles the Segment discriminated union, re-exports - _common.py # TransportationSegment and what the arms share - road.py # RoadSegment and its supporting types - rail.py # RailSegment and its supporting types - water.py # WaterSegment +``` +✓ Successfully validated examples/buildings/building-polygon.yaml ``` -Every `__init__.py` re-exports its public names and declares `__all__`, so consumers -import from the package rather than reaching into the defining module: -`from overture.schema.buildings import Building`, not -`from overture.schema.buildings.building import Building`. Entry points name the -package root for the same reason -- `building = "overture.schema.buildings:Building"`. +Failures come back as a rendered table showing the offending value in context: -There is no `models.py` / `enums.py` / `types.py` split. An enum lives in the module -whose type uses it, and moves up to `_common.py` when a second type needs it. +```bash +overture-schema validate --show-field id counterexamples/buildings/negative-height.json +``` -##### Import Organization +``` + ─ Validation Failed id=foo ──────────────────────────────────────────────────── + ... + id "foo" + version 0 + height -1.23 ← Input should be greater than 0 + ────────────────────────────────────────────────────────────────────────────── +``` -```python -# Standard library imports first -from enum import Enum -from typing import Annotated, Literal, NewType +For a collection, errors are indexed and labeled by the model that best fit: -# Third-party imports -from pydantic import BaseModel, ConfigDict, Field +``` + ─ [1] (Building) ────────────────────────────────────────────────────────────── + ... + version 0 + height -1.23 ← Input should be greater than 0 + ────────────────────────────────────────────────────────────────────────────── +``` -# Cross-theme and system imports -from overture.schema.common.scoping import Heading, Scope, scoped -from overture.schema.system.doc import DocumentedEnum -from overture.schema.system.field_constraint import UniqueItemsConstraint -from overture.schema.system.model_constraint import no_extra_fields +Narrow the candidate set to sharpen the error messages — with `--type building` the CLI +stops guessing which model you meant: -# Local imports last -- siblings in the theme, then the module's own package -from ..connector import Connector -from ._common import SegmentSubtype, TransportationSegment +```bash +overture-schema validate --type building data.json ``` -`uv run ruff format ` will sort your imports in this order automatically. - -##### Why Not Use @field_validator or @model_validator? +Every check comes from the model definition — nothing is hand-written per file. Copy an +example, edit a field, and you can watch each kind fire: -This project uses a custom validation system that generates better JSON Schema output and supports code generation (without additional work, `@field_validator` and `@model_validator` don't make their constraints discoverable). Always use constraints from `overture.schema.system` instead of using Pydantic validation decorators: +| Edit | What validation says | +|---|---| +| `class: parking` → `class: skyscraper` | `Input should be 'agricultural', 'allotment_house', ...` | +| `num_floors: 4` → `num_floors: 4.7` | `Input should be a valid integer, got a number with a fractional part` | +| delete the `theme:` line | `Ambiguous: Data matches multiple types equally` | -```python -# Don't do this -@field_validator("categories") -def validate_categories_unique(cls, v): - if v and len(v) != len(set(v)): - raise ValueError("Categories must be unique") - return v +What it does **not** catch: free-form string fields accept any string, and nothing checks +fields against each other. Validation enforces the schema, not correctness. +### 5.2 In a Python pipeline -# Do this instead -from overture.schema.system.field_constraint import UniqueItemsConstraint +```python +from pydantic import ValidationError +from overture.schema.validation import validate_json +ok, bad = 0, [] +for line in open("features.ndjson"): + try: + validate_json(line) + ok += 1 + except ValidationError as e: + bad.append((line[:60], e.errors())) -class Building(OvertureFeature): - categories: Annotated[ - list[str] | None, - Field(min_length=1, description="Building categories"), - UniqueItemsConstraint(), - ] = None +print(f"{ok} valid, {len(bad)} invalid") ``` -#### Migrating from JSON Schema - -If you're familiar with JSON Schema files (like `schema/schema.yaml`), this section helps translate those patterns to Pydantic models. - -##### How $defs and $ref Translate +`e.errors()` gives you structured dicts with `loc`, `msg`, `type`, and `input` — the +right thing to log or turn into a report. Package reference: +[`packages/overture-schema-validation/README.md`](packages/overture-schema-validation/README.md). -**JSON Schema approach:** +If you know the type, validate against it directly for better errors and speed: -```yaml -# In defs.yaml -"$defs": - propertyDefinitions: - address: - type: object - properties: - freeform: { type: string } - locality: { type: string } +```python +from overture.schema.buildings import Building -# In building.yaml -properties: - address: { "$ref": "../defs.yaml#/$defs/propertyDefinitions/address" } +Building.model_validate_json(line) ``` -**Pydantic approach:** +### 5.3 At scale — PySpark ```python -# In overture-schema-theme-addresses/src/overture/schema/addresses/address.py -@no_extra_fields -class Address(BaseModel): - """A postal address.""" +from pyspark.sql import SparkSession +from overture.schema.pyspark import validate_model, explain_errors - freeform: str | None = None - locality: str | None = None +spark = SparkSession.builder.getOrCreate() +df = spark.read.parquet("s3a://.../theme=buildings/type=building/") + +result = validate_model(df, "building") +result.evaluated.cache() +total = result.evaluated.count() +errors = result.error_rows().count() +print(f"{errors} / {total} rows with errors") -# In overture-schema-theme-buildings/src/overture/schema/buildings/building.py -class Building(OvertureFeature): - address: Address | None = None +if errors: + violations = explain_errors(result.evaluated, result.checks) + violations.select("id", "field", "check", "message").show(truncate=False) ``` -**Primary differences:** - -- JSON Schema uses `$ref` to reference definitions; Pydantic uses direct Python imports -- JSON Schema definitions live in `$defs`; Pydantic models are regular Python classes grouped into modules -- JSON Schema allows inline definitions; Pydantic encourages separate model classes +`validate_model` accepts either the short name (`"building"`) or the full entry-point key +(`"overture.schema.buildings:Building"`). It looks up the feature type in the registry, +compares the DataFrame schema against the expected one, and evaluates every check in a +single pass — no per-row Python, so it scales. -##### How Containers Work +| Function | Returns | Purpose | +|---|---|---| +| `validate_model(df, type)` | `ValidationResult` | Registry lookup, schema comparison, check evaluation | +| `result.error_rows()` | `DataFrame` | Rows with at least one violation | +| `explain_errors(evaluated, checks)` | `DataFrame` | One row per violation: `field`, `check`, `message` | +| `model_names()` | `list[str]` | Available type names | -**JSON Schema containers** (like `namesContainer`, `shapeContainer`) are reusable property groups: +Tuning, partition handling, and the rest of the PySpark API are documented in +[`packages/overture-schema-pyspark/README.md`](packages/overture-schema-pyspark/README.md). +If `model_names()` comes back empty, see +[Troubleshooting](TROUBLESHOOTING.md#model_names-returns--or-keyerror-on-a-feature-type). -```yaml -# In defs.yaml -propertyContainers: - namesContainer: - properties: - names: { "$ref": "#/$defs/propertyDefinitions/allNames" } +### 5.4 Validating the schema itself - shapeContainer: - properties: - height: { type: number } - num_floors: { type: integer } +If you're changing the models rather than the data: -# In building.yaml -allOf: - - "$ref": ../defs.yaml#/$defs/propertyContainers/namesContainer - - "$ref": ./defs.yaml#/$defs/propertyContainers/shapeContainer +```bash +make check +make test +make update-baselines ``` -**Pydantic equivalent** uses **mixin classes**: +The theme packages carry golden-file baseline tests of their generated JSON Schema, so +unintended schema drift fails CI. After an intentional change, run `make +update-baselines` and inspect the `git diff` on the regenerated golden files before +committing. -```python -# namesContainer -> Named, in -# overture-schema-common/src/overture/schema/common/names.py -class Named(BaseModel): - """Properties defining the names of a feature.""" +--- - names: Names | None = None +## 6. Converting the schema to other formats +Three built-in targets, plus everything reachable through JSON Schema. -# shapeContainer -> Appearance, in buildings/_common.py, -# shared by Building and BuildingPart -class Appearance(BaseModel): - """Physical and visual properties of a building.""" +| Target | Command | Output | +|---|---|---| +| JSON Schema | `overture-schema json-schema` | A JSON Schema document on stdout | +| Markdown | `overture-codegen generate --format markdown` | Docusaurus-ready reference pages | +| PySpark | `overture-codegen generate --format pyspark` | Python modules of `Check` builders + `StructType` | +| Spark `StructType` | (Python, via the pyspark registry) | A live Spark schema object | - height: float64 | None = None - num_floors: int32 | None = None - # ... roof and facade fields +### 6.1 JSON Schema +The interop format — this is your bridge to every other ecosystem. -# Usage with multiple inheritance -- this is how Building is actually declared -class Building( - OvertureFeature[Literal["buildings"], Literal["building"]], - Named, - Stacked, - Appearance, -): ... # inherits names, level, height, num_floors, and the rest from its parents -``` +```bash +# One type +overture-schema json-schema --type building > building.schema.json -JSON Schema containers become **mixin classes** in Pydantic that you inherit from. +# One theme +overture-schema json-schema --tag overture:theme=transportation > transportation.schema.json -##### Common Translation Patterns +# Everything (an `anyOf` over all installed types) +overture-schema json-schema > overture.schema.json +``` -| JSON Schema | Pydantic | Notes | -|-------------|----------|-------| -| `"$ref": "other.yaml#/path"` | `from other import Model` | Direct Python imports | -| `allOf: [ref1, ref2]` | `class Model(Base1, Base2)` | Multiple inheritance | -| `minLength: 1` | `Field(min_length=1)` | Field constraints | -| `minimum: 0, maximum: 100` | `Field(ge=0, le=100)` | Numeric ranges | -| `uniqueItems: true` | `UniqueItemsConstraint()` | Custom constraint | -| `enum: [a, b, c]` | `class E(str, Enum): A="a"` | Enum class | -| `type: ["string", "null"]` | `str \| None = None` | Optional types | -| `if/then` conditional | Custom validation constraints | Model constraints | +A single type produces a self-contained document with its dependencies inlined under +`$defs`: ---- +``` +$ jq 'keys' building.schema.json +["$defs", "additionalProperties", "description", "properties", "required", "title", "type"] +$ jq '.title, (.["$defs"] | length)' building.schema.json +"building" +13 +``` ---- +All types produce `{"anyOf": [...], "$defs": {...}}`. -## 11. Development workflow +In Python: +```python +from overture.schema.system.json_schema import json_schema +from overture.schema.buildings import Building -This project uses [uv](https://docs.astral.sh/uv/) for dependency management: +schema = json_schema(Building) +``` -```bash -# Install dependencies for the entire workspace -uv sync --all-packages +### 6.2 Markdown -# Run all tests and type/code quality checks -make check +Covered in [Generate browsable reference docs](#28-generate-browsable-reference-docs). Output is Docusaurus-flavored +(frontmatter plus `_category_.json` files), but it's plain markdown underneath and reads +fine in any editor or static site generator. -# Run tests for a specific package -uv run pytest packages/overture-schema-theme-buildings/ +### 6.3 PySpark expressions and Spark schemas -# Run tests matching a pattern -uv run pytest -k "buildings" +```bash +overture-codegen generate --format pyspark --output-dir ./ps --test-output-dir ./ps-tests ``` -Auto-format / fix code to align with project expectations: +You get one module per feature type, mirroring the Python package layout: -```shell -uv run ruff check --fix -uv run ruff format -uv run docformatter --in-place --recursive packages/ +``` +ps/overture/schema/buildings/building.py +ps/overture/schema/buildings/building_part.py +ps-tests/overture/schema/buildings/test_building.py ``` ---- +Each module is auto-generated (`# Do not edit`) and contains one builder function per +constraint, returning a `Check` with an unevaluated PySpark `Column`: -# Part III — Reference +```python +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0), + shape=CheckShape.SCALAR, + root_field="version", + ) +``` -## 12. Gotchas +Plus a `MODEL_VALIDATION` constant pairing the checks with the expected `StructType`. -Things that cost time if you don't know them. +**To get a Spark schema for a feature type** — useful for `spark.read.schema(...)`, +Delta table creation, or comparing against your own tables: -| Gotcha | What happens | Fix | -|---|---|---| -| `model_validate(geojson_dict)` | `ValidationError`: missing `theme`/`version`, `type` is `'Feature'` | Use `model_validate_json()`. Python mode expects flat data, JSON mode expects GeoJSON. | -| `model_dump()` without `by_alias` | Emits `class_`, not `class`; output won't re-validate | Always `by_alias=True` | -| `Segment.model_validate(...)` | `AttributeError` — it's a union alias, not a class | `TypeAdapter(Segment).validate_json(...)` | -| `model_names()` returns `[]` | PySpark expressions are generated, not committed | `make generate-pyspark` (or `make install`) | -| Dumps full of `null` | Unset optionals serialize explicitly | `exclude_none=True` | -| Absent list → `[]` → won't re-validate | An omitted optional list defaults to `[]` on the model, dumps as `[]`, then fails a `min_length` check on the way back in | `exclude_defaults=True`, or drop empty lists before re-validating | +```python +from overture.schema.pyspark._registry import REGISTRY +from overture.schema.pyspark.validate import resolve_entry_point_key -Asymmetric round-trip, worth knowing about: an optional list that is simply *absent* -from the input becomes an empty list on the model, and an empty list is not the same as -absent on the way back out. +key = resolve_entry_point_key("building", REGISTRY) +struct = REGISTRY[key].schema -```python -segments = TypeAdapter(Segment) -seg = segments.validate_json( - open("road-indoors.yaml-as-json").read() -) # no `connectors` key -seg.connectors # [] ← not None +type(struct).__name__ # 'StructType' +[(f.name, f.dataType.simpleString()) for f in struct.fields][:5] +``` -flat = seg.model_dump(mode="python", by_alias=True, exclude_none=True) -flat["connectors"] # [] ← exclude_none doesn't drop it -segments.validate_python(flat) -# ValidationError: road.connectors -# List should have at least 2 items after validation, not 0 +``` +[('id', 'string'), + ('bbox', 'struct'), + ('geometry', 'binary'), + ('theme', 'string'), + ('type', 'string')] ``` -The same document validates fine on the way in (the CLI accepts it) and fails on the way -back. `exclude_defaults=True` avoids it, as does pruning empty lists before re-validating. +`REGISTRY` is keyed by the full entry-point string +(`"overture.schema.buildings:Building"`), which is why the `resolve_entry_point_key` +step is there — it accepts the short alias too. `ModelValidation` exposes `.schema`, +`.checks`, and `.geometry_types`. -### Docs in the repo that are currently wrong +Since `StructType` has `.json()` and `.jsonValue()`, this is also your route to an +Arrow/Parquet schema. -- **`pip install overture-schema`** — in every package README. Nothing is on PyPI yet; - see [7.3](#73-what-changes-once-these-packages-are-published). Every other item that - stood here has been fixed, and `tests/test_documented_imports.py` now imports every - `overture.*` statement in every tracked Markdown file, so a broken one fails the suite - rather than accumulating here. +For the generator's own architecture and programmatic API — the shape extraction layer +these modules are rendered from — see +[`packages/overture-schema-codegen/README.md`](packages/overture-schema-codegen/README.md). +To write a new output format, see [AUTHORING.md](AUTHORING.md#write-a-new-codegen-target). --- -## 13. Templates and quick reference +## 7. Using the packages from your own project -### Reference +Everything above assumes you're working *inside* the schema repo. If instead you're +building your own application that depends on these models, you don't use the workspace +at all — you point `uv` at the package directories on disk. -#### Complete Templates +Depend on the **theme packages you actually need**, not the workspace root: -##### Basic Model Template +```toml +# myapp/pyproject.toml +[project] +name = "myapp" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [ + "overture-schema-theme-buildings", + "overture-schema-cli", +] -```python -from typing import Annotated -from pydantic import BaseModel, Field -from overture.schema.system.model_constraint import no_extra_fields -from overture.schema.system.numeric import int8, float64 +[tool.uv.sources] +overture-schema-theme-buildings = { path = "/path/to/schema/packages/overture-schema-theme-buildings", editable = true } +overture-schema-cli = { path = "/path/to/schema/packages/overture-schema-cli", editable = true } +``` +```bash +cd myapp +uv sync +uv run python -c "from overture.schema.buildings import Building; print(Building.__name__)" +``` +``` +Building +``` -@no_extra_fields -class MyCustomType(BaseModel): - """Brief description of what this represents.""" +`editable = true` means edits in your schema clone take effect immediately in `myapp` — +useful if you're changing both. - # Required fields (no default value) - name: str - category: str +### 7.1 The payoff: install set = runtime set - # Optional fields (with default values) - description: str | None = None +In the `myapp` project above, only the buildings theme is installed. So: - # Field with constraints and description - priority: Annotated[ - int8 | None, - Field( - ge=1, le=10, description="Priority level from 1 (lowest) to 10 (highest)" - ), - ] = None +```bash +cd myapp && uv run overture-schema list-types +``` +``` +building feature overture:theme=buildings +building_part feature overture:theme=buildings ``` -##### Feature Template - -```python -from typing import Annotated, Literal -from pydantic import Field -from overture.schema.common import OvertureFeature -from overture.schema.system.geometric import ( - Geometry, - GeometryType, - GeometryTypeConstraint, -) +Two types, not fifteen. **The same CLI binary, scoped by what's installed.** Nothing was +configured to make that happen — the models register themselves through entry points, and +the tooling discovers whatever is present. +This is worth internalizing early, because it's the whole extension story: add your own +package that registers models, and your feature types appear in `list-types`, validate +through the same commands, and show up in generated docs, alongside Overture's. See +[Register your own feature types](AUTHORING.md#register-your-own-feature-types). -class MyFeature(OvertureFeature[Literal["my_theme"], Literal["my_type"]]): - """Description of what this feature represents.""" +### 7.2 If you'd rather have everything - # Geometry with constraints - geometry: Annotated[ - Geometry, - GeometryTypeConstraint(GeometryType.POINT), - Field(description="Location of this feature"), - ] +`overture-schema` is a metapackage depending on all six themes plus validation and the +CLI: - # Custom fields - my_field: str | None = None +```toml +[tool.uv.sources] +overture-schema = { path = "/path/to/schema/packages/overture-schema", editable = true } ``` -##### Enum Template +One caveat: `import overture.schema` gives you nothing directly. It's a namespace root +that ships only a `py.typed` marker — no models, no functions. Always import from the +theme packages: ```python -from enum import Enum +from overture.schema.buildings import Building # ✓ +from overture.schema import Building # ✗ ImportError +``` -class MyEnum(str, Enum): - """Description of what this enum represents.""" +--- - VALUE_ONE = "value_one" - VALUE_TWO = "value_two" - VALUE_THREE = "value_three" -``` +### 7.3 What changes once these packages are published -##### Model with Validation Constraints +Some of this section is scaffolding for the fact that nothing is on a package index yet. +Worth knowing which parts, so you don't over-invest in learning them. -```python -from typing import Annotated -from pydantic import BaseModel, Field -from overture.schema.system.field_constraint import UniqueItemsConstraint -from overture.schema.system.model_constraint import no_extra_fields +| Part of this section | After publishing | +|---|---| +| What a workspace is, the shared `.venv`, `uv run` | **Stays** — but becomes reading for contributors only | +| `git clone` + `uv sync --all-packages` | **Stays** — contributors only | +| `make generate-pyspark` | **Gone for consumers.** Published wheels ship the generated expressions already. | +| The `SPARK_HOME` fix | **Stays forever.** It's an environment problem, unrelated to packaging. | +| Empty registry, `exclude-newer` warning | Contributors only | +| The `[tool.uv.sources]` path blocks above | **Deleted entirely.** This is the pure workaround. | +| "Install set = runtime set" | **Stays.** That's entry-point discovery, not packaging. | +The whole of this subsection collapses to one line: -@no_extra_fields -class Contact(BaseModel): - """Contact information with validation constraints.""" +```bash +uv add overture-schema-theme-buildings overture-schema-cli +``` - name: str - email: str | None = None - phone: str | None = None +or, for everything: - # List with constraints - tags: Annotated[ - list[str] | None, - Field(min_length=1, description="Contact tags"), - UniqueItemsConstraint(), # No duplicate tags - ] = None +```bash +uv add overture-schema ``` -##### Association Feature Template +**On the PySpark step specifically:** the release pipeline already handles it. +`.github/workflows/publish-python-packages.yaml` runs `make generate-pyspark` before +`uv build`, and aborts the release if the resulting wheel contains no +`expressions/generated/*.py`. Its own comment explains why: -```python -from typing import Annotated, Literal -from pydantic import Field -from overture.schema.common import OvertureFeature -from overture.schema.system.numeric import float64 -from overture.schema.system.ref import Id, Reference, Relationship +> the tree must be generated here before the build, or the published wheel ships without +> its `expressions/generated/` modules and `validate_model()` discovers nothing to run +So consumers of a published wheel never run that step. -class MyAssociation( - OvertureFeature[Literal["associations"], Literal["my_association"]] -): - """Represents a relationship between two features with metadata.""" +**Which index?** The publish workflow currently pushes to AWS CodeArtifact (the +`overture-pypi` domain), even though its step names say PyPI. `CONTRIBUTING.md` describes +version bumps reaching public PyPI while interim builds stay internal. Either way the +consumer instruction is a one-line install — only the index URL differs. - # References to the associated features - # Relationship takes a *kind* (COMPOSITION / AGGREGATION / HIERARCHY / - # ASSOCIATION); what the reference means is carried by `role`. Two - # references to related features need distinct roles to stay unambiguous. - feature_a_id: Annotated[ - Id, - Reference(Relationship.ASSOCIATION, FeatureA, role="connects_from"), - Field(description="First feature in the relationship"), - ] +Nothing in sections 2 through 7 changes. The behavior you learn there — entry-point +discovery, the two representations, `by_alias`, `TypeAdapter` for `Segment` — is independent +of how the packages get onto your machine. - feature_b_id: Annotated[ - Id, - Reference(Relationship.ASSOCIATION, FeatureB, role="connects_to"), - Field(description="Second feature in the relationship"), - ] +--- - # Relationship metadata - relationship_type: Literal["primary", "secondary"] = "primary" - confidence: Annotated[float64 | None, Field(ge=0.0, le=1.0)] = None +## 8. Building tools on the models - # Optional contextual information - notes: str | None = None -``` +Sections 1–7 use the schema. This one builds *on* it: generating a client library in +another language, or writing your own command-line tool that stays correct as the +installed packages change. Neither requires touching the schema itself — for that, see +[AUTHORING.md](AUTHORING.md). -#### Quick Reference +### 8.1 Generate an SDK from JSON Schema (any language) -##### Essential Patterns (Most Common) +Emit JSON Schema, then hand it to a standard generator. Both of these were run against +the output of `overture-schema json-schema --type building` and produced working code. -```python -# Basic field types -name: str # Required string -name: str | None = None # Optional string -count: int32 # Required integer -priority: Literal["high", "medium", "low"] | None = None # Constrained values +**Python (datamodel-code-generator):** -# Validated fields -height: Annotated[float64 | None, Field(ge=0, description="Height in meters")] = None -tags: Annotated[list[str] | None, Field(min_length=1), UniqueItemsConstraint()] = None +```bash +uv run overture-schema json-schema --type building > building.schema.json -# Association patterns -- Relationship is the kind, role is the meaning -parent_id: Annotated[ - Id | None, - Reference(Relationship.HIERARCHY, ParentModel, role="child_of"), -] = None -connector_ids: list[Id] # References to multiple related features +uvx --from datamodel-code-generator datamodel-codegen \ + --input building.schema.json \ + --input-file-type jsonschema \ + --output building_models.py ``` -##### Model Templates +Produces standalone Pydantic v2 models with no Overture dependency — useful for a +service that shouldn't take the whole workspace as a dependency: ```python -# Non-feature model -@no_extra_fields -class Address(BaseModel): - street: str - city: str | None = None +# generated by datamodel-codegen: +# filename: building.schema.json +from pydantic import BaseModel, ConfigDict, Field, confloat, conint, constr +``` +**TypeScript (quicktype):** -# Feature model -class Building(OvertureFeature[Literal["buildings"], Literal["building"]]): - geometry: Geometry - height: float64 | None = None +```bash +npx -y quicktype --src-lang schema --lang typescript \ + -o Building.ts building.schema.json +``` +Field descriptions survive as JSDoc: -# Enum -class Status(str, Enum): - ACTIVE = "active" - INACTIVE = "inactive" +```typescript +/** + * Buildings are man-made structures with roofs that exist permanently in one place. + * ... + */ +export interface Building { + bbox?: [number, number, number, number, ...number[]]; + /** The building's footprint or roofprint... */ + geometry: Geometry; ``` -##### Constraint Reference +quicktype also targets Go, Rust, Java, Kotlin, Swift, C#, and others from the same input. +Anything that reads JSON Schema — OpenAPI toolchains, `go-jsonschema`, `schemars`, +`jsonschema2pojo` — works the same way. + +The tradeoff: you get types and structural validation, but not the semantic layer. +Cross-field model constraints (`@require_any_of`, `@radio_group`) do translate into JSON +Schema `if`/`then`/`anyOf` constructs, but domain-specific error messages and the +NewType vocabulary flatten out. -| Type | Constraint | JSON Schema | Example | -|------|------------|-------------|---------| -| **Numeric** | `ge=0, le=100` | `minimum`, `maximum` | `Field(ge=0, le=100)` | -| **String** | `min_length=1, pattern=r"..."` | `minLength`, `pattern` | `Field(min_length=1, pattern=r"^[A-Z]+$")` | -| **List** | `min_length=1, UniqueItemsConstraint()` | `minItems`, `uniqueItems` | `Field(min_length=1), UniqueItemsConstraint()` | -| **Custom** | `LanguageTagConstraint()` | Custom validation | `LanguageTagConstraint()` | +### 8.2 Build a CLI on discovery and tags -##### Import Cheatsheet +If you're staying in Python, don't hardcode a type list. Discover, and let the installed +packages decide what exists — same as the built-in CLI. Your tool then automatically +covers new themes and third-party extensions. ```python -# Essential imports for most models -from typing import Annotated, Literal -from enum import Enum -from pydantic import Field -from overture.schema.common import OvertureFeature -from overture.schema.system.field_constraint import UniqueItemsConstraint -from overture.schema.system.model_constraint import no_extra_fields -from overture.schema.system.numeric import int32, float64 +import click +from overture.schema.system.discovery import discover_models, filter_models, TagSelector +from overture.schema.cli.tag_options import tag_selection_options, build_selector -# For associations and references -from overture.schema.system.ref import Id, Reference, Relationship -``` -##### Naming Conventions +@click.command() +@tag_selection_options # gives you --tag / --filter / --exclude for free +def report(tags, filters, excludes): + """Report field counts for the selected feature types.""" + models = filter_models(discover_models(), build_selector(tags, filters, excludes)) + for key, model in sorted(models.items(), key=lambda kv: kv[0].name): + n = len(model.model_fields) if hasattr(model, "model_fields") else "—" + click.echo(f"{key.name:20} {n}") +``` -- **Classes**: `PascalCase` (`Building`, `AccessRule`) -- **Fields**: `snake_case` (`construction_year`, `has_parts`) -- **Enums**: `UPPER_SNAKE_CASE = "value"` (`ACTIVE = "active"`) +`overture.schema.cli` also exports `resolve_types`, `create_union_type_from_models`, +`load_input`, `perform_validation`, `handle_validation_error`, and +`handle_generic_error` — so you can reuse the file-loading and error-rendering behavior +rather than reimplementing it. diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md new file mode 100644 index 000000000..62048e875 --- /dev/null +++ b/TROUBLESHOOTING.md @@ -0,0 +1,370 @@ +# Troubleshooting + +Symptom-first. Find the error you actually saw, read that entry, and skip the rest. +**Nothing on this page is setup you need to perform.** If your commands run cleanly, +you do not need this page at all. + +The page has two halves. Most of it is **errors** — something failed and printed a +message you can search for. The last section is **gotchas**: cases where nothing fails, +and you get wrong output instead of an error. Those are worth reading once before you +trust a round-trip. + +| Symptom | Section | +|---|---| +| `ModuleNotFoundError: No module named 'overture'` | [Wrong interpreter](#modulenotfounderror-no-module-named-overture) | +| `command not found: overture-schema` | [Missing `uv run`](#command-not-found-overture-schema) | +| A wall of `exclude-newer` warning text, and a dirty `uv.lock` | [uv is out of date](#uv-warns-about-exclude-newer--and-quietly-rewrites-your-lockfile) | +| `FileNotFoundError: ... spark-submit` | [Stale `SPARK_HOME`](#filenotfounderror-on-spark-submit) | +| `model_names()` returns `[]`, or `KeyError` on a feature type | [PySpark expressions not generated](#model_names-returns--or-keyerror-on-a-feature-type) | +| `wc: #: open: No such file or directory` when pasting | [zsh and `#`](#-is-not-a-comment-in-interactive-zsh) | +| A pasted command turns into something from your shell history | [zsh and `!`](#-runs-a-command-out-of-your-history) | +| `ValidationError` you didn't expect from a model | [Model gotchas](#model-gotchas) | + +--- + +## Install and environment + +### `ModuleNotFoundError: No module named 'overture'` + +You started Python without `uv run`, so you're in your system or `pyenv` interpreter +rather than the project's `.venv`. Use `uv run python` instead of `python`. See +[Running Python against the models](SCHEMA_GUIDE.md#21-running-python-against-the-models). + +### `command not found: overture-schema` + +You're missing the `uv run` prefix, or you're not in the repo root. Every command in this +guide is `uv run overture-schema …`, run from the directory containing `pyproject.toml`. + +### uv warns about `exclude-newer` — and quietly rewrites your lockfile + +``` +warning: Failed to parse `pyproject.toml` during settings discovery: + TOML parse error at line 10, column 17 + | + 10 | exclude-newer = "1 week" + | ^^^^^^^^ + failed to parse year in date "1 week": failed to parse "1 we" as year ... +``` + +> **Do you actually have this problem?** Only if that banner appears when you run a +> command. If your commands print their output cleanly, skip this entire subsection — +> there is nothing to fix, and the four steps below are not maintenance you need to +> perform. Confirm in one line: +> +> ```bash +> uv run overture-schema --version +> ``` +> +> A single line of output means you're fine. A wall of warning text above it means read on. + +**If you do see it: this is not cosmetic. Fix it before you do anything else.** + +The root `pyproject.toml` writes `exclude-newer` as a relative duration (`"1 week"`), +which caps how new a package `uv` will consider during dependency resolution. Only +reasonably recent `uv` versions parse that form. + +An older `uv` fails to read the whole `[tool.uv]` block, says so, **and carries on +without the cap** — then, because its resolution no longer matches the committed +lockfile, rewrites `uv.lock` in place. You end up with a modified tracked file you never +asked to change: + +```bash +git status --short uv.lock +git diff --stat uv.lock +``` + +On an affected machine: + +``` + M uv.lock + uv.lock | 807 +++++++++++++++++++------------------- + 1 file changed, 464 insertions(+), 343 deletions(-) +``` + +> **Both commands printing nothing is the healthy result.** `git status --short` and +> `git diff --stat` say nothing about a file that hasn't changed. Empty output here means +> your lockfile is untouched and there is nothing to fix. If you'd rather have an explicit +> answer than read silence: +> +> ```bash +> git diff --quiet uv.lock && echo "uv.lock: unmodified" || echo "uv.lock: MODIFIED" +> ``` + +It drops the `[options]` block that records the resolution settings, and pulls in +dependency versions past the cutoff the repo intended. Nothing breaks immediately — the +install works fine — but you're now building against a different dependency set than the +project pinned, and `git status` is dirty. + +**Step 1 — check your version:** + +```bash +uv --version +brew outdated uv +``` + +**Step 2 — upgrade:** + +```bash +brew upgrade uv +``` + +**Step 3 — restore the lockfile if the old `uv` already rewrote it:** + +```bash +git checkout uv.lock +``` + +**Step 4 — confirm:** + +```bash +uv sync --all-packages --locked +``` + +``` +Resolved 64 packages in 12ms +Audited 60 packages in 0.81ms +``` + +`--locked` fails outright if the lockfile isn't authoritative, so a clean pass means your +`uv`, the lockfile, and your `.venv` all agree. Run `git status --short uv.lock` once more +too — it should print nothing, which means the file is unmodified. + +> **If you see `error: The lockfile at uv.lock needs to be updated, but --locked was +> provided`,** you're at step 3, not step 4. A previous `uv sync` under the old `uv` +> already modified the lock. `git checkout uv.lock` and re-run. + +Once you're on a current `uv`, ordinary use leaves the lockfile alone — including the +`uv sync --all-packages --all-extras` that `make install`, `make check`, and +`make generate-pyspark` all run internally. + +--- + +### `FileNotFoundError` on `spark-submit` + +``` +FileNotFoundError: [Errno 2] No such file or directory: +'/opt/homebrew/Cellar/apache-spark/3.5.3/libexec/./bin/spark-submit' +``` + +**This has nothing to do with the generation step in "Do you need PySpark?".** It's a stale `SPARK_HOME` +environment variable, and it will happen whether or not you ran `make generate-pyspark`. + +What's going on: you have a `SPARK_HOME` exported in your shell profile pointing at a +**version-specific Homebrew path**. Homebrew has since upgraded Spark, so that exact +directory no longer exists: + +```bash +echo $SPARK_HOME +ls /opt/homebrew/Cellar/apache-spark/ +``` + +``` +/opt/homebrew/Cellar/apache-spark/3.5.3/libexec +4.0.0 +``` + +The variable names `3.5.3`; the only version present is `4.0.0`. It points at nothing. + +Meanwhile the `pyspark` in your venv (4.2.0) **ships its own copy of Spark** and doesn't +need the Homebrew one at all. But when `SPARK_HOME` is set, PySpark obeys it and looks +for `spark-submit` at that dead path. + +**Confirm that's your problem:** + +```bash +env -u SPARK_HOME uv run python -c " +from pyspark.sql import SparkSession +s = SparkSession.builder.master('local[1]').getOrCreate() +print('SUCCESS — spark', s.version) +s.stop()" +``` + +``` +SUCCESS — spark 4.2.0 +``` + +**Fix it permanently.** The variable is set in *two* files — fixing only one won't help, +because `.zprofile` runs for login shells and `.zshrc` for interactive ones: + +``` +~/.zshrc:8 export SPARK_HOME=/opt/homebrew/Cellar/apache-spark/3.5.3/libexec +~/.zshrc:9 export PATH="$SPARK_HOME/bin/:$PATH" +~/.zprofile:5 export SPARK_HOME=/opt/homebrew/Cellar/apache-spark/3.5.3/libexec +~/.zprofile:6 export PATH="$SPARK_HOME/bin/:$PATH" +``` + +Pick one: + +- **Simplest — delete all four lines.** If you only use Spark through Python projects + like this one, you don't need `SPARK_HOME` at all; each venv's `pyspark` brings its + own. +- **Keep Homebrew Spark for other work — stop hardcoding the version.** Replace the two + `SPARK_HOME` lines with the version-independent symlink Homebrew maintains: + + ```bash + export SPARK_HOME=/opt/homebrew/opt/apache-spark/libexec + ``` + + This survives upgrades. But note it points at Spark **4.0.0** while this project's + `pyspark` is **4.2.0** — mismatched versions cause their own confusing failures, so + prefer the first option while working in this repo. + +Then open a new terminal, or `exec zsh`, and re-run the check above. + +> **Per-shell workaround** if you don't want to touch your profile right now: +> `unset SPARK_HOME` in the terminal you're working in. It lasts until you close it. + +### `model_names()` returns `[]`, or `KeyError` on a feature type + +The PySpark validation expressions are generated code that is **not committed to git** — +they're in `.gitignore`, and `uv sync` alone cannot produce them. Skipping that step +doesn't raise an error; it leaves you with an empty registry, which is why this is easy +to miss. + +```bash +make generate-pyspark +``` + +Then confirm — don't infer it from the output, which is silent by design: + +```bash +uv run python -c "from overture.schema.pyspark import model_names; print(model_names())" +``` + +Before, an empty list. After, 30 entries — 15 feature types, each reachable by two names: + +``` +['address', 'bathymetry', 'building', 'building_part', 'connector', 'division', ...] +``` + +Only people working from a git clone ever need this. Published wheels ship the generated +expressions already; see +[Why generated code is gitignored](CONCEPTS.md#why-generated-code-is-gitignored). + +--- + +## Pasting commands into zsh + +macOS defaults to **zsh**, and two of its interactive behaviors mangle commands copied +out of documentation. Neither affects scripts or non-interactive shells. + +### `#` is not a comment in interactive zsh + +macOS defaults to **zsh**, and interactive zsh does *not* treat `#` as a comment unless +you turn that on. Paste a line like this and zsh hands `#`, `→`, and `15` to `wc` as +filenames: + +``` +find ... | wc -l # → 15 +wc: #: open: No such file or directory +wc: →: open: No such file or directory +wc: 15: open: No such file or directory + 0 total +``` + +A line that *starts* with `#` fails more obviously — `command not found: #`. + +This guide keeps `#` comments only as standalone label lines inside multi-command blocks, +never trailing after a command. To paste those blocks whole, enable comments once: + +```bash +setopt interactive_comments +``` + +Add it to `~/.zshrc` to make it permanent. Otherwise, skip the `#` lines when copying — +they are labels, not commands. Scripts and non-interactive shells are unaffected; this is +purely an interactive-zsh behavior. + +### `!` runs a command out of your history + +This one is worth understanding because it can do real damage. In an interactive shell, +`!` triggers **history expansion**, and it fires *inside double quotes*. `!r` means "the +most recent command starting with `r`" — the shell splices that command's text into your +line before running it. + +So a Python one-liner containing `{value!r}`, pasted into zsh as + +``` +uv run python -c "... f'default={f.default!r}' ..." +``` + +becomes something else entirely. What you get depends on your own shell history: + +``` +SyntaxError: f-string: invalid syntax + (f.defaultrm -rf schema) +``` + +That is a past command of yours, pasted into the middle of a Python f-string. Here it only +produced a syntax error — Python never ran and nothing was deleted. But the same mechanism +can land text somewhere the shell *will* execute. + +**The fix used throughout this guide:** multi-line Python is passed via a heredoc with a +**quoted** delimiter, not `-c "..."`. + +```python +from overture.schema.buildings import Building +print(f"{Building.__name__!r} is safe here") +``` + +Quoting the delimiter (`<<'""" + D + """'` rather than `<<""" + D + """`) disables every +form of expansion in the body — history, variables, command substitution. The text reaches +Python exactly as written. + +Verified rather than assumed: + +``` +double-quoted -c -> !r expanded into a command from history +quoted heredoc -> !r left alone +``` + +Single quotes also block history expansion, but the Python in this guide uses single +quotes internally, so heredocs are the practical choice. If you hit this in your own +one-liners, `{value!r}` can always be written `{repr(value)}` instead — no `!` at all. + +--- + +--- + +## Model gotchas + +Things that cost time if you don't know them. Every one of these is a consequence of +[the two representations](SCHEMA_GUIDE.md#32-the-one-thing-to-understand-two-representations). + +| Gotcha | What happens | Fix | +|---|---|---| +| `model_validate(geojson_dict)` | `ValidationError`: missing `theme`/`version`, `type` is `'Feature'` | Use `model_validate_json()`. Python mode expects flat data, JSON mode expects GeoJSON. | +| `model_dump()` without `by_alias` | Emits `class_`, not `class`; output won't re-validate | Always `by_alias=True` | +| `Segment.model_validate(...)` | `AttributeError` — it's a union alias, not a class | `TypeAdapter(Segment).validate_json(...)` | +| `model_names()` returns `[]` | PySpark expressions are generated, not committed | `make generate-pyspark` (or `make install`) | +| Dumps full of `null` | Unset optionals serialize explicitly | `exclude_none=True` | +| Absent list → `[]` → won't re-validate | An omitted optional list defaults to `[]` on the model, dumps as `[]`, then fails a `min_length` check on the way back in | `exclude_defaults=True`, or drop empty lists before re-validating | + +Asymmetric round-trip, worth knowing about: an optional list that is simply *absent* +from the input becomes an empty list on the model, and an empty list is not the same as +absent on the way back out. + +```python +segments = TypeAdapter(Segment) +seg = segments.validate_json( + open("road-indoors.yaml-as-json").read() +) # no `connectors` key +seg.connectors # [] ← not None + +flat = seg.model_dump(mode="python", by_alias=True, exclude_none=True) +flat["connectors"] # [] ← exclude_none doesn't drop it +segments.validate_python(flat) +# ValidationError: road.connectors +# List should have at least 2 items after validation, not 0 +``` + +The same document validates fine on the way in (the CLI accepts it) and fails on the way +back. `exclude_defaults=True` avoids it, as does pruning empty lists before re-validating. + +## Docs in the repo that are currently wrong + +- **`pip install overture-schema`** — in every package README. Nothing is on PyPI yet; + see [7.3](SCHEMA_GUIDE.md#73-what-changes-once-these-packages-are-published). Every other item that + stood here has been fixed, and `tests/test_documented_imports.py` now imports every + `overture.*` statement in every tracked Markdown file, so a broken one fails the suite + rather than accumulating here. From bfa48d35dfc64ab219b6acae26e21400bfc6af1a Mon Sep 17 00:00:00 2001 From: Dana Bauer Date: Thu, 20 Aug 2026 18:54:41 -0400 Subject: [PATCH 9/9] expand README to cover the whole repo, record the tenets and doctrine Signed-off-by: Dana Bauer --- AUTHORING.md | 6 +- CONCEPTS.md | 36 ++++++++++++ README.md | 135 ++++++++++++++++++++++++++++++++++++++------- SCHEMA_GUIDE.md | 2 +- TROUBLESHOOTING.md | 17 ++---- 5 files changed, 160 insertions(+), 36 deletions(-) diff --git a/AUTHORING.md b/AUTHORING.md index 10a82c6ce..0c0300281 100644 --- a/AUTHORING.md +++ b/AUTHORING.md @@ -1,8 +1,8 @@ # Authoring and Extending the Schema -For people **writing** schema — adding feature types, building tools on top of the models, +This page is for people **writing** to the schema: adding feature types, building tools on top of the models, or authoring new Pydantic models for Overture itself. If you only want to *use* the -schema — validate data, explore models, generate artifacts — you want +schema to validate data, explore models, or generate artifacts, you want [SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) instead. | If you want to | Read | @@ -24,7 +24,7 @@ package's `README.md` under `packages/`, versioned alongside the code it documen page covers what spans packages. See also [CONCEPTS.md](CONCEPTS.md) for why the schema is Pydantic and how the packages -fit together, and [SCHEMA_CONVENTIONS.md](SCHEMA_CONVENTIONS.md) for naming rules. +fit together. *Every code block on this page has been executed against the repo; `tests/test_documented_imports.py` keeps the imports honest.* diff --git a/CONCEPTS.md b/CONCEPTS.md index 366748a17..8f079400e 100644 --- a/CONCEPTS.md +++ b/CONCEPTS.md @@ -10,6 +10,7 @@ the glossary is faster. This page is for why they exist. | Question | Section | |---|---| +| What principles is the schema designed under? | [The tenets](#the-tenets) | | Why have a schema at all? | [Beyond raw data](#beyond-raw-data) | | Why Pydantic instead of JSON Schema? | [Why Pydantic](#why-pydantic-rather-than-json-schema) | | Why is the schema split into a dozen packages? | [Many packages](#why-the-schema-is-many-packages) | @@ -25,6 +26,41 @@ the glossary is faster. This page is for why they exist. --- +## The tenets + +Six principles the schema is designed under, set down by the working group early in the +project under the heading *"These are our tenets unless you know better ones"*. They still +decide arguments, so they are worth knowing before you propose a change. + +1. **Address the core, enable the periphery.** The Overture schema doesn't solve every + problem. It describes fully-formed solutions only for the most fundamental use cases + ("the core") while enabling less common use cases ("the periphery") via extensibility. +2. **Invent across the gap.** Many excellent solutions — published standards, best + practices, open-source tools — already exist and are well understood in the community. + The Overture schema reuses them to maximize compatibility and to focus effort on + unaddressed high-priority pain points. +3. **Backward-compatible is forward-compatible.** No design is future-proof, but good + designs stay relevant by adding features without breaking existing use cases. +4. **The world is neither flat nor still.** The Overture schema links representations of + 2- and 3-dimensional objects in space and time. +5. **Empower, don't dictate.** The schema provides a framework that lets users bring + together the data they need for their own use cases — Overture and non-Overture + sources alike — according to their own viewpoints and perspectives. +6. **Always open, never closed.** The schema and format aim for compatibility with free + and open-source tools, and avoid depending on closed-source or proprietary ones. + +Where they show up in this repository: + +| Tenet | In practice | +|---|---| +| Address the core, enable the periphery | Your own feature types register through entry points and become first-class — nothing in the tooling special-cases Overture. See [AUTHORING.md](AUTHORING.md#register-your-own-feature-types). | +| Invent across the gap | JSON Schema, OGC geometries, GeoJSON, GeoParquet, and Pydantic are all reused rather than reinvented. See [Why Pydantic](#why-pydantic-rather-than-json-schema) and [The GeoJSON envelope](#the-geojson-envelope). | +| Backward-compatible is forward-compatible | Backward-compatible changes land in both the Pydantic models and the deprecated YAML while both are live; major changes wait for `vnext`. See [CONTRIBUTING.md](CONTRIBUTING.md). | +| Always open, never closed | Every published artifact — JSON Schema, PySpark expressions, documentation — is generated by open tooling in this repository. | + +The doctrine the working group built on these tenets in 2023 is recorded in the +[project history](README.md#the-tenets-and-the-doctrine). + ## Why this exists This project provides type-safe Python models for validating and working with diff --git a/README.md b/README.md index 4969c2726..47b7d8b43 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,46 @@ -Overture Maps Schema +Overture Schema === -The Overture Maps schema working group is responsible for designing the Overture Maps Data Schema and the Global Entity Reference System (GERS). +This code in this repository defines the Overture schema. -## Documentation -The contents of this repository are presented in a more human-friendly format at [docs.overturemaps.org](https://docs.overturemaps.org/) + +_ Note: You'll find reference documentation, tutorials, and examples of working with Overture data +at [docs.overturemaps.org](https://docs.overturemaps.org/).__ + +## What's in this repository + +| Path | What it is | +|---|---| +| `packages/` | The schema, authored as [Pydantic](https://docs.pydantic.dev/latest/) models and published as Python packages. | +| `reference/examples/` | Feature instances that are **expected to validate** — one file per case, organized by theme. | +| `reference/counterexamples/` | Feature instances that are **expected to fail**, most carrying the specific error they should raise. | +| `tests/` | Tests that span packages, including the check that keeps the imports in these docs working. | +| `docs/` | Source for the schema pages on docs.overturemaps.org, plus the versioning reference. | +| `schema/` | **Deprecated.** The YAML JSON Schema — see [The YAML schema](#the-yaml-schema-deprecated). | +| `examples/`, `counterexamples/` | **Deprecated.** Fixtures for the YAML schema, not the Pydantic models. | + +Note the two sets of examples. `reference/examples/` and `reference/counterexamples/` are +the current ones, exercised by the Python test suite. The top-level `examples/` and +`counterexamples/` belong to the deprecated YAML schema. They overlap heavily but have +drifted apart; when you add a case, add it under `reference/`. ## Python packages -The schema is authored as [Pydantic](https://docs.pydantic.dev/latest/) models, published -as a set of Python packages under `packages/`. -These pages are for people working with the packages in code. **To read the schema itself -— what feature types exist, what fields they carry, what values are valid — use -[docs.overturemaps.org](https://docs.overturemaps.org/).** +Fourteen packages under `packages/`, versioned and released independently: + +- **`overture-schema`** — the umbrella package. Depends on all themes and support + packages for a coherent set. **This is what consumers pin.** +- **Themes** — `theme-addresses`, `theme-base`, `theme-buildings`, `theme-divisions`, + `theme-places`, `theme-transportation`. One package per theme, each holding its feature + types and the structures they share. +- **Foundation** — `system` (the base types every model builds on) and `common` + (structures shared across themes, like names and sources). +- **Tooling** — `cli` (validate data, generate JSON Schema), `codegen` (generate docs and + code from the models), `validation` (validate against the union of all discovered + models), `pyspark` (validation expressions for Spark). +- **Extensions** — `extensions-operating-hours`, an optional add-on. + +These pages are for people working with the packages in code: - [SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) — install the packages, explore the models, validate data, generate artifacts. **Start here.** @@ -21,21 +49,88 @@ These pages are for people working with the packages in code. **To read the sche - [CONCEPTS.md](CONCEPTS.md) — why the schema is Pydantic, why it is many packages, and what the GeoJSON envelope is. - [TROUBLESHOOTING.md](TROUBLESHOOTING.md) — symptom-indexed fixes and known gotchas. - -## Schema reference - - [GLOSSARY.md](GLOSSARY.md) — vocabulary for both the data model (entity, feature type, theme) and the Python toolchain (entry point, workspace, discriminated union). -- [SCHEMA_CONVENTIONS.md](SCHEMA_CONVENTIONS.md) — naming and modelling conventions. - **Out of date:** it predates the Pydantic packages and still describes JSON Schema as - the way the schema is defined, spells `subtype` as `subType`, and leaves the extensions - section unfinished. Useful for the conventions themselves; check anything structural - against [SCHEMA_GUIDE.md](SCHEMA_GUIDE.md) and [AUTHORING.md](AUTHORING.md). + +Run the full test and quality suite with: + +```bash +make check +``` + +## The YAML schema (deprecated) + +`schema/` holds the previous definition of the Overture schema as JSON Schema written in +YAML, with `schema/schema.yaml` as its entry point. It is validated by `./test.sh` (which +needs [`jv`](https://github.com/santhosh-tekuri/jsonschema)) against the top-level +`examples/` and `counterexamples/`, wired up in +[test-schema.yaml](.github/workflows/test-schema.yaml). + +**It is deprecated and scheduled for removal in December 2026.** It stays until then +because docs.overturemaps.org still builds from it: the interactive schema blocks come +from `schema/`, and the sample features come from `examples/`. See +[docs/README.md](docs/README.md). + +**Until it is removed, backward-compatible changes belong in both places.** A change to +the published data should land in the Pydantic models *and* in the YAML, so the two +definitions stay in step for as long as both are live. + +## Project history + +The schema working group opened this repository in January 2023, and the artifacts of +every phase of development are still here. + +| | | +|---|---| +| **Jan 2023** | Repository created as `schema-wg`, chartered to design the data schema and GERS. | +| **Mar 2023** | The schema takes shape as JSON Schema written in YAML: `schema/`, `examples/`, `counterexamples/`, and `test.sh` all arrive together. | +| **Aug 2023** | The Schema Task Force writes a *doctrine* on top of the project's tenets, to aim the work at a vision rather than react week to week. | +| **Jul 2024** | Overture data reaches general availability. | +| **Jul 2025** | Pydantic rewrite begins. +| **Nov 2025** | `packages/` are merged to main. The Pydantic schema and the YAML schema are developed in parallel. | +| **Aug 2026** | Repository docs consolidated into the guides listed above. | +| **Dec 2026** | `schema/`, `examples/`, and `counterexamples/` scheduled for removal. | + +1,788 commits from 71 contributors. + +### The tenets and the doctrine + +The working group set down six tenets early on, under the heading *"These are our tenets +unless you know better ones"* — address the core and enable the periphery; invent across +the gap; backward-compatible is forward-compatible; the world is neither flat nor still; +empower, don't dictate; always open, never closed. They still govern the design, and are +recorded with their consequences in [CONCEPTS.md](CONCEPTS.md#the-tenets). + +At the Schema Task Force meeting on 2023-08-30, the group decided to work toward a stated +vision rather than react week to week, and wrote a short doctrine built on those tenets for +the Working Group to ratify. It made five commitments: + +- **The schema optimizes for specific use cases, but allows extension.** Structure targets + selected use cases; user extensions unblock everyone else. +- **The schema gets better over time.** Properties for core use cases keep landing, known + issues keep getting fixed, and the schema advances in a backward-compatible way. +- **The schema is a cohesive whole.** Same problems get same solutions and same things get + same names, within and across themes, so that understanding one theme carries to the rest. +- **The schema is usable for data consumption by humans.** +- **The schema is usable for data consumption by open source tools.** + +The doctrine flagged the last two as an unresolved tension and said the project should lean +toward one. The 2025 move to Pydantic is what settled it: the models are authored for humans +to read and edit, and the artifacts tools need — JSON Schema, PySpark expressions, +documentation — are generated from them. See +[Why Pydantic rather than JSON Schema](CONCEPTS.md#why-pydantic-rather-than-json-schema). + +The specifics the doctrine argued over have all since been resolved. The `admins` theme is +now `divisions`; the camelCase property names it cited (`isoCountryCodeAlpha2`, +`road.roadNames`) became snake_case in February 2024; and the `entityId` property it +proposed as the schema/GERS interface never shipped — features carry a GERS `id` directly. ## Contributing -See [CONTRIBUTING.md](CONTRIBUTING.md) for branching strategy, workflow, and contribution guidelines. -## Feedback -Please provide feedback or ask questions at [Discussions](https://github.com/orgs/OvertureMaps/discussions). +See [CONTRIBUTING.md](CONTRIBUTING.md) for branching strategy, workflow, and contribution +guidelines. +## Feedback +Please provide feedback or ask questions at +[Discussions](https://github.com/orgs/OvertureMaps/discussions). diff --git a/SCHEMA_GUIDE.md b/SCHEMA_GUIDE.md index 943a699d9..d70ade4d7 100644 --- a/SCHEMA_GUIDE.md +++ b/SCHEMA_GUIDE.md @@ -1,6 +1,6 @@ # Overture Schema Guide -A practical guide to installing the Overture Maps schema packages, exploring the models, +This is a practical guide to installing the Overture schema packages, exploring the models, writing code against them, validating data, and generating artifacts from the schema. Three other pages cover material this guide points to rather than repeats: diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 62048e875..3203ca2ba 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -1,13 +1,8 @@ # Troubleshooting -Symptom-first. Find the error you actually saw, read that entry, and skip the rest. -**Nothing on this page is setup you need to perform.** If your commands run cleanly, -you do not need this page at all. +This page is meant to help you handle **errors** and navigate **gotchas** when you are installing and working with the schema models. It's written for early testers of the Pydantic schema who are installing locally, prior to the publication of the packages on PyPI and the `v2.0.0` launch of the schema. + -The page has two halves. Most of it is **errors** — something failed and printed a -message you can search for. The last section is **gotchas**: cases where nothing fails, -and you get wrong output instead of an error. Those are worth reading once before you -trust a round-trip. | Symptom | Section | |---|---| @@ -55,7 +50,7 @@ warning: Failed to parse `pyproject.toml` during settings discovery: > uv run overture-schema --version > ``` > -> A single line of output means you're fine. A wall of warning text above it means read on. +> A single line of output means you're fine. A wall of warning text above it means you have a problem to sort out. **If you do see it: this is not cosmetic. Fix it before you do anything else.** @@ -328,8 +323,7 @@ one-liners, `{value!r}` can always be written `{repr(value)}` instead — no `!` ## Model gotchas -Things that cost time if you don't know them. Every one of these is a consequence of -[the two representations](SCHEMA_GUIDE.md#32-the-one-thing-to-understand-two-representations). +Navigating [the two representations](SCHEMA_GUIDE.md#32-the-one-thing-to-understand-two-representations) of the models can be challenging. | Gotcha | What happens | Fix | |---|---|---| @@ -340,8 +334,7 @@ Things that cost time if you don't know them. Every one of these is a consequenc | Dumps full of `null` | Unset optionals serialize explicitly | `exclude_none=True` | | Absent list → `[]` → won't re-validate | An omitted optional list defaults to `[]` on the model, dumps as `[]`, then fails a `min_length` check on the way back in | `exclude_defaults=True`, or drop empty lists before re-validating | -Asymmetric round-trip, worth knowing about: an optional list that is simply *absent* -from the input becomes an empty list on the model, and an empty list is not the same as +Note: optional list that is simply *absent* from the input becomes an empty list on the model, and an empty list is not the same as absent on the way back out. ```python