From df24b0f808d02ff23f972dbb1f84f474cc9a6336 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Wed, 23 Sep 2026 16:55:07 +0300 Subject: [PATCH 01/27] fix: enforce schema dialect semantics Require one supported dialect across each derivation chain and its transitive GTS references, rejecting unknown declarations instead of allowing jsonschema to fall back. Validate trait schema integrity under the host type dialect. Signed-off-by: Artifizer --- gts/src/gts/store.py | 105 ++++++++++++++++++++++++++++++++++++-- gts/src/gts/traits.py | 18 +++++-- tests/test_store_extra.py | 45 ++++++++++++++++ tests/test_traits.py | 17 ++++++ 4 files changed, 176 insertions(+), 9 deletions(-) diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index f8c601e..1e3cd42 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -410,6 +410,98 @@ def _content_is_abstract(content: dict[str, Any]) -> bool: def _content_is_final(content: dict[str, Any]) -> bool: return content.get("x-gts-final") is True + @staticmethod + def _schema_dialect(schema: dict[str, Any]) -> str: + dialect = schema.get("$schema") + if dialect is None: + return "draft-07" + if not isinstance(dialect, str) or not dialect: + raise ValueError("$schema must declare a supported JSON Schema dialect") + normalized = dialect.removesuffix("#").lower().replace("https://", "http://", 1) + supported = { + "http://json-schema.org/draft-07/schema": "draft-07", + "http://json-schema.org/draft/2019-09/schema": "2019-09", + "http://json-schema.org/draft/2020-12/schema": "2020-12", + } + try: + return supported[normalized] + except KeyError as error: + raise ValueError(f"Unsupported JSON Schema dialect: {dialect}") from error + + def _validate_chain_dialect( + self, + gts_id: str, + chain_ids: list[str], + transient_schema: dict[str, Any] | None, + ) -> None: + root_entity = self.get(chain_ids[0]) + root_content = ( + transient_schema + if chain_ids[0] == gts_id and transient_schema is not None + else root_entity.content + if root_entity + else None + ) + if not isinstance(root_content, dict): + return + root_dialect = self._schema_dialect(root_content) + + for chain_id in chain_ids: + entity = self.get(chain_id) + content = ( + transient_schema + if chain_id == gts_id and transient_schema is not None + else entity.content + if entity + else None + ) + if ( + isinstance(content, dict) + and self._schema_dialect(content) != root_dialect + ): + raise ValueError( + "GTS derivation chain mixes JSON Schema dialects: " + f"root type '{chain_ids[0]}' uses {root_dialect} but " + f"'{chain_id}' uses {self._schema_dialect(content)}; every type " + "in a chained $id hierarchy must use the root type's dialect" + ) + + visited: set[str] = set() + queue: list[tuple[str, dict[str, Any]]] = ( + [(gts_id, transient_schema)] if transient_schema is not None else [] + ) + if not queue: + entity = self.get(gts_id) + if entity and isinstance(entity.content, dict): + queue.append((gts_id, entity.content)) + while queue: + current_id, content = queue.pop(0) + if current_id in visited: + continue + visited.add(current_id) + for dependency_id, dependency_is_type in self._schema_dependencies( + content, include_gts_refs=False + ): + if not dependency_is_type or dependency_id == current_id: + continue + target = self.get(dependency_id) + if ( + not target + or not target.is_schema + or not isinstance(target.content, dict) + ): + continue + target_dialect = self._schema_dialect(target.content) + if target_dialect != root_dialect: + raise ValueError( + "GTS derivation mixes JSON Schema dialects: " + f"root type '{chain_ids[0]}' uses {root_dialect} but $ref " + f"target '{dependency_id}' uses {target_dialect}; every type " + "in the chain and its transitive gts:// $ref targets must use " + "the root type's dialect" + ) + queue.append((dependency_id, target.content)) + def _validate_schema_chain( self, gts_id: str, transient_schema: dict[str, Any] | None = None ) -> None: @@ -417,17 +509,18 @@ def _validate_schema_chain( gid = GtsID(gts_id) segments = gid.gts_id_segments - # Single-segment schemas have no parent to validate against - if len(segments) < 2: - return - - # Build chain IDs chain_ids = [] prefix = "gts." for seg in segments: chain_ids.append(prefix + seg.segment) prefix = prefix + seg.segment + self._validate_chain_dialect(gts_id, chain_ids, transient_schema) + + # Single-segment schemas have no parent to validate against + if len(segments) < 2: + return + # Validate each adjacent pair for i in range(len(chain_ids) - 1): base_id = chain_ids[i] @@ -688,6 +781,7 @@ def validate_schema_content( f"Invalid $schema URL '{meta_schema_url}': must be a standard JSON Schema URL, not a GTS ID" ) + self._schema_dialect(schema_content) logger.info(f"Validating schema {schema_id.id}") self._validate_schema_refs(schema_content, "") self._validate_schema_ref_targets(schema_content) @@ -916,6 +1010,7 @@ def validate_instance_content( except KeyError as error: raise StoreGtsSchemaNotFound(schema_type.id) from error + self._schema_dialect(schema) if isinstance(schema, dict) and self._content_is_abstract(schema): raise ValueError( f"type '{schema_type.id}' is abstract and cannot have direct instances" diff --git a/gts/src/gts/traits.py b/gts/src/gts/traits.py index 36c40c5..2ca9571 100644 --- a/gts/src/gts/traits.py +++ b/gts/src/gts/traits.py @@ -41,11 +41,13 @@ def __init__( values: Any, resolved_trait_schemas: list[Any], merged_traits: dict[str, Any], + dialect: str | None, ) -> None: self.schema = schema self.values = values self.resolved_trait_schemas = resolved_trait_schemas self.merged_traits = merged_traits + self.dialect = dialect def _has_schema(self) -> bool: return len(self.resolved_trait_schemas) > 0 @@ -61,7 +63,9 @@ def validate( selected_type_id: str | None = None, ) -> list[str]: """Return a list of error strings (empty means valid).""" - errors = _validate_trait_schema_integrity(self.resolved_trait_schemas) + errors = _validate_trait_schema_integrity( + self.resolved_trait_schemas, self.dialect + ) if errors: return errors errors = _validate_trait_schema_compatibility(self.resolved_trait_schemas) @@ -191,6 +195,7 @@ def build_effective_traits( values=values, resolved_trait_schemas=list(resolved_trait_schemas), merged_traits=copy.deepcopy(merged_traits), + dialect=dialect, ) @@ -298,14 +303,19 @@ def strip(node: Any) -> Any: return map_schema_nodes(copy.deepcopy(schema), strip) -def _validate_trait_schema_integrity(resolved_trait_schemas: list[Any]) -> list[str]: +def _validate_trait_schema_integrity( + resolved_trait_schemas: list[Any], dialect: str | None +) -> list[str]: for i, ts in enumerate(resolved_trait_schemas): if isinstance(ts, bool): continue if isinstance(ts, dict): + schema = copy.deepcopy(ts) + if dialect: + schema["$schema"] = dialect try: - cls = validator_for(ts) - cls.check_schema(ts) + cls = validator_for(schema) + cls.check_schema(schema) except Exception as e: # noqa: BLE001 - surfaced as validation error message return [f"{X_GTS_TRAITS_SCHEMA}[{i}] is not a valid JSON Schema: {e}"] else: diff --git a/tests/test_store_extra.py b/tests/test_store_extra.py index 1850bcc..8ae3c87 100644 --- a/tests/test_store_extra.py +++ b/tests/test_store_extra.py @@ -285,6 +285,51 @@ def test_missing_base_schema_raises(self): with pytest.raises(ValueError, match="not found for chain validation"): store._validate_schema_chain("gts.x.test._.base.v1~x.test._.derived.v1~") + @pytest.mark.parametrize( + "dialect", + [ + "https://example.invalid/not-a-json-schema-dialect", + "https://json-schema.org/draft/2020-21/schema", + ], + ) + def test_unsupported_schema_dialect_raises(self, dialect): + with pytest.raises(ValueError, match="Unsupported JSON Schema dialect"): + GtsStore._schema_dialect({"$schema": dialect}) + + def test_mixed_dialect_chain_raises(self): + base = _schema_entity("gts.x.test._.base.v1~") + derived = _schema_entity( + "gts.x.test._.base.v1~x.test._.derived.v1~", + {"$schema": "https://json-schema.org/draft/2020-12/schema"}, + ) + store = GtsStore(reader=None) + store.register(base) + store.register(derived) + + with pytest.raises(ValueError, match="mixes JSON Schema dialects"): + store._validate_schema_chain("gts.x.test._.base.v1~x.test._.derived.v1~") + + def test_transitive_ref_dialect_mismatch_raises(self): + foreign = _schema_entity( + "gts.x.test._.foreign.v1~", + {"$schema": "https://json-schema.org/draft/2020-12/schema"}, + ) + middle = _schema_entity( + "gts.x.test._.middle.v1~", + {"allOf": [{"$ref": "gts://gts.x.test._.foreign.v1~"}]}, + ) + host = _schema_entity( + "gts.x.test._.host.v1~", + {"allOf": [{"$ref": "gts://gts.x.test._.middle.v1~"}]}, + ) + store = GtsStore(reader=None) + store.register(foreign) + store.register(middle) + store.register(host) + + with pytest.raises(ValueError, match="gts.x.test._.foreign.v1~"): + store._validate_schema_chain("gts.x.test._.host.v1~") + def test_incompatible_derivation_raises(self): base = _schema_entity( "gts.x.test._.base.v1~", diff --git a/tests/test_traits.py b/tests/test_traits.py index 3b8a6da..9746c53 100644 --- a/tests/test_traits.py +++ b/tests/test_traits.py @@ -308,6 +308,23 @@ def test_invalid_trait_schema_integrity_flagged(self): errors = effective.validate(check_unresolved=True) assert any("not a valid JSON Schema" in e for e in errors) + def test_draft7_tuple_schema_integrity_uses_host_dialect(self): + schema = { + "type": "object", + "properties": { + "pair": { + "type": "array", + "items": [{"type": "string"}], + } + }, + } + effective = build_effective_traits( + [schema], + {"pair": ["ok"]}, + "http://json-schema.org/draft-07/schema#", + ) + assert effective.validate(check_unresolved=True) == [] + def test_dialect_applied_to_effective_schema(self): effective = build_effective_traits( [{"type": "object"}], {}, "https://json-schema.org/draft/2020-12/schema" From 0917306a63d3214de51d6b13ba98a00a5deb5491 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Wed, 23 Sep 2026 20:54:12 +0300 Subject: [PATCH 02/27] fix: reject mixed local references Resolve local JSON Pointer targets during GTS dialect validation and reject embedded resources that declare a different dialect from the hierarchy root. Add regression coverage for cross-dialect compound schemas. Signed-off-by: Artifizer --- gts/src/gts/store.py | 42 +++++++++++++++++++++++++++++---------- tests/test_store_extra.py | 21 ++++++++++++++++++++ 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index 1e3cd42..853bde2 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -11,6 +11,7 @@ from referencing.jsonschema import DRAFT202012 from . import compatibility, derivation, traits +from ._json_pointer import resolve as resolve_json_pointer from ._naming import looks_like_gts, strip_scheme, with_scheme from .entities import GtsEntity from .gts import GtsID, GtsRef, GtsWildcard @@ -428,6 +429,24 @@ def _schema_dialect(schema: dict[str, Any]) -> str: except KeyError as error: raise ValueError(f"Unsupported JSON Schema dialect: {dialect}") from error + def _validate_local_ref_dialects( + self, schema: dict[str, Any], root_id: str, root_dialect: str + ) -> None: + for subschema, _schema_path in iter_schema_nodes(schema): + ref = subschema.get("$ref") + if not isinstance(ref, str) or not ref.startswith("#"): + continue + target = resolve_json_pointer(schema, ref) + if not isinstance(target, dict) or "$schema" not in target: + continue + target_dialect = self._schema_dialect(target) + if target_dialect != root_dialect: + raise ValueError( + "GTS schema reference graph mixes JSON Schema dialects: " + f"root type '{root_id}' uses {root_dialect} but local $ref " + f"target '{ref}' uses {target_dialect}" + ) + def _validate_chain_dialect( self, gts_id: str, @@ -455,16 +474,16 @@ def _validate_chain_dialect( if entity else None ) - if ( - isinstance(content, dict) - and self._schema_dialect(content) != root_dialect - ): - raise ValueError( - "GTS derivation chain mixes JSON Schema dialects: " - f"root type '{chain_ids[0]}' uses {root_dialect} but " - f"'{chain_id}' uses {self._schema_dialect(content)}; every type " - "in a chained $id hierarchy must use the root type's dialect" - ) + if isinstance(content, dict): + chain_dialect = self._schema_dialect(content) + if chain_dialect != root_dialect: + raise ValueError( + "GTS derivation chain mixes JSON Schema dialects: " + f"root type '{chain_ids[0]}' uses {root_dialect} but " + f"'{chain_id}' uses {chain_dialect}; every type in a chained " + "$id hierarchy must use the root type's dialect" + ) + self._validate_local_ref_dialects(content, chain_ids[0], root_dialect) visited: set[str] = set() queue: list[tuple[str, dict[str, Any]]] = ( @@ -500,6 +519,9 @@ def _validate_chain_dialect( "in the chain and its transitive gts:// $ref targets must use " "the root type's dialect" ) + self._validate_local_ref_dialects( + target.content, chain_ids[0], root_dialect + ) queue.append((dependency_id, target.content)) def _validate_schema_chain( diff --git a/tests/test_store_extra.py b/tests/test_store_extra.py index 8ae3c87..9a7f56c 100644 --- a/tests/test_store_extra.py +++ b/tests/test_store_extra.py @@ -296,6 +296,27 @@ def test_unsupported_schema_dialect_raises(self, dialect): with pytest.raises(ValueError, match="Unsupported JSON Schema dialect"): GtsStore._schema_dialect({"$schema": dialect}) + def test_local_ref_dialect_mismatch_raises(self): + schema = _schema_entity( + "gts.x.test._.embedded.v1~", + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"legacy": {"$ref": "#/$defs/legacy"}}, + "$defs": { + "legacy": { + "$id": "legacy", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "string", + } + }, + }, + ) + store = GtsStore(reader=None) + store.register(schema) + + with pytest.raises(ValueError, match=r"local \$ref target"): + store._validate_schema_chain(schema.gts_id.id) + def test_mixed_dialect_chain_raises(self): base = _schema_entity("gts.x.test._.base.v1~") derived = _schema_entity( From 317b24f018bd96740945ec0220be6856f3f4694d Mon Sep 17 00:00:00 2001 From: Artifizer Date: Thu, 24 Sep 2026 11:53:27 +0300 Subject: [PATCH 03/27] chore: support GTS spec v0.14.1 Signed-off-by: Artifizer --- .gts-spec | 2 +- README.md | 2 +- gts/README.md | 2 +- gts/openapi.json | 2 +- gts/pyproject.toml | 2 +- gts/src/gts/_server.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gts-spec b/.gts-spec index 1bffb45..3ccf2c0 160000 --- a/.gts-spec +++ b/.gts-spec @@ -1 +1 @@ -Subproject commit 1bffb45de77982a0852a351cceba5787e36bd810 +Subproject commit 3ccf2c0c0f43afc10d7ee7ab20b0a087214e86c1 diff --git a/README.md b/README.md index a356620..3d93988 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A minimal, idiomatic Python library for working with **GTS** ([Global Type System](https://github.com/gts-spec/gts-spec)) identifiers and JSON/JSON Schema artifacts. -Current supported GTS spec version: `0.14.0` +Current supported GTS spec version: `0.14.1` ## Roadmap diff --git a/gts/README.md b/gts/README.md index 43f66ff..950b412 100644 --- a/gts/README.md +++ b/gts/README.md @@ -2,7 +2,7 @@ Python helpers and a reference HTTP service for the [Global Type System (GTS)](https://github.com/globaltypesystem/gts-spec). The package supports GTS identifier parsing, JSON Schema-backed validation, schema compatibility and derivation checks, traits, casting, queries, file loading, a CLI, and a FastAPI application. -The package targets GTS specification v0.14.0 and requires Python 3.9 or later. +The package targets GTS specification v0.14.1 and requires Python 3.9 or later. ## Installation diff --git a/gts/openapi.json b/gts/openapi.json index e4a2b05..11d7908 100644 --- a/gts/openapi.json +++ b/gts/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "GTS Server", - "version": "0.14.0" + "version": "0.14.1" }, "paths": { "/entities": { diff --git a/gts/pyproject.toml b/gts/pyproject.toml index 2d5a04a..9dfd049 100644 --- a/gts/pyproject.toml +++ b/gts/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "gts" -version = "0.14.0" +version = "0.14.1" description = "Global Type System (GTS) helpers: identifiers, parsing, validation, and operations" readme = "README.md" authors = [{ name = "GTS Community" }] diff --git a/gts/src/gts/_server.py b/gts/src/gts/_server.py index 5f13b89..8554328 100644 --- a/gts/src/gts/_server.py +++ b/gts/src/gts/_server.py @@ -193,7 +193,7 @@ def __init__( self.host = host self.port = port self.base_url = f"http://{self.host}:{self.port}" - self.app = FastAPI(title="GTS Server", version="0.14.0") + self.app = FastAPI(title="GTS Server", version="0.14.1") self.app.add_middleware( _RequestLoggingMiddleware, verbose=self.ops.verbose, From e93a33cd5b58e8a3a8253fd41118c71df4ea5093 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Thu, 24 Sep 2026 12:57:02 +0300 Subject: [PATCH 04/27] test(files_reader): avoid assuming filesystem discovery order Verify discovered entities by their GTS IDs and retain assertions for their source-file metadata. This prevents the test from failing when os.walk returns JSON and YAML files in a different order across platforms or filesystems. Signed-off-by: Artifizer --- tests/test_files_reader_coverage.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_files_reader_coverage.py b/tests/test_files_reader_coverage.py index 2583029..7d6d3c9 100644 --- a/tests/test_files_reader_coverage.py +++ b/tests/test_files_reader_coverage.py @@ -35,13 +35,16 @@ def test_reader_discovers_json_yaml_and_list_entities_and_skips_invalid_files(tm entities = list(GtsFileReader(str(source))) - assert [entity.gts_id.id for entity in entities] == [ + entities_by_id = {entity.gts_id.id: entity for entity in entities} + assert set(entities_by_id) == { "gts.acme.catalog._.item.v1~acme.catalog._.one.v1", "gts.acme.catalog._.item.v1~", - ] - assert entities[0].label == "entities.json#0" - assert entities[0].file.sequencesCount == 2 - assert entities[1].file.name == "schema.yaml" + } + list_entity = entities_by_id["gts.acme.catalog._.item.v1~acme.catalog._.one.v1"] + schema_entity = entities_by_id["gts.acme.catalog._.item.v1~"] + assert list_entity.label == "entities.json#0" + assert list_entity.file.sequencesCount == 2 + assert schema_entity.file.name == "schema.yaml" def test_reader_accepts_multiple_paths_and_reset_recollects_files(tmp_path): From 0bce0fc6eb29661202f8631e22528e41e4115c53 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Thu, 24 Sep 2026 18:27:21 +0300 Subject: [PATCH 05/27] fix: require canonical type schema identity Make legacy and HTTP-backed explicit Type Schema registration require canonical JSON metadata. Require a supported $schema, a gts:// $id that denotes a GTS Type, and equality between the normalized $id and type_id. Preserve the dialect validator fix by canonicalizing accepted Draft-07 aliases before meta-schema and instance validation, and extend regression coverage for both behaviors. Signed-off-by: Artifizer --- gts/src/gts/store.py | 50 +++++++++++++++++++++++++++------------ tests/test_ops.py | 22 ++++++++++++++--- tests/test_server.py | 17 ++++++++++--- tests/test_store.py | 30 +++++++++++++++++++++++ tests/test_store_extra.py | 31 ++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 21 deletions(-) diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index 853bde2..3d543f8 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -170,6 +170,20 @@ def register_schema(self, type_id: str, schema: dict[str, Any]) -> None: Creates a JsonEntity from the schema dict. """ gts_id = GtsID.parse_type(type_id) + dialect = schema.get("$schema") + if not isinstance(dialect, str) or not dialect: + raise ValueError("Type Schema must contain a top-level $schema field") + embedded_id = schema.get("$id") + if not isinstance(embedded_id, str) or not embedded_id.startswith("gts://gts."): + raise ValueError("Type Schema must contain a top-level $id in gts:// form") + try: + normalized_id = GtsID.parse_type(strip_scheme(embedded_id)) + except ValueError as error: + raise ValueError(f"Invalid GTS Type Schema $id: {embedded_id}") from error + if normalized_id.id != gts_id.id: + raise ValueError( + f"Embedded $id '{embedded_id}' must match external type_id '{type_id}'" + ) entity = GtsEntity(content=schema, gts_id=gts_id, is_schema=True) self._by_id[gts_id.id] = entity @@ -414,8 +428,6 @@ def _content_is_final(content: dict[str, Any]) -> bool: @staticmethod def _schema_dialect(schema: dict[str, Any]) -> str: dialect = schema.get("$schema") - if dialect is None: - return "draft-07" if not isinstance(dialect, str) or not dialect: raise ValueError("$schema must declare a supported JSON Schema dialect") normalized = dialect.removesuffix("#").lower().replace("https://", "http://", 1) @@ -429,6 +441,14 @@ def _schema_dialect(schema: dict[str, Any]) -> str: except KeyError as error: raise ValueError(f"Unsupported JSON Schema dialect: {dialect}") from error + @staticmethod + def _schema_dialect_uri(schema: dict[str, Any]) -> str: + return { + "draft-07": "http://json-schema.org/draft-07/schema#", + "2019-09": "https://json-schema.org/draft/2019-09/schema", + "2020-12": "https://json-schema.org/draft/2020-12/schema", + }[GtsStore._schema_dialect(schema)] + def _validate_local_ref_dialects( self, schema: dict[str, Any], root_id: str, root_dialect: str ) -> None: @@ -714,11 +734,11 @@ def _build_effective_traits( if leaf else None ) - dialect = None - if isinstance(leaf_content, dict): - ds = leaf_content.get("$schema") - if isinstance(ds, str): - dialect = ds + dialect = ( + self._schema_dialect_uri(leaf_content) + if isinstance(leaf_content, dict) + else None + ) return traits.build_effective_traits(trait_schemas, merged_traits, dialect) @@ -812,13 +832,10 @@ def validate_schema_content( self._validate_schema_chain(schema_id.id, schema_content) try: - from jsonschema import Draft7Validator - - if meta_schema_url: - validator_class = validator_for({"$schema": meta_schema_url}) - validator_class.check_schema(schema_content) - else: - Draft7Validator.check_schema(schema_content) + validator_class = validator_for( + {"$schema": self._schema_dialect_uri(schema_content)} + ) + validator_class.check_schema(schema_content) logger.info( f"Schema {schema_id.id} passed JSON Schema meta-schema validation" @@ -1038,7 +1055,10 @@ def validate_instance_content( f"type '{schema_type.id}' is abstract and cannot have direct instances" ) - schema_for_validation = _without_x_gts_ref(schema) + schema_for_validation = { + **_without_x_gts_ref(schema), + "$schema": self._schema_dialect_uri(schema), + } validator_class = validator_for(schema_for_validation) validator = validator_class( schema_for_validation, diff --git a/tests/test_ops.py b/tests/test_ops.py index 77e42fb..839abbf 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -155,13 +155,29 @@ def test_add_entities_rejects_unsupported_x_gts_ref_pointer(self, ops): class TestAddSchemaLegacy: def test_add_schema_legacy_success(self, ops): - result = ops.add_schema("gts.x.test._.legacy.v1~", {"type": "object"}) + schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "gts://gts.x.test._.legacy.v1~", + "type": "object", + } + result = ops.add_schema("gts.x.test._.legacy.v1~", schema) assert result.ok is True assert result.id == "gts.x.test._.legacy.v1~" def test_add_schema_legacy_changed_content_is_conflict(self, ops): - assert ops.add_schema("gts.x.test._.legacy.v1~", {"type": "object"}).ok is True - result = ops.add_schema("gts.x.test._.legacy.v1~", {"type": "string"}) + dialect = "http://json-schema.org/draft-07/schema#" + schema_id = "gts://gts.x.test._.legacy.v1~" + assert ( + ops.add_schema( + "gts.x.test._.legacy.v1~", + {"$schema": dialect, "$id": schema_id, "type": "object"}, + ).ok + is True + ) + result = ops.add_schema( + "gts.x.test._.legacy.v1~", + {"$schema": dialect, "$id": schema_id, "type": "string"}, + ) assert result.ok is False assert result.conflict is True diff --git a/tests/test_server.py b/tests/test_server.py index d6a6856..f199556 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -108,18 +108,29 @@ def test_add_entities(self, server): def test_add_schema(self, server): from gts._server import SchemaRegister - body = SchemaRegister(type_id="gts.x.test._.bar.v1~", type_schema={"type": "object"}) + body = SchemaRegister( + type_id="gts.x.test._.bar.v1~", + type_schema={ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "gts://gts.x.test._.bar.v1~", + "type": "object", + }, + ) resp = run(server.add_schema(body)) assert resp.status_code == 200 def test_add_schema_changed_content_conflict(self, server): from gts._server import SchemaRegister + dialect = "http://json-schema.org/draft-07/schema#" + schema_id = "gts://gts.x.test._.bar.v1~" initial = SchemaRegister( - type_id="gts.x.test._.bar.v1~", type_schema={"type": "object"} + type_id="gts.x.test._.bar.v1~", + type_schema={"$schema": dialect, "$id": schema_id, "type": "object"}, ) changed = SchemaRegister( - type_id="gts.x.test._.bar.v1~", type_schema={"type": "string"} + type_id="gts.x.test._.bar.v1~", + type_schema={"$schema": dialect, "$id": schema_id, "type": "string"}, ) assert run(server.add_schema(initial)).status_code == 200 diff --git a/tests/test_store.py b/tests/test_store.py index d8ea42c..b942b1a 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -92,6 +92,8 @@ def test_store_register_schema(self): store = GtsStore(reader) schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "gts://gts.vendor.package.namespace.type.v1~", "type": "object", "properties": {"name": {"type": "string"}}, } @@ -101,6 +103,33 @@ def test_store_register_schema(self): assert result is not None assert result.is_schema is True + def test_store_register_schema_requires_schema_marker(self): + """An explicit type ID does not turn an instance document into a schema.""" + store = GtsStore(MockGtsReader([])) + + with pytest.raises(ValueError, match=r"top-level \$schema"): + store.register_schema( + "gts.vendor.package.namespace.type.v1~", {"type": "object"} + ) + + def test_store_register_schema_requires_matching_embedded_id(self): + store = GtsStore(MockGtsReader([])) + type_id = "gts.vendor.package.namespace.type.v1~" + + with pytest.raises(ValueError, match=r"top-level \$id"): + store.register_schema( + type_id, + {"$schema": "http://json-schema.org/draft-07/schema#"}, + ) + with pytest.raises(ValueError, match="must match"): + store.register_schema( + type_id, + { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "gts://gts.vendor.package.namespace.other.v1~", + }, + ) + def test_store_register_schema_invalid_id(self): """Test registering schema with invalid ID (not ending with ~).""" reader = MockGtsReader([]) @@ -307,6 +336,7 @@ def test_validate_instance_content_enforces_standard_formats(self): type_id, { "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "gts://gts.vendor.package.namespace.type.v1~", "type": "object", "properties": { "uuid": {"type": "string", "format": "uuid"}, diff --git a/tests/test_store_extra.py b/tests/test_store_extra.py index 9a7f56c..f0773dc 100644 --- a/tests/test_store_extra.py +++ b/tests/test_store_extra.py @@ -13,6 +13,7 @@ StoreGtsEntityNotFound, StoreGtsObjectNotFound, ) +from jsonschema import ValidationError class MockGtsReader(GtsReader): @@ -296,6 +297,36 @@ def test_unsupported_schema_dialect_raises(self, dialect): with pytest.raises(ValueError, match="Unsupported JSON Schema dialect"): GtsStore._schema_dialect({"$schema": dialect}) + def test_missing_schema_dialect_raises(self): + """A document without $schema is not eligible for schema validation.""" + with pytest.raises(ValueError, match=r"\$schema must declare"): + GtsStore._schema_dialect({}) + + def test_draft7_alias_uses_draft7_validator(self): + """Canonicalize accepted aliases before schema and instance validation.""" + schema_id = "gts.x.test._.draft7_alias.v1~" + schema = _schema_entity( + schema_id, + { + "$schema": "https://json-schema.org/draft-07/schema#", + "required": ["pair"], + "properties": { + "pair": { + "type": "array", + "items": [{"type": "string"}, {"type": "integer"}], + "additionalItems": False, + } + }, + }, + ) + store = GtsStore(reader=None) + store.register(schema) + + store.validate_schema(schema_id) + store.validate_instance_content({"pair": ["ok", 1]}, schema_id) + with pytest.raises(ValidationError): + store.validate_instance_content({"pair": ["ok", "wrong"]}, schema_id) + def test_local_ref_dialect_mismatch_raises(self): schema = _schema_entity( "gts.x.test._.embedded.v1~", From b34ca2cbfeb27121342ff752f906846af53b7d8a Mon Sep 17 00:00:00 2001 From: Artifizer Date: Fri, 25 Sep 2026 15:31:59 +0300 Subject: [PATCH 06/27] chore: support gts-spec v0.14.2 Adopt batch Type Schema registration and derive each type identifier from canonical embedded metadata. Pin the conformance submodule to v0.14.2, bump the breaking package release, and regenerate the OpenAPI document. Signed-off-by: Artifizer --- .gts-spec | 2 +- README.md | 2 +- gts/README.md | 6 +- gts/openapi.json | 453 ++++++++++++++++++++++++++++++++--------- gts/pyproject.toml | 2 +- gts/src/gts/_server.py | 21 +- gts/src/gts/ops.py | 54 ++++- tests/test_ops.py | 35 +++- tests/test_server.py | 38 ++-- 9 files changed, 461 insertions(+), 152 deletions(-) diff --git a/.gts-spec b/.gts-spec index 3ccf2c0..4ec63d6 160000 --- a/.gts-spec +++ b/.gts-spec @@ -1 +1 @@ -Subproject commit 3ccf2c0c0f43afc10d7ee7ab20b0a087214e86c1 +Subproject commit 4ec63d6978919465175931f040745a8a75333d10 diff --git a/README.md b/README.md index 3d93988..1bbdb03 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A minimal, idiomatic Python library for working with **GTS** ([Global Type System](https://github.com/gts-spec/gts-spec)) identifiers and JSON/JSON Schema artifacts. -Current supported GTS spec version: `0.14.1` +Current supported GTS spec version: `0.14.2` ## Roadmap diff --git a/gts/README.md b/gts/README.md index 950b412..47ff0d7 100644 --- a/gts/README.md +++ b/gts/README.md @@ -2,7 +2,7 @@ Python helpers and a reference HTTP service for the [Global Type System (GTS)](https://github.com/globaltypesystem/gts-spec). The package supports GTS identifier parsing, JSON Schema-backed validation, schema compatibility and derivation checks, traits, casting, queries, file loading, a CLI, and a FastAPI application. -The package targets GTS specification v0.14.1 and requires Python 3.9 or later. +The package targets GTS specification v0.14.2 and requires Python 3.9 or later. ## Installation @@ -185,7 +185,7 @@ ops.reload_from_path("replacement-directory") ops.add_entity(content, validate=False) ops.add_entities([content_a, content_b]) -ops.add_schema(type_id, schema) +ops.add_schemas([schema_a, schema_b]) ops.extract_id(content) ops.validate_id(gts_id) ops.parse_id(gts_id) @@ -268,7 +268,7 @@ app = GtsHttpServer(ops=GtsOps()).app | `/entities/{gts_id}` | `GET` | Retrieves one entity. | | `/entities` | `POST` | Entity/schema body; optional `validate=true` runs full validation. Failed registration returns 422 and is rolled back. | | `/entities/bulk` | `POST` | JSON array of entity/schema objects. | -| `/type-schemas` | `POST` | `{"type_id": "...~", "type_schema": {...}}`. | +| `/type-schemas` | `POST` | JSON array of GTS Type Schema objects (batch); each entry's `type_id` is derived from its embedded `$id`. Returns `{"ok": ..., "results": [{"ok": ..., "type_id": ..., "error": ...}]}`. | | `/validate-id` | `GET` | `gts_id` query parameter. | | `/extract-id` | `POST` | JSON entity/schema object. | | `/parse-id` | `GET` | `gts_id` query parameter. | diff --git a/gts/openapi.json b/gts/openapi.json index 11d7908..872e139 100644 --- a/gts/openapi.json +++ b/gts/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "GTS Server", - "version": "0.14.1" + "version": "0.15.0" }, "paths": { "/entities": { @@ -11,16 +11,16 @@ "operationId": "get_entities_entities_get", "parameters": [ { + "name": "limit", + "in": "query", "required": false, "schema": { "type": "integer", - "maximum": 1000.0, - "minimum": 1.0, - "title": "Limit", - "default": 100 - }, - "name": "limit", - "in": "query" + "maximum": 1000, + "minimum": 1, + "default": 100, + "title": "Limit" + } } ], "responses": { @@ -30,6 +30,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Get Entities Entities Get" } } @@ -52,26 +53,46 @@ "operationId": "add_entity_entities_post", "parameters": [ { + "name": "validate", + "in": "query", "required": false, "schema": { "type": "boolean", - "title": "Validate", - "default": false - }, - "name": "validate", - "in": "query" + "default": false, + "title": "Validate" + } + }, + { + "name": "validation", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "default": false, + "title": "Validation" + } + }, + { + "name": "gts-ref-validation", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/GtsRefValidationMode", + "default": "any-valid" + } } ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Body" } } - }, - "required": true + } }, "responses": { "200": { @@ -101,13 +122,13 @@ "operationId": "get_entity_entities__gts_id__get", "parameters": [ { + "name": "gts_id", + "in": "path", "required": true, "schema": { "type": "string", "title": "Gts Id" - }, - "name": "gts_id", - "in": "path" + } } ], "responses": { @@ -117,6 +138,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Get Entity Entities Gts Id Get" } } @@ -144,6 +166,7 @@ "application/json": { "schema": { "items": { + "additionalProperties": true, "type": "object" }, "type": "array", @@ -175,19 +198,19 @@ } } }, - "/schemas": { + "/type-schemas": { "post": { - "summary": "Register schema by explicit type_id", - "operationId": "add_schema_schemas_post", + "summary": "Register a batch of GTS Type Schemas", + "operationId": "add_schemas_type_schemas_post", "requestBody": { "content": { "application/json": { "schema": { - "allOf": [ - { - "$ref": "#/components/schemas/SchemaRegister" - } - ], + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array", "title": "Body" } } @@ -222,13 +245,13 @@ "operationId": "validate_id_validate_id_get", "parameters": [ { + "name": "gts_id", + "in": "query", "required": true, "schema": { "type": "string", "title": "Gts Id" - }, - "name": "gts_id", - "in": "query" + } } ], "responses": { @@ -238,6 +261,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Validate Id Validate Id Get" } } @@ -264,6 +288,7 @@ "content": { "application/json": { "schema": { + "additionalProperties": true, "type": "object", "title": "Body" } @@ -277,6 +302,7 @@ "content": { "application/json": { "schema": { + "additionalProperties": true, "type": "object", "title": "Response Extract Id Extract Id Post" } @@ -302,13 +328,13 @@ "operationId": "parse_parse_id_get", "parameters": [ { + "name": "gts_id", + "in": "query", "required": true, "schema": { "type": "string", "title": "Gts Id" - }, - "name": "gts_id", - "in": "query" + } } ], "responses": { @@ -318,6 +344,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Parse Parse Id Get" } } @@ -342,22 +369,22 @@ "operationId": "match_id_pattern_match_id_pattern_get", "parameters": [ { + "name": "candidate", + "in": "query", "required": true, "schema": { "type": "string", "title": "Candidate" - }, - "name": "candidate", - "in": "query" + } }, { + "name": "pattern", + "in": "query", "required": true, "schema": { "type": "string", "title": "Pattern" - }, - "name": "pattern", - "in": "query" + } } ], "responses": { @@ -367,6 +394,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Match Id Pattern Match Id Pattern Get" } } @@ -391,13 +419,13 @@ "operationId": "id_to_uuid_uuid_get", "parameters": [ { + "name": "gts_id", + "in": "query", "required": true, "schema": { "type": "string", "title": "Gts Id" - }, - "name": "gts_id", - "in": "query" + } } ], "responses": { @@ -407,6 +435,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Id To Uuid Uuid Get" } } @@ -429,15 +458,63 @@ "post": { "summary": "Validate instance by GTS ID", "operationId": "validate_instance_validate_instance_post", + "parameters": [ + { + "name": "gts-ref-validation", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/GtsRefValidationMode", + "default": "any-valid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidateInstanceRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Validate Instance Validate Instance Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/validate-json": { + "post": { + "summary": "Validate unregistered JSON entity", + "operationId": "validate_json_validate_json_post", "requestBody": { "content": { "application/json": { "schema": { - "allOf": [ - { - "$ref": "#/components/schemas/ValidateInstanceRequest" - } - ], + "additionalProperties": true, + "type": "object", "title": "Body" } } @@ -450,8 +527,164 @@ "content": { "application/json": { "schema": { + "additionalProperties": true, "type": "object", - "title": "Response Validate Instance Validate Instance Post" + "title": "Response Validate Json Validate Json Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/validate-json/{gts_type}": { + "post": { + "summary": "Validate unregistered JSON instance against a type", + "operationId": "validate_json_as_type_validate_json__gts_type__post", + "parameters": [ + { + "name": "gts_type", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Gts Type" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Body" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Validate Json As Type Validate Json Gts Type Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/validate-type-schema": { + "post": { + "summary": "Validate that a derived GTS Type Schema correctly extends its base chain", + "operationId": "validate_type_schema_validate_type_schema_post", + "parameters": [ + { + "name": "gts-ref-validation", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/GtsRefValidationMode", + "default": "any-valid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidateTypeSchemaRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Validate Type Schema Validate Type Schema Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/validate-entity": { + "post": { + "summary": "Validate entity (instance or type schema) by GTS Identifier", + "operationId": "validate_entity_validate_entity_post", + "parameters": [ + { + "name": "gts-ref-validation", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/GtsRefValidationMode", + "default": "any-valid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidateEntityRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Validate Entity Validate Entity Post" } } } @@ -475,13 +708,13 @@ "operationId": "schema_graph_resolve_relationships_get", "parameters": [ { + "name": "gts_id", + "in": "query", "required": true, "schema": { "type": "string", "title": "Gts Id" - }, - "name": "gts_id", - "in": "query" + } } ], "responses": { @@ -491,6 +724,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Schema Graph Resolve Relationships Get" } } @@ -511,26 +745,26 @@ }, "/compatibility": { "get": { - "summary": "Check minor version compatibility", + "summary": "Check Type Schema evolution compatibility", "operationId": "compatibility_compatibility_get", "parameters": [ { + "name": "old_type_id", + "in": "query", "required": true, "schema": { "type": "string", - "title": "Old Schema Id" - }, - "name": "old_schema_id", - "in": "query" + "title": "Old Type Id" + } }, { + "name": "new_type_id", + "in": "query", "required": true, "schema": { "type": "string", - "title": "New Schema Id" - }, - "name": "new_schema_id", - "in": "query" + "title": "New Type Id" + } } ], "responses": { @@ -540,6 +774,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Compatibility Compatibility Get" } } @@ -566,12 +801,7 @@ "content": { "application/json": { "schema": { - "allOf": [ - { - "$ref": "#/components/schemas/CastRequest" - } - ], - "title": "Body" + "$ref": "#/components/schemas/CastRequest" } } }, @@ -583,6 +813,7 @@ "content": { "application/json": { "schema": { + "additionalProperties": true, "type": "object", "title": "Response Cast Cast Post" } @@ -608,25 +839,25 @@ "operationId": "query_query_get", "parameters": [ { + "name": "expr", + "in": "query", "required": true, "schema": { "type": "string", "title": "Expr" - }, - "name": "expr", - "in": "query" + } }, { + "name": "limit", + "in": "query", "required": false, "schema": { "type": "integer", - "maximum": 1000.0, - "minimum": 1.0, - "title": "Limit", - "default": 100 - }, - "name": "limit", - "in": "query" + "maximum": 1000, + "minimum": 1, + "default": 100, + "title": "Limit" + } } ], "responses": { @@ -636,6 +867,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Query Query Get" } } @@ -660,13 +892,13 @@ "operationId": "attr_attr_get", "parameters": [ { + "name": "gts_with_path", + "in": "query", "required": true, "schema": { "type": "string", "title": "Gts With Path" - }, - "name": "gts_with_path", - "in": "query" + } } ], "responses": { @@ -676,6 +908,7 @@ "application/json": { "schema": { "type": "object", + "additionalProperties": true, "title": "Response Attr Attr Get" } } @@ -703,18 +936,27 @@ "type": "string", "title": "Instance Id" }, - "to_schema_id": { + "to_type_id": { "type": "string", - "title": "To Schema Id" + "title": "To Type Id" } }, "type": "object", "required": [ "instance_id", - "to_schema_id" + "to_type_id" ], "title": "CastRequest" }, + "GtsRefValidationMode": { + "type": "string", + "enum": [ + "none", + "any-present", + "any-valid" + ], + "title": "GtsRefValidationMode" + }, "HTTPValidationError": { "properties": { "detail": { @@ -728,23 +970,33 @@ "type": "object", "title": "HTTPValidationError" }, - "SchemaRegister": { + "ValidateEntityRequest": { "properties": { - "type_id": { - "type": "string", - "title": "Type Id" + "entity_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Entity Id" }, - "schema": { - "type": "object", - "title": "Schema" + "gts_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Gts Id" } }, "type": "object", - "required": [ - "type_id", - "schema" - ], - "title": "SchemaRegister" + "title": "ValidateEntityRequest" }, "ValidateInstanceRequest": { "properties": { @@ -759,6 +1011,19 @@ ], "title": "ValidateInstanceRequest" }, + "ValidateTypeSchemaRequest": { + "properties": { + "type_id": { + "type": "string", + "title": "Type Id" + } + }, + "type": "object", + "required": [ + "type_id" + ], + "title": "ValidateTypeSchemaRequest" + }, "ValidationError": { "properties": { "loc": { @@ -794,4 +1059,4 @@ } } } -} +} \ No newline at end of file diff --git a/gts/pyproject.toml b/gts/pyproject.toml index 9dfd049..f7116e0 100644 --- a/gts/pyproject.toml +++ b/gts/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "gts" -version = "0.14.1" +version = "0.15.0" description = "Global Type System (GTS) helpers: identifiers, parsing, validation, and operations" readme = "README.md" authors = [{ name = "GTS Community" }] diff --git a/gts/src/gts/_server.py b/gts/src/gts/_server.py index 8554328..4f162a5 100644 --- a/gts/src/gts/_server.py +++ b/gts/src/gts/_server.py @@ -146,11 +146,6 @@ async def receive(): return response -class SchemaRegister(BaseModel): - type_id: str - type_schema: dict[str, Any] - - class CastRequest(BaseModel): instance_id: str to_type_id: str @@ -193,7 +188,7 @@ def __init__( self.host = host self.port = port self.base_url = f"http://{self.host}:{self.port}" - self.app = FastAPI(title="GTS Server", version="0.14.1") + self.app = FastAPI(title="GTS Server", version="0.15.0") self.app.add_middleware( _RequestLoggingMiddleware, verbose=self.ops.verbose, @@ -235,9 +230,9 @@ def _register_routes(self) -> None: ) app.add_api_route( "/type-schemas", - self.add_schema, + self.add_schemas, methods=["POST"], - summary="Register a GTS Type Schema under an explicit type_id", + summary="Register a batch of GTS Type Schemas", response_class=JSONResponse, ) @@ -367,11 +362,11 @@ async def add_entities( ) -> JSONResponse: return JSONResponse(self.ops.add_entities(body).to_dict()) - async def add_schema(self, body: SchemaRegister) -> JSONResponse: - result = self.ops.add_schema(body.type_id, body.type_schema) - return JSONResponse( - result.to_dict(), status_code=409 if result.conflict else 200 - ) + async def add_schemas( + self, + body: list[dict[str, Any]] = Body(...), + ) -> JSONResponse: + return JSONResponse(self.ops.add_schemas(body).to_dict()) async def validate_id(self, id: str = Query(..., alias="gts_id")) -> dict[str, Any]: return self.ops.validate_id(id).to_dict() diff --git a/gts/src/gts/ops.py b/gts/src/gts/ops.py index f9709b9..905fc49 100644 --- a/gts/src/gts/ops.py +++ b/gts/src/gts/ops.py @@ -6,7 +6,7 @@ from pathlib import Path as SysPath from typing import Any -from ._naming import looks_like_gts +from ._naming import looks_like_gts, strip_scheme from .entities import DEFAULT_GTS_CONFIG, GtsConfig, GtsEntity from .files_reader import GtsFileReader from .gts import GtsID, GtsWildcard @@ -291,22 +291,34 @@ def to_dict(self) -> dict[str, Any]: @dataclass class GtsAddSchemaResult: - """Result of adding a schema to the store.""" + """Result of adding a single GTS Type Schema to the store.""" ok: bool - id: str = "" + type_id: str | None = None error: str = "" conflict: bool = False def to_dict(self) -> dict[str, Any]: - result: dict[str, Any] = {"ok": self.ok} - if self.ok: - result["id"] = self.id - else: + result: dict[str, Any] = {"ok": self.ok, "type_id": self.type_id} + if not self.ok: result["error"] = self.error return result +@dataclass +class GtsAddSchemasResult: + """Result of registering a batch of GTS Type Schemas.""" + + ok: bool + results: list[GtsAddSchemaResult] + + def to_dict(self) -> dict[str, Any]: + return { + "ok": self.ok, + "results": [r.to_dict() for r in self.results], + } + + @dataclass class GtsExtractIdResult: """Result of extracting ID information from content.""" @@ -476,7 +488,28 @@ def add_entities( ok = all(r.ok for r in results) return GtsAddEntitiesResult(ok=ok, results=results) - def add_schema(self, type_id: str, schema: dict[str, Any]) -> GtsAddSchemaResult: + def add_schemas( + self, schemas: builtins.list[dict[str, Any]] + ) -> GtsAddSchemasResult: + """Register a batch of GTS Type Schemas. + + Each entry's GTS Type Identifier is derived from its embedded ``$id``; + the aggregate ``ok`` is ``True`` only when every entry registered. + """ + results = [self.add_schema(schema) for schema in schemas] + ok = all(r.ok for r in results) + return GtsAddSchemasResult(ok=ok, results=results) + + def add_schema(self, schema: dict[str, Any]) -> GtsAddSchemaResult: + """Register a single GTS Type Schema, deriving its type_id from ``$id``.""" + embedded_id = schema.get("$id") if isinstance(schema, dict) else None + if not isinstance(embedded_id, str) or not embedded_id: + return GtsAddSchemaResult( + ok=False, + type_id=None, + error="GTS Type Schema must contain a top-level $id in gts:// form", + ) + type_id = strip_scheme(embedded_id) try: previous = self.store.get(type_id) if ( @@ -486,13 +519,14 @@ def add_schema(self, type_id: str, schema: dict[str, Any]) -> GtsAddSchemaResult ): return GtsAddSchemaResult( ok=False, + type_id=type_id, error=f"Entity '{type_id}' is already registered with different content", conflict=True, ) self.store.register_schema(type_id, schema) - return GtsAddSchemaResult(ok=True, id=type_id) + return GtsAddSchemaResult(ok=True, type_id=type_id) except Exception as e: # noqa: BLE001 - converted to a result object at API boundary - return GtsAddSchemaResult(ok=False, error=str(e)) + return GtsAddSchemaResult(ok=False, type_id=type_id, error=str(e)) def validate_id(self, gts_id: str) -> GtsIdValidationResult: # Check if it's a wildcard pattern (contains *) diff --git a/tests/test_ops.py b/tests/test_ops.py index 839abbf..c5234cd 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -153,39 +153,54 @@ def test_add_entities_rejects_unsupported_x_gts_ref_pointer(self, ops): assert "must be a GTS identifier" in result.results[0].error -class TestAddSchemaLegacy: - def test_add_schema_legacy_success(self, ops): +class TestAddSchema: + def test_add_schema_success(self, ops): schema = { "$schema": "http://json-schema.org/draft-07/schema#", "$id": "gts://gts.x.test._.legacy.v1~", "type": "object", } - result = ops.add_schema("gts.x.test._.legacy.v1~", schema) + result = ops.add_schema(schema) assert result.ok is True - assert result.id == "gts.x.test._.legacy.v1~" + assert result.type_id == "gts.x.test._.legacy.v1~" - def test_add_schema_legacy_changed_content_is_conflict(self, ops): + def test_add_schema_changed_content_is_conflict(self, ops): dialect = "http://json-schema.org/draft-07/schema#" schema_id = "gts://gts.x.test._.legacy.v1~" assert ( ops.add_schema( - "gts.x.test._.legacy.v1~", {"$schema": dialect, "$id": schema_id, "type": "object"}, ).ok is True ) result = ops.add_schema( - "gts.x.test._.legacy.v1~", {"$schema": dialect, "$id": schema_id, "type": "string"}, ) assert result.ok is False assert result.conflict is True - def test_add_schema_legacy_failure(self, ops): - result = ops.add_schema("gts.x.test._.legacy.v1", {"type": "object"}) + def test_add_schema_missing_id_failure(self, ops): + result = ops.add_schema({"type": "object"}) assert result.ok is False - assert result.error + assert "$id" in result.error + + def test_add_schemas_batch_partial(self, ops): + dialect = "http://json-schema.org/draft-07/schema#" + result = ops.add_schemas( + [ + { + "$schema": dialect, + "$id": "gts://gts.x.test._.batch_ok.v1~", + "type": "object", + }, + {"$schema": dialect, "type": "object"}, + ] + ) + assert result.ok is False + assert result.results[0].ok is True + assert result.results[0].type_id == "gts.x.test._.batch_ok.v1~" + assert result.results[1].ok is False class TestValidateId: diff --git a/tests/test_server.py b/tests/test_server.py index f199556..63fc9e2 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -105,36 +105,36 @@ def test_add_entities(self, server): resp = run(server.add_entities(body=[SCHEMA, INSTANCE])) assert resp.status_code == 200 - def test_add_schema(self, server): - from gts._server import SchemaRegister + def test_add_schemas(self, server): + import json - body = SchemaRegister( - type_id="gts.x.test._.bar.v1~", - type_schema={ + body = [ + { "$schema": "http://json-schema.org/draft-07/schema#", "$id": "gts://gts.x.test._.bar.v1~", "type": "object", }, - ) - resp = run(server.add_schema(body)) + ] + resp = run(server.add_schemas(body)) assert resp.status_code == 200 + payload = json.loads(resp.body) + assert payload["ok"] is True + assert payload["results"][0]["type_id"] == "gts.x.test._.bar.v1~" - def test_add_schema_changed_content_conflict(self, server): - from gts._server import SchemaRegister + def test_add_schemas_changed_content_conflict(self, server): + import json dialect = "http://json-schema.org/draft-07/schema#" schema_id = "gts://gts.x.test._.bar.v1~" - initial = SchemaRegister( - type_id="gts.x.test._.bar.v1~", - type_schema={"$schema": dialect, "$id": schema_id, "type": "object"}, - ) - changed = SchemaRegister( - type_id="gts.x.test._.bar.v1~", - type_schema={"$schema": dialect, "$id": schema_id, "type": "string"}, - ) + initial = [{"$schema": dialect, "$id": schema_id, "type": "object"}] + changed = [{"$schema": dialect, "$id": schema_id, "type": "string"}] - assert run(server.add_schema(initial)).status_code == 200 - assert run(server.add_schema(changed)).status_code == 409 + assert run(server.add_schemas(initial)).status_code == 200 + resp = run(server.add_schemas(changed)) + assert resp.status_code == 200 + payload = json.loads(resp.body) + assert payload["ok"] is False + assert payload["results"][0]["ok"] is False def test_validate_id(self, server): result = run(server.validate_id(id="gts.x.test._.foo.v1~")) From 0f7ac5448b58397955835feb42915966c7113c1c Mon Sep 17 00:00:00 2001 From: Artifizer Date: Fri, 25 Sep 2026 17:06:48 +0300 Subject: [PATCH 07/27] fix(validation): enforce dialect checks on instance paths Reject trait schemas that declare a dialect different from their host type. Validate the registered schema chain before checking transient instance content so mixed-dialect reference graphs cannot be accepted through /validate-json. Signed-off-by: Artifizer --- gts/src/gts/store.py | 8 ++++++++ tests/test_ops.py | 27 +++++++++++++++++++++++++++ tests/test_store_extra.py | 19 +++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index 3d543f8..abb1c78 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -717,6 +717,13 @@ def _build_effective_traits( level_schemas: list[Any] = [] traits.collect_trait_schema_from_value(content, level_schemas) for ts in level_schemas: + if isinstance(ts, dict) and "$schema" in ts: + trait_dialect = self._schema_dialect(ts) + host_dialect = self._schema_dialect(content) + if trait_dialect != host_dialect: + raise ValueError( + f"trait schema dialect {trait_dialect} differs from host dialect {host_dialect}" + ) # Inline local JSON Pointer refs against the host document, then # resolve any gts:// refs so the composed schema is self-contained. inlined = traits.inline_local_pointers(ts, content) @@ -1049,6 +1056,7 @@ def validate_instance_content( except KeyError as error: raise StoreGtsSchemaNotFound(schema_type.id) from error + self._validate_schema_chain(schema_type.id) self._schema_dialect(schema) if isinstance(schema, dict) and self._content_is_abstract(schema): raise ValueError( diff --git a/tests/test_ops.py b/tests/test_ops.py index c5234cd..f322a3a 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -354,6 +354,33 @@ def test_validates_instance_against_explicit_type_without_storing_it(self, ops): assert result.ok is True assert result.type_id == "gts.x.test._.foo.v1~" + def test_rejects_instance_of_mixed_dialect_schema_graph(self, ops): + ops.add_entity( + { + "$id": "gts://gts.x.test._.foreign.v1~", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + } + ) + ops.add_entity( + { + "$id": "gts://gts.x.test._.host.v1~", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "allOf": [{"$ref": "gts://gts.x.test._.foreign.v1~"}], + } + ) + + result = ops.validate_json( + { + "id": "gts.x.test._.host.v1~x.test._.item.v1", + "type": "gts.x.test._.host.v1~", + } + ) + + assert result.ok is False + assert "mixes JSON Schema dialects" in result.error + def test_does_not_mutate_registry(self, ops, monkeypatch): monkeypatch.setattr( ops.store, diff --git a/tests/test_store_extra.py b/tests/test_store_extra.py index f0773dc..3eecf93 100644 --- a/tests/test_store_extra.py +++ b/tests/test_store_extra.py @@ -382,6 +382,25 @@ def test_transitive_ref_dialect_mismatch_raises(self): with pytest.raises(ValueError, match="gts.x.test._.foreign.v1~"): store._validate_schema_chain("gts.x.test._.host.v1~") + def test_trait_resource_dialect_mismatch_raises(self): + schema_id = "gts.x.test._.trait_resource.v1~" + schema = _schema_entity( + schema_id, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "x-gts-traits-schema": { + "$id": "https://example.com/gts/legacy-traits", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + }, + }, + ) + store = GtsStore(reader=None) + store.register(schema) + + with pytest.raises(ValueError, match="differs from host dialect"): + store.validate_schema(schema_id) + def test_incompatible_derivation_raises(self): base = _schema_entity( "gts.x.test._.base.v1~", From 00969bd72b4a91078625d69f4a8d41944b9781c2 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Fri, 25 Sep 2026 20:33:45 +0300 Subject: [PATCH 08/27] fix(store): make registration atomic and isolated Serialize conflict checks, registration, validation, and rollback under a reentrant store transaction so concurrent writers cannot overwrite or remove each other's state. Store and return defensive entity copies, including reader-backed entries and collection snapshots.\n\nAdd a schema reference expansion budget to fail safely on pathological graphs, and cover transaction serialization, mutation isolation, and bounded expansion with regression tests. Signed-off-by: Artifizer --- gts/src/gts/_json_validation.py | 8 +-- gts/src/gts/ops.py | 90 ++++++++++++++++--------------- gts/src/gts/store.py | 96 +++++++++++++++++++++------------ tests/test_store_extra.py | 53 ++++++++++++++++++ 4 files changed, 166 insertions(+), 81 deletions(-) diff --git a/gts/src/gts/_json_validation.py b/gts/src/gts/_json_validation.py index c7e23df..b1bb05c 100644 --- a/gts/src/gts/_json_validation.py +++ b/gts/src/gts/_json_validation.py @@ -256,7 +256,8 @@ def _validate_schemas(self, store: GtsStore) -> None: if not entity.is_schema or not entity.gts_id: continue gid = entity.gts_id - if store.get(gid.id) is not entity: + stored = store.get(gid.id) + if stored is None or stored.content != entity.content: continue depth = len(gid.gts_id_segments) file = entity.file.path if entity.file else entity.label @@ -290,8 +291,9 @@ def _validate_instances(self, store: GtsStore) -> None: key = self._registry_key(entity) if key is None: continue - # Skip rejected duplicates: only validate the registered entity - if store.get(key) is not entity: + # Skip rejected duplicates: only validate content retained by the registry. + stored = store.get(key) + if stored is None or stored.content != entity.content: continue depth = self._entity_depth(entity) gts_id_str = entity.gts_id.id if entity.gts_id else "" diff --git a/gts/src/gts/ops.py b/gts/src/gts/ops.py index 905fc49..c022f16 100644 --- a/gts/src/gts/ops.py +++ b/gts/src/gts/ops.py @@ -437,38 +437,39 @@ def add_entity( ) store_key = entity.gts_id.id if entity.is_schema else entity.raw_id - previous = self.store.get(store_key) - if ( - previous - and not self.allow_entity_updates - and previous.content != entity.content - ): - return GtsAddEntityResult( - ok=False, - error=f"Entity '{store_key}' is already registered with different content", - is_type_schema=entity.is_schema, - conflict=True, - ) - self.store.register(entity) + with self.store.transaction(): + previous = self.store.get(store_key) + if ( + previous + and not self.allow_entity_updates + and previous.content != entity.content + ): + return GtsAddEntityResult( + ok=False, + error=f"Entity '{store_key}' is already registered with different content", + is_type_schema=entity.is_schema, + conflict=True, + ) + self.store.register(entity) - try: - if entity.is_schema: - self.store.validate_schema_basic(entity.gts_id.id) - if validate: - self.store.validate_schema(entity.gts_id.id, gts_ref_validation) - elif validate: - self.store.validate_instance( - entity.raw_id or entity.gts_id.id, gts_ref_validation + try: + if entity.is_schema: + self.store.validate_schema_basic(entity.gts_id.id) + if validate: + self.store.validate_schema(entity.gts_id.id, gts_ref_validation) + elif validate: + self.store.validate_instance( + entity.raw_id or entity.gts_id.id, gts_ref_validation + ) + except Exception as e: # noqa: BLE001 - converted to a result object at API boundary + self.store.unregister(store_key) + if previous: + self.store.register(previous) + return GtsAddEntityResult( + ok=False, + error=f"Validation failed: {e!s}", + is_type_schema=entity.is_schema, ) - except Exception as e: # noqa: BLE001 - converted to a result object at API boundary - self.store.unregister(store_key) - if previous: - self.store.register(previous) - return GtsAddEntityResult( - ok=False, - error=f"Validation failed: {e!s}", - is_type_schema=entity.is_schema, - ) # Return gts_id if available, otherwise raw_id entity_id = entity.gts_id.id if entity.gts_id else (entity.raw_id or "") @@ -511,20 +512,21 @@ def add_schema(self, schema: dict[str, Any]) -> GtsAddSchemaResult: ) type_id = strip_scheme(embedded_id) try: - previous = self.store.get(type_id) - if ( - previous - and not self.allow_entity_updates - and previous.content != schema - ): - return GtsAddSchemaResult( - ok=False, - type_id=type_id, - error=f"Entity '{type_id}' is already registered with different content", - conflict=True, - ) - self.store.register_schema(type_id, schema) - return GtsAddSchemaResult(ok=True, type_id=type_id) + with self.store.transaction(): + previous = self.store.get(type_id) + if ( + previous + and not self.allow_entity_updates + and previous.content != schema + ): + return GtsAddSchemaResult( + ok=False, + type_id=type_id, + error=f"Entity '{type_id}' is already registered with different content", + conflict=True, + ) + self.store.register_schema(type_id, schema) + return GtsAddSchemaResult(ok=True, type_id=type_id) except Exception as e: # noqa: BLE001 - converted to a result object at API boundary return GtsAddSchemaResult(ok=False, type_id=type_id, error=str(e)) diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index abb1c78..0567adc 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -1,9 +1,12 @@ from __future__ import annotations +import copy import logging +import threading import uuid from abc import ABC, abstractmethod from collections.abc import Iterator +from contextlib import contextmanager from typing import Any from jsonschema import RefResolver @@ -22,6 +25,8 @@ logger = logging.getLogger(__name__) +MAX_SCHEMA_REF_EXPANSIONS = 10_000 + def _require_schema_id(value: str) -> GtsID: try: @@ -123,6 +128,7 @@ def __init__(self, reader: GtsReader) -> None: """ self._by_id: dict[str, GtsEntity] = {} self._reader = reader + self._lock = threading.RLock() # Populate entities from reader if provided if self._reader: @@ -137,7 +143,7 @@ def _populate_from_reader(self) -> None: for entity in self._reader: if entity.gts_id and entity.gts_id.id: - self._by_id[entity.gts_id.id] = entity + self._by_id[entity.gts_id.id] = copy.deepcopy(entity) def register(self, entity: GtsEntity) -> None: """Register a GtsEntity in the store. @@ -148,21 +154,29 @@ def register(self, entity: GtsEntity) -> None: # Instances should remain addressable by the id value they carry. # For plain UUID anonymous instances, `gts_id` may be inferred from # the `type` field while `raw_id` is the UUID we must look up by. - if not entity.is_schema and entity.raw_id: - self._by_id[entity.raw_id] = entity - return + stored = copy.deepcopy(entity) + with self._lock: + if not stored.is_schema and stored.raw_id: + self._by_id[stored.raw_id] = stored + return - if entity.gts_id and entity.gts_id.id: - self._by_id[entity.gts_id.id] = entity - elif entity.raw_id: - # Allow non-GTS entities with raw_id (e.g., UUIDs or simple strings) - self._by_id[entity.raw_id] = entity - else: - raise ValueError("Entity must have a valid gts_id or raw_id") + if stored.gts_id and stored.gts_id.id: + self._by_id[stored.gts_id.id] = stored + elif stored.raw_id: + # Allow non-GTS entities with raw_id (e.g., UUIDs or simple strings) + self._by_id[stored.raw_id] = stored + else: + raise ValueError("Entity must have a valid gts_id or raw_id") def unregister(self, entity_id: str) -> None: """Remove an entity from the in-memory registry if it is present.""" - self._by_id.pop(entity_id, None) + with self._lock: + self._by_id.pop(entity_id, None) + + @contextmanager + def transaction(self): + with self._lock: + yield def register_schema(self, type_id: str, schema: dict[str, Any]) -> None: """ @@ -184,8 +198,9 @@ def register_schema(self, type_id: str, schema: dict[str, Any]) -> None: raise ValueError( f"Embedded $id '{embedded_id}' must match external type_id '{type_id}'" ) - entity = GtsEntity(content=schema, gts_id=gts_id, is_schema=True) - self._by_id[gts_id.id] = entity + entity = GtsEntity(content=copy.deepcopy(schema), gts_id=gts_id, is_schema=True) + with self._lock: + self._by_id[gts_id.id] = entity def get(self, entity_id: str) -> GtsEntity | None: """ @@ -198,18 +213,19 @@ def get(self, entity_id: str) -> GtsEntity | None: scheme themselves. """ entity_id = strip_scheme(entity_id) - # Check cache first - if entity_id in self._by_id: - return self._by_id[entity_id] + with self._lock: + entity = self._by_id.get(entity_id) + if entity is not None: + return copy.deepcopy(entity) - # Try to fetch from reader - if self._reader: - entity = self._reader.read_by_id(entity_id) - if entity: - self._by_id[entity_id] = entity - return entity + if self._reader: + entity = self._reader.read_by_id(entity_id) + if entity: + stored = copy.deepcopy(entity) + self._by_id[entity_id] = stored + return copy.deepcopy(stored) - return None + return None def get_schema_content(self, type_id: str) -> dict[str, Any]: """Get schema content as dict (legacy method for backward compatibility).""" @@ -233,7 +249,7 @@ def resolve_gts_ref(uri: str) -> dict[str, Any]: # Create a store dict that maps GTS IDs to their schema content store = {} - for entity_id, entity in self._by_id.items(): + for entity_id, entity in self.items(): if entity.is_schema and isinstance(entity.content, dict): store[entity_id] = entity.content @@ -245,7 +261,7 @@ def resolve_gts_ref(uri: str) -> dict[str, Any]: def _create_reference_registry(self) -> Registry: registry = Registry() - for entity_id, entity in self._by_id.items(): + for entity_id, entity in self.items(): if entity.is_schema and isinstance(entity.content, dict): resource = Resource.from_contents( _without_x_gts_ref(entity.content), @@ -256,7 +272,8 @@ def _create_reference_registry(self) -> Registry: def items(self): """Return all entity ID and entity pairs.""" - return self._by_id.items() + with self._lock: + return tuple((entity_id, copy.deepcopy(entity)) for entity_id, entity in self._by_id.items()) @staticmethod def _validate_schema_refs(schema: dict[str, Any], path: str = "") -> None: @@ -627,7 +644,7 @@ def _resolve_schema_refs(self, schema: Any) -> Any: import copy return self._inline_refs( - copy.deepcopy(schema), set(), self._supports_ref_siblings(schema) + copy.deepcopy(schema), set(), self._supports_ref_siblings(schema), [0] ) @staticmethod @@ -638,9 +655,14 @@ def _supports_ref_siblings(schema: Any) -> bool: ) def _inline_refs( - self, node: Any, seen: set[str], supports_ref_siblings: bool + self, + node: Any, + seen: set[str], + supports_ref_siblings: bool, + expansions: list[int] | None = None, ) -> Any: """Recursively inline $ref references, guarding against cycles.""" + expansions = expansions if expansions is not None else [0] if isinstance(node, dict): ref_uri = node.get("$ref") if isinstance(ref_uri, str): @@ -652,6 +674,11 @@ def _inline_refs( if ref_id in seen: # Cycle detected: leave the $ref unresolved. return node + expansions[0] += 1 + if expansions[0] > MAX_SCHEMA_REF_EXPANSIONS: + raise ValueError( + f"schema reference expansion exceeds limit of {MAX_SCHEMA_REF_EXPANSIONS}" + ) try: ref_schema = self.get_schema_content(ref_id) except KeyError: @@ -662,6 +689,7 @@ def _inline_refs( copy.deepcopy(ref_schema), seen | {ref_id}, self._supports_ref_siblings(ref_schema), + expansions, ) if supports_ref_siblings and len(node) > 1: siblings = { @@ -671,18 +699,18 @@ def _inline_refs( "allOf": [ resolved, self._inline_refs( - siblings, seen, supports_ref_siblings + siblings, seen, supports_ref_siblings, expansions ), ] } return resolved return { - key: self._inline_refs(value, seen, supports_ref_siblings) + key: self._inline_refs(value, seen, supports_ref_siblings, expansions) for key, value in node.items() } if isinstance(node, list): return [ - self._inline_refs(item, seen, supports_ref_siblings) for item in node + self._inline_refs(item, seen, supports_ref_siblings, expansions) for item in node ] return node @@ -1030,7 +1058,7 @@ def _has_valid_wildcard_match( gts_ref_validation: GtsRefValidationMode, ) -> bool: wildcard = GtsWildcard(pattern) - for entity_id in self._by_id: + for entity_id, _ in self.items(): try: if not GtsID(entity_id).wildcard_match(wildcard): continue @@ -1481,7 +1509,7 @@ def query(self, expr: str, limit: int = 100) -> GtsStoreQueryResult: return result # Filter entities - for entity in self._by_id.values(): + for _, entity in self.items(): if len(result.results) >= limit: break if not isinstance(entity.content, dict) or not entity.gts_id: diff --git a/tests/test_store_extra.py b/tests/test_store_extra.py index 3eecf93..2a6c846 100644 --- a/tests/test_store_extra.py +++ b/tests/test_store_extra.py @@ -1,5 +1,6 @@ """Additional coverage-focused tests for gts.store.GtsStore.""" +import threading from collections.abc import Iterator from typing import Optional @@ -8,6 +9,7 @@ from gts.gts import GtsID from gts.schema_validation import PATTERN_TIMEOUT_SECONDS, validator_for from gts.store import ( + MAX_SCHEMA_REF_EXPANSIONS, GtsReader, GtsStore, StoreGtsEntityNotFound, @@ -88,6 +90,57 @@ def test_unregister_missing_id_noop(self): store = GtsStore(reader=None) store.unregister("gts.x.test._.missing.v1~") # should not raise + def test_store_uses_defensive_copies(self): + store = GtsStore(reader=None) + entity = _schema_entity("gts.x.test._.copy.v1~") + store.register(entity) + entity.content["type"] = "array" + first = store.get("gts.x.test._.copy.v1~") + assert first.content["type"] == "object" + first.content["type"] = "number" + items = dict(store.items()) + items["gts.x.test._.copy.v1~"].content["type"] = "boolean" + assert store.get("gts.x.test._.copy.v1~").content["type"] == "object" + + def test_transaction_serializes_writers(self): + store = GtsStore(reader=None) + entered = threading.Event() + release = threading.Event() + completed = threading.Event() + + def first_writer(): + with store.transaction(): + entered.set() + release.wait() + + def second_writer(): + entered.wait() + store.register(_schema_entity("gts.x.test._.serialized.v1~")) + completed.set() + + first = threading.Thread(target=first_writer) + second = threading.Thread(target=second_writer) + first.start() + second.start() + assert not completed.wait(0.05) + release.set() + first.join() + second.join() + assert completed.is_set() + + def test_reference_expansion_budget(self): + store = GtsStore(reader=None) + target_id = "gts.x.test._.budget.v1~" + store.register(_schema_entity(target_id)) + schema = { + "allOf": [ + {"$ref": f"gts://{target_id}"} + for _ in range(MAX_SCHEMA_REF_EXPANSIONS + 1) + ] + } + with pytest.raises(ValueError, match="expansion exceeds limit"): + store._resolve_schema_refs(schema) + class TestValidateSchemaRefs: def test_local_ref_valid(self): From aef60729d7c809db39f2dc5b12bd590dd598afd2 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Fri, 25 Sep 2026 23:54:52 +0300 Subject: [PATCH 09/27] fix(store): cache schema registry and secure diagnostics Cache and invalidate immutable schema registries, use linear-time dependency queues, and replace deprecated RefResolver use. Expose the high-level operations API and stop verbose HTTP logging from buffering or disclosing entity bodies. Signed-off-by: Artifizer --- gts/src/gts/__init__.py | 36 ++++++++++++++++ gts/src/gts/_server.py | 83 ++----------------------------------- gts/src/gts/schema_cast.py | 5 ++- gts/src/gts/store.py | 85 +++++++++++++++++++------------------- tests/test_ops.py | 5 +++ tests/test_server.py | 25 +++++++++-- tests/test_store_extra.py | 17 ++++++++ 7 files changed, 129 insertions(+), 127 deletions(-) diff --git a/gts/src/gts/__init__.py b/gts/src/gts/__init__.py index 3bc3e95..8933097 100644 --- a/gts/src/gts/__init__.py +++ b/gts/src/gts/__init__.py @@ -15,6 +15,25 @@ GtsWildcard, ) from .gts_ref_validation import GtsRefValidationMode +from .ops import ( + GtsAddEntitiesResult, + GtsAddEntityResult, + GtsAddSchemaResult, + GtsAddSchemasResult, + GtsEntitiesListResult, + GtsEntityInfo, + GtsEntityValidationResult, + GtsExtractIdResult, + GtsGetEntityResult, + GtsIdMatchResult, + GtsIdParseResult, + GtsIdValidationResult, + GtsJsonValidationResult, + GtsOps, + GtsSchemaGraphResult, + GtsUuidResult, + GtsValidationResult, +) from .path_resolver import GtsPathResolver from .store import ( GtsReader, @@ -23,16 +42,33 @@ __all__ = [ "DEFAULT_GTS_CONFIG", + "GtsAddEntitiesResult", + "GtsAddEntityResult", + "GtsAddSchemaResult", + "GtsAddSchemasResult", "GtsConfig", + "GtsEntitiesListResult", "GtsEntity", + "GtsEntityInfo", + "GtsEntityValidationResult", + "GtsExtractIdResult", "GtsFile", "GtsFileReader", + "GtsGetEntityResult", "GtsID", + "GtsIdMatchResult", + "GtsIdParseResult", "GtsIdSegment", + "GtsIdValidationResult", + "GtsJsonValidationResult", + "GtsOps", "GtsPathResolver", "GtsReader", "GtsRefValidationMode", + "GtsSchemaGraphResult", "GtsStore", + "GtsUuidResult", + "GtsValidationResult", "GtsWildcard", "JsonEntity", # Backward compatibility aliases diff --git a/gts/src/gts/_server.py b/gts/src/gts/_server.py index 4f162a5..9087aaf 100644 --- a/gts/src/gts/_server.py +++ b/gts/src/gts/_server.py @@ -45,32 +45,13 @@ def __init__(self, app: FastAPI, verbose: int) -> None: self.verbose = verbose async def dispatch(self, request, call_next): - if not self.verbose: - response = await call_next(request) - response.headers["connection"] = "close" - return response - start = time.time() - - # Cache request body for DEBUG logging (verbose >= 2) - cached_body = None - if self.verbose >= 2: - # Read and cache the request body - cached_body = await request.body() - - # Create a new request with the cached body - from starlette.requests import Request - - async def receive(): - return {"type": "http.request", "body": cached_body} - - request = Request(request.scope, receive) - response = await call_next(request) response.headers["connection"] = "close" - dur = (time.time() - start) * 1000.0 + if not self.verbose: + return response - # Determine status color + dur = (time.time() - start) * 1000.0 if 200 <= response.status_code < 300: status_color = Colors.GREEN elif 300 <= response.status_code < 400: @@ -78,10 +59,6 @@ async def receive(): else: status_color = Colors.RED - # Log response at INFO level (verbose >= 1). - # Neutralize CR/LF in the request-derived path to prevent log forging - # (CWE-117); ASGI percent-decodes scope["path"], so it may contain - # newlines that would otherwise inject forged log records. safe_path = request.url.path.replace("\r", "\\r").replace("\n", "\\n") logger.info( f"{Colors.CYAN}{request.method}{Colors.RESET} " @@ -89,60 +66,6 @@ async def receive(): f"{status_color}{response.status_code}{Colors.RESET} " f"in {Colors.MAGENTA}{dur:.1f}ms{Colors.RESET}" ) - - # Log request body at DEBUG level (verbose >= 2) - if self.verbose >= 2 and cached_body: - try: - import json - - body_json = json.loads(cached_body.decode("utf-8")) - body_str = json.dumps(body_json, indent=2) - logger.debug( - f"{Colors.DIM}Request body:{Colors.RESET}\n" - f"{Colors.GRAY}{body_str}{Colors.RESET}" - ) - except Exception: # noqa: BLE001 - best-effort debug logging - body_str = cached_body.decode("utf-8", errors="replace") - logger.debug( - f"{Colors.DIM}Request body (raw):{Colors.RESET}\n" - f"{Colors.GRAY}{body_str}{Colors.RESET}" - ) - - # Log response body at DEBUG level (verbose >= 2) - if self.verbose >= 2: - # Read response body - from starlette.responses import Response, StreamingResponse - - if isinstance(response, (Response, StreamingResponse)): - response_body = b"" - async for chunk in response.body_iterator: - response_body += chunk - - if response_body: - try: - import json - - body_json = json.loads(response_body.decode("utf-8")) - body_str = json.dumps(body_json, indent=2) - logger.debug( - f"{Colors.DIM}Response body:{Colors.RESET}\n" - f"{Colors.GRAY}{body_str}{Colors.RESET}" - ) - except Exception: # noqa: BLE001 - best-effort debug logging - body_str = response_body.decode("utf-8", errors="replace") - logger.debug( - f"{Colors.DIM}Response body (raw):{Colors.RESET}\n" - f"{Colors.GRAY}{body_str}{Colors.RESET}" - ) - - # Recreate response with the body - return Response( - content=response_body, - status_code=response.status_code, - headers=dict(response.headers), - media_type=response.media_type, - ) - return response diff --git a/gts/src/gts/schema_cast.py b/gts/src/gts/schema_cast.py index b197d9c..db5c71b 100644 --- a/gts/src/gts/schema_cast.py +++ b/gts/src/gts/schema_cast.py @@ -6,6 +6,7 @@ from typing import Any from jsonschema import exceptions as js_exceptions +from referencing import Registry from .compatibility import UNKNOWN, dialects_differ from .gts import GtsID @@ -399,7 +400,9 @@ def _validate_with_gts_id_tolerance( modified_schema = GtsEntityCastResult._remove_gts_const_constraints(schema) validator_class = validator_for(modified_schema) - if resolver is not None: + if isinstance(resolver, Registry): + validator = validator_class(modified_schema, registry=resolver) + elif resolver is not None: validator = validator_class(modified_schema, resolver=resolver) else: validator = validator_class(modified_schema) diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index 0567adc..c49bd1d 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -5,11 +5,11 @@ import threading import uuid from abc import ABC, abstractmethod +from collections import deque from collections.abc import Iterator from contextlib import contextmanager from typing import Any -from jsonschema import RefResolver from referencing import Registry, Resource from referencing.jsonschema import DRAFT202012 @@ -119,7 +119,7 @@ def to_dict(self) -> dict[str, Any]: class GtsStore: - def __init__(self, reader: GtsReader) -> None: + def __init__(self, reader: GtsReader | None = None) -> None: """ Initialize GtsStore with an optional GtsReader. @@ -129,6 +129,7 @@ def __init__(self, reader: GtsReader) -> None: self._by_id: dict[str, GtsEntity] = {} self._reader = reader self._lock = threading.RLock() + self._reference_registry: Registry | None = None # Populate entities from reader if provided if self._reader: @@ -144,6 +145,10 @@ def _populate_from_reader(self) -> None: for entity in self._reader: if entity.gts_id and entity.gts_id.id: self._by_id[entity.gts_id.id] = copy.deepcopy(entity) + self._reference_registry = None + + def _invalidate_reference_registry(self) -> None: + self._reference_registry = None def register(self, entity: GtsEntity) -> None: """Register a GtsEntity in the store. @@ -167,11 +172,15 @@ def register(self, entity: GtsEntity) -> None: self._by_id[stored.raw_id] = stored else: raise ValueError("Entity must have a valid gts_id or raw_id") + if stored.is_schema: + self._invalidate_reference_registry() def unregister(self, entity_id: str) -> None: """Remove an entity from the in-memory registry if it is present.""" with self._lock: - self._by_id.pop(entity_id, None) + removed = self._by_id.pop(entity_id, None) + if removed is not None and removed.is_schema: + self._invalidate_reference_registry() @contextmanager def transaction(self): @@ -201,6 +210,7 @@ def register_schema(self, type_id: str, schema: dict[str, Any]) -> None: entity = GtsEntity(content=copy.deepcopy(schema), gts_id=gts_id, is_schema=True) with self._lock: self._by_id[gts_id.id] = entity + self._invalidate_reference_registry() def get(self, entity_id: str) -> GtsEntity | None: """ @@ -223,6 +233,8 @@ def get(self, entity_id: str) -> GtsEntity | None: if entity: stored = copy.deepcopy(entity) self._by_id[entity_id] = stored + if stored.is_schema: + self._invalidate_reference_registry() return copy.deepcopy(stored) return None @@ -234,47 +246,34 @@ def get_schema_content(self, type_id: str) -> dict[str, Any]: return entity.content raise KeyError(f"Schema not found: {type_id}") - def _create_ref_resolver(self, schema: dict[str, Any]) -> RefResolver: - """Create a custom RefResolver that can resolve GTS ID references from the store.""" - - def resolve_gts_ref(uri: str) -> dict[str, Any]: - """Resolve a GTS ID reference to its schema content. - - ``get_schema_content`` normalizes the ``gts://`` scheme internally. - """ - try: - return self.get_schema_content(uri) - except KeyError as e: - raise ValueError(f"Unresolvable: {strip_scheme(uri)}") from e - - # Create a store dict that maps GTS IDs to their schema content - store = {} - for entity_id, entity in self.items(): - if entity.is_schema and isinstance(entity.content, dict): - store[entity_id] = entity.content - - # Create RefResolver with custom handlers - # Issue #32: Support "gts" scheme - handlers = {"": resolve_gts_ref, "gts": resolve_gts_ref} - resolver = RefResolver.from_schema(schema, store=store, handlers=handlers) - return resolver - def _create_reference_registry(self) -> Registry: - registry = Registry() - for entity_id, entity in self.items(): - if entity.is_schema and isinstance(entity.content, dict): - resource = Resource.from_contents( - _without_x_gts_ref(entity.content), - default_specification=DRAFT202012, - ) - registry = registry.with_resource(with_scheme(entity_id), resource) - return registry + with self._lock: + if self._reference_registry is not None: + return self._reference_registry + registry = Registry() + for entity_id, entity in self._by_id.items(): + if entity.is_schema and isinstance(entity.content, dict): + resource = Resource.from_contents( + _without_x_gts_ref(entity.content), + default_specification=DRAFT202012, + ) + registry = registry.with_resource(with_scheme(entity_id), resource) + self._reference_registry = registry + return registry def items(self): """Return all entity ID and entity pairs.""" with self._lock: return tuple((entity_id, copy.deepcopy(entity)) for entity_id, entity in self._by_id.items()) + def keys(self) -> tuple[str, ...]: + with self._lock: + return tuple(self._by_id) + + def values(self) -> tuple[GtsEntity, ...]: + with self._lock: + return tuple(copy.deepcopy(entity) for entity in self._by_id.values()) + @staticmethod def _validate_schema_refs(schema: dict[str, Any], path: str = "") -> None: """ @@ -523,7 +522,7 @@ def _validate_chain_dialect( self._validate_local_ref_dialects(content, chain_ids[0], root_dialect) visited: set[str] = set() - queue: list[tuple[str, dict[str, Any]]] = ( + queue: deque[tuple[str, dict[str, Any]]] = deque( [(gts_id, transient_schema)] if transient_schema is not None else [] ) if not queue: @@ -531,7 +530,7 @@ def _validate_chain_dialect( if entity and isinstance(entity.content, dict): queue.append((gts_id, entity.content)) while queue: - current_id, content = queue.pop(0) + current_id, content = queue.popleft() if current_id in visited: continue visited.add(current_id) @@ -1058,7 +1057,7 @@ def _has_valid_wildcard_match( gts_ref_validation: GtsRefValidationMode, ) -> bool: wildcard = GtsWildcard(pattern) - for entity_id, _ in self.items(): + for entity_id in self.keys(): try: if not GtsID(entity_id).wildcard_match(wildcard): continue @@ -1236,9 +1235,9 @@ def cast( raise StoreGtsObjectNotFound(from_schema_id) # Create a resolver to handle $ref in schemas - resolver = self._create_ref_resolver(to_schema.content) + registry = self._create_reference_registry() - return from_entity.cast(to_schema, from_schema, resolver=resolver) + return from_entity.cast(to_schema, from_schema, resolver=registry) def is_minor_compatible( self, @@ -1509,7 +1508,7 @@ def query(self, expr: str, limit: int = 100) -> GtsStoreQueryResult: return result # Filter entities - for _, entity in self.items(): + for entity in self.values(): if len(result.results) >= limit: break if not isinstance(entity.content, dict) or not entity.gts_id: diff --git a/tests/test_ops.py b/tests/test_ops.py index f322a3a..0887732 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -3,6 +3,8 @@ import pytest from gts.ops import GtsOps +from gts import GtsOps as PublicGtsOps + SCHEMA = { "$schema": "http://json-schema.org/draft-07/schema#", "$id": "gts://gts.x.test._.foo.v1~", @@ -24,6 +26,9 @@ def ops(): class TestConstructionAndConfig: + def test_high_level_facade_is_exported(self): + assert PublicGtsOps is GtsOps + def test_default_config_used_when_no_path(self, ops): assert "$id" in ops.cfg.entity_id_fields diff --git a/tests/test_server.py b/tests/test_server.py index 63fc9e2..0d5c2d3 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -6,10 +6,8 @@ import pytest from fastapi.responses import JSONResponse - -from gts.ops import GtsOps from gts._server import GtsHttpServer, ValidateEntityRequest, _RequestLoggingMiddleware - +from gts.ops import GtsOps SCHEMA = { "$schema": "http://json-schema.org/draft-07/schema#", @@ -237,6 +235,27 @@ def test_get_entities(self, server): class TestRequestLoggingMiddlewareVerboseOff: + def test_verbose_logging_does_not_read_or_log_bodies(self, server, caplog): + class URL: + path = "/entities" + + class Request: + method = "POST" + url = URL() + + async def body(self): + raise AssertionError("request body must not be read by logging middleware") + + middleware = _RequestLoggingMiddleware(server.app, verbose=2) + + async def call_next(request): + return JSONResponse({"secret": "must-not-be-logged"}) + + with caplog.at_level("DEBUG"): + result = run(middleware.dispatch(request=Request(), call_next=call_next)) + assert result.headers["connection"] == "close" + assert "must-not-be-logged" not in caplog.text + def test_dispatch_skips_when_not_verbose(self, server): middleware = _RequestLoggingMiddleware(server.app, verbose=0) diff --git a/tests/test_store_extra.py b/tests/test_store_extra.py index 2a6c846..34ebdc1 100644 --- a/tests/test_store_extra.py +++ b/tests/test_store_extra.py @@ -102,6 +102,23 @@ def test_store_uses_defensive_copies(self): items["gts.x.test._.copy.v1~"].content["type"] = "boolean" assert store.get("gts.x.test._.copy.v1~").content["type"] == "object" + def test_reference_registry_is_cached_and_invalidated_by_schema_changes(self): + store = GtsStore(reader=None) + store.register(_schema_entity("gts.x.test._.cached.v1~")) + first = store._create_reference_registry() + second = store._create_reference_registry() + assert first is second + + instance = GtsEntity( + content={"id": "gts.x.test._.cached.v1~x.test._.i.v1.0"}, + gts_id=GtsID("gts.x.test._.cached.v1~x.test._.i.v1.0"), + ) + store.register(instance) + assert store._create_reference_registry() is first + + store.register(_schema_entity("gts.x.test._.cached2.v1~")) + assert store._create_reference_registry() is not first + def test_transaction_serializes_writers(self): store = GtsStore(reader=None) entered = threading.Event() From 86c1c25376c745b95a34c22fbbbfa5341a00c686 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Sat, 26 Sep 2026 01:43:34 +0300 Subject: [PATCH 10/27] chore(spec): pin conformance suite to gts-spec v0.14.3 Bump the targeted gts-spec version to v0.14.3 and make the pin reproducible, mirroring the Rust reference implementation. - Record the version in .gts-spec-version (vMAJOR.MINOR.PATCH) and advance the .gts-spec submodule to the matching tag. - Rework `make update-spec` to check the submodule out at the pinned tag instead of floating to the remote's latest. - Add `make verify-spec-version`, a prerequisite of `make e2e`, that fails when the checked-out submodule drifts from the pin. - Update the README spec-version references to v0.14.3. Signed-off-by: Artifizer --- .gts-spec | 2 +- .gts-spec-version | 1 + Makefile | 27 ++++++++++++++++++++++----- README.md | 2 +- gts/README.md | 2 +- 5 files changed, 26 insertions(+), 8 deletions(-) create mode 100644 .gts-spec-version diff --git a/.gts-spec b/.gts-spec index 4ec63d6..98a16f3 160000 --- a/.gts-spec +++ b/.gts-spec @@ -1 +1 @@ -Subproject commit 4ec63d6978919465175931f040745a8a75333d10 +Subproject commit 98a16f3f9815dd4b852484ae7e52713d797d4dbb diff --git a/.gts-spec-version b/.gts-spec-version new file mode 100644 index 0000000..818cabd --- /dev/null +++ b/.gts-spec-version @@ -0,0 +1 @@ +v0.14.3 diff --git a/Makefile b/Makefile index 8ae4e27..2182d97 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ $(error PYTHON must be set for local package targets (examples: venv: PYTHON=.ve endif endif -.PHONY: help py-env install build install-local uninstall-local clean dev-fmt all check fmt lint clippy mypy test security update-spec e2e coverage gts-server +.PHONY: help py-env install build install-local uninstall-local clean dev-fmt all check fmt lint clippy mypy test security update-spec verify-spec-version e2e coverage gts-server # Default target - show help .DEFAULT_GOAL := help @@ -125,8 +125,8 @@ PORT ?= 8000 gts-server: install $(PYTHON) -m gts server --host 127.0.0.1 --port $(PORT) -# Run end-to-end tests against gts-spec -e2e: install +# Run end-to-end tests against gts-spec (pinned via .gts-spec-version) +e2e: install verify-spec-version @echo "Starting server in background..." @$(PYTHON) -m gts server --port 8000 & echo $$! > .server.pid @sleep 2 @@ -144,9 +144,26 @@ security: py-env $(PYTHON) -m pip install pip-audit $(PYTHON) -m pip_audit -# Update gts-spec submodule to latest +# Spec conformance suite is pinned in .gts-spec-version (format vMAJOR.MINOR.PATCH) +# so every checkout reproduces the same e2e run, mirroring the Rust reference. +GTS_SPEC_VERSION ?= $(shell cat .gts-spec-version 2>/dev/null) + +# Check out the gts-spec submodule at the pinned .gts-spec-version tag update-spec: - git submodule update --remote .gts-spec + git submodule update --init .gts-spec + git -C .gts-spec fetch --tags --quiet origin + git -C .gts-spec checkout --quiet "$(GTS_SPEC_VERSION)" + @echo "gts-spec pinned to $(GTS_SPEC_VERSION)" + +# Fail if the checked-out gts-spec submodule does not match the pinned version +verify-spec-version: + @current="$$(git -C .gts-spec describe --tags 2>/dev/null)"; \ + if [ "$$current" != "$(GTS_SPEC_VERSION)" ]; then \ + echo "gts-spec is at '$$current' but .gts-spec-version pins '$(GTS_SPEC_VERSION)'"; \ + echo "run 'make update-spec' to sync"; \ + exit 1; \ + fi; \ + echo "gts-spec matches pinned $(GTS_SPEC_VERSION)" # Run all checks and build all: check build diff --git a/README.md b/README.md index 1bbdb03..c79e83e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A minimal, idiomatic Python library for working with **GTS** ([Global Type System](https://github.com/gts-spec/gts-spec)) identifiers and JSON/JSON Schema artifacts. -Current supported GTS spec version: `0.14.2` +Current supported GTS spec version: `0.14.3` ## Roadmap diff --git a/gts/README.md b/gts/README.md index 47ff0d7..6e18b3a 100644 --- a/gts/README.md +++ b/gts/README.md @@ -2,7 +2,7 @@ Python helpers and a reference HTTP service for the [Global Type System (GTS)](https://github.com/globaltypesystem/gts-spec). The package supports GTS identifier parsing, JSON Schema-backed validation, schema compatibility and derivation checks, traits, casting, queries, file loading, a CLI, and a FastAPI application. -The package targets GTS specification v0.14.2 and requires Python 3.9 or later. +The package targets GTS specification v0.14.3 and requires Python 3.9 or later. ## Installation From 98711c98a85b1b1425ecc91e1d3f6d71957f4c14 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Sat, 26 Sep 2026 01:43:53 +0300 Subject: [PATCH 11/27] feat(core): structured errors, focused schema modules, and compatibility diagnostics Adopt several design ideas from the Rust reference implementation while keeping the code idiomatic Python. Errors: - Add a GtsError hierarchy (GtsNotFoundError, GtsValidationError, GtsConflictError, GtsUnresolvedRefError) in gts._errors and reparent the existing store exceptions onto it. GtsValidationError also subclasses ValueError so callers/tests that catch ValueError keep working. The hierarchy is exported from the package root. Schema modules: - Extract dialect detection into gts.schema_dialect and $ref inlining into gts.schema_resolver (behind a provider callable, the Python analogue of Rust's SchemaProvider), leaving thin delegators on GtsStore so the public surface is unchanged. - Reject subschemas that switch JSON Schema dialect: a type is read under the single dialect its top-level $schema selects. - Centralize the bare gts. / gts:// literals behind the naming constants. Compatibility diagnostics (OP#8): - /compatibility now reports why a direction failed (backward_errors, forward_errors, incompatibility_reasons) and classifies the content model (open / closed / partially_open) of every object level of the candidate schema (candidate_object_levels). Reasons are derived from the already-computed verdict in a single inclusion pass, so they never contradict it; verdicts themselves are unchanged. - Replace the mutable default-list pattern on GtsEntityCastResult with dataclass field(default_factory=list). The full gts-spec conformance suite still passes; behavior is additive. Signed-off-by: Artifizer --- gts/src/gts/__init__.py | 12 +++ gts/src/gts/_errors.py | 45 +++++++++ gts/src/gts/compatibility.py | 171 +++++++++++++++++++++++++++++++ gts/src/gts/schema_cast.py | 37 +++---- gts/src/gts/schema_dialect.py | 99 ++++++++++++++++++ gts/src/gts/schema_resolver.py | 103 +++++++++++++++++++ gts/src/gts/store.py | 180 +++++++++++++-------------------- gts/src/gts/x_gts_ref.py | 4 +- tests/test_compatibility.py | 97 ++++++++++++++++++ tests/test_store_extra.py | 68 +++++++++++++ 10 files changed, 682 insertions(+), 134 deletions(-) create mode 100644 gts/src/gts/_errors.py create mode 100644 gts/src/gts/schema_dialect.py create mode 100644 gts/src/gts/schema_resolver.py diff --git a/gts/src/gts/__init__.py b/gts/src/gts/__init__.py index 8933097..7f9a496 100644 --- a/gts/src/gts/__init__.py +++ b/gts/src/gts/__init__.py @@ -1,3 +1,10 @@ +from ._errors import ( + GtsConflictError, + GtsError, + GtsNotFoundError, + GtsUnresolvedRefError, + GtsValidationError, +) from .entities import ( DEFAULT_GTS_CONFIG, GtsConfig, @@ -47,10 +54,12 @@ "GtsAddSchemaResult", "GtsAddSchemasResult", "GtsConfig", + "GtsConflictError", "GtsEntitiesListResult", "GtsEntity", "GtsEntityInfo", "GtsEntityValidationResult", + "GtsError", "GtsExtractIdResult", "GtsFile", "GtsFileReader", @@ -61,13 +70,16 @@ "GtsIdSegment", "GtsIdValidationResult", "GtsJsonValidationResult", + "GtsNotFoundError", "GtsOps", "GtsPathResolver", "GtsReader", "GtsRefValidationMode", "GtsSchemaGraphResult", "GtsStore", + "GtsUnresolvedRefError", "GtsUuidResult", + "GtsValidationError", "GtsValidationResult", "GtsWildcard", "JsonEntity", diff --git a/gts/src/gts/_errors.py b/gts/src/gts/_errors.py new file mode 100644 index 0000000..d8ad218 --- /dev/null +++ b/gts/src/gts/_errors.py @@ -0,0 +1,45 @@ +"""Structured GTS exception hierarchy. + +The Rust reference models store failures as a single ``StoreError`` enum whose +variants (not-found, invalid, conflict, unresolved-ref, ...) let callers react +by category rather than by matching on message text. The idiomatic Python +analogue is a small exception hierarchy rooted at :class:`GtsError`. + +:class:`GtsValidationError` also derives from the builtin :class:`ValueError`: +the library historically raised bare ``ValueError`` for validation failures and +callers/tests catch it, so keeping that base preserves backward compatibility +while letting new code catch :class:`GtsError` / :class:`GtsValidationError`. +""" + +from __future__ import annotations + + +class GtsError(Exception): + """Base class for every error raised by the GTS library.""" + + +class GtsNotFoundError(GtsError): + """An entity, schema or instance was not present in the store.""" + + +class GtsValidationError(GtsError, ValueError): + """An entity failed GTS validation. + + Subclasses :class:`ValueError` so existing ``except ValueError`` callers + keep working while new code can catch the narrower GTS categories. + """ + + +class GtsConflictError(GtsError): + """An entity id is already registered with different content.""" + + def __init__(self, entity_id: str, message: str | None = None) -> None: + self.entity_id = entity_id + super().__init__( + message + or f"Entity '{entity_id}' is already registered with different content" + ) + + +class GtsUnresolvedRefError(GtsValidationError): + """A ``$ref`` or ``x-gts-ref`` target could not be resolved.""" diff --git a/gts/src/gts/compatibility.py b/gts/src/gts/compatibility.py index 76a1270..5feb5b4 100644 --- a/gts/src/gts/compatibility.py +++ b/gts/src/gts/compatibility.py @@ -244,3 +244,174 @@ def full_verdict(backward: str, forward: str) -> str: def check_accepted_set_inclusion(subset: Any, superset: Any) -> bool | None: """Shared inclusion primitive used by OP#12 derivation admission.""" return _is_subschema(subset, superset) + + +# --- diagnostics (spec sec 4.4) ------------------------------------------- +# +# The inclusion verdict tells you *whether* two definitions are compatible; a +# caller admitting a new version also wants to know *why* a direction failed and +# whether a level can still gain optional properties later. The Rust reference +# surfaces both (``SchemaComparison`` carries diagnostics plus the content model +# of every object level). We keep ``jsonsubschema`` as the inclusion primitive +# and add the same reporting on top of it. + +OPEN = "open" +CLOSED = "closed" +PARTIAL = "partially_open" + +_APPLICATOR_OBJECT_KEYWORDS = ( + "properties", + "patternProperties", + "$defs", + "definitions", +) + + +def _level_content_model(node: dict[str, Any]) -> str: + """Classify how one object level treats undeclared properties. + + - ``open``: accepts an undeclared property with any value; + - ``closed``: rejects every undeclared property; + - ``partially_open``: accepts some undeclared names or constrains their + values (schema-valued ``additionalProperties``, ``patternProperties`` or + ``propertyNames``). + """ + has_pattern = bool(node.get("patternProperties")) + has_property_names = "propertyNames" in node + additional = node.get("additionalProperties") + if additional is None: + additional_kind = "open" + else: + boolean = boolean_schema_value(additional) + if boolean is True: + additional_kind = "open" + elif boolean is False: + additional_kind = "closed" + else: + additional_kind = "schema" + + if additional_kind == "closed" and not has_pattern and not has_property_names: + return CLOSED + if additional_kind == "open" and not has_pattern and not has_property_names: + return OPEN + return PARTIAL + + +def _is_object_level(node: Any) -> bool: + if not isinstance(node, dict): + return False + if node.get("type") == "object" or "properties" in node: + return True + return any( + key in node + for key in ("additionalProperties", "patternProperties", "propertyNames") + ) + + +def classify_object_levels(schema: Any) -> list[dict[str, str]]: + """Content model of every object level of a (resolved) schema. + + Returns one entry per object level, e.g. + ``[{"path": "$", "content_model": "closed"}, ...]``. Callers use it to + report, per level, whether a later definition can add an optional property + there (only a ``closed`` level can, per spec sec 4.4). + """ + levels: list[dict[str, str]] = [] + seen: set[int] = set() + + def walk(node: Any, path: str, depth: int) -> None: + if depth > 64 or not isinstance(node, dict): + return + marker = id(node) + if marker in seen: + return + seen.add(marker) + + if _is_object_level(node): + levels.append({"path": path, "content_model": _level_content_model(node)}) + + properties = node.get("properties") + if isinstance(properties, dict): + for name, child in properties.items(): + walk(child, f"{path}.{name}", depth + 1) + + # Undeclared-property and pattern-property values are object levels of + # the instance too, so classify their schemas (a bare boolean/absent + # additionalProperties carries no nested level). + pattern_properties = node.get("patternProperties") + if isinstance(pattern_properties, dict): + for pattern, child in pattern_properties.items(): + walk(child, f"{path}.patternProperties[{pattern}]", depth + 1) + additional = node.get("additionalProperties") + if isinstance(additional, dict): + walk(additional, f"{path}.additionalProperties", depth + 1) + + # Array element schemas: `items` as a single schema, and the tuple forms + # (`prefixItems`, or `items`/`additionalItems` as a list). + items = node.get("items") + if isinstance(items, dict): + walk(items, f"{path}[]", depth + 1) + elif isinstance(items, list): + for index, child in enumerate(items): + walk(child, f"{path}[{index}]", depth + 1) + prefix_items = node.get("prefixItems") + if isinstance(prefix_items, list): + for index, child in enumerate(prefix_items): + walk(child, f"{path}[{index}]", depth + 1) + additional_items = node.get("additionalItems") + if isinstance(additional_items, dict): + walk(additional_items, f"{path}[].additionalItems", depth + 1) + + for combinator in ("allOf", "anyOf", "oneOf"): + branches = node.get(combinator) + if isinstance(branches, list): + for branch in branches: + walk(branch, path, depth + 1) + + walk(schema, "$", 0) + return levels + + +def explain_verdict( + verdict: str, *, backward: bool, differing_dialects: bool = False +) -> list[str]: + """Human-readable reasons for a non-``compatible`` directional verdict. + + Pure function of an already-computed ``verdict`` (``compatible`` / + ``incompatible`` / ``unknown``); it does not re-run the inclusion check, so + the caller pays for the accepted-instance-set comparison exactly once. + ``backward`` selects the direction (backward is ``Valid(old) subset-of + Valid(new)``, forward the reverse) for message wording; ``differing_dialects`` + distinguishes an ``unknown`` caused by incomparable dialects from one the + checker could not decide. Returns ``[]`` for a compatible verdict. + """ + if verdict == COMPATIBLE: + return [] + if verdict == UNKNOWN: + if differing_dialects: + return [ + ( + "compatibility is unknown: the two definitions declare " + "different JSON Schema dialects, so their accepted-instance " + "sets are not comparable" + ) + ] + return [ + ( + "compatibility is unknown: the accepted-instance-set inclusion " + "could not be proved or disproved for this direction" + ) + ] + if backward: + return [ + ( + "backward incompatible: Valid(old) is not a subset of Valid(new); " + "the new definition rejects instances the old definition accepts" + ) + ] + return [ + ( + "forward incompatible: Valid(new) is not a subset of Valid(old); the " + "old definition rejects instances the new definition accepts" + ) + ] diff --git a/gts/src/gts/schema_cast.py b/gts/src/gts/schema_cast.py index db5c71b..0d70bff 100644 --- a/gts/src/gts/schema_cast.py +++ b/gts/src/gts/schema_cast.py @@ -2,7 +2,7 @@ import copy import logging -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any from jsonschema import exceptions as js_exceptions @@ -24,15 +24,15 @@ class GtsEntityCastResult: from_id: str = "" to_id: str = "" direction: str = "unknown" - added_properties: list[str] = None # type: ignore - removed_properties: list[str] = None # type: ignore - changed_properties: list[dict[str, str]] = None # type: ignore + added_properties: list[str] = field(default_factory=list) + removed_properties: list[str] = field(default_factory=list) + changed_properties: list[dict[str, str]] = field(default_factory=list) is_fully_compatible: bool | None = False is_backward_compatible: bool | None = False is_forward_compatible: bool | None = False - incompatibility_reasons: list[str] = None # type: ignore - backward_errors: list[str] = None # type: ignore - forward_errors: list[str] = None # type: ignore + incompatibility_reasons: list[str] = field(default_factory=list) + backward_errors: list[str] = field(default_factory=list) + forward_errors: list[str] = field(default_factory=list) casted_entity: dict[str, Any] | None = None error: str = "" # Optional explicit verdict strings ("compatible"/"incompatible"/"unknown"). @@ -40,21 +40,9 @@ class GtsEntityCastResult: backward_verdict: str | None = None forward_verdict: str | None = None full_verdict: str | None = None - - def __post_init__(self): - # Initialize list fields if None - if self.added_properties is None: - self.added_properties = [] - if self.removed_properties is None: - self.removed_properties = [] - if self.changed_properties is None: - self.changed_properties = [] - if self.incompatibility_reasons is None: - self.incompatibility_reasons = [] - if self.backward_errors is None: - self.backward_errors = [] - if self.forward_errors is None: - self.forward_errors = [] + # Content model (open/closed/partially_open) of every object level of the + # candidate ("new") schema, per spec sec 4.4. Empty for the cast op. + candidate_object_levels: list[dict[str, str]] = field(default_factory=list) def to_dict(self) -> dict[str, Any]: def _compat_str(val: bool | None) -> str: @@ -85,6 +73,7 @@ def _compat_str(val: bool | None) -> str: "backward_errors": self.backward_errors, "forward_errors": self.forward_errors, "casted_entity": self.casted_entity, + "candidate_object_levels": self.candidate_object_levels, } if self.error: result["error"] = self.error @@ -256,7 +245,7 @@ def _cast_instance_to_schema( incompatibility_reasons = [] added: list[str] = [] removed: list[str] = [] - incompatibility_reasons: list[str] = [] + incompatibility_reasons = [] if not isinstance(instance, dict): raise SchemaCastError("Instance must be an object for casting") @@ -416,7 +405,7 @@ def _remove_gts_const_constraints(schema: Any) -> Any: if not isinstance(schema, dict): return schema - result = {} + result: dict[str, Any] = {} for key, value in schema.items(): if key == "const" and isinstance(value, str) and GtsID.is_valid(value): # Replace const with a type constraint instead diff --git a/gts/src/gts/schema_dialect.py b/gts/src/gts/schema_dialect.py new file mode 100644 index 0000000..70acac2 --- /dev/null +++ b/gts/src/gts/schema_dialect.py @@ -0,0 +1,99 @@ +"""JSON Schema dialects a GTS Type Schema may declare (spec sec 2.4 / 11). + +GTS admits Draft-07, Draft 2019-09 and Draft 2020-12 and nothing else. This +module is the single home for detecting the dialect a document declares and +mapping it to its canonical meta-schema URI, mirroring the Rust reference's +``schema_dialect`` module. Keeping it out of :class:`~gts.store.GtsStore` makes +the dialect rules unit-testable on their own. +""" + +from __future__ import annotations + +from typing import Any + +# Canonical (http, unfragmented) meta-schema -> short dialect label. +_SUPPORTED = { + "http://json-schema.org/draft-07/schema": "draft-07", + "http://json-schema.org/draft/2019-09/schema": "2019-09", + "http://json-schema.org/draft/2020-12/schema": "2020-12", +} + +_DIALECT_URI = { + "draft-07": "http://json-schema.org/draft-07/schema#", + "2019-09": "https://json-schema.org/draft/2019-09/schema", + "2020-12": "https://json-schema.org/draft/2020-12/schema", +} + + +def document_dialect(schema: dict[str, Any]) -> str: + """Short label of the dialect ``schema`` declares in its ``$schema``. + + The ``http``/``https`` spellings and a trailing ``#`` fragment are treated + as equivalent; anything outside the supported set is refused. + + Raises: + ValueError: if no supported dialect is declared. + """ + dialect = schema.get("$schema") + if not isinstance(dialect, str) or not dialect: + raise ValueError("$schema must declare a supported JSON Schema dialect") + normalized = dialect.removesuffix("#").lower().replace("https://", "http://", 1) + try: + return _SUPPORTED[normalized] + except KeyError as error: + raise ValueError(f"Unsupported JSON Schema dialect: {dialect}") from error + + +def dialect_uri(schema: dict[str, Any]) -> str: + """Canonical meta-schema URI for the dialect ``schema`` declares.""" + return _DIALECT_URI[document_dialect(schema)] + + +def supports_ref_siblings(schema: Any) -> bool: + """Whether the declared dialect evaluates keywords alongside ``$ref``. + + Draft 2019-09 and 2020-12 do; Draft-07 ignores ``$ref`` siblings. + """ + dialect = schema.get("$schema") if isinstance(schema, dict) else None + return isinstance(dialect, str) and ( + "/draft/2019-09/" in dialect or "/draft/2020-12/" in dialect + ) + + +def check_subschemas(schema: dict[str, Any]) -> None: + """Reject a nested subschema that switches JSON Schema dialect. + + JSON Schema lets any subschema restate ``$schema`` and switch dialect, but + GTS reads every part of a type under the single dialect its top-level + ``$schema`` selects (spec sec 11). This mirrors the Rust reference's + ``schema_dialect::check_subschemas``: a nested ``$schema`` may only restate + the document's own dialect. An unrecognized nested ``$schema`` is left to + the meta-schema check rather than reported here. + + Raises: + ValueError: on the first subschema that changes dialect, by location. + """ + # Imported lazily to avoid any import-order coupling with schema_validation. + from .schema_validation import iter_schema_nodes + + root_dialect = document_dialect(schema) + for node, path in iter_schema_nodes(schema): + if not path: + continue # the document root defines the dialect + # Trait schemas are dialect-checked by the traits subsystem, which + # reports a more specific "differs from host dialect" message. + if any(segment == "x-gts-traits-schema" for segment in path.split("/")): + continue + declared = node.get("$schema") + if not isinstance(declared, str) or not declared: + continue + try: + nested_dialect = document_dialect(node) + except ValueError: + continue # not a recognized dialect; the meta-schema check owns it + if nested_dialect != root_dialect: + raise ValueError( + f"subschema at '{path}' declares dialect {nested_dialect} but the " + f"type is read under {root_dialect}; a subschema must not change " + "JSON Schema dialect" + ) diff --git a/gts/src/gts/schema_resolver.py b/gts/src/gts/schema_resolver.py new file mode 100644 index 0000000..594e85a --- /dev/null +++ b/gts/src/gts/schema_resolver.py @@ -0,0 +1,103 @@ +"""Inlining of GTS ``$ref`` targets (spec sec 4.3). + +Extracted from :class:`~gts.store.GtsStore` so reference resolution is testable +without a full store. Resolution takes a ``provider`` callable that returns a +schema's content by GTS id (or raises :class:`KeyError` when absent) — the +Python analogue of the Rust reference's ``SchemaProvider`` trait. + +References are inlined recursively so a schema reached through an intermediate +is fully expanded. Cyclic references are left unresolved on purpose: the +surviving ``$ref`` makes the effective schema unprovable, which is the intended +admission failure. +""" + +from __future__ import annotations + +import copy +from collections.abc import Callable +from typing import Any + +from .gts import GtsRef +from .schema_dialect import supports_ref_siblings + +# A ``$ref`` provider maps a bare GTS id to its schema content, raising +# ``KeyError`` when the id is unknown. +SchemaProvider = Callable[[str], dict] + +MAX_SCHEMA_REF_EXPANSIONS = 10_000 + + +def resolve_schema_refs(schema: Any, provider: SchemaProvider) -> Any: + """Return ``schema`` with external ``$ref`` targets inlined via ``provider``.""" + return inline_refs( + copy.deepcopy(schema), set(), supports_ref_siblings(schema), provider, [0] + ) + + +def inline_refs( + node: Any, + seen: set[str], + supports_ref_siblings_flag: bool, + provider: SchemaProvider, + expansions: list[int] | None = None, +) -> Any: + """Recursively inline ``$ref`` references, guarding against cycles.""" + expansions = expansions if expansions is not None else [0] + if isinstance(node, dict): + ref_uri = node.get("$ref") + if isinstance(ref_uri, str): + ref = GtsRef.parse(ref_uri) + # Local (#/...) refs are resolved by JSON Schema itself; only + # external targets are inlined from the provider. + ref_id = None if ref.is_local else ref.target_id + if ref_id is not None: + if ref_id in seen: + # Cycle detected: leave the $ref unresolved. + return node + expansions[0] += 1 + if expansions[0] > MAX_SCHEMA_REF_EXPANSIONS: + raise ValueError( + f"schema reference expansion exceeds limit of " + f"{MAX_SCHEMA_REF_EXPANSIONS}" + ) + try: + ref_schema = provider(ref_id) + except KeyError: + return node # Leave unresolved + + resolved = inline_refs( + copy.deepcopy(ref_schema), + seen | {ref_id}, + supports_ref_siblings(ref_schema), + provider, + expansions, + ) + if supports_ref_siblings_flag and len(node) > 1: + siblings = { + key: value for key, value in node.items() if key != "$ref" + } + return { + "allOf": [ + resolved, + inline_refs( + siblings, + seen, + supports_ref_siblings_flag, + provider, + expansions, + ), + ] + } + return resolved + return { + key: inline_refs( + value, seen, supports_ref_siblings_flag, provider, expansions + ) + for key, value in node.items() + } + if isinstance(node, list): + return [ + inline_refs(item, seen, supports_ref_siblings_flag, provider, expansions) + for item in node + ] + return node diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index c49bd1d..e4397fd 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -13,20 +13,32 @@ from referencing import Registry, Resource from referencing.jsonschema import DRAFT202012 -from . import compatibility, derivation, traits +from . import compatibility, derivation, schema_dialect, schema_resolver, traits +from ._errors import ( + GtsNotFoundError, + GtsUnresolvedRefError, + GtsValidationError, +) from ._json_pointer import resolve as resolve_json_pointer -from ._naming import looks_like_gts, strip_scheme, with_scheme +from ._naming import ( + GTS_PREFIX, + GTS_URI_PREFIX, + looks_like_gts, + strip_scheme, + with_scheme, +) from .entities import GtsEntity from .gts import GtsID, GtsRef, GtsWildcard from .gts_ref_validation import GtsRefValidationMode from .schema_cast import GtsEntityCastResult + +# Re-exported for backward compatibility; the limit now lives in schema_resolver. +from .schema_resolver import MAX_SCHEMA_REF_EXPANSIONS # noqa: F401 from .schema_validation import FORMAT_CHECKER, iter_schema_nodes, validator_for from .x_gts_ref import XGtsRefValidator, _without_x_gts_ref logger = logging.getLogger(__name__) -MAX_SCHEMA_REF_EXPANSIONS = 10_000 - def _require_schema_id(value: str) -> GtsID: try: @@ -35,7 +47,7 @@ def _require_schema_id(value: str) -> GtsID: raise ValueError(f"ID '{value}' is not a schema (must end with '~')") from error -class StoreGtsObjectNotFound(Exception): +class StoreGtsObjectNotFound(GtsNotFoundError): """Exception raised when a GTS entity is not found in the store.""" def __init__(self, entity_id: str): @@ -43,7 +55,7 @@ def __init__(self, entity_id: str): self.entity_id = entity_id -class StoreGtsSchemaNotFound(Exception): +class StoreGtsSchemaNotFound(GtsNotFoundError): """Exception raised when a GTS schema is not found in the store.""" def __init__(self, entity_id: str): @@ -51,7 +63,7 @@ def __init__(self, entity_id: str): self.entity_id = entity_id -class StoreGtsEntityNotFound(Exception): +class StoreGtsEntityNotFound(GtsNotFoundError): """Exception raised when a GTS entity is not found in the store.""" def __init__(self, entity_id: str): @@ -59,7 +71,7 @@ def __init__(self, entity_id: str): self.entity_id = entity_id -class StoreGtsSchemaForInstanceNotFound(Exception): +class StoreGtsSchemaForInstanceNotFound(GtsNotFoundError): """Exception raised when a GTS schema for an instance is not found in the store.""" def __init__(self, entity_id: str): @@ -69,7 +81,7 @@ def __init__(self, entity_id: str): self.entity_id = entity_id -class StoreGtsCastFromSchemaNotAllowed(Exception): +class StoreGtsCastFromSchemaNotAllowed(GtsValidationError): """Exception raised when attempting to cast from a schema ID.""" def __init__(self, from_id: str): @@ -197,7 +209,9 @@ def register_schema(self, type_id: str, schema: dict[str, Any]) -> None: if not isinstance(dialect, str) or not dialect: raise ValueError("Type Schema must contain a top-level $schema field") embedded_id = schema.get("$id") - if not isinstance(embedded_id, str) or not embedded_id.startswith("gts://gts."): + if not isinstance(embedded_id, str) or not embedded_id.startswith( + GTS_URI_PREFIX + GTS_PREFIX + ): raise ValueError("Type Schema must contain a top-level $id in gts:// form") try: normalized_id = GtsID.parse_type(strip_scheme(embedded_id)) @@ -264,7 +278,10 @@ def _create_reference_registry(self) -> Registry: def items(self): """Return all entity ID and entity pairs.""" with self._lock: - return tuple((entity_id, copy.deepcopy(entity)) for entity_id, entity in self._by_id.items()) + return tuple( + (entity_id, copy.deepcopy(entity)) + for entity_id, entity in self._by_id.items() + ) def keys(self) -> tuple[str, ...]: with self._lock: @@ -340,7 +357,7 @@ def _validate_schema_ref_targets( try: target = self.get_schema_content(ref.target_id) except KeyError as error: - raise ValueError( + raise GtsUnresolvedRefError( f"Unresolvable $ref at '{current_path}': '{ref_uri}'" ) from error if ref.target_id not in visited: @@ -443,27 +460,11 @@ def _content_is_final(content: dict[str, Any]) -> bool: @staticmethod def _schema_dialect(schema: dict[str, Any]) -> str: - dialect = schema.get("$schema") - if not isinstance(dialect, str) or not dialect: - raise ValueError("$schema must declare a supported JSON Schema dialect") - normalized = dialect.removesuffix("#").lower().replace("https://", "http://", 1) - supported = { - "http://json-schema.org/draft-07/schema": "draft-07", - "http://json-schema.org/draft/2019-09/schema": "2019-09", - "http://json-schema.org/draft/2020-12/schema": "2020-12", - } - try: - return supported[normalized] - except KeyError as error: - raise ValueError(f"Unsupported JSON Schema dialect: {dialect}") from error + return schema_dialect.document_dialect(schema) @staticmethod def _schema_dialect_uri(schema: dict[str, Any]) -> str: - return { - "draft-07": "http://json-schema.org/draft-07/schema#", - "2019-09": "https://json-schema.org/draft/2019-09/schema", - "2020-12": "https://json-schema.org/draft/2020-12/schema", - }[GtsStore._schema_dialect(schema)] + return schema_dialect.dialect_uri(schema) def _validate_local_ref_dialects( self, schema: dict[str, Any], root_id: str, root_dialect: str @@ -568,7 +569,7 @@ def _validate_schema_chain( segments = gid.gts_id_segments chain_ids = [] - prefix = "gts." + prefix = GTS_PREFIX for seg in segments: chain_ids.append(prefix + seg.segment) prefix = prefix + seg.segment @@ -633,25 +634,16 @@ def _validate_schema_chain( ) def _resolve_schema_refs(self, schema: Any) -> Any: - """Resolve $ref references in a schema by inlining referenced schemas. + """Resolve external ``$ref`` targets by inlining them from the store. - References are inlined recursively so that a schema reached through an - intermediate (A referenced via A~B) is fully expanded. Cyclic - references are left unresolved: the surviving $ref makes the effective - schema unprovable, which is the intended admission failure. + Thin adapter over :func:`schema_resolver.resolve_schema_refs`, passing + the store's :meth:`get_schema_content` as the reference provider. """ - import copy - - return self._inline_refs( - copy.deepcopy(schema), set(), self._supports_ref_siblings(schema), [0] - ) + return schema_resolver.resolve_schema_refs(schema, self.get_schema_content) @staticmethod def _supports_ref_siblings(schema: Any) -> bool: - dialect = schema.get("$schema") if isinstance(schema, dict) else None - return isinstance(dialect, str) and ( - "/draft/2019-09/" in dialect or "/draft/2020-12/" in dialect - ) + return schema_dialect.supports_ref_siblings(schema) def _inline_refs( self, @@ -660,58 +652,10 @@ def _inline_refs( supports_ref_siblings: bool, expansions: list[int] | None = None, ) -> Any: - """Recursively inline $ref references, guarding against cycles.""" - expansions = expansions if expansions is not None else [0] - if isinstance(node, dict): - ref_uri = node.get("$ref") - if isinstance(ref_uri, str): - ref = GtsRef.parse(ref_uri) - # Local (#/...) refs are resolved by JSON Schema itself; only - # external targets are inlined from the store. - ref_id = None if ref.is_local else ref.target_id - if ref_id is not None: - if ref_id in seen: - # Cycle detected: leave the $ref unresolved. - return node - expansions[0] += 1 - if expansions[0] > MAX_SCHEMA_REF_EXPANSIONS: - raise ValueError( - f"schema reference expansion exceeds limit of {MAX_SCHEMA_REF_EXPANSIONS}" - ) - try: - ref_schema = self.get_schema_content(ref_id) - except KeyError: - return node # Leave unresolved - import copy - - resolved = self._inline_refs( - copy.deepcopy(ref_schema), - seen | {ref_id}, - self._supports_ref_siblings(ref_schema), - expansions, - ) - if supports_ref_siblings and len(node) > 1: - siblings = { - key: value for key, value in node.items() if key != "$ref" - } - return { - "allOf": [ - resolved, - self._inline_refs( - siblings, seen, supports_ref_siblings, expansions - ), - ] - } - return resolved - return { - key: self._inline_refs(value, seen, supports_ref_siblings, expansions) - for key, value in node.items() - } - if isinstance(node, list): - return [ - self._inline_refs(item, seen, supports_ref_siblings, expansions) for item in node - ] - return node + """Recursively inline ``$ref`` references, guarding against cycles.""" + return schema_resolver.inline_refs( + node, seen, supports_ref_siblings, self.get_schema_content, expansions + ) def _build_effective_traits( self, gts_id: str, transient_schema: dict[str, Any] | None = None @@ -721,7 +665,7 @@ def _build_effective_traits( segments = gid.gts_id_segments chain_ids: list[str] = [] - prefix = "gts." + prefix = GTS_PREFIX for seg in segments: chain_ids.append(prefix + seg.segment) prefix = prefix + seg.segment @@ -858,6 +802,9 @@ def validate_schema_content( ) self._schema_dialect(schema_content) + # A subschema may not switch JSON Schema dialect (spec sec 11); the whole + # type is read under the dialect its top-level $schema selects. + schema_dialect.check_subschemas(schema_content) logger.info(f"Validating schema {schema_id.id}") self._validate_schema_refs(schema_content, "") self._validate_schema_ref_targets(schema_content) @@ -968,7 +915,7 @@ def _validate_schema_transitive( ) chain_ids: list[str] = [] - prefix = "gts." + prefix = GTS_PREFIX for segment in schema_id.gts_id_segments: chain_ids.append(prefix + segment.segment) prefix += segment.segment @@ -1025,7 +972,7 @@ def _schema_dependencies( ) if ( isinstance(x_gts_ref, str) - and x_gts_ref.startswith("gts.") + and x_gts_ref.startswith(GTS_PREFIX) and "*" not in x_gts_ref ): yield x_gts_ref, True @@ -1292,6 +1239,22 @@ def is_minor_compatible( # Determine direction direction = GtsEntityCastResult._infer_direction(old_schema_id, new_schema_id) + # Surface *why* a direction failed and classify the candidate's object + # levels so a caller can see whether a level can still gain optional + # properties later (spec sec 4.4). Reasons are derived from the verdicts + # already computed above (no second inclusion pass), so they never + # contradict the verdict. + differing_dialects = compatibility.dialects_differ(old_resolved, new_resolved) + backward_errors = compatibility.explain_verdict( + backward, backward=True, differing_dialects=differing_dialects + ) + forward_errors = compatibility.explain_verdict( + forward, backward=False, differing_dialects=differing_dialects + ) + # Union the directional reasons for the summary, dropping the duplicate + # a single shared cause (e.g. differing dialects) would produce. + incompatibility_reasons = list(dict.fromkeys(backward_errors + forward_errors)) + return GtsEntityCastResult( from_id=old_schema_id, to_id=new_schema_id, @@ -1302,20 +1265,21 @@ def is_minor_compatible( is_fully_compatible=full == compatibility.COMPATIBLE, is_backward_compatible=backward == compatibility.COMPATIBLE, is_forward_compatible=forward == compatibility.COMPATIBLE, - incompatibility_reasons=[], - backward_errors=[], - forward_errors=[], + incompatibility_reasons=incompatibility_reasons, + backward_errors=backward_errors, + forward_errors=forward_errors, casted_entity=None, backward_verdict=backward, forward_verdict=forward, full_verdict=full, + candidate_object_levels=compatibility.classify_object_levels(new_resolved), ) - def build_schema_graph(self, gts_id: str) -> tuple[dict[str, set[str]], list[str]]: - seen_gts_ids = set() + def build_schema_graph(self, gts_id: str) -> dict[str, Any]: + seen_gts_ids: set[str] = set() - def gts2node(gts_id: str, seen_gts_ids: set[str]) -> str: - ret = {"id": gts_id} + def gts2node(gts_id: str, seen_gts_ids: set[str]) -> dict[str, Any]: + ret: dict[str, Any] = {"id": gts_id} if gts_id in seen_gts_ids: return ret @@ -1324,7 +1288,7 @@ def gts2node(gts_id: str, seen_gts_ids: set[str]) -> str: entity = self.get(gts_id) if entity: - refs = {} + refs: dict[str, Any] = {} for r in entity.gts_refs: if r["id"] == gts_id: continue diff --git a/gts/src/gts/x_gts_ref.py b/gts/src/gts/x_gts_ref.py index ae7595e..3810a98 100644 --- a/gts/src/gts/x_gts_ref.py +++ b/gts/src/gts/x_gts_ref.py @@ -419,7 +419,7 @@ def _validate_gts_id_or_pattern( self, pattern: str, field_path: str ) -> XGtsRefValidationError | None: """Validate a GTS ID or pattern in schema definition.""" - if pattern == "gts.*": + if pattern == GTS_PREFIX + "*": return None # Valid wildcard if "*" in pattern: @@ -465,7 +465,7 @@ def _validate_gts_pattern( ) # Check pattern match - if pattern == "gts.*": + if pattern == GTS_PREFIX + "*": pass # Any valid GTS ID matches elif pattern.endswith("*"): prefix = pattern[:-1] diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py index ed0f355..d7602d9 100644 --- a/tests/test_compatibility.py +++ b/tests/test_compatibility.py @@ -1,18 +1,115 @@ """Tests for gts.compatibility (spec sec 4, OP#8 & OP#12 inclusion primitive).""" from gts.compatibility import ( + CLOSED, COMPATIBLE, INCOMPATIBLE, + OPEN, + PARTIAL, UNKNOWN, boolean_schema_value, sanitize, check_backward_compatibility, check_forward_compatibility, + classify_object_levels, + explain_verdict, full_verdict, check_accepted_set_inclusion, ) +class TestObjectLevelClassification: + def test_open_when_additional_properties_absent(self): + levels = classify_object_levels( + {"type": "object", "properties": {"a": {"type": "string"}}} + ) + assert levels == [{"path": "$", "content_model": OPEN}] + + def test_closed_when_additional_properties_false(self): + levels = classify_object_levels( + {"type": "object", "properties": {}, "additionalProperties": False} + ) + assert levels == [{"path": "$", "content_model": CLOSED}] + + def test_partial_when_additional_properties_is_schema(self): + levels = classify_object_levels( + {"type": "object", "additionalProperties": {"type": "string"}} + ) + assert levels == [{"path": "$", "content_model": PARTIAL}] + + def test_partial_when_pattern_properties_present(self): + levels = classify_object_levels( + { + "type": "object", + "additionalProperties": False, + "patternProperties": {"^x-": {"type": "string"}}, + } + ) + assert levels == [{"path": "$", "content_model": PARTIAL}] + + def test_nested_levels_are_reported(self): + levels = classify_object_levels( + { + "type": "object", + "additionalProperties": False, + "properties": { + "payload": {"type": "object", "properties": {}}, + }, + } + ) + paths = {level["path"]: level["content_model"] for level in levels} + assert paths == {"$": CLOSED, "$.payload": OPEN} + + def test_levels_under_applicator_keywords_are_reported(self): + levels = classify_object_levels( + { + "type": "object", + "additionalProperties": {"type": "object", "properties": {}}, + "properties": { + "items": { + "type": "array", + "items": {"type": "object", "additionalProperties": False}, + }, + "map": { + "type": "object", + "patternProperties": { + "^x-": {"type": "object", "additionalProperties": False} + }, + }, + }, + } + ) + paths = {level["path"]: level["content_model"] for level in levels} + # Object levels nested under additionalProperties, array items, and + # patternProperties are all classified now, not only `properties`. + assert paths["$.additionalProperties"] == OPEN + assert paths["$.items[]"] == CLOSED + assert paths["$.map.patternProperties[^x-]"] == CLOSED + + +class TestExplainVerdict: + def test_compatible_direction_has_no_reasons(self): + assert explain_verdict(COMPATIBLE, backward=True) == [] + + def test_incompatible_backward_explains_relation(self): + reasons = explain_verdict(INCOMPATIBLE, backward=True) + assert reasons + assert "backward incompatible" in reasons[0] + + def test_incompatible_forward_explains_relation(self): + reasons = explain_verdict(INCOMPATIBLE, backward=False) + assert reasons + assert "forward incompatible" in reasons[0] + + def test_unknown_on_differing_dialects(self): + reasons = explain_verdict(UNKNOWN, backward=True, differing_dialects=True) + assert reasons and "different JSON Schema dialects" in reasons[0] + + def test_unknown_without_dialect_difference(self): + reasons = explain_verdict(UNKNOWN, backward=False, differing_dialects=False) + assert reasons and "could not be proved or disproved" in reasons[0] + + class TestBooleanSchemaValue: def test_bool_passthrough(self): assert boolean_schema_value(True) is True diff --git a/tests/test_store_extra.py b/tests/test_store_extra.py index 34ebdc1..fc87c55 100644 --- a/tests/test_store_extra.py +++ b/tests/test_store_extra.py @@ -740,3 +740,71 @@ def test_validate_instance_invalid_non_uuid_non_gts_raises(self): store = GtsStore(reader=None) with pytest.raises(StoreGtsObjectNotFound): store.validate_instance("totally-not-valid") + + +class TestSubschemaDialect: + def test_nested_schema_switching_dialect_is_rejected(self): + schema_id = "gts.x.test._.nested_dialect.v1~" + schema = _schema_entity( + schema_id, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "inner": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + } + }, + }, + ) + store = GtsStore(reader=None) + store.register(schema) + with pytest.raises(ValueError, match="must not change JSON Schema dialect"): + store.validate_schema(schema_id) + + def test_nested_schema_restating_dialect_is_allowed(self): + schema_id = "gts.x.test._.nested_same_dialect.v1~" + schema = _schema_entity( + schema_id, + { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "inner": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + } + }, + }, + ) + store = GtsStore(reader=None) + store.register(schema) + store.validate_schema(schema_id) # must not raise + + +class TestExceptionHierarchy: + def test_not_found_errors_share_a_base(self): + from gts import GtsError, GtsNotFoundError + from gts.store import StoreGtsObjectNotFound, StoreGtsSchemaNotFound + + assert issubclass(StoreGtsObjectNotFound, GtsNotFoundError) + assert issubclass(StoreGtsSchemaNotFound, GtsNotFoundError) + assert issubclass(GtsNotFoundError, GtsError) + + def test_validation_error_is_value_error(self): + from gts import GtsError, GtsValidationError + + assert issubclass(GtsValidationError, ValueError) + assert issubclass(GtsValidationError, GtsError) + + def test_unresolvable_ref_is_catchable_as_gts_error(self): + from gts import GtsError + + schema_id = "gts.x.test._.dangling_ref.v1~" + schema = _schema_entity( + schema_id, + {"properties": {"other": {"$ref": "gts://gts.x.test._.missing.v1~"}}}, + ) + store = GtsStore(reader=None) + store.register(schema) + with pytest.raises(GtsError, match="Unresolvable \\$ref"): + store.validate_schema(schema_id) From fdce27db6ce6944eb24cac82a7635eff1e5c9f5e Mon Sep 17 00:00:00 2001 From: Artifizer Date: Sat, 26 Sep 2026 01:44:34 +0300 Subject: [PATCH 12/27] chore(types): fix mypy findings and gate `make check` on mypy Resolve the pre-existing mypy errors across the package so the type checker runs clean, then add `mypy` to the `make check` gate to keep it that way. - Narrow Optional GtsID / type_id accesses in ops (store-key selection, validate_json) and guard the cast entry points in entities. - Declare the GtsIdSegment uuid-tail attribute and allow an absent major version (int | None). - Correct the build_schema_graph return/annotations and the get_graph return type. - Add a targeted type-ignore for the FastAPI middleware factory. Signed-off-by: Artifizer --- Makefile | 2 +- gts/src/gts/_server.py | 2 +- gts/src/gts/entities.py | 14 +++++++++++--- gts/src/gts/gts.py | 4 +++- gts/src/gts/ops.py | 22 +++++++++++++++------- 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 2182d97..1b5780b 100644 --- a/Makefile +++ b/Makefile @@ -169,4 +169,4 @@ verify-spec-version: all: check build # Run all quality checks -check: fmt lint test e2e +check: fmt lint mypy test e2e diff --git a/gts/src/gts/_server.py b/gts/src/gts/_server.py index 9087aaf..cd3ca97 100644 --- a/gts/src/gts/_server.py +++ b/gts/src/gts/_server.py @@ -113,7 +113,7 @@ def __init__( self.base_url = f"http://{self.host}:{self.port}" self.app = FastAPI(title="GTS Server", version="0.15.0") self.app.add_middleware( - _RequestLoggingMiddleware, + _RequestLoggingMiddleware, # type: ignore[arg-type] verbose=self.ops.verbose, ) self._register_routes() diff --git a/gts/src/gts/entities.py b/gts/src/gts/entities.py index 94b61c3..5223d8d 100644 --- a/gts/src/gts/entities.py +++ b/gts/src/gts/entities.py @@ -184,6 +184,10 @@ def cast( from_schema: GtsEntity, resolver: Any | None = None, ) -> GtsEntityCastResult: + if self.gts_id is None: + raise SchemaCastError("source entity has no GTS identifier") + if to_schema.gts_id is None: + raise SchemaCastError("target schema has no GTS identifier") if ( self.is_schema and from_schema.gts_id @@ -386,8 +390,12 @@ def _calc_json_schema_id(self, cfg: GtsConfig) -> str | None: # No schema reference found for instance return None - def get_graph(self) -> dict[str, set[str]]: - refs = {} + def get_graph(self) -> dict[str, Any]: + refs: dict[str, str] = {} for r in self.gts_refs: refs[r["sourcePath"]] = r["id"] - return {"id": self.gts_id.id, "type_id": self.type_id, "refs": refs} + return { + "id": self.gts_id.id if self.gts_id else None, + "type_id": self.type_id, + "refs": refs, + } diff --git a/gts/src/gts/gts.py b/gts/src/gts/gts.py index 6a21b00..bdc795a 100644 --- a/gts/src/gts/gts.py +++ b/gts/src/gts/gts.py @@ -81,10 +81,12 @@ def __init__(self, num: int, offset: int, segment: str): self.package: str = "" self.namespace: str = "" self.type: str = "" - self.ver_major: int = 0 + self.ver_major: int | None = 0 self.ver_minor: int | None = None self.is_type: bool = False self.is_wildcard: bool = False + # Marks the synthetic UUID tail of a combined anonymous instance id. + self._is_uuid_tail: bool = False self._parse_segment_id(num, offset, segment) diff --git a/gts/src/gts/ops.py b/gts/src/gts/ops.py index c022f16..f65f507 100644 --- a/gts/src/gts/ops.py +++ b/gts/src/gts/ops.py @@ -436,7 +436,14 @@ def add_entity( is_type_schema=entity.is_schema, ) - store_key = entity.gts_id.id if entity.is_schema else entity.raw_id + # Both branches are guaranteed non-None by the guards above: a schema + # without gts_id and an instance without raw_id have already returned. + if entity.is_schema: + assert entity.gts_id is not None + store_key = entity.gts_id.id + else: + assert entity.raw_id is not None + store_key = entity.raw_id with self.store.transaction(): previous = self.store.get(store_key) if ( @@ -454,13 +461,11 @@ def add_entity( try: if entity.is_schema: - self.store.validate_schema_basic(entity.gts_id.id) + self.store.validate_schema_basic(store_key) if validate: - self.store.validate_schema(entity.gts_id.id, gts_ref_validation) + self.store.validate_schema(store_key, gts_ref_validation) elif validate: - self.store.validate_instance( - entity.raw_id or entity.gts_id.id, gts_ref_validation - ) + self.store.validate_instance(store_key, gts_ref_validation) except Exception as e: # noqa: BLE001 - converted to a result object at API boundary self.store.unregister(store_key) if previous: @@ -558,6 +563,7 @@ def parse_id(self, gts_id: str) -> GtsIdParseResult: # Check if it's a wildcard pattern (contains *) is_wildcard = "*" in gts_id try: + parsed: GtsID if is_wildcard: parsed = GtsWildcard(gts_id) segs = parsed.gts_id_segments @@ -680,8 +686,10 @@ def validate_json( try: if entity.is_schema: - self.store.validate_schema_content(entity.gts_id.id, content) # type: ignore[union-attr] + assert entity.gts_id is not None + self.store.validate_schema_content(entity.gts_id.id, content) else: + assert entity.type_id is not None self.store.validate_instance_content(content, entity.type_id) except Exception as error: # noqa: BLE001 - converted to a result object at API boundary error_message = str(error) From b5d5f1a32819ca11a162f885bbf4eb6367e99cd8 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Sat, 26 Sep 2026 11:45:59 +0300 Subject: [PATCH 13/27] fix(store): check ancestor $ref dialects during chain validation _validate_chain_dialect seeded its reference walk from the selected type only, so a cross-dialect gts:// $ref on an ancestor was accepted by validate_instance_content and OP#12. Seed the walk from every type in the chain so the whole chain plus its reference closure shares the root dialect (spec 11.0/12), matching the Rust reference. Signed-off-by: Artifizer --- gts/src/gts/store.py | 17 ++++++------ tests/test_store_extra.py | 54 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index e4397fd..030b494 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -502,6 +502,8 @@ def _validate_chain_dialect( return root_dialect = self._schema_dialect(root_content) + visited: set[str] = set() + queue: deque[tuple[str, dict[str, Any]]] = deque() for chain_id in chain_ids: entity = self.get(chain_id) content = ( @@ -521,15 +523,12 @@ def _validate_chain_dialect( "$id hierarchy must use the root type's dialect" ) self._validate_local_ref_dialects(content, chain_ids[0], root_dialect) - - visited: set[str] = set() - queue: deque[tuple[str, dict[str, Any]]] = deque( - [(gts_id, transient_schema)] if transient_schema is not None else [] - ) - if not queue: - entity = self.get(gts_id) - if entity and isinstance(entity.content, dict): - queue.append((gts_id, entity.content)) + # Seed the reference walk from every chain member, not just the + # leaf, so a cross-dialect gts:// $ref on an ancestor is caught + # even when the leaf does not reference it (spec §11.0/§12; + # matches the Rust reference which validates each related type in + # the closure). + queue.append((chain_id, content)) while queue: current_id, content = queue.popleft() if current_id in visited: diff --git a/tests/test_store_extra.py b/tests/test_store_extra.py index fc87c55..a2b5a13 100644 --- a/tests/test_store_extra.py +++ b/tests/test_store_extra.py @@ -452,6 +452,60 @@ def test_transitive_ref_dialect_mismatch_raises(self): with pytest.raises(ValueError, match="gts.x.test._.foreign.v1~"): store._validate_schema_chain("gts.x.test._.host.v1~") + def test_ancestor_ref_dialect_mismatch_raises(self): + # Issue C: the cross-dialect $ref lives on an ancestor; the descendant + # derives by re-declaration and references neither the ancestor nor the + # 2020-12 target. A leaf-only reference walk would accept it; walking the + # whole chain closure must reject it (OP#12 path). + foreign = _schema_entity( + "gts.x.test._.ancforeign.v1~", + {"$schema": "https://json-schema.org/draft/2020-12/schema"}, + ) + base = _schema_entity( + "gts.x.test._.ancbase.v1~", + {"properties": {"ext": {"$ref": "gts://gts.x.test._.ancforeign.v1~"}}}, + ) + child = _schema_entity( + "gts.x.test._.ancbase.v1~x.test._.ancchild.v1~", + {"properties": {"label": {"type": "string"}}}, + ) + store = GtsStore(reader=None) + store.register(foreign) + store.register(base) + store.register(child) + + with pytest.raises(ValueError, match="gts.x.test._.ancforeign.v1~"): + store._validate_schema_chain( + "gts.x.test._.ancbase.v1~x.test._.ancchild.v1~" + ) + + def test_instance_content_rejects_ancestor_cross_dialect_ref(self): + # Issue C on the OP#6 instance path: validating an instance of the + # descendant must reject it because an ancestor references a schema of a + # different dialect. + foreign = _schema_entity( + "gts.x.test._.ancforeign2.v1~", + {"$schema": "https://json-schema.org/draft/2020-12/schema"}, + ) + base = _schema_entity( + "gts.x.test._.ancbase2.v1~", + {"properties": {"ext": {"$ref": "gts://gts.x.test._.ancforeign2.v1~"}}}, + ) + child = _schema_entity( + "gts.x.test._.ancbase2.v1~x.test._.ancchild2.v1~", + {"properties": {"label": {"type": "string"}}}, + ) + store = GtsStore(reader=None) + store.register(foreign) + store.register(base) + store.register(child) + + with pytest.raises(ValueError, match="gts.x.test._.ancforeign2.v1~"): + store.validate_instance_content( + {"label": "ok"}, + "gts.x.test._.ancbase2.v1~x.test._.ancchild2.v1~", + ) + def test_trait_resource_dialect_mismatch_raises(self): schema_id = "gts.x.test._.trait_resource.v1~" schema = _schema_entity( From ec5ffd56d248bb9682be38fe61f526ed9163fcc4 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Sat, 26 Sep 2026 14:56:43 +0300 Subject: [PATCH 14/27] Register x-gts-ref as a JSON Schema keyword; fix id parsing and matching - x_gts_ref: register x-gts-ref as a jsonschema keyword via validators.extend, so oneOf/anyOf/allOf resolve correctly during structural validation. Branches that differ only by x-gts-ref are now genuinely distinct instead of collapsing to identical match-all schemas once stripped, so a oneOf of such branches no longer rejects every value. Replaces the previous strip-and-reconstruct workaround; a shared pattern matcher backs both the keyword and the XGtsRefValidator walker, which still owns /$id resolution and registry existence. - x_gts_ref: enforce a segment boundary for exact (non-wildcard) patterns, so "...w.v1" no longer matches "...w.v12"/"...w.v1.5". - GtsID: reject the gts:// URI form in the core parser. The scheme is a JSON Schema $id/$ref serialization detail and is stripped by those callers before parsing, matching the gts-rust/gts-go reference parsers. - Update the id-parsing unit tests to assert URI-form rejection. Signed-off-by: Artifizer --- gts/src/gts/gts.py | 12 +++-- gts/src/gts/store.py | 10 ++-- gts/src/gts/x_gts_ref.py | 105 ++++++++++++++++++++++++++++----------- tests/test_gts_id.py | 13 +++-- 4 files changed, 100 insertions(+), 40 deletions(-) diff --git a/gts/src/gts/gts.py b/gts/src/gts/gts.py index bdc795a..9303a4e 100644 --- a/gts/src/gts/gts.py +++ b/gts/src/gts/gts.py @@ -208,8 +208,12 @@ class GtsID: def __init__(self, id: str): raw = id.strip() - # Normalize to the canonical bare form at this boundary. - raw = _strip_scheme(raw) + # A GtsID is always the bare canonical form ("gts.…"). The "gts://" URI + # form is a JSON Schema serialization detail ($id/$ref) and is stripped + # by those URI-specific callers (e.g. entity extraction via + # ``strip_scheme``) before reaching here, mirroring the gts-rust/gts-go + # reference implementations. Accepting it here would let URI-form values + # pass validate-id/parse-id and disagree with the canonical ``id``. # Validate it's lower case if raw != raw.lower(): @@ -337,7 +341,9 @@ def to_uuid(self) -> uuid.UUID: @classmethod def is_valid(cls, s: str) -> bool: - if not _strip_scheme(s).startswith(GTS_PREFIX): + # Only the bare canonical form is a valid id; the "gts://" URI form is + # stripped by URI-specific callers before validation (see GtsID.__init__). + if not s.startswith(GTS_PREFIX): return False try: _ = cls(s) diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index 030b494..87a4e1c 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -35,7 +35,7 @@ # Re-exported for backward compatibility; the limit now lives in schema_resolver. from .schema_resolver import MAX_SCHEMA_REF_EXPANSIONS # noqa: F401 from .schema_validation import FORMAT_CHECKER, iter_schema_nodes, validator_for -from .x_gts_ref import XGtsRefValidator, _without_x_gts_ref +from .x_gts_ref import XGtsRefValidator, _without_x_gts_ref, extended_validator_for logger = logging.getLogger(__name__) @@ -1036,11 +1036,15 @@ def validate_instance_content( f"type '{schema_type.id}' is abstract and cannot have direct instances" ) + # Keep x-gts-ref in the schema and evaluate it with the extended + # validator so the engine resolves oneOf/anyOf/allOf correctly (branches + # that differ only by x-gts-ref stay distinct). Existence and /$id are + # still enforced by the XGtsRefValidator walker below. schema_for_validation = { - **_without_x_gts_ref(schema), + **schema, "$schema": self._schema_dialect_uri(schema), } - validator_class = validator_for(schema_for_validation) + validator_class = extended_validator_for(schema_for_validation) validator = validator_class( schema_for_validation, registry=self._create_reference_registry(), diff --git a/gts/src/gts/x_gts_ref.py b/gts/src/gts/x_gts_ref.py index 3810a98..c2eaf66 100644 --- a/gts/src/gts/x_gts_ref.py +++ b/gts/src/gts/x_gts_ref.py @@ -14,7 +14,8 @@ from typing import Any -from jsonschema.validators import validator_for +from jsonschema.exceptions import ValidationError +from jsonschema.validators import extend, validator_for from ._json_pointer import resolve as resolve_json_pointer from ._naming import GTS_PREFIX, strip_scheme @@ -25,7 +26,77 @@ X_GTS_REF_SELF = "/$id" +def _gts_pattern_violation(value: str, pattern: str) -> str | None: + """Return why the string ``value`` does not satisfy ``pattern``, or None. + + Pure pattern matching (concrete id, type prefix, or single trailing + wildcard). Registry existence and the ``/$id`` self-reference are NOT decided + here — they need the store and the selected-type context, which the + :class:`XGtsRefValidator` walker owns. This helper is shared by that walker + and by the structural ``x-gts-ref`` keyword so both agree on matching. + """ + if not GtsID.is_valid(value): + return f"Value '{value}' is not a valid GTS identifier" + if pattern == GTS_PREFIX + "*": + return None + if pattern.endswith("*"): + if not value.startswith(pattern[:-1]): + return f"Value '{value}' does not match pattern '{pattern}'" + return None + # Prefix matching alone ignores segment boundaries: an exact constraint such + # as "gts.a.b.c.d.v1~x.y.z.w.v1" would otherwise also accept "…w.v12"/"…w.v1.5". + # Type patterns (ending with '~') admit derived identifiers; any other (exact) + # pattern requires a full match or a '~' boundary right after the pattern. + if not value.startswith(pattern) or not ( + len(value) == len(pattern) + or pattern.endswith("~") + or value[len(pattern)] == "~" + ): + return f"Value '{value}' does not match pattern '{pattern}'" + return None + + +def _x_gts_ref_keyword(validator, ref_pattern, instance, schema): + """``jsonschema`` keyword handler that makes ``x-gts-ref`` a first-class + assertion, so ``oneOf``/``anyOf``/``allOf`` resolve correctly: two branches + that differ only by ``x-gts-ref`` are genuinely different schemas rather than + identical match-all schemas. This is the same design gts-go and gts-rust use + (a registered keyword/vocabulary) and removes the need to strip x-gts-ref and + rewrite ``oneOf``→``anyOf``. + + Only concrete/wildcard patterns are enforced here. The ``/$id`` self-reference + (needs the selected type) and registry existence stay with XGtsRefValidator. + """ + if not isinstance(ref_pattern, str) or ref_pattern == X_GTS_REF_SELF: + return + if not isinstance(instance, str): + return + reason = _gts_pattern_violation(instance, strip_scheme(ref_pattern)) + if reason is not None: + yield ValidationError(reason) + + +_EXTENDED_VALIDATORS: dict[type, type] = {} + + +def extended_validator_for(schema: Any) -> type: + """Return the ``jsonschema`` validator class for ``schema``'s dialect, + extended so ``x-gts-ref`` is evaluated as a real keyword during structural + validation (including inside combinators).""" + base = validator_for(schema) + extended = _EXTENDED_VALIDATORS.get(base) + if extended is None: + extended = extend(base, {"x-gts-ref": _x_gts_ref_keyword}) + _EXTENDED_VALIDATORS[base] = extended + return extended + + def _without_x_gts_ref(schema: Any) -> Any: + # Plain removal of x-gts-ref for the few places that deliberately need the + # bare structural shape: the walker's per-branch structural check + # (_is_structurally_valid) and the $ref reference registry. Structural + # instance validation does NOT use this — it uses extended_validator_for so + # the engine evaluates x-gts-ref natively (see _x_gts_ref_keyword). def strip(node: Any) -> Any: if not isinstance(node, dict): return node @@ -455,34 +526,10 @@ def _validate_gts_pattern( Returns: Error if validation fails, None otherwise """ - # Validate it's a valid GTS ID - if not GtsID.is_valid(value): - return XGtsRefValidationError( - field_path, - value, - pattern, - f"Value '{value}' is not a valid GTS identifier", - ) - - # Check pattern match - if pattern == GTS_PREFIX + "*": - pass # Any valid GTS ID matches - elif pattern.endswith("*"): - prefix = pattern[:-1] - if not value.startswith(prefix): - return XGtsRefValidationError( - field_path, - value, - pattern, - f"Value '{value}' does not match pattern '{pattern}'", - ) - elif not value.startswith(pattern): - return XGtsRefValidationError( - field_path, - value, - pattern, - f"Value '{value}' does not match pattern '{pattern}'", - ) + # Shared pattern matching (also used by the structural x-gts-ref keyword). + reason = _gts_pattern_violation(value, pattern) + if reason is not None: + return XGtsRefValidationError(field_path, value, pattern, reason) # Referenced values use exact registry lookup in presence/full modes. if self.store and self.mode != GtsRefValidationMode.NONE: diff --git a/tests/test_gts_id.py b/tests/test_gts_id.py index ba613f5..745b071 100644 --- a/tests/test_gts_id.py +++ b/tests/test_gts_id.py @@ -89,10 +89,12 @@ def test_valid_gts_id_basic(self): assert gts_id.is_type is True assert len(gts_id.gts_id_segments) == 1 - def test_valid_gts_id_with_uri_prefix(self): - """Test GTS ID with URI prefix is normalized.""" - gts_id = GtsID("gts://gts.vendor.package.namespace.type.v1~") - assert gts_id.id == "gts.vendor.package.namespace.type.v1~" + def test_uri_prefix_rejected_by_core_parser(self): + """The core parser accepts only the bare canonical form; the ``gts://`` + URI form is a $id/$ref serialization detail stripped by URI-specific + callers before parsing (mirroring gts-rust/gts-go).""" + with pytest.raises(GtsInvalidId): + GtsID("gts://gts.vendor.package.namespace.type.v1~") def test_valid_gts_id_instance(self): """Test instance GTS ID (not ending with ~).""" @@ -193,7 +195,8 @@ def test_to_uuid(self): def test_is_valid_static_method(self): """Test static is_valid method.""" assert GtsID.is_valid("gts.vendor.package.namespace.type.v1~") is True - assert GtsID.is_valid("gts://gts.vendor.package.namespace.type.v1~") is True + # The URI form is not a canonical id; callers strip the scheme first. + assert GtsID.is_valid("gts://gts.vendor.package.namespace.type.v1~") is False assert GtsID.is_valid("invalid") is False assert GtsID.is_valid("") is False From 9b052b993d925b9ea9209cba06f048b7d4203577 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Sat, 26 Sep 2026 18:27:39 +0300 Subject: [PATCH 15/27] fix(store): resolve /$id in x-gts-ref during combinator resolution The extended-validator x-gts-ref keyword returned early for the /$id self-reference, deferring it to the XGtsRefValidator walker. But the jsonschema engine performs oneOf/anyOf branch selection, so a /$id branch matched every string. In a oneOf that mixes a /$id branch with a concrete sibling pattern, a value matching only the sibling satisfied both branches and the exactly-one rule rejected a valid value; the walker runs afterwards and cannot undo it. Thread the selected type id into the keyword (mirroring the gts-rust design) so /$id resolves to the type being validated and participates in branch selection like any other pattern. Extended validator classes are cached per (base, selected type). Registry existence and the standalone /$id checks stay in XGtsRefValidator. Signed-off-by: Artifizer --- gts/src/gts/store.py | 4 ++- gts/src/gts/x_gts_ref.py | 65 ++++++++++++++++++++++++++-------------- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index 87a4e1c..4eb24ad 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -1044,7 +1044,9 @@ def validate_instance_content( **schema, "$schema": self._schema_dialect_uri(schema), } - validator_class = extended_validator_for(schema_for_validation) + validator_class = extended_validator_for( + schema_for_validation, selected_type_id=schema_type.id + ) validator = validator_class( schema_for_validation, registry=self._create_reference_registry(), diff --git a/gts/src/gts/x_gts_ref.py b/gts/src/gts/x_gts_ref.py index c2eaf66..90ff898 100644 --- a/gts/src/gts/x_gts_ref.py +++ b/gts/src/gts/x_gts_ref.py @@ -56,38 +56,57 @@ def _gts_pattern_violation(value: str, pattern: str) -> str | None: return None -def _x_gts_ref_keyword(validator, ref_pattern, instance, schema): - """``jsonschema`` keyword handler that makes ``x-gts-ref`` a first-class - assertion, so ``oneOf``/``anyOf``/``allOf`` resolve correctly: two branches - that differ only by ``x-gts-ref`` are genuinely different schemas rather than - identical match-all schemas. This is the same design gts-go and gts-rust use - (a registered keyword/vocabulary) and removes the need to strip x-gts-ref and - rewrite ``oneOf``→``anyOf``. - - Only concrete/wildcard patterns are enforced here. The ``/$id`` self-reference - (needs the selected type) and registry existence stay with XGtsRefValidator. +def _make_x_gts_ref_keyword(selected_type_id: str | None): + """Build a ``jsonschema`` keyword handler that makes ``x-gts-ref`` a + first-class assertion, so ``oneOf``/``anyOf``/``allOf`` resolve correctly: + two branches that differ only by ``x-gts-ref`` are genuinely different + schemas rather than identical match-all schemas. This is the same design + gts-go and gts-rust use (a registered keyword/vocabulary) and removes the + need to strip x-gts-ref and rewrite ``oneOf``→``anyOf``. + + ``selected_type_id`` (the type being validated) lets the ``/$id`` + self-reference resolve here so a ``/$id`` branch matches only that type + instead of matching unconditionally; without it, ``/$id`` is deferred to + XGtsRefValidator. Registry existence always stays with XGtsRefValidator. """ - if not isinstance(ref_pattern, str) or ref_pattern == X_GTS_REF_SELF: - return - if not isinstance(instance, str): - return - reason = _gts_pattern_violation(instance, strip_scheme(ref_pattern)) - if reason is not None: - yield ValidationError(reason) + def _keyword(validator, ref_pattern, instance, schema): + if not isinstance(ref_pattern, str): + return + if not isinstance(instance, str): + return + pattern = ref_pattern + if ref_pattern == X_GTS_REF_SELF: + if not selected_type_id: + return + pattern = selected_type_id + reason = _gts_pattern_violation(instance, strip_scheme(pattern)) + if reason is not None: + yield ValidationError(reason) + + return _keyword -_EXTENDED_VALIDATORS: dict[type, type] = {} +# Backwards-compatible symbol: the /$id-deferring keyword (no selected type). +_x_gts_ref_keyword = _make_x_gts_ref_keyword(None) -def extended_validator_for(schema: Any) -> type: + +_EXTENDED_VALIDATORS: dict[tuple[type, str | None], type] = {} + + +def extended_validator_for(schema: Any, selected_type_id: str | None = None) -> type: """Return the ``jsonschema`` validator class for ``schema``'s dialect, extended so ``x-gts-ref`` is evaluated as a real keyword during structural - validation (including inside combinators).""" + validation (including inside combinators). ``selected_type_id`` is threaded + into the keyword so ``/$id`` resolves during combinator resolution.""" base = validator_for(schema) - extended = _EXTENDED_VALIDATORS.get(base) + key = (base, selected_type_id) + extended = _EXTENDED_VALIDATORS.get(key) if extended is None: - extended = extend(base, {"x-gts-ref": _x_gts_ref_keyword}) - _EXTENDED_VALIDATORS[base] = extended + extended = extend( + base, {"x-gts-ref": _make_x_gts_ref_keyword(selected_type_id)} + ) + _EXTENDED_VALIDATORS[key] = extended return extended From df2159884691fe3a688446701aa37b63829bf50f Mon Sep 17 00:00:00 2001 From: Artifizer Date: Sun, 27 Sep 2026 12:16:52 +0300 Subject: [PATCH 16/27] feat(ops): honor validate and gts-ref-validation on batch add_schemas Batch type-schema registration ignored the validate flag and gts-ref-validation mode: add_schemas called add_schema, which registered each entry directly via the store with no validation. This diverged from add_entity (POST /entities), where those parameters drive per-entry validation. Thread validate and gts-ref-validation through add_schemas/add_schema and route registration through the single-entity add_entity path so each batch entry is validated identically. The embedded $schema/$id presence and GTS type-id checks remain batch-specific and now reject a missing $schema, a non-gts:// $id, or a malformed type identifier up front. The HTTP handler wires the ?validate/?validation aliases and ?gts-ref-validation query parameters into the call. Signed-off-by: Artifizer --- gts/src/gts/_server.py | 11 +++++- gts/src/gts/ops.py | 84 ++++++++++++++++++++++++++++++------------ 2 files changed, 70 insertions(+), 25 deletions(-) diff --git a/gts/src/gts/_server.py b/gts/src/gts/_server.py index cd3ca97..f406341 100644 --- a/gts/src/gts/_server.py +++ b/gts/src/gts/_server.py @@ -288,8 +288,17 @@ async def add_entities( async def add_schemas( self, body: list[dict[str, Any]] = Body(...), + validate: bool = Query(False), + validation: bool = Query(False), + gts_ref_validation: GtsRefValidationMode = GTS_REF_VALIDATION_QUERY, ) -> JSONResponse: - return JSONResponse(self.ops.add_schemas(body).to_dict()) + return JSONResponse( + self.ops.add_schemas( + body, + validate=validate is True or validation is True, + gts_ref_validation=gts_ref_validation, + ).to_dict() + ) async def validate_id(self, id: str = Query(..., alias="gts_id")) -> dict[str, Any]: return self.ops.validate_id(id).to_dict() diff --git a/gts/src/gts/ops.py b/gts/src/gts/ops.py index f65f507..500a6f9 100644 --- a/gts/src/gts/ops.py +++ b/gts/src/gts/ops.py @@ -6,7 +6,7 @@ from pathlib import Path as SysPath from typing import Any -from ._naming import looks_like_gts, strip_scheme +from ._naming import GTS_PREFIX, GTS_URI_PREFIX, looks_like_gts, strip_scheme from .entities import DEFAULT_GTS_CONFIG, GtsConfig, GtsEntity from .files_reader import GtsFileReader from .gts import GtsID, GtsWildcard @@ -495,21 +495,55 @@ def add_entities( return GtsAddEntitiesResult(ok=ok, results=results) def add_schemas( - self, schemas: builtins.list[dict[str, Any]] + self, + schemas: builtins.list[dict[str, Any]], + validate: bool = False, + gts_ref_validation: GtsRefValidationMode = GtsRefValidationMode.ANY_VALID, ) -> GtsAddSchemasResult: """Register a batch of GTS Type Schemas. Each entry's GTS Type Identifier is derived from its embedded ``$id``; the aggregate ``ok`` is ``True`` only when every entry registered. + ``validate`` / ``gts_ref_validation`` apply to every entry exactly as + they do on ``POST /entities``. """ - results = [self.add_schema(schema) for schema in schemas] + results = [ + self.add_schema(schema, validate=validate, gts_ref_validation=gts_ref_validation) + for schema in schemas + ] ok = all(r.ok for r in results) return GtsAddSchemasResult(ok=ok, results=results) - def add_schema(self, schema: dict[str, Any]) -> GtsAddSchemaResult: - """Register a single GTS Type Schema, deriving its type_id from ``$id``.""" - embedded_id = schema.get("$id") if isinstance(schema, dict) else None - if not isinstance(embedded_id, str) or not embedded_id: + def add_schema( + self, + schema: dict[str, Any], + validate: bool = False, + gts_ref_validation: GtsRefValidationMode = GtsRefValidationMode.ANY_VALID, + ) -> GtsAddSchemaResult: + """Register a single GTS Type Schema, deriving its type_id from ``$id``. + + The embedded ``$schema`` / ``$id`` presence checks are batch-specific; + the actual registration and (when requested) validation reuse the + single-entity :meth:`add_entity` path, so each entry honors ``validate`` + / ``gts_ref_validation`` exactly like a ``POST /entities`` call. + """ + if not isinstance(schema, dict): + return GtsAddSchemaResult( + ok=False, + type_id=None, + error="GTS Type Schema entry must be a JSON object", + ) + dialect = schema.get("$schema") + if not isinstance(dialect, str) or not dialect: + return GtsAddSchemaResult( + ok=False, + type_id=None, + error="GTS Type Schema must contain a top-level $schema field", + ) + embedded_id = schema.get("$id") + if not isinstance(embedded_id, str) or not embedded_id.startswith( + GTS_URI_PREFIX + GTS_PREFIX + ): return GtsAddSchemaResult( ok=False, type_id=None, @@ -517,23 +551,25 @@ def add_schema(self, schema: dict[str, Any]) -> GtsAddSchemaResult: ) type_id = strip_scheme(embedded_id) try: - with self.store.transaction(): - previous = self.store.get(type_id) - if ( - previous - and not self.allow_entity_updates - and previous.content != schema - ): - return GtsAddSchemaResult( - ok=False, - type_id=type_id, - error=f"Entity '{type_id}' is already registered with different content", - conflict=True, - ) - self.store.register_schema(type_id, schema) - return GtsAddSchemaResult(ok=True, type_id=type_id) - except Exception as e: # noqa: BLE001 - converted to a result object at API boundary - return GtsAddSchemaResult(ok=False, type_id=type_id, error=str(e)) + GtsID.parse_type(type_id) + except ValueError: + return GtsAddSchemaResult( + ok=False, + type_id=type_id, + error=f"Invalid GTS Type Schema $id: {embedded_id}", + ) + + result = self.add_entity( + schema, validate=validate, gts_ref_validation=gts_ref_validation + ) + if result.ok: + return GtsAddSchemaResult(ok=True, type_id=type_id) + return GtsAddSchemaResult( + ok=False, + type_id=type_id, + error=result.error, + conflict=result.conflict, + ) def validate_id(self, gts_id: str) -> GtsIdValidationResult: # Check if it's a wildcard pattern (contains *) From bb5f4b058eae6cb2bee881aab8a7884f74eb9db1 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Sun, 27 Sep 2026 15:28:43 +0300 Subject: [PATCH 17/27] feat(store): stage validate=true registration behind a committed overlay Add a staging overlay to GtsStore: staged entities are visible to internal validation and the $ref reference registry but invisible to public reads (get_committed) until commit. A validate=true registration stages, validates, then commits on success or discards on failure, so a concurrent reader never observes an unvalidated entity and no lock is held across validation. Batch add_schemas now runs two-phase - stage every entry, validate each against the fully-staged set, then commit the survivors and discard the rest - making it order-independent while never publishing an entry that fails. Add a threaded concurrency test. Signed-off-by: Artifizer --- gts/src/gts/ops.py | 240 ++++++++++++++++++-------- gts/src/gts/store.py | 77 ++++++++- gts/tests/test_staging_concurrency.py | 78 +++++++++ 3 files changed, 317 insertions(+), 78 deletions(-) create mode 100644 gts/tests/test_staging_concurrency.py diff --git a/gts/src/gts/ops.py b/gts/src/gts/ops.py index 500a6f9..5ef6bd5 100644 --- a/gts/src/gts/ops.py +++ b/gts/src/gts/ops.py @@ -444,37 +444,40 @@ def add_entity( else: assert entity.raw_id is not None store_key = entity.raw_id - with self.store.transaction(): - previous = self.store.get(store_key) - if ( - previous - and not self.allow_entity_updates - and previous.content != entity.content - ): - return GtsAddEntityResult( - ok=False, - error=f"Entity '{store_key}' is already registered with different content", - is_type_schema=entity.is_schema, - conflict=True, - ) - self.store.register(entity) + previous = self.store.get_committed(store_key) + if ( + previous + and not self.allow_entity_updates + and previous.content != entity.content + ): + return GtsAddEntityResult( + ok=False, + error=f"Entity '{store_key}' is already registered with different content", + is_type_schema=entity.is_schema, + conflict=True, + ) - try: - if entity.is_schema: - self.store.validate_schema_basic(store_key) - if validate: - self.store.validate_schema(store_key, gts_ref_validation) - elif validate: - self.store.validate_instance(store_key, gts_ref_validation) - except Exception as e: # noqa: BLE001 - converted to a result object at API boundary - self.store.unregister(store_key) - if previous: - self.store.register(previous) - return GtsAddEntityResult( - ok=False, - error=f"Validation failed: {e!s}", - is_type_schema=entity.is_schema, - ) + # Stage the entity (invisible to public reads) and validate it before + # publishing. A failure discards the staged copy, so a reader never + # observes an entity that has not passed validation, and the committed + # state (any prior version under this id) is never touched. No lock is + # held across validation, so concurrent reads are not blocked. + self.store.stage(entity) + try: + if entity.is_schema: + self.store.validate_schema_basic(store_key) + if validate: + self.store.validate_schema(store_key, gts_ref_validation) + elif validate: + self.store.validate_instance(store_key, gts_ref_validation) + except Exception as e: # noqa: BLE001 - converted to a result object at API boundary + self.store.discard(store_key) + return GtsAddEntityResult( + ok=False, + error=f"Validation failed: {e!s}", + is_type_schema=entity.is_schema, + ) + self.store.commit(store_key) # Return gts_id if available, otherwise raw_id entity_id = entity.gts_id.id if entity.gts_id else (entity.raw_id or "") @@ -494,39 +497,33 @@ def add_entities( ok = all(r.ok for r in results) return GtsAddEntitiesResult(ok=ok, results=results) - def add_schemas( + def _validate_staged( self, - schemas: builtins.list[dict[str, Any]], - validate: bool = False, - gts_ref_validation: GtsRefValidationMode = GtsRefValidationMode.ANY_VALID, - ) -> GtsAddSchemasResult: - """Register a batch of GTS Type Schemas. - - Each entry's GTS Type Identifier is derived from its embedded ``$id``; - the aggregate ``ok`` is ``True`` only when every entry registered. - ``validate`` / ``gts_ref_validation`` apply to every entry exactly as - they do on ``POST /entities``. - """ - results = [ - self.add_schema(schema, validate=validate, gts_ref_validation=gts_ref_validation) - for schema in schemas - ] - ok = all(r.ok for r in results) - return GtsAddSchemasResult(ok=ok, results=results) - - def add_schema( - self, - schema: dict[str, Any], - validate: bool = False, - gts_ref_validation: GtsRefValidationMode = GtsRefValidationMode.ANY_VALID, - ) -> GtsAddSchemaResult: - """Register a single GTS Type Schema, deriving its type_id from ``$id``. - - The embedded ``$schema`` / ``$id`` presence checks are batch-specific; - the actual registration and (when requested) validation reuse the - single-entity :meth:`add_entity` path, so each entry honors ``validate`` - / ``gts_ref_validation`` exactly like a ``POST /entities`` call. - """ + entity: GtsEntity, + store_key: str, + validate: bool, + gts_ref_validation: GtsRefValidationMode, + ) -> str | None: + """Validate a staged entity against the current (staged + committed) + set, returning an error message on failure or ``None`` on success. The + caller stages before and commits/discards after.""" + try: + if entity.is_schema: + self.store.validate_schema_basic(store_key) + if validate: + self.store.validate_schema(store_key, gts_ref_validation) + elif validate: + self.store.validate_instance(store_key, gts_ref_validation) + except Exception as e: # noqa: BLE001 - converted to a result object at API boundary + return f"Validation failed: {e!s}" + return None + + def _prepare_type_schema( + self, schema: dict[str, Any] + ) -> tuple[str, GtsEntity, str] | GtsAddSchemaResult: + """Run the batch-specific $schema/$id checks and build the entity. On + success returns ``(type_id, entity, store_key)``; on failure returns the + per-item error result. Does not stage or register anything.""" if not isinstance(schema, dict): return GtsAddSchemaResult( ok=False, @@ -558,18 +555,112 @@ def add_schema( type_id=type_id, error=f"Invalid GTS Type Schema $id: {embedded_id}", ) + entity = GtsEntity(content=schema, cfg=self.cfg) + if not entity.is_schema or not entity.gts_id: + return GtsAddSchemaResult( + ok=False, type_id=type_id, error="Unable to detect GTS ID in schema" + ) + store_key = entity.gts_id.id + previous = self.store.get_committed(store_key) + if ( + previous + and not self.allow_entity_updates + and previous.content != entity.content + ): + return GtsAddSchemaResult( + ok=False, + type_id=type_id, + error=f"Entity '{store_key}' is already registered with different content", + conflict=True, + ) + return (type_id, entity, store_key) - result = self.add_entity( - schema, validate=validate, gts_ref_validation=gts_ref_validation - ) - if result.ok: - return GtsAddSchemaResult(ok=True, type_id=type_id) - return GtsAddSchemaResult( - ok=False, - type_id=type_id, - error=result.error, - conflict=result.conflict, - ) + def add_schemas( + self, + schemas: builtins.list[dict[str, Any]], + validate: bool = False, + gts_ref_validation: GtsRefValidationMode = GtsRefValidationMode.ANY_VALID, + ) -> GtsAddSchemasResult: + """Register a batch of GTS Type Schemas. + + Each entry's GTS Type Identifier is derived from its embedded ``$id``; + the aggregate ``ok`` is ``True`` only when every entry registered. + ``validate`` / ``gts_ref_validation`` apply to every entry exactly as + they do on ``POST /entities``. + + With ``validate`` the batch runs in two phases so the outcome is + order-independent and nothing invalid is ever published: every + structurally-valid entry is staged first (invisible to public reads), + then each is validated against the fully-staged set - so an entry can + resolve intra-batch references/ancestors regardless of position - and + finally the entries that passed are committed while the rest are + discarded. + """ + gts_ref_validation = _normalize_gts_ref_validation(gts_ref_validation) + if not validate: + results = [ + self.add_schema(schema, validate=False, gts_ref_validation=gts_ref_validation) + for schema in schemas + ] + return GtsAddSchemasResult(ok=all(r.ok for r in results), results=results) + + results: list[GtsAddSchemaResult | None] = [None] * len(schemas) + # Phase 1: stage every structurally-valid entry. + staged: list[tuple[int, str, GtsEntity, str]] = [] + for index, schema in enumerate(schemas): + prepared = self._prepare_type_schema(schema) + if isinstance(prepared, GtsAddSchemaResult): + results[index] = prepared + continue + type_id, entity, store_key = prepared + self.store.stage(entity) + staged.append((index, type_id, entity, store_key)) + + # Phase 2: validate every staged entry against the fully-staged set. + verdicts: list[tuple[int, str, str, str | None]] = [] + for index, type_id, entity, store_key in staged: + error = self._validate_staged(entity, store_key, True, gts_ref_validation) + verdicts.append((index, type_id, store_key, error)) + + # Phase 3: publish the entries that passed, discard the ones that failed. + for index, type_id, store_key, error in verdicts: + if error is None: + self.store.commit(store_key) + results[index] = GtsAddSchemaResult(ok=True, type_id=type_id) + else: + self.store.discard(store_key) + results[index] = GtsAddSchemaResult( + ok=False, type_id=type_id, error=error + ) + + final = [r for r in results if r is not None] + return GtsAddSchemasResult(ok=all(r.ok for r in final), results=final) + + def add_schema( + self, + schema: dict[str, Any], + validate: bool = False, + gts_ref_validation: GtsRefValidationMode = GtsRefValidationMode.ANY_VALID, + ) -> GtsAddSchemaResult: + """Register a single GTS Type Schema, deriving its type_id from ``$id``. + + Stages the entity (invisible to public reads), validates it, then + commits on success or discards on failure, so each entry honors + ``validate`` / ``gts_ref_validation`` exactly like a ``POST /entities`` + call and an invalid schema is never observable. + """ + gts_ref_validation = _normalize_gts_ref_validation(gts_ref_validation) + prepared = self._prepare_type_schema(schema) + if isinstance(prepared, GtsAddSchemaResult): + return prepared + type_id, entity, store_key = prepared + self.store.stage(entity) + error = self._validate_staged(entity, store_key, validate, gts_ref_validation) + if error is not None: + self.store.discard(store_key) + return GtsAddSchemaResult(ok=False, type_id=type_id, error=error) + self.store.commit(store_key) + return GtsAddSchemaResult(ok=True, type_id=type_id) def validate_id(self, gts_id: str) -> GtsIdValidationResult: # Check if it's a wildcard pattern (contains *) @@ -855,7 +946,8 @@ def get_entity(self, gts_id: str) -> GtsGetEntityResult: GtsGetEntityResult with entity details or error """ try: - entity = self.store.get(gts_id) + # Public read: never expose a staged (not-yet-committed) entity. + entity = self.store.get_committed(gts_id) if not entity: return GtsGetEntityResult( ok=False, error=f"Entity '{gts_id}' not found" diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index 4eb24ad..69275b0 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -139,6 +139,14 @@ def __init__(self, reader: GtsReader | None = None) -> None: reader: GtsReader instance to populate entities from """ self._by_id: dict[str, GtsEntity] = {} + # Entities registered but not yet published. A staged entity is visible + # to internal validation/resolution (via `get`, the reference registry + # and x-gts-ref existence checks) so a batch can resolve intra-batch + # references regardless of entry order, but it is invisible to public + # reads (`get_committed`, `values`, `keys`, `items`, `query`) until + # `commit`. This is how a validate=true registration avoids ever + # exposing an entity that has not yet passed validation. + self._staged: dict[str, GtsEntity] = {} self._reader = reader self._lock = threading.RLock() self._reference_registry: Registry | None = None @@ -228,15 +236,31 @@ def register_schema(self, type_id: str, schema: dict[str, Any]) -> None: def get(self, entity_id: str) -> GtsEntity | None: """ - Get a JsonEntity by its ID. - If not found in cache, try to fetch from reader. - Returns None if not found. + Get a JsonEntity by its ID for INTERNAL use (validation, $ref/existence + resolution). The staging overlay is consulted first so an entity being + validated as part of a batch can resolve its not-yet-committed siblings + regardless of order. Public/API reads MUST use :meth:`get_committed` so + uncommitted entities are never exposed. Lookups are normalized to the canonical bare form here, so callers may pass either a bare ``gts.`` id or a ``gts://`` URI without stripping the scheme themselves. """ entity_id = strip_scheme(entity_id) + with self._lock: + staged = self._staged.get(entity_id) + if staged is not None: + return copy.deepcopy(staged) + return self.get_committed(entity_id) + + def get_committed(self, entity_id: str) -> GtsEntity | None: + """ + Committed-only lookup: the staging overlay is never consulted, so a + staged-but-not-yet-committed entity is invisible here. This is the read + path for public/API consumers. Falls back to the reader for ids not yet + cached. Returns None if not found. + """ + entity_id = strip_scheme(entity_id) with self._lock: entity = self._by_id.get(entity_id) if entity is not None: @@ -253,6 +277,47 @@ def get(self, entity_id: str) -> GtsEntity | None: return None + @staticmethod + def _entity_key(entity: GtsEntity) -> str: + """The registry key for an entity: the raw id for instances that carry + one (anonymous/UUID instances), otherwise the canonical GTS id.""" + if not entity.is_schema and entity.raw_id: + return entity.raw_id + if entity.gts_id and entity.gts_id.id: + return entity.gts_id.id + if entity.raw_id: + return entity.raw_id + raise ValueError("Entity must have a valid gts_id or raw_id") + + def stage(self, entity: GtsEntity) -> str: + """Place an entity into the staging overlay WITHOUT publishing it, and + return its registry key. A staged entity is visible to internal + validation but invisible to public reads until :meth:`commit`. Raises + ``EntityConflictError`` when a committed entity with different content + already holds the key and updates are not allowed. Callers MUST + eventually :meth:`commit` or :meth:`discard` the staged key.""" + stored = copy.deepcopy(entity) + key = self._entity_key(stored) + with self._lock: + self._staged[key] = stored + self._invalidate_reference_registry() + return key + + def commit(self, entity_id: str) -> None: + """Publish a previously staged entity, making it visible to public reads.""" + with self._lock: + staged = self._staged.pop(entity_id, None) + if staged is not None: + self._by_id[entity_id] = staged + self._invalidate_reference_registry() + + def discard(self, entity_id: str) -> None: + """Drop a staged entity that failed validation. The committed state is + untouched, so a client never observes the discarded (invalid) entity.""" + with self._lock: + if self._staged.pop(entity_id, None) is not None: + self._invalidate_reference_registry() + def get_schema_content(self, type_id: str) -> dict[str, Any]: """Get schema content as dict (legacy method for backward compatibility).""" entity = self.get(type_id) @@ -265,7 +330,11 @@ def _create_reference_registry(self) -> Registry: if self._reference_registry is not None: return self._reference_registry registry = Registry() - for entity_id, entity in self._by_id.items(): + # Committed entities plus the staging overlay (staged overrides a + # committed entry of the same id), so a schema being validated as + # part of a batch resolves `$ref`s to its not-yet-committed siblings. + merged = {**self._by_id, **self._staged} + for entity_id, entity in merged.items(): if entity.is_schema and isinstance(entity.content, dict): resource = Resource.from_contents( _without_x_gts_ref(entity.content), diff --git a/gts/tests/test_staging_concurrency.py b/gts/tests/test_staging_concurrency.py new file mode 100644 index 0000000..8e96060 --- /dev/null +++ b/gts/tests/test_staging_concurrency.py @@ -0,0 +1,78 @@ +"""Concurrency test for validate=true batch staging. + +A ``validate=true`` batch stages entries invisibly, validates them, and commits +only the survivors, so a concurrent reader must never observe an entity that has +not passed validation. This test runs a batch containing a deliberately-invalid +entry while several threads hammer the committed read path for that entry's id, +asserting it is never observable. The cycle is repeated many times to widen the +window. ``GtsStore`` is thread-safe (an ``RLock`` guards the maps) and no lock is +held across validation, so the reads run concurrently with the batch writer. +""" + +import threading +import uuid + +from gts.ops import GtsOps + +_DRAFT7 = "http://json-schema.org/draft-07/schema#" +_CYCLES = 10 +_PROBE_THREADS = 4 +_VALID_PER_BATCH = 80 + + +def _batch(ns: str): + valid = [ + { + "$schema": _DRAFT7, + "$id": f"gts://gts.x.{ns}._.t{i}.v1~", + "type": "object", + "properties": {"p": {"type": "string"}}, + } + for i in range(_VALID_PER_BATCH) + ] + invalid_id = f"gts.x.{ns}._.invalid.v1~" + invalid = { + "$schema": _DRAFT7, + "$id": f"gts://{invalid_id}", + "type": "object", + "properties": {"a": {"$ref": f"gts://gts.x.{ns}._.never.v1~"}}, + } + return valid + [invalid], invalid_id + + +def test_validate_batch_never_exposes_uncommitted_entities(): + ops = GtsOps(path=None) + leaks: list[object] = [] + leaks_lock = threading.Lock() + + for cycle in range(_CYCLES): + ns = f"pyconc{uuid.uuid4().hex[:8]}c{cycle}" + batch, invalid_id = _batch(ns) + stop = threading.Event() + + def probe(invalid_id=invalid_id): + while not stop.is_set(): + entity = ops.store.get_committed(invalid_id) + if entity is not None: + with leaks_lock: + leaks.append(entity) + + probers = [threading.Thread(target=probe) for _ in range(_PROBE_THREADS)] + for prober in probers: + prober.start() + try: + result = ops.add_schemas(batch, validate=True) + finally: + stop.set() + for prober in probers: + prober.join(timeout=10) + + assert result.ok is False, "batch with an invalid entry must report ok=false" + # The valid entries are committed; the invalid one is not. + assert ops.store.get_committed(f"gts.x.{ns}._.t0.v1~") is not None + assert ops.store.get_committed(invalid_id) is None + + assert leaks == [], ( + "an uncommitted/invalid entity was exposed to a concurrent reader " + f"({len(leaks)} time(s))" + ) From 39e9387bf711e4a31da21b4e0ee7a60fb68a0ced Mon Sep 17 00:00:00 2001 From: Artifizer Date: Sun, 27 Sep 2026 15:38:20 +0300 Subject: [PATCH 18/27] fix: fix formatting issues Signed-off-by: Artifizer --- gts/src/gts/ops.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gts/src/gts/ops.py b/gts/src/gts/ops.py index 5ef6bd5..d5baf0c 100644 --- a/gts/src/gts/ops.py +++ b/gts/src/gts/ops.py @@ -599,7 +599,9 @@ def add_schemas( gts_ref_validation = _normalize_gts_ref_validation(gts_ref_validation) if not validate: results = [ - self.add_schema(schema, validate=False, gts_ref_validation=gts_ref_validation) + self.add_schema( + schema, validate=False, gts_ref_validation=gts_ref_validation + ) for schema in schemas ] return GtsAddSchemasResult(ok=all(r.ok for r in results), results=results) From a4c00f534fc8a370664affb7fc3d28efb9802662 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Sun, 27 Sep 2026 15:55:12 +0300 Subject: [PATCH 19/27] test: run the staging concurrency test under `make test` `make test` runs `pytest tests/`, so the staging concurrency test placed under `gts/tests/` never executed. Move it to `tests/` alongside the other unit tests so it runs as part of the suite. Also make `test_add_schema_missing_id_failure` precise: it fed a document missing both `$schema` and `$id` and asserted the error mentioned `$id`, but a Type Schema entry is now checked for `$schema` before `$id` (consistent with the other implementations). Give it a valid `$schema` so it specifically exercises the absent `$id`. Signed-off-by: Artifizer --- tests/test_ops.py | 6 +++++- {gts/tests => tests}/test_staging_concurrency.py | 0 2 files changed, 5 insertions(+), 1 deletion(-) rename {gts/tests => tests}/test_staging_concurrency.py (100%) diff --git a/tests/test_ops.py b/tests/test_ops.py index 0887732..b4b42ca 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -186,7 +186,11 @@ def test_add_schema_changed_content_is_conflict(self, ops): assert result.conflict is True def test_add_schema_missing_id_failure(self, ops): - result = ops.add_schema({"type": "object"}) + # $schema is present so the failure is specifically the absent $id + # (a Type Schema entry is checked for $schema before $id). + result = ops.add_schema( + {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object"} + ) assert result.ok is False assert "$id" in result.error diff --git a/gts/tests/test_staging_concurrency.py b/tests/test_staging_concurrency.py similarity index 100% rename from gts/tests/test_staging_concurrency.py rename to tests/test_staging_concurrency.py From bcc2881e0978b707c74c036502a5f0da73a3cc5b Mon Sep 17 00:00:00 2001 From: Artifizer Date: Sun, 27 Sep 2026 21:12:52 +0300 Subject: [PATCH 20/27] fix(store): make validate=true batch staging leak-, conflict- and dependency-safe The two-phase add_schemas(validate=True) flow had three defects: staged entries leaked if an exception occurred before the commit phase, a survivor could be committed even though a staged sibling it depends on was discarded, and staging keyed purely by the registry key let concurrent batches (or a duplicated batch entry) clobber each other while commit bypassed the conflict check. - Key the staging overlay by a unique token, with a by-key overlay for internal resolution and the reference registry, so two entries that resolve to the same key never overwrite each other and a commit/discard only affects the entry it names. - Commit atomically with a compare-and-swap against the committed store, returning "added"/"unchanged"/"conflict" so a conflicting publish is reported rather than silently overwriting existing content. - Track pending tokens and discard the leftovers in a finally block, so a raised exception anywhere in the batch never leaves unvalidated entries in the read overlay. - Iteratively discard failures and re-validate the survivors against the now-smaller staged set until a round is clean, so nothing is committed with a dangling parent or reference to a discarded sibling. Adds tests for token isolation, compare-and-swap commit, the dependent-of-discarded case and the duplicate-id case. Signed-off-by: Artifizer --- gts/src/gts/ops.py | 115 ++++++++++++++++++------- gts/src/gts/store.py | 81 ++++++++++++----- tests/test_staging_commit_integrity.py | 112 ++++++++++++++++++++++++ 3 files changed, 255 insertions(+), 53 deletions(-) create mode 100644 tests/test_staging_commit_integrity.py diff --git a/gts/src/gts/ops.py b/gts/src/gts/ops.py index d5baf0c..0340151 100644 --- a/gts/src/gts/ops.py +++ b/gts/src/gts/ops.py @@ -462,7 +462,7 @@ def add_entity( # observes an entity that has not passed validation, and the committed # state (any prior version under this id) is never touched. No lock is # held across validation, so concurrent reads are not blocked. - self.store.stage(entity) + token = self.store.stage(entity) try: if entity.is_schema: self.store.validate_schema_basic(store_key) @@ -471,13 +471,19 @@ def add_entity( elif validate: self.store.validate_instance(store_key, gts_ref_validation) except Exception as e: # noqa: BLE001 - converted to a result object at API boundary - self.store.discard(store_key) + self.store.discard(token) return GtsAddEntityResult( ok=False, error=f"Validation failed: {e!s}", is_type_schema=entity.is_schema, ) - self.store.commit(store_key) + if self.store.commit(token, self.allow_entity_updates) == "conflict": + return GtsAddEntityResult( + ok=False, + error=f"Entity '{store_key}' is already registered with different content", + is_type_schema=entity.is_schema, + conflict=True, + ) # Return gts_id if available, otherwise raw_id entity_id = entity.gts_id.id if entity.gts_id else (entity.raw_id or "") @@ -607,33 +613,70 @@ def add_schemas( return GtsAddSchemasResult(ok=all(r.ok for r in results), results=results) results: list[GtsAddSchemaResult | None] = [None] * len(schemas) - # Phase 1: stage every structurally-valid entry. - staged: list[tuple[int, str, GtsEntity, str]] = [] - for index, schema in enumerate(schemas): - prepared = self._prepare_type_schema(schema) - if isinstance(prepared, GtsAddSchemaResult): - results[index] = prepared - continue - type_id, entity, store_key = prepared - self.store.stage(entity) - staged.append((index, type_id, entity, store_key)) - - # Phase 2: validate every staged entry against the fully-staged set. - verdicts: list[tuple[int, str, str, str | None]] = [] - for index, type_id, entity, store_key in staged: - error = self._validate_staged(entity, store_key, True, gts_ref_validation) - verdicts.append((index, type_id, store_key, error)) - - # Phase 3: publish the entries that passed, discard the ones that failed. - for index, type_id, store_key, error in verdicts: - if error is None: - self.store.commit(store_key) - results[index] = GtsAddSchemaResult(ok=True, type_id=type_id) - else: - self.store.discard(store_key) - results[index] = GtsAddSchemaResult( - ok=False, type_id=type_id, error=error - ) + # Track every still-staged token so an exception anywhere below discards + # the leftovers (in the finally block) instead of leaking unvalidated + # entries into the staging overlay. + pending: set[str] = set() + try: + # Phase 1: stage every structurally-valid entry. + # survivors: (index, type_id, entity, store_key, token) + survivors: list[tuple[int, str, GtsEntity, str, str]] = [] + for index, schema in enumerate(schemas): + prepared = self._prepare_type_schema(schema) + if isinstance(prepared, GtsAddSchemaResult): + results[index] = prepared + continue + type_id, entity, store_key = prepared + token = self.store.stage(entity) + pending.add(token) + survivors.append((index, type_id, entity, store_key, token)) + + # Phase 2: validate the staged entries, discarding failures and + # RE-validating the survivors against the now-smaller staged set until + # a round produces no new failures. This stops an entry that only + # validated because a sibling was staged (e.g. its parent or $ref + # target) from being committed after that sibling was discarded. + while True: + still_good: list[tuple[int, str, GtsEntity, str, str]] = [] + failed: list[tuple[int, str, str, str]] = [] + for index, type_id, entity, store_key, token in survivors: + error = self._validate_staged( + entity, store_key, True, gts_ref_validation + ) + if error is None: + still_good.append((index, type_id, entity, store_key, token)) + else: + failed.append((index, type_id, token, error)) + if not failed: + break + for index, type_id, token, error in failed: + self.store.discard(token) + pending.discard(token) + results[index] = GtsAddSchemaResult( + ok=False, type_id=type_id, error=error + ) + survivors = still_good + + # Phase 3: publish the entries that passed. A commit can still report + # a conflict if the same id was already committed with different + # content in the meantime. + for index, type_id, _entity, store_key, token in survivors: + outcome = self.store.commit(token, self.allow_entity_updates) + pending.discard(token) + if outcome == "conflict": + results[index] = GtsAddSchemaResult( + ok=False, + type_id=type_id, + error=f"Entity '{store_key}' is already registered with different content", + conflict=True, + ) + else: + results[index] = GtsAddSchemaResult(ok=True, type_id=type_id) + finally: + # Discard anything still staged (e.g. a validation call raised) so no + # unvalidated entry lingers in the staging overlay. + for token in pending: + self.store.discard(token) final = [r for r in results if r is not None] return GtsAddSchemasResult(ok=all(r.ok for r in final), results=final) @@ -656,12 +699,18 @@ def add_schema( if isinstance(prepared, GtsAddSchemaResult): return prepared type_id, entity, store_key = prepared - self.store.stage(entity) + token = self.store.stage(entity) error = self._validate_staged(entity, store_key, validate, gts_ref_validation) if error is not None: - self.store.discard(store_key) + self.store.discard(token) return GtsAddSchemaResult(ok=False, type_id=type_id, error=error) - self.store.commit(store_key) + if self.store.commit(token, self.allow_entity_updates) == "conflict": + return GtsAddSchemaResult( + ok=False, + type_id=type_id, + error=f"Entity '{store_key}' is already registered with different content", + conflict=True, + ) return GtsAddSchemaResult(ok=True, type_id=type_id) def validate_id(self, gts_id: str) -> GtsIdValidationResult: diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index 69275b0..fc44cb8 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -146,7 +146,16 @@ def __init__(self, reader: GtsReader | None = None) -> None: # reads (`get_committed`, `values`, `keys`, `items`, `query`) until # `commit`. This is how a validate=true registration avoids ever # exposing an entity that has not yet passed validation. - self._staged: dict[str, GtsEntity] = {} + # + # Keyed by a unique staging TOKEN (not the registry key) so two entries + # that resolve to the same id - a duplicated batch entry or two + # concurrent batches - never clobber each other, and commit/discard only + # affect the entry they name. ``_staged_by_key`` is the by-key overlay + # ``get`` and the reference registry consult; it holds the most recently + # staged entity per key and is recomputed on commit/discard. + self._staged: dict[str, tuple[str, GtsEntity]] = {} + self._staged_by_key: dict[str, GtsEntity] = {} + self._stage_seq = 0 self._reader = reader self._lock = threading.RLock() self._reference_registry: Registry | None = None @@ -248,7 +257,7 @@ def get(self, entity_id: str) -> GtsEntity | None: """ entity_id = strip_scheme(entity_id) with self._lock: - staged = self._staged.get(entity_id) + staged = self._staged_by_key.get(entity_id) if staged is not None: return copy.deepcopy(staged) return self.get_committed(entity_id) @@ -291,31 +300,63 @@ def _entity_key(entity: GtsEntity) -> str: def stage(self, entity: GtsEntity) -> str: """Place an entity into the staging overlay WITHOUT publishing it, and - return its registry key. A staged entity is visible to internal - validation but invisible to public reads until :meth:`commit`. Raises - ``EntityConflictError`` when a committed entity with different content - already holds the key and updates are not allowed. Callers MUST - eventually :meth:`commit` or :meth:`discard` the staged key.""" + return a unique staging token. A staged entity is visible to internal + validation but invisible to public reads until :meth:`commit`. Callers + MUST eventually :meth:`commit` or :meth:`discard` the returned token.""" stored = copy.deepcopy(entity) key = self._entity_key(stored) with self._lock: - self._staged[key] = stored + self._stage_seq += 1 + token = f"stg-{self._stage_seq}" + self._staged[token] = (key, stored) + self._staged_by_key[key] = stored self._invalidate_reference_registry() - return key - - def commit(self, entity_id: str) -> None: - """Publish a previously staged entity, making it visible to public reads.""" + return token + + def _remove_staged_locked(self, token: str, key: str) -> None: + """Drop the staged entry named by ``token`` and rebuild the by-key + overlay for ``key`` from any other tokens that still target it. The + caller must hold ``self._lock``.""" + self._staged.pop(token, None) + self._staged_by_key.pop(key, None) + for other_key, other_entity in self._staged.values(): + if other_key == key: + self._staged_by_key[key] = other_entity + + def commit(self, token: str, allow_updates: bool = False) -> str: + """Publish the entity named by ``token``, making it visible to public + reads. The publish is atomic with a conflict check against the committed + store: returns ``"added"`` when it was inserted, ``"unchanged"`` when an + identical entity already held the key, and ``"conflict"`` when a + different entity already held it and ``allow_updates`` is false (nothing + is published in that case). An unknown token also yields ``"conflict"``.""" with self._lock: - staged = self._staged.pop(entity_id, None) - if staged is not None: - self._by_id[entity_id] = staged + entry = self._staged.get(token) + if entry is None: + return "conflict" + key, staged = entry + self._remove_staged_locked(token, key) + existing = self._by_id.get(key) + if ( + existing is not None + and not allow_updates + and existing.content != staged.content + ): self._invalidate_reference_registry() + return "conflict" + outcome = "unchanged" if existing is not None else "added" + self._by_id[key] = staged + self._invalidate_reference_registry() + return outcome - def discard(self, entity_id: str) -> None: - """Drop a staged entity that failed validation. The committed state is - untouched, so a client never observes the discarded (invalid) entity.""" + def discard(self, token: str) -> None: + """Drop the staged entity named by ``token`` that failed validation. The + committed state is untouched, so a client never observes the discarded + (invalid) entity.""" with self._lock: - if self._staged.pop(entity_id, None) is not None: + entry = self._staged.get(token) + if entry is not None: + self._remove_staged_locked(token, entry[0]) self._invalidate_reference_registry() def get_schema_content(self, type_id: str) -> dict[str, Any]: @@ -333,7 +374,7 @@ def _create_reference_registry(self) -> Registry: # Committed entities plus the staging overlay (staged overrides a # committed entry of the same id), so a schema being validated as # part of a batch resolves `$ref`s to its not-yet-committed siblings. - merged = {**self._by_id, **self._staged} + merged = {**self._by_id, **self._staged_by_key} for entity_id, entity in merged.items(): if entity.is_schema and isinstance(entity.content, dict): resource = Resource.from_contents( diff --git a/tests/test_staging_commit_integrity.py b/tests/test_staging_commit_integrity.py new file mode 100644 index 0000000..d035660 --- /dev/null +++ b/tests/test_staging_commit_integrity.py @@ -0,0 +1,112 @@ +"""Integrity tests for the validate=true staging/commit flow. + +These cover the guarantees a validate=true batch must uphold beyond "a staged +entity is never publicly visible" (see ``test_staging_concurrency.py``): + +* A staged entry is identified by a unique token, so a duplicated id (or two + concurrent batches) never clobber each other and a commit never silently + overwrites conflicting committed content. +* A survivor is never published when a sibling it depends on is discarded in the + same batch (iterative discard-then-revalidate). +""" + +from gts.ops import GtsOps, GtsRefValidationMode + +_D7 = "http://json-schema.org/draft-07/schema#" + + +def _schema(type_id: str, title: str) -> dict: + return { + "$schema": _D7, + "$id": f"gts://{type_id}", + "type": "object", + "title": title, + } + + +def test_staging_same_id_twice_yields_distinct_tokens_and_commit_does_not_overwrite(): + ops = GtsOps(path=None) + tid = "gts.x.pyunit._.dup.v1~" + prep_a = ops._prepare_type_schema(_schema(tid, "a")) + prep_b = ops._prepare_type_schema(_schema(tid, "b")) + _, entity_a, _ = prep_a + _, entity_b, _ = prep_b + + token_a = ops.store.stage(entity_a) + token_b = ops.store.stage(entity_b) + assert token_a != token_b + + # First commit publishes A; committing B (different content) must conflict, + # not clobber A. + assert ops.store.commit(token_a) == "added" + assert ops.store.commit(token_b) == "conflict" + committed = ops.store.get_committed(tid) + assert committed is not None + assert committed.content["title"] == "a" + + +def test_committing_identical_staged_content_twice_is_unchanged(): + ops = GtsOps(path=None) + tid = "gts.x.pyunit._.same.v1~" + _, entity_a, _ = ops._prepare_type_schema(_schema(tid, "x")) + _, entity_b, _ = ops._prepare_type_schema(_schema(tid, "x")) + token_a = ops.store.stage(entity_a) + token_b = ops.store.stage(entity_b) + assert ops.store.commit(token_a) == "added" + assert ops.store.commit(token_b) == "unchanged" + + +def test_discarding_one_token_leaves_another_staged_entry_for_the_same_id(): + ops = GtsOps(path=None) + tid = "gts.x.pyunit._.iso.v1~" + _, entity_a, _ = ops._prepare_type_schema(_schema(tid, "keep")) + _, entity_b, _ = ops._prepare_type_schema(_schema(tid, "drop")) + token_a = ops.store.stage(entity_a) + token_b = ops.store.stage(entity_b) + ops.store.discard(token_b) + assert ops.store.commit(token_a) == "added" + assert ops.store.get_committed(tid).content["title"] == "keep" + + +def test_batch_does_not_commit_dependent_of_discarded_sibling(): + """Under any-present ref validation, B x-gts-refs A while A x-gts-refs a type + that is never registered. Validated against the fully staged set, B passes + (A is present) while A fails (its target is missing) - a single-pass + implementation would then commit B with a dangling reference to the discarded + A. The iterative discard-then-revalidate must reject B too, so neither is + retrievable afterwards.""" + ops = GtsOps(path=None) + a = { + "$schema": _D7, + "$id": "gts://gts.x.pydep._.a.v1~", + "type": "object", + "properties": { + "r": {"type": "string", "x-gts-ref": "gts.x.pydep._.missing.v1~"} + }, + } + b = { + "$schema": _D7, + "$id": "gts://gts.x.pydep._.b.v1~", + "type": "object", + "properties": {"x": {"type": "string", "x-gts-ref": "gts.x.pydep._.a.v1~"}}, + } + res = ops.add_schemas( + [a, b], validate=True, gts_ref_validation=GtsRefValidationMode.ANY_PRESENT + ) + assert res.ok is False + assert all(not r.ok for r in res.results) + assert ops.store.get_committed("gts.x.pydep._.a.v1~") is None + assert ops.store.get_committed("gts.x.pydep._.b.v1~") is None + + +def test_batch_with_conflicting_duplicate_id_reports_conflict(): + """A batch that carries the same $id twice with different content must not + silently keep only the last entry: exactly one commits and the conflicting + duplicate is reported as not-ok.""" + ops = GtsOps(path=None) + tid = "gts.x.pydup._.t.v1~" + res = ops.add_schemas([_schema(tid, "a"), _schema(tid, "b")], validate=True) + assert res.ok is False + oks = [r.ok for r in res.results] + assert oks.count(True) == 1 + assert oks.count(False) == 1 From 9ca6d3898637bf7b3e1792fe60363c6b146bcee8 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Mon, 28 Sep 2026 16:52:32 +0300 Subject: [PATCH 21/27] fix(validation): bound patternProperties key matching to prevent ReDoS The custom validator only overrode the `pattern` keyword with a timeout-bounded matcher; `patternProperties` fell through to jsonschema's default applicator, which matches property names with `re.search` and no timeout. An untrusted schema pattern matched against an adversarially long property name could therefore still cause catastrophic backtracking (ReDoS, CWE-1333). Override `patternProperties` too, matching property names with the `regex` module under PATTERN_TIMEOUT_SECONDS and yielding "regular expression match timed out" on timeout, exactly as the `pattern` keyword already does. Adds a regression test asserting a catastrophic patternProperties key is bounded. Signed-off-by: Artifizer --- gts/src/gts/schema_validation.py | 32 +++++++++++++++++++++++++++++++- tests/test_store_extra.py | 12 ++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/gts/src/gts/schema_validation.py b/gts/src/gts/schema_validation.py index 9496e8a..798c405 100644 --- a/gts/src/gts/schema_validation.py +++ b/gts/src/gts/schema_validation.py @@ -141,7 +141,37 @@ def _validate_pattern( yield ValidationError(f"invalid regular expression: {error}") +def _validate_pattern_properties( + validator: Any, pattern_properties: Any, instance: Any, schema: Any +) -> Iterator[ValidationError]: + # Mirrors jsonschema's default ``patternProperties`` applicator, but matches + # property names with the ``regex`` module under ``PATTERN_TIMEOUT_SECONDS`` + # so an untrusted schema pattern cannot cause catastrophic backtracking + # (ReDoS, CWE-1333) against an adversarially long property name. The default + # implementation uses ``re.search`` with no timeout. + if not validator.is_type(instance, "object"): + return + for pattern, subschema in pattern_properties.items(): + for name, value in instance.items(): + try: + matched = regex.search(pattern, name, timeout=PATTERN_TIMEOUT_SECONDS) + except TimeoutError: + yield ValidationError("regular expression match timed out") + continue + except regex.error as error: + yield ValidationError(f"invalid regular expression: {error}") + continue + if matched is not None: + yield from validator.descend( + value, subschema, path=name, schema_path=pattern + ) + + def validator_for(schema: Any) -> Any: return validators.extend( - jsonschema_validator_for(schema), {"pattern": _validate_pattern} + jsonschema_validator_for(schema), + { + "pattern": _validate_pattern, + "patternProperties": _validate_pattern_properties, + }, ) diff --git a/tests/test_store_extra.py b/tests/test_store_extra.py index a2b5a13..3d465c9 100644 --- a/tests/test_store_extra.py +++ b/tests/test_store_extra.py @@ -65,6 +65,18 @@ def test_catastrophic_pattern_times_out(self): assert errors[0].message == "regular expression match timed out" assert PATTERN_TIMEOUT_SECONDS == 1.0 + def test_catastrophic_pattern_properties_key_times_out(self): + # patternProperties matches property NAMES; an adversarially long key + # against a catastrophic pattern must be bounded by the same timeout as + # the `pattern` keyword rather than backtracking indefinitely. + schema = {"patternProperties": {"(a+)+$": {"type": "string"}}} + validator = validator_for(schema)(schema) + instance = {"a" * 30_000 + "!": "value"} + errors = list(validator.iter_errors(instance)) + assert any( + error.message == "regular expression match timed out" for error in errors + ) + class TestRegisterEdgeCases: def test_register_raises_without_id(self): From add874a9f0296c85851aeee0980b1e4b70b60f3f Mon Sep 17 00:00:00 2001 From: Artifizer Date: Mon, 28 Sep 2026 22:23:03 +0300 Subject: [PATCH 22/27] Isolate staging per session and publish batches atomically The validate=true registration overlay was global: one request's staged, not-yet-validated entries were visible to another request's validation, so a batch could resolve a parent or $ref target against another batch's unvalidated entry and commit a dangling reference. Separately, the batch commit loop published survivors one-by-one, so a dependent could be published against a target whose committed content changed after the dependent had been validated. Scope the staging overlay to a per-request staging session (committed-only when no session is active) so a batch resolves only its own siblings, and replace the per-entry commit loop with a store-level all-or-nothing batch compare-and-swap that publishes nothing when any target conflicts. Signed-off-by: Artifizer --- gts/src/gts/ops.py | 207 ++++++++++++++----------- gts/src/gts/store.py | 187 +++++++++++++++++----- tests/test_staging_commit_integrity.py | 64 +++++++- 3 files changed, 319 insertions(+), 139 deletions(-) diff --git a/gts/src/gts/ops.py b/gts/src/gts/ops.py index 0340151..4ad68b5 100644 --- a/gts/src/gts/ops.py +++ b/gts/src/gts/ops.py @@ -462,28 +462,32 @@ def add_entity( # observes an entity that has not passed validation, and the committed # state (any prior version under this id) is never touched. No lock is # held across validation, so concurrent reads are not blocked. - token = self.store.stage(entity) - try: - if entity.is_schema: - self.store.validate_schema_basic(store_key) - if validate: - self.store.validate_schema(store_key, gts_ref_validation) - elif validate: - self.store.validate_instance(store_key, gts_ref_validation) - except Exception as e: # noqa: BLE001 - converted to a result object at API boundary - self.store.discard(token) - return GtsAddEntityResult( - ok=False, - error=f"Validation failed: {e!s}", - is_type_schema=entity.is_schema, - ) - if self.store.commit(token, self.allow_entity_updates) == "conflict": - return GtsAddEntityResult( - ok=False, - error=f"Entity '{store_key}' is already registered with different content", - is_type_schema=entity.is_schema, - conflict=True, - ) + # A staging session isolates this registration's own staged entity so its + # validation resolves itself but never another concurrent request's + # not-yet-validated staged entries. + with self.store.staging_session(): + token = self.store.stage(entity) + try: + if entity.is_schema: + self.store.validate_schema_basic(store_key) + if validate: + self.store.validate_schema(store_key, gts_ref_validation) + elif validate: + self.store.validate_instance(store_key, gts_ref_validation) + except Exception as e: # noqa: BLE001 - converted to a result object at API boundary + self.store.discard(token) + return GtsAddEntityResult( + ok=False, + error=f"Validation failed: {e!s}", + is_type_schema=entity.is_schema, + ) + if self.store.commit(token, self.allow_entity_updates) == "conflict": + return GtsAddEntityResult( + ok=False, + error=f"Entity '{store_key}' is already registered with different content", + is_type_schema=entity.is_schema, + conflict=True, + ) # Return gts_id if available, otherwise raw_id entity_id = entity.gts_id.id if entity.gts_id else (entity.raw_id or "") @@ -617,66 +621,81 @@ def add_schemas( # the leftovers (in the finally block) instead of leaking unvalidated # entries into the staging overlay. pending: set[str] = set() - try: - # Phase 1: stage every structurally-valid entry. - # survivors: (index, type_id, entity, store_key, token) - survivors: list[tuple[int, str, GtsEntity, str, str]] = [] - for index, schema in enumerate(schemas): - prepared = self._prepare_type_schema(schema) - if isinstance(prepared, GtsAddSchemaResult): - results[index] = prepared - continue - type_id, entity, store_key = prepared - token = self.store.stage(entity) - pending.add(token) - survivors.append((index, type_id, entity, store_key, token)) - - # Phase 2: validate the staged entries, discarding failures and - # RE-validating the survivors against the now-smaller staged set until - # a round produces no new failures. This stops an entry that only - # validated because a sibling was staged (e.g. its parent or $ref - # target) from being committed after that sibling was discarded. - while True: - still_good: list[tuple[int, str, GtsEntity, str, str]] = [] - failed: list[tuple[int, str, str, str]] = [] - for index, type_id, entity, store_key, token in survivors: - error = self._validate_staged( - entity, store_key, True, gts_ref_validation - ) - if error is None: - still_good.append((index, type_id, entity, store_key, token)) + # One staging session for the whole batch: entries resolve their own + # intra-batch siblings, while another request's staged entries stay + # invisible to this batch's validation. + with self.store.staging_session(): + try: + # Phase 1: stage every structurally-valid entry. + # survivors: (index, type_id, entity, store_key, token) + survivors: list[tuple[int, str, GtsEntity, str, str]] = [] + for index, schema in enumerate(schemas): + prepared = self._prepare_type_schema(schema) + if isinstance(prepared, GtsAddSchemaResult): + results[index] = prepared + continue + type_id, entity, store_key = prepared + token = self.store.stage(entity) + pending.add(token) + survivors.append((index, type_id, entity, store_key, token)) + + # Phase 2: validate the staged entries, discarding failures and + # RE-validating the survivors against the now-smaller staged set + # until a round produces no new failures. This stops an entry that + # only validated because a sibling was staged (e.g. its parent or + # $ref target) from being committed after that sibling was discarded. + while True: + still_good: list[tuple[int, str, GtsEntity, str, str]] = [] + failed: list[tuple[int, str, str, str]] = [] + for index, type_id, entity, store_key, token in survivors: + error = self._validate_staged( + entity, store_key, True, gts_ref_validation + ) + if error is None: + still_good.append( + (index, type_id, entity, store_key, token) + ) + else: + failed.append((index, type_id, token, error)) + if not failed: + break + for index, type_id, token, error in failed: + self.store.discard(token) + pending.discard(token) + results[index] = GtsAddSchemaResult( + ok=False, type_id=type_id, error=error + ) + survivors = still_good + + # Phase 3: publish the survivors AS ONE ATOMIC UNIT. A per-entry + # commit loop could publish a dependent schema after a concurrent + # batch won its parent or $ref target id with different content + # than the dependent was validated against; the store-level batch + # compare-and-swap publishes none if any target conflicts. + survivor_tokens = [token for _i, _t, _e, _sk, token in survivors] + outcomes = self.store.commit_batch( + survivor_tokens, self.allow_entity_updates + ) + for (index, type_id, _entity, store_key, token), outcome in zip( + survivors, outcomes + ): + if outcome == "conflict": + # Nothing was published for a conflicting entry; leave its + # token in `pending` so the finally discards it. + results[index] = GtsAddSchemaResult( + ok=False, + type_id=type_id, + error=f"Entity '{store_key}' is already registered with different content", + conflict=True, + ) else: - failed.append((index, type_id, token, error)) - if not failed: - break - for index, type_id, token, error in failed: + pending.discard(token) + results[index] = GtsAddSchemaResult(ok=True, type_id=type_id) + finally: + # Discard anything still staged (e.g. a validation call raised) so + # no unvalidated entry lingers in the staging overlay. + for token in pending: self.store.discard(token) - pending.discard(token) - results[index] = GtsAddSchemaResult( - ok=False, type_id=type_id, error=error - ) - survivors = still_good - - # Phase 3: publish the entries that passed. A commit can still report - # a conflict if the same id was already committed with different - # content in the meantime. - for index, type_id, _entity, store_key, token in survivors: - outcome = self.store.commit(token, self.allow_entity_updates) - pending.discard(token) - if outcome == "conflict": - results[index] = GtsAddSchemaResult( - ok=False, - type_id=type_id, - error=f"Entity '{store_key}' is already registered with different content", - conflict=True, - ) - else: - results[index] = GtsAddSchemaResult(ok=True, type_id=type_id) - finally: - # Discard anything still staged (e.g. a validation call raised) so no - # unvalidated entry lingers in the staging overlay. - for token in pending: - self.store.discard(token) final = [r for r in results if r is not None] return GtsAddSchemasResult(ok=all(r.ok for r in final), results=final) @@ -699,18 +718,24 @@ def add_schema( if isinstance(prepared, GtsAddSchemaResult): return prepared type_id, entity, store_key = prepared - token = self.store.stage(entity) - error = self._validate_staged(entity, store_key, validate, gts_ref_validation) - if error is not None: - self.store.discard(token) - return GtsAddSchemaResult(ok=False, type_id=type_id, error=error) - if self.store.commit(token, self.allow_entity_updates) == "conflict": - return GtsAddSchemaResult( - ok=False, - type_id=type_id, - error=f"Entity '{store_key}' is already registered with different content", - conflict=True, + # A staging session isolates this schema's own staged entity so its + # validation resolves itself but never another concurrent request's + # not-yet-validated staged entries. + with self.store.staging_session(): + token = self.store.stage(entity) + error = self._validate_staged( + entity, store_key, validate, gts_ref_validation ) + if error is not None: + self.store.discard(token) + return GtsAddSchemaResult(ok=False, type_id=type_id, error=error) + if self.store.commit(token, self.allow_entity_updates) == "conflict": + return GtsAddSchemaResult( + ok=False, + type_id=type_id, + error=f"Entity '{store_key}' is already registered with different content", + conflict=True, + ) return GtsAddSchemaResult(ok=True, type_id=type_id) def validate_id(self, gts_id: str) -> GtsIdValidationResult: diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index fc44cb8..910c149 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextvars import copy import logging import threading @@ -39,6 +40,14 @@ logger = logging.getLogger(__name__) +# The staging session active on the current thread/task. A staged entry is +# overlaid onto internal reads (``get``, the reference registry) only while its +# OWN session is active, so one request's not-yet-validated staged entries never +# leak into another request's validation. ``None`` means committed-only. +_active_staging_session: contextvars.ContextVar[str | None] = contextvars.ContextVar( + "gts_active_staging_session", default=None +) + def _require_schema_id(value: str) -> GtsID: try: @@ -150,11 +159,13 @@ def __init__(self, reader: GtsReader | None = None) -> None: # Keyed by a unique staging TOKEN (not the registry key) so two entries # that resolve to the same id - a duplicated batch entry or two # concurrent batches - never clobber each other, and commit/discard only - # affect the entry they name. ``_staged_by_key`` is the by-key overlay - # ``get`` and the reference registry consult; it holds the most recently - # staged entity per key and is recomputed on commit/discard. - self._staged: dict[str, tuple[str, GtsEntity]] = {} - self._staged_by_key: dict[str, GtsEntity] = {} + # affect the entry they name. Each entry also records the staging SESSION + # it belongs to. ``_staged_by_key`` is the by-key overlay ``get`` and the + # reference registry consult, but scoped per session: it maps + # session -> key -> most-recently-staged entity, so a snapshot only ever + # overlays its own session's staged entries and never another request's. + self._staged: dict[str, tuple[str, GtsEntity, str]] = {} + self._staged_by_key: dict[str, dict[str, GtsEntity]] = {} self._stage_seq = 0 self._reader = reader self._lock = threading.RLock() @@ -256,10 +267,12 @@ def get(self, entity_id: str) -> GtsEntity | None: scheme themselves. """ entity_id = strip_scheme(entity_id) - with self._lock: - staged = self._staged_by_key.get(entity_id) - if staged is not None: - return copy.deepcopy(staged) + session = _active_staging_session.get() + if session is not None: + with self._lock: + staged = self._staged_by_key.get(session, {}).get(entity_id) + if staged is not None: + return copy.deepcopy(staged) return self.get_committed(entity_id) def get_committed(self, entity_id: str) -> GtsEntity | None: @@ -298,30 +311,53 @@ def _entity_key(entity: GtsEntity) -> str: return entity.raw_id raise ValueError("Entity must have a valid gts_id or raw_id") + @contextmanager + def staging_session(self, session_id: str | None = None): + """Bind a staging SESSION for the duration of the block so every + :meth:`stage` inside it, and the internal reads (:meth:`get`, the + reference registry) that a validation performs, share one isolated + overlay. Entries staged under this session are visible only while this + session is active, so one request never observes another's unvalidated + staged entries. Returns the session id.""" + session_id = session_id or f"ses-{uuid.uuid4().hex}" + token = _active_staging_session.set(session_id) + try: + yield session_id + finally: + _active_staging_session.reset(token) + def stage(self, entity: GtsEntity) -> str: """Place an entity into the staging overlay WITHOUT publishing it, and return a unique staging token. A staged entity is visible to internal - validation but invisible to public reads until :meth:`commit`. Callers - MUST eventually :meth:`commit` or :meth:`discard` the returned token.""" + validation (only within its own :meth:`staging_session`) but invisible to + public reads until :meth:`commit`. Callers MUST eventually :meth:`commit` + or :meth:`discard` the returned token.""" stored = copy.deepcopy(entity) key = self._entity_key(stored) + # No active session => a private per-entry session, so the entry is + # visible to no snapshot. with self._lock: self._stage_seq += 1 token = f"stg-{self._stage_seq}" - self._staged[token] = (key, stored) - self._staged_by_key[key] = stored - self._invalidate_reference_registry() + session = _active_staging_session.get() or token + self._staged[token] = (key, stored, session) + self._staged_by_key.setdefault(session, {})[key] = stored return token - def _remove_staged_locked(self, token: str, key: str) -> None: + def _remove_staged_locked(self, token: str, key: str, session: str) -> None: """Drop the staged entry named by ``token`` and rebuild the by-key - overlay for ``key`` from any other tokens that still target it. The - caller must hold ``self._lock``.""" + overlay for ``key`` within ``session`` from any other tokens that still + target it. The caller must hold ``self._lock``.""" self._staged.pop(token, None) - self._staged_by_key.pop(key, None) - for other_key, other_entity in self._staged.values(): - if other_key == key: - self._staged_by_key[key] = other_entity + session_overlay = self._staged_by_key.get(session) + if session_overlay is None: + return + session_overlay.pop(key, None) + for other_key, other_entity, other_session in self._staged.values(): + if other_session == session and other_key == key: + session_overlay[key] = other_entity + if not session_overlay: + self._staged_by_key.pop(session, None) def commit(self, token: str, allow_updates: bool = False) -> str: """Publish the entity named by ``token``, making it visible to public @@ -334,21 +370,73 @@ def commit(self, token: str, allow_updates: bool = False) -> str: entry = self._staged.get(token) if entry is None: return "conflict" - key, staged = entry - self._remove_staged_locked(token, key) + key, staged, session = entry + self._remove_staged_locked(token, key, session) existing = self._by_id.get(key) if ( existing is not None and not allow_updates and existing.content != staged.content ): - self._invalidate_reference_registry() return "conflict" outcome = "unchanged" if existing is not None else "added" self._by_id[key] = staged self._invalidate_reference_registry() return outcome + def commit_batch( + self, tokens: list[str], allow_updates: bool = False + ) -> list[str]: + """Atomically publish a whole set of staged ``tokens`` as one + all-or-nothing unit. Every token's target is checked against the + committed store (and against its batch siblings) under a single lock; if + ANY would conflict - a concurrent commit won the id with different + content, an intra-batch duplicate disagrees, or a token is unknown - + NOTHING is published. This closes the gap where a per-entry commit loop + could publish a dependent schema against a target whose content changed + after the dependent was validated. Returns one outcome per token, + positionally aligned with ``tokens``; on a batch conflict the whole set + reports ``"conflict"`` and the tokens remain staged for the caller to + discard.""" + with self._lock: + outcomes: list[str] = [] + entries: list[tuple[str, GtsEntity, str] | None] = [] + pending_by_key: dict[str, GtsEntity] = {} + any_conflict = False + for token in tokens: + entry = self._staged.get(token) + if entry is None: + entries.append(None) + outcomes.append("conflict") + any_conflict = True + continue + key, staged, _session = entry + entries.append(entry) + existing = pending_by_key.get(key, self._by_id.get(key)) + if existing is not None: + identical = existing.content == staged.content + outcome = "unchanged" if identical or allow_updates else "conflict" + if outcome == "conflict": + any_conflict = True + outcomes.append(outcome) + else: + outcomes.append("added") + pending_by_key[key] = staged + + # All-or-nothing: if any target conflicts, publish none and report + # every entry as a conflict. The tokens stay staged for the caller. + if any_conflict: + return ["conflict"] * len(tokens) + + for token, entry, outcome in zip(tokens, entries, outcomes): + assert entry is not None + key, staged, session = entry + self._remove_staged_locked(token, key, session) + if outcome == "added": + self._by_id[key] = staged + self._invalidate_reference_registry() + return outcomes + def discard(self, token: str) -> None: """Drop the staged entity named by ``token`` that failed validation. The committed state is untouched, so a client never observes the discarded @@ -356,8 +444,7 @@ def discard(self, token: str) -> None: with self._lock: entry = self._staged.get(token) if entry is not None: - self._remove_staged_locked(token, entry[0]) - self._invalidate_reference_registry() + self._remove_staged_locked(token, entry[0], entry[2]) def get_schema_content(self, type_id: str) -> dict[str, Any]: """Get schema content as dict (legacy method for backward compatibility).""" @@ -366,24 +453,40 @@ def get_schema_content(self, type_id: str) -> dict[str, Any]: return entity.content raise KeyError(f"Schema not found: {type_id}") + @staticmethod + def _registry_with_schemas( + registry: Registry, entities: dict[str, GtsEntity] + ) -> Registry: + for entity_id, entity in entities.items(): + if entity.is_schema and isinstance(entity.content, dict): + resource = Resource.from_contents( + _without_x_gts_ref(entity.content), + default_specification=DRAFT202012, + ) + registry = registry.with_resource(with_scheme(entity_id), resource) + return registry + def _create_reference_registry(self) -> Registry: with self._lock: - if self._reference_registry is not None: - return self._reference_registry - registry = Registry() - # Committed entities plus the staging overlay (staged overrides a - # committed entry of the same id), so a schema being validated as - # part of a batch resolves `$ref`s to its not-yet-committed siblings. - merged = {**self._by_id, **self._staged_by_key} - for entity_id, entity in merged.items(): - if entity.is_schema and isinstance(entity.content, dict): - resource = Resource.from_contents( - _without_x_gts_ref(entity.content), - default_specification=DRAFT202012, - ) - registry = registry.with_resource(with_scheme(entity_id), resource) - self._reference_registry = registry - return registry + # The committed-only registry is cached and only invalidated when the + # committed set changes (never on stage/discard), so staging never + # corrupts another request's view. + if self._reference_registry is None: + self._reference_registry = self._registry_with_schemas( + Registry(), self._by_id + ) + committed = self._reference_registry + # Overlay ONLY the active session's staged schemas (staged overrides a + # committed entry of the same id), so a schema being validated as part + # of a batch resolves `$ref`s to its own not-yet-committed siblings - + # and never to another request's staged entries. + session = _active_staging_session.get() + if session is None: + return committed + session_overlay = self._staged_by_key.get(session) + if not session_overlay: + return committed + return self._registry_with_schemas(committed, session_overlay) def items(self): """Return all entity ID and entity pairs.""" diff --git a/tests/test_staging_commit_integrity.py b/tests/test_staging_commit_integrity.py index d035660..f22b5dd 100644 --- a/tests/test_staging_commit_integrity.py +++ b/tests/test_staging_commit_integrity.py @@ -10,6 +10,7 @@ same batch (iterative discard-then-revalidate). """ +from gts.entities import GtsEntity from gts.ops import GtsOps, GtsRefValidationMode _D7 = "http://json-schema.org/draft-07/schema#" @@ -100,13 +101,64 @@ def test_batch_does_not_commit_dependent_of_discarded_sibling(): def test_batch_with_conflicting_duplicate_id_reports_conflict(): - """A batch that carries the same $id twice with different content must not - silently keep only the last entry: exactly one commits and the conflicting - duplicate is reported as not-ok.""" + """A batch that carries the same $id twice with different content is + internally inconsistent, so the atomic publish must keep NEITHER entry (no + silent last-wins): the batch is rejected as a whole and nothing is committed.""" ops = GtsOps(path=None) tid = "gts.x.pydup._.t.v1~" res = ops.add_schemas([_schema(tid, "a"), _schema(tid, "b")], validate=True) assert res.ok is False - oks = [r.ok for r in res.results] - assert oks.count(True) == 1 - assert oks.count(False) == 1 + assert all(not r.ok for r in res.results) + # All-or-nothing: nothing from the inconsistent batch is published. + assert ops.store.get_committed(tid) is None + + +def test_snapshot_overlay_is_scoped_to_its_own_staging_session(): + """Regression for the cross-request staging leak: a schema staged under one + session must be invisible to reads made under a different session (or none), + so one batch never resolves a $ref/parent against another batch's + not-yet-validated entry.""" + ops = GtsOps(path=None) + tid = "gts.x.pysess._.a.v1~" + _, entity, _ = ops._prepare_type_schema(_schema(tid, "a")) + + with ops.store.staging_session() as session_a: + ops.store.stage(entity) + # Visible to its own session. + assert ops.store.get(tid) is not None + # Invisible to a different session and to committed-only reads. + with ops.store.staging_session(): + assert ops.store.get(tid) is None + assert ops.store.get_committed(tid) is None + # Outside any session (committed-only), still invisible. + assert ops.store.get(tid) is None + assert session_a # session id was returned + + +def test_commit_batch_publishes_nothing_when_a_target_conflicts_with_committed(): + """Regression for the non-atomic commit loop: if a survivor's target was + committed with different content by a concurrent writer, the whole batch + publishes nothing rather than committing the dependent against changed + content.""" + ops = GtsOps(path=None) + parent = "gts.x.pyatomic._.parent.v1~" + child = "gts.x.pyatomic._.child.v1~" + + # A concurrent writer already committed the parent id with different content. + committed = ops.add_schema(_schema(parent, "committed")) + assert committed.ok + + # Build entities directly to simulate a batch whose parent target was won by + # a concurrent writer AFTER staging (the ops pre-check would otherwise reject + # staging a conflicting id up front). + with ops.store.staging_session(): + p_entity = GtsEntity(content=_schema(parent, "staged"), cfg=ops.cfg) + c_entity = GtsEntity(content=_schema(child, "ok"), cfg=ops.cfg) + c_token = ops.store.stage(c_entity) + p_token = ops.store.stage(p_entity) + outcomes = ops.store.commit_batch([c_token, p_token]) + assert outcomes == ["conflict", "conflict"] + + # Nothing from the batch was published; the parent keeps its committed content. + assert ops.store.get_committed(child) is None + assert ops.store.get_committed(parent).content["title"] == "committed" From a7ccbf24bd425d5105557611c2dcf1ff87bbe4c9 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Mon, 28 Sep 2026 23:12:30 +0300 Subject: [PATCH 23/27] chore: update to gts-spec v0.14.4 Signed-off-by: Artifizer --- .gts-spec | 2 +- .gts-spec-version | 2 +- README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gts-spec b/.gts-spec index 98a16f3..386870a 160000 --- a/.gts-spec +++ b/.gts-spec @@ -1 +1 @@ -Subproject commit 98a16f3f9815dd4b852484ae7e52713d797d4dbb +Subproject commit 386870a2f4e7b79c1af019b9d13fc30e06e2efef diff --git a/.gts-spec-version b/.gts-spec-version index 818cabd..6cf9fa5 100644 --- a/.gts-spec-version +++ b/.gts-spec-version @@ -1 +1 @@ -v0.14.3 +v0.14.4 diff --git a/README.md b/README.md index c79e83e..61b148f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A minimal, idiomatic Python library for working with **GTS** ([Global Type System](https://github.com/gts-spec/gts-spec)) identifiers and JSON/JSON Schema artifacts. -Current supported GTS spec version: `0.14.3` +Current supported GTS spec version: `0.14.4` ## Roadmap From d68e70ab9ae89280689ca10912918d4ad0522924 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Tue, 29 Sep 2026 01:59:06 +0300 Subject: [PATCH 24/27] fix: fix formatting Signed-off-by: Artifizer --- gts/src/gts/store.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index 910c149..105db29 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -384,9 +384,7 @@ def commit(self, token: str, allow_updates: bool = False) -> str: self._invalidate_reference_registry() return outcome - def commit_batch( - self, tokens: list[str], allow_updates: bool = False - ) -> list[str]: + def commit_batch(self, tokens: list[str], allow_updates: bool = False) -> list[str]: """Atomically publish a whole set of staged ``tokens`` as one all-or-nothing unit. Every token's target is checked against the committed store (and against its batch siblings) under a single lock; if From 0a6faeb5de2ec1236bf9b4b240d1d81a2164181e Mon Sep 17 00:00:00 2001 From: Artifizer Date: Tue, 29 Sep 2026 16:48:36 +0300 Subject: [PATCH 25/27] fix(validation): make regex dialect handling explicit Use bounded regex callbacks for pattern evaluation, centralize supported dialects, require inherited dialect context for fragments, and preserve that context through references, traits, casts, and x-gts-ref validation. Signed-off-by: Artifizer --- gts/src/gts/compatibility.py | 13 ++++-- gts/src/gts/schema_cast.py | 16 ++++++-- gts/src/gts/schema_dialect.py | 22 ++++++++-- gts/src/gts/schema_resolver.py | 11 +++-- gts/src/gts/schema_validation.py | 48 ++++++++++++++++++---- gts/src/gts/store.py | 29 +++++++++---- gts/src/gts/traits.py | 5 +-- gts/src/gts/x_gts_ref.py | 20 +++++---- tests/test_regressions.py | 10 ++--- tests/test_schema_cast.py | 3 ++ tests/test_store_extra.py | 40 +++++++++++++----- tests/test_traits.py | 38 +++++++++-------- tests/test_x_gts_ref.py | 70 ++++++++++++++++---------------- 13 files changed, 216 insertions(+), 109 deletions(-) diff --git a/gts/src/gts/compatibility.py b/gts/src/gts/compatibility.py index 5feb5b4..edf7926 100644 --- a/gts/src/gts/compatibility.py +++ b/gts/src/gts/compatibility.py @@ -16,9 +16,10 @@ from typing import Any -from jsonschema.validators import validator_for from jsonsubschema import isSubschema +from .schema_validation import validator_for + COMPATIBLE = "compatible" INCOMPATIBLE = "incompatible" UNKNOWN = "unknown" @@ -85,7 +86,9 @@ def _value_constraint_makes_type_redundant(schema: dict[Any, Any]) -> bool: if values is None: return False try: - validator = validator_for({"type": schema["type"]})({"type": schema["type"]}) + validator = validator_for( + {"type": schema["type"]}, inherited_dialect="draft-07" + )({"type": schema["type"]}) return all(validator.is_valid(value) for value in values) except Exception: # noqa: BLE001 - intentional broad fallback return False @@ -96,8 +99,10 @@ def _finite_subset(subset: Any, superset: Any) -> bool | None: if values is None: return None try: - subset_validator = validator_for(subset)(subset) - superset_validator = validator_for(superset)(superset) + subset_validator = validator_for(subset, inherited_dialect="draft-07")(subset) + superset_validator = validator_for(superset, inherited_dialect="draft-07")( + superset + ) return all( not subset_validator.is_valid(value) or superset_validator.is_valid(value) for value in values diff --git a/gts/src/gts/schema_cast.py b/gts/src/gts/schema_cast.py index 0d70bff..c123cb4 100644 --- a/gts/src/gts/schema_cast.py +++ b/gts/src/gts/schema_cast.py @@ -10,6 +10,7 @@ from .compatibility import UNKNOWN, dialects_differ from .gts import GtsID +from .schema_dialect import effective_dialect from .schema_validation import validator_for logger = logging.getLogger(__name__) @@ -88,7 +89,9 @@ def cast( from_schema_content: dict, to_schema_content: dict, resolver: Any | None = None, + inherited_dialect: str | None = None, ) -> GtsEntityCastResult: + target_dialect = effective_dialect(to_schema_content, inherited_dialect) # Flatten target schema to merge allOf and get all properties including const values target_schema = cls._flatten_schema(to_schema_content) @@ -154,9 +157,13 @@ def cast( # Allow GTS ID changes in const values try: if resolver is not None: - cls._validate_with_gts_id_tolerance(casted, to_schema_content, resolver) + cls._validate_with_gts_id_tolerance( + casted, to_schema_content, target_dialect, resolver + ) else: - cls._validate_with_gts_id_tolerance(casted, to_schema_content, None) + cls._validate_with_gts_id_tolerance( + casted, to_schema_content, target_dialect, None + ) is_fully_compatible = is_backward and is_forward except js_exceptions.ValidationError as ve: reasons.append(ve.message) @@ -382,13 +389,16 @@ def _cast_instance_to_schema( def _validate_with_gts_id_tolerance( instance: dict[str, Any], schema: dict[str, Any], + inherited_dialect: str, resolver: Any | None = None, ) -> None: """Validate instance against schema, but allow const values to differ if both are GTS IDs.""" # Create a modified schema that removes const constraints for GTS IDs modified_schema = GtsEntityCastResult._remove_gts_const_constraints(schema) - validator_class = validator_for(modified_schema) + validator_class = validator_for( + modified_schema, inherited_dialect=inherited_dialect + ) if isinstance(resolver, Registry): validator = validator_class(modified_schema, registry=resolver) elif resolver is not None: diff --git a/gts/src/gts/schema_dialect.py b/gts/src/gts/schema_dialect.py index 70acac2..7be92fd 100644 --- a/gts/src/gts/schema_dialect.py +++ b/gts/src/gts/schema_dialect.py @@ -23,6 +23,21 @@ "2019-09": "https://json-schema.org/draft/2019-09/schema", "2020-12": "https://json-schema.org/draft/2020-12/schema", } +SUPPORTED_DIALECTS = frozenset(_DIALECT_URI) + + +def require_dialect(dialect: str | None) -> str: + if dialect not in SUPPORTED_DIALECTS: + raise ValueError(f"Unsupported JSON Schema dialect: {dialect}") + return dialect + + +def effective_dialect(schema: Any, inherited_dialect: str | None = None) -> str: + if isinstance(schema, dict) and "$schema" in schema: + return document_dialect(schema) + if inherited_dialect is None: + raise ValueError("schema fragments require an explicit inherited dialect") + return require_dialect(inherited_dialect) def document_dialect(schema: dict[str, Any]) -> str: @@ -54,10 +69,9 @@ def supports_ref_siblings(schema: Any) -> bool: Draft 2019-09 and 2020-12 do; Draft-07 ignores ``$ref`` siblings. """ - dialect = schema.get("$schema") if isinstance(schema, dict) else None - return isinstance(dialect, str) and ( - "/draft/2019-09/" in dialect or "/draft/2020-12/" in dialect - ) + if not isinstance(schema, dict): + raise TypeError("schema must be a JSON object with a supported $schema dialect") + return document_dialect(schema) != "draft-07" def check_subschemas(schema: dict[str, Any]) -> None: diff --git a/gts/src/gts/schema_resolver.py b/gts/src/gts/schema_resolver.py index 594e85a..b4cce86 100644 --- a/gts/src/gts/schema_resolver.py +++ b/gts/src/gts/schema_resolver.py @@ -18,7 +18,7 @@ from typing import Any from .gts import GtsRef -from .schema_dialect import supports_ref_siblings +from .schema_dialect import require_dialect, supports_ref_siblings # A ``$ref`` provider maps a bare GTS id to its schema content, raising # ``KeyError`` when the id is unknown. @@ -27,10 +27,15 @@ MAX_SCHEMA_REF_EXPANSIONS = 10_000 -def resolve_schema_refs(schema: Any, provider: SchemaProvider) -> Any: +def resolve_schema_refs(schema: Any, provider: SchemaProvider, dialect: str) -> Any: """Return ``schema`` with external ``$ref`` targets inlined via ``provider``.""" + effective_dialect = require_dialect(dialect) return inline_refs( - copy.deepcopy(schema), set(), supports_ref_siblings(schema), provider, [0] + copy.deepcopy(schema), + set(), + effective_dialect != "draft-07", + provider, + [0], ) diff --git a/gts/src/gts/schema_validation.py b/gts/src/gts/schema_validation.py index 798c405..3bcfbd4 100644 --- a/gts/src/gts/schema_validation.py +++ b/gts/src/gts/schema_validation.py @@ -2,13 +2,22 @@ import copy from collections.abc import Callable, Iterator +from functools import cache from typing import Any import regex -from jsonschema import Draft7Validator, FormatChecker, ValidationError, validators -from jsonschema.validators import validator_for as jsonschema_validator_for +from jsonschema import ( + Draft7Validator, + Draft201909Validator, + Draft202012Validator, + FormatChecker, + ValidationError, + validators, +) -PATTERN_TIMEOUT_SECONDS = 1.0 +from . import schema_dialect + +PATTERN_TIMEOUT_SECONDS = 0.25 _SCHEMA_MAP_KEYWORDS = { "$defs", @@ -121,12 +130,21 @@ def map_schema_nodes(schema: Any, transform: Callable[[Any], Any]) -> Any: # # Starting from the bare checker (which provides "uuid") and overlaying the # draft-07 checkers (which restore correct RFC 3339 "time"/"date-time") yields -# the complete, correct standard-format set. This uses only built-in jsonschema -# checkers -- no custom format functions. +# the complete standard-format set. The regex checker is replaced below with +# the ECMA-compatible engine used by pattern validation. FORMAT_CHECKER = FormatChecker() FORMAT_CHECKER.checkers.update(Draft7Validator.FORMAT_CHECKER.checkers) +def _is_ecma_regex(value: object) -> bool: + if isinstance(value, str): + regex.compile(value) + return True + + +FORMAT_CHECKER.checks("regex", raises=regex.error)(_is_ecma_regex) + + def _validate_pattern( validator: Any, pattern: str, instance: Any, schema: Any ) -> Iterator[ValidationError]: @@ -167,11 +185,27 @@ def _validate_pattern_properties( ) -def validator_for(schema: Any) -> Any: +@cache +def _bounded_pattern_validator(base: Any) -> Any: return validators.extend( - jsonschema_validator_for(schema), + base, { "pattern": _validate_pattern, "patternProperties": _validate_pattern_properties, }, + format_checker=FORMAT_CHECKER, ) + + +def validator_for(schema: Any, *, inherited_dialect: str | None = None) -> Any: + dialect = schema_dialect.effective_dialect(schema, inherited_dialect) + base = { + "draft-07": Draft7Validator, + "2019-09": Draft201909Validator, + "2020-12": Draft202012Validator, + }[dialect] + return _bounded_pattern_validator(base) + + +def check_schema(schema: Any, dialect: Any = None) -> None: + validator_for(dialect or schema).check_schema(schema, format_checker=FORMAT_CHECKER) diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index 105db29..6b1ca18 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -12,7 +12,7 @@ from typing import Any from referencing import Registry, Resource -from referencing.jsonschema import DRAFT202012 +from referencing.jsonschema import DRAFT7, DRAFT201909, DRAFT202012 from . import compatibility, derivation, schema_dialect, schema_resolver, traits from ._errors import ( @@ -35,7 +35,7 @@ # Re-exported for backward compatibility; the limit now lives in schema_resolver. from .schema_resolver import MAX_SCHEMA_REF_EXPANSIONS # noqa: F401 -from .schema_validation import FORMAT_CHECKER, iter_schema_nodes, validator_for +from .schema_validation import FORMAT_CHECKER, check_schema, iter_schema_nodes from .x_gts_ref import XGtsRefValidator, _without_x_gts_ref, extended_validator_for logger = logging.getLogger(__name__) @@ -457,9 +457,15 @@ def _registry_with_schemas( ) -> Registry: for entity_id, entity in entities.items(): if entity.is_schema and isinstance(entity.content, dict): + dialect = schema_dialect.document_dialect(entity.content) + specification = { + "draft-07": DRAFT7, + "2019-09": DRAFT201909, + "2020-12": DRAFT202012, + }[dialect] resource = Resource.from_contents( _without_x_gts_ref(entity.content), - default_specification=DRAFT202012, + default_specification=specification, ) registry = registry.with_resource(with_scheme(entity_id), resource) return registry @@ -843,13 +849,16 @@ def _validate_schema_chain( + "; ".join(errors) ) - def _resolve_schema_refs(self, schema: Any) -> Any: + def _resolve_schema_refs(self, schema: Any, dialect: str | None = None) -> Any: """Resolve external ``$ref`` targets by inlining them from the store. Thin adapter over :func:`schema_resolver.resolve_schema_refs`, passing the store's :meth:`get_schema_content` as the reference provider. """ - return schema_resolver.resolve_schema_refs(schema, self.get_schema_content) + effective_dialect = dialect or self._schema_dialect(schema) + return schema_resolver.resolve_schema_refs( + schema, self.get_schema_content, effective_dialect + ) @staticmethod def _supports_ref_siblings(schema: Any) -> bool: @@ -908,7 +917,9 @@ def _build_effective_traits( # Inline local JSON Pointer refs against the host document, then # resolve any gts:// refs so the composed schema is self-contained. inlined = traits.inline_local_pointers(ts, content) - trait_schemas.append(self._resolve_schema_refs(inlined)) + trait_schemas.append( + self._resolve_schema_refs(inlined, self._schema_dialect(content)) + ) level_traits: dict[str, Any] = {} traits.collect_traits_from_value(content, level_traits) @@ -1023,10 +1034,10 @@ def validate_schema_content( self._validate_schema_chain(schema_id.id, schema_content) try: - validator_class = validator_for( - {"$schema": self._schema_dialect_uri(schema_content)} + check_schema( + schema_content, + {"$schema": self._schema_dialect_uri(schema_content)}, ) - validator_class.check_schema(schema_content) logger.info( f"Schema {schema_id.id} passed JSON Schema meta-schema validation" diff --git a/gts/src/gts/traits.py b/gts/src/gts/traits.py index 2ca9571..a25baac 100644 --- a/gts/src/gts/traits.py +++ b/gts/src/gts/traits.py @@ -23,7 +23,7 @@ from ._json_pointer import resolve as resolve_json_pointer from .gts_ref_validation import GtsRefValidationMode from .schema_validation import FORMAT_CHECKER as _FORMAT_CHECKER -from .schema_validation import map_schema_nodes, validator_for +from .schema_validation import check_schema, map_schema_nodes, validator_for from .x_gts_ref import XGtsRefValidator X_GTS_TRAITS_SCHEMA = "x-gts-traits-schema" @@ -314,8 +314,7 @@ def _validate_trait_schema_integrity( if dialect: schema["$schema"] = dialect try: - cls = validator_for(schema) - cls.check_schema(schema) + check_schema(schema) except Exception as e: # noqa: BLE001 - surfaced as validation error message return [f"{X_GTS_TRAITS_SCHEMA}[{i}] is not a valid JSON Schema: {e}"] else: diff --git a/gts/src/gts/x_gts_ref.py b/gts/src/gts/x_gts_ref.py index 90ff898..9e450f3 100644 --- a/gts/src/gts/x_gts_ref.py +++ b/gts/src/gts/x_gts_ref.py @@ -15,13 +15,14 @@ from typing import Any from jsonschema.exceptions import ValidationError -from jsonschema.validators import extend, validator_for +from jsonschema.validators import extend from ._json_pointer import resolve as resolve_json_pointer from ._naming import GTS_PREFIX, strip_scheme from .gts import GtsID, GtsWildcard from .gts_ref_validation import GtsRefValidationMode -from .schema_validation import iter_schema_nodes, map_schema_nodes +from .schema_dialect import effective_dialect +from .schema_validation import iter_schema_nodes, map_schema_nodes, validator_for X_GTS_REF_SELF = "/$id" @@ -143,9 +144,11 @@ def _is_x_gts_ref_only_combinator(branches: list[Any]) -> bool: return True -def _is_structurally_valid(instance: Any, schema: Any) -> bool: +def _is_structurally_valid(instance: Any, schema: Any, dialect: str) -> bool: try: - validator = validator_for(schema)(_without_x_gts_ref(schema)) + validator = validator_for(schema, inherited_dialect=dialect)( + _without_x_gts_ref(schema) + ) return validator.is_valid(instance) except Exception: # noqa: BLE001 - treat any validation error as "not valid" return False @@ -173,6 +176,7 @@ def __init__( mode: GtsRefValidationMode | bool | str = GtsRefValidationMode.ANY_VALID, *, enforce_existence: bool | None = None, + inherited_dialect: str | None = None, ): if enforce_existence is not None: mode = ( @@ -186,6 +190,7 @@ def __init__( ) self.store = store self.mode = GtsRefValidationMode(mode) + self.inherited_dialect = inherited_dialect self.referenced_ids: set[str] = set() self.referenced_wildcard_patterns: set[str] = set() @@ -219,6 +224,7 @@ def validate_instance( List of validation errors (empty if valid) """ errors: list[XGtsRefValidationError] = [] + dialect = effective_dialect(schema, self.inherited_dialect) selected_type_id = self.selected_type_id(schema, selected_type_id) def resolve_local_ref(ref: str) -> Any | None: @@ -271,7 +277,7 @@ def visit_instance(inst, sch, path, errs, refs=None): matching_branches = [ branch for branch in one_of - if _is_structurally_valid(inst, branch) + if _is_structurally_valid(inst, branch, dialect) ] if len(matching_branches) == 1: errs.extend(_validate_branch(inst, matching_branches[0], path)) @@ -292,7 +298,7 @@ def visit_instance(inst, sch, path, errs, refs=None): matching_branches = [ branch for branch in any_of - if _is_structurally_valid(inst, branch) + if _is_structurally_valid(inst, branch, dialect) ] branch_errors = [ _validate_branch(inst, branch, path) @@ -310,7 +316,7 @@ def visit_instance(inst, sch, path, errs, refs=None): all_of = sch.get("allOf") if isinstance(all_of, list): for branch in all_of: - if _is_structurally_valid(inst, branch): + if _is_structurally_valid(inst, branch, dialect): errs.extend(_validate_branch(inst, branch, path)) properties = sch.get("properties") diff --git a/tests/test_regressions.py b/tests/test_regressions.py index e905f41..49757d2 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -25,7 +25,7 @@ def _schema_entity(gts_id, content): class TestXGtsRefCombinators: def test_plain_one_of_is_left_to_jsonschema(self): - errors = XGtsRefValidator().validate_instance( + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_instance( "value", {"oneOf": [{"type": "string"}, {"type": "integer"}]} ) @@ -42,16 +42,16 @@ def test_x_gts_ref_only_one_of_still_requires_one_match(self): valid = "gts.x.test._.first.v1~x.test._.item.v1" invalid = "gts.x.test._.other.v1~x.test._.item.v1" - assert XGtsRefValidator().validate_instance(valid, schema) == [] + assert XGtsRefValidator(inherited_dialect="draft-07").validate_instance(valid, schema) == [] assert [ error.reason - for error in XGtsRefValidator().validate_instance(invalid, schema) + for error in XGtsRefValidator(inherited_dialect="draft-07").validate_instance(invalid, schema) ] == ["oneOf: no branch matched"] class TestReferenceResolution: def test_boolean_schema_does_not_require_reference_resolution(self): - assert GtsStore(reader=None)._resolve_schema_refs(True) is True + assert GtsStore(reader=None)._resolve_schema_refs(True, "draft-07") is True def test_modern_ref_siblings_are_preserved_during_instance_validation(self): store = GtsStore(reader=None) @@ -103,7 +103,7 @@ def test_null_default_is_materialized_and_overrides_ancestor_default(self): {"properties": {"value": {"default": None}}}, ], {}, - None, + "http://json-schema.org/draft-07/schema#", ) assert effective.values == {"value": None} diff --git a/tests/test_schema_cast.py b/tests/test_schema_cast.py index f0a9451..bf27bb1 100644 --- a/tests/test_schema_cast.py +++ b/tests/test_schema_cast.py @@ -517,6 +517,7 @@ def test_cast_backward_incompatible_and_validation_error(self): {"a": "x"}, from_schema, to_schema, + inherited_dialect="draft-07", ) assert result.is_fully_compatible is False assert result.incompatibility_reasons @@ -536,6 +537,7 @@ def test_cast_fully_compatible(self): {"a": "x"}, from_schema, to_schema, + inherited_dialect="draft-07", ) assert result.is_fully_compatible is True assert result.casted_entity == {"a": "x"} @@ -572,5 +574,6 @@ def test_cast_with_non_dict_instance_content_defaults_to_empty(self): "not-a-dict", {}, {}, + inherited_dialect="draft-07", ) assert result.casted_entity == {} diff --git a/tests/test_store_extra.py b/tests/test_store_extra.py index 3d465c9..16f655c 100644 --- a/tests/test_store_extra.py +++ b/tests/test_store_extra.py @@ -7,7 +7,7 @@ import pytest from gts.entities import DEFAULT_GTS_CONFIG, GtsEntity from gts.gts import GtsID -from gts.schema_validation import PATTERN_TIMEOUT_SECONDS, validator_for +from gts.schema_validation import PATTERN_TIMEOUT_SECONDS, check_schema, validator_for from gts.store import ( MAX_SCHEMA_REF_EXPANSIONS, GtsReader, @@ -15,7 +15,7 @@ StoreGtsEntityNotFound, StoreGtsObjectNotFound, ) -from jsonschema import ValidationError +from jsonschema import SchemaError, ValidationError class MockGtsReader(GtsReader): @@ -58,19 +58,34 @@ def _schema_entity(gts_id: str, content_extra=None): class TestBoundedPatternValidation: + def test_schema_check_uses_ecma_compatible_regex_format(self): + schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "pattern": r"^\p{Lu}(?=\p{Ll})", + } + check_schema(schema) + + def test_schema_check_rejects_invalid_regex(self): + schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "pattern": "[", + } + with pytest.raises(SchemaError): + check_schema(schema) + def test_catastrophic_pattern_times_out(self): - validator = validator_for({"pattern": "(a+)+$"})({"pattern": "(a+)+$"}) + validator = validator_for({"pattern": "(a+)+$"}, inherited_dialect="draft-07")({"pattern": "(a+)+$"}) errors = list(validator.iter_errors("a" * 30_000 + "!")) assert len(errors) == 1 assert errors[0].message == "regular expression match timed out" - assert PATTERN_TIMEOUT_SECONDS == 1.0 + assert PATTERN_TIMEOUT_SECONDS == 0.25 def test_catastrophic_pattern_properties_key_times_out(self): # patternProperties matches property NAMES; an adversarially long key # against a catastrophic pattern must be bounded by the same timeout as # the `pattern` keyword rather than backtracking indefinitely. schema = {"patternProperties": {"(a+)+$": {"type": "string"}}} - validator = validator_for(schema)(schema) + validator = validator_for(schema, inherited_dialect="draft-07")(schema) instance = {"a" * 30_000 + "!": "value"} errors = list(validator.iter_errors(instance)) assert any( @@ -168,7 +183,7 @@ def test_reference_expansion_budget(self): ] } with pytest.raises(ValueError, match="expansion exceeds limit"): - store._resolve_schema_refs(schema) + store._resolve_schema_refs(schema, "draft-07") class TestValidateSchemaRefs: @@ -371,6 +386,8 @@ def test_missing_base_schema_raises(self): @pytest.mark.parametrize( "dialect", [ + "http://json-schema.org/draft-06/schema#", + "https://json-schema.org/draft/2025-01/schema", "https://example.invalid/not-a-json-schema-dialect", "https://json-schema.org/draft/2020-21/schema", ], @@ -559,13 +576,13 @@ def test_resolves_gts_ref(self): store = GtsStore(reader=None) store.register(target) schema = {"$ref": "gts://gts.x.test._.target.v1~"} - resolved = store._resolve_schema_refs(schema) + resolved = store._resolve_schema_refs(schema, "draft-07") assert resolved["type"] == "object" def test_unresolvable_ref_left_unresolved(self): store = GtsStore(reader=None) schema = {"$ref": "gts://gts.x.test._.missing.v1~"} - resolved = store._resolve_schema_refs(schema) + resolved = store._resolve_schema_refs(schema, "draft-07") assert resolved == schema def test_cyclic_ref_left_unresolved(self): @@ -586,11 +603,12 @@ def test_ref_with_siblings_creates_allof(self): "$ref": "gts://gts.x.test._.target.v1~", "title": "sibling", } - resolved = store._resolve_schema_refs(schema) + resolved = store._resolve_schema_refs(schema, "2020-12") assert "allOf" in resolved - def test_supports_ref_siblings_false_for_non_dict(self): - assert GtsStore._supports_ref_siblings("nope") is False + def test_supports_ref_siblings_rejects_non_schema(self): + with pytest.raises(TypeError, match="schema must be a JSON object"): + GtsStore._supports_ref_siblings("nope") def test_inline_refs_list_recursion(self): store = GtsStore(reader=None) diff --git a/tests/test_traits.py b/tests/test_traits.py index 9746c53..c783a3b 100644 --- a/tests/test_traits.py +++ b/tests/test_traits.py @@ -11,6 +11,8 @@ merge_rfc7396_into, ) +DRAFT7 = "http://json-schema.org/draft-07/schema#" + class TestSchemaTraversal: def test_maps_draft3_schema_forms_only(self): @@ -161,26 +163,26 @@ def test_multiple_schemas_composed_via_allof(self): class TestBuildEffectiveTraits: def test_no_schema_no_values(self): - effective = build_effective_traits([], {}, None) + effective = build_effective_traits([], {}, DRAFT7) assert effective.validate(check_unresolved=True) == [] def test_values_without_schema_is_error(self): - effective = build_effective_traits([], {"a": 1}, None) + effective = build_effective_traits([], {"a": 1}, DRAFT7) errors = effective.validate(check_unresolved=True) assert any("no x-gts-traits-schema is defined" in e for e in errors) def test_schema_false_prohibits_values(self): - effective = build_effective_traits([False], {"a": 1}, None) + effective = build_effective_traits([False], {"a": 1}, DRAFT7) errors = effective.validate(check_unresolved=True) assert any("values are prohibited" in e for e in errors) def test_schema_false_with_no_values_is_ok(self): - effective = build_effective_traits([False], {}, None) + effective = build_effective_traits([False], {}, DRAFT7) assert effective.validate(check_unresolved=True) == [] def test_default_materialized(self): effective = build_effective_traits( - [{"type": "object", "properties": {"a": {"default": "x"}}}], {}, None + [{"type": "object", "properties": {"a": {"default": "x"}}}], {}, DRAFT7 ) assert effective.values == {"a": "x"} @@ -190,7 +192,7 @@ def test_valid_trait_values_pass(self): "properties": {"a": {"type": "string"}}, "required": ["a"], } - effective = build_effective_traits([schema], {"a": "hi"}, None) + effective = build_effective_traits([schema], {"a": "hi"}, DRAFT7) assert effective.validate(check_unresolved=True) == [] def test_standard_trait_formats_are_enforced(self): @@ -204,12 +206,12 @@ def test_standard_trait_formats_are_enforced(self): assert ( build_effective_traits( - [schema], {"email": "user@example.com", "time": "10:30:00Z"}, None + [schema], {"email": "user@example.com", "time": "10:30:00Z"}, DRAFT7 ).validate(check_unresolved=True) == [] ) errors = build_effective_traits( - [schema], {"email": "not-an-email", "time": "10:30:00Z"}, None + [schema], {"email": "not-an-email", "time": "10:30:00Z"}, DRAFT7 ).validate(check_unresolved=True) assert any("is not a 'email'" in error for error in errors) @@ -218,7 +220,7 @@ def test_invalid_trait_type_fails(self): "type": "object", "properties": {"a": {"type": "string"}}, } - effective = build_effective_traits([schema], {"a": 5}, None) + effective = build_effective_traits([schema], {"a": 5}, DRAFT7) errors = effective.validate(check_unresolved=True) assert any("trait validation" in e for e in errors) @@ -228,7 +230,7 @@ def test_required_without_default_unresolved(self): "properties": {"a": {"type": "string"}}, "required": ["a"], } - effective = build_effective_traits([schema], {}, None) + effective = build_effective_traits([schema], {}, DRAFT7) errors = effective.validate(check_unresolved=True) assert any("is not resolved" in e for e in errors) @@ -238,7 +240,7 @@ def test_abstract_skips_unresolved_check(self): "properties": {"a": {"type": "string"}}, "required": ["a"], } - effective = build_effective_traits([schema], {}, None) + effective = build_effective_traits([schema], {}, DRAFT7) errors = effective.validate(check_unresolved=False) assert errors == [] @@ -247,7 +249,7 @@ def test_abstract_validates_provided_trait_values(self): "type": "object", "properties": {"a": {"type": "string"}}, } - effective = build_effective_traits([schema], {"a": 5}, None) + effective = build_effective_traits([schema], {"a": 5}, DRAFT7) errors = effective.validate(check_unresolved=False) assert any("is not of type 'string'" in error for error in errors) @@ -258,7 +260,7 @@ def test_abstract_preserves_required_in_const_value(self): "required": ["missing"], } effective = build_effective_traits( - [schema], {"config": {"required": ["a"]}}, None + [schema], {"config": {"required": ["a"]}}, DRAFT7 ) assert effective.validate(check_unresolved=False) == [] @@ -273,7 +275,7 @@ def test_abstract_skips_required_in_nested_dialect_schema(self): } ], } - effective = build_effective_traits([schema], {}, None) + effective = build_effective_traits([schema], {}, DRAFT7) assert effective.validate(check_unresolved=False) == [] def test_abstract_checks_x_gts_ref_constraint_type_existence(self): @@ -287,7 +289,7 @@ def get(self, value): "ref": {"type": "string", "x-gts-ref": "gts.x.test._.foo.v1~"} }, } - errors = build_effective_traits([schema], {}, None).validate( + errors = build_effective_traits([schema], {}, DRAFT7).validate( check_unresolved=False, reference_store=FakeStore() ) assert any( @@ -298,13 +300,13 @@ def get(self, value): def test_incompatible_trait_schema_chain_flagged(self): # Second schema narrows type incompatibly with the ancestor. effective = build_effective_traits( - [{"type": "string"}, {"type": "integer"}], {}, None + [{"type": "string"}, {"type": "integer"}], {}, DRAFT7 ) errors = effective.validate(check_unresolved=True) assert any("incompatible with ancestor trait schema" in e for e in errors) def test_invalid_trait_schema_integrity_flagged(self): - effective = build_effective_traits([{"type": "not-a-real-type"}], {}, None) + effective = build_effective_traits([{"type": "not-a-real-type"}], {}, DRAFT7) errors = effective.validate(check_unresolved=True) assert any("not a valid JSON Schema" in e for e in errors) @@ -339,6 +341,6 @@ def test_x_gts_ref_errors_prefixed(self): "type": "object", "properties": {"ref": {"type": "string", "x-gts-ref": "not-valid"}}, } - effective = build_effective_traits([schema], {"ref": "also-not-valid"}, None) + effective = build_effective_traits([schema], {"ref": "also-not-valid"}, DRAFT7) errors = effective.validate(check_unresolved=True) assert any(e.startswith("trait x-gts-ref:") for e in errors) diff --git a/tests/test_x_gts_ref.py b/tests/test_x_gts_ref.py index e2ed54d..5ff96e6 100644 --- a/tests/test_x_gts_ref.py +++ b/tests/test_x_gts_ref.py @@ -5,46 +5,46 @@ class TestValidateSchema: def test_valid_absolute_pattern(self): - errors = XGtsRefValidator().validate_schema( + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_schema( {"x-gts-ref": "gts.x.test._.foo.v1~"} ) assert errors == [] def test_valid_wildcard_pattern(self): - errors = XGtsRefValidator().validate_schema({"x-gts-ref": "gts.*"}) + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_schema({"x-gts-ref": "gts.*"}) assert errors == [] def test_valid_prefix_wildcard(self): - errors = XGtsRefValidator().validate_schema({"x-gts-ref": "gts.x.test.*"}) + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_schema({"x-gts-ref": "gts.x.test.*"}) assert errors == [] def test_invalid_wildcard_prefix_direct(self): - error = XGtsRefValidator()._validate_gts_id_or_pattern("notgts*", "path") + error = XGtsRefValidator(inherited_dialect="draft-07")._validate_gts_id_or_pattern("notgts*", "path") assert error is not None assert "Invalid GTS wildcard pattern" in error.reason def test_invalid_specific_gts_id(self): - errors = XGtsRefValidator().validate_schema({"x-gts-ref": "gts.bad id"}) + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_schema({"x-gts-ref": "gts.bad id"}) assert len(errors) == 1 assert "Invalid GTS identifier" in errors[0].reason def test_non_string_ref_value(self): - errors = XGtsRefValidator().validate_schema({"x-gts-ref": 123}) + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_schema({"x-gts-ref": 123}) assert len(errors) == 1 assert "must be a string" in errors[0].reason def test_invalid_prefix_value(self): - errors = XGtsRefValidator().validate_schema({"x-gts-ref": "nope"}) + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_schema({"x-gts-ref": "nope"}) assert len(errors) == 1 assert "must be a GTS identifier" in errors[0].reason def test_selected_type_self_reference_is_valid(self): - validator = XGtsRefValidator() + validator = XGtsRefValidator(inherited_dialect="draft-07") assert validator.is_self_reference(X_GTS_REF_SELF) assert validator.validate_schema({"x-gts-ref": X_GTS_REF_SELF}) == [] def test_other_pointers_are_invalid(self): - validator = XGtsRefValidator() + validator = XGtsRefValidator(inherited_dialect="draft-07") for pointer in ("/missing/path", "/properties/id"): errors = validator.validate_schema({"x-gts-ref": pointer}) assert len(errors) == 1 @@ -56,13 +56,13 @@ def test_recurses_into_nested_structures(self): "child": {"x-gts-ref": "notgts.*"}, } } - errors = XGtsRefValidator().validate_schema(schema) + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_schema(schema) assert len(errors) == 1 assert "properties/child/x-gts-ref" in errors[0].field_path def test_recurses_into_list_of_dicts(self): schema = {"allOf": [{"x-gts-ref": "notgts.*"}]} - errors = XGtsRefValidator().validate_schema(schema) + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_schema(schema) assert len(errors) == 1 assert "allOf[0]/x-gts-ref" in errors[0].field_path @@ -75,7 +75,7 @@ def test_ignores_x_gts_ref_in_annotation_data(self): } } } - assert XGtsRefValidator().validate_schema(schema) == [] + assert XGtsRefValidator(inherited_dialect="draft-07").validate_schema(schema) == [] def test_property_named_x_gts_ref_is_not_a_keyword(self): schema = { @@ -83,7 +83,7 @@ def test_property_named_x_gts_ref_is_not_a_keyword(self): "x-gts-ref": {"x-gts-ref": "notgts.*"}, } } - errors = XGtsRefValidator().validate_schema(schema) + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_schema(schema) assert len(errors) == 1 assert errors[0].field_path == "properties/x-gts-ref/x-gts-ref" @@ -94,7 +94,7 @@ def test_recurses_into_draft3_schema_forms(self): "type": ["object", {"x-gts-ref": "invalid-type"}], "disallow": ["array", {"x-gts-ref": "invalid-disallow"}], } - errors = XGtsRefValidator().validate_schema(schema) + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_schema(schema) assert [error.field_path for error in errors] == [ "extends/x-gts-ref", "type[1]/x-gts-ref", @@ -108,7 +108,7 @@ class FakeStore: def get(self, value): return None - errors = XGtsRefValidator(store=FakeStore()).validate_schema_ref_existence( + errors = XGtsRefValidator(store=FakeStore(), inherited_dialect="draft-07").validate_schema_ref_existence( {"properties": {"ref": {"x-gts-ref": "gts.x.test._.foo.v1~"}}} ) @@ -127,7 +127,7 @@ def get(self, value): def items(self): return [("gts.x.test._.foo.v1~", object())] - errors = XGtsRefValidator(store=FakeStore()).validate_schema_ref_existence( + errors = XGtsRefValidator(store=FakeStore(), inherited_dialect="draft-07").validate_schema_ref_existence( { "allOf": [ {"x-gts-ref": "gts.x.test._.foo.v1~"}, @@ -143,7 +143,7 @@ class FakeStore: def get(self, value): return None - errors = XGtsRefValidator(store=FakeStore()).validate_schema_ref_existence( + errors = XGtsRefValidator(store=FakeStore(), inherited_dialect="draft-07").validate_schema_ref_existence( {"const": {"x-gts-ref": "gts.x.test._.missing.v1~"}} ) assert errors == [] @@ -153,7 +153,7 @@ class FakeStore: def get(self, value): return object() if value == "gts.x.test._.leaf.v1~" else None - errors = XGtsRefValidator(store=FakeStore()).validate_schema_ref_existence( + errors = XGtsRefValidator(store=FakeStore(), inherited_dialect="draft-07").validate_schema_ref_existence( {"x-gts-ref": X_GTS_REF_SELF}, selected_type_id="gts.x.test._.leaf.v1~", ) @@ -162,7 +162,7 @@ def get(self, value): class TestValidateInstanceValue: def test_non_string_instance_value_error(self): - error = XGtsRefValidator()._validate_ref_value(123, "gts.*", "ref", None) + error = XGtsRefValidator(inherited_dialect="draft-07")._validate_ref_value(123, "gts.*", "ref", None) assert error is not None assert "Value must be a string" in error.reason @@ -172,7 +172,7 @@ def test_relative_ref_pattern_resolution_on_instance(self): "type": "object", "properties": {"ref": {"x-gts-ref": "/$id"}}, } - errors = XGtsRefValidator().validate_instance( + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_instance( {"ref": "gts.x.test._.foo.v1~x.test._.bar.v1"}, schema ) assert errors == [] @@ -185,32 +185,32 @@ def test_self_reference_uses_explicit_selected_leaf(self): } leaf = "gts.x.test._.base.v1~x.test._.leaf.v1~" assert ( - XGtsRefValidator().validate_instance( + XGtsRefValidator(inherited_dialect="draft-07").validate_instance( {"ref": leaf}, schema, selected_type_id=leaf ) == [] ) - errors = XGtsRefValidator().validate_instance( + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_instance( {"ref": "gts.x.test._.base.v1~"}, schema, selected_type_id=leaf ) assert len(errors) == 1 assert "does not match pattern" in errors[0].reason def test_wildcard_pattern_matches_prefix(self): - errors = XGtsRefValidator().validate_instance( + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_instance( "gts.x.test._.foo.v1~", {"x-gts-ref": "gts.x.test.*"} ) assert errors == [] def test_wildcard_pattern_mismatch(self): - errors = XGtsRefValidator().validate_instance( + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_instance( "gts.x.other._.foo.v1~", {"x-gts-ref": "gts.x.test.*"} ) assert len(errors) == 1 assert "does not match pattern" in errors[0].reason def test_exact_pattern_mismatch(self): - errors = XGtsRefValidator().validate_instance( + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_instance( "gts.x.other._.foo.v1~", {"x-gts-ref": "gts.x.test._.foo.v1~"} ) assert len(errors) == 1 @@ -221,7 +221,7 @@ class FakeStore: def get(self, value): return None - errors = XGtsRefValidator(store=FakeStore()).validate_instance( + errors = XGtsRefValidator(store=FakeStore(), inherited_dialect="draft-07").validate_instance( "gts.x.test._.foo.v1~", {"x-gts-ref": "gts.*"} ) assert len(errors) == 1 @@ -232,7 +232,7 @@ class FakeStore: def get(self, value): return object() - errors = XGtsRefValidator(store=FakeStore()).validate_instance( + errors = XGtsRefValidator(store=FakeStore(), inherited_dialect="draft-07").validate_instance( "gts.x.test._.foo.v1~", {"x-gts-ref": "gts.*"} ) assert errors == [] @@ -242,7 +242,7 @@ def test_array_items_recursion(self): "type": "array", "items": {"x-gts-ref": "gts.x.test.*"}, } - errors = XGtsRefValidator().validate_instance(["gts.x.other.v1~"], schema) + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_instance(["gts.x.other.v1~"], schema) assert len(errors) == 1 def test_tuple_additional_items_recursion(self): @@ -254,7 +254,7 @@ def test_tuple_additional_items_recursion(self): "x-gts-ref": "gts.x.test._.target.v1~", }, } - errors = XGtsRefValidator().validate_instance( + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_instance( ["tuple-prefix", "gts.x.other._.target.v1~"], schema ) assert len(errors) == 1 @@ -269,7 +269,7 @@ def test_prefix_items_recursion(self): "x-gts-ref": "gts.x.test._.target.v1~", }, } - errors = XGtsRefValidator().validate_instance( + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_instance( ["tuple-prefix", "gts.x.other._.target.v1~"], schema ) assert len(errors) == 1 @@ -280,7 +280,7 @@ def test_object_properties_recursion(self): "type": "object", "properties": {"ref": {"x-gts-ref": "gts.x.test.*"}}, } - errors = XGtsRefValidator().validate_instance( + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_instance( {"ref": "gts.x.other.v1~"}, schema ) assert len(errors) == 1 @@ -294,7 +294,7 @@ def test_root_ref_traverses_nested_constraints(self): }, } - errors = XGtsRefValidator().validate_instance( + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_instance( {"child": {"link": "gts.x.other.v1~"}}, schema ) @@ -308,7 +308,7 @@ def test_any_of_no_branch_matched(self): {"x-gts-ref": "gts.x.test._.b.v1~"}, ] } - errors = XGtsRefValidator().validate_instance( + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_instance( "gts.x.test._.c.v1~x.test._.item.v1", schema ) assert any("anyOf: no branch matched" in e.reason for e in errors) @@ -320,7 +320,7 @@ def test_any_of_matches_one_branch(self): {"x-gts-ref": "gts.x.test._.b.v1~"}, ] } - errors = XGtsRefValidator().validate_instance( + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_instance( "gts.x.test._.a.v1~x.test._.item.v1", schema ) assert errors == [] @@ -331,5 +331,5 @@ def test_all_of_validates_each_matching_branch(self): {"type": "string", "x-gts-ref": "gts.x.test.*"}, ] } - errors = XGtsRefValidator().validate_instance("gts.x.other.v1~", schema) + errors = XGtsRefValidator(inherited_dialect="draft-07").validate_instance("gts.x.other.v1~", schema) assert len(errors) == 1 From dcca589d9ef4fa17ad5ce5ce78db6936cc88824f Mon Sep 17 00:00:00 2001 From: Artifizer Date: Tue, 29 Sep 2026 17:22:40 +0300 Subject: [PATCH 26/27] fix: persist batch updates with inherited dialects Publish changed entries from validated batches when updates are enabled. Preserve inherited JSON Schema dialects for boolean trait schemas and x-gts-ref validation, and remove the batch result type redefinition caught by mypy. Signed-off-by: Artifizer --- gts/src/gts/ops.py | 6 ++++-- gts/src/gts/schema_dialect.py | 10 +++++++--- gts/src/gts/store.py | 14 ++++++++++++-- gts/src/gts/traits.py | 17 +++++++++++++---- tests/test_staging_commit_integrity.py | 12 ++++++++++++ 5 files changed, 48 insertions(+), 11 deletions(-) diff --git a/gts/src/gts/ops.py b/gts/src/gts/ops.py index 4ad68b5..6d3930a 100644 --- a/gts/src/gts/ops.py +++ b/gts/src/gts/ops.py @@ -608,13 +608,15 @@ def add_schemas( """ gts_ref_validation = _normalize_gts_ref_validation(gts_ref_validation) if not validate: - results = [ + direct_results = [ self.add_schema( schema, validate=False, gts_ref_validation=gts_ref_validation ) for schema in schemas ] - return GtsAddSchemasResult(ok=all(r.ok for r in results), results=results) + return GtsAddSchemasResult( + ok=all(r.ok for r in direct_results), results=direct_results + ) results: list[GtsAddSchemaResult | None] = [None] * len(schemas) # Track every still-staged token so an exception anywhere below discards diff --git a/gts/src/gts/schema_dialect.py b/gts/src/gts/schema_dialect.py index 7be92fd..42273da 100644 --- a/gts/src/gts/schema_dialect.py +++ b/gts/src/gts/schema_dialect.py @@ -27,9 +27,13 @@ def require_dialect(dialect: str | None) -> str: - if dialect not in SUPPORTED_DIALECTS: - raise ValueError(f"Unsupported JSON Schema dialect: {dialect}") - return dialect + if dialect in SUPPORTED_DIALECTS: + return dialect + if isinstance(dialect, str): + normalized = dialect.removesuffix("#").lower().replace("https://", "http://", 1) + if normalized in _SUPPORTED: + return _SUPPORTED[normalized] + raise ValueError(f"Unsupported JSON Schema dialect: {dialect}") def effective_dialect(schema: Any, inherited_dialect: str | None = None) -> str: diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index 6b1ca18..21a8b81 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -413,7 +413,13 @@ def commit_batch(self, tokens: list[str], allow_updates: bool = False) -> list[s existing = pending_by_key.get(key, self._by_id.get(key)) if existing is not None: identical = existing.content == staged.content - outcome = "unchanged" if identical or allow_updates else "conflict" + outcome = ( + "unchanged" + if identical + else "added" + if allow_updates + else "conflict" + ) if outcome == "conflict": any_conflict = True outcomes.append(outcome) @@ -1101,7 +1107,11 @@ def _validate_schema_transitive( ) effective_traits = self._build_effective_traits(schema_id.id) - trait_ref_validator = XGtsRefValidator(store=self, mode=gts_ref_validation) + trait_ref_validator = XGtsRefValidator( + store=self, + mode=gts_ref_validation, + inherited_dialect=effective_traits.dialect, + ) trait_ref_validator.validate_schema_ref_existence( effective_traits.schema, selected_type_id=schema_id.id ) diff --git a/gts/src/gts/traits.py b/gts/src/gts/traits.py index a25baac..e6bf533 100644 --- a/gts/src/gts/traits.py +++ b/gts/src/gts/traits.py @@ -98,6 +98,7 @@ def validate( reference_store, gts_ref_validation, selected_type_id, + self.dialect, ) @@ -358,12 +359,15 @@ def _validate_trait_schema_compatibility( def _validate_traits_against_schema( - trait_schema: Any, effective_traits: Any, check_unresolved: bool + trait_schema: Any, + effective_traits: Any, + check_unresolved: bool, + inherited_dialect: str | None, ) -> list[str]: errors: list[str] = [] try: - cls = validator_for(trait_schema) + cls = validator_for(trait_schema, inherited_dialect=inherited_dialect) validator = cls(trait_schema, format_checker=_FORMAT_CHECKER) for error in validator.iter_errors(effective_traits): errors.append(f"trait validation: {error.message}") @@ -403,6 +407,7 @@ def _validate_trait_values( reference_store: Any | None, gts_ref_validation: GtsRefValidationMode, selected_type_id: str | None, + inherited_dialect: str | None, ) -> list[str]: schema_for_values = ( effective_traits_schema @@ -410,9 +415,13 @@ def _validate_trait_values( else _without_required(effective_traits_schema) ) errors = _validate_traits_against_schema( - schema_for_values, effective_traits, check_unresolved + schema_for_values, effective_traits, check_unresolved, inherited_dialect + ) + xref = XGtsRefValidator( + store=reference_store, + mode=gts_ref_validation, + inherited_dialect=inherited_dialect, ) - xref = XGtsRefValidator(store=reference_store, mode=gts_ref_validation) for err in xref.validate_schema_ref_existence( effective_traits_schema, selected_type_id=selected_type_id ): diff --git a/tests/test_staging_commit_integrity.py b/tests/test_staging_commit_integrity.py index f22b5dd..6b6fc60 100644 --- a/tests/test_staging_commit_integrity.py +++ b/tests/test_staging_commit_integrity.py @@ -162,3 +162,15 @@ def test_commit_batch_publishes_nothing_when_a_target_conflicts_with_committed() # Nothing from the batch was published; the parent keeps its committed content. assert ops.store.get_committed(child) is None assert ops.store.get_committed(parent).content["title"] == "committed" + + +def test_commit_batch_publishes_changed_content_when_updates_are_enabled(): + ops = GtsOps(path=None, allow_entity_updates=True) + type_id = "gts.x.pyatomic3._.updated.v1~" + assert ops.add_schema(_schema(type_id, "old")).ok + + result = ops.add_schemas([_schema(type_id, "new")], validate=True) + + assert result.ok + assert result.results[0].ok + assert ops.store.get_committed(type_id).content["title"] == "new" From 46b09b2e5701bbbe2d4acad17ff8e6fae4af6fa8 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Wed, 30 Sep 2026 00:44:52 +0300 Subject: [PATCH 27/27] chore: bump gts-spec pin to v0.14.5 Update the pinned spec release from v0.14.4 to v0.14.5, advance the .gts-spec submodule to the matching tag, and update the supported-version notes in the READMEs. v0.14.5 requires ECMA-262 regex semantics for schema patterns; the full gts-spec conformance suite passes at the new pin. Signed-off-by: Artifizer --- .gts-spec | 2 +- .gts-spec-version | 2 +- README.md | 2 +- gts/README.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gts-spec b/.gts-spec index 386870a..6f91ebd 160000 --- a/.gts-spec +++ b/.gts-spec @@ -1 +1 @@ -Subproject commit 386870a2f4e7b79c1af019b9d13fc30e06e2efef +Subproject commit 6f91ebd2f3e958dac71026b14fe719128831ed82 diff --git a/.gts-spec-version b/.gts-spec-version index 6cf9fa5..574be60 100644 --- a/.gts-spec-version +++ b/.gts-spec-version @@ -1 +1 @@ -v0.14.4 +v0.14.5 diff --git a/README.md b/README.md index 61b148f..c6ac7fe 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A minimal, idiomatic Python library for working with **GTS** ([Global Type System](https://github.com/gts-spec/gts-spec)) identifiers and JSON/JSON Schema artifacts. -Current supported GTS spec version: `0.14.4` +Current supported GTS spec version: `0.14.5` ## Roadmap diff --git a/gts/README.md b/gts/README.md index 6e18b3a..e78c553 100644 --- a/gts/README.md +++ b/gts/README.md @@ -2,7 +2,7 @@ Python helpers and a reference HTTP service for the [Global Type System (GTS)](https://github.com/globaltypesystem/gts-spec). The package supports GTS identifier parsing, JSON Schema-backed validation, schema compatibility and derivation checks, traits, casting, queries, file loading, a CLI, and a FastAPI application. -The package targets GTS specification v0.14.3 and requires Python 3.9 or later. +The package targets GTS specification v0.14.5 and requires Python 3.9 or later. ## Installation