From 52d3e71bf28acdc54c48a73a0e63b9383ea6847a Mon Sep 17 00:00:00 2001 From: Michael Heller <21163552+mdheller@users.noreply.github.com> Date: Mon, 3 Aug 2026 01:24:06 -0400 Subject: [PATCH] feat: content-addressed, hash-chained model carry manifest Adds ModelCarryManifest: a content-addressed catalog for the model-router carry layer that names every base and adapter with a verifiable identity, targeting predicate, authoritative SHA-256 content hash, and policy hash. Closes three gaps from the SourceOS-vs-Apple model-carry analysis: - no content-addressed model manifest (identity asserted, not verifiable); - adapter/base version binding not enforced (silent drift at deploy); - integrity-checked transport not an invariant. Teeth (tools/validate_model_carry_manifests.py, wired into make validate): - SHA-256 authoritative (FIPS 180-4); content+policy hashes non-null; - integrityFailureIsHardStop and transportMustBeIntegrityChecked const true; - manifest hash-chain (genesis-only null prev link); - adapter must bind to an existing base and pin its exact content hash. Includes valid example plus negative fixtures for adapter/base drift, missing base, and soft integrity. --- Makefile | 7 +- contracts/model-carry-manifest.schema.json | 117 ++++++++++++++ docs/model-carry-manifest.md | 51 +++++++ ...y-manifest.adapter-base-drift.invalid.json | 35 +++++ ...manifest.adapter-missing-base.invalid.json | 25 +++ ...nifest.integrity-not-hardstop.invalid.json | 25 +++ .../model-carry-manifest.laptop-safe.json | 43 ++++++ tools/validate_model_carry_manifests.py | 144 ++++++++++++++++++ 8 files changed, 445 insertions(+), 2 deletions(-) create mode 100644 contracts/model-carry-manifest.schema.json create mode 100644 docs/model-carry-manifest.md create mode 100644 examples/model-carry-manifest.adapter-base-drift.invalid.json create mode 100644 examples/model-carry-manifest.adapter-missing-base.invalid.json create mode 100644 examples/model-carry-manifest.integrity-not-hardstop.invalid.json create mode 100644 examples/model-carry-manifest.laptop-safe.json create mode 100644 tools/validate_model_carry_manifests.py diff --git a/Makefile b/Makefile index d9380ab..904fe15 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test validate validate-portable-ai validate-model-carry-boundary dist release-dry-run clean +.PHONY: build test validate validate-portable-ai validate-model-carry-boundary validate-model-carry-manifest dist release-dry-run clean BIN := sourceos-ai DIST_DIR := dist @@ -23,7 +23,10 @@ validate-portable-ai: validate-model-carry-boundary: python3 tools/validate_model_carry_authorization_boundaries.py -validate: build validate-portable-ai validate-model-carry-boundary +validate-model-carry-manifest: + python3 tools/validate_model_carry_manifests.py + +validate: build validate-portable-ai validate-model-carry-boundary validate-model-carry-manifest python3 tools/validate_carry_refs.py bin/$(BIN) carry validate --refs examples bin/$(BIN) carry list --refs examples diff --git a/contracts/model-carry-manifest.schema.json b/contracts/model-carry-manifest.schema.json new file mode 100644 index 0000000..34ee4eb --- /dev/null +++ b/contracts/model-carry-manifest.schema.json @@ -0,0 +1,117 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.srcos.ai/model-carry/model-carry-manifest.schema.json", + "title": "SourceOS Model Carry Manifest", + "description": "A content-addressed, hash-chained model manifest for the SourceOS model-router carry layer. Names every carried model and adapter with a base-version binding, a targeting predicate, an authoritative SHA-256 content hash, and a policy hash. This is the model-layer analogue of Exodus provenance: model identity is verifiable, not asserted. Integrity failure is a hard stop, not a routing hint. SHA-256 is the authoritative content-hash algorithm (FIPS 180-4).", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "kind", + "manifestId", + "version", + "prevManifestSha256", + "integrity", + "entries" + ], + "properties": { + "schemaVersion": { "const": "v0.1" }, + "kind": { "const": "ModelCarryManifest" }, + "manifestId": { "type": "string", "pattern": "^urn:srcos:model-carry-manifest:" }, + "version": { + "type": "integer", + "minimum": 1, + "description": "Monotonic manifest version. version 1 is the genesis manifest and MUST have a null prevManifestSha256; every later version MUST link the previous manifest by SHA-256." + }, + "prevManifestSha256": { + "type": ["string", "null"], + "pattern": "^[a-f0-9]{64}$", + "description": "SHA-256 of the previous manifest in the chain, or null for the genesis manifest (version 1)." + }, + "integrity": { + "type": "object", + "additionalProperties": false, + "required": [ + "contentHashAlgorithm", + "transportMustBeIntegrityChecked", + "integrityFailureIsHardStop" + ], + "properties": { + "contentHashAlgorithm": { + "const": "sha256", + "description": "Authoritative content-hash algorithm. SHA-256 (FIPS 180-4) only." + }, + "transportMustBeIntegrityChecked": { + "const": true, + "description": "Carried artifacts MUST arrive through integrity-checked transport; unchecked transport is non-conformant." + }, + "integrityFailureIsHardStop": { + "const": true, + "description": "A carried artifact whose content hash does not match its manifest entry MUST be rejected. Integrity failure is a hard stop surfaced as an uncertainty signal to the decision surface, never a silent fallback." + } + } + }, + "entries": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "entryId", + "entryKind", + "modelRef", + "contentSha256", + "policySha256", + "targeting", + "baseEntryRef", + "baseContentSha256" + ], + "properties": { + "entryId": { "type": "string", "pattern": "^urn:srcos:model-carry-entry:" }, + "entryKind": { "type": "string", "enum": ["base", "adapter"] }, + "modelRef": { "type": "string", "minLength": 1 }, + "contentSha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$", + "description": "Authoritative SHA-256 of the carried artifact content. Never null: content-addressing is not optional at the manifest layer." + }, + "policySha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$", + "description": "SHA-256 of the policy object bound to this entry." + }, + "targeting": { + "type": "object", + "additionalProperties": false, + "required": ["deviceClass"], + "description": "Targeting predicate deciding which device class, OS version range, and locale this entry applies to.", + "properties": { + "deviceClass": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "enum": ["laptop", "workstation", "field-kit", "server", "any"] } + }, + "osVersionRange": { "type": ["string", "null"] }, + "locale": { + "type": ["array", "null"], + "items": { "type": "string" } + } + } + }, + "baseEntryRef": { + "type": ["string", "null"], + "pattern": "^urn:srcos:model-carry-entry:", + "description": "For entryKind=adapter: the entryId of the base this adapter is registered against. Null for entryKind=base." + }, + "baseContentSha256": { + "type": ["string", "null"], + "pattern": "^[a-f0-9]{64}$", + "description": "For entryKind=adapter: the pinned SHA-256 of the base content this adapter was registered against. Must equal the referenced base entry contentSha256. Null for entryKind=base." + } + } + } + }, + "notes": { "type": "string" } + } +} diff --git a/docs/model-carry-manifest.md b/docs/model-carry-manifest.md new file mode 100644 index 0000000..ce6206f --- /dev/null +++ b/docs/model-carry-manifest.md @@ -0,0 +1,51 @@ +# Model Carry Manifest + +The model carry manifest is the content-addressed, hash-chained catalog the +SourceOS model-router carries. It names every base model and adapter with a +verifiable identity, a targeting predicate, and the policy bound to it. It is the +model-layer analogue of Exodus provenance: model identity is verifiable, not +asserted. + +- Contract: `contracts/model-carry-manifest.schema.json` +- Validator: `tools/validate_model_carry_manifests.py` (`make validate-model-carry-manifest`) +- Example: `examples/model-carry-manifest.laptop-safe.json` + +## Why + +This closes three gaps identified in the SourceOS-vs-Apple model-carry analysis, +where Apple's delivery discipline exposed missing primitives in our carry path: + +1. **No content-addressed model manifest.** The router routed, but nothing named + every model+adapter with its targeting predicate and integrity hash. Without + it, provenance at the model layer was asserted, not verifiable. +2. **Adapter/base version binding was not enforced.** A base update did not + mechanically force adapter re-delivery, inviting silent drift between an + adapter and the base it was trained against. +3. **Encrypted, integrity-checked transport was not an invariant.** Integrity was + optional rather than a hard stop. + +## Invariants (enforced by the validator) + +| Invariant | Rule | +|---|---| +| Authoritative hash | `integrity.contentHashAlgorithm` is `sha256` (FIPS 180-4). | +| Content-addressed | Every entry carries a non-null `contentSha256` and `policySha256`. | +| Integrity is a hard stop | `integrityFailureIsHardStop` and `transportMustBeIntegrityChecked` are `true`. | +| Hash chain | `version` 1 has null `prevManifestSha256`; every later version links the previous manifest by SHA-256. | +| Adapter/base binding | Every adapter references an existing base entry and pins a `baseContentSha256` that equals that base's `contentSha256`. Drift is rejected by construction. | +| Base entries | A base entry sets `baseEntryRef` and `baseContentSha256` to null. | +| Targeting predicate | Every entry carries a `targeting` predicate with at least a `deviceClass`. | + +## Carry boundary + +The manifest is a reference/verification object. It does not authorize runtime +execution, prompt egress, tool use, model download, training, or promotion. Those +remain outside the mutable workstation image, consistent with the carry-only +doctrine in `repo.maturity.yaml`. + +## Not yet in scope (tracked as issues) + +- Governed staged pre-load plus atomic swap under a disk/CPU resource governor. +- Tiered escalation contract: local-by-default to an attested confidential-compute + target, with the escalation decision and its confidence propagated as a + first-class provenance record. diff --git a/examples/model-carry-manifest.adapter-base-drift.invalid.json b/examples/model-carry-manifest.adapter-base-drift.invalid.json new file mode 100644 index 0000000..744a381 --- /dev/null +++ b/examples/model-carry-manifest.adapter-base-drift.invalid.json @@ -0,0 +1,35 @@ +{ + "schemaVersion": "v0.1", + "kind": "ModelCarryManifest", + "manifestId": "urn:srcos:model-carry-manifest:laptop-safe", + "version": 2, + "prevManifestSha256": "4888b1850bddd6715434b02a37378ba12fab3102a20277d2434c92082a749e1a", + "integrity": { + "contentHashAlgorithm": "sha256", + "transportMustBeIntegrityChecked": true, + "integrityFailureIsHardStop": true + }, + "entries": [ + { + "entryId": "urn:srcos:model-carry-entry:llama32-3b-base", + "entryKind": "base", + "modelRef": "llama3.2:3b", + "contentSha256": "7372678075509b60673cfe1df40dc56801ace22cad6e56277cf4a0b4a359d8e7", + "policySha256": "af853ebc3648bc457c414e81b29f88c40931d5c744a89ac42dbd0dacc3aa23b0", + "targeting": { "deviceClass": ["laptop"], "osVersionRange": null, "locale": null }, + "baseEntryRef": null, + "baseContentSha256": null + }, + { + "entryId": "urn:srcos:model-carry-entry:office-rewrite-adapter", + "entryKind": "adapter", + "modelRef": "office-rewrite@llama3.2:3b", + "contentSha256": "d2a13f41da9fe4b1c1ac77bd73f18db1ff393a2c570706e9d0308018ef82f256", + "policySha256": "5ebc47cbb9475e64eb84aae73b2f4e39cee5a4eba4486cf4012c171a5693a0c2", + "targeting": { "deviceClass": ["laptop"], "osVersionRange": null, "locale": null }, + "baseEntryRef": "urn:srcos:model-carry-entry:llama32-3b-base", + "baseContentSha256": "326bf933ccae9bd3e98fe7e85dc4dea4124436199e298200c5f86abc62b43df8" + } + ], + "notes": "INVALID: adapter pins a base content hash that does not match the base entry (silent adapter/base drift)." +} diff --git a/examples/model-carry-manifest.adapter-missing-base.invalid.json b/examples/model-carry-manifest.adapter-missing-base.invalid.json new file mode 100644 index 0000000..0514264 --- /dev/null +++ b/examples/model-carry-manifest.adapter-missing-base.invalid.json @@ -0,0 +1,25 @@ +{ + "schemaVersion": "v0.1", + "kind": "ModelCarryManifest", + "manifestId": "urn:srcos:model-carry-manifest:laptop-safe", + "version": 1, + "prevManifestSha256": null, + "integrity": { + "contentHashAlgorithm": "sha256", + "transportMustBeIntegrityChecked": true, + "integrityFailureIsHardStop": true + }, + "entries": [ + { + "entryId": "urn:srcos:model-carry-entry:office-rewrite-adapter", + "entryKind": "adapter", + "modelRef": "office-rewrite@llama3.2:3b", + "contentSha256": "d2a13f41da9fe4b1c1ac77bd73f18db1ff393a2c570706e9d0308018ef82f256", + "policySha256": "5ebc47cbb9475e64eb84aae73b2f4e39cee5a4eba4486cf4012c171a5693a0c2", + "targeting": { "deviceClass": ["laptop"], "osVersionRange": null, "locale": null }, + "baseEntryRef": "urn:srcos:model-carry-entry:llama32-3b-base", + "baseContentSha256": "7372678075509b60673cfe1df40dc56801ace22cad6e56277cf4a0b4a359d8e7" + } + ], + "notes": "INVALID: adapter is bound to a base entry that is not present in the manifest." +} diff --git a/examples/model-carry-manifest.integrity-not-hardstop.invalid.json b/examples/model-carry-manifest.integrity-not-hardstop.invalid.json new file mode 100644 index 0000000..3e14d5f --- /dev/null +++ b/examples/model-carry-manifest.integrity-not-hardstop.invalid.json @@ -0,0 +1,25 @@ +{ + "schemaVersion": "v0.1", + "kind": "ModelCarryManifest", + "manifestId": "urn:srcos:model-carry-manifest:laptop-safe", + "version": 1, + "prevManifestSha256": null, + "integrity": { + "contentHashAlgorithm": "sha256", + "transportMustBeIntegrityChecked": true, + "integrityFailureIsHardStop": false + }, + "entries": [ + { + "entryId": "urn:srcos:model-carry-entry:llama32-3b-base", + "entryKind": "base", + "modelRef": "llama3.2:3b", + "contentSha256": "7372678075509b60673cfe1df40dc56801ace22cad6e56277cf4a0b4a359d8e7", + "policySha256": "af853ebc3648bc457c414e81b29f88c40931d5c744a89ac42dbd0dacc3aa23b0", + "targeting": { "deviceClass": ["laptop"], "osVersionRange": null, "locale": null }, + "baseEntryRef": null, + "baseContentSha256": null + } + ], + "notes": "INVALID: integrityFailureIsHardStop is false; integrity failure must be a hard stop." +} diff --git a/examples/model-carry-manifest.laptop-safe.json b/examples/model-carry-manifest.laptop-safe.json new file mode 100644 index 0000000..96a5718 --- /dev/null +++ b/examples/model-carry-manifest.laptop-safe.json @@ -0,0 +1,43 @@ +{ + "schemaVersion": "v0.1", + "kind": "ModelCarryManifest", + "manifestId": "urn:srcos:model-carry-manifest:laptop-safe", + "version": 2, + "prevManifestSha256": "4888b1850bddd6715434b02a37378ba12fab3102a20277d2434c92082a749e1a", + "integrity": { + "contentHashAlgorithm": "sha256", + "transportMustBeIntegrityChecked": true, + "integrityFailureIsHardStop": true + }, + "entries": [ + { + "entryId": "urn:srcos:model-carry-entry:llama32-3b-base", + "entryKind": "base", + "modelRef": "llama3.2:3b", + "contentSha256": "7372678075509b60673cfe1df40dc56801ace22cad6e56277cf4a0b4a359d8e7", + "policySha256": "af853ebc3648bc457c414e81b29f88c40931d5c744a89ac42dbd0dacc3aa23b0", + "targeting": { + "deviceClass": ["laptop", "workstation"], + "osVersionRange": ">=1.0.0", + "locale": ["en_US", "en_GB"] + }, + "baseEntryRef": null, + "baseContentSha256": null + }, + { + "entryId": "urn:srcos:model-carry-entry:office-rewrite-adapter", + "entryKind": "adapter", + "modelRef": "office-rewrite@llama3.2:3b", + "contentSha256": "d2a13f41da9fe4b1c1ac77bd73f18db1ff393a2c570706e9d0308018ef82f256", + "policySha256": "5ebc47cbb9475e64eb84aae73b2f4e39cee5a4eba4486cf4012c171a5693a0c2", + "targeting": { + "deviceClass": ["laptop", "workstation"], + "osVersionRange": ">=1.0.0", + "locale": ["en_US"] + }, + "baseEntryRef": "urn:srcos:model-carry-entry:llama32-3b-base", + "baseContentSha256": "7372678075509b60673cfe1df40dc56801ace22cad6e56277cf4a0b4a359d8e7" + } + ], + "notes": "Genuine carry manifest: base plus one task adapter bound to the exact base content hash it was registered against." +} diff --git a/tools/validate_model_carry_manifests.py b/tools/validate_model_carry_manifests.py new file mode 100644 index 0000000..032d867 --- /dev/null +++ b/tools/validate_model_carry_manifests.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Validate SourceOS ModelCarryManifest examples. + +Enforces the content-addressed carry invariants that the model-router relies on: + + 1. SHA-256 is the authoritative content-hash algorithm (FIPS 180-4). + 2. Every entry carries a non-null content hash and policy hash. + 3. Integrity failure is a hard stop, and transport must be integrity-checked. + 4. The manifest is hash-chained: only the genesis manifest (version 1) may omit + the previous-manifest link. + 5. Every adapter is bound to a base entry that exists in the manifest, and the + adapter's pinned base content hash matches that base. This eliminates silent + adapter/base drift by construction. + +Boundary: this checker is structural. It does not fetch artifacts, verify real +cryptographic content, or authorize runtime execution. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +SCHEMA = ROOT / "contracts" / "model-carry-manifest.schema.json" +VALID = ROOT / "examples" / "model-carry-manifest.laptop-safe.json" +INVALID_DRIFT = ROOT / "examples" / "model-carry-manifest.adapter-base-drift.invalid.json" +INVALID_MISSING_BASE = ROOT / "examples" / "model-carry-manifest.adapter-missing-base.invalid.json" +INVALID_SOFT_INTEGRITY = ROOT / "examples" / "model-carry-manifest.integrity-not-hardstop.invalid.json" + +SHA256_RE = re.compile(r"^[a-f0-9]{64}$") + + +class ValidationError(Exception): + pass + + +def load_json(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValidationError(f"{path.name}: expected JSON object") + return payload + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValidationError(message) + + +def is_sha256(value: Any) -> bool: + return isinstance(value, str) and bool(SHA256_RE.match(value)) + + +def validate_schema(schema: dict[str, Any]) -> None: + require(schema.get("$schema") == "https://json-schema.org/draft/2020-12/schema", "schema draft mismatch") + require(schema.get("type") == "object", "schema must describe object") + require(schema.get("additionalProperties") is False, "schema must be closed") + + +def validate_manifest(name: str, record: dict[str, Any]) -> None: + require(record.get("schemaVersion") == "v0.1", f"{name}: schemaVersion must be v0.1") + require(record.get("kind") == "ModelCarryManifest", f"{name}: kind must be ModelCarryManifest") + require(str(record.get("manifestId", "")).startswith("urn:srcos:model-carry-manifest:"), + f"{name}: manifestId must be a SourceOS model-carry-manifest URN") + + version = record.get("version") + require(isinstance(version, int) and version >= 1, f"{name}: version must be an integer >= 1") + + prev = record.get("prevManifestSha256", "missing") + require(prev != "missing", f"{name}: prevManifestSha256 must be present (null only for the genesis manifest)") + if version == 1: + require(prev is None, f"{name}: genesis manifest (version 1) must have null prevManifestSha256") + else: + require(is_sha256(prev), f"{name}: non-genesis manifest must link previous manifest by SHA-256 (hash chain)") + + integrity = record.get("integrity", {}) + require(integrity.get("contentHashAlgorithm") == "sha256", + f"{name}: contentHashAlgorithm must be sha256 (FIPS 180-4 authoritative)") + require(integrity.get("transportMustBeIntegrityChecked") is True, + f"{name}: transportMustBeIntegrityChecked must be true") + require(integrity.get("integrityFailureIsHardStop") is True, + f"{name}: integrityFailureIsHardStop must be true (integrity failure is a hard stop, not a fallback)") + + entries = record.get("entries", []) + require(isinstance(entries, list) and entries, f"{name}: entries must be a non-empty array") + + by_id: dict[str, dict[str, Any]] = {} + for entry in entries: + entry_id = entry.get("entryId", "") + require(str(entry_id).startswith("urn:srcos:model-carry-entry:"), + f"{name}: entryId must be a SourceOS model-carry-entry URN") + require(entry_id not in by_id, f"{name}: duplicate entryId {entry_id}") + require(is_sha256(entry.get("contentSha256")), f"{name}: {entry_id} contentSha256 must be a SHA-256 digest") + require(is_sha256(entry.get("policySha256")), f"{name}: {entry_id} policySha256 must be a SHA-256 digest") + targeting = entry.get("targeting", {}) + require(isinstance(targeting, dict) and targeting.get("deviceClass"), + f"{name}: {entry_id} must carry a targeting predicate with deviceClass") + by_id[entry_id] = entry + + for entry in entries: + entry_id = entry.get("entryId", "") + kind = entry.get("entryKind") + require(kind in ("base", "adapter"), f"{name}: {entry_id} entryKind must be base or adapter") + if kind == "base": + require(entry.get("baseEntryRef") is None and entry.get("baseContentSha256") is None, + f"{name}: {entry_id} base entry must not set baseEntryRef/baseContentSha256") + continue + # adapter: enforce base-version binding + base_ref = entry.get("baseEntryRef") + require(base_ref in by_id, f"{name}: {entry_id} adapter references unknown base {base_ref}") + base = by_id[base_ref] + require(base.get("entryKind") == "base", f"{name}: {entry_id} adapter must bind to a base entry, not {base_ref}") + require(is_sha256(entry.get("baseContentSha256")), + f"{name}: {entry_id} adapter must pin baseContentSha256") + require(entry.get("baseContentSha256") == base.get("contentSha256"), + f"{name}: {entry_id} adapter/base drift: pinned baseContentSha256 does not match base contentSha256") + + +def expect_invalid(path: Path) -> None: + try: + validate_manifest(path.name, load_json(path)) + except ValidationError: + return + raise ValidationError(f"invalid fixture unexpectedly validated: {path.name}") + + +def main() -> int: + try: + validate_schema(load_json(SCHEMA)) + validate_manifest(VALID.name, load_json(VALID)) + expect_invalid(INVALID_DRIFT) + expect_invalid(INVALID_MISSING_BASE) + expect_invalid(INVALID_SOFT_INTEGRITY) + except (OSError, json.JSONDecodeError, ValidationError) as exc: + print(f"ERR: {exc}") + return 1 + print("Model carry manifest validation passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())