diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5407564..3df8131 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -204,5 +204,13 @@ jobs: - name: Run the model-signing provenance example run: | - python -m pip install "weight-custody-manifest[model-signing]>=0.21.0" + python -m pip install "weight-custody-manifest[model-signing]>=0.27.0" python provenance_model_signing.py + + # These existed and were never run by any workflow. They cover the artifact + # digest, which is the value a manifest binds, so a change to it that no + # demo happens to exercise would otherwise land unnoticed. + - name: Run the offline unit tests + run: | + python -m pip install pytest + python -m pytest . -q diff --git a/weight-custody-manifest/channel_binding.py b/weight-custody-manifest/channel_binding.py index 1860b66..86b9a16 100644 --- a/weight-custody-manifest/channel_binding.py +++ b/weight-custody-manifest/channel_binding.py @@ -30,15 +30,16 @@ CompositeEvidence, CpuQuote, Ed25519Signer, + generate_ed25519, + generate_transport_keypair, JsonQuoteParser, KeyBrokerService, + manifest_identity, + open_sealed, QuoteVerifier, SealError, TrustStore, WeightCustodyManifest, - generate_ed25519, - generate_transport_keypair, - open_sealed, ) NOW = datetime(2026, 1, 1, tzinfo=timezone.utc) @@ -154,6 +155,11 @@ def main() -> int: now=lambda: NOW, cpu_quote_verifier=QuoteVerifier(JsonQuoteParser(), trust), require_channel_binding=True, + # weight-custody-manifest 0.27.0 refuses to release unless the manifest's + # identity is pinned out of band. Without it, a caller could present an + # attacker-authored policy that reused a weights hash the broker already + # held and be released against terms nobody agreed. + trusted_manifest_identities=[manifest_identity(manifest)], ) enclave_priv, enclave_pub = generate_transport_keypair() diff --git a/weight-custody-manifest/closed_model_e2e.py b/weight-custody-manifest/closed_model_e2e.py index c97407c..6b34aec 100644 --- a/weight-custody-manifest/closed_model_e2e.py +++ b/weight-custody-manifest/closed_model_e2e.py @@ -25,12 +25,13 @@ from wcm import ( Ed25519Signer, EnclaveSession, + generate_ed25519, KeyBrokerService, + manifest_identity, SoftwareProvider, VerificationContext, - WeightCustodyManifest, - generate_ed25519, verify_manifest, + WeightCustodyManifest, ) @@ -107,7 +108,14 @@ def main() -> int: return 1 rule("Step 2 - Attestation gate: the decryption key releases only into the approved enclave") - kbs = KeyBrokerService({weights_hash: b"the-secret-weight-decryption-key"}) + kbs = KeyBrokerService( + {weights_hash: b"the-secret-weight-decryption-key"}, + # weight-custody-manifest 0.27.0 refuses to release unless the manifest's + # identity is pinned out of band. Without it, a caller could present an + # attacker-authored policy that reused a weights hash the broker already + # held and be released against terms nobody agreed. + trusted_manifest_identities=[manifest_identity(manifest)], + ) challenge = kbs.issue_challenge() evidence = SoftwareProvider().produce( challenge, serving_image_measurement=serving, gpu_measurement="nvidia-rim:golden" diff --git a/weight-custody-manifest/open_model_e2e.py b/weight-custody-manifest/open_model_e2e.py index af826f3..6aa8466 100644 --- a/weight-custody-manifest/open_model_e2e.py +++ b/weight-custody-manifest/open_model_e2e.py @@ -35,17 +35,18 @@ import hashlib from wcm import ( + Ed25519Signer, EnclaveSession, + generate_ed25519, + is_root, KeyBrokerService, KeyWipedError, + manifest_identity, SoftwareProvider, VerificationContext, - WeightCustodyManifest, - generate_ed25519, - Ed25519Signer, - is_root, verify_lineage, verify_manifest, + WeightCustodyManifest, ) @@ -170,7 +171,14 @@ def main() -> None: # ---- Step 2: attestation-gated load ------------------------------------- rule("Step 2 - Attestation gate: only load the CERTIFIED serving stack") - kbs = KeyBrokerService({base.weights_hash: b"loading-key-protects-no-secret-here"}) + kbs = KeyBrokerService( + {base.weights_hash: b"loading-key-protects-no-secret-here"}, + # weight-custody-manifest 0.27.0 refuses to release unless the manifest's + # identity is pinned out of band. Without it, a caller could present an + # attacker-authored policy that reused a weights hash the broker already + # held and be released against terms nobody agreed. + trusted_manifest_identities=[manifest_identity(base)], + ) challenge = kbs.issue_challenge() evidence = SoftwareProvider().produce( # a REAL enclave would produce a hardware quote challenge, diff --git a/weight-custody-manifest/real_lora_custody.py b/weight-custody-manifest/real_lora_custody.py index 091038f..fdc2e9c 100644 --- a/weight-custody-manifest/real_lora_custody.py +++ b/weight-custody-manifest/real_lora_custody.py @@ -36,6 +36,7 @@ SoftwareProvider, VerificationContext, WeightCustodyManifest, + artifact_files, generate_ed25519, generate_transport_keypair, model_signing_digest, @@ -44,7 +45,7 @@ verify_provenance, ) -from real_open_model import artifact_files, build_manifest, resolve_artifact, sha256_artifact +from real_open_model import build_manifest, resolve_artifact, sha256_artifact FORMAT = "wcm-encrypted-derivative/v1" @@ -72,7 +73,7 @@ def sign_artifact(root: pathlib.Path, output: pathlib.Path) -> tuple[str, pathli def _relative_files(root: pathlib.Path) -> Iterable[tuple[pathlib.Path, str]]: - for path in artifact_files(root): + for path in artifact_files(root, follow_symlinks=True): yield path, path.relative_to(root).as_posix() diff --git a/weight-custody-manifest/real_open_model.py b/weight-custody-manifest/real_open_model.py index 36d032c..d523998 100644 --- a/weight-custody-manifest/real_open_model.py +++ b/weight-custody-manifest/real_open_model.py @@ -27,19 +27,24 @@ import hashlib import os import pathlib +import shutil +import tempfile from wcm import ( + artifact_digest, + artifact_files, + combine_shares, # noqa: F401 - kept importable for the curious Ed25519Signer, EnclaveSession, + generate_ed25519, + is_root, KeyBrokerService, + manifest_identity, SoftwareProvider, VerificationContext, - WeightCustodyManifest, - combine_shares, # noqa: F401 - kept importable for the curious - generate_ed25519, - is_root, verify_lineage, verify_manifest, + WeightCustodyManifest, ) @@ -47,47 +52,54 @@ def rule(title: str) -> None: print(f"\n{'=' * 72}\n{title}\n{'=' * 72}") -def artifact_files(path: pathlib.Path) -> list[pathlib.Path]: - """Return the deterministic file inventory bound by the manifest. +def sha256_artifact(path: pathlib.Path) -> str: + """Hash a file or a complete model directory. - A directory snapshot includes weights, shard indexes, configuration, and - tokenizer assets. Hidden Hugging Face cache metadata is deliberately not - part of the serving artifact. + A thin wrapper over ``wcm.artifact_digest``, which is where this recipe + lives as of weight-custody-manifest 0.27.0. It used to be implemented here, + and copied into two Marketplace integrations, with nothing keeping the three + in step; a digest recipe that exists three times stops being one recipe, and + the drift shows up as ``weights_hash`` not matching, which reads as tampered + weights. + + ``follow_symlinks=True`` because a Hugging Face snapshot directory is a + symlink tree: ``snapshot_download`` populates ``snapshots//`` with + links into a content-addressed ``blobs/`` directory in the same cache. The + SDK refuses symlinks by default, which is right for an artifact somebody + handed you and wrong for a cache you just populated yourself. """ - if path.is_file(): - return [path] - files = [ - p for p in path.rglob("*") - if p.is_file() and ".cache" not in p.relative_to(path).parts - ] - if not files: - raise ValueError(f"model artifact contains no files: {path}") - return sorted(files, key=lambda p: p.relative_to(path).as_posix()) - - -def sha256_artifact(path: pathlib.Path, *, flip_first_byte: bool = False) -> str: - """Hash a file or complete model directory with names and boundaries. - - Length-prefixing the relative path and content prevents two different - directory layouts from producing the same concatenated byte stream. + return str(artifact_digest(path, follow_symlinks=True)) + + +def tampered_digest(path: pathlib.Path) -> str: + """The digest a silently modified fork of these weights would have. + + Copies the artifact, flips one byte, hashes the copy and throws it away, so + the model you downloaded is never written to. There is a test asserting the + original bytes are unchanged afterwards. + + This used to be a ``flip_first_byte`` flag threaded through the hash + function, which avoided the copy but meant maintaining a second hashing path + that existed only to fake tampering. Copying costs disk on a demo that has + already downloaded the model, and buys a demonstration where the bytes + genuinely differ rather than one where the arithmetic was nudged. """ - root = path if path.is_dir() else path.parent - h = hashlib.sha256() - flipped = False - for file_path in artifact_files(path): - relative = file_path.relative_to(root).as_posix().encode("utf-8") - h.update(len(relative).to_bytes(8, "big")) - h.update(relative) - h.update(file_path.stat().st_size.to_bytes(8, "big")) - with file_path.open("rb") as fh: - for chunk in iter(lambda: fh.read(1 << 20), b""): - if flip_first_byte and not flipped and chunk: - changed = bytearray(chunk) - changed[0] ^= 0xFF - chunk = bytes(changed) - flipped = True - h.update(chunk) - return "sha256:" + h.hexdigest() + with tempfile.TemporaryDirectory() as tmp: + staged = pathlib.Path(tmp) / "fork" + if path.is_dir(): + shutil.copytree(path, staged, symlinks=False) + else: + staged.mkdir() + shutil.copy2(path, staged / path.name) + + target = artifact_files(staged, follow_symlinks=True)[0] + data = bytearray(target.read_bytes()) + if not data: + raise ValueError(f"cannot tamper an empty file: {target}") + data[0] ^= 0xFF + target.write_bytes(bytes(data)) + + return str(artifact_digest(staged, follow_symlinks=True)) def run_inference(verified_path: pathlib.Path) -> None: @@ -188,7 +200,7 @@ def main() -> int: args = ap.parse_args() artifact_path, model_id = resolve_artifact(args) - files = artifact_files(artifact_path) + files = artifact_files(artifact_path, follow_symlinks=True) size_mb = sum(p.stat().st_size for p in files) / 1e6 rule(f"Real open model: {model_id} ({size_mb:.1f} MB of real weights)") @@ -224,7 +236,14 @@ def main() -> int: # ---- Step 2: attestation-gated load (only the certified stack) ---------- rule("Step 2 - Attestation gate (load only the certified serving stack)") - kbs = KeyBrokerService({base.weights_hash: b"loading-key-not-a-secret-for-open-weights"}) + kbs = KeyBrokerService( + {base.weights_hash: b"loading-key-not-a-secret-for-open-weights"}, + # weight-custody-manifest 0.27.0 refuses to release unless the manifest's + # identity is pinned out of band. Without it, a caller could present an + # attacker-authored policy that reused a weights hash the broker already + # held and be released against terms nobody agreed. + trusted_manifest_identities=[manifest_identity(base)], + ) challenge = kbs.issue_challenge() evidence = SoftwareProvider().produce( challenge, serving_image_measurement=serving, gpu_measurement="nvidia-rim:demo-golden") @@ -273,7 +292,7 @@ def main() -> int: # ---- Step 7: integrity, made concrete ----------------------------------- rule("Step 7 - A silently tampered fork is caught by the hash") - tampered = sha256_artifact(artifact_path, flip_first_byte=True) + tampered = tampered_digest(artifact_path) print("certified weights_hash :", base_hash) print("tampered-fork hash :", tampered) print("tampered matches manifest? :", tampered == base.weights_hash) diff --git a/weight-custody-manifest/refuse_and_wipe.py b/weight-custody-manifest/refuse_and_wipe.py index 0076289..a81f928 100644 --- a/weight-custody-manifest/refuse_and_wipe.py +++ b/weight-custody-manifest/refuse_and_wipe.py @@ -22,11 +22,12 @@ from wcm import ( Ed25519Signer, EnclaveSession, + generate_ed25519, KeyBrokerService, KeyWipedError, + manifest_identity, SoftwareProvider, WeightCustodyManifest, - generate_ed25519, ) @@ -88,7 +89,14 @@ def main() -> None: Ed25519Signer(custodian).sign(manifest.unsigned_dict(), role="custodian", signer=org), ]) - kbs = KeyBrokerService({weights_hash: b"the-weight-decryption-key"}) + kbs = KeyBrokerService( + {weights_hash: b"the-weight-decryption-key"}, + # weight-custody-manifest 0.27.0 refuses to release unless the manifest's + # identity is pinned out of band. Without it, a caller could present an + # attacker-authored policy that reused a weights hash the broker already + # held and be released against terms nobody agreed. + trusted_manifest_identities=[manifest_identity(manifest)], + ) # -- Moment 1: REFUSE ------------------------------------------------------ banner("1. A tampered enclave asks for the key. It gets nothing.") diff --git a/weight-custody-manifest/requirements.txt b/weight-custody-manifest/requirements.txt index fc4d81a..b4126b0 100644 --- a/weight-custody-manifest/requirements.txt +++ b/weight-custody-manifest/requirements.txt @@ -1,4 +1,4 @@ # The Weight Custody Manifest reference SDK, from PyPI. # Covers the offline examples. provenance_model_signing.py additionally needs the # [model-signing] extra: pip install "weight-custody-manifest[model-signing]". -weight-custody-manifest>=0.25.0 +weight-custody-manifest>=0.27.0 diff --git a/weight-custody-manifest/revocation_kill_switch.py b/weight-custody-manifest/revocation_kill_switch.py index 10cf262..c78eec1 100644 --- a/weight-custody-manifest/revocation_kill_switch.py +++ b/weight-custody-manifest/revocation_kill_switch.py @@ -22,11 +22,12 @@ from wcm import ( Ed25519Signer, EnclaveSession, + generate_ed25519, KeyBrokerService, KeyWipedError, + manifest_identity, SoftwareProvider, WeightCustodyManifest, - generate_ed25519, ) NOW = datetime(2026, 1, 1, tzinfo=timezone.utc) @@ -96,7 +97,15 @@ def main() -> int: key = b"the-weight-decryption-key" rule("Step 1 - The model is attested, released, and serving") - kbs = KeyBrokerService({weights_hash: key}, now=lambda: NOW) + kbs = KeyBrokerService( + {weights_hash: key}, + now=lambda: NOW, + # weight-custody-manifest 0.27.0 refuses to release unless the manifest's + # identity is pinned out of band. Without it, a caller could present an + # attacker-authored policy that reused a weights hash the broker already + # held and be released against terms nobody agreed. + trusted_manifest_identities=[manifest_identity(manifest)], + ) decision = _release(kbs, manifest, serving) print("released :", decision.released) session = EnclaveSession.from_release(manifest, decision, now=lambda: NOW) diff --git a/weight-custody-manifest/sovereign_self_custody.py b/weight-custody-manifest/sovereign_self_custody.py index d5f1329..fc49f4e 100644 --- a/weight-custody-manifest/sovereign_self_custody.py +++ b/weight-custody-manifest/sovereign_self_custody.py @@ -29,17 +29,22 @@ from __future__ import annotations import hashlib +import os from wcm import ( + BytearrayMemoryRange, + combine_shares, Ed25519Signer, + generate_ed25519, KeyBrokerService, + manifest_identity, + memory_sweep_public_key, + run_memory_sweep, SoftwareProvider, - VerificationContext, - WeightCustodyManifest, - combine_shares, - generate_ed25519, split_secret, + VerificationContext, verify_manifest, + WeightCustodyManifest, ) @@ -138,18 +143,47 @@ def main() -> int: # The self-custody KBS holds NO weight key (empty entry): the gate proves the # enclave is the builder-measured release coordinator; the key comes from the # quorum, not from the KBS. - kbs = KeyBrokerService({weights_hash: b""}) + # The hostile-owner posture requires a memory-fingerprint challenge, and since + # weight-custody-manifest 0.27.0 that means a SIGNED sweep: the runtime writes + # unpredictable nonce-derived data across its declared range, reads it back in + # a different nonce-derived order, and signs the transcript. The broker needs + # the sweep key to check it. An unsigned fingerprint is refused rather than + # accepted on trust, which is the point: an aliasing attack that can fake the + # readback can also fake an unsigned claim about it. + sweep_key = generate_ed25519().private_key + + kbs = KeyBrokerService( + {weights_hash: b""}, + # 0.27.0 also refuses to release unless the manifest's identity is pinned + # out of band. Without it, a caller could present an attacker-authored + # policy that reused a weights hash the broker already held and be + # released against terms nobody agreed. + trusted_manifest_identities=[manifest_identity(manifest)], + memory_fingerprint_public_key_b64url=memory_sweep_public_key(sweep_key), + ) challenge = kbs.issue_challenge() evidence = SoftwareProvider().produce( challenge, serving_image_measurement=serving, gpu_measurement="nvidia-rim:demo-golden", - include_memory_fingerprint=True, # mandatory in the hostile-owner posture + ) + evidence = evidence.model_copy( + update={ + "memory_fingerprint": run_memory_sweep( + # A stand-in for the enclave's real DRAM range. The algorithm is + # the same; what a bytearray cannot do is prove anything about + # physical memory, which is why WCM issue #79 is still open. + BytearrayMemoryRange(64 * 4096), + challenge_nonce=challenge.nonce, + signing_key=sweep_key, + sweep_secret=os.urandom(32), + ) + } ) decision = kbs.verify_and_release(manifest, evidence) print("attestation gate passed :", decision.released) if not decision.released: - for c in decision.failures(): + for c in decision.failures: print(" failed:", c.name, "-", c.detail) return 1 print("each party checks this attestation before contributing its share.") diff --git a/weight-custody-manifest/test_real_open_model_artifact.py b/weight-custody-manifest/test_real_open_model_artifact.py index c0b1232..3a0869a 100644 --- a/weight-custody-manifest/test_real_open_model_artifact.py +++ b/weight-custody-manifest/test_real_open_model_artifact.py @@ -3,7 +3,9 @@ import pathlib -from real_open_model import artifact_files, sha256_artifact +from wcm import artifact_files + +from real_open_model import sha256_artifact, tampered_digest def test_artifact_hash_is_order_independent_and_covers_all_files(tmp_path: pathlib.Path) -> None: @@ -12,7 +14,7 @@ def test_artifact_hash_is_order_independent_and_covers_all_files(tmp_path: pathl (tmp_path / "model-00001-of-00002.safetensors").write_bytes(b"first") before = sha256_artifact(tmp_path) - assert [p.name for p in artifact_files(tmp_path)] == [ + assert [p.name for p in artifact_files(tmp_path, follow_symlinks=True)] == [ "config.json", "model-00001-of-00002.safetensors", "model-00002-of-00002.safetensors", @@ -33,10 +35,33 @@ def test_cache_metadata_is_not_part_of_serving_artifact(tmp_path: pathlib.Path) assert sha256_artifact(tmp_path) == digest -def test_tamper_simulation_changes_digest_without_writing(tmp_path: pathlib.Path) -> None: +def test_tampered_fork_hashes_differently_and_leaves_the_original_alone( + tmp_path: pathlib.Path, +) -> None: + """The demo must never write to the model somebody just downloaded. + + This used to be a flip_first_byte flag threaded through the hash function. + That avoided the copy but meant a second hashing path existing only to fake + tampering; the recipe now comes from the SDK and the fork is a real one. + """ artifact = tmp_path / "model.safetensors" artifact.write_bytes(b"weights") original = artifact.read_bytes() - assert sha256_artifact(artifact, flip_first_byte=True) != sha256_artifact(artifact) + assert tampered_digest(artifact) != sha256_artifact(artifact) assert artifact.read_bytes() == original + + +def test_tampered_fork_of_a_directory_also_differs(tmp_path: pathlib.Path) -> None: + (tmp_path / "model.safetensors").write_bytes(b"weights") + (tmp_path / "config.json").write_text("{}", encoding="utf-8") + + assert tampered_digest(tmp_path) != sha256_artifact(tmp_path) + + +def test_the_recipe_comes_from_the_sdk() -> None: + """One implementation. It lived here and in two integrations until 0.27.0.""" + import wcm + + assert artifact_files is wcm.artifact_files + assert wcm.ARTIFACT_DIGEST_RECIPE == "wcm-artifact-digest/v1"