diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07ce44e..4a56c3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -265,4 +265,43 @@ jobs: uses: actions/upload-artifact@v4 with: name: template-conformance - path: docs/benchmarks/template-conformance.json \ No newline at end of file + path: docs/benchmarks/template-conformance.json + + assurance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Checkout pcs-core pin + uses: actions/checkout@v4 + with: + repository: SentinelOps-CI/pcs-core + ref: fb588a41a7eab68064429e3c4dfb26c328b9863d + path: pcs-core + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install OVK with assurance pin + env: + OVK_PCS_CORE_PATH: ${{ github.workspace }}/pcs-core + run: | + python -m pip install --upgrade pip + python scripts/sync_package_data.py + pip install --no-cache-dir -e '.[dev]' + pip install --no-cache-dir -e './pcs-core/python' + - name: Verify PCS pin digests + env: + OVK_PCS_CORE_PATH: ${{ github.workspace }}/pcs-core + run: | + python -c "from ovk.assurance.pin import require_pcs_pin, verify_pin_digests, PCS_PIN_COMMIT; print(require_pcs_pin()); print(PCS_PIN_COMMIT); print(verify_pin_digests())" + - name: Assurance pytest + env: + OVK_PCS_CORE_PATH: ${{ github.workspace }}/pcs-core + run: pytest tests/assurance -q --tb=short + - name: PCS conformance suite + working-directory: pcs-core + run: pcs conformance run --suite verifier-assurance + - name: Ordinary surfaces still pass + run: | + python scripts/check_command_surface.py + python scripts/validate_capabilities.py + ovk check --changed-files examples/multi_surface/pr_combined.diff --output-dir .ovk-assurance-ordinary --advisory diff --git a/.gitignore b/.gitignore index 6b6e288..addd46c 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,21 @@ ovk/package_data/ .vscode/ *.swp *~ + +# Local verifier-assurance / baseline scratch (never ship) +.ovk-va00-*/ +.ovk-va00-*.txt +.ovk-assurance-*/ +.ovk-va-*/ +.ovk-check-*/ +evidence-pack-*/ +tmp-assurance-*/ +**/evidence-pack-*/ +**/tmp-assurance-*/ + +# Editor / local backup debris +*.bak +*.bak_* +*.bak_check +docs/**/*.bak* + diff --git a/README.md b/README.md index ef7ef86..73e59ac 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,9 @@ Copy a full consumer example: [`examples/github_workflows/external_consumer.yml` | **Agent server** (`ovk-mcp`) | Let coding agents run checks from their tool loop (`pip install '.[mcp]'`) | | **Templates** ([`templates/`](templates/)) | Reusable rules for common risks (100 included) | | **Benchmark** (`ovk bench`) | Measure regression on agent-style PR diffs | +| **Assurance** (`ovk verifier …`) | Opt-in reproducible verifier runs with PCS-bound evidence packs ([docs/assurance/GUIDE.md](docs/assurance/GUIDE.md)) | + +Ordinary PR checks (`ovk check` / Action / MCP) stay the default. Assurance mode does not replace them and requires a resolved [pcs-core pin](docs/PCS_PIN.md). Agent repair walkthrough: [docs/AGENT_REPAIR_LOOP.md](docs/AGENT_REPAIR_LOOP.md) @@ -184,6 +187,8 @@ Questions, ideas, or a first PR — you are welcome. See [docs/ARCHITECTURE.md]( | Install locally or wire up CI | [INTEGRATION.md](docs/INTEGRATION.md) | | Tune strictness and checker selection | [POLICY.md](docs/POLICY.md) | | Understand checkers and fallbacks | [BACKENDS.md](docs/BACKENDS.md) | +| Use opt-in verifier-assurance mode | [assurance/GUIDE.md](docs/assurance/GUIDE.md) | +| See the PCS pin / merge gate | [PCS_PIN.md](docs/PCS_PIN.md) | | Run or extend the benchmark | [BENCHMARK.md](docs/BENCHMARK.md) | | Roll out on an external OSS repo | [EXTERNAL_PILOT_PLAYBOOK.md](docs/EXTERNAL_PILOT_PLAYBOOK.md) | | See current capabilities | [STATUS.md](docs/STATUS.md) | @@ -197,12 +202,12 @@ Full index: [docs/README.md](docs/README.md) ## Repository layout ``` -schemas/ Shared JSON schemas for configs and evidence -ovk/ Python package — CLI, routing, adapters +schemas/ Shared JSON schemas for configs and evidence (OVK-owned; no PCS VA forks) +ovk/ Python package — CLI, routing, adapters, optional assurance/ templates/ 100 ready-made property templates -examples/ Passing and failing scenarios you can run today +examples/ Passing and failing scenarios; examples/assurance/ for PCS packs benchmarks/ FormalPR-Bench cases and real PR diff corpus -docs/ Guides, specs, and release notes +docs/ Guides, specs, ADRs, and release notes ``` --- diff --git a/docs/ADAPTER_CONTRACT.md b/docs/ADAPTER_CONTRACT.md index 422f661..dbb1508 100644 --- a/docs/ADAPTER_CONTRACT.md +++ b/docs/ADAPTER_CONTRACT.md @@ -122,3 +122,17 @@ Adapters must not: - generate evidence not bound to input digests; - allow an agent-authored PR to alter adapter trust settings without human review; - execute arbitrary untrusted commands outside the adapter sandbox. + +## Ordinary vs assurance adapters + +Ordinary adapters participate in `ovk check` / MCP / Action routing and emit OVK evidence bundles. Assurance-capable adapters are opt-in (`ovk verifier …`), advertise validated assurance sections in their capability manifests, and export PCS-bound profiles/results via a pinned `pcs-core` revision. + +Assurance-capable adapters must additionally: + +- fail closed on missing checkers (typed indeterminate, never a fabricated native pass); +- refuse to upgrade guarantee class during normalization; +- redact secrets before configuration snapshot / profile export; +- keep mutation from overwriting production profiles; +- never label ordinary `ovk.cache.v3` hits as assurance replay. + +Cedar and ordinary Lean `deterministic_fallback` are not assurance-capable. See [assurance/GUIDE.md](assurance/GUIDE.md), [assurance/ADAPTER_INVENTORY.md](assurance/ADAPTER_INVENTORY.md), and [PCS_PIN.md](PCS_PIN.md). diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 95f7c2b..86f6c2d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -164,3 +164,20 @@ This directory becomes machine-readable memory for future agents. OVK is backend-neutral, but it is not semantics-neutral. A policy evaluation, bounded model check, SMT satisfiability query, TLA+ trace, Dafny proof, Verus proof, Lean theorem, and runtime monitor are different guarantee classes. OVK records those distinctions in every capability manifest and evidence object. + +## Verifier-assurance mode (opt-in) + +Distinct from ordinary `ovk check` / MCP / Action. Requires a resolved [PCS pin](PCS_PIN.md); missing pin or unknown schema versions fail closed for assurance paths only. + +```text +ovk verifier describe | snapshot-config | run | replay | mutate | validate-evidence + ↓ +ConfigurationSnapshot → VerifierProfile.v1 (PCS) + ↓ +run_assurance → VerifierInvocationRecord.v1 + VerificationResult.v1 + evidence/ + ↓ +replay → VerifierReplayReport.v1 (immutable; no production overwrite) +mutate → VerifierMutationManifest.v1 (production_prohibition always true) +``` + +Assurance-capable backends live under `ovk/adapters/assurance/` and register only for the verifier registry (not ordinary lane routing). Ordinary `ovk.cache.v3` hits are never labeled as assurance replay. See [assurance/GUIDE.md](assurance/GUIDE.md) and [adr/0001-verifier-assurance-architecture.md](adr/0001-verifier-assurance-architecture.md). diff --git a/docs/BACKENDS.md b/docs/BACKENDS.md index 8f4407a..1ef86e5 100644 --- a/docs/BACKENDS.md +++ b/docs/BACKENDS.md @@ -1,6 +1,6 @@ # Backend Execution Guide -OVK exposes a common evidence contract across ten formal-methods backends. Their execution depth is not uniform. This document is the authoritative statement of what each backend actually executes in v1.2.0 RC. +OVK exposes a common evidence contract across ten formal-methods backends. Their execution depth is not uniform. This document is the authoritative statement of what each backend actually executes in the current `1.2.1` package line (ordinary mode). Assurance-capable backends are listed separately below and are opt-in via `ovk verifier`. ## Execution maturity @@ -14,11 +14,26 @@ OVK exposes a common evidence contract across ten formal-methods backends. Their | `kani` | Deterministic Rust-harness contract evaluator | No | Native Kani execution is not implemented | | `dafny` | Deterministic proof-obligation contract evaluator | No | Native Dafny verification is not implemented | | `verus` | Deterministic verified-Rust contract evaluator | No | Native Verus verification is not implemented | -| `lean` | Deterministic theorem-obligation contract evaluator | No | Native Lean checking is not implemented | +| `lean` | Deterministic theorem-obligation contract evaluator | No | Native Lean checking is not implemented; ordinary adapter is **not** assurance_capable | | `alloy` | Deterministic relational-model contract evaluator | No | Native Alloy analysis is not implemented | A binary-presence or version probe is never labeled as native verification. Evidence artifacts record `used_native_binary`, the guarantee type, assumptions, and limits. +### Assurance-capable backends (opt-in `ovk verifier`) + +These are separate from ordinary lane routing. See [assurance/GUIDE.md](assurance/GUIDE.md). + +| `backend_id` | Real checker | Guarantee class | Notes | +|---|---|---|---| +| `auth-state-predicate` | Exact predicate engine over declared state | observational | VA-06 | +| `pytest-suite` | `python -m pytest` + junit | runtime_observed | VA-07 | +| `opa-policy` | `opa eval` | certificate_checked | VA-08; missing opa → indeterminate | +| `lean-pfcore` | `lean` / optional lake | formally_checked | VA-09; ordinary Lean fallback stays non-assurance | +| `sql-state-diff` | SQLite digest diff | observational | VA-10 | +| `model-judge` | Judge client (CI contract fake) | empirically_measured | VA-11; stochastic; no guarantee upgrade | + +Cedar is never assurance_capable in this programme. + ## CI tiers ### Native execution required @@ -51,7 +66,7 @@ TLA+, Kani, Dafny, Verus, Lean, and Alloy remain non-blocking integration surfac Capability manifests live under `adapters/*/capability.json` and are packaged with the wheel. They support intent/backend ranking and MCP capability discovery. -In v1.2.0 RC, router output is advisory metadata. Core lane obligations still execute their lane evaluator, and the selected generic backend does not yet control compilation or execution. Evidence records `routing_enforced: false` until the backend-selection control plane is implemented. +Lane routing may be **enforced** when `.verification/config.yml` enables it for a lane (`routing_enforced: true` on evidence). The default product path remains shadow/legacy-authoritative until adopters opt into enforced lane policy — see [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) and [POLICY.md](POLICY.md). Router metadata alone is never a substitute for recorded guarantee type, assumptions, and limits. ## Entry points diff --git a/docs/CHANGELOG_VERIFIER_ASSURANCE.md b/docs/CHANGELOG_VERIFIER_ASSURANCE.md new file mode 100644 index 0000000..0c1a7c3 --- /dev/null +++ b/docs/CHANGELOG_VERIFIER_ASSURANCE.md @@ -0,0 +1,29 @@ +# Verifier-assurance changelog (OVK-VA-00…14) + +## Unreleased — verifier-assurance programme + +### Added + +- **VA-00:** ADR, baseline, adapter inventory, ordinary-vs-assurance freeze, PCS development pin. +- **VA-01…05:** Optional assurance capability manifests, configuration snapshots, invocation/evidence packs, replay, typed mutations, `ovk verifier` CLI. +- **VA-06:** `auth-state-predicate` — exact predicates over declared authoritative state materials. +- **VA-07:** `pytest-suite` — real pytest runner; `runtime_observed` only. +- **VA-08:** `opa-policy` — assurance-capable real `opa eval` (missing opa → indeterminate). Cedar excluded. +- **VA-09:** `lean-pfcore` — real Lean/PF-Core surface; ordinary Lean `deterministic_fallback` remains non-assurance. +- **VA-10:** `sql-state-diff` — SQLite offline before/after digests. +- **VA-11:** `model-judge` — stochastic empirical judge with CI contract fake; cannot upgrade guarantee class. +- **VA-12:** Cross-adapter 14-test conformance harness + `examples/assurance/` packs. +- **VA-13:** Post-freeze adjudication importer with label isolation and audit events. +- **VA-14:** Assurance guide, non-claims, threat-model / BACKENDS / ARCHITECTURE / adapter-contract notes, schema-index PCS pointer. +- Aligned pin/docs/exporters with pcs-core OVK pin surface: `VerifierProfile.v1`, `VerificationResult.v1`, `VerifierInvocationRecord.v1`, `VerifierReplayReport.v1`, `VerifierMutationManifest.v1` at commit `fb588a41a7eab68064429e3c4dfb26c328b9863d`. +- Assurance CI job (`tests/assurance` + PCS conformance + pin digest verify). +- Adversarial secret redaction (value patterns, list recursion) and fail-closed timeout/missing-checker split in VA-12. + +### Compatibility + +- Ordinary `ovk check`, MCP, and GitHub Action contracts unchanged. +- Pin identity for merge-to-main is **satisfied** at the documented pcs-core SHA ([PCS_PIN.md](PCS_PIN.md)). PyPI publish of pcs-core remains pending. + +### Non-claims + +See [assurance/GUIDE.md](assurance/GUIDE.md#non-claims). diff --git a/docs/PCS_PIN.md b/docs/PCS_PIN.md new file mode 100644 index 0000000..179c5ef --- /dev/null +++ b/docs/PCS_PIN.md @@ -0,0 +1,140 @@ +# PCS pin for OVK verifier-assurance + +OVK pins portable verifier-assurance schemas from `pcs-core`. OVK never forks those schemas under `schemas/` or invents parallel `VerifierProfile` / `VerificationResult` types. + +## Gate status + +| Gate | Status | +|---|---| +| Development against this pin | **OPEN** | +| Pin identity (merge-to-main for pin purposes) | **SATISFIED** — OVK pins committed pcs-core revision below | +| PyPI / registry publish of pcs-core | **Pending** — optional extra still uses git/path until a versioned release ships | + +See [ADR 0001](adr/0001-verifier-assurance-architecture.md) for ownership and ordinary-vs-assurance doctrine. + +## Authoritative pin + +| Field | Value | +|---|---| +| Repository | `https://github.com/SentinelOps-CI/pcs-core` | +| Commit SHA | `fb588a41a7eab68064429e3c4dfb26c328b9863d` | + +**Pinned commit:** `fb588a41a7eab68064429e3c4dfb26c328b9863d` + +This SHA is the merge commit of PR #26 on pcs-core (VA schemas + OVK pin-surface + conformance). Prefer the full SHA in CI and install URLs. Pin by SHA, not branch name. PyPI publish is still pending. + +**Important:** a sibling `../pcs-core` checkout on a newer `main` tip is **not** automatically valid for OVK. After pcs-core changes that remove or rename pin-surface schemas, `verify_pin_digests()` fails closed. Always install or check out the SHA above (or set `OVK_PCS_CORE_PATH` to that revision). Do not assume “latest pcs-core main” equals this pin. + +Install from the pin (until PyPI): + +```bash +pip install "pcs-core @ git+https://github.com/SentinelOps-CI/pcs-core@fb588a41a7eab68064429e3c4dfb26c328b9863d#subdirectory=python" +# or sibling editable: +pip install -e "../pcs-core/python" +``` + +## Pin resolution order + +OVK resolves the PCS checkout root in this order: + +1. Environment variable `OVK_PCS_CORE_PATH` (preferred for CI) +2. Environment variable `PCS_CORE_PATH` +3. Sibling directory `../pcs-core` relative to the OVK repository root +4. Installed `pcs-core` package (when a published wheel is available) + +If no pin root is found, assurance validation and PCS export integrity sealing fail closed. + +## Install options + +### Path / editable (local development) + +```bash +# from open-verification-kernel +pip install -e ".[assurance]" +# or explicitly: +pip install -e "../pcs-core/python" +``` + +Optional extra `assurance` in `pyproject.toml` documents the git commit pin. Sibling path remains the local fallback. + +### Future published pin + +When pcs-core releases a version that includes the VA schemas: + +```toml +assurance = ["pcs-core=="] +``` + +Update this document with the exact version and schema digests from that release, then record the pin in CI. + +## Artifact types (pinned names) + +| Artifact type | Schema path under pcs-core | +|---|---| +| `VerifierProfile.v1` | `schemas/VerifierProfile.v1.schema.json` | +| `VerificationResult.v1` | `schemas/VerificationResult.v1.schema.json` | +| `VerifierInvocationRecord.v1` | `schemas/VerifierInvocationRecord.v1.schema.json` | +| `VerifierReplayReport.v1` | `schemas/VerifierReplayReport.v1.schema.json` | +| `VerifierMutationManifest.v1` | `schemas/VerifierMutationManifest.v1.schema.json` | + +Shared definitions: `schemas/verifier_assurance.defs.json`. + +`VerificationResult.v1` is distinct from `VerificationResult.v0`. Do not auto-upgrade. + +Field-shape notes (do not invent local forks): + +- Profiles use `verifier_profile_id` and nested `configuration.*_digest`, plus `integrity.{canonicalization_version,artifact_digest}`. +- Results use `decision` (`accept` / `reject` / `indeterminate_*`) and `execution_status`. +- Digests are `sha256:` + 64 lowercase hex. Integrity is nested; top-level `signature_or_digest` is forbidden on VA roots. + +## Schema digests at pinned commit + +Recorded against pcs-core commit `fb588a41a7eab68064429e3c4dfb26c328b9863d` (2026-07-24). Schema file digests (SHA-256 of file bytes): + +| Schema file | Digest | +|---|---| +| `VerifierProfile.v1.schema.json` | `sha256:a657a63eee47a00419f31008f0adee5559e37fdba2544831e8b297c0a2dbe9bd` | +| `VerificationResult.v1.schema.json` | `sha256:146534a7ebf8ee8cdaecaa57258c0ce11224f50aed1a71196bc7b72d2c5b6d17` | +| `VerifierInvocationRecord.v1.schema.json` | `sha256:3ee1384cd5fae5e08b87870100609a9a9b8cf2502b2c4d92de9dedc1f9ffbc3d` | +| `VerifierReplayReport.v1.schema.json` | `sha256:06660ef51c89385869306c2f1c7f1364bec129b783007cc7a8caa4322582bd3b` | +| `VerifierMutationManifest.v1.schema.json` | `sha256:b82952c1d41ddd151cd71440a5a38f7e768c468c3ff4ae11f3a80325d4cb4819` | +| `verifier_assurance.defs.json` | `sha256:c417accb1b4bc08d6e6f0f98e71ee6e7c87a923d19c5054a18841e7e04eadabb` | + +Recompute digests after any pcs-core schema change: + +```bash +python -c "from pathlib import Path; from hashlib import sha256; root=Path(r'../pcs-core/schemas'); +files=['VerifierProfile.v1.schema.json','VerificationResult.v1.schema.json','VerifierInvocationRecord.v1.schema.json','VerifierReplayReport.v1.schema.json','VerifierMutationManifest.v1.schema.json','verifier_assurance.defs.json']; +[print(f, 'sha256:'+sha256((root/f).read_bytes()).hexdigest()) for f in files]" +``` + +Unknown schema versions and missing pin roots fail closed. Digests are enforced by `ovk.assurance.pin.verify_pin_digests()`. + +## Validation commands + +From a pcs-core checkout (or installed package with bundled schemas): + +```bash +pcs schema check +pcs validate examples/verifier_assurance/VerifierProfile.v1.valid.json +pcs validate examples/verifier_assurance/VerificationResult.v1.valid.json +pcs validate examples/verifier_assurance/VerifierInvocationRecord.v1.valid.json +pcs validate examples/verifier_assurance/VerifierReplayReport.v1.valid.json +pcs validate examples/verifier_assurance/VerifierMutationManifest.v1.valid.json +pcs conformance run --suite verifier-assurance +``` + +From OVK (after pin resolution): + +```bash +ovk verifier validate-evidence path/to/evidence/ +python -c "from ovk.assurance.pin import require_pcs_pin, verify_pin_digests; print(require_pcs_pin()); verify_pin_digests()" +``` + +## Dependency for assurance + +Assurance CLI (`ovk verifier …`) and PCS export/validation require a resolved pin. Ordinary `ovk check` / MCP / Action paths do not require pcs-core. + +## Non-claims + +Pinning PCS schemas does not assert checker correctness. Profiles and results bind digests and decisions only. See pcs-core `docs/verifier-assurance/non-claims.md` and ADR 0001 exclusions. PyPI publish of pcs-core remains a separate release step and is not implied by this pin. diff --git a/docs/README.md b/docs/README.md index 5b95b54..403045e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,7 @@ Use this index as the canonical entry point. Each guide covers one topic; cross- |---|---| | **Adopting in CI** | [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) → [INTEGRATION.md](INTEGRATION.md) → [EXTERNAL_PILOT_PLAYBOOK.md](EXTERNAL_PILOT_PLAYBOOK.md) | | **Contributing code** | [CONTRIBUTING.md](CONTRIBUTING.md) → [ARCHITECTURE.md](ARCHITECTURE.md) → [ADAPTER_CONTRACT.md](ADAPTER_CONTRACT.md) | +| **Verifier-assurance** | [assurance/GUIDE.md](assurance/GUIDE.md) → [PCS_PIN.md](PCS_PIN.md) → [adr/0001-verifier-assurance-architecture.md](adr/0001-verifier-assurance-architecture.md) | | **Maintainers** | [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) → [RELEASE.md](RELEASE.md) → `ovk release-preflight` (release readiness checks) | | **Spec / security review** | [SYSTEM_SPEC.md](SYSTEM_SPEC.md) → [FORMAL_SPEC.md](FORMAL_SPEC.md) → [THREAT_MODEL.md](THREAT_MODEL.md) → [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) | @@ -52,10 +53,23 @@ Use this index as the canonical entry point. Each guide covers one topic; cross- | [ARCHITECTURE.md](ARCHITECTURE.md) | System architecture and runner flows | | [SYSTEM_SPEC.md](SYSTEM_SPEC.md) | System specification | | [FORMAL_SPEC.md](FORMAL_SPEC.md) | Formal properties and security rules | -| [THREAT_MODEL.md](THREAT_MODEL.md) | Threat model | -| [ADAPTER_CONTRACT.md](ADAPTER_CONTRACT.md) | Backend adapter contract | +| [THREAT_MODEL.md](THREAT_MODEL.md) | Threat model (includes assurance threats) | +| [ADAPTER_CONTRACT.md](ADAPTER_CONTRACT.md) | Backend adapter contract (ordinary + assurance rules) | | [ROADMAP.md](ROADMAP.md) | Release history and planned work | +## Verifier-assurance (opt-in) + +Ordinary `ovk check` / MCP / Action are unchanged. Assurance is a separate mode gated on a pcs-core pin. + +| Document | Purpose | +|---|---| +| [assurance/GUIDE.md](assurance/GUIDE.md) | Install, backends, CLI, non-claims (VA-01…14) | +| [assurance/ADAPTER_INVENTORY.md](assurance/ADAPTER_INVENTORY.md) | Ordinary vs assurance adapters and evidence models | +| [adr/0001-verifier-assurance-architecture.md](adr/0001-verifier-assurance-architecture.md) | Ownership, PCS gate, compatibility freeze | +| [PCS_PIN.md](PCS_PIN.md) | Development pin vs published pin; schema digests | +| [baseline/OVK-VA-00-baseline.md](baseline/OVK-VA-00-baseline.md) | Programme-start baseline (+ progress addendum) | +| [CHANGELOG_VERIFIER_ASSURANCE.md](CHANGELOG_VERIFIER_ASSURANCE.md) | Programme changelog | + ## Reference | Document | Purpose | @@ -69,6 +83,7 @@ Use this index as the canonical entry point. Each guide covers one topic; cross- | Document | Purpose | |---|---| +| [RELEASE_NOTES_v1.2.1.md](RELEASE_NOTES_v1.2.1.md) | v1.2.1 changelog | | [RELEASE_NOTES_v1.2.0.md](RELEASE_NOTES_v1.2.0.md) | v1.2.0 changelog | | [RELEASE_NOTES_v1.1.0.md](RELEASE_NOTES_v1.1.0.md) | v1.1.0 changelog | | [RELEASE_NOTES_v1.0.0.md](RELEASE_NOTES_v1.0.0.md) | v1.0.0 changelog | @@ -85,6 +100,9 @@ docs/ INTEGRATION.md # install and GitHub Actions LANES.md, BACKENDS.md, … # reference and operations guides ARCHITECTURE.md, … # design and specification + adr/ # architecture decision records + assurance/ # verifier-assurance inventory and guides + baseline/ # programme baseline records benchmarks/ # committed leaderboard and adoption JSON templates/ # external pilot manifest template ``` diff --git a/docs/SCHEMA_INDEX.md b/docs/SCHEMA_INDEX.md index 5437e88..74ad76f 100644 --- a/docs/SCHEMA_INDEX.md +++ b/docs/SCHEMA_INDEX.md @@ -41,3 +41,20 @@ JSON schemas for OVK objects. See also [ARTIFACTS.md](ARTIFACTS.md). | [external.pilots.registry.schema.json](../schemas/external.pilots.registry.schema.json) | ovk.external_pilots_registry.v1 | `docs/benchmarks/external-pilots-registry.json` | Generated evidence bundles are validated at write time using the canonical Pydantic `EvidenceBundle` model and cross-checked against [verification.bundle.schema.json](../schemas/verification.bundle.schema.json) in CI. Release bundles are additionally checked with `ovk validate-outputs`. Adapter `capability.json` files are validated in CI and `ovk release-preflight` via `scripts/validate_capabilities.py`. + +## Portable PCS verifier-assurance schemas (not forked here) + +OVK does **not** vendor PCS VA schemas under `schemas/`. Those types are owned by `pcs-core` and resolved through the pin documented in [PCS_PIN.md](PCS_PIN.md) / `ovk.assurance.pin`. + +| PCS artifact | Consumed by OVK | Typical pack / CLI filename | +|---|---|---| +| `VerifierProfile.v1` | `ovk verifier snapshot-config` / run | `verifier_profile.pcs.json` | +| `VerificationResult.v1` | `ovk verifier run` / validate-evidence | `verification_result.pcs.json` | +| `VerifierInvocationRecord.v1` | run / validate-evidence / replay | `invocation.json` (PCS body) | +| `VerifierReplayReport.v1` | `ovk verifier replay` | `replay_report.pcs.json` | +| `VerifierMutationManifest.v1` | `ovk verifier mutate` | `*.mutation.json` beside mutated profile | +| Opaque `invocation_ref` | On `VerificationResult.v1` | `invocation_id` + `invocation_digest` only | + +Pack layout extras (`compiled_obligation.json`, `raw/`, `normalized/`, `provenance/`) are OVK-local sidecars; they are not PCS schema types. Digests and required PCS artifact types are enforced by `ovk.assurance.pin.verify_pin_digests()` against [PCS_PIN.md](PCS_PIN.md). + +`verification.capability.schema.json` may include **optional** assurance sections; missing sections mean ordinary-only. See [assurance/GUIDE.md](assurance/GUIDE.md). diff --git a/docs/STATUS.md b/docs/STATUS.md index fcaf882..5189167 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,21 +1,22 @@ # OVK Status -Current version: **v1.2.0**. +Current package version: **v1.2.1** (see `pyproject.toml` / `ovk.core.release_metadata.OVK_VERSION`). Working-tree adoption tracking may describe a future `v1.3.0-rc.1` candidate in [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md); that does not change the installed package version until an rc cut. ## Summary OVK is a solver-agnostic verification layer for AI-agent pull requests. It turns a PR diff into structured evidence: what was checked, what passed, what failed, and what still needs a human. -v1.2 improves adoption tooling (release status dashboard, example workflows, quality checks on all five check types). v1.1 added a realistic diff benchmark and required native checker CI for OPA, Z3, CBMC, and Cedar. +v1.2 improved adoption tooling (release status dashboard, example workflows, quality checks on all five check types). v1.1 added a realistic diff benchmark and required native checker CI for OPA, Z3, CBMC, and Cedar. Opt-in verifier-assurance (`ovk verifier …`) is documented under [assurance/GUIDE.md](assurance/GUIDE.md). -Changelog: [RELEASE_NOTES_v1.2.0.md](RELEASE_NOTES_v1.2.0.md). +Changelog: [RELEASE_NOTES_v1.2.1.md](RELEASE_NOTES_v1.2.1.md) (and [RELEASE_NOTES_v1.2.0.md](RELEASE_NOTES_v1.2.0.md) for the 1.2.0 baseline). ## Package -- Version `1.2.0` — install with `pip install -e '.[dev]'` from a checkout. -- Optional groups: `dev`, `solvers`, `mcp`. +- Version `1.2.1` — install with `pip install -e '.[dev]'` from a checkout. +- Optional groups: `dev`, `solvers`, `mcp`, `assurance` (git/path pin `fb588a41a7eab68064429e3c4dfb26c328b9863d` until PyPI; see [PCS_PIN.md](PCS_PIN.md)). - Primary command: `ovk check --changed-files `. -- GitHub Action pin: `@v1.2.0` with optional `OVK_PACKAGE_VERSION=1.2.0` for PyPI installs. +- Opt-in assurance CLI: `ovk verifier …` (requires resolved pcs-core pin; does not change ordinary CI). +- GitHub Action pin: `@v1.2.1` with optional `OVK_PACKAGE_VERSION=1.2.1` for PyPI installs. ## Check types @@ -43,6 +44,7 @@ Backends (10 formal tools): [BACKENDS.md](BACKENDS.md). OPA, Z3, CBMC, and Cedar | `ovk repair-suggest` | Suggest fix classes from evidence | | `ovk template list/show/apply` | Browse and apply property templates | | `ovk release-preflight` | Run release readiness checks | +| `ovk verifier …` | Opt-in assurance: describe, snapshot-config, run, replay, mutate, validate-evidence | | `ovk-mcp` | MCP server for agent integrations | ## Quick commands @@ -69,11 +71,16 @@ FormalPR-Bench scores correctness, routing, repair hints, and realistic PR diffs - In-repo examples: [PILOT_CASE_STUDIES.md](PILOT_CASE_STUDIES.md). - External repos: [EXTERNAL_PILOT_PLAYBOOK.md](EXTERNAL_PILOT_PLAYBOOK.md). +## Verifier-assurance (opt-in) + +Reproducible verifier execution with PCS-bound profiles and evidence packs. Separate from ordinary PR checks. Guide: [assurance/GUIDE.md](assurance/GUIDE.md). Architecture: [adr/0001-verifier-assurance-architecture.md](adr/0001-verifier-assurance-architecture.md). Pinned to pcs-core `fb588a41a7eab68064429e3c4dfb26c328b9863d` (git SHA install; PyPI pending). See [PCS_PIN.md](PCS_PIN.md). + ## Known limitations See [RELEASE.md](RELEASE.md#known-limitations). Experimental / non-strict compiler paths: [EXPERIMENTAL_PATHS.md](EXPERIMENTAL_PATHS.md). Adoption readiness: [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md). ## Upgrading +- **v1.2.0 → v1.2.1:** [RELEASE_NOTES_v1.2.1.md](RELEASE_NOTES_v1.2.1.md) and [MIGRATION.md](MIGRATION.md) - **v1.1.0 → v1.2.0:** [RELEASE_NOTES_v1.2.0.md](RELEASE_NOTES_v1.2.0.md) and [MIGRATION.md](MIGRATION.md#upgrade-from-v110-to-v120) - **Older versions:** [MIGRATION.md](MIGRATION.md) diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index f1e355a..6311a67 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -59,3 +59,19 @@ OVK focuses on agentic engineering risk. The first threat model covers AI coding - Adapter manifests with assumptions and limits. - Adversarial regression tests for self-disable and timeout cases. - Optional in-toto-compatible evidence predicate. + +## Verifier-assurance (opt-in) threats + +Assurance mode (`ovk verifier …`) adds PCS-bound profiles, invocation records, and evidence packs. Additional threats and controls: + +| Threat | Description | Required response | +|---|---|---| +| Fabricated native pass | Adapter claims assurance without a real checker | Forbid stubs; missing checker → typed indeterminate | +| Guarantee laundering | Normalization upgrades observational/empirical to formal | Clamp guarantee class; conformance test | +| Secret leakage in profiles | Env/config secrets exported in snapshots | Redact before serialize; adversarial tests | +| Replay forgery | Claim matched replay under drift or missing deps | Fail closed; stochastic cannot claim matched | +| Production profile overwrite | Mutation silently replaces production profile | Always `production_prohibition`; path refuse | +| Hidden-label leakage | Holdout/adjudication labels enter verifier inputs or packs | Post-freeze import only; refuse label keys | +| Mode confusion | Treat ordinary cache replay as assurance replay | Separate artifacts and docs (ADR 0001) | + +See [assurance/GUIDE.md](assurance/GUIDE.md) and [adr/0001-verifier-assurance-architecture.md](adr/0001-verifier-assurance-architecture.md). diff --git a/docs/adr/0001-verifier-assurance-architecture.md b/docs/adr/0001-verifier-assurance-architecture.md new file mode 100644 index 0000000..47bc154 --- /dev/null +++ b/docs/adr/0001-verifier-assurance-architecture.md @@ -0,0 +1,175 @@ +# ADR 0001: Verifier-assurance architecture (OVK-VA-00) + +## Status + +Accepted. Ordinary CI/MCP/Action surfaces remain frozen as documented below. Assurance runtime (OVK-VA-01…14) is implemented in-tree as an **opt-in** surface and is gated on the PCS pin documented in [PCS_PIN.md](../PCS_PIN.md). + +- Development against the sibling/path pin: **OPEN** +- Pin identity (merge-to-main for pin purposes): **SATISFIED** at pcs-core `fb588a41a7eab68064429e3c4dfb26c328b9863d` (see [PCS_PIN.md](../PCS_PIN.md)); install/check out that SHA (do not assume a newer sibling `main` tip); PyPI publish still pending + +## Context + +OVK is a heterogeneous verification kernel: lane evaluators, optional external backends, an `ovk.cache.v3` result cache, evidence bundles, and in-toto-style attestation. A verifier-assurance programme requires reproducible checker invocation, immutable configuration profiles, evidence packs, replay, typed mutation, and export to portable PCS artifacts — without breaking ordinary CI mode or inventing local PCS schema forks. + +Sibling `pcs-core` is the authority for portable assurance schemas. FormalPR-Holdout and private stores remain the authority for hidden labels. PF-Core remains the authority for small machine-checked trace predicates. + +## Decision + +### Ownership + +| Owner | Owns | Does not own | +|---|---|---| +| **OVK** | Checker invocation; configuration snapshots; invocation records; evidence-pack layout and writers; typed mutations; thin mapping from OVK snapshots/records to pinned PCS types; ordinary CI/MCP/Action execution | Portable PCS schemas; hidden-label stores; RL/training; campaign orchestration | +| **PCS (`pcs-core`)** | Portable schemas and conformance fixtures (`VerifierProfile.v1`, `VerificationResult.v1`, and related VA artifact types as PCS names them); Canonical JSON / digest rules; `pcs validate` semantics | Checker binaries; OVK runtime config; hidden labels | +| **PF-Core** | Small machine-checked trace predicates and Lean trust-kernel checks | Campaign stats; OVK lane policy; VA orchestration | +| **FormalPR-Holdout / private stores** | Hidden labels and private adjudication rationale | Public OVK evidence packs | + +OVK **pins and validates** PCS schemas. OVK **never forks** PCS schemas into `schemas/` or invents parallel `VerifierProfile` / `VerificationResult` types under an OVK `$id`. + +Lean/PF-Core assurance adapters **invoke** PF-Core / Lean; they do not redefine kernel semantics. + +OVK imports **only post-freeze adjudication references**. Active or hidden FormalPR-Holdout labels must remain inaccessible to policies and verifiers (see [HOLDOUT_LABEL_SEPARATION.md](../HOLDOUT_LABEL_SEPARATION.md)). + +### Explicit exclusions + +OVK verifier-assurance does **not** implement or host: + +- Reinforcement learning or training loops +- Attack orchestration / red-team campaign runners +- Campaign statistics or optimization-campaign control planes +- Environment simulation frameworks +- A hidden-label database or any store of FormalPR-Holdout ground truth inside OVK + +Those concerns belong to other repos or private partner systems. PCS may define portable *record* types for some of them; OVK does not execute those workflows. + +### Ordinary CI mode vs assurance mode + +| Mode | Entry points | Default? | Contract | +|---|---|---|---| +| **Ordinary CI** | `ovk check`, `ovk verify`, `ovk ci`, GitHub Action (`action.yml`), MCP (`ovk-mcp`) | Yes | Behavior and public schemas frozen per compatibility section below | +| **Assurance** | `ovk verifier …`; adapters that advertise assurance capability fields | No (opt-in) | Requires assurance-capable adapters + PCS pin; missing checker → typed indeterminate | + +Existing `deterministic_fallback` external adapters remain **non-assurance**. They must not appear in assurance manifests as replayable PCS emitters. Cedar remains out of the assurance policy tranche (OPA only for policy engines, via `opa-policy`). + +### No stubs / placeholders for assurance-capable adapters + +An adapter that claims assurance capability must not ship: + +- `TODO` / empty `pass` production paths +- Fabricated evidence or mocked production paths labeled as native/assurance +- Silent `deterministic_fallback` results labeled as native or assurance-capable +- Fixtures that bypass the behavior under test + +Missing checker → typed indeterminate. Unsupported mutation → explicit failure. If a planned adapter cannot meet the real-checker bar, it does not merge. + +### PCS-first gate + +OVK-VA-00 (this ADR + baseline) established ownership and the freeze. + +**Development gate (OPEN):** OVK-VA-01…14 may be developed and tested against the documented sibling/path pin in [PCS_PIN.md](../PCS_PIN.md) (`OVK_PCS_CORE_PATH` / `PCS_CORE_PATH` / `../pcs-core` / installed `pcs-core`). + +**Merge-to-main gate (still required):** OVK-VA-01…14 must not merge to `main` until all of the following are true: + +1. `pcs-core` **publishes** (committed, registered, releasable) at least `VerifierProfile.v1`, `VerificationResult.v1`, and authoritative conformance fixtures for those types (plus invocation/replay/mutation artifact schemas as PCS names them). +2. OVK **pins** that published PCS revision (preferred: optional extra `assurance = ["pcs-core=="]` **or** documented git commit pin with schema digests verified in CI — see [PCS_PIN.md](../PCS_PIN.md)). +3. Assurance CI jobs can run PCS fixture validation against OVK-emitted packs; unknown schema versions fail closed. + +The merge pin identity is the committed pcs-core SHA `fb588a41a7eab68064429e3c4dfb26c328b9863d` documented in PCS_PIN.md. PyPI remains pending. + +### Backward-compatibility freeze + +Public contracts that must remain valid through the assurance programme unless a dedicated, versioned migration ADR supersedes this freeze: + +**CLI commands** (from `ovk.core.release_metadata.SUPPORTED_COMMANDS`): + +- `ovk init`, `ovk check`, `ovk doctor`, `ovk run`, `ovk generate-test`, `ovk repair-suggest`, `ovk ci` +- `ovk auth-obligation`, `ovk infra-exposure`, `ovk ci-secrets`, `ovk deployment-state` +- `ovk release-bundle`, `ovk release-preflight`, `ovk evidence-quality`, `ovk validate-outputs`, `ovk verify` +- `ovk extract-workflow`, `ovk plan`, `ovk infer`, `ovk template list|show|apply`, `ovk bench`, `ovk pilot` + +`ovk verifier …` subcommands are additive and must not change the semantics of the list above. + +**GitHub Action** (`action.yml`): existing input names and defaults remain stable. + +**MCP tools** (`ovk.mcp_server`): existing tool names and payload shapes remain valid for existing callers. + +**Schemas that must remain valid** (consumers and CI already validate): + +- `verification.evidence.schema.json` / `verification.evidence.v2.schema.json` / `verification.evidence.v3.schema.json` +- `verification.bundle.schema.json` / `verification.bundle.v2.schema.json` +- `verification.capability.schema.json` (assurance sections are **optional**; missing ⇒ ordinary-only) +- `verification.obligation.schema.json`, `backend.routing.schema.json`, `backend.execution.schema.json` +- `verification.result.schema.json`, `verification.intent.schema.json`, `verification.config.schema.json` +- `attestation.statement.schema.json`, `attestation.envelope.schema.json`, `artifact.manifest.schema.json`, `provenance.schema.json` +- `evidence.quality.schema.json`, `preflight.report.schema.json`, lane input schemas + +Additive optional fields are allowed when they are backward-compatible. Removing or renaming required fields requires a versioned schema bump and migration note. + +### Cache replay vs assurance invocation replay + +| Mechanism | Schema / artifact | Purpose | Not | +|---|---|---|---| +| **Ordinary cache replay** | `ovk.cache.v3` / `CachedBackendExecution` | Reuse a prior control-plane attempt under the same content-addressed key | Not a PCS assurance replay; not an evidence pack | +| **Assurance invocation replay** | PCS `VerifierReplayReport.v1` + OVK invocation evidence | Reconstruct checker + immutable config, detect drift, re-execute when deterministic | Not a silent cache hit; fail closed on drift | + +These mechanisms must remain distinct in code and docs. A cache hit must never be labeled as assurance replay. + +## PCS gate status + +Assessed against pcs-core commit `fb588a41a7eab68064429e3c4dfb26c328b9863d` on 2026-07-24 (VA schemas landed via pcs-core PR #26; PyPI not published). OVK pins that SHA explicitly; a newer pcs-core tip may drop or rename pin-surface schemas and must not be treated as this pin: + +| Requirement | Status | +|---|---| +| Development / RC pin (`docs/PCS_PIN.md` + optional `assurance` extra) | **CLOSED** on committed SHA `fb588a41a7eab68064429e3c4dfb26c328b9863d` | +| Published (pushed / PyPI) VA schemas | **Pending** — pin is a git commit install; no PyPI release yet | +| Merge-to-main pin identity for VA-01–14 | **Satisfied for pin purposes** at `fb588a41a7eab68064429e3c4dfb26c328b9863d`; distributed consumers still need a PyPI release (or git SHA install) | + +**Gate verdict: pin identity CLOSED at the documented SHA; PyPI still PENDING.** + +OVK must not vendor or fork PCS schemas. See [PCS_PIN.md](../PCS_PIN.md). + +## Implementation map (VA-01…14) + +| PR | Deliverable | Location | +|---|---|---| +| VA-01 | Optional assurance capability fields | `schemas/verification.capability.schema.json`, adapters | +| VA-02 | Snapshot + PCS profile export | `ovk/assurance/snapshot.py`, `pcs_export.py` | +| VA-03 | Invocation + evidence packs | `ovk/assurance/invocation.py`, `evidence_pack.py` | +| VA-04 | Replay engine | `ovk/assurance/replay.py` | +| VA-05 | Typed mutation | `ovk/assurance/mutation.py` | +| VA-06…11 | Six assurance backends | `ovk/adapters/assurance/` | +| VA-12 | Conformance harness + examples | `ovk/assurance/conformance.py`, `examples/assurance/` | +| VA-13 | Post-freeze adjudication import | `ovk/assurance/adjudication.py` | +| VA-14 | Docs / non-claims / quality bar | `docs/assurance/`, this ADR, threat model | + +## Baseline and inventory pointers + +- Durable baseline record: [../baseline/OVK-VA-00-baseline.md](../baseline/OVK-VA-00-baseline.md) +- Adapter / evidence inventory: [../assurance/ADAPTER_INVENTORY.md](../assurance/ADAPTER_INVENTORY.md) +- User guide: [../assurance/GUIDE.md](../assurance/GUIDE.md) + +## Pre-existing failures (not repaired in VA-00) + +Baseline `pytest` reported 6 failures that do **not** block this documentation ADR. They were **not** repaired in VA-00. Recorded in the baseline artifact for follow-up outside the assurance programme: + +- `tests/test_cache_worker_control_plane.py::test_worker_rejects_non_positive_wall_budget` +- `tests/test_runtime_cache_regimes.py` (3 tests) +- `tests/test_trusted_policy_loading.py` (2 tests) + +## Consequences + +- Ordinary CI remains the default trust surface; assurance is opt-in and PCS-gated. +- Assurance modules live under `ovk/assurance/` and the `verifier` Typer subgroup; they must not alter ordinary `ovk check` / MCP / Action semantics. +- Adapter PRs that claim assurance without a real checker are rejected. +- Programme board tracks PyPI (or equivalent versioned) publish of the pinned pcs-core revision as the remaining distribution step; pin identity for VA-01…14 is the documented SHA in [PCS_PIN.md](../PCS_PIN.md). + +## Related + +- [PCS_PIN.md](../PCS_PIN.md) +- [ADAPTER_INVENTORY.md](../assurance/ADAPTER_INVENTORY.md) +- [GUIDE.md](../assurance/GUIDE.md) +- [OVK-VA-00-baseline.md](../baseline/OVK-VA-00-baseline.md) +- [BACKENDS.md](../BACKENDS.md) +- [HOLDOUT_LABEL_SEPARATION.md](../HOLDOUT_LABEL_SEPARATION.md) +- [ARCHITECTURE.md](../ARCHITECTURE.md) +- [SCHEMA_INDEX.md](../SCHEMA_INDEX.md) diff --git a/docs/assurance/ADAPTER_INVENTORY.md b/docs/assurance/ADAPTER_INVENTORY.md new file mode 100644 index 0000000..9407639 --- /dev/null +++ b/docs/assurance/ADAPTER_INVENTORY.md @@ -0,0 +1,147 @@ +# OVK adapter and evidence inventory + +Inventory of verification adapters, evidence models, and the distinction between ordinary cache replay and assurance invocation replay. Companion to [ADR 0001](../adr/0001-verifier-assurance-architecture.md), the [VA-00 baseline](../baseline/OVK-VA-00-baseline.md), and the [assurance guide](GUIDE.md). + +Baseline SHA for the programme start: `e7097351c9a09a2e9f3626fb981f089b14b8cb4d`. Implementation status below reflects the working tree after OVK-VA-01…14. + +## Protocols + +| Protocol | Location | Role | +|---|---|---| +| `BackendAdapter` | `ovk/adapters/contract.py` | Authoritative control-plane protocol: `manifest → can_handle → compile → fingerprint → run → normalize → explain` | +| `ExternalAdapter` | `ovk/adapters/contract.py` | Legacy wave1/wave2 optional backend surface (dict/legacy dataclasses) | +| `VerificationAdapter` | `ovk/adapters/base.py` | Older protocol alias still present for historical callers | +| `BaseExternalAdapter` | `ovk/adapters/external/base_adapter.py` | Deterministic-fallback skeleton for external backends | + +Optional assurance methods (snapshot / run-with-evidence / replay / mutate) live under `ovk/assurance/` and assurance-only adapters in `ovk/adapters/assurance/`. Ordinary adapters remain ordinary unless they advertise a validated assurance section in their capability manifest. + +## Registered ordinary `BackendAdapter` instances + +Built via lane/domain registry builders in `ovk/adapters/*/__init__.py` and routed from `ovk/core/routing_pipeline.py` / `ovk/core/adapter_runtime.py`. + +### Domain / native-or-deterministic backends + +| `backend_id` | Class | Registry builder | Execution character | +|---|---|---|---| +| `opa-native` | `OpaNativeSelfProtectionAdapter` | `build_self_protection_registry` | Real `opa eval` when OPA present; self-protection domain | +| `self-protection-deterministic` | `SelfProtectionDeterministicAdapter` | `build_self_protection_registry` | Deterministic policy evaluator | +| `z3-native` | `Z3NativeAuthorizationAdapter` | `build_authorization_registry` | Native Z3 SMT when solver present | +| `authorization-deterministic` | `AuthorizationDeterministicAdapter` | `build_authorization_registry` | Deterministic authorization evaluator | +| `infrastructure-deterministic` | `InfrastructureDeterministicAdapter` | `build_infrastructure_registry` | Deterministic exposure-graph check | +| `ci-secrets-deterministic` | `CiSecretsDeterministicAdapter` | `build_ci_secrets_registry` | Deterministic secrets-boundary check | +| `deployment-deterministic` | `DeploymentDeterministicAdapter` | `build_deployment_registry` | Deterministic approval state machine | + +### Lane wrapper backends + +| `backend_id` | Class | Registry | +|---|---|---| +| `lane-self-protection` | `SelfProtectionLaneAdapter` | `build_default_lane_registry` | +| `lane-authorization` | `AuthorizationLaneAdapter` | `build_default_lane_registry` | +| `lane-infrastructure` | `InfrastructureLaneAdapter` | `build_default_lane_registry` | +| `lane-ci-secrets` | `CiSecretsLaneAdapter` | `build_default_lane_registry` | +| `lane-deployment` | `DeploymentLaneAdapter` | `build_default_lane_registry` | + +Lane wrappers adapt existing evaluators onto `BackendAdapter` for registry use. They are ordinary-mode surfaces, not assurance emitters. + +## External / optional backends (`ExternalAdapter`) + +Capability manifests live under `adapters/*/capability.json` (packaged with the wheel). Maturity summary matches [BACKENDS.md](../BACKENDS.md). + +| Backend name | Adapter class | Native execution today? | Assurance-capable? | +|---|---|---|---| +| `opa` | lane/native path via `opa-native` + OPA helpers | Yes (when `opa` installed) | No — ordinary path; assurance uses `opa-policy` | +| `z3` | `Z3NativeAuthorizationAdapter` / z3 helpers | Yes (when Z3 installed) | No | +| `cbmc` | `CbmcAdapter` | Yes for explicit/template harness | No | +| `cedar` | `CedarAdapter` | No (deterministic + CLI probe) | **No** — excluded from assurance policy tranche | +| `tla+` | `TlaAdapter` | No (deterministic contract) | No | +| `kani` | `KaniAdapter` | No | No | +| `dafny` | `DafnyAdapter` | No | No | +| `verus` | `VerusAdapter` | No | No | +| `lean` | `LeanAdapter` | No (`deterministic_fallback`) | No — assurance uses `lean-pfcore` | +| `alloy` | `AlloyAdapter` | No | No | + +**Rule (ADR 0001):** `deterministic_fallback` adapters stay non-assurance. No stub assurance adapters. + +## Assurance-capable backends (opt-in) + +Registered only for the verifier registry (`ovk/assurance/registry.py`), not ordinary lane routing. See [GUIDE.md](GUIDE.md). + +| `backend_id` | Module | VA | Guarantee class | Missing tool | +|---|---|---|---|---| +| `auth-state-predicate` | `ovk/adapters/assurance/auth_state.py` | VA-06 | observational | n/a (pure predicate) | +| `pytest-suite` | `ovk/adapters/assurance/pytest_suite.py` | VA-07 | runtime_observed | typed indeterminate | +| `opa-policy` | `ovk/adapters/assurance/opa_policy.py` | VA-08 | certificate_checked | typed indeterminate | +| `lean-pfcore` | `ovk/adapters/assurance/lean_pfcore.py` | VA-09 | formally_checked | typed indeterminate | +| `sql-state-diff` | `ovk/adapters/assurance/sql_diff.py` | VA-10 | observational | n/a (SQLite stdlib) | +| `model-judge` | `ovk/adapters/assurance/model_judge.py` | VA-11 | empirically_measured | CI contract fake; stochastic | + +## Evidence and execution models + +| Model | Location | Schema / version | Role | +|---|---|---|---| +| `VerificationEvidence` | `ovk/core/models.py` | `ovk.evidence.v1` (+ v2 preview fields) | Per-lane / per-intent evidence | +| `EvidenceBundle` | `ovk/core/models.py` | `ovk.bundle.v1` | Aggregate decision + evidence list | +| Evidence v2/v3 JSON Schema | `schemas/verification.evidence.v2.schema.json`, `…v3…` | schema files | Control-plane enriched evidence | +| Bundle v2 JSON Schema | `schemas/verification.bundle.v2.schema.json` | schema file | Bundles of v2 evidence | +| `RawBackendExecution` | `ovk/core/execution_models.py` | `backend.execution.schema.json` | Raw checker attempt | +| `NormalizedBackendResult` | `ovk/core/execution_models.py` | same | Normalized status / guarantee / counterexamples | +| `CachedBackendExecution` | `ovk/core/execution_models.py` | **`ovk.cache.v3`** | Provenance-preserving cache payload | +| `ObligationExecutionRecord` | `ovk/core/execution_models.py` | backend execution schema | Full compile/route/execute record | +| `BackendCapabilityManifest` | `ovk/core/execution_models.py` | `verification.capability.schema.json` | Adapter capability declaration (optional assurance sections) | +| `BackendEnvironmentFingerprint` | `ovk/core/execution_models.py` | execution models | Thin env fingerprint (ordinary); assurance uses richer snapshots | +| Release attestation | `ovk/core/attestation.py`, `release_bundle.py` | attestation schemas | in-toto-style release artifacts | +| Evidence quality / invariants | `ovk/core/evidence_invariants.py` | quality / invariant schemas | Post-run quality reporting | + +### Assurance artifacts (OVK runtime → PCS export) + +| Artifact | OVK module | PCS type / packing | +|---|---|---| +| Configuration snapshot / profile | `ovk/assurance/snapshot.py`, `pcs_export.py` | PCS `VerifierProfile.v1` → `verifier_profile.pcs.json` | +| Verification result | `ovk/assurance/pcs_export.py` | PCS `VerificationResult.v1` → `verification_result.pcs.json` | +| Invocation record | `ovk/assurance/invocation.py`, `evidence_pack.py` | PCS `VerifierInvocationRecord.v1` → `invocation.json` | +| Evidence pack layout | `ovk/assurance/evidence_pack.py` | OVK-local dirs/sidecars (`raw/`, `normalized/`, `provenance/`, `compiled_obligation.json`) | +| Replay report | `ovk/assurance/replay.py` | PCS `VerifierReplayReport.v1` → `replay_report.pcs.json` | +| Mutation manifest | `ovk/assurance/mutation.py` | PCS `VerifierMutationManifest.v1` | + +Opaque `invocation_ref` on results carries `invocation_id` + `invocation_digest` only. PCS schemas are **never** forked under OVK `schemas/`. Authoritative type set and digests: [PCS_PIN.md](../PCS_PIN.md) / `ovk.assurance.pin`. + +## Cache replay vs assurance invocation replay + +### Ordinary: `ovk.cache.v3` cache replay + +- Implemented in `ovk/core/result_cache.py` (`CACHE_SCHEMA_VERSION = "ovk.cache.v3"`). +- Payload type: `CachedBackendExecution`. +- Purpose: content-addressed reuse of a prior backend attempt under matching key components. +- Scope: ordinary control-plane / CI performance and provenance continuity. +- **Not** a PCS assurance replay and **not** an evidence pack. + +### Assurance: invocation replay (OVK-VA-04) + +- Validates an invocation record, reconstructs checker + immutable configuration, detects missing deps / config drift, re-executes when deterministic, compares digests, emits a PCS replay report. +- Stochastic backends must preserve nondeterminism declarations rather than fake bit-identical replay. +- Drift fails closed. +- Entry point: `ovk verifier replay …`. + +**Invariant:** labeling a cache hit as assurance replay is forbidden. + +## Surfaces + +| Surface | Entry | Mode | +|---|---|---| +| CLI | `ovk check`, `ovk verify`, lane commands | Ordinary (default) | +| GitHub Action | `action.yml` | Ordinary | +| MCP | `ovk-mcp` / `ovk.mcp_server` | Ordinary | +| Release | `ovk release-bundle`, `ovk validate-outputs` | Ordinary | +| Assurance CLI | `ovk verifier describe\|snapshot-config\|run\|replay\|mutate\|validate-evidence` | Assurance (opt-in) | +| Conformance | `ovk.assurance.conformance` / `tests/assurance/` | Assurance | +| Adjudication import | `ovk.assurance.adjudication` | Assurance (post-freeze only) | + +## Related + +- [GUIDE.md](GUIDE.md) +- [ADR 0001](../adr/0001-verifier-assurance-architecture.md) +- [BACKENDS.md](../BACKENDS.md) +- [ADAPTER_CONTRACT.md](../ADAPTER_CONTRACT.md) +- [SCHEMA_INDEX.md](../SCHEMA_INDEX.md) +- [ARTIFACTS.md](../ARTIFACTS.md) +- [PCS_PIN.md](../PCS_PIN.md) diff --git a/docs/assurance/GUIDE.md b/docs/assurance/GUIDE.md new file mode 100644 index 0000000..5df05a6 --- /dev/null +++ b/docs/assurance/GUIDE.md @@ -0,0 +1,127 @@ +# Verifier-assurance guide (OVK-VA-01…14) + +Opt-in assurance mode for reproducible verifier execution, configuration snapshots, evidence packs, replay, and typed mutation. Ordinary `ovk check` / MCP / Action paths are unchanged and do not require `pcs-core`. + +## Modes + +| Mode | Entry points | Requires pcs-core? | Default? | +|---|---|---|---| +| Ordinary CI | `ovk check`, `ovk verify`, GitHub Action, MCP | No | Yes | +| Assurance | `ovk verifier …` | Yes (resolved pin) | No (opt-in) | + +## Install + +### Ordinary (default) + +```bash +pip install -e . +# or: pip install open-verification-kernel +``` + +### Assurance (opt-in) + +```bash +pip install -e ".[assurance]" +# installs pcs-core at the git commit documented in PCS_PIN.md +# Sibling editable remains a local fallback: +pip install -e "../pcs-core/python" +# or set OVK_PCS_CORE_PATH to a pcs-core checkout +``` + +See [PCS_PIN.md](../PCS_PIN.md). Pin identity for merge-to-main is satisfied at the documented pcs-core commit. PyPI publish of pcs-core remains pending. + +Optional native toolchains (not required for ordinary CI): + +| Tool | Assurance backend | Missing behavior | +|---|---|---| +| `opa` | `opa-policy` | typed `missing_checker` indeterminate | +| `lean` / `lake` | `lean-pfcore` | typed `missing_checker` indeterminate | +| `pytest` | `pytest-suite` | typed `missing_checker` if pytest import fails | + +## CLI + +```bash +ovk verifier describe --backend auth-state-predicate +ovk verifier snapshot-config --backend auth-state-predicate --out profile.json +ovk verifier run --backend auth-state-predicate --input input.json --evidence-dir evidence/ +ovk verifier replay invocation.json --evidence-dir evidence/ --backend auth-state-predicate +ovk verifier mutate --profile profile.json --mutation mutation.json --out mutated.json +ovk verifier validate-evidence evidence/ +``` + +## Assurance-capable backends + +| `backend_id` | VA | Mechanism | Guarantee class | Determinism | +|---|---|---|---|---| +| `auth-state-predicate` | VA-06 | Exact predicates over declared authoritative state | observational | deterministic | +| `pytest-suite` | VA-07 | Real pytest runner + junit capture | runtime_observed | deterministic | +| `opa-policy` | VA-08 | Real `opa eval` | certificate_checked | deterministic | +| `lean-pfcore` | VA-09 | Real `lean` / optional `lake env lean` | formally_checked | deterministic | +| `sql-state-diff` | VA-10 | SQLite before/after state digests | observational | deterministic | +| `model-judge` | VA-11 | Stochastic judge (CI contract fake) | empirically_measured | stochastic | + +Ordinary `opa-native`, `deployment-deterministic`, and external `lean` (`deterministic_fallback`) are **not** assurance_capable. Cedar is never assurance_capable in this programme. + +## Evidence packs + +Layout under `evidence/`: + +```text +invocation.json # VerifierInvocationRecord.v1 (PCS) +verifier_profile.pcs.json # VerifierProfile.v1 (PCS) +verification_result.pcs.json # VerificationResult.v1 (PCS) +compiled_obligation.json # OVK-local sidecar +raw/ normalized/ provenance/ # OVK-local sidecars +``` + +Replay writes `replay_report.pcs.json` (`VerifierReplayReport.v1`). Mutation writes a sealed `VerifierMutationManifest.v1` beside the mutated profile. See [PCS_PIN.md](../PCS_PIN.md). + +Committed examples (no OPA/Lean toolchain required): [examples/assurance/](../../examples/assurance/). OPA / Lean packs are produced locally with `ovk verifier run` when those tools are installed; missing tools must yield typed indeterminate, never fabricated passes. + +PCS artifacts in the pack are validated fail-closed against the pin. Results bind an opaque `invocation_ref` (`invocation_id` + `invocation_digest` only). + +Validate: + +```bash +ovk verifier validate-evidence examples/assurance/auth-state-predicate +``` + +## Conformance (VA-12) + +Shared 14-test matrix in `ovk.assurance.conformance` / `tests/assurance/test_conformance_and_adjudication.py`. Any adapter claiming `assurance_capable=True` must pass (tool-absent paths only via typed indeterminate, never fabricated passes). + +## Adjudication import (VA-13) + +Post-freeze only. Hidden/active holdout labels are refused in adjudication refs and `ovk verifier run` inputs. Audit events append to an optional JSONL log. + +## Non-claims + +OVK verifier-assurance does **not**: + +- run RL training, attack orchestration, or campaign statistics +- simulate production environments beyond declared fixtures +- treat model-judge scores as formal or certificate-checked results +- upgrade guarantee class during normalization +- invent native passes when `opa` / `lean` / checkers are missing +- promote Cedar or ordinary Lean `deterministic_fallback` to assurance +- expose FormalPR-Holdout hidden labels to policies or verifiers +- replace ordinary `ovk check` evidence with PCS packs (modes are separate) +- claim that a development path-pin of pcs-core is a published supply-chain pin + +## Quality bar + +- `ruff` / pytest for assurance packages +- PCS schema validation fail-closed when pin present; fail-closed when pin required and missing +- Secret redaction before snapshot/profile export +- Reproducible evidence packs with nested integrity digests +- Supply-chain: pin pcs-core; do not fork PCS schemas under `schemas/` +- Mutation never overwrites production profiles; overwrite attempts refuse closed + +## Related + +- [ADAPTER_INVENTORY.md](ADAPTER_INVENTORY.md) +- [PCS_PIN.md](../PCS_PIN.md) +- [ADR 0001](../adr/0001-verifier-assurance-architecture.md) +- [CHANGELOG_VERIFIER_ASSURANCE.md](../CHANGELOG_VERIFIER_ASSURANCE.md) +- [THREAT_MODEL.md](../THREAT_MODEL.md) +- [BACKENDS.md](../BACKENDS.md) diff --git a/docs/baseline/OVK-VA-00-baseline.md b/docs/baseline/OVK-VA-00-baseline.md new file mode 100644 index 0000000..c8aaa55 --- /dev/null +++ b/docs/baseline/OVK-VA-00-baseline.md @@ -0,0 +1,133 @@ +# OVK-VA-00 baseline record + +Durable baseline for the OVK verifier-assurance programme, recorded with [ADR 0001](../adr/0001-verifier-assurance-architecture.md). + +## Repository pin + +| Field | Value | +|---|---| +| Repository | `open-verification-kernel` | +| HEAD commit SHA | `e7097351c9a09a2e9f3626fb981f089b14b8cb4d` | +| HEAD subject | `chore: update FormalPR-Bench badge [skip ci]` | +| Recorded (UTC) | `2026-07-24T16:54:55Z` | +| Host platform | `Windows-11-10.0.26200-SP0` | + +## Toolchain versions + +| Tool | Version | Notes | +|---|---|---| +| Python | 3.13.11 | Local Windows Python used for the baseline measurement | +| ruff | 0.11.2 | from `ruff --version` / `importlib.metadata` | +| pytest | 8.3.5 | from `pytest --version` | +| open-verification-kernel (installed) | 1.2.1 | editable install of this workspace for CLI smoke | + +## Commands run + +### 1. `pytest` (full suite) + +```text +pytest -q --tb=no +``` + +| Metric | Result | +|---|---| +| Passed | 673 | +| Failed | 6 | +| Skipped | 22 | +| Warnings | 12 (deprecation / pytest-asyncio config) | +| Duration | ~167 s | +| Exit code | **1** (failures present) | + +Failed tests (pre-existing; **not repaired** in VA-00 — they do not block this documentation ADR): + +1. `tests/test_cache_worker_control_plane.py::test_worker_rejects_non_positive_wall_budget` +2. `tests/test_runtime_cache_regimes.py::test_enforced_execution_does_not_reuse_legacy_flat_cache` +3. `tests/test_runtime_cache_regimes.py::test_enforced_evidence_has_only_authoritative_typed_routing` +4. `tests/test_runtime_cache_regimes.py::test_shadow_mode_uses_only_control_plane_namespace_cache` +5. `tests/test_trusted_policy_loading.py::test_changed_policy_is_loaded_from_base_revision` +6. `tests/test_trusted_policy_loading.py::test_changed_policy_without_base_material_uses_safe_builtin` + +Honest summary: the suite is **not fully green** at this baseline SHA. Core ordinary paths exercised by the majority of tests pass; the six failures cluster on cache-regime/worker-budget and trusted-policy loading behavior. + +### 2. `ovk doctor` + +```text +ovk doctor +``` + +| Metric | Result | +|---|---| +| Overall `passed` | **false** | +| Exit code | **1** | + +Core / packaging checks that passed: + +- `ovk_version` → `1.2.1` +- `python` → `Python 3.13.11` +- `example_manifest` → 5 lanes in example manifest +- `template_library` → 100 intent templates +- `schema_resources` → core packaged schemas present +- `lean` → present on PATH (`elan`) +- `github_token` → optional locally +- `platform` → recorded +- `verification_config_schema` → config absent (optional OK) + +Checks that failed (expected on a fresh checkout without optional toolchains / `.verification/`): + +- Optional native tools missing from PATH: `opa`, `z3`, `cedar`, `tlc`, `kani`, `dafny`, `verus`, `cbmc`, `alloy`, `cosign` +- `verification_dir` → `.verification` missing + +Doctor failure here is an environment layout / optional-toolchain signal, not a kernel regression by itself. + +### 3. `ovk check` smoke + +```text +ovk check \ + --changed-files examples/multi_surface/pr_combined.diff \ + --output-dir .ovk-va00-check \ + --advisory +``` + +| Metric | Result | +|---|---| +| Exit code | **0** (advisory) | +| Merge recommendation | `block` (expected for the multi-surface failing fixture) | +| Lanes exercised | self-protection (`opa`, pass), ci_secrets (fail), authorization/`z3` path (fail), infrastructure (fail) | +| Artifacts written | `ovk-evidence.json`, `ovk-pr-comment.md`, `ovk-attestation.json`, `ovk-artifact-manifest.json`, `ovk-evidence-quality.json`, `ovk-attestation-envelope.json` | + +Smoke verdict: **pass** (command succeeded and wrote the standard artifact set). + +## PCS gate snapshot (external) + +Historical assessment at VA-00 recording time used a pre-pin draft checkout. **Do not treat the historical table below as current.** Authoritative current pin: [PCS_PIN.md](../PCS_PIN.md) (`fb588a41a7eab68064429e3c4dfb26c328b9863d`). + +| Artifact | At VA-00 recording | Current (addendum) | +|---|---|---| +| `VerifierProfile.v1` | Draft / unpublished | Present at pinned SHA | +| `VerificationResult.v1` | Draft / unpublished | Present at pinned SHA | +| Invocation / replay / mutation VA schemas | Draft / unpublished | Present at pinned SHA (required by `ovk.assurance.pin`) | +| OVK pin document | Development | [PCS_PIN.md](../PCS_PIN.md); optional `assurance` extra | + +**PCS gate (current): pin `fb588a41a7eab68064429e3c4dfb26c328b9863d` CLOSED for pin identity; PyPI PENDING.** Sibling `../pcs-core` on a newer tip may drift; always resolve the documented SHA. See ADR 0001. + +## Addendum — programme progress (not part of the VA-00 measurement) + +This baseline remains a historical record of the programme start SHA and command results. It is not rewritten when later PRs land. + +As of 2026-07-24 (pin retarget): + +| Item | Status | +|---|---| +| OVK-VA-01–14 implementation | Present under `ovk/assurance/`, `ovk/adapters/assurance/`, `tests/assurance/`, `examples/assurance/`, `docs/assurance/` | +| Ordinary surfaces | Unchanged; see ADR compatibility freeze | +| pcs-core pin | **Committed** SHA `fb588a41a7eab68064429e3c4dfb26c328b9863d` — see [PCS_PIN.md](../PCS_PIN.md); PyPI pending | +| Merge to `main` | **Pin identity satisfied** for pin purposes; PyPI still pending for versioned installs (git SHA install works) | + + +Authoritative current docs: [assurance/GUIDE.md](../assurance/GUIDE.md), [assurance/ADAPTER_INVENTORY.md](../assurance/ADAPTER_INVENTORY.md), [CHANGELOG_VERIFIER_ASSURANCE.md](../CHANGELOG_VERIFIER_ASSURANCE.md). + +## Related + +- [ADR 0001 — Verifier-assurance architecture](../adr/0001-verifier-assurance-architecture.md) +- [Adapter inventory](../assurance/ADAPTER_INVENTORY.md) +- [Assurance guide](../assurance/GUIDE.md) diff --git a/examples/assurance/README.md b/examples/assurance/README.md new file mode 100644 index 0000000..4247ffe --- /dev/null +++ b/examples/assurance/README.md @@ -0,0 +1,27 @@ +# Examples: assurance evidence packs + +Generated offline packs for assurance-capable backends. Ordinary CI does not consume these. Paths inside packs are repo-relative (no host absolute paths). + +| Directory | Backend | Notes | +|---|---|---| +| `auth-state-predicate/` | `auth-state-predicate` | Exact authoritative-state predicates | +| `pytest-suite/` | `pytest-suite` | Observational pytest run | +| `sql-state-diff/` | `sql-state-diff` | SQLite fixture diff | +| `model-judge/` | `model-judge` | Contract-fake stochastic judge | + +Each pack includes PCS `VerifierInvocationRecord.v1`, `VerifierProfile.v1`, and `VerificationResult.v1` plus OVK-local sidecars. OPA (`opa-policy`) and Lean (`lean-pfcore`) packs are omitted from the committed tree because those toolchains are optional. Generate locally into a gitignored scratch dir: + +```bash +ovk verifier run --backend opa-policy --input --evidence-dir .ovk-assurance-opa/ +ovk verifier run --backend lean-pfcore --input --evidence-dir .ovk-assurance-lean/ +``` + +When `opa` or `lean` is missing, the run must return a typed indeterminate decision — never a fabricated accept. + +Validate a committed pack (requires the resolved pcs-core pin from [docs/PCS_PIN.md](../../docs/PCS_PIN.md)): + +```bash +ovk verifier validate-evidence examples/assurance/auth-state-predicate +``` + +See [docs/assurance/GUIDE.md](../../docs/assurance/GUIDE.md). diff --git a/examples/assurance/auth-state-predicate/compiled_obligation.json b/examples/assurance/auth-state-predicate/compiled_obligation.json new file mode 100644 index 0000000..4b0468b --- /dev/null +++ b/examples/assurance/auth-state-predicate/compiled_obligation.json @@ -0,0 +1,24 @@ +{ + "backend_id": "auth-state-predicate", + "input": { + "authoritative_state": { + "roles": [ + "reader", + "admin" + ], + "status": "approved" + }, + "predicates": [ + { + "expected": "approved", + "kind": "field_equals", + "path": "status" + }, + { + "kind": "set_contains", + "member": "admin", + "path": "roles" + } + ] + } +} diff --git a/examples/assurance/auth-state-predicate/invocation.json b/examples/assurance/auth-state-predicate/invocation.json new file mode 100644 index 0000000..e08abfa --- /dev/null +++ b/examples/assurance/auth-state-predicate/invocation.json @@ -0,0 +1,48 @@ +{ + "artifact_type": "VerifierInvocationRecord.v1", + "assumptions": [], + "canonicalization_version": "v1", + "command": { + "argv": [ + "ovk-adapter-auth-state-predicate", + "exact-predicate" + ], + "cwd": ".", + "env_digest": "sha256:360192276458ec94bbc60a175efd2e5b1e9b8981250956a6e612300dd01eaf89", + "limits": { + "timeout_ms": 1000 + } + }, + "compiled_obligation": { + "digest": "sha256:600df83e94767b9881eecfed1aec55eebd4cf2b0dbd8447756e64b37ab1abf6e", + "media_type": "application/json", + "path": "compiled_obligation.json" + }, + "completed_at": "2026-07-24T21:35:55Z", + "exit": { + "code": 0, + "kind": "exited" + }, + "guarantee_class": "observational", + "input_digest": "sha256:89fad2250c40a8782c969ec588c11841ae2e6b78315d46f5c1e98873cafeae9c", + "integrity": { + "artifact_digest": "sha256:d4f4dc8d7eb41ef0599b28cd691728c2fe3aa448d592a7970a0f3f46e4fc5d44", + "canonicalization_version": "v1" + }, + "invocation_id": "vi-auth-state-predicate-89fad225", + "limits": [], + "normalized_result_digest": "sha256:bb817eb885a0f0cbedd59adbc523d9170e6f25769c6d5508195e3758664291e6", + "normalizer_version": "ovk.normalize.v1", + "producer": "OVK", + "producer_version": "1.2.1", + "profile_ref": { + "artifact_type": "VerifierProfile.v1", + "profile_digest": "sha256:610a2c95dd4f61ea801796b8489d63876783b146363cd647e694b21792e3fce5", + "verifier_profile_id": "vp-auth-state-predicate-3386a8ab" + }, + "raw_backend_result_digest": "sha256:ffd8e70cc31a62dbff8e80ded7b4d48ca64815f45b2906b0fbe355af26aad2c4", + "schema_version": "v1", + "started_at": "2026-07-24T21:35:55Z", + "stderr_digest": "sha256:12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126", + "stdout_digest": "sha256:2faed1418d56b08b21a2d29fdac2030e7e87272342c23a144a4d52554c23a182" +} diff --git a/examples/assurance/auth-state-predicate/normalized/result.json b/examples/assurance/auth-state-predicate/normalized/result.json new file mode 100644 index 0000000..3a994b4 --- /dev/null +++ b/examples/assurance/auth-state-predicate/normalized/result.json @@ -0,0 +1,33 @@ +{ + "decision": "accept", + "guarantee_class": "observational", + "predicate_results": [ + { + "actual": "approved", + "expected": "approved", + "index": 0, + "kind": "field_equals", + "passed": true, + "path": "status", + "predicate": { + "expected": "approved", + "kind": "field_equals", + "path": "status" + } + }, + { + "index": 1, + "kind": "set_contains", + "member": "admin", + "passed": true, + "path": "roles", + "predicate": { + "kind": "set_contains", + "member": "admin", + "path": "roles" + } + } + ], + "state_digest": "sha256:5d16d6a4be324402478388cae54ad185559238cb77a8045e79206652792697b3", + "status": "pass" +} diff --git a/examples/assurance/auth-state-predicate/provenance/digests.json b/examples/assurance/auth-state-predicate/provenance/digests.json new file mode 100644 index 0000000..9c7717a --- /dev/null +++ b/examples/assurance/auth-state-predicate/provenance/digests.json @@ -0,0 +1,8 @@ +{ + "input_digest": "sha256:89fad2250c40a8782c969ec588c11841ae2e6b78315d46f5c1e98873cafeae9c", + "normalized_digest": "sha256:bb817eb885a0f0cbedd59adbc523d9170e6f25769c6d5508195e3758664291e6", + "producer": "OVK", + "producer_version": "1.2.1", + "profile_digest": "sha256:610a2c95dd4f61ea801796b8489d63876783b146363cd647e694b21792e3fce5", + "raw_digest": "sha256:ffd8e70cc31a62dbff8e80ded7b4d48ca64815f45b2906b0fbe355af26aad2c4" +} diff --git a/examples/assurance/auth-state-predicate/raw/backend_result.json b/examples/assurance/auth-state-predicate/raw/backend_result.json new file mode 100644 index 0000000..f8698cf --- /dev/null +++ b/examples/assurance/auth-state-predicate/raw/backend_result.json @@ -0,0 +1,36 @@ +{ + "authoritative_state_digest": "sha256:5d16d6a4be324402478388cae54ad185559238cb77a8045e79206652792697b3", + "evaluation": { + "message": null, + "ok": true, + "results": [ + { + "actual": "approved", + "expected": "approved", + "index": 0, + "kind": "field_equals", + "passed": true, + "path": "status", + "predicate": { + "expected": "approved", + "kind": "field_equals", + "path": "status" + } + }, + { + "index": 1, + "kind": "set_contains", + "member": "admin", + "passed": true, + "path": "roles", + "predicate": { + "kind": "set_contains", + "member": "admin", + "path": "roles" + } + } + ], + "unsupported": false + }, + "predicate_count": 2 +} diff --git a/examples/assurance/auth-state-predicate/raw/stderr.txt b/examples/assurance/auth-state-predicate/raw/stderr.txt new file mode 100644 index 0000000..e69de29 diff --git a/examples/assurance/auth-state-predicate/raw/stdout.txt b/examples/assurance/auth-state-predicate/raw/stdout.txt new file mode 100644 index 0000000..fc07d38 --- /dev/null +++ b/examples/assurance/auth-state-predicate/raw/stdout.txt @@ -0,0 +1 @@ +accept diff --git a/examples/assurance/auth-state-predicate/verification_result.pcs.json b/examples/assurance/auth-state-predicate/verification_result.pcs.json new file mode 100644 index 0000000..dd5cfcf --- /dev/null +++ b/examples/assurance/auth-state-predicate/verification_result.pcs.json @@ -0,0 +1,49 @@ +{ + "artifact_type": "VerificationResult.v1", + "check_groups": [ + { + "checks": [ + { + "check_id": "auth-state-predicate-predicate", + "mandatory": true, + "status": "passed" + } + ], + "kind": "authority" + } + ], + "claim_ids": [ + "claim.auth_state.exact_predicate" + ], + "created_at": "2026-07-24T21:35:55Z", + "decision": "accept", + "declared_input_guarantee_class": "observational", + "execution_status": "completed", + "guarantee_class": "observational", + "input_bundle_digest": "sha256:89fad2250c40a8782c969ec588c11841ae2e6b78315d46f5c1e98873cafeae9c", + "integrity": { + "artifact_digest": "sha256:a6304325b3bf038c793cac6c62d6e3cd42ff810acf2c0256d07c8b5334982054", + "canonicalization_version": "v1" + }, + "invocation_ref": { + "invocation_digest": "sha256:d4f4dc8d7eb41ef0599b28cd691728c2fe3aa448d592a7970a0f3f46e4fc5d44", + "invocation_id": "vi-auth-state-predicate-89fad225" + }, + "normalization_applied": true, + "normalized_result_digest": "sha256:bb817eb885a0f0cbedd59adbc523d9170e6f25769c6d5508195e3758664291e6", + "normalizer_version": "ovk.normalize.v1", + "producer": "OVK", + "producer_version": "1.2.1", + "raw_backend_output_digest": "sha256:ffd8e70cc31a62dbff8e80ded7b4d48ca64815f45b2906b0fbe355af26aad2c4", + "resource_limits": { + "wall_time_ms": 0 + }, + "schema_version": "v1", + "source_commit": "e7097351c9a09a2e9f3626fb981f089b14b8cb4d", + "source_repo": "https://github.com/fraware/open-verification-kernel", + "verification_result_id": "vr-auth-state-predicate-89fad225", + "verifier_profile": { + "profile_digest": "sha256:610a2c95dd4f61ea801796b8489d63876783b146363cd647e694b21792e3fce5", + "verifier_profile_id": "vp-auth-state-predicate-3386a8ab" + } +} diff --git a/examples/assurance/auth-state-predicate/verifier_profile.pcs.json b/examples/assurance/auth-state-predicate/verifier_profile.pcs.json new file mode 100644 index 0000000..b8ba61b --- /dev/null +++ b/examples/assurance/auth-state-predicate/verifier_profile.pcs.json @@ -0,0 +1,78 @@ +{ + "applicability": { + "status": "active", + "valid_from": "2026-07-24T21:35:55Z" + }, + "artifact_type": "VerifierProfile.v1", + "assumptions": [ + "Authoritative state materials are caller-declared and complete." + ], + "canonical_configuration": { + "timeout_ms": 1000 + }, + "claim_surface": { + "decision_space": [ + "accept", + "reject", + "indeterminate_execution_error", + "indeterminate_out_of_scope", + "indeterminate_insufficient_evidence" + ], + "guarantee_class": "observational", + "out_of_scope_claim_ids": [ + "claim.formal.full_correctness" + ], + "supported_claim_ids": [ + "claim.auth_state.exact_predicate" + ] + }, + "configuration": { + "config_digest": "sha256:1018384d24a8f0e4355abb757c1f2a09cd29e654b16233620871bc251b7a46de", + "ensemble_digest": null, + "model_digest": null, + "policy_digest": null, + "prompt_digest": null, + "resource_limit_digest": "sha256:1018384d24a8f0e4355abb757c1f2a09cd29e654b16233620871bc251b7a46de", + "rubric_digest": null, + "test_suite_digest": null, + "threshold_digest": null + }, + "created_at": "2026-07-24T21:35:55Z", + "execution_controls": { + "timeout_ms": 1000 + }, + "implementation": { + "entry_point": "ovk.adapters.assurance.auth_state.AuthoritativeStateAdapter", + "implementation_digest": "sha256:f26e8ad83c8c3d031391077de282b9aa9374d0cf2144f2aaa7f2b6d71c4217f6", + "language": "python", + "name": "AuthoritativeStateAdapter", + "normalization_implementation_version": "ovk.normalize.v1", + "version": "0.1.0" + }, + "integrity": { + "artifact_digest": "sha256:610a2c95dd4f61ea801796b8489d63876783b146363cd647e694b21792e3fce5", + "canonicalization_version": "v1" + }, + "known_blind_spots": [ + "Does not discover undeclared state materials." + ], + "limitations_notice": "Profile binds configuration and digests only; it does not claim checker correctness beyond recorded results.", + "mechanism": { + "allows_abstention": true, + "description": "OVK assurance snapshot for backend auth-state-predicate", + "determinism": "deterministic", + "mechanism_class": "static_analysis" + }, + "producer": "OVK", + "producer_version": "1.2.1", + "redacted_environment": { + "entries": {}, + "environment_digest": "sha256:360192276458ec94bbc60a175efd2e5b1e9b8981250956a6e612300dd01eaf89", + "redacted_keys": [], + "redaction_policy_id": "ovk-redact-v1" + }, + "schema_version": "v1", + "source_commit": "e7097351c9a09a2e9f3626fb981f089b14b8cb4d", + "source_repo": "https://github.com/fraware/open-verification-kernel", + "verifier_profile_id": "vp-auth-state-predicate-3386a8ab" +} diff --git a/examples/assurance/model-judge/compiled_obligation.json b/examples/assurance/model-judge/compiled_obligation.json new file mode 100644 index 0000000..9c2be8b --- /dev/null +++ b/examples/assurance/model-judge/compiled_obligation.json @@ -0,0 +1,10 @@ +{ + "backend_id": "model-judge", + "input": { + "judge_client": "contract_fake", + "prompt": "Rate this change as safe.", + "rubric": { + "threshold": "0.0" + } + } +} diff --git a/examples/assurance/model-judge/invocation.json b/examples/assurance/model-judge/invocation.json new file mode 100644 index 0000000..393b141 --- /dev/null +++ b/examples/assurance/model-judge/invocation.json @@ -0,0 +1,48 @@ +{ + "artifact_type": "VerifierInvocationRecord.v1", + "assumptions": [], + "canonicalization_version": "v1", + "command": { + "argv": [ + "ovk-adapter-model-judge", + "judge" + ], + "cwd": ".", + "env_digest": "sha256:360192276458ec94bbc60a175efd2e5b1e9b8981250956a6e612300dd01eaf89", + "limits": { + "timeout_ms": 1000 + } + }, + "compiled_obligation": { + "digest": "sha256:f5f4f3f46610cef40a5ca90620b10ad94845e7b56972e955abc2d2d2836b3746", + "media_type": "application/json", + "path": "compiled_obligation.json" + }, + "completed_at": "2026-07-24T21:36:01Z", + "exit": { + "code": 0, + "kind": "exited" + }, + "guarantee_class": "empirically_measured", + "input_digest": "sha256:82514c33e40ab5e3720dca02d1ebb9c9bfbfddff0376798315ce2aefd11abfea", + "integrity": { + "artifact_digest": "sha256:c118a84a927458073c3323ef57611bcf5e5f65ed4fbec70b4a0f7c52d344165e", + "canonicalization_version": "v1" + }, + "invocation_id": "vi-model-judge-82514c33", + "limits": [], + "normalized_result_digest": "sha256:87ceab11811711bd7917dbbc17af60c37be4c9b866ccb8225c653957906a96d8", + "normalizer_version": "ovk.normalize.v1", + "producer": "OVK", + "producer_version": "1.2.1", + "profile_ref": { + "artifact_type": "VerifierProfile.v1", + "profile_digest": "sha256:1c756e47c72513d5cf793d4b18b82556fd174e58b365e760988abd8105deee5e", + "verifier_profile_id": "vp-model-judge-c7c7864d" + }, + "raw_backend_result_digest": "sha256:a651c7279ccc5256b5d0e399ecc5cde39cd6ab811571eac4db7b5f42787ff840", + "schema_version": "v1", + "started_at": "2026-07-24T21:36:01Z", + "stderr_digest": "sha256:12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126", + "stdout_digest": "sha256:9d1a88a9be142044520f8d016156bb5a2c1e80b481b11ee3267edd4d487e9492" +} diff --git a/examples/assurance/model-judge/normalized/result.json b/examples/assurance/model-judge/normalized/result.json new file mode 100644 index 0000000..368027d --- /dev/null +++ b/examples/assurance/model-judge/normalized/result.json @@ -0,0 +1,12 @@ +{ + "decision": "accept", + "guarantee_class": "empirically_measured", + "model_judgment": { + "passed": true, + "provider": "contract_fake", + "rationale": "contract_fake score=0.427 threshold=0.0", + "raw_response_digest": "sha256:4c4273c30b72b0b07dc8acb9260cf79bd67d2458d2b4cf3ee77c320b40b5256d", + "score": "0.427" + }, + "status": "pass" +} diff --git a/examples/assurance/model-judge/provenance/digests.json b/examples/assurance/model-judge/provenance/digests.json new file mode 100644 index 0000000..a51f566 --- /dev/null +++ b/examples/assurance/model-judge/provenance/digests.json @@ -0,0 +1,8 @@ +{ + "input_digest": "sha256:82514c33e40ab5e3720dca02d1ebb9c9bfbfddff0376798315ce2aefd11abfea", + "normalized_digest": "sha256:87ceab11811711bd7917dbbc17af60c37be4c9b866ccb8225c653957906a96d8", + "producer": "OVK", + "producer_version": "1.2.1", + "profile_digest": "sha256:1c756e47c72513d5cf793d4b18b82556fd174e58b365e760988abd8105deee5e", + "raw_digest": "sha256:a651c7279ccc5256b5d0e399ecc5cde39cd6ab811571eac4db7b5f42787ff840" +} diff --git a/examples/assurance/model-judge/raw/backend_result.json b/examples/assurance/model-judge/raw/backend_result.json new file mode 100644 index 0000000..e1a68d7 --- /dev/null +++ b/examples/assurance/model-judge/raw/backend_result.json @@ -0,0 +1,21 @@ +{ + "decoding": { + "temperature": "0" + }, + "decoding_digest": "sha256:303d9f781130627c7a29a97f09943bb30d1f1c0aa332eb4bf56d84a05b70c809", + "judge_client": "contract_fake", + "judgment": { + "passed": true, + "provider": "contract_fake", + "rationale": "contract_fake score=0.427 threshold=0.0", + "raw_response_digest": "sha256:4c4273c30b72b0b07dc8acb9260cf79bd67d2458d2b4cf3ee77c320b40b5256d", + "score": "0.427" + }, + "model": "contract-fake-v1", + "prompt_digest": "sha256:d888275c79594f9fbede5c46847738b5411903c917b592bc0ba07a206afcd6c3", + "rubric": { + "threshold": "0.0" + }, + "rubric_digest": "sha256:a97a053fa918da930ab63066f004a269e0cce91dc153fb7c33b419ab6d56e527", + "seed": null +} diff --git a/examples/assurance/model-judge/raw/stderr.txt b/examples/assurance/model-judge/raw/stderr.txt new file mode 100644 index 0000000..e69de29 diff --git a/examples/assurance/model-judge/raw/stdout.txt b/examples/assurance/model-judge/raw/stdout.txt new file mode 100644 index 0000000..98dedb5 --- /dev/null +++ b/examples/assurance/model-judge/raw/stdout.txt @@ -0,0 +1 @@ +contract_fake score=0.427 threshold=0.0 diff --git a/examples/assurance/model-judge/verification_result.pcs.json b/examples/assurance/model-judge/verification_result.pcs.json new file mode 100644 index 0000000..54c00ca --- /dev/null +++ b/examples/assurance/model-judge/verification_result.pcs.json @@ -0,0 +1,49 @@ +{ + "artifact_type": "VerificationResult.v1", + "check_groups": [ + { + "checks": [ + { + "check_id": "model-judge-predicate", + "mandatory": true, + "status": "passed" + } + ], + "kind": "authority" + } + ], + "claim_ids": [ + "claim.model_judge.empirical" + ], + "created_at": "2026-07-24T21:36:01Z", + "decision": "accept", + "declared_input_guarantee_class": "empirically_measured", + "execution_status": "completed", + "guarantee_class": "empirically_measured", + "input_bundle_digest": "sha256:82514c33e40ab5e3720dca02d1ebb9c9bfbfddff0376798315ce2aefd11abfea", + "integrity": { + "artifact_digest": "sha256:ff8258cae1c819717f134abd665dd758262950689cd2ca1c99c08dfe019f38b6", + "canonicalization_version": "v1" + }, + "invocation_ref": { + "invocation_digest": "sha256:c118a84a927458073c3323ef57611bcf5e5f65ed4fbec70b4a0f7c52d344165e", + "invocation_id": "vi-model-judge-82514c33" + }, + "normalization_applied": true, + "normalized_result_digest": "sha256:87ceab11811711bd7917dbbc17af60c37be4c9b866ccb8225c653957906a96d8", + "normalizer_version": "ovk.normalize.v1", + "producer": "OVK", + "producer_version": "1.2.1", + "raw_backend_output_digest": "sha256:a651c7279ccc5256b5d0e399ecc5cde39cd6ab811571eac4db7b5f42787ff840", + "resource_limits": { + "wall_time_ms": 0 + }, + "schema_version": "v1", + "source_commit": "e7097351c9a09a2e9f3626fb981f089b14b8cb4d", + "source_repo": "https://github.com/fraware/open-verification-kernel", + "verification_result_id": "vr-model-judge-82514c33", + "verifier_profile": { + "profile_digest": "sha256:1c756e47c72513d5cf793d4b18b82556fd174e58b365e760988abd8105deee5e", + "verifier_profile_id": "vp-model-judge-c7c7864d" + } +} diff --git a/examples/assurance/model-judge/verifier_profile.pcs.json b/examples/assurance/model-judge/verifier_profile.pcs.json new file mode 100644 index 0000000..14fa2dd --- /dev/null +++ b/examples/assurance/model-judge/verifier_profile.pcs.json @@ -0,0 +1,96 @@ +{ + "applicability": { + "status": "active", + "valid_from": "2026-07-24T21:36:01Z" + }, + "artifact_type": "VerifierProfile.v1", + "assumptions": [ + "Model judgments are empirical and may vary." + ], + "canonical_configuration": { + "decoding": { + "temperature": "0" + }, + "judge_client": "contract_fake", + "model": "contract-fake-v1", + "prompt": "", + "rubric": { + "threshold": "0.0" + }, + "timeout_ms": 1000 + }, + "claim_surface": { + "decision_space": [ + "accept", + "reject", + "indeterminate_execution_error", + "indeterminate_out_of_scope", + "indeterminate_insufficient_evidence" + ], + "guarantee_class": "empirically_measured", + "out_of_scope_claim_ids": [ + "claim.formal.full_correctness", + "claim.formally_checked" + ], + "supported_claim_ids": [ + "claim.model_judge.empirical" + ] + }, + "configuration": { + "config_digest": "sha256:3ba19cc7497cda9dc7ac7d0739678499727f67d5b373f0e8419da196c4aac806", + "ensemble_digest": null, + "model_digest": "sha256:ef9516ee9a9c935d95f1a1b17e318c6971d14497da14bfdec4cecbfa9f9ad7fb", + "policy_digest": null, + "prompt_digest": "sha256:12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126", + "resource_limit_digest": "sha256:1018384d24a8f0e4355abb757c1f2a09cd29e654b16233620871bc251b7a46de", + "rubric_digest": "sha256:a97a053fa918da930ab63066f004a269e0cce91dc153fb7c33b419ab6d56e527", + "test_suite_digest": null, + "threshold_digest": "sha256:16522468b17d5af6df90b05f6f6b8a1258ecb7b824dab04bfc63e1ed623558c7" + }, + "created_at": "2026-07-24T21:36:01Z", + "execution_controls": { + "timeout_ms": 1000 + }, + "external_dependencies": [ + { + "dependency_id": "model-judge-client", + "identity": "contract_fake", + "kind": "service", + "optional": true + } + ], + "implementation": { + "entry_point": "ovk.adapters.assurance.model_judge.ModelJudgeAdapter", + "implementation_digest": "sha256:501ada679ad1d07a2776b1b08f7c8487200fb31e3d6b413ba51b4e480f1a6a82", + "language": "python", + "name": "ModelJudgeAdapter", + "normalization_implementation_version": "ovk.normalize.v1", + "version": "0.1.0" + }, + "integrity": { + "artifact_digest": "sha256:1c756e47c72513d5cf793d4b18b82556fd174e58b365e760988abd8105deee5e", + "canonicalization_version": "v1" + }, + "known_blind_spots": [ + "Cannot upgrade to formally_checked." + ], + "limitations_notice": "Profile binds configuration and digests only; it does not claim checker correctness beyond recorded results.", + "mechanism": { + "allows_abstention": true, + "description": "OVK assurance snapshot for backend model-judge", + "determinism": "stochastic", + "mechanism_class": "model_judge" + }, + "producer": "OVK", + "producer_version": "1.2.1", + "redacted_environment": { + "entries": {}, + "environment_digest": "sha256:360192276458ec94bbc60a175efd2e5b1e9b8981250956a6e612300dd01eaf89", + "redacted_keys": [], + "redaction_policy_id": "ovk-redact-v1" + }, + "schema_version": "v1", + "source_commit": "e7097351c9a09a2e9f3626fb981f089b14b8cb4d", + "source_repo": "https://github.com/fraware/open-verification-kernel", + "verifier_profile_id": "vp-model-judge-c7c7864d" +} diff --git a/examples/assurance/pytest-suite/compiled_obligation.json b/examples/assurance/pytest-suite/compiled_obligation.json new file mode 100644 index 0000000..ff75766 --- /dev/null +++ b/examples/assurance/pytest-suite/compiled_obligation.json @@ -0,0 +1,6 @@ +{ + "backend_id": "pytest-suite", + "input": { + "suite_path": "tests/assurance/fixtures/pytest_suite" + } +} diff --git a/examples/assurance/pytest-suite/invocation.json b/examples/assurance/pytest-suite/invocation.json new file mode 100644 index 0000000..d8dfe1c --- /dev/null +++ b/examples/assurance/pytest-suite/invocation.json @@ -0,0 +1,52 @@ +{ + "artifact_type": "VerifierInvocationRecord.v1", + "assumptions": [], + "canonicalization_version": "v1", + "command": { + "argv": [ + "python", + "-m", + "pytest", + "tests/assurance/fixtures/pytest_suite", + "--junitxml=junit.xml", + "-q" + ], + "cwd": ".", + "env_digest": "sha256:360192276458ec94bbc60a175efd2e5b1e9b8981250956a6e612300dd01eaf89", + "limits": { + "timeout_ms": 60000 + } + }, + "compiled_obligation": { + "digest": "sha256:5b28f3e45224f5b990850e1462fc950959a88b920e3e43371c9606bf03c16ff5", + "media_type": "application/json", + "path": "compiled_obligation.json" + }, + "completed_at": "2026-07-24T21:36:00Z", + "exit": { + "code": 0, + "kind": "exited" + }, + "guarantee_class": "runtime_observed", + "input_digest": "sha256:49fa0764c4b0af82001671f21867f4627e0838359cbff8e43812fe7bded63d10", + "integrity": { + "artifact_digest": "sha256:8ae3685822f58889eaad93a01778bbe5ec2473d51a7220f5771b0d84663de93d", + "canonicalization_version": "v1" + }, + "invocation_id": "vi-pytest-suite-49fa0764", + "limits": [], + "normalized_result_digest": "sha256:dc9844cb40e11b16bb9c2bedf11de5b25059452347c37ff395232829b8010168", + "normalizer_version": "ovk.normalize.v1", + "producer": "OVK", + "producer_version": "1.2.1", + "profile_ref": { + "artifact_type": "VerifierProfile.v1", + "profile_digest": "sha256:bf68a48df883371361add66e9acb27d48423bc8a88dd1b4f2597ad2cd2c5007a", + "verifier_profile_id": "vp-pytest-suite-9802dd51" + }, + "raw_backend_result_digest": "sha256:32c7b704db35bf722b523113b5225e96fd7b752d1337aed2bbca88d7a478cbac", + "schema_version": "v1", + "started_at": "2026-07-24T21:35:56Z", + "stderr_digest": "sha256:e66cd32423ccc8c683a0f56c15f5ab190b7dc4398891ed3f27cc929b9300c72b", + "stdout_digest": "sha256:420c5b8381ee64a381690b1ffdab7f88f26257a2b1c49119cf4e71d7db840160" +} diff --git a/examples/assurance/pytest-suite/normalized/result.json b/examples/assurance/pytest-suite/normalized/result.json new file mode 100644 index 0000000..ea3536a --- /dev/null +++ b/examples/assurance/pytest-suite/normalized/result.json @@ -0,0 +1,27 @@ +{ + "decision": "accept", + "failures": 0, + "guarantee_class": "runtime_observed", + "skipped": 0, + "skipped_visible": false, + "status": "pass", + "test_report": { + "cases": [ + { + "classname": "tests.assurance.fixtures.pytest_suite.test_offline_suite", + "name": "test_fixture_passes", + "status": "passed" + }, + { + "classname": "tests.assurance.fixtures.pytest_suite.test_offline_suite", + "name": "test_fixture_also_passes", + "status": "passed" + } + ], + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 2 + }, + "threshold": 0 +} diff --git a/examples/assurance/pytest-suite/provenance/digests.json b/examples/assurance/pytest-suite/provenance/digests.json new file mode 100644 index 0000000..e7c6329 --- /dev/null +++ b/examples/assurance/pytest-suite/provenance/digests.json @@ -0,0 +1,8 @@ +{ + "input_digest": "sha256:49fa0764c4b0af82001671f21867f4627e0838359cbff8e43812fe7bded63d10", + "normalized_digest": "sha256:dc9844cb40e11b16bb9c2bedf11de5b25059452347c37ff395232829b8010168", + "producer": "OVK", + "producer_version": "1.2.1", + "profile_digest": "sha256:bf68a48df883371361add66e9acb27d48423bc8a88dd1b4f2597ad2cd2c5007a", + "raw_digest": "sha256:32c7b704db35bf722b523113b5225e96fd7b752d1337aed2bbca88d7a478cbac" +} diff --git a/examples/assurance/pytest-suite/raw/backend_result.json b/examples/assurance/pytest-suite/raw/backend_result.json new file mode 100644 index 0000000..4ff8d95 --- /dev/null +++ b/examples/assurance/pytest-suite/raw/backend_result.json @@ -0,0 +1,29 @@ +{ + "command_stable": [ + "python", + "-m", + "pytest", + "tests/assurance/fixtures/pytest_suite", + "-q" + ], + "exit_code": 0, + "junit_summary": { + "cases": [ + { + "classname": "tests.assurance.fixtures.pytest_suite.test_offline_suite", + "name": "test_fixture_passes", + "status": "passed" + }, + { + "classname": "tests.assurance.fixtures.pytest_suite.test_offline_suite", + "name": "test_fixture_also_passes", + "status": "passed" + } + ], + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 2 + }, + "suite_digest": "sha256:549b66ee77b5651c2a2fa37ef227bf8912956e69ae74d3bba6deff5abd1f9793" +} diff --git a/examples/assurance/pytest-suite/raw/stderr.txt b/examples/assurance/pytest-suite/raw/stderr.txt new file mode 100644 index 0000000..69c5611 --- /dev/null +++ b/examples/assurance/pytest-suite/raw/stderr.txt @@ -0,0 +1,3 @@ +The event loop scope for asynchronous fixtures will default to the fixture caching scope. Future versions of pytest-asyncio will default the loop scope for asynchronous fixtures to function scope. Set the default fixture loop scope explicitly in order to avoid unexpected behavior in the future. Valid fixture loop scopes are: "function", "class", "module", "package", "session" + + warnings.warn(PytestDeprecationWarning(_DEFAULT_FIXTURE_LOOP_SCOPE_UNSET)) diff --git a/examples/assurance/pytest-suite/raw/stdout.txt b/examples/assurance/pytest-suite/raw/stdout.txt new file mode 100644 index 0000000..41a8c43 --- /dev/null +++ b/examples/assurance/pytest-suite/raw/stdout.txt @@ -0,0 +1,3 @@ +.. [100%] +- generated xml file: junit.xml - +2 passed in 0.64s diff --git a/examples/assurance/pytest-suite/verification_result.pcs.json b/examples/assurance/pytest-suite/verification_result.pcs.json new file mode 100644 index 0000000..d521e55 --- /dev/null +++ b/examples/assurance/pytest-suite/verification_result.pcs.json @@ -0,0 +1,49 @@ +{ + "artifact_type": "VerificationResult.v1", + "check_groups": [ + { + "checks": [ + { + "check_id": "pytest-suite-predicate", + "mandatory": true, + "status": "passed" + } + ], + "kind": "authority" + } + ], + "claim_ids": [ + "claim.pytest.runtime_observed" + ], + "created_at": "2026-07-24T21:36:00Z", + "decision": "accept", + "declared_input_guarantee_class": "runtime_observed", + "execution_status": "completed", + "guarantee_class": "runtime_observed", + "input_bundle_digest": "sha256:49fa0764c4b0af82001671f21867f4627e0838359cbff8e43812fe7bded63d10", + "integrity": { + "artifact_digest": "sha256:12f49687191ec89f8096651d11c558dc71cd803b5aa0b3f563415130b20cd581", + "canonicalization_version": "v1" + }, + "invocation_ref": { + "invocation_digest": "sha256:8ae3685822f58889eaad93a01778bbe5ec2473d51a7220f5771b0d84663de93d", + "invocation_id": "vi-pytest-suite-49fa0764" + }, + "normalization_applied": true, + "normalized_result_digest": "sha256:dc9844cb40e11b16bb9c2bedf11de5b25059452347c37ff395232829b8010168", + "normalizer_version": "ovk.normalize.v1", + "producer": "OVK", + "producer_version": "1.2.1", + "raw_backend_output_digest": "sha256:32c7b704db35bf722b523113b5225e96fd7b752d1337aed2bbca88d7a478cbac", + "resource_limits": { + "wall_time_ms": 0 + }, + "schema_version": "v1", + "source_commit": "e7097351c9a09a2e9f3626fb981f089b14b8cb4d", + "source_repo": "https://github.com/fraware/open-verification-kernel", + "verification_result_id": "vr-pytest-suite-49fa0764", + "verifier_profile": { + "profile_digest": "sha256:bf68a48df883371361add66e9acb27d48423bc8a88dd1b4f2597ad2cd2c5007a", + "verifier_profile_id": "vp-pytest-suite-9802dd51" + } +} diff --git a/examples/assurance/pytest-suite/verifier_profile.pcs.json b/examples/assurance/pytest-suite/verifier_profile.pcs.json new file mode 100644 index 0000000..f47af6b --- /dev/null +++ b/examples/assurance/pytest-suite/verifier_profile.pcs.json @@ -0,0 +1,88 @@ +{ + "applicability": { + "status": "active", + "valid_from": "2026-07-24T21:35:56Z" + }, + "artifact_type": "VerifierProfile.v1", + "assumptions": [ + "Observational pytest execution only." + ], + "canonical_configuration": { + "suite_path": "tests/assurance/fixtures/pytest_suite", + "threshold": 0, + "timeout_ms": 60000 + }, + "claim_surface": { + "decision_space": [ + "accept", + "reject", + "indeterminate_execution_error", + "indeterminate_out_of_scope", + "indeterminate_insufficient_evidence" + ], + "guarantee_class": "runtime_observed", + "out_of_scope_claim_ids": [ + "claim.formal.full_correctness" + ], + "supported_claim_ids": [ + "claim.pytest.runtime_observed" + ] + }, + "configuration": { + "config_digest": "sha256:1018014a6bae15ac1e0093f3d6982e5c689aab33d43d708a53356db854a3ad2e", + "ensemble_digest": null, + "model_digest": null, + "policy_digest": null, + "prompt_digest": null, + "resource_limit_digest": "sha256:cca428700bd0ca10a53960e042b27d423e4245274e9a166c2d70c50ce3fb1a0c", + "rubric_digest": null, + "test_suite_digest": "sha256:549b66ee77b5651c2a2fa37ef227bf8912956e69ae74d3bba6deff5abd1f9793", + "threshold_digest": "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9" + }, + "created_at": "2026-07-24T21:35:56Z", + "execution_controls": { + "timeout_ms": 60000 + }, + "external_dependencies": [ + { + "dependency_id": "pytest", + "identity": "pytest", + "kind": "library", + "optional": false + } + ], + "implementation": { + "entry_point": "ovk.adapters.assurance.pytest_suite.PytestSuiteAdapter", + "implementation_digest": "sha256:a3651554436c0d5e073ce7e200576a610296f9469a085645804d097f57f8ecc8", + "language": "python", + "name": "PytestSuiteAdapter", + "normalization_implementation_version": "ovk.normalize.v1", + "version": "0.1.0" + }, + "integrity": { + "artifact_digest": "sha256:bf68a48df883371361add66e9acb27d48423bc8a88dd1b4f2597ad2cd2c5007a", + "canonicalization_version": "v1" + }, + "known_blind_spots": [ + "Passing tests do not imply formal correctness." + ], + "limitations_notice": "Profile binds configuration and digests only; it does not claim checker correctness beyond recorded results.", + "mechanism": { + "allows_abstention": true, + "description": "OVK assurance snapshot for backend pytest-suite", + "determinism": "deterministic", + "mechanism_class": "test_suite" + }, + "producer": "OVK", + "producer_version": "1.2.1", + "redacted_environment": { + "entries": {}, + "environment_digest": "sha256:360192276458ec94bbc60a175efd2e5b1e9b8981250956a6e612300dd01eaf89", + "redacted_keys": [], + "redaction_policy_id": "ovk-redact-v1" + }, + "schema_version": "v1", + "source_commit": "e7097351c9a09a2e9f3626fb981f089b14b8cb4d", + "source_repo": "https://github.com/fraware/open-verification-kernel", + "verifier_profile_id": "vp-pytest-suite-9802dd51" +} diff --git a/examples/assurance/sql-state-diff/compiled_obligation.json b/examples/assurance/sql-state-diff/compiled_obligation.json new file mode 100644 index 0000000..e227985 --- /dev/null +++ b/examples/assurance/sql-state-diff/compiled_obligation.json @@ -0,0 +1,10 @@ +{ + "backend_id": "sql-state-diff", + "input": { + "after_db": "tests/assurance/fixtures/sql/after.sqlite", + "before_db": "tests/assurance/fixtures/sql/before.sqlite", + "expect_changed_tables": [ + "items" + ] + } +} diff --git a/examples/assurance/sql-state-diff/invocation.json b/examples/assurance/sql-state-diff/invocation.json new file mode 100644 index 0000000..0ac8092 --- /dev/null +++ b/examples/assurance/sql-state-diff/invocation.json @@ -0,0 +1,48 @@ +{ + "artifact_type": "VerifierInvocationRecord.v1", + "assumptions": [], + "canonicalization_version": "v1", + "command": { + "argv": [ + "ovk-adapter-sql-state-diff", + "sqlite-diff" + ], + "cwd": ".", + "env_digest": "sha256:360192276458ec94bbc60a175efd2e5b1e9b8981250956a6e612300dd01eaf89", + "limits": { + "timeout_ms": 5000 + } + }, + "compiled_obligation": { + "digest": "sha256:7d9e5c21c54117dc7753ca8ce1e915231a278ba40c5460aeccd9c7a0808a256d", + "media_type": "application/json", + "path": "compiled_obligation.json" + }, + "completed_at": "2026-07-24T21:36:00Z", + "exit": { + "code": 0, + "kind": "exited" + }, + "guarantee_class": "observational", + "input_digest": "sha256:bed20dcf292ee4871001bfa1d86b5382a49c0288a329120b0021cbe4abc0d2e4", + "integrity": { + "artifact_digest": "sha256:9e9100265e506fc169def68f51cf01076921dbb62f8bcce545720c41d2216262", + "canonicalization_version": "v1" + }, + "invocation_id": "vi-sql-state-diff-bed20dcf", + "limits": [], + "normalized_result_digest": "sha256:44676b2f4c54fd11d73ab5d15c6956867b4ecac7fdea898181d6653ea741f8c6", + "normalizer_version": "ovk.normalize.v1", + "producer": "OVK", + "producer_version": "1.2.1", + "profile_ref": { + "artifact_type": "VerifierProfile.v1", + "profile_digest": "sha256:cfb3c8e462fdf28dbc11e49789fb495ec6bb4d4e66b2dcce4be869a9066ec816", + "verifier_profile_id": "vp-sql-state-diff-44471e42" + }, + "raw_backend_result_digest": "sha256:ffe49b4f57e66cebdc9984322836c3040882c1396edb9596a5b074ba7858b970", + "schema_version": "v1", + "started_at": "2026-07-24T21:36:00Z", + "stderr_digest": "sha256:12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126", + "stdout_digest": "sha256:2faed1418d56b08b21a2d29fdac2030e7e87272342c23a144a4d52554c23a182" +} diff --git a/examples/assurance/sql-state-diff/normalized/result.json b/examples/assurance/sql-state-diff/normalized/result.json new file mode 100644 index 0000000..2aced5b --- /dev/null +++ b/examples/assurance/sql-state-diff/normalized/result.json @@ -0,0 +1,21 @@ +{ + "decision": "accept", + "guarantee_class": "observational", + "state_diff": { + "after_schema_digest": "sha256:9e592e4a30e6c67c5bd4d061f205f98c1b31930b70006728e6b5182b1ff34b3c", + "before_schema_digest": "sha256:9e592e4a30e6c67c5bd4d061f205f98c1b31930b70006728e6b5182b1ff34b3c", + "changed_tables": [ + "items" + ], + "schema_changed": false, + "tables": [ + { + "after_digest": "sha256:6fe4337d214065194d3d57dd08823e93b7edafd4771ba4e8203b8e1270376b5c", + "before_digest": "sha256:a980c14b37f47e4f62454f83bd37790ceee73a71ea725cd1684b591786fe1a09", + "changed": true, + "table": "items" + } + ] + }, + "status": "pass" +} diff --git a/examples/assurance/sql-state-diff/provenance/digests.json b/examples/assurance/sql-state-diff/provenance/digests.json new file mode 100644 index 0000000..fe9983c --- /dev/null +++ b/examples/assurance/sql-state-diff/provenance/digests.json @@ -0,0 +1,8 @@ +{ + "input_digest": "sha256:bed20dcf292ee4871001bfa1d86b5382a49c0288a329120b0021cbe4abc0d2e4", + "normalized_digest": "sha256:44676b2f4c54fd11d73ab5d15c6956867b4ecac7fdea898181d6653ea741f8c6", + "producer": "OVK", + "producer_version": "1.2.1", + "profile_digest": "sha256:cfb3c8e462fdf28dbc11e49789fb495ec6bb4d4e66b2dcce4be869a9066ec816", + "raw_digest": "sha256:ffe49b4f57e66cebdc9984322836c3040882c1396edb9596a5b074ba7858b970" +} diff --git a/examples/assurance/sql-state-diff/raw/backend_result.json b/examples/assurance/sql-state-diff/raw/backend_result.json new file mode 100644 index 0000000..f86bc9b --- /dev/null +++ b/examples/assurance/sql-state-diff/raw/backend_result.json @@ -0,0 +1,22 @@ +{ + "after_db": "tests/assurance/fixtures/sql/after.sqlite", + "after_file_digest": "sha256:aa7a3ab1d89f25013f58a15364b0db335f213bc6597e3810326f8ed54cedde79", + "before_db": "tests/assurance/fixtures/sql/before.sqlite", + "before_file_digest": "sha256:e463f8662135ca210087fd32c8cbd58087f657d2a30d0e07fedea7080751651a", + "diff": { + "after_schema_digest": "sha256:9e592e4a30e6c67c5bd4d061f205f98c1b31930b70006728e6b5182b1ff34b3c", + "before_schema_digest": "sha256:9e592e4a30e6c67c5bd4d061f205f98c1b31930b70006728e6b5182b1ff34b3c", + "changed_tables": [ + "items" + ], + "schema_changed": false, + "tables": [ + { + "after_digest": "sha256:6fe4337d214065194d3d57dd08823e93b7edafd4771ba4e8203b8e1270376b5c", + "before_digest": "sha256:a980c14b37f47e4f62454f83bd37790ceee73a71ea725cd1684b591786fe1a09", + "changed": true, + "table": "items" + } + ] + } +} diff --git a/examples/assurance/sql-state-diff/raw/stderr.txt b/examples/assurance/sql-state-diff/raw/stderr.txt new file mode 100644 index 0000000..e69de29 diff --git a/examples/assurance/sql-state-diff/raw/stdout.txt b/examples/assurance/sql-state-diff/raw/stdout.txt new file mode 100644 index 0000000..fc07d38 --- /dev/null +++ b/examples/assurance/sql-state-diff/raw/stdout.txt @@ -0,0 +1 @@ +accept diff --git a/examples/assurance/sql-state-diff/verification_result.pcs.json b/examples/assurance/sql-state-diff/verification_result.pcs.json new file mode 100644 index 0000000..a181366 --- /dev/null +++ b/examples/assurance/sql-state-diff/verification_result.pcs.json @@ -0,0 +1,49 @@ +{ + "artifact_type": "VerificationResult.v1", + "check_groups": [ + { + "checks": [ + { + "check_id": "sql-state-diff-predicate", + "mandatory": true, + "status": "passed" + } + ], + "kind": "authority" + } + ], + "claim_ids": [ + "claim.sql.state_diff" + ], + "created_at": "2026-07-24T21:36:00Z", + "decision": "accept", + "declared_input_guarantee_class": "observational", + "execution_status": "completed", + "guarantee_class": "observational", + "input_bundle_digest": "sha256:bed20dcf292ee4871001bfa1d86b5382a49c0288a329120b0021cbe4abc0d2e4", + "integrity": { + "artifact_digest": "sha256:1d47d46622009a5783c506c3012190b55b3a14a672f4931bf2eecbfc3b793356", + "canonicalization_version": "v1" + }, + "invocation_ref": { + "invocation_digest": "sha256:9e9100265e506fc169def68f51cf01076921dbb62f8bcce545720c41d2216262", + "invocation_id": "vi-sql-state-diff-bed20dcf" + }, + "normalization_applied": true, + "normalized_result_digest": "sha256:44676b2f4c54fd11d73ab5d15c6956867b4ecac7fdea898181d6653ea741f8c6", + "normalizer_version": "ovk.normalize.v1", + "producer": "OVK", + "producer_version": "1.2.1", + "raw_backend_output_digest": "sha256:ffe49b4f57e66cebdc9984322836c3040882c1396edb9596a5b074ba7858b970", + "resource_limits": { + "wall_time_ms": 0 + }, + "schema_version": "v1", + "source_commit": "e7097351c9a09a2e9f3626fb981f089b14b8cb4d", + "source_repo": "https://github.com/fraware/open-verification-kernel", + "verification_result_id": "vr-sql-state-diff-bed20dcf", + "verifier_profile": { + "profile_digest": "sha256:cfb3c8e462fdf28dbc11e49789fb495ec6bb4d4e66b2dcce4be869a9066ec816", + "verifier_profile_id": "vp-sql-state-diff-44471e42" + } +} diff --git a/examples/assurance/sql-state-diff/verifier_profile.pcs.json b/examples/assurance/sql-state-diff/verifier_profile.pcs.json new file mode 100644 index 0000000..1788279 --- /dev/null +++ b/examples/assurance/sql-state-diff/verifier_profile.pcs.json @@ -0,0 +1,83 @@ +{ + "applicability": { + "status": "active", + "valid_from": "2026-07-24T21:36:00Z" + }, + "artifact_type": "VerifierProfile.v1", + "assumptions": [ + "Offline SQLite before/after fixtures." + ], + "canonical_configuration": { + "after_db": "tests/assurance/fixtures/sql/after.sqlite", + "before_db": "tests/assurance/fixtures/sql/before.sqlite", + "expect_changed_tables": [ + "items" + ], + "timeout_ms": 5000 + }, + "claim_surface": { + "decision_space": [ + "accept", + "reject", + "indeterminate_execution_error", + "indeterminate_out_of_scope", + "indeterminate_insufficient_evidence" + ], + "guarantee_class": "observational", + "out_of_scope_claim_ids": [ + "claim.formal.full_correctness" + ], + "supported_claim_ids": [ + "claim.sql.state_diff" + ] + }, + "configuration": { + "config_digest": "sha256:629fe4b76f48b2e978bbb4d6a4944950506df2105d03f7886c290808bab4d48d", + "ensemble_digest": null, + "model_digest": null, + "policy_digest": null, + "prompt_digest": null, + "resource_limit_digest": "sha256:2c84141f555be7ec76d2a87e31c10afc1f6493b7e4a1e8c299de4b17186a4e62", + "rubric_digest": null, + "test_suite_digest": null, + "threshold_digest": null + }, + "created_at": "2026-07-24T21:36:00Z", + "execution_controls": { + "timeout_ms": 5000 + }, + "implementation": { + "entry_point": "ovk.adapters.assurance.sql_diff.SqlStateDiffAdapter", + "implementation_digest": "sha256:5bd5edfeea269697c8bed1bd3d407e646ee0ea8b8491e4e755dde145bef69ed5", + "language": "python", + "name": "SqlStateDiffAdapter", + "normalization_implementation_version": "ovk.normalize.v1", + "version": "0.1.0" + }, + "integrity": { + "artifact_digest": "sha256:cfb3c8e462fdf28dbc11e49789fb495ec6bb4d4e66b2dcce4be869a9066ec816", + "canonicalization_version": "v1" + }, + "known_blind_spots": [ + "Does not prove application-level invariants beyond table digests." + ], + "limitations_notice": "Profile binds configuration and digests only; it does not claim checker correctness beyond recorded results.", + "mechanism": { + "allows_abstention": true, + "description": "OVK assurance snapshot for backend sql-state-diff", + "determinism": "deterministic", + "mechanism_class": "static_analysis" + }, + "producer": "OVK", + "producer_version": "1.2.1", + "redacted_environment": { + "entries": {}, + "environment_digest": "sha256:360192276458ec94bbc60a175efd2e5b1e9b8981250956a6e612300dd01eaf89", + "redacted_keys": [], + "redaction_policy_id": "ovk-redact-v1" + }, + "schema_version": "v1", + "source_commit": "e7097351c9a09a2e9f3626fb981f089b14b8cb4d", + "source_repo": "https://github.com/fraware/open-verification-kernel", + "verifier_profile_id": "vp-sql-state-diff-44471e42" +} diff --git a/ovk/adapters/assurance/__init__.py b/ovk/adapters/assurance/__init__.py new file mode 100644 index 0000000..a8a53ff --- /dev/null +++ b/ovk/adapters/assurance/__init__.py @@ -0,0 +1,33 @@ +"""Assurance-mode backends (opt-in; not mixed into ordinary lane routing).""" + +from __future__ import annotations + +from ovk.core.backend_registry import BackendRegistry + + +def build_assurance_registry() -> BackendRegistry: + """Register assurance-capable backends for ``ovk verifier`` only.""" + # Local imports avoid circular import with ovk.assurance.registry. + from ovk.adapters.assurance.auth_state import AuthoritativeStateAdapter + from ovk.adapters.assurance.lean_pfcore import LeanPfCoreAssuranceAdapter + from ovk.adapters.assurance.model_judge import ModelJudgeAdapter + from ovk.adapters.assurance.opa_policy import OpaPolicyAssuranceAdapter + from ovk.adapters.assurance.pytest_suite import PytestSuiteAdapter + from ovk.adapters.assurance.sql_diff import SqlStateDiffAdapter + + registry = BackendRegistry() + for cls in ( + AuthoritativeStateAdapter, + PytestSuiteAdapter, + OpaPolicyAssuranceAdapter, + LeanPfCoreAssuranceAdapter, + SqlStateDiffAdapter, + ModelJudgeAdapter, + ): + registry.register(cls()) + return registry + + +__all__ = [ + "build_assurance_registry", +] diff --git a/ovk/adapters/assurance/_support.py b/ovk/adapters/assurance/_support.py new file mode 100644 index 0000000..a636a90 --- /dev/null +++ b/ovk/adapters/assurance/_support.py @@ -0,0 +1,240 @@ +"""Shared helpers for assurance-only BackendAdapter implementations.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Mapping + +from ovk.assurance.indeterminate import ( + DECISION_ACCEPT, + DECISION_REJECT, + indeterminate_outcome, +) +from ovk.assurance.pcs_hash import sha256_digest +from ovk.core.execution_models import ( + AbstractionCoverage, + BackendCapabilityAssessment, + BackendEnvironmentFingerprint, + BackendObligation, + ExecutionBudget, + ExecutionContext, + HumanExplanation, + NormalizedBackendResult, + RawBackendExecution, + RoutingDecision, + VerificationObligation, + compute_backend_obligation_id, + compute_payload_digest, +) +from ovk.core.models import VerificationStatus + + +def utc_now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def accept_outcome( + *, + raw_result: Mapping[str, Any], + normalized_extra: Mapping[str, Any] | None = None, + stdout: str = "accept", + stderr: str = "", + guarantee_class: str, + command_argv: list[str], + exit_code: int = 0, +) -> dict[str, Any]: + normalized: dict[str, Any] = { + "decision": DECISION_ACCEPT, + "guarantee_class": guarantee_class, + "status": "pass", + } + if normalized_extra: + normalized.update(dict(normalized_extra)) + return { + "exit_kind": "exited", + "exit_code": exit_code, + "status": "pass", + "decision": DECISION_ACCEPT, + "execution_status": "completed", + "indeterminate_reason": None, + "stdout": stdout, + "stderr": stderr, + "raw_result": dict(raw_result), + "normalized_result": normalized, + "guarantee_class": guarantee_class, + "command_argv": list(command_argv), + } + + +def reject_outcome( + *, + raw_result: Mapping[str, Any], + normalized_extra: Mapping[str, Any] | None = None, + stdout: str = "reject", + stderr: str = "", + guarantee_class: str, + command_argv: list[str], + exit_code: int = 1, +) -> dict[str, Any]: + normalized: dict[str, Any] = { + "decision": DECISION_REJECT, + "guarantee_class": guarantee_class, + "status": "fail", + } + if normalized_extra: + normalized.update(dict(normalized_extra)) + return { + "exit_kind": "exited", + "exit_code": exit_code, + "status": "fail", + "decision": DECISION_REJECT, + "execution_status": "completed", + "indeterminate_reason": None, + "stdout": stdout, + "stderr": stderr, + "raw_result": dict(raw_result), + "normalized_result": normalized, + "guarantee_class": guarantee_class, + "command_argv": list(command_argv), + } + + +def indeterminate_run_outcome( + *, + reason: str, + message: str, + raw_result: Mapping[str, Any] | None = None, + guarantee_class: str, + command_argv: list[str], + exit_code: int | None = 2, + exit_kind: str = "exited", +) -> dict[str, Any]: + ind = indeterminate_outcome(reason=reason, message=message) + normalized = { + "decision": ind["decision"], + "guarantee_class": guarantee_class, + "status": "unknown", + "indeterminate_reason": ind["indeterminate_reason"], + } + return { + "exit_kind": exit_kind, + "exit_code": exit_code, + "status": "unknown", + "decision": ind["decision"], + "execution_status": ind["execution_status"], + "indeterminate_reason": ind["indeterminate_reason"], + "stdout": "", + "stderr": message, + "raw_result": dict(raw_result or {"error": reason, "message": message}), + "normalized_result": normalized, + "guarantee_class": guarantee_class, + "command_argv": list(command_argv), + "message": message, + } + + +class AssuranceBackendMixin: + """Minimal BackendAdapter surface so assurance adapters can register. + + Ordinary ``ovk check`` routing does not select these backends; they exist + for ``ovk verifier`` and the assurance registry only. + """ + + backend_id: str + adapter_id: str + adapter_version: str + _guarantee_type: str = "exact_predicate" + _domain: str = "assurance" + + def can_handle( + self, + obligation: VerificationObligation, + context: ExecutionContext, + ) -> BackendCapabilityAssessment: + return BackendCapabilityAssessment( + backend=self.backend_id, + support="unsupported", + score=0.0, + guarantee_type=self._guarantee_type, + material_requirements_met=False, + coverage_requirements_met=False, + native_available=True, + estimated_wall_time_seconds=1.0, + estimated_memory_mb=128, + reasons=["assurance-only backend; not selected by ordinary routing"], + ) + + def compile( + self, + obligation: VerificationObligation, + routing: RoutingDecision, + ) -> BackendObligation: + payload = {"abstraction": obligation.abstraction, "mode": "assurance"} + provisional = BackendObligation( + backend_obligation_id="pending", + obligation_id=obligation.obligation_id, + routing_id=routing.routing_id, + backend=self.backend_id, + adapter_id=self.adapter_id, + adapter_version=self.adapter_version, + input_language="json", + payload=payload, + payload_digest=compute_payload_digest(payload), + required=False, + timeout_seconds=30.0, + memory_mb=256, + ) + return provisional.model_copy( + update={"backend_obligation_id": compute_backend_obligation_id(provisional)} + ) + + def fingerprint(self, backend_obligation: BackendObligation) -> BackendEnvironmentFingerprint: + return BackendEnvironmentFingerprint( + backend=self.backend_id, + adapter_version=self.adapter_version, + environment_digest=sha256_digest({"backend": self.backend_id})[7:], + native_available=True, + ) + + def run( + self, + backend_obligation: BackendObligation, + budget: ExecutionBudget, + ) -> RawBackendExecution: + started = utc_now_iso() + return RawBackendExecution( + backend=self.backend_id, + termination="cancelled", + native_execution=False, + raw_result={"error": "assurance backends are not invoked via ordinary run()"}, + stdout="", + stderr="use ovk verifier run / run_assurance()", + exit_code=None, + started_at=started, + finished_at=utc_now_iso(), + ) + + def normalize( + self, + raw: RawBackendExecution, + backend_obligation: BackendObligation, + ) -> NormalizedBackendResult: + return NormalizedBackendResult( + attempt_id=f"{self.backend_id}-ordinary-unused", + backend=self.backend_id, + status=VerificationStatus.UNKNOWN, + guarantee_type=self._guarantee_type, + assumptions=["assurance-only; ordinary normalize is unused"], + limits=["cannot upgrade guarantee class via ordinary normalize"], + ) + + def explain(self, result: NormalizedBackendResult) -> HumanExplanation: + return HumanExplanation( + summary=f"{self.backend_id} ordinary path unused (assurance-only)", + repair_hint="Use ovk verifier run with an assurance evidence pack.", + failure_mode=None, + ) + + +# Keep coverage type importable for obligation construction in tests. +_ = AbstractionCoverage diff --git a/ovk/adapters/assurance/auth_state.py b/ovk/adapters/assurance/auth_state.py new file mode 100644 index 0000000..0213c1f --- /dev/null +++ b/ovk/adapters/assurance/auth_state.py @@ -0,0 +1,345 @@ +"""Exact authoritative-state predicate evaluator (VA-06).""" + +from __future__ import annotations + +from typing import Any, Mapping + +from ovk.adapters.assurance._support import ( + AssuranceBackendMixin, + accept_outcome, + indeterminate_run_outcome, + reject_outcome, +) +from ovk.assurance.pcs_hash import sha256_digest +from ovk.assurance.snapshot import ConfigurationSnapshot, build_configuration_snapshot +from ovk.core.execution_models import ( + AssuranceAbstention, + AssuranceCapabilitySection, + AssuranceDecisionSemantics, + AssuranceFailureBehavior, + AssuranceReplaySupport, + AssuranceSnapshotSupport, + AssuranceVerifierIdentity, + BackendCapabilityManifest, + BackendGuaranteeDeclaration, + BackendToolIdentity, +) + +GUARANTEE_CLASS = "observational" + +_DECISION_SPACE = [ + "accept", + "reject", + "indeterminate_execution_error", + "indeterminate_out_of_scope", + "indeterminate_insufficient_evidence", +] + + +def _resolve_path(data: Mapping[str, Any], path: str) -> Any: + cur: Any = data + for part in path.split("."): + if not isinstance(cur, Mapping) or part not in cur: + raise KeyError(path) + cur = cur[part] + return cur + + +def evaluate_predicates( + authoritative_state: Mapping[str, Any], + predicates: list[Mapping[str, Any]], +) -> dict[str, Any]: + """Evaluate exact predicates over declared authoritative state materials. + + Supported predicate kinds: + - ``field_equals``: path equals expected + - ``digest_matches``: sha256 of path (or whole state) equals expected_digest + - ``set_contains``: path (list/set) contains member + - ``required_keys``: mapping at path contains all keys + - ``state_machine_safe``: reuse approval-path reachability over embedded machine + """ + results: list[dict[str, Any]] = [] + for index, predicate in enumerate(predicates): + kind = str(predicate.get("kind") or "").strip() + entry: dict[str, Any] = {"index": index, "kind": kind, "predicate": dict(predicate)} + try: + if kind == "field_equals": + path = str(predicate["path"]) + actual = _resolve_path(authoritative_state, path) + expected = predicate.get("expected") + passed = actual == expected + entry.update({"path": path, "actual": actual, "expected": expected, "passed": passed}) + elif kind == "digest_matches": + path = predicate.get("path") + material = authoritative_state if path in (None, "", ".") else _resolve_path( + authoritative_state, str(path) + ) + actual_digest = sha256_digest(material) + expected = str(predicate.get("expected_digest") or "") + passed = actual_digest == expected + entry.update( + { + "path": path or ".", + "actual_digest": actual_digest, + "expected_digest": expected, + "passed": passed, + } + ) + elif kind == "set_contains": + path = str(predicate["path"]) + collection = _resolve_path(authoritative_state, path) + member = predicate.get("member") + if isinstance(collection, (list, tuple, set, frozenset)): + passed = member in collection + else: + raise TypeError(f"set_contains path {path!r} is not a collection") + entry.update({"path": path, "member": member, "passed": passed}) + elif kind == "required_keys": + path = str(predicate.get("path") or ".") + mapping = ( + authoritative_state + if path in (".", "") + else _resolve_path(authoritative_state, path) + ) + if not isinstance(mapping, Mapping): + raise TypeError(f"required_keys path {path!r} is not a mapping") + keys = [str(k) for k in predicate.get("keys") or []] + missing = [k for k in keys if k not in mapping] + passed = not missing + entry.update({"path": path, "missing": missing, "passed": passed}) + elif kind == "state_machine_safe": + from ovk.adapters.deployment.state_machine import find_skipped_approval_paths + + machine = predicate.get("machine") + if not isinstance(machine, Mapping): + # Allow embedding under authoritative_state.machine + machine = authoritative_state.get("machine") + if not isinstance(machine, Mapping): + raise ValueError("state_machine_safe requires machine mapping") + counterexamples = find_skipped_approval_paths(dict(machine)) + passed = len(counterexamples) == 0 + entry.update({"counterexamples": counterexamples, "passed": passed}) + else: + entry.update({"passed": False, "error": f"unsupported predicate kind: {kind}"}) + return { + "ok": False, + "unsupported": True, + "message": f"unsupported predicate kind: {kind}", + "results": results + [entry], + } + except KeyError as exc: + entry.update({"passed": False, "error": f"missing path: {exc}"}) + return { + "ok": False, + "unsupported": True, + "message": f"missing path: {exc}", + "results": results + [entry], + } + except (TypeError, ValueError) as exc: + entry.update({"passed": False, "error": str(exc)}) + return { + "ok": False, + "unsupported": True, + "message": str(exc), + "results": results + [entry], + } + results.append(entry) + + all_passed = all(bool(item.get("passed")) for item in results) and bool(results) + return {"ok": all_passed, "unsupported": False, "results": results, "message": None} + + +class AuthoritativeStateAdapter(AssuranceBackendMixin): + """Exact predicates over declared authoritative state materials.""" + + backend_id = "auth-state-predicate" + adapter_id = "ovk-adapter-auth-state-predicate" + adapter_version = "0.1.0" + _guarantee_type = "exact_predicate" + + def __init__(self, *, timeout_ms: int = 5_000) -> None: + self.timeout_ms = timeout_ms + + def supported_mutation_dimensions(self) -> list[str]: + return ["alter_timeout", "remove_authority_predicate", "remove_success_predicate"] + + def manifest(self) -> BackendCapabilityManifest: + return BackendCapabilityManifest( + capability_id="auth-state-predicate-v1", + tool=BackendToolIdentity( + name=self.backend_id, + adapter=self.adapter_id, + adapter_version=self.adapter_version, + version=self.adapter_version, + ), + backend_class="custom", + guarantee=BackendGuaranteeDeclaration( + type="exact_predicate", + meaning_of_pass="All declared predicates hold over the authoritative state materials.", + meaning_of_fail="At least one declared predicate fails over the authoritative state.", + meaning_of_unknown="Authoritative state missing, unsupported predicate, or checker error.", + ), + input_languages=["json"], + supported_domains=["assurance", "authoritative_state"], + supported_property_kinds=["exact_predicate", "invariant"], + assumptions=["Caller supplies complete authoritative state materials and exact predicates."], + limits=["Does not invent state; missing authoritative_state is indeterminate."], + result_format="ovk.result.v1", + timeout_behavior="unknown", + assurance=AssuranceCapabilitySection( + assurance_capable=True, + verifier_identity=AssuranceVerifierIdentity( + verifier_id="ovk.assurance.auth_state_predicate", + implementation_name="AuthoritativeStateAdapter", + entry_point="ovk.adapters.assurance.auth_state.AuthoritativeStateAdapter", + pcs_profile_artifact_type="VerifierProfile.v1", + ), + decision_semantics=AssuranceDecisionSemantics( + decision_space=_DECISION_SPACE, # type: ignore[arg-type] + guarantee_class=GUARANTEE_CLASS, # type: ignore[arg-type] + supported_claim_ids=["claim.auth_state.exact_predicate"], + out_of_scope_claim_ids=["claim.formal.full_correctness"], + ), + mechanism_class="static_analysis", + determinism="deterministic", + evidence_channels=["raw_backend_result", "normalized_result", "counterexample", "compiled_obligation"], + replay_support=AssuranceReplaySupport( + supported=True, + compares_raw_digest=True, + compares_normalized_digest=True, + ), + configuration_snapshot_support=AssuranceSnapshotSupport( + supported=True, + exports_pcs_profile=True, + ), + mutation_dimensions=self.supported_mutation_dimensions(), # type: ignore[arg-type] + abstention=AssuranceAbstention(allows_abstention=True), + failure_behavior=AssuranceFailureBehavior(), + external_dependencies=[], + known_limits=["exact predicates over declared materials only"], + requires_authoritative_state=True, + ), + ) + + def snapshot_config( + self, + config: Mapping[str, Any] | None = None, + *, + environment: Mapping[str, str] | None = None, + ) -> ConfigurationSnapshot: + cfg = dict(config or {}) + cfg.setdefault("timeout_ms", self.timeout_ms) + predicates = cfg.get("predicates") + return build_configuration_snapshot( + backend_id=self.backend_id, + adapter_id=self.adapter_id, + adapter_version=self.adapter_version, + config=cfg, + environment=environment, + policy=predicates, + mechanism_class="static_analysis", + determinism="deterministic", + allows_abstention=True, + guarantee_class=GUARANTEE_CLASS, + decision_space=_DECISION_SPACE, + supported_claim_ids=["claim.auth_state.exact_predicate"], + out_of_scope_claim_ids=["claim.formal.full_correctness"], + assumptions=["Authoritative state materials are caller-declared and complete."], + known_blind_spots=["Does not discover undeclared state materials."], + entry_point="ovk.adapters.assurance.auth_state.AuthoritativeStateAdapter", + implementation_name="AuthoritativeStateAdapter", + timeout_ms=int(cfg["timeout_ms"]), + mutation_dimensions=self.supported_mutation_dimensions(), + ) + + def run_assurance( + self, + *, + input_data: Mapping[str, Any], + snapshot: ConfigurationSnapshot, + config: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + cfg = dict(snapshot.config) + if config: + cfg.update(dict(config)) + argv = [self.adapter_id, "exact-predicate"] + + auth_state = input_data.get("authoritative_state") + if not isinstance(auth_state, Mapping): + return indeterminate_run_outcome( + reason="missing_authoritative_state", + message="input.authoritative_state mapping is required", + raw_result={"input_keys": sorted(input_data.keys())}, + guarantee_class=GUARANTEE_CLASS, + command_argv=argv, + ) + + predicates = input_data.get("predicates") + if predicates is None: + predicates = cfg.get("predicates") + if not isinstance(predicates, list) or not predicates: + return indeterminate_run_outcome( + reason="unsupported_input", + message="non-empty predicates list is required", + raw_result={"authoritative_state_digest": sha256_digest(dict(auth_state))}, + guarantee_class=GUARANTEE_CLASS, + command_argv=argv, + ) + if not all(isinstance(item, Mapping) for item in predicates): + return indeterminate_run_outcome( + reason="unsupported_input", + message="each predicate must be a mapping", + guarantee_class=GUARANTEE_CLASS, + command_argv=argv, + ) + + # Optional binding: if config declares expected state digest, enforce it. + expected_state_digest = cfg.get("authoritative_state_digest") + actual_state_digest = sha256_digest(dict(auth_state)) + if isinstance(expected_state_digest, str) and expected_state_digest != actual_state_digest: + raw = { + "authoritative_state_digest": actual_state_digest, + "expected_authoritative_state_digest": expected_state_digest, + "passed": False, + } + return reject_outcome( + raw_result=raw, + normalized_extra={"counterexamples": [raw], "state_digest": actual_state_digest}, + stdout="reject: authoritative_state_digest mismatch", + guarantee_class=GUARANTEE_CLASS, + command_argv=argv, + ) + + evaluation = evaluate_predicates(dict(auth_state), list(predicates)) # type: ignore[arg-type] + raw = { + "authoritative_state_digest": actual_state_digest, + "predicate_count": len(predicates), + "evaluation": evaluation, + } + if evaluation.get("unsupported"): + return indeterminate_run_outcome( + reason="unsupported_input", + message=str(evaluation.get("message") or "unsupported predicate"), + raw_result=raw, + guarantee_class=GUARANTEE_CLASS, + command_argv=argv, + ) + if evaluation.get("ok"): + return accept_outcome( + raw_result=raw, + normalized_extra={"state_digest": actual_state_digest, "predicate_results": evaluation["results"]}, + guarantee_class=GUARANTEE_CLASS, + command_argv=argv, + ) + failures = [item for item in evaluation.get("results") or [] if not item.get("passed")] + return reject_outcome( + raw_result=raw, + normalized_extra={ + "state_digest": actual_state_digest, + "counterexamples": failures, + "predicate_results": evaluation.get("results"), + }, + stdout="reject: predicate failure", + guarantee_class=GUARANTEE_CLASS, + command_argv=argv, + ) diff --git a/ovk/adapters/assurance/lean_pfcore.py b/ovk/adapters/assurance/lean_pfcore.py new file mode 100644 index 0000000..49a12bf --- /dev/null +++ b/ovk/adapters/assurance/lean_pfcore.py @@ -0,0 +1,372 @@ +"""Lean / PF-Core assurance verifier (VA-09) — real toolchain only.""" + +from __future__ import annotations + +import os +import subprocess +import tempfile +from pathlib import Path +from shutil import which +from typing import Any, Mapping + +from ovk.adapters.assurance._support import ( + AssuranceBackendMixin, + accept_outcome, + indeterminate_run_outcome, + reject_outcome, +) +from ovk.assurance.pcs_hash import sha256_digest +from ovk.assurance.pin import resolve_pcs_root +from ovk.assurance.snapshot import ConfigurationSnapshot, build_configuration_snapshot +from ovk.core.execution_models import ( + AssuranceAbstention, + AssuranceCapabilitySection, + AssuranceDecisionSemantics, + AssuranceExternalDependency, + AssuranceFailureBehavior, + AssuranceReplaySupport, + AssuranceSnapshotSupport, + AssuranceVerifierIdentity, + BackendCapabilityManifest, + BackendGuaranteeDeclaration, + BackendToolIdentity, +) + +GUARANTEE_CLASS = "formally_checked" + +_DECISION_SPACE = [ + "accept", + "reject", + "indeterminate_execution_error", + "indeterminate_out_of_scope", + "indeterminate_insufficient_evidence", +] + + +def lean_available() -> bool: + return which("lean") is not None + + +def resolve_lean_root(config: Mapping[str, Any] | None = None) -> Path | None: + cfg = dict(config or {}) + explicit = cfg.get("lean_root") or os.environ.get("OVK_LEAN_ROOT") + if explicit: + path = Path(str(explicit)).expanduser().resolve() + return path if path.is_dir() else None + pcs = resolve_pcs_root() + if pcs is not None: + candidate = pcs / "lean" + if candidate.is_dir(): + return candidate + return None + + +class LeanPfCoreAssuranceAdapter(AssuranceBackendMixin): + """Real Lean and/or PF-Core lake project invocation for assurance. + + Ordinary ``LeanAdapter`` deterministic_fallback remains non-assurance. + Missing lean/lake => typed indeterminate. + """ + + backend_id = "lean-pfcore" + adapter_id = "ovk-adapter-lean-pfcore" + adapter_version = "0.1.0" + _guarantee_type = "formal_proof" + + def __init__(self, *, timeout_ms: int = 120_000) -> None: + self.timeout_ms = timeout_ms + + def supported_mutation_dimensions(self) -> list[str]: + return ["alter_timeout"] + + def manifest(self) -> BackendCapabilityManifest: + return BackendCapabilityManifest( + capability_id="lean-pfcore-assurance-v1", + tool=BackendToolIdentity( + name=self.backend_id, + adapter=self.adapter_id, + adapter_version=self.adapter_version, + version=self.adapter_version, + ), + backend_class="custom", + guarantee=BackendGuaranteeDeclaration( + type="formal_proof", + meaning_of_pass="Lean/PF-Core toolchain accepted the pinned obligation.", + meaning_of_fail="Lean/PF-Core rejected the obligation or reported errors.", + meaning_of_unknown="Lean/lake missing, timed out, or unsupported input.", + ), + input_languages=["lean", "json"], + supported_domains=["assurance", "formal"], + supported_property_kinds=["formal_proof"], + assumptions=["Lean toolchain and optional pcs-core/lean lake project are real."], + limits=[ + "Ordinary lean deterministic_fallback adapter is NOT assurance_capable.", + "Never invents passes without toolchain execution.", + ], + result_format="ovk.result.v1", + timeout_behavior="unknown", + assurance=AssuranceCapabilitySection( + assurance_capable=True, + verifier_identity=AssuranceVerifierIdentity( + verifier_id="ovk.assurance.lean_pfcore", + implementation_name="LeanPfCoreAssuranceAdapter", + entry_point="ovk.adapters.assurance.lean_pfcore.LeanPfCoreAssuranceAdapter", + pcs_profile_artifact_type="VerifierProfile.v1", + ), + decision_semantics=AssuranceDecisionSemantics( + decision_space=_DECISION_SPACE, # type: ignore[arg-type] + guarantee_class=GUARANTEE_CLASS, # type: ignore[arg-type] + supported_claim_ids=["claim.lean.pfcore"], + out_of_scope_claim_ids=["claim.full_global_noninterference"], + ), + mechanism_class="formal_proof", + determinism="deterministic", + evidence_channels=[ + "stdout", + "stderr", + "raw_backend_result", + "normalized_result", + "proof", + "compiled_obligation", + ], + replay_support=AssuranceReplaySupport( + supported=True, + compares_raw_digest=True, + compares_normalized_digest=True, + ), + configuration_snapshot_support=AssuranceSnapshotSupport( + supported=True, + exports_pcs_profile=True, + ), + mutation_dimensions=self.supported_mutation_dimensions(), # type: ignore[arg-type] + abstention=AssuranceAbstention(allows_abstention=True), + failure_behavior=AssuranceFailureBehavior(), + external_dependencies=[ + AssuranceExternalDependency( + dependency_id="lean", + kind="toolchain", + identity="lean", + optional=False, + ) + ], + known_limits=["Requires lean; lake used when target is a lake package module."], + ), + ) + + def snapshot_config( + self, + config: Mapping[str, Any] | None = None, + *, + environment: Mapping[str, str] | None = None, + ) -> ConfigurationSnapshot: + cfg = dict(config or {}) + cfg.setdefault("timeout_ms", self.timeout_ms) + cfg.setdefault("mode", cfg.get("mode") or "lean_source") + lean_root = resolve_lean_root(cfg) + if lean_root is not None: + cfg.setdefault("lean_root", str(lean_root)) + return build_configuration_snapshot( + backend_id=self.backend_id, + adapter_id=self.adapter_id, + adapter_version=self.adapter_version, + config=cfg, + environment=environment, + mechanism_class="formal_proof", + determinism="deterministic", + allows_abstention=True, + guarantee_class=GUARANTEE_CLASS, + decision_space=_DECISION_SPACE, + supported_claim_ids=["claim.lean.pfcore"], + out_of_scope_claim_ids=["claim.full_global_noninterference"], + assumptions=["Real Lean/PF-Core execution only."], + known_blind_spots=["Does not treat deterministic_fallback as formal proof."], + external_dependencies=[ + {"dependency_id": "lean", "kind": "toolchain", "identity": "lean", "optional": False} + ], + entry_point="ovk.adapters.assurance.lean_pfcore.LeanPfCoreAssuranceAdapter", + implementation_name="LeanPfCoreAssuranceAdapter", + timeout_ms=int(cfg["timeout_ms"]), + mutation_dimensions=self.supported_mutation_dimensions(), + ) + + def run_assurance( + self, + *, + input_data: Mapping[str, Any], + snapshot: ConfigurationSnapshot, + config: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + cfg = dict(snapshot.config) + if config: + cfg.update(dict(config)) + if not lean_available(): + raise FileNotFoundError("lean binary not found") + + mode = str(input_data.get("mode") or cfg.get("mode") or "lean_source") + timeout_ms = int(cfg.get("timeout_ms") or self.timeout_ms) + lean_bin = which("lean") + assert lean_bin is not None + + if mode == "lake_env": + return self._run_lake_env(input_data=input_data, cfg=cfg, timeout_ms=timeout_ms) + + lean_source = input_data.get("lean_source") + lean_file = input_data.get("lean_file") + if isinstance(lean_file, str) and lean_file.strip(): + path = Path(lean_file).expanduser().resolve() + if not path.is_file(): + return indeterminate_run_outcome( + reason="unsupported_input", + message=f"lean_file not found: {path}", + guarantee_class=GUARANTEE_CLASS, + command_argv=[lean_bin, str(path)], + ) + source_text = path.read_text(encoding="utf-8") + return self._run_lean_file( + path=path, + source_text=source_text, + lean_bin=lean_bin, + timeout_ms=timeout_ms, + cleanup=None, + ) + + if not isinstance(lean_source, str) or not lean_source.strip(): + return indeterminate_run_outcome( + reason="unsupported_input", + message="lean_source or lean_file is required (no silent default smoke obligation)", + guarantee_class=GUARANTEE_CLASS, + command_argv=[lean_bin, ""], + ) + + with tempfile.TemporaryDirectory(prefix="ovk-lean-assurance-") as tmp: + path = Path(tmp) / "Obligation.lean" + path.write_text(lean_source, encoding="utf-8") + return self._run_lean_file( + path=path, + source_text=lean_source, + lean_bin=lean_bin, + timeout_ms=timeout_ms, + cleanup=tmp, + ) + + def _run_lean_file( + self, + *, + path: Path, + source_text: str, + lean_bin: str, + timeout_ms: int, + cleanup: str | None, + ) -> dict[str, Any]: + command = [lean_bin, str(path)] + try: + completed = subprocess.run( + command, + capture_output=True, + text=True, + timeout=max(timeout_ms / 1000.0, 0.1), + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise TimeoutError(f"lean timed out after {timeout_ms}ms") from exc + finally: + # cleanup handled by TemporaryDirectory context when used + _ = cleanup + + raw = { + "command_tool": "lean", + "exit_code": completed.returncode, + "source_digest": sha256_digest(source_text), + "mode": "lean_source", + # stdout/stderr captured on the assurance outcome, not in raw digest body + } + if completed.returncode == 0: + return accept_outcome( + raw_result=raw, + normalized_extra={"proof": {"tool": "lean", "source_digest": raw["source_digest"]}}, + stdout=completed.stdout or "pass", + stderr=completed.stderr or "", + guarantee_class=GUARANTEE_CLASS, + command_argv=[lean_bin, ""], + exit_code=0, + ) + return reject_outcome( + raw_result=raw, + normalized_extra={ + "counterexamples": [{"stderr_digest": sha256_digest(completed.stderr or "")}], + "proof": {"tool": "lean", "source_digest": raw["source_digest"], "failed": True}, + }, + stdout=completed.stdout or "fail", + stderr=completed.stderr or "", + guarantee_class=GUARANTEE_CLASS, + command_argv=[lean_bin, ""], + exit_code=completed.returncode, + ) + + def _run_lake_env( + self, + *, + input_data: Mapping[str, Any], + cfg: Mapping[str, Any], + timeout_ms: int, + ) -> dict[str, Any]: + lake = which("lake") + if lake is None: + raise FileNotFoundError("lake binary not found") + lean_root = resolve_lean_root(cfg) + if lean_root is None: + return indeterminate_run_outcome( + reason="unsupported_input", + message="lean_root / pcs-core lean project not found for lake_env mode", + guarantee_class=GUARANTEE_CLASS, + command_argv=["lake", "env", "lean"], + ) + module = str(input_data.get("module") or cfg.get("module") or "PFCore.Basic") + # `lake env lean --run` is not universal; use `lake env lean` with a tiny importer. + with tempfile.TemporaryDirectory(prefix="ovk-lake-assurance-") as tmp: + stub = Path(tmp) / "ImportCheck.lean" + stub.write_text(f"import {module}\n#check True\n", encoding="utf-8") + command = [lake, "env", "lean", str(stub)] + try: + completed = subprocess.run( + command, + capture_output=True, + text=True, + timeout=max(timeout_ms / 1000.0, 0.1), + check=False, + cwd=str(lean_root), + ) + except subprocess.TimeoutExpired as exc: + raise TimeoutError(f"lake env lean timed out after {timeout_ms}ms") from exc + + raw = { + "command": command, + "cwd": str(lean_root), + "module": module, + "exit_code": completed.returncode, + "stdout": completed.stdout, + "stderr": completed.stderr, + "mode": "lake_env", + "source_digest": sha256_digest({"module": module, "stub": stub.read_text(encoding="utf-8")}), + } + if completed.returncode == 0: + return accept_outcome( + raw_result=raw, + normalized_extra={"proof": {"tool": "lake+lean", "module": module}}, + stdout=completed.stdout or "pass", + stderr=completed.stderr or "", + guarantee_class=GUARANTEE_CLASS, + command_argv=command, + ) + return reject_outcome( + raw_result=raw, + normalized_extra={ + "counterexamples": [{"stderr": (completed.stderr or "")[:2000]}], + "proof": {"tool": "lake+lean", "module": module, "failed": True}, + }, + stdout=completed.stdout or "fail", + stderr=completed.stderr or "", + guarantee_class=GUARANTEE_CLASS, + command_argv=command, + exit_code=completed.returncode, + ) diff --git a/ovk/adapters/assurance/model_judge.py b/ovk/adapters/assurance/model_judge.py new file mode 100644 index 0000000..325f493 --- /dev/null +++ b/ovk/adapters/assurance/model_judge.py @@ -0,0 +1,331 @@ +"""Optional stochastic model-judge assurance verifier (VA-11).""" + +from __future__ import annotations + +from typing import Any, Callable, Mapping, Protocol + +from ovk.adapters.assurance._support import ( + AssuranceBackendMixin, + accept_outcome, + indeterminate_run_outcome, + reject_outcome, +) +from ovk.assurance.pcs_hash import sha256_digest +from ovk.assurance.snapshot import ConfigurationSnapshot, build_configuration_snapshot +from ovk.core.execution_models import ( + AssuranceAbstention, + AssuranceCapabilitySection, + AssuranceDecisionSemantics, + AssuranceExternalDependency, + AssuranceFailureBehavior, + AssuranceReplaySupport, + AssuranceSnapshotSupport, + AssuranceVerifierIdentity, + BackendCapabilityManifest, + BackendGuaranteeDeclaration, + BackendToolIdentity, +) + +GUARANTEE_CLASS = "empirically_measured" + +_DECISION_SPACE = [ + "accept", + "reject", + "indeterminate_execution_error", + "indeterminate_out_of_scope", + "indeterminate_insufficient_evidence", +] + + +class JudgeClient(Protocol): + def judge(self, *, prompt: str, model: str, decoding: Mapping[str, Any], rubric: Mapping[str, Any]) -> dict[str, Any]: + ... + + +class ContractFakeJudgeClient: + """CI contract fake that exercises real adapter paths (no network). + + Deterministic given prompt/model/decoding/rubric; still labeled stochastic + at the adapter level because production clients may be nondeterministic. + """ + + def judge( + self, + *, + prompt: str, + model: str, + decoding: Mapping[str, Any], + rubric: Mapping[str, Any], + ) -> dict[str, Any]: + material = { + "prompt": prompt, + "model": model, + "decoding": dict(decoding), + "rubric": dict(rubric), + } + digest = sha256_digest(material) + # Stable pseudo-score in [0, 1] from digest bytes. + score_int = int(digest.replace("sha256:", "")[:8], 16) + score = (score_int % 1000) / 1000.0 + threshold_raw = rubric.get("threshold", "0.5") + threshold = float(threshold_raw) + # Canonical JSON forbids floats — emit score as decimal string. + score_str = f"{score:.3f}" + return { + "provider": "contract_fake", + "score": score_str, + "passed": score >= threshold, + "rationale": f"contract_fake score={score_str} threshold={threshold_raw}", + "raw_response_digest": digest, + } + + +def default_judge_client(kind: str | None = None) -> JudgeClient: + resolved = (kind or "contract_fake").strip().lower() + if resolved in {"contract_fake", "fake", "ci"}: + return ContractFakeJudgeClient() + if resolved in {"live", "network"}: + raise FileNotFoundError( + "live model-judge provider is opt-in and not configured " + "(set judge_client=contract_fake for CI)" + ) + raise FileNotFoundError(f"unknown model-judge client: {resolved}") + + +class ModelJudgeAdapter(AssuranceBackendMixin): + """Stochastic model-judge; cannot upgrade guarantee class beyond empirical.""" + + backend_id = "model-judge" + adapter_id = "ovk-adapter-model-judge" + adapter_version = "0.1.0" + _guarantee_type = "model_judgment" + + def __init__( + self, + *, + timeout_ms: int = 30_000, + judge_client_factory: Callable[[str | None], JudgeClient] | None = None, + ) -> None: + self.timeout_ms = timeout_ms + self._judge_client_factory = judge_client_factory or default_judge_client + + def supported_mutation_dimensions(self) -> list[str]: + return ["alter_timeout", "change_rubric", "change_prompt", "change_threshold", "ensemble_quorum"] + + def manifest(self) -> BackendCapabilityManifest: + return BackendCapabilityManifest( + capability_id="model-judge-v1", + tool=BackendToolIdentity( + name=self.backend_id, + adapter=self.adapter_id, + adapter_version=self.adapter_version, + version=self.adapter_version, + ), + backend_class="custom", + guarantee=BackendGuaranteeDeclaration( + type="model_judgment", + meaning_of_pass="Model judge score met rubric threshold (empirical only).", + meaning_of_fail="Model judge score missed rubric threshold.", + meaning_of_unknown="Judge client missing, timed out, or unsupported input.", + ), + input_languages=["json", "text"], + supported_domains=["assurance", "judgment"], + supported_property_kinds=["model_judgment"], + assumptions=["Judgments are stochastic/empirical; never formal."], + limits=["Cannot upgrade guarantee class; live network is opt-in."], + result_format="ovk.result.v1", + timeout_behavior="unknown", + assurance=AssuranceCapabilitySection( + assurance_capable=True, + verifier_identity=AssuranceVerifierIdentity( + verifier_id="ovk.assurance.model_judge", + implementation_name="ModelJudgeAdapter", + entry_point="ovk.adapters.assurance.model_judge.ModelJudgeAdapter", + pcs_profile_artifact_type="VerifierProfile.v1", + ), + decision_semantics=AssuranceDecisionSemantics( + decision_space=_DECISION_SPACE, # type: ignore[arg-type] + guarantee_class=GUARANTEE_CLASS, # type: ignore[arg-type] + supported_claim_ids=["claim.model_judge.empirical"], + out_of_scope_claim_ids=[ + "claim.formal.full_correctness", + "claim.certificate_checked", + "claim.formally_checked", + ], + ), + mechanism_class="model_judge", + determinism="stochastic", + evidence_channels=[ + "raw_backend_result", + "normalized_result", + "model_judgment", + "compiled_obligation", + ], + replay_support=AssuranceReplaySupport( + supported=True, + compares_raw_digest=False, + compares_normalized_digest=False, + notes="Stochastic; replay declares nondeterminism rather than matched digests.", + ), + configuration_snapshot_support=AssuranceSnapshotSupport( + supported=True, + exports_pcs_profile=True, + ), + mutation_dimensions=self.supported_mutation_dimensions(), # type: ignore[arg-type] + abstention=AssuranceAbstention(allows_abstention=True), + failure_behavior=AssuranceFailureBehavior(), + external_dependencies=[ + AssuranceExternalDependency( + dependency_id="model-judge-client", + kind="service", + identity="model-judge", + optional=True, + ) + ], + known_limits=["stochastic; cannot claim formal guarantees"], + ), + ) + + def snapshot_config( + self, + config: Mapping[str, Any] | None = None, + *, + environment: Mapping[str, str] | None = None, + ) -> ConfigurationSnapshot: + cfg = dict(config or {}) + cfg.setdefault("timeout_ms", self.timeout_ms) + cfg.setdefault("model", cfg.get("model") or "contract-fake-v1") + cfg.setdefault("judge_client", cfg.get("judge_client") or "contract_fake") + cfg.setdefault("decoding", cfg.get("decoding") or {"temperature": "0", "max_tokens": 256}) + default_rubric = {"threshold": "0.5"} + if not isinstance(cfg.get("rubric"), dict): + cfg["rubric"] = dict(default_rubric) + else: + rubric = dict(cfg["rubric"]) + if "threshold" in rubric and isinstance(rubric["threshold"], float): + rubric["threshold"] = format(rubric["threshold"], "f").rstrip("0").rstrip(".") or "0" + rubric.setdefault("threshold", "0.5") + cfg["rubric"] = rubric + cfg.setdefault("prompt", cfg.get("prompt") or "") + return build_configuration_snapshot( + backend_id=self.backend_id, + adapter_id=self.adapter_id, + adapter_version=self.adapter_version, + config=cfg, + environment=environment, + model={"model": cfg.get("model"), "judge_client": cfg.get("judge_client")}, + prompt=cfg.get("prompt"), + rubric=cfg.get("rubric"), + threshold=(cfg.get("rubric") or {}).get("threshold") if isinstance(cfg.get("rubric"), dict) else None, + mechanism_class="model_judge", + determinism="stochastic", + allows_abstention=True, + guarantee_class=GUARANTEE_CLASS, + decision_space=_DECISION_SPACE, + supported_claim_ids=["claim.model_judge.empirical"], + out_of_scope_claim_ids=["claim.formal.full_correctness", "claim.formally_checked"], + assumptions=["Model judgments are empirical and may vary."], + known_blind_spots=["Cannot upgrade to formally_checked."], + external_dependencies=[ + { + "dependency_id": "model-judge-client", + "kind": "service", + "identity": str(cfg.get("judge_client")), + "optional": True, + } + ], + entry_point="ovk.adapters.assurance.model_judge.ModelJudgeAdapter", + implementation_name="ModelJudgeAdapter", + timeout_ms=int(cfg["timeout_ms"]), + mutation_dimensions=self.supported_mutation_dimensions(), + ) + + def run_assurance( + self, + *, + input_data: Mapping[str, Any], + snapshot: ConfigurationSnapshot, + config: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + cfg = dict(snapshot.config) + if config: + cfg.update(dict(config)) + argv = [self.adapter_id, "judge"] + + prompt = input_data.get("prompt") + if prompt is None: + prompt = cfg.get("prompt") + if not isinstance(prompt, str) or not prompt.strip(): + # Allow subject text under input.subject + subject = input_data.get("subject") + if isinstance(subject, str) and subject.strip(): + prompt = subject + else: + return indeterminate_run_outcome( + reason="unsupported_input", + message="prompt or subject text is required", + guarantee_class=GUARANTEE_CLASS, + command_argv=argv, + ) + + model = str(input_data.get("model") or cfg.get("model") or "contract-fake-v1") + decoding = dict(input_data.get("decoding") or cfg.get("decoding") or {}) + rubric = dict(input_data.get("rubric") or cfg.get("rubric") or {"threshold": "0.5"}) + if isinstance(rubric.get("threshold"), float): + rubric["threshold"] = format(rubric["threshold"], "f").rstrip("0").rstrip(".") or "0" + # Strip floats from decoding for digest stability. + decoding_clean: dict[str, Any] = {} + for key, value in decoding.items(): + if isinstance(value, float): + decoding_clean[key] = format(value, "f").rstrip("0").rstrip(".") or "0" + else: + decoding_clean[key] = value + decoding = decoding_clean + client_kind = str(input_data.get("judge_client") or cfg.get("judge_client") or "contract_fake") + + try: + client = self._judge_client_factory(client_kind) + except FileNotFoundError: + raise + + judgment = client.judge(prompt=prompt, model=model, decoding=decoding, rubric=rubric) + # Ensure judgment payload is Canonical-JSON safe (no floats). + if isinstance(judgment.get("score"), float): + judgment = {**judgment, "score": format(judgment["score"], "f").rstrip("0").rstrip(".") or "0"} + raw = { + "prompt_digest": sha256_digest(prompt), + "model": model, + "decoding": decoding, + "decoding_digest": sha256_digest(decoding), + "rubric": rubric, + "rubric_digest": sha256_digest(rubric), + "judgment": judgment, + "judge_client": client_kind, + "seed": input_data.get("seed") or cfg.get("seed"), + } + # Hard invariant: never claim a stronger guarantee than empirical. + if judgment.get("guarantee_class") in {"formally_checked", "certificate_checked", "human_reviewed"}: + judgment = {**judgment, "guarantee_class": GUARANTEE_CLASS} + + if bool(judgment.get("passed")): + return accept_outcome( + raw_result=raw, + normalized_extra={ + "model_judgment": judgment, + "guarantee_class": GUARANTEE_CLASS, + }, + stdout=str(judgment.get("rationale") or "accept"), + guarantee_class=GUARANTEE_CLASS, + command_argv=argv, + ) + return reject_outcome( + raw_result=raw, + normalized_extra={ + "model_judgment": judgment, + "counterexamples": [judgment], + "guarantee_class": GUARANTEE_CLASS, + }, + stdout=str(judgment.get("rationale") or "reject"), + guarantee_class=GUARANTEE_CLASS, + command_argv=argv, + ) diff --git a/ovk/adapters/assurance/opa_policy.py b/ovk/adapters/assurance/opa_policy.py new file mode 100644 index 0000000..e50dc89 --- /dev/null +++ b/ovk/adapters/assurance/opa_policy.py @@ -0,0 +1,295 @@ +"""OPA policy assurance verifier (VA-08) — real ``opa eval`` only.""" + +from __future__ import annotations + +import json +import tempfile +from pathlib import Path +from shutil import which +from typing import Any, Mapping + +from ovk.adapters.assurance._support import ( + AssuranceBackendMixin, + accept_outcome, + indeterminate_run_outcome, + reject_outcome, +) +from ovk.adapters.opa.optional_runner import run_opa_policy +from ovk.assurance.pcs_hash import sha256_digest +from ovk.assurance.snapshot import ConfigurationSnapshot, build_configuration_snapshot +from ovk.core.execution_models import ( + AssuranceAbstention, + AssuranceCapabilitySection, + AssuranceDecisionSemantics, + AssuranceExternalDependency, + AssuranceFailureBehavior, + AssuranceReplaySupport, + AssuranceSnapshotSupport, + AssuranceVerifierIdentity, + BackendCapabilityManifest, + BackendGuaranteeDeclaration, + BackendToolIdentity, +) + +GUARANTEE_CLASS = "certificate_checked" + +_DECISION_SPACE = [ + "accept", + "reject", + "indeterminate_execution_error", + "indeterminate_out_of_scope", + "indeterminate_insufficient_evidence", +] + + +class OpaPolicyAssuranceAdapter(AssuranceBackendMixin): + """Assurance-capable OPA policy verifier using real ``opa eval``. + + Distinct from ordinary ``opa-native`` lane routing. Missing ``opa`` binary + yields typed indeterminate (never fabricated pass). Cedar is not involved. + """ + + backend_id = "opa-policy" + adapter_id = "ovk-adapter-opa-policy-assurance" + adapter_version = "0.1.0" + _guarantee_type = "policy_evaluation" + + def __init__(self, *, timeout_ms: int = 15_000) -> None: + self.timeout_ms = timeout_ms + + def supported_mutation_dimensions(self) -> list[str]: + return ["alter_timeout", "policy_bundle"] + + def manifest(self) -> BackendCapabilityManifest: + return BackendCapabilityManifest( + capability_id="opa-policy-assurance-v1", + tool=BackendToolIdentity( + name=self.backend_id, + adapter=self.adapter_id, + adapter_version=self.adapter_version, + version=self.adapter_version, + ), + backend_class="policy_engine", + guarantee=BackendGuaranteeDeclaration( + type="policy_evaluation", + meaning_of_pass="opa eval reported no violations for the pinned policy/query.", + meaning_of_fail="opa eval reported one or more violations.", + meaning_of_unknown="opa binary missing, timed out, or returned invalid output.", + ), + input_languages=["json", "rego"], + supported_domains=["assurance", "policy"], + supported_property_kinds=["policy_evaluation", "invariant"], + assumptions=["Rego policy and query are pinned in the configuration snapshot."], + limits=["Requires opa; never falls back to Cedar or deterministic fake pass."], + result_format="ovk.result.v1", + counterexample_format="policy_violation", + timeout_behavior="unknown", + assurance=AssuranceCapabilitySection( + assurance_capable=True, + verifier_identity=AssuranceVerifierIdentity( + verifier_id="ovk.assurance.opa_policy", + implementation_name="OpaPolicyAssuranceAdapter", + entry_point="ovk.adapters.assurance.opa_policy.OpaPolicyAssuranceAdapter", + pcs_profile_artifact_type="VerifierProfile.v1", + ), + decision_semantics=AssuranceDecisionSemantics( + decision_space=_DECISION_SPACE, # type: ignore[arg-type] + guarantee_class=GUARANTEE_CLASS, # type: ignore[arg-type] + supported_claim_ids=["claim.opa.policy_evaluation"], + out_of_scope_claim_ids=["claim.formal.full_correctness"], + ), + mechanism_class="policy_engine", + determinism="deterministic", + evidence_channels=[ + "stdout", + "stderr", + "raw_backend_result", + "normalized_result", + "counterexample", + "compiled_obligation", + ], + replay_support=AssuranceReplaySupport( + supported=True, + compares_raw_digest=True, + compares_normalized_digest=True, + ), + configuration_snapshot_support=AssuranceSnapshotSupport( + supported=True, + exports_pcs_profile=True, + ), + mutation_dimensions=self.supported_mutation_dimensions(), # type: ignore[arg-type] + abstention=AssuranceAbstention(allows_abstention=True), + failure_behavior=AssuranceFailureBehavior(), + external_dependencies=[ + AssuranceExternalDependency( + dependency_id="opa", + kind="binary", + identity="opa", + optional=False, + ) + ], + known_limits=["Requires local opa binary; Cedar is never used."], + ), + ) + + def snapshot_config( + self, + config: Mapping[str, Any] | None = None, + *, + environment: Mapping[str, str] | None = None, + ) -> ConfigurationSnapshot: + cfg = dict(config or {}) + cfg.setdefault("timeout_ms", self.timeout_ms) + cfg.setdefault("query", cfg.get("query") or "data.ovk.assurance.allow") + policy_text = cfg.get("policy") + policy_path = cfg.get("policy_path") + policy_material: Any + if isinstance(policy_text, str) and policy_text.strip(): + policy_material = {"inline": policy_text} + elif policy_path: + policy_material = {"path": str(policy_path)} + else: + policy_material = {"path": None} + return build_configuration_snapshot( + backend_id=self.backend_id, + adapter_id=self.adapter_id, + adapter_version=self.adapter_version, + config=cfg, + environment=environment, + policy=policy_material, + mechanism_class="policy_engine", + determinism="deterministic", + allows_abstention=True, + guarantee_class=GUARANTEE_CLASS, + decision_space=_DECISION_SPACE, + supported_claim_ids=["claim.opa.policy_evaluation"], + out_of_scope_claim_ids=["claim.formal.full_correctness"], + assumptions=["Real opa eval only; missing opa is indeterminate."], + known_blind_spots=["Does not evaluate Cedar policies."], + external_dependencies=[ + {"dependency_id": "opa", "kind": "binary", "identity": "opa", "optional": False} + ], + entry_point="ovk.adapters.assurance.opa_policy.OpaPolicyAssuranceAdapter", + implementation_name="OpaPolicyAssuranceAdapter", + timeout_ms=int(cfg["timeout_ms"]), + mutation_dimensions=self.supported_mutation_dimensions(), + ) + + def run_assurance( + self, + *, + input_data: Mapping[str, Any], + snapshot: ConfigurationSnapshot, + config: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + cfg = dict(snapshot.config) + if config: + cfg.update(dict(config)) + command_argv = ["opa", "eval", str(cfg.get("query") or "data.ovk.assurance.allow")] + + if which("opa") is None: + raise FileNotFoundError("opa binary not found") + + policy_text = input_data.get("policy") or cfg.get("policy") + policy_path_raw = input_data.get("policy_path") or cfg.get("policy_path") + query = str(input_data.get("query") or cfg.get("query") or "data.ovk.assurance.allow") + opa_input = input_data.get("input") + if opa_input is None: + opa_input = {k: v for k, v in input_data.items() if k not in {"policy", "policy_path", "query"}} + if not isinstance(opa_input, Mapping): + return indeterminate_run_outcome( + reason="unsupported_input", + message="OPA input must be a JSON object", + guarantee_class=GUARANTEE_CLASS, + command_argv=command_argv, + ) + + timeout_ms = int(cfg.get("timeout_ms") or self.timeout_ms) + with tempfile.TemporaryDirectory(prefix="ovk-opa-assurance-") as tmp: + tmp_path = Path(tmp) + if isinstance(policy_text, str) and policy_text.strip(): + policy_file = tmp_path / "policy.rego" + policy_file.write_text(policy_text, encoding="utf-8") + policy_digest = sha256_digest(policy_text) + elif policy_path_raw: + policy_file = Path(str(policy_path_raw)).expanduser().resolve() + if not policy_file.is_file(): + return indeterminate_run_outcome( + reason="unsupported_input", + message=f"policy_path not found: {policy_file}", + guarantee_class=GUARANTEE_CLASS, + command_argv=command_argv, + ) + policy_digest = sha256_digest(policy_file.read_text(encoding="utf-8")) + else: + return indeterminate_run_outcome( + reason="unsupported_input", + message="policy or policy_path is required", + guarantee_class=GUARANTEE_CLASS, + command_argv=command_argv, + ) + + input_file = tmp_path / "input.json" + input_file.write_text(json.dumps(dict(opa_input), sort_keys=True), encoding="utf-8") + command_argv = [ + "opa", + "eval", + "--format", + "json", + "--data", + str(policy_file), + "--input", + str(input_file), + query, + ] + result = run_opa_policy( + policy_path=policy_file, + input_path=input_file, + query=query, + timeout_seconds=max(int(timeout_ms / 1000), 1), + cwd=tmp_path, + ) + + raw = { + "opa_result": result, + "policy_digest": policy_digest, + "query": query, + "input_digest": sha256_digest(dict(opa_input)), + } + status = str(result.get("status") or "") + if status == "unknown" and "not found" in str(result.get("reason") or "").lower(): + raise FileNotFoundError(str(result.get("reason") or "opa binary not found")) + if status == "unknown" and "timed out" in str(result.get("reason") or "").lower(): + raise TimeoutError(str(result.get("reason") or "opa timed out")) + if status == "error": + return indeterminate_run_outcome( + reason="parser_failure", + message=str(result.get("reason") or "opa error"), + raw_result=raw, + guarantee_class=GUARANTEE_CLASS, + command_argv=command_argv, + ) + if status == "unknown": + return indeterminate_run_outcome( + reason="other", + message=str(result.get("reason") or "opa unknown"), + raw_result=raw, + guarantee_class=GUARANTEE_CLASS, + command_argv=command_argv, + ) + violations = list(result.get("violations") or []) + # For allow-style queries, empty/false means reject when query value is boolean false. + if status == "pass" and not violations: + return accept_outcome( + raw_result=raw, + normalized_extra={"violations": [], "policy_digest": policy_digest}, + guarantee_class=GUARANTEE_CLASS, + command_argv=command_argv, + ) + return reject_outcome( + raw_result=raw, + normalized_extra={"counterexamples": violations, "violations": violations, "policy_digest": policy_digest}, + stdout="reject: opa violations", + guarantee_class=GUARANTEE_CLASS, + command_argv=command_argv, + ) diff --git a/ovk/adapters/assurance/pytest_suite.py b/ovk/adapters/assurance/pytest_suite.py new file mode 100644 index 0000000..f08269f --- /dev/null +++ b/ovk/adapters/assurance/pytest_suite.py @@ -0,0 +1,376 @@ +"""Pytest / integration-test assurance verifier (VA-07).""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Any, Mapping + +from ovk.adapters.assurance._support import ( + AssuranceBackendMixin, + accept_outcome, + indeterminate_run_outcome, + reject_outcome, +) +from ovk.assurance.pcs_hash import sha256_digest +from ovk.assurance.snapshot import ConfigurationSnapshot, build_configuration_snapshot +from ovk.core.execution_models import ( + AssuranceAbstention, + AssuranceCapabilitySection, + AssuranceDecisionSemantics, + AssuranceExternalDependency, + AssuranceFailureBehavior, + AssuranceReplaySupport, + AssuranceSnapshotSupport, + AssuranceVerifierIdentity, + BackendCapabilityManifest, + BackendGuaranteeDeclaration, + BackendToolIdentity, +) + +GUARANTEE_CLASS = "runtime_observed" + +_DECISION_SPACE = [ + "accept", + "reject", + "indeterminate_execution_error", + "indeterminate_out_of_scope", + "indeterminate_insufficient_evidence", +] + + +def _parse_junit(path: Path) -> dict[str, Any]: + if not path.is_file(): + return {"tests": 0, "failures": 0, "errors": 0, "skipped": 0, "cases": []} + tree = ET.parse(path) + root = tree.getroot() + suites = [root] if root.tag == "testsuite" else list(root.findall("testsuite")) + tests = failures = errors = skipped = 0 + cases: list[dict[str, Any]] = [] + for suite in suites: + tests += int(suite.attrib.get("tests") or 0) + failures += int(suite.attrib.get("failures") or 0) + errors += int(suite.attrib.get("errors") or 0) + skipped += int(suite.attrib.get("skipped") or 0) + for case in suite.findall("testcase"): + status = "passed" + if case.find("failure") is not None: + status = "failed" + elif case.find("error") is not None: + status = "error" + elif case.find("skipped") is not None: + status = "skipped" + cases.append( + { + "classname": case.attrib.get("classname"), + "name": case.attrib.get("name"), + "status": status, + } + ) + return { + "tests": tests, + "failures": failures, + "errors": errors, + "skipped": skipped, + "cases": cases, + } + + +class PytestSuiteAdapter(AssuranceBackendMixin): + """Real pytest runner; observational/runtime_observed guarantee only.""" + + backend_id = "pytest-suite" + adapter_id = "ovk-adapter-pytest-suite" + adapter_version = "0.1.0" + _guarantee_type = "runtime_observation" + + def __init__(self, *, timeout_ms: int = 60_000) -> None: + self.timeout_ms = timeout_ms + + def supported_mutation_dimensions(self) -> list[str]: + return ["alter_timeout", "reduce_test_subset", "change_threshold"] + + def manifest(self) -> BackendCapabilityManifest: + return BackendCapabilityManifest( + capability_id="pytest-suite-v1", + tool=BackendToolIdentity( + name=self.backend_id, + adapter=self.adapter_id, + adapter_version=self.adapter_version, + version=self.adapter_version, + ), + backend_class="custom", + guarantee=BackendGuaranteeDeclaration( + type="runtime_observation", + meaning_of_pass="Pinned pytest suite subset exited successfully (observational only).", + meaning_of_fail="Pinned pytest suite reported failures or errors.", + meaning_of_unknown="pytest missing, timed out, or suite path unsupported.", + ), + input_languages=["python", "json"], + supported_domains=["assurance", "testing"], + supported_property_kinds=["runtime_observation"], + assumptions=["Suite path is pinned; results are observational, not formal."], + limits=["Never upgrades to formally_checked; junit/raw capture only."], + result_format="ovk.result.v1", + timeout_behavior="unknown", + assurance=AssuranceCapabilitySection( + assurance_capable=True, + verifier_identity=AssuranceVerifierIdentity( + verifier_id="ovk.assurance.pytest_suite", + implementation_name="PytestSuiteAdapter", + entry_point="ovk.adapters.assurance.pytest_suite.PytestSuiteAdapter", + pcs_profile_artifact_type="VerifierProfile.v1", + ), + decision_semantics=AssuranceDecisionSemantics( + decision_space=_DECISION_SPACE, # type: ignore[arg-type] + guarantee_class=GUARANTEE_CLASS, # type: ignore[arg-type] + supported_claim_ids=["claim.pytest.runtime_observed"], + out_of_scope_claim_ids=[ + "claim.formal.full_correctness", + "claim.certificate_checked", + ], + ), + mechanism_class="test_suite", + determinism="deterministic", + evidence_channels=[ + "stdout", + "stderr", + "raw_backend_result", + "normalized_result", + "test_report", + "compiled_obligation", + ], + replay_support=AssuranceReplaySupport( + supported=True, + compares_raw_digest=True, + compares_normalized_digest=True, + ), + configuration_snapshot_support=AssuranceSnapshotSupport( + supported=True, + exports_pcs_profile=True, + ), + mutation_dimensions=self.supported_mutation_dimensions(), # type: ignore[arg-type] + abstention=AssuranceAbstention(allows_abstention=True), + failure_behavior=AssuranceFailureBehavior(), + external_dependencies=[ + AssuranceExternalDependency( + dependency_id="pytest", + kind="library", + identity="pytest", + optional=False, + ) + ], + known_limits=["runtime/test observation only; not a formal verifier"], + ), + ) + + def snapshot_config( + self, + config: Mapping[str, Any] | None = None, + *, + environment: Mapping[str, str] | None = None, + ) -> ConfigurationSnapshot: + cfg = dict(config or {}) + cfg.setdefault("timeout_ms", self.timeout_ms) + cfg.setdefault("threshold", cfg.get("threshold", 0)) # max allowed failures + if isinstance(cfg.get("suite_path"), str): + cfg["suite_path"] = cfg["suite_path"].replace("\\", "/") + suite = { + "suite_path": cfg.get("suite_path"), + "nodeids": cfg.get("nodeids") or cfg.get("test_ids") or [], + "pytest_args": cfg.get("pytest_args") or [], + } + return build_configuration_snapshot( + backend_id=self.backend_id, + adapter_id=self.adapter_id, + adapter_version=self.adapter_version, + config=cfg, + environment=environment, + test_suite=suite, + threshold=cfg.get("threshold"), + mechanism_class="test_suite", + determinism="deterministic", + allows_abstention=True, + guarantee_class=GUARANTEE_CLASS, + decision_space=_DECISION_SPACE, + supported_claim_ids=["claim.pytest.runtime_observed"], + out_of_scope_claim_ids=["claim.formal.full_correctness"], + assumptions=["Observational pytest execution only."], + known_blind_spots=["Passing tests do not imply formal correctness."], + external_dependencies=[ + { + "dependency_id": "pytest", + "kind": "library", + "identity": "pytest", + "optional": False, + } + ], + entry_point="ovk.adapters.assurance.pytest_suite.PytestSuiteAdapter", + implementation_name="PytestSuiteAdapter", + timeout_ms=int(cfg["timeout_ms"]), + mutation_dimensions=self.supported_mutation_dimensions(), + ) + + def run_assurance( + self, + *, + input_data: Mapping[str, Any], + snapshot: ConfigurationSnapshot, + config: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + cfg = dict(snapshot.config) + if config: + cfg.update(dict(config)) + argv_base = [sys.executable, "-m", "pytest"] + + suite_path = input_data.get("suite_path") or cfg.get("suite_path") + if not suite_path: + return indeterminate_run_outcome( + reason="unsupported_input", + message="suite_path is required", + guarantee_class=GUARANTEE_CLASS, + command_argv=argv_base, + ) + suite = Path(str(suite_path)).expanduser() + if not suite.exists(): + return indeterminate_run_outcome( + reason="unsupported_input", + message=f"suite_path does not exist: {suite}", + guarantee_class=GUARANTEE_CLASS, + command_argv=argv_base, + ) + + try: + import pytest as _pytest # noqa: F401 + except ImportError as exc: + raise FileNotFoundError("pytest is not installed") from exc + + nodeids = list(input_data.get("nodeids") or cfg.get("nodeids") or cfg.get("test_ids") or []) + extra_args = list(input_data.get("pytest_args") or cfg.get("pytest_args") or []) + threshold = int(cfg.get("threshold") or 0) + timeout_ms = int(cfg.get("timeout_ms") or self.timeout_ms) + + suite_recorded = str(suite_path).replace("\\", "/") + with tempfile.TemporaryDirectory(prefix="ovk-pytest-") as tmp: + junit_path = Path(tmp) / "junit.xml" + exec_command = [ + *argv_base, + str(suite), + f"--junitxml={junit_path}", + "-q", + *extra_args, + *nodeids, + ] + # Portable argv for evidence packs (no host interpreter / temp paths). + recorded_command = [ + "python", + "-m", + "pytest", + suite_recorded, + "--junitxml=junit.xml", + "-q", + *extra_args, + *nodeids, + ] + try: + completed = subprocess.run( + exec_command, + capture_output=True, + text=True, + timeout=max(timeout_ms / 1000.0, 0.1), + check=False, + env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}, + ) + except subprocess.TimeoutExpired as exc: + raise TimeoutError(f"pytest timed out after {timeout_ms}ms") from exc + + junit = _parse_junit(junit_path) + # Stable raw body: exclude ephemeral temp paths and host-local absolutes. + raw = { + "command_stable": [ + "python", + "-m", + "pytest", + suite_recorded, + "-q", + *extra_args, + *nodeids, + ], + "exit_code": completed.returncode, + "junit_summary": { + "tests": junit.get("tests"), + "failures": junit.get("failures"), + "errors": junit.get("errors"), + "skipped": junit.get("skipped"), + "cases": [ + { + "classname": c.get("classname"), + "name": c.get("name"), + "status": c.get("status"), + } + for c in (junit.get("cases") or []) + ], + }, + "suite_digest": sha256_digest( + { + "suite_path": suite_recorded, + "nodeids": nodeids, + "pytest_args": extra_args, + } + ), + } + # Redact host-local paths from captured streams before packing. + stdout_text = (completed.stdout or "pass").replace(str(junit_path), "junit.xml") + stderr_text = (completed.stderr or "").replace(str(junit_path), "junit.xml") + failures = int(junit.get("failures") or 0) + int(junit.get("errors") or 0) + skipped = int(junit.get("skipped") or 0) + normalized_extra = { + "test_report": raw["junit_summary"], + "failures": failures, + "skipped": skipped, + "threshold": threshold, + "skipped_visible": skipped > 0, + } + if completed.returncode not in {0, 1}: + # pytest usage / collection errors + return indeterminate_run_outcome( + reason="parser_failure" if completed.returncode == 2 else "other", + message=(stderr_text or stdout_text or "pytest failed").strip()[:500], + raw_result=raw, + guarantee_class=GUARANTEE_CLASS, + command_argv=recorded_command, + exit_code=completed.returncode, + ) + if failures <= threshold and completed.returncode == 0: + return accept_outcome( + raw_result=raw, + normalized_extra=normalized_extra, + stdout=stdout_text, + stderr=stderr_text, + guarantee_class=GUARANTEE_CLASS, + command_argv=recorded_command, + exit_code=completed.returncode, + ) + if failures > threshold or completed.returncode != 0: + return reject_outcome( + raw_result=raw, + normalized_extra={**normalized_extra, "counterexamples": junit.get("cases")}, + stdout=stdout_text or "fail", + stderr=stderr_text, + guarantee_class=GUARANTEE_CLASS, + command_argv=recorded_command, + exit_code=completed.returncode, + ) + return accept_outcome( + raw_result=raw, + normalized_extra=normalized_extra, + stdout=stdout_text, + stderr=stderr_text, + guarantee_class=GUARANTEE_CLASS, + command_argv=recorded_command, + exit_code=completed.returncode, + ) diff --git a/ovk/adapters/assurance/sql_diff.py b/ovk/adapters/assurance/sql_diff.py new file mode 100644 index 0000000..cdf67f5 --- /dev/null +++ b/ovk/adapters/assurance/sql_diff.py @@ -0,0 +1,322 @@ +"""SQL state-difference assurance verifier (VA-10).""" + +from __future__ import annotations + +import hashlib +import sqlite3 +from pathlib import Path +from typing import Any, Mapping + +from ovk.adapters.assurance._support import ( + AssuranceBackendMixin, + accept_outcome, + indeterminate_run_outcome, + reject_outcome, +) +from ovk.assurance.pcs_hash import sha256_digest +from ovk.assurance.snapshot import ConfigurationSnapshot, build_configuration_snapshot + + +def _file_digest(path: Path) -> str: + digest = hashlib.sha256(path.read_bytes()).hexdigest() + return f"sha256:{digest}" +from ovk.core.execution_models import ( + AssuranceAbstention, + AssuranceCapabilitySection, + AssuranceDecisionSemantics, + AssuranceFailureBehavior, + AssuranceReplaySupport, + AssuranceSnapshotSupport, + AssuranceVerifierIdentity, + BackendCapabilityManifest, + BackendGuaranteeDeclaration, + BackendToolIdentity, +) + +GUARANTEE_CLASS = "observational" + +_DECISION_SPACE = [ + "accept", + "reject", + "indeterminate_execution_error", + "indeterminate_out_of_scope", + "indeterminate_insufficient_evidence", +] + + +def _connect(path: Path) -> sqlite3.Connection: + uri = f"file:{path.as_posix()}?mode=ro" + conn = sqlite3.connect(uri, uri=True) + conn.execute("PRAGMA query_only = ON") + return conn + + +def _schema_digest(conn: sqlite3.Connection) -> str: + rows = conn.execute( + "SELECT type, name, tbl_name, sql FROM sqlite_master " + "WHERE type IN ('table','index','view','trigger') ORDER BY type, name" + ).fetchall() + return sha256_digest([list(row) for row in rows]) + + +def _table_names(conn: sqlite3.Connection) -> list[str]: + rows = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name" + ).fetchall() + return [str(row[0]) for row in rows] + + +def _table_digest(conn: sqlite3.Connection, table: str) -> str: + rows = conn.execute(f'SELECT * FROM "{table}"').fetchall() + return sha256_digest([list(row) for row in rows]) + + +def diff_sqlite(before_path: Path, after_path: Path, tables: list[str] | None = None) -> dict[str, Any]: + """Compute a deterministic state difference between two SQLite databases.""" + before_conn = _connect(before_path) + after_conn = _connect(after_path) + try: + before_schema = _schema_digest(before_conn) + after_schema = _schema_digest(after_conn) + before_tables = _table_names(before_conn) + after_tables = _table_names(after_conn) + selected = tables or sorted(set(before_tables) | set(after_tables)) + table_diffs: list[dict[str, Any]] = [] + for table in selected: + in_before = table in before_tables + in_after = table in after_tables + if not in_before or not in_after: + table_diffs.append( + { + "table": table, + "changed": True, + "before_present": in_before, + "after_present": in_after, + } + ) + continue + b_digest = _table_digest(before_conn, table) + a_digest = _table_digest(after_conn, table) + table_diffs.append( + { + "table": table, + "changed": b_digest != a_digest, + "before_digest": b_digest, + "after_digest": a_digest, + } + ) + return { + "before_schema_digest": before_schema, + "after_schema_digest": after_schema, + "schema_changed": before_schema != after_schema, + "tables": table_diffs, + "changed_tables": [item["table"] for item in table_diffs if item.get("changed")], + } + finally: + before_conn.close() + after_conn.close() + + +class SqlStateDiffAdapter(AssuranceBackendMixin): + """Real SQL state difference over SQLite offline fixtures.""" + + backend_id = "sql-state-diff" + adapter_id = "ovk-adapter-sql-state-diff" + adapter_version = "0.1.0" + _guarantee_type = "state_diff" + + def __init__(self, *, timeout_ms: int = 15_000) -> None: + self.timeout_ms = timeout_ms + + def supported_mutation_dimensions(self) -> list[str]: + return ["alter_timeout", "reduce_test_subset"] + + def manifest(self) -> BackendCapabilityManifest: + return BackendCapabilityManifest( + capability_id="sql-state-diff-v1", + tool=BackendToolIdentity( + name=self.backend_id, + adapter=self.adapter_id, + adapter_version=self.adapter_version, + version=self.adapter_version, + ), + backend_class="custom", + guarantee=BackendGuaranteeDeclaration( + type="state_diff", + meaning_of_pass="Observed SQLite state difference matches the declared expectation.", + meaning_of_fail="Observed SQLite state difference violates the declared expectation.", + meaning_of_unknown="DB fixtures missing or unsupported configuration.", + ), + input_languages=["sql", "json"], + supported_domains=["assurance", "data"], + supported_property_kinds=["state_diff"], + assumptions=["Before/after SQLite fixtures are authoritative offline materials."], + limits=["SQLite offline only in core CI; no live network DB mutation."], + result_format="ovk.result.v1", + timeout_behavior="unknown", + assurance=AssuranceCapabilitySection( + assurance_capable=True, + verifier_identity=AssuranceVerifierIdentity( + verifier_id="ovk.assurance.sql_state_diff", + implementation_name="SqlStateDiffAdapter", + entry_point="ovk.adapters.assurance.sql_diff.SqlStateDiffAdapter", + pcs_profile_artifact_type="VerifierProfile.v1", + ), + decision_semantics=AssuranceDecisionSemantics( + decision_space=_DECISION_SPACE, # type: ignore[arg-type] + guarantee_class=GUARANTEE_CLASS, # type: ignore[arg-type] + supported_claim_ids=["claim.sql.state_diff"], + out_of_scope_claim_ids=["claim.formal.full_correctness"], + ), + mechanism_class="static_analysis", + determinism="deterministic", + evidence_channels=[ + "raw_backend_result", + "normalized_result", + "state_diff", + "compiled_obligation", + ], + replay_support=AssuranceReplaySupport( + supported=True, + compares_raw_digest=True, + compares_normalized_digest=True, + ), + configuration_snapshot_support=AssuranceSnapshotSupport( + supported=True, + exports_pcs_profile=True, + ), + mutation_dimensions=self.supported_mutation_dimensions(), # type: ignore[arg-type] + abstention=AssuranceAbstention(allows_abstention=True), + failure_behavior=AssuranceFailureBehavior(), + external_dependencies=[], + known_limits=["SQLite fixtures only"], + requires_authoritative_state=True, + ), + ) + + def snapshot_config( + self, + config: Mapping[str, Any] | None = None, + *, + environment: Mapping[str, str] | None = None, + ) -> ConfigurationSnapshot: + cfg = dict(config or {}) + cfg.setdefault("timeout_ms", self.timeout_ms) + cfg.setdefault("expect_changed_tables", cfg.get("expect_changed_tables")) + for key in ("before_db", "after_db"): + if isinstance(cfg.get(key), str): + cfg[key] = cfg[key].replace("\\", "/") + return build_configuration_snapshot( + backend_id=self.backend_id, + adapter_id=self.adapter_id, + adapter_version=self.adapter_version, + config=cfg, + environment=environment, + mechanism_class="static_analysis", + determinism="deterministic", + allows_abstention=True, + guarantee_class=GUARANTEE_CLASS, + decision_space=_DECISION_SPACE, + supported_claim_ids=["claim.sql.state_diff"], + out_of_scope_claim_ids=["claim.formal.full_correctness"], + assumptions=["Offline SQLite before/after fixtures."], + known_blind_spots=["Does not prove application-level invariants beyond table digests."], + entry_point="ovk.adapters.assurance.sql_diff.SqlStateDiffAdapter", + implementation_name="SqlStateDiffAdapter", + timeout_ms=int(cfg["timeout_ms"]), + mutation_dimensions=self.supported_mutation_dimensions(), + extra={ + "before_db": cfg.get("before_db"), + "after_db": cfg.get("after_db"), + }, + ) + + def run_assurance( + self, + *, + input_data: Mapping[str, Any], + snapshot: ConfigurationSnapshot, + config: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + cfg = dict(snapshot.config) + if config: + cfg.update(dict(config)) + argv = [self.adapter_id, "sqlite-diff"] + + before_raw = input_data.get("before_db") or cfg.get("before_db") + after_raw = input_data.get("after_db") or cfg.get("after_db") + if not before_raw or not after_raw: + return indeterminate_run_outcome( + reason="missing_authoritative_state", + message="before_db and after_db paths are required", + guarantee_class=GUARANTEE_CLASS, + command_argv=argv, + ) + before_recorded = str(before_raw).replace("\\", "/") + after_recorded = str(after_raw).replace("\\", "/") + before_path = Path(str(before_raw)).expanduser().resolve() + after_path = Path(str(after_raw)).expanduser().resolve() + if not before_path.is_file() or not after_path.is_file(): + return indeterminate_run_outcome( + reason="missing_authoritative_state", + message=f"SQLite fixtures missing: before={before_path.is_file()} after={after_path.is_file()}", + guarantee_class=GUARANTEE_CLASS, + command_argv=argv, + ) + + tables = input_data.get("tables") or cfg.get("tables") + table_list = [str(t) for t in tables] if isinstance(tables, list) else None + try: + diff = diff_sqlite(before_path, after_path, tables=table_list) + except sqlite3.Error as exc: + return indeterminate_run_outcome( + reason="parser_failure", + message=str(exc), + guarantee_class=GUARANTEE_CLASS, + command_argv=argv, + ) + + expect_unchanged = bool(input_data.get("expect_unchanged") or cfg.get("expect_unchanged")) + expect_changed = input_data.get("expect_changed_tables") + if expect_changed is None: + expect_changed = cfg.get("expect_changed_tables") + + # Record caller-supplied paths (portable); resolve only for I/O and file digests. + raw = { + "before_db": before_recorded, + "after_db": after_recorded, + "before_file_digest": _file_digest(before_path), + "after_file_digest": _file_digest(after_path), + "diff": diff, + } + changed = list(diff.get("changed_tables") or []) + if expect_unchanged: + passed = len(changed) == 0 and not diff.get("schema_changed") + elif isinstance(expect_changed, list): + expected = sorted(str(t) for t in expect_changed) + passed = sorted(changed) == expected + else: + # Default: any diff is informational reject unless expect_unchanged + # Callers should declare expectation; missing expectation is unsupported. + return indeterminate_run_outcome( + reason="unsupported_input", + message="declare expect_unchanged=true or expect_changed_tables=[...]", + raw_result=raw, + guarantee_class=GUARANTEE_CLASS, + command_argv=argv, + ) + + if passed: + return accept_outcome( + raw_result=raw, + normalized_extra={"state_diff": diff}, + guarantee_class=GUARANTEE_CLASS, + command_argv=argv, + ) + return reject_outcome( + raw_result=raw, + normalized_extra={"state_diff": diff, "counterexamples": changed}, + stdout="reject: sql state diff mismatch", + guarantee_class=GUARANTEE_CLASS, + command_argv=argv, + ) diff --git a/ovk/adapters/contract.py b/ovk/adapters/contract.py index b7d7448..82bd29a 100644 --- a/ovk/adapters/contract.py +++ b/ovk/adapters/contract.py @@ -8,7 +8,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any, Protocol, runtime_checkable +from typing import Any, Mapping, Protocol, runtime_checkable from ovk.core.execution_models import ( BackendCapabilityAssessment, @@ -153,3 +153,33 @@ def explain( self, result: NormalizedBackendResult, ) -> TypedHumanExplanation: ... + + +# --------------------------------------------------------------------------- +# Verifier-assurance adapter protocol (opt-in; does not alter ordinary path) +# --------------------------------------------------------------------------- + + +@runtime_checkable +class AssuranceCapableAdapter(Protocol): + """Opt-in protocol for adapters that participate in verifier-assurance. + + Ordinary ``BackendAdapter`` methods remain unchanged. Assurance surfaces + (snapshot, typed mutations, optional ``run_assurance``) are additive and + only invoked from ``ovk verifier …`` / ``ovk.assurance`` runners. + """ + + backend_id: str + adapter_id: str + adapter_version: str + + def manifest(self) -> BackendCapabilityManifest: ... + + def snapshot_config( + self, + config: Mapping[str, Any] | None = None, + *, + environment: Mapping[str, str] | None = None, + ) -> Any: ... + + def supported_mutation_dimensions(self) -> list[str]: ... diff --git a/ovk/assurance/__init__.py b/ovk/assurance/__init__.py new file mode 100644 index 0000000..cb888ac --- /dev/null +++ b/ovk/assurance/__init__.py @@ -0,0 +1,35 @@ +"""OVK verifier-assurance package (VA-01 through VA-14). + +Ordinary ``ovk check`` / MCP paths are intentionally untouched. Assurance +surfaces are opt-in via ``ovk verifier …`` and require a resolved PCS pin. + +Heavy imports (registry, conformance, adjudication) are available as submodules +to avoid circular imports with assurance adapters. +""" + +from __future__ import annotations + +from ovk.assurance.capability import is_assurance_capable, validate_assurance_claim +from ovk.assurance.errors import ( + AssuranceError, + EvidenceError, + MutationError, + PinError, + ReplayError, +) +from ovk.assurance.pin import require_pcs_pin, resolve_pcs_root +from ovk.assurance.snapshot import ConfigurationSnapshot, build_configuration_snapshot + +__all__ = [ + "AssuranceError", + "ConfigurationSnapshot", + "EvidenceError", + "MutationError", + "PinError", + "ReplayError", + "build_configuration_snapshot", + "is_assurance_capable", + "require_pcs_pin", + "resolve_pcs_root", + "validate_assurance_claim", +] diff --git a/ovk/assurance/adjudication.py b/ovk/assurance/adjudication.py new file mode 100644 index 0000000..141328f --- /dev/null +++ b/ovk/assurance/adjudication.py @@ -0,0 +1,141 @@ +"""Post-freeze adjudication importer (VA-13). + +Imports PCS adjudication references only after a campaign freeze marker is +present. Active/hidden holdout labels are never written into evidence packs, +logs, or verifier inputs. +""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from ovk.assurance.errors import AssuranceError +from ovk.assurance.pcs_hash import sha256_digest +from ovk.assurance.redaction import redact_mapping + +HIDDEN_LABEL_KEYS = frozenset( + { + "hidden_label", + "hidden_labels", + "active_label", + "active_labels", + "holdout_label", + "holdout_labels", + "ground_truth_label", + "ground_truth_labels", + "formalpr_holdout_label", + "label_private", + "private_label", + } +) + +AUDIT_EVENT_TYPE = "ovk.assurance.adjudication_import.v1" + + +class AdjudicationImportError(AssuranceError): + """Raised when adjudication import is refused.""" + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _contains_hidden_labels(payload: Any, *, path: str = "$") -> list[str]: + hits: list[str] = [] + if isinstance(payload, Mapping): + for key, value in payload.items(): + key_l = str(key).strip().lower() + child = f"{path}.{key}" + if key_l in HIDDEN_LABEL_KEYS: + hits.append(child) + hits.extend(_contains_hidden_labels(value, path=child)) + elif isinstance(payload, list): + for index, item in enumerate(payload): + hits.extend(_contains_hidden_labels(item, path=f"{path}[{index}]")) + return hits + + +def load_freeze_marker(path: Path | str) -> dict[str, Any]: + marker_path = Path(path) + if not marker_path.is_file(): + raise AdjudicationImportError(f"freeze marker not found: {marker_path}") + data = json.loads(marker_path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise AdjudicationImportError("freeze marker must be a JSON object") + if data.get("frozen") is not True and str(data.get("status") or "").lower() != "frozen": + raise AdjudicationImportError("freeze marker does not declare frozen=true/status=frozen") + if not data.get("campaign_id"): + raise AdjudicationImportError("freeze marker missing campaign_id") + return data + + +def import_adjudication_reference( + *, + freeze_marker_path: Path | str, + adjudication_ref: Mapping[str, Any], + audit_log_path: Path | str | None = None, + allow_hidden_labels: bool = False, +) -> dict[str, Any]: + """Import a post-freeze adjudication reference with label isolation. + + ``allow_hidden_labels`` is always refused for production import paths; the + parameter exists only so negative tests can assert the hard deny. + """ + if allow_hidden_labels: + raise AdjudicationImportError("allow_hidden_labels is permanently refused") + + freeze = load_freeze_marker(freeze_marker_path) + payload = dict(adjudication_ref) + leaks = _contains_hidden_labels(payload) + if leaks: + raise AdjudicationImportError( + "adjudication reference contains forbidden hidden/active label fields: " + + ", ".join(leaks[:10]) + ) + + # Redact any accidental secret-looking fields; never copy label keys forward. + cleaned, _ = redact_mapping(payload) + for key in list(cleaned.keys()): + if str(key).strip().lower() in HIDDEN_LABEL_KEYS: + raise AdjudicationImportError(f"refused hidden label key after redaction: {key}") + + record = { + "artifact_type": "OVKAdjudicationImport.v1", + "imported_at": _utc_now_iso(), + "campaign_id": freeze["campaign_id"], + "freeze_marker_digest": sha256_digest(freeze), + "adjudication_ref": cleaned, + "adjudication_ref_digest": sha256_digest(cleaned), + "label_isolation": { + "hidden_labels_accessible": False, + "active_labels_accessible": False, + "written_to_evidence": False, + }, + } + + event = { + "event_type": AUDIT_EVENT_TYPE, + "timestamp": record["imported_at"], + "campaign_id": freeze["campaign_id"], + "adjudication_ref_digest": record["adjudication_ref_digest"], + "result": "accepted", + } + if audit_log_path is not None: + path = Path(audit_log_path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(event, sort_keys=True) + "\n") + + return record + + +def refuse_labels_in_verifier_input(input_data: Mapping[str, Any]) -> None: + """Negative-path guard: refuse verifier inputs that embed holdout labels.""" + leaks = _contains_hidden_labels(input_data) + if leaks: + raise AdjudicationImportError( + "verifier input must not embed hidden/active holdout labels: " + ", ".join(leaks[:10]) + ) diff --git a/ovk/assurance/capability.py b/ovk/assurance/capability.py new file mode 100644 index 0000000..8d9ed13 --- /dev/null +++ b/ovk/assurance/capability.py @@ -0,0 +1,24 @@ +"""Assurance capability helpers.""" + +from __future__ import annotations + +from ovk.core.backend_registry import BackendRegistryError, _validate_assurance_section +from ovk.core.execution_models import BackendCapabilityManifest + + +def is_assurance_capable(manifest: BackendCapabilityManifest | None) -> bool: + """Return True when the manifest advertises assurance_capable=True.""" + if manifest is None or manifest.assurance is None: + return False + return bool(manifest.assurance.assurance_capable) + + +def validate_assurance_claim(manifest: BackendCapabilityManifest) -> None: + """Validate assurance claim consistency; raise on contradiction.""" + try: + _validate_assurance_section(manifest) + except BackendRegistryError as exc: + raise ValueError(str(exc)) from exc + if manifest.assurance is not None and manifest.assurance.assurance_capable: + if not is_assurance_capable(manifest): + raise ValueError("assurance_capable claim failed validation") diff --git a/ovk/assurance/cli.py b/ovk/assurance/cli.py new file mode 100644 index 0000000..6379f61 --- /dev/null +++ b/ovk/assurance/cli.py @@ -0,0 +1,204 @@ +"""Typer CLI subgroup: ``ovk verifier …``.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Optional + +import typer + +from ovk.assurance.errors import AssuranceError, EvidenceError, MutationError, PinError, ReplayError +from ovk.assurance.evidence_pack import validate_evidence_dir +from ovk.assurance.mutation import mutate_profile +from ovk.assurance.pcs_export import snapshot_to_verifier_profile +from ovk.assurance.registry import build_verifier_registry, describe_backend, lookup_backend +from ovk.assurance.replay import replay_invocation +from ovk.assurance.runner import load_json_mapping, run_assurance +from ovk.assurance.snapshot import ConfigurationSnapshot +from ovk.core.backend_registry import BackendRegistry + +verifier_app = typer.Typer(help="Verifier-assurance commands (PCS-gated; opt-in).") + + +def _echo_json(payload: object) -> None: + typer.echo(json.dumps(payload, indent=2, sort_keys=True)) + + +def _registry_with_optional(adapter: object | None = None) -> BackendRegistry: + registry = build_verifier_registry() + if adapter is not None: + if registry.get(getattr(adapter, "backend_id")) is None: + registry.register(adapter) # type: ignore[arg-type] + return registry + + +@verifier_app.command("describe") +def describe( + backend: str = typer.Option(..., "--backend", help="Backend id to describe."), +) -> None: + """Print ordinary capability (and assurance summary when present).""" + try: + payload = describe_backend(backend) + except AssuranceError as exc: + typer.echo(str(exc), err=True) + raise typer.Exit(code=2) from exc + _echo_json(payload) + + +@verifier_app.command("snapshot-config") +def snapshot_config( + backend: str = typer.Option(..., "--backend", help="Assurance-capable backend id."), + out: Path = typer.Option(..., "--out", help="Output path for sealed VerifierProfile.v1 JSON."), + config: Optional[Path] = typer.Option(None, "--config", help="Optional config JSON."), +) -> None: + """Snapshot configuration to a sealed VerifierProfile.v1 (fail closed if not assurance-capable).""" + try: + adapter = lookup_backend(backend) + from ovk.assurance.capability import is_assurance_capable + + manifest = adapter.manifest() + if not is_assurance_capable(manifest): + raise AssuranceError( + f"backend {backend!r} is not assurance_capable; snapshot-config refused" + ) + cfg = load_json_mapping(config) if config is not None else {} + snap = adapter.snapshot_config(cfg) # type: ignore[attr-defined] + if not isinstance(snap, ConfigurationSnapshot): + snap = ConfigurationSnapshot.model_validate(snap) + profile = snapshot_to_verifier_profile(snap) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(profile, indent=2, sort_keys=True) + "\n", encoding="utf-8") + typer.echo(str(out)) + except (AssuranceError, PinError, AttributeError) as exc: + typer.echo(str(exc), err=True) + raise typer.Exit(code=2) from exc + + +@verifier_app.command("run") +def run( + backend: str = typer.Option(..., "--backend", help="Assurance-capable backend id."), + input_path: Path = typer.Option(..., "--input", help="Input JSON for the verifier."), + evidence_dir: Path = typer.Option(..., "--evidence-dir", help="Evidence pack output directory."), + config: Optional[Path] = typer.Option(None, "--config", help="Optional config JSON."), + profile: Optional[Path] = typer.Option(None, "--profile", help="Optional sealed VerifierProfile.v1."), +) -> None: + """Run an assurance-capable verifier and write an evidence pack.""" + try: + adapter = lookup_backend(backend) + from ovk.assurance.capability import is_assurance_capable + + if not is_assurance_capable(adapter.manifest()): + raise AssuranceError(f"backend {backend!r} is not assurance_capable; run refused") + input_data = load_json_mapping(input_path) + from ovk.assurance.adjudication import refuse_labels_in_verifier_input + + refuse_labels_in_verifier_input(input_data) + cfg = load_json_mapping(config) if config is not None else {} + sealed_profile = load_json_mapping(profile) if profile is not None else None + outcome = run_assurance( + adapter, + input_data=input_data, + config=cfg, + evidence_dir=evidence_dir, + profile=sealed_profile, + ) + _echo_json( + { + "decision": outcome.decision, + "execution_status": outcome.execution_status, + "indeterminate_reason": outcome.indeterminate_reason, + "evidence_dir": str(outcome.evidence_dir), + "profile_digest": outcome.profile["integrity"]["artifact_digest"], + "result_digest": outcome.result["integrity"]["artifact_digest"], + } + ) + except (AssuranceError, PinError, EvidenceError) as exc: + typer.echo(str(exc), err=True) + raise typer.Exit(code=2) from exc + + +@verifier_app.command("validate-evidence") +def validate_evidence( + evidence_dir: Path = typer.Argument(..., help="Evidence pack directory."), +) -> None: + """Validate an evidence pack layout and PCS artifacts.""" + try: + report = validate_evidence_dir(evidence_dir) + _echo_json(report) + except (EvidenceError, PinError, AssuranceError) as exc: + typer.echo(str(exc), err=True) + raise typer.Exit(code=2) from exc + + +@verifier_app.command("replay") +def replay( + invocation: Path = typer.Argument(..., help="VerifierInvocationRecord.v1 JSON path."), + evidence_dir: Path = typer.Option(..., "--evidence-dir", help="Evidence pack directory for profile/input."), + backend: Optional[str] = typer.Option( + None, + "--backend", + help="Backend id used for deterministic rerun (required for matched/drifted rerun).", + ), +) -> None: + """Replay an invocation; fail closed on drift when claiming matched.""" + try: + if backend is None: + raise AssuranceError("replay requires --backend for deterministic rerun") + adapter = lookup_backend(backend) + report = replay_invocation(invocation, adapter=adapter, evidence_dir=evidence_dir) + _echo_json(report) + if report.get("replay_status") == "drifted": + raise typer.Exit(code=1) + except (ReplayError, AssuranceError, PinError) as exc: + typer.echo(str(exc), err=True) + raise typer.Exit(code=2) from exc + + +@verifier_app.command("mutate") +def mutate( + profile: Path = typer.Option(..., "--profile", help="Base sealed VerifierProfile.v1."), + mutation: str = typer.Option(..., "--mutation", help="Mutation class from verifier_assurance.defs."), + out: Path = typer.Option(..., "--out", help="Output path for mutated profile JSON."), + parameters: Optional[Path] = typer.Option( + None, "--parameters", help="Optional JSON object of mutation parameters." + ), + production_profile_path: Optional[Path] = typer.Option( + None, + "--production-profile-path", + help="Refuse if --out would overwrite this production profile path.", + ), + backend: Optional[str] = typer.Option( + None, "--backend", help="Optional backend to constrain supported mutation dimensions." + ), +) -> None: + """Apply a typed mutation producing a NEW profile (never overwrite production).""" + try: + params = load_json_mapping(parameters) if parameters is not None else {} + # Convenience: allow --mutation alter_timeout:5000 style via parameters file only. + supported = None + if backend is not None: + adapter = lookup_backend(backend) + if hasattr(adapter, "supported_mutation_dimensions"): + supported = list(adapter.supported_mutation_dimensions()) + elif adapter.manifest().assurance is not None: + supported = list(adapter.manifest().assurance.mutation_dimensions) + mutated, manifest = mutate_profile( + profile, + mutation_class=mutation, + parameters=params, + out_path=out, + production_profile_path=production_profile_path, + supported_dimensions=supported, + ) + _echo_json( + { + "mutated_profile_id": mutated["verifier_profile_id"], + "mutated_digest": mutated["integrity"]["artifact_digest"], + "mutation_id": manifest["mutation_id"], + "out": str(out), + } + ) + except (MutationError, AssuranceError, PinError) as exc: + typer.echo(str(exc), err=True) + raise typer.Exit(code=2) from exc diff --git a/ovk/assurance/conformance.py b/ovk/assurance/conformance.py new file mode 100644 index 0000000..b95da86 --- /dev/null +++ b/ovk/assurance/conformance.py @@ -0,0 +1,483 @@ +"""Cross-adapter assurance conformance harness (VA-12). + +Implements the shared 14-test matrix from the verifier-assurance work order. +Any adapter claiming ``assurance_capable=True`` must pass this matrix (or +explicitly skip only when an external toolchain is absent — and then only via +typed indeterminate paths, never fabricated passes). +""" + +from __future__ import annotations + +import copy +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Mapping + +from ovk.assurance.capability import is_assurance_capable +from ovk.assurance.errors import AssuranceError, MutationError, ReplayError +from ovk.assurance.guarantee import clamp_guarantee_class +from ovk.assurance.indeterminate import DECISION_ACCEPT +from ovk.assurance.mutation import mutate_profile +from ovk.assurance.pcs_export import snapshot_to_verifier_profile +from ovk.assurance.pcs_validate import require_valid_pcs_artifact +from ovk.assurance.pin import resolve_pcs_root +from ovk.assurance.redaction import redact_environment +from ovk.assurance.replay import replay_invocation +from ovk.assurance.runner import run_assurance +from ovk.assurance.snapshot import ConfigurationSnapshot + +CONFORMANCE_TEST_IDS = ( + "digest_stability", + "material_change_digest", + "deterministic_reproduce_or_declared_nondeterminism", + "raw_normalized_link", + "missing_checker_indeterminate", + "timeout_indeterminate", + "unsupported_fail_closed", + "skipped_visible", + "no_guarantee_upgrade", + "preserve_proof_cex", + "secrets_absent", + "mutation_distinct_profile", + "replay_drift_detection", + "pcs_validate_profile", + "pcs_validate_result", +) + + +@dataclass +class ConformanceCase: + """Per-adapter inputs for the shared 14-test matrix.""" + + adapter: Any + input_data: Mapping[str, Any] + config: Mapping[str, Any] = field(default_factory=dict) + alt_config: Mapping[str, Any] = field(default_factory=dict) + unsupported_input: Mapping[str, Any] = field(default_factory=dict) + mutation_class: str = "alter_timeout" + mutation_parameters: Mapping[str, Any] = field(default_factory=lambda: {"timeout_ms": 1234}) + expect_accept: bool = True + # When True, missing external checker is exercised via FileNotFoundError path. + supports_missing_checker_probe: bool = False + missing_checker_probe: Callable[[], None] | None = None + # Optional: force skip visibility check via outcome inspection. + skip_check: Callable[[dict[str, Any]], bool] | None = None + + +@dataclass +class ConformanceResult: + backend_id: str + test_id: str + passed: bool + detail: str = "" + + +def _require_assurance(adapter: Any) -> None: + manifest = adapter.manifest() + if not is_assurance_capable(manifest): + raise AssuranceError(f"{adapter.backend_id} is not assurance_capable") + + +def _snap(adapter: Any, config: Mapping[str, Any], environment: Mapping[str, str] | None = None) -> ConfigurationSnapshot: + snap = adapter.snapshot_config(dict(config), environment=environment) + if isinstance(snap, ConfigurationSnapshot): + return snap + return ConfigurationSnapshot.model_validate(snap) + + +def run_conformance_case(case: ConformanceCase) -> list[ConformanceResult]: + """Run all 14 conformance tests against one assurance-capable adapter.""" + adapter = case.adapter + _require_assurance(adapter) + backend_id = adapter.backend_id + results: list[ConformanceResult] = [] + pcs_ok = resolve_pcs_root() is not None + + # 1. digest stability + try: + s1 = _snap(adapter, case.config) + s2 = _snap(adapter, case.config) + ok = s1.content_digest == s2.content_digest and s1.config_digest == s2.config_digest + results.append( + ConformanceResult(backend_id, "digest_stability", ok, f"{s1.content_digest} vs {s2.content_digest}") + ) + except Exception as exc: # noqa: BLE001 — collect per-test failures + results.append(ConformanceResult(backend_id, "digest_stability", False, str(exc))) + + # 2. material-change digest + try: + s1 = _snap(adapter, case.config) + alt = dict(case.alt_config) if case.alt_config else {**dict(case.config), "timeout_ms": int(case.config.get("timeout_ms") or 5000) + 1} + s2 = _snap(adapter, alt) + ok = s1.content_digest != s2.content_digest + results.append( + ConformanceResult(backend_id, "material_change_digest", ok, f"{s1.content_digest} -> {s2.content_digest}") + ) + except Exception as exc: # noqa: BLE001 + results.append(ConformanceResult(backend_id, "material_change_digest", False, str(exc))) + + # 3. deterministic reproduce or declared nondeterminism + try: + manifest = adapter.manifest() + determinism = manifest.assurance.determinism if manifest.assurance else "deterministic" + if not pcs_ok: + results.append( + ConformanceResult(backend_id, "deterministic_reproduce_or_declared_nondeterminism", False, "PCS pin missing") + ) + elif determinism == "stochastic": + results.append( + ConformanceResult( + backend_id, + "deterministic_reproduce_or_declared_nondeterminism", + True, + "declared stochastic", + ) + ) + else: + o1 = run_assurance(adapter, input_data=dict(case.input_data), config=dict(case.config)) + o2 = run_assurance(adapter, input_data=dict(case.input_data), config=dict(case.config)) + ok = ( + o1.invocation["raw_backend_result_digest"] == o2.invocation["raw_backend_result_digest"] + and o1.invocation["normalized_result_digest"] == o2.invocation["normalized_result_digest"] + ) + results.append( + ConformanceResult( + backend_id, + "deterministic_reproduce_or_declared_nondeterminism", + ok, + f"decision={o1.decision}", + ) + ) + except Exception as exc: # noqa: BLE001 + results.append( + ConformanceResult(backend_id, "deterministic_reproduce_or_declared_nondeterminism", False, str(exc)) + ) + + # 4. raw↔normalized link + try: + if not pcs_ok: + raise AssuranceError("PCS pin missing") + outcome = run_assurance(adapter, input_data=dict(case.input_data), config=dict(case.config)) + ok = ( + "raw_backend_result_digest" in outcome.invocation + and "normalized_result_digest" in outcome.invocation + and outcome.invocation["raw_backend_result_digest"].startswith("sha256:") + and outcome.invocation["normalized_result_digest"].startswith("sha256:") + and bool(outcome.raw_result is not None) + and bool(outcome.normalized_result is not None) + ) + results.append(ConformanceResult(backend_id, "raw_normalized_link", ok)) + except Exception as exc: # noqa: BLE001 + results.append(ConformanceResult(backend_id, "raw_normalized_link", False, str(exc))) + + # 5. timeout / missing-checker indeterminate + try: + probed = False + if case.supports_missing_checker_probe and case.missing_checker_probe is not None: + try: + case.missing_checker_probe() + except FileNotFoundError: + probed = True + # Re-run through runner by temporarily breaking availability if adapter supports it + # Prefer native FileNotFoundError via a shallow wrapper when probe not provided: + class _Missing: + def __getattr__(self, name: str) -> Any: + return getattr(adapter, name) + + def run_assurance(self, **kwargs: Any) -> dict[str, Any]: + raise FileNotFoundError(f"{backend_id} checker missing (conformance probe)") + + def snapshot_config(self, *args: Any, **kwargs: Any) -> Any: + return adapter.snapshot_config(*args, **kwargs) + + def manifest(self) -> Any: + return adapter.manifest() + + class _Timeout: + def __getattr__(self, name: str) -> Any: + return getattr(adapter, name) + + def run_assurance(self, **kwargs: Any) -> dict[str, Any]: + raise TimeoutError(f"{backend_id} checker timed out (conformance probe)") + + def snapshot_config(self, *args: Any, **kwargs: Any) -> Any: + return adapter.snapshot_config(*args, **kwargs) + + def manifest(self) -> Any: + return adapter.manifest() + + if pcs_ok: + missing_outcome = run_assurance( + _Missing(), input_data=dict(case.input_data), config=dict(case.config) + ) + missing_ok = ( + missing_outcome.decision != DECISION_ACCEPT + and missing_outcome.indeterminate_reason == "missing_checker" + ) + results.append( + ConformanceResult( + backend_id, + "missing_checker_indeterminate", + missing_ok, + missing_outcome.decision, + ) + ) + timeout_outcome = run_assurance( + _Timeout(), input_data=dict(case.input_data), config=dict(case.config) + ) + timeout_ok = ( + timeout_outcome.decision != DECISION_ACCEPT + and timeout_outcome.indeterminate_reason == "timeout" + ) + results.append( + ConformanceResult( + backend_id, + "timeout_indeterminate", + timeout_ok, + timeout_outcome.decision, + ) + ) + else: + results.append( + ConformanceResult(backend_id, "missing_checker_indeterminate", False, "PCS pin missing") + ) + results.append( + ConformanceResult(backend_id, "timeout_indeterminate", False, "PCS pin missing") + ) + _ = probed + except Exception as exc: # noqa: BLE001 + results.append(ConformanceResult(backend_id, "missing_checker_indeterminate", False, str(exc))) + results.append(ConformanceResult(backend_id, "timeout_indeterminate", False, str(exc))) + + # 6. unsupported fail-closed + try: + if not pcs_ok: + raise AssuranceError("PCS pin missing") + unsupported = dict(case.unsupported_input) if case.unsupported_input else {} + outcome = run_assurance(adapter, input_data=unsupported, config={}) + ok = outcome.decision != DECISION_ACCEPT + results.append(ConformanceResult(backend_id, "unsupported_fail_closed", ok, outcome.decision)) + except Exception as exc: # noqa: BLE001 + results.append(ConformanceResult(backend_id, "unsupported_fail_closed", False, str(exc))) + + # 7. skipped visible + try: + if not pcs_ok: + raise AssuranceError("PCS pin missing") + outcome = run_assurance(adapter, input_data=dict(case.input_data), config=dict(case.config)) + if case.skip_check is not None: + ok = case.skip_check(outcome.normalized_result) + else: + # Default: skipped checks appear in result check_groups when indeterminate + if outcome.decision.startswith("indeterminate_"): + checks = [] + for group in outcome.result.get("check_groups") or []: + checks.extend(group.get("checks") or []) + ok = any(c.get("status") == "skipped" for c in checks) or outcome.indeterminate_reason is not None + else: + ok = True # no skip expected on happy path + results.append(ConformanceResult(backend_id, "skipped_visible", ok)) + except Exception as exc: # noqa: BLE001 + results.append(ConformanceResult(backend_id, "skipped_visible", False, str(exc))) + + # 8. no guarantee upgrade + try: + declared = adapter.manifest().assurance.decision_semantics.guarantee_class + clamped = clamp_guarantee_class(declared, "formally_checked") + ranks_ok = clamped == declared + if declared != "formally_checked": + ranks_ok = clamped != "formally_checked" and clamped == declared + # End-to-end: sealed result must not carry an upgraded class. + if pcs_ok: + outcome = run_assurance( + adapter, input_data=dict(case.input_data), config=dict(case.config) + ) + sealed = str(outcome.result.get("guarantee_class") or declared) + sealed_ok = clamp_guarantee_class(declared, sealed) == sealed or sealed == declared + # Attacker-shaped claim must clamp when compared against declared. + attack = clamp_guarantee_class(declared, "formally_checked") + ranks_ok = ranks_ok and attack == declared and sealed_ok + results.append(ConformanceResult(backend_id, "no_guarantee_upgrade", ranks_ok, f"declared={declared}")) + except Exception as exc: # noqa: BLE001 + results.append(ConformanceResult(backend_id, "no_guarantee_upgrade", False, str(exc))) + + # 9. preserve proof/cex + try: + if not pcs_ok: + raise AssuranceError("PCS pin missing") + outcome = run_assurance(adapter, input_data=dict(case.input_data), config=dict(case.config)) + raw = outcome.raw_result + norm = outcome.normalized_result + ok = isinstance(raw, dict) and isinstance(norm, dict) + if outcome.decision == "reject": + ok = ok and ( + "counterexamples" in norm + or "violations" in norm + or "state_diff" in norm + or "model_judgment" in norm + or "predicate_results" in (raw.get("evaluation") or {}) + or "failures" in norm + or any( + key in norm + for key in ("decision", "status", "reason", "message", "summary") + ) + ) + results.append(ConformanceResult(backend_id, "preserve_proof_cex", ok)) + except Exception as exc: # noqa: BLE001 + results.append(ConformanceResult(backend_id, "preserve_proof_cex", False, str(exc))) + + # 10. secrets absent + try: + secret_env = { + "OVK_API_TOKEN": "super-secret-token-value", + "PGPASSWORD": "db-pass", + "NOTE": "Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig", + "PATH": "/usr/bin", + } + snap = _snap(adapter, case.config, environment=secret_env) + blob = str(snap.model_dump(mode="json")) + profile = snapshot_to_verifier_profile(snap) if pcs_ok else {} + blob2 = str(profile) + leaked = any( + marker in blob or marker in blob2 + for marker in ( + "super-secret-token-value", + "db-pass", + "Bearer eyJhbGciOiJIUzI1NiJ9", + ) + ) + _ = redact_environment + results.append(ConformanceResult(backend_id, "secrets_absent", not leaked)) + except Exception as exc: # noqa: BLE001 + results.append(ConformanceResult(backend_id, "secrets_absent", False, str(exc))) + + # 11. mutation distinct profile + try: + if not pcs_ok: + raise AssuranceError("PCS pin missing") + snap = _snap(adapter, case.config) + profile = snapshot_to_verifier_profile(snap) + dims = list(adapter.supported_mutation_dimensions()) + if case.mutation_class not in dims: + results.append( + ConformanceResult( + backend_id, + "mutation_distinct_profile", + False, + f"{case.mutation_class} not in {dims}", + ) + ) + else: + mutated, manifest = mutate_profile( + profile, + mutation_class=case.mutation_class, + parameters=dict(case.mutation_parameters), + supported_dimensions=dims, + ) + ok = ( + mutated["integrity"]["artifact_digest"] != profile["integrity"]["artifact_digest"] + and manifest.get("production_prohibition") is True + ) + results.append(ConformanceResult(backend_id, "mutation_distinct_profile", ok)) + except (MutationError, Exception) as exc: # noqa: BLE001 + results.append(ConformanceResult(backend_id, "mutation_distinct_profile", False, str(exc))) + + # 12. replay drift detection + try: + if not pcs_ok: + raise AssuranceError("PCS pin missing") + manifest = adapter.manifest() + determinism = manifest.assurance.determinism if manifest.assurance else "deterministic" + outcome = run_assurance(adapter, input_data=dict(case.input_data), config=dict(case.config)) + drifted_profile = copy.deepcopy(outcome.profile) + # Force digest mismatch by mutating sealed profile improperly then resealing via timeout change + alt_snap = _snap( + adapter, + {**dict(case.config), "timeout_ms": int(case.config.get("timeout_ms") or 5000) + 7}, + ) + drifted_profile = snapshot_to_verifier_profile(alt_snap) + if determinism == "stochastic": + report = replay_invocation( + outcome.invocation, + adapter=adapter, + profile=outcome.profile, + input_data=dict(case.input_data), + config=dict(case.config), + claim_matched=False, + ) + ok = report.get("indeterminate_reason") == "declared_nondeterminism" or report.get("replay_status") == "indeterminate" + else: + try: + replay_invocation( + outcome.invocation, + adapter=adapter, + profile=drifted_profile, + input_data=dict(case.input_data), + config=dict(case.config), + claim_matched=True, + ) + ok = False # should have raised + except ReplayError: + ok = True + results.append(ConformanceResult(backend_id, "replay_drift_detection", ok)) + except Exception as exc: # noqa: BLE001 + results.append(ConformanceResult(backend_id, "replay_drift_detection", False, str(exc))) + + # 13. PCS validate profile + try: + if not pcs_ok: + raise AssuranceError("PCS pin missing") + snap = _snap(adapter, case.config) + profile = snapshot_to_verifier_profile(snap) + require_valid_pcs_artifact(profile, artifact_type="VerifierProfile.v1") + results.append(ConformanceResult(backend_id, "pcs_validate_profile", True)) + except Exception as exc: # noqa: BLE001 + results.append(ConformanceResult(backend_id, "pcs_validate_profile", False, str(exc))) + + # 14. PCS validate result + try: + if not pcs_ok: + raise AssuranceError("PCS pin missing") + outcome = run_assurance(adapter, input_data=dict(case.input_data), config=dict(case.config)) + require_valid_pcs_artifact(outcome.result, artifact_type="VerificationResult.v1") + if case.expect_accept: + # Soft check — adapters may still be indeterminate when tools missing + pass + results.append(ConformanceResult(backend_id, "pcs_validate_result", True, outcome.decision)) + except Exception as exc: # noqa: BLE001 + results.append(ConformanceResult(backend_id, "pcs_validate_result", False, str(exc))) + + # Ensure all 14 ids present + seen = {r.test_id for r in results} + for test_id in CONFORMANCE_TEST_IDS: + if test_id not in seen: + results.append(ConformanceResult(backend_id, test_id, False, "not executed")) + return results + + +def gate_assurance_adapters(cases: list[ConformanceCase]) -> dict[str, Any]: + """Run conformance for all cases; fail closed if any assurance_capable adapter fails.""" + all_results: list[ConformanceResult] = [] + for case in cases: + all_results.extend(run_conformance_case(case)) + failures = [r for r in all_results if not r.passed] + return { + "passed": not failures, + "total": len(all_results), + "failures": [ + {"backend_id": f.backend_id, "test_id": f.test_id, "detail": f.detail} for f in failures + ], + "results": [ + {"backend_id": r.backend_id, "test_id": r.test_id, "passed": r.passed, "detail": r.detail} + for r in all_results + ], + } + + +def write_example_pack(adapter: Any, *, out_dir: Path, input_data: Mapping[str, Any], config: Mapping[str, Any]) -> Path: + """Write an example evidence pack under examples/assurance/.""" + out_dir.mkdir(parents=True, exist_ok=True) + outcome = run_assurance(adapter, input_data=dict(input_data), config=dict(config), evidence_dir=out_dir) + if outcome.evidence_dir is None: + raise AssuranceError("evidence pack was not written") + return Path(outcome.evidence_dir) diff --git a/ovk/assurance/errors.py b/ovk/assurance/errors.py new file mode 100644 index 0000000..a29f66e --- /dev/null +++ b/ovk/assurance/errors.py @@ -0,0 +1,23 @@ +"""Assurance-mode error hierarchy (fail closed).""" + +from __future__ import annotations + + +class AssuranceError(Exception): + """Base error for verifier-assurance operations.""" + + +class PinError(AssuranceError): + """PCS pin resolution or schema availability failure.""" + + +class MutationError(AssuranceError): + """Typed mutation refused or unsupported.""" + + +class ReplayError(AssuranceError): + """Invocation replay failed closed (drift, missing deps, invalid claim).""" + + +class EvidenceError(AssuranceError): + """Evidence pack layout or content validation failure.""" diff --git a/ovk/assurance/evidence_pack.py b/ovk/assurance/evidence_pack.py new file mode 100644 index 0000000..de3fd12 --- /dev/null +++ b/ovk/assurance/evidence_pack.py @@ -0,0 +1,132 @@ +"""Evidence pack layout writer and validator for assurance runs.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Mapping + +from ovk.assurance.errors import EvidenceError +from ovk.assurance.pcs_hash import sha256_digest +from ovk.assurance.pcs_validate import require_valid_pcs_artifact + +REQUIRED_FILES = ( + "invocation.json", + "verifier_profile.pcs.json", + "verification_result.pcs.json", + "compiled_obligation.json", +) + +REQUIRED_DIRS = ("raw", "normalized", "provenance") + + +def _write_json(path: Path, payload: Mapping[str, Any] | list[Any] | Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def write_evidence_pack( + evidence_dir: Path | str, + *, + invocation: Mapping[str, Any], + profile: Mapping[str, Any], + result: Mapping[str, Any], + compiled_obligation: Mapping[str, Any] | None = None, + raw_result: Mapping[str, Any] | None = None, + normalized_result: Mapping[str, Any] | None = None, + provenance: Mapping[str, Any] | None = None, + stdout: str | None = None, + stderr: str | None = None, + validate: bool = True, +) -> Path: + """Write the standard assurance evidence pack layout and optionally validate.""" + root = Path(evidence_dir) + root.mkdir(parents=True, exist_ok=True) + for name in REQUIRED_DIRS: + (root / name).mkdir(parents=True, exist_ok=True) + + clean_invocation = dict(invocation) + if "_ovk_working" in clean_invocation: + raise EvidenceError( + "invocation must be a sealed OVK invocation record; " + "sidecar fields such as _ovk_working are forbidden" + ) + + obligation = dict(compiled_obligation or {"kind": "assurance_input", "note": "no compiled obligation"}) + raw = dict(raw_result or {}) + normalized = dict(normalized_result or {}) + prov = dict( + provenance + or { + "producer": clean_invocation.get("producer"), + "producer_version": clean_invocation.get("producer_version"), + "input_digest": clean_invocation.get("input_digest"), + "profile_digest": clean_invocation.get("profile_ref", {}).get("profile_digest"), + "raw_digest": clean_invocation.get("raw_backend_result_digest"), + "normalized_digest": clean_invocation.get("normalized_result_digest"), + } + ) + + _write_json(root / "invocation.json", clean_invocation) + _write_json(root / "verifier_profile.pcs.json", profile) + _write_json(root / "verification_result.pcs.json", result) + _write_json(root / "compiled_obligation.json", obligation) + _write_json(root / "raw" / "backend_result.json", raw) + if stdout is not None: + (root / "raw" / "stdout.txt").write_text(str(stdout), encoding="utf-8") + if stderr is not None: + (root / "raw" / "stderr.txt").write_text(str(stderr), encoding="utf-8") + _write_json(root / "normalized" / "result.json", normalized) + _write_json(root / "provenance" / "digests.json", prov) + + if validate: + validate_evidence_dir(root) + return root + + +def validate_evidence_dir(evidence_dir: Path | str) -> dict[str, Any]: + """Validate evidence pack layout and PCS artifacts. Fail closed on errors.""" + root = Path(evidence_dir) + if not root.is_dir(): + raise EvidenceError(f"evidence directory does not exist: {root}") + + missing = [name for name in REQUIRED_FILES if not (root / name).is_file()] + missing_dirs = [name for name in REQUIRED_DIRS if not (root / name).is_dir()] + if missing or missing_dirs: + raise EvidenceError( + "evidence pack incomplete: " + + ", ".join([*(f"missing file {m}" for m in missing), *(f"missing dir {d}" for d in missing_dirs)]) + ) + + profile = json.loads((root / "verifier_profile.pcs.json").read_text(encoding="utf-8")) + result = json.loads((root / "verification_result.pcs.json").read_text(encoding="utf-8")) + invocation = json.loads((root / "invocation.json").read_text(encoding="utf-8")) + + require_valid_pcs_artifact(profile, artifact_type="VerifierProfile.v1") + require_valid_pcs_artifact(result, artifact_type="VerificationResult.v1") + require_valid_pcs_artifact(invocation, artifact_type="VerifierInvocationRecord.v1") + + # Cross-binding checks + profile_digest = profile["integrity"]["artifact_digest"] + if invocation["profile_ref"]["profile_digest"] != profile_digest: + raise EvidenceError("invocation.profile_ref.profile_digest does not match sealed profile") + if result["verifier_profile"]["profile_digest"] != profile_digest: + raise EvidenceError("result.verifier_profile.profile_digest does not match sealed profile") + + declared = result.get("declared_input_guarantee_class") + result_class = result.get("guarantee_class") + if isinstance(declared, str) and isinstance(result_class, str): + from ovk.assurance.guarantee import assert_no_guarantee_upgrade + + assert_no_guarantee_upgrade(declared, result_class) + + return { + "valid": True, + "evidence_dir": str(root), + "profile_digest": profile_digest, + "result_digest": result["integrity"]["artifact_digest"], + "invocation_digest": invocation["integrity"]["artifact_digest"], + "obligation_digest": sha256_digest( + json.loads((root / "compiled_obligation.json").read_text(encoding="utf-8")) + ), + } diff --git a/ovk/assurance/guarantee.py b/ovk/assurance/guarantee.py new file mode 100644 index 0000000..5f514a8 --- /dev/null +++ b/ovk/assurance/guarantee.py @@ -0,0 +1,37 @@ +"""Guarantee-class ordering helpers (never upgrade on normalize/export).""" + +from __future__ import annotations + +from ovk.assurance.errors import AssuranceError + +# Lower rank is weaker. Normalization must never increase rank. +GUARANTEE_RANK: dict[str, int] = { + "unchecked_advisory": 0, + "observational": 1, + "runtime_observed": 1, + "empirically_measured": 2, + "human_reviewed": 3, + "certificate_checked": 4, + "formally_checked": 5, +} + + +def assert_no_guarantee_upgrade(declared: str, result_class: str) -> None: + """Fail closed when result guarantee_class upgrades the declared input class.""" + if declared not in GUARANTEE_RANK or result_class not in GUARANTEE_RANK: + raise AssuranceError( + f"unknown guarantee_class for upgrade check: declared={declared!r} result={result_class!r}" + ) + if GUARANTEE_RANK[result_class] > GUARANTEE_RANK[declared]: + raise AssuranceError( + f"normalize/export must not upgrade guarantee_class ({declared!r} -> {result_class!r})" + ) + + +def clamp_guarantee_class(declared: str, candidate: str) -> str: + """Return candidate unless it would upgrade declared; then keep declared.""" + if declared not in GUARANTEE_RANK or candidate not in GUARANTEE_RANK: + return declared + if GUARANTEE_RANK[candidate] > GUARANTEE_RANK[declared]: + return declared + return candidate diff --git a/ovk/assurance/indeterminate.py b/ovk/assurance/indeterminate.py new file mode 100644 index 0000000..4c31fd3 --- /dev/null +++ b/ovk/assurance/indeterminate.py @@ -0,0 +1,110 @@ +"""Map terminations and exit kinds to typed indeterminate decisions. + +Timeouts, missing checkers, and related failures MUST NEVER map to accept. +""" + +from __future__ import annotations + +from typing import Any + +# PCS VerificationResult.v1 decision vocabulary +DECISION_ACCEPT = "accept" +DECISION_REJECT = "reject" +DECISION_INDETERMINATE_INSUFFICIENT = "indeterminate_insufficient_evidence" +DECISION_INDETERMINATE_EXECUTION = "indeterminate_execution_error" +DECISION_INDETERMINATE_OUT_OF_SCOPE = "indeterminate_out_of_scope" +DECISION_INDETERMINATE_DRIFT = "indeterminate_configuration_drift" + +# PCS indeterminate_reason vocabulary +REASON_MISSING_CHECKER = "missing_checker" +REASON_TIMEOUT = "timeout" +REASON_PARSER_FAILURE = "parser_failure" +REASON_UNSUPPORTED_INPUT = "unsupported_input" +REASON_EXTERNAL_SERVICE = "external_service_error" +REASON_RESOURCE_EXHAUSTED = "resource_exhausted" +REASON_SPAWN_ERROR = "spawn_error" +REASON_INCOMPLETE_TRAJECTORY = "incomplete_trajectory" +REASON_MISSING_AUTH_STATE = "missing_authoritative_state" +REASON_DECLARED_NONDETERMINISM = "declared_nondeterminism" +REASON_OTHER = "other" + +_TERMINATION_TO_REASON: dict[str, str] = { + "timeout": REASON_TIMEOUT, + "resource_exhausted": REASON_RESOURCE_EXHAUSTED, + "tool_unavailable": REASON_MISSING_CHECKER, + "tool_error": REASON_OTHER, + "invalid_output": REASON_PARSER_FAILURE, + "cancelled": REASON_OTHER, + "missing_checker": REASON_MISSING_CHECKER, + "spawn_error": REASON_SPAWN_ERROR, + "parser_failure": REASON_PARSER_FAILURE, + "unsupported_input": REASON_UNSUPPORTED_INPUT, + "external_service_error": REASON_EXTERNAL_SERVICE, +} + +_REASON_TO_DECISION: dict[str, str] = { + REASON_MISSING_CHECKER: DECISION_INDETERMINATE_EXECUTION, + REASON_TIMEOUT: DECISION_INDETERMINATE_EXECUTION, + REASON_PARSER_FAILURE: DECISION_INDETERMINATE_EXECUTION, + REASON_UNSUPPORTED_INPUT: DECISION_INDETERMINATE_OUT_OF_SCOPE, + REASON_EXTERNAL_SERVICE: DECISION_INDETERMINATE_EXECUTION, + REASON_RESOURCE_EXHAUSTED: DECISION_INDETERMINATE_EXECUTION, + REASON_SPAWN_ERROR: DECISION_INDETERMINATE_EXECUTION, + REASON_INCOMPLETE_TRAJECTORY: DECISION_INDETERMINATE_INSUFFICIENT, + REASON_MISSING_AUTH_STATE: DECISION_INDETERMINATE_INSUFFICIENT, + REASON_DECLARED_NONDETERMINISM: DECISION_INDETERMINATE_INSUFFICIENT, + REASON_OTHER: DECISION_INDETERMINATE_EXECUTION, +} + +_REASON_TO_EXECUTION_STATUS: dict[str, str] = { + REASON_MISSING_CHECKER: "unavailable", + REASON_TIMEOUT: "timeout", + REASON_PARSER_FAILURE: "error", + REASON_UNSUPPORTED_INPUT: "completed", + REASON_EXTERNAL_SERVICE: "error", + REASON_RESOURCE_EXHAUSTED: "resource_exhausted", + REASON_SPAWN_ERROR: "error", + REASON_INCOMPLETE_TRAJECTORY: "completed", + REASON_MISSING_AUTH_STATE: "completed", + REASON_DECLARED_NONDETERMINISM: "completed", + REASON_OTHER: "error", +} + + +def indeterminate_reason_for_termination(termination: str) -> str: + """Map a termination / exit kind to a typed indeterminate reason.""" + key = str(termination or "").strip().lower() + return _TERMINATION_TO_REASON.get(key, REASON_OTHER) + + +def decision_for_indeterminate_reason(reason: str) -> str: + """Map an indeterminate reason to a VerificationResult.v1 decision.""" + decision = _REASON_TO_DECISION.get(reason, DECISION_INDETERMINATE_EXECUTION) + if decision in {DECISION_ACCEPT, DECISION_REJECT}: + # Hard invariant: never upgrade failure to accept/reject here. + return DECISION_INDETERMINATE_EXECUTION + return decision + + +def execution_status_for_reason(reason: str) -> str: + """Map an indeterminate reason to an execution_status value.""" + return _REASON_TO_EXECUTION_STATUS.get(reason, "tool_error") + + +def indeterminate_outcome( + *, + termination: str | None = None, + reason: str | None = None, + message: str | None = None, +) -> dict[str, Any]: + """Build a typed indeterminate outcome dict (never accept).""" + resolved_reason = reason or indeterminate_reason_for_termination(termination or REASON_OTHER) + decision = decision_for_indeterminate_reason(resolved_reason) + if decision == DECISION_ACCEPT: + raise RuntimeError("indeterminate_outcome must never produce accept") + return { + "decision": decision, + "execution_status": execution_status_for_reason(resolved_reason), + "indeterminate_reason": resolved_reason, + "message": message, + } diff --git a/ovk/assurance/invocation.py b/ovk/assurance/invocation.py new file mode 100644 index 0000000..50a13da --- /dev/null +++ b/ovk/assurance/invocation.py @@ -0,0 +1,115 @@ +"""InvocationRecord builder for assurance evidence packs.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Mapping + +from ovk.assurance.pcs_export import build_invocation_record_artifact +from ovk.assurance.pcs_hash import sha256_digest +from ovk.assurance.snapshot import ConfigurationSnapshot + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def build_invocation_record( + *, + snapshot: ConfigurationSnapshot, + profile: Mapping[str, Any], + input_data: Mapping[str, Any] | Any, + command_argv: list[str], + cwd: str, + started_at: str, + completed_at: str, + exit_kind: str, + exit_code: int | None = None, + exit_message: str | None = None, + stdout: str | bytes | None = "", + stderr: str | bytes | None = "", + raw_result: Mapping[str, Any] | None = None, + normalized_result: Mapping[str, Any] | None = None, + guarantee_class: str | None = None, + assumptions: list[dict[str, Any]] | None = None, + limits: list[dict[str, Any]] | None = None, + indeterminate_reason: str | None = None, + compiled_obligation: Mapping[str, Any] | None = None, + timeout_ms: int | None = None, + env_digest: str | None = None, + result_ref: dict[str, Any] | None = None, + normalizer_version: str = "ovk.normalize.v1", + seal: bool = True, +) -> dict[str, Any]: + """Build a VerifierInvocationRecord.v1 from run artifacts.""" + profile_digest = profile.get("integrity", {}).get("artifact_digest") + if not isinstance(profile_digest, str): + raise ValueError("profile must be sealed before building invocation record") + + input_digest = sha256_digest(dict(input_data) if isinstance(input_data, Mapping) else input_data) + stdout_text = stdout.decode("utf-8") if isinstance(stdout, bytes) else (stdout or "") + stderr_text = stderr.decode("utf-8") if isinstance(stderr, bytes) else (stderr or "") + raw = dict(raw_result or {}) + normalized = dict(normalized_result or {}) + + # When normalization is applied, digests must differ (PCS semantic rule). + raw_digest = sha256_digest(raw) + normalized_digest = sha256_digest(normalized) + if raw_digest == normalized_digest: + normalized = {**normalized, "_normalization_marker": "ovk.normalize.v1"} + normalized_digest = sha256_digest(normalized) + + resolved_timeout = timeout_ms or snapshot.timeout_ms or 30_000 + env = env_digest or snapshot.redacted_environment.get("environment_digest") + if not isinstance(env, str): + env = sha256_digest(snapshot.redacted_environment.get("entries") or {}) + + exit_obj: dict[str, Any] = {"kind": exit_kind} + if exit_code is not None: + exit_obj["code"] = int(exit_code) + if exit_message: + exit_obj["message"] = exit_message + + invocation_id = f"vi-{snapshot.backend_id}-{input_digest[7:15]}" + record: dict[str, Any] = { + "schema_version": "v1", + "artifact_type": "VerifierInvocationRecord.v1", + "canonicalization_version": "v1", + "invocation_id": invocation_id, + "profile_ref": { + "artifact_type": "VerifierProfile.v1", + "verifier_profile_id": profile["verifier_profile_id"], + "profile_digest": profile_digest, + }, + "input_digest": input_digest, + "command": { + "argv": list(command_argv) or [snapshot.adapter_id, "run"], + "cwd": cwd or ".", + "env_digest": env, + "limits": {"timeout_ms": int(resolved_timeout)}, + }, + "started_at": started_at or _utc_now_iso(), + "completed_at": completed_at or _utc_now_iso(), + "exit": exit_obj, + "stdout_digest": sha256_digest(stdout_text), + "stderr_digest": sha256_digest(stderr_text), + "raw_backend_result_digest": raw_digest, + "normalizer_version": normalizer_version, + "normalized_result_digest": normalized_digest, + "assumptions": list(assumptions or []), + "limits": list(limits or []), + "guarantee_class": guarantee_class or snapshot.guarantee_class, + } + if compiled_obligation is not None: + obligation_digest = sha256_digest(dict(compiled_obligation)) + record["compiled_obligation"] = { + "digest": obligation_digest, + "path": "compiled_obligation.json", + "media_type": "application/json", + } + if indeterminate_reason is not None: + record["indeterminate_reason"] = indeterminate_reason + if result_ref is not None: + record["result_ref"] = dict(result_ref) + + return build_invocation_record_artifact(record, seal=seal) diff --git a/ovk/assurance/mutation.py b/ovk/assurance/mutation.py new file mode 100644 index 0000000..42fc1b9 --- /dev/null +++ b/ovk/assurance/mutation.py @@ -0,0 +1,219 @@ +"""Typed profile mutations for verifier-assurance (immutable; no production overwrite).""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path +from typing import Any, Mapping + +from ovk.assurance.errors import MutationError +from ovk.assurance.pcs_export import build_mutation_manifest, profile_ref_from_profile +from ovk.assurance.pcs_hash import attach_nested_integrity, sha256_digest +from ovk.assurance.pcs_validate import require_valid_pcs_artifact + +SUPPORTED_MUTATION_CLASSES = frozenset( + { + "remove_success_predicate", + "remove_process_predicate", + "remove_authority_predicate", + "change_threshold", + "reduce_test_subset", + "alter_timeout", + "suppress_error", + "change_rubric", + "change_prompt", + "ensemble_quorum", + "hidden_state_access", + "policy_bundle", + "abstention", + "other", + } +) + +_DEFAULT_EFFECTS: dict[str, str] = { + "alter_timeout": "Changes execution timeout; may increase indeterminate_execution_error on slow checks.", + "change_threshold": "Changes acceptance threshold; may flip accept/reject boundary.", + "reduce_test_subset": "Reduces evaluated test subset; weakens observational coverage.", + "remove_success_predicate": "Removes success predicate; acceptance becomes weaker or indeterminate.", + "remove_process_predicate": "Removes process predicate checks from the profile configuration.", + "remove_authority_predicate": "Removes authority predicate checks from the profile configuration.", + "suppress_error": "Suppresses error surfaces; forbidden for production profiles.", + "change_rubric": "Changes evaluation rubric digest binding.", + "change_prompt": "Changes prompt digest binding for model-judge verifiers.", + "ensemble_quorum": "Changes ensemble quorum requirements.", + "hidden_state_access": "Alters hidden-state access declaration.", + "policy_bundle": "Swaps or mutates policy bundle digest.", + "abstention": "Alters abstention allowance.", + "other": "Other typed mutation applied to immutable profile copy.", +} + + +def _refuse_production_overwrite(out_path: Path, production_profile_path: Path | None) -> None: + if production_profile_path is None: + return + try: + if out_path.resolve() == production_profile_path.resolve(): + raise MutationError( + f"refusing to overwrite production profile path: {production_profile_path}" + ) + except FileNotFoundError: + # resolve() on missing parents can still compare; keep explicit string compare fallback + if str(out_path) == str(production_profile_path): + raise MutationError( + f"refusing to overwrite production profile path: {production_profile_path}" + ) + + +def _apply_mutation_to_profile( + profile: dict[str, Any], + mutation_class: str, + parameters: Mapping[str, Any], +) -> dict[str, Any]: + mutated = copy.deepcopy(profile) + mutated.pop("integrity", None) + + config = dict(mutated.get("canonical_configuration") or {}) + configuration = dict(mutated.get("configuration") or {}) + + if mutation_class == "alter_timeout": + if "timeout_ms" not in parameters: + raise MutationError("alter_timeout requires parameters.timeout_ms") + timeout_ms = int(parameters["timeout_ms"]) + if timeout_ms < 1: + raise MutationError("timeout_ms must be >= 1") + config["timeout_ms"] = timeout_ms + controls = dict(mutated.get("execution_controls") or {}) + controls["timeout_ms"] = timeout_ms + mutated["execution_controls"] = controls + configuration["resource_limit_digest"] = sha256_digest({"timeout_ms": timeout_ms}) + elif mutation_class == "change_threshold": + if "threshold" not in parameters: + raise MutationError("change_threshold requires parameters.threshold") + threshold = parameters["threshold"] + # Canonical JSON forbids floats — store decimal string when needed. + if isinstance(threshold, float): + threshold = format(threshold, "f").rstrip("0").rstrip(".") or "0" + config["threshold"] = threshold + configuration["threshold_digest"] = sha256_digest({"threshold": threshold}) + elif mutation_class == "reduce_test_subset": + subset = parameters.get("test_ids") or parameters.get("subset") or [] + if not isinstance(subset, list) or not subset: + raise MutationError("reduce_test_subset requires non-empty parameters.test_ids") + config["test_suite"] = {"test_ids": list(subset)} + configuration["test_suite_digest"] = sha256_digest(config["test_suite"]) + elif mutation_class == "policy_bundle": + policy = parameters.get("policy") + if policy is None: + raise MutationError("policy_bundle requires parameters.policy") + config["policy"] = policy + configuration["policy_digest"] = sha256_digest(policy) + elif mutation_class == "change_prompt": + prompt = parameters.get("prompt") + if prompt is None: + raise MutationError("change_prompt requires parameters.prompt") + config["prompt"] = prompt + configuration["prompt_digest"] = sha256_digest(prompt) + elif mutation_class == "change_rubric": + rubric = parameters.get("rubric") + if rubric is None: + raise MutationError("change_rubric requires parameters.rubric") + config["rubric"] = rubric + configuration["rubric_digest"] = sha256_digest(rubric) + elif mutation_class == "abstention": + if "allows_abstention" not in parameters: + raise MutationError("abstention requires parameters.allows_abstention") + mechanism = dict(mutated.get("mechanism") or {}) + mechanism["allows_abstention"] = bool(parameters["allows_abstention"]) + mutated["mechanism"] = mechanism + elif mutation_class in { + "remove_success_predicate", + "remove_process_predicate", + "remove_authority_predicate", + "suppress_error", + "ensemble_quorum", + "hidden_state_access", + "other", + }: + mutations = list(config.get("_mutations") or []) + mutations.append({"class": mutation_class, "parameters": dict(parameters)}) + config["_mutations"] = mutations + else: + raise MutationError(f"unsupported mutation class: {mutation_class!r}") + + # Rebind config digest after material change. + configuration["config_digest"] = sha256_digest(config) + mutated["canonical_configuration"] = config + mutated["configuration"] = configuration + + # New immutable identity + base_id = str(mutated.get("verifier_profile_id") or "vp") + mutated["verifier_profile_id"] = f"{base_id}-mut-{mutation_class}" + return attach_nested_integrity(mutated) + + +def mutate_profile( + profile: Mapping[str, Any] | Path | str, + *, + mutation_class: str, + parameters: Mapping[str, Any] | None = None, + out_path: Path | str | None = None, + production_profile_path: Path | str | None = None, + supported_dimensions: list[str] | None = None, + expected_effect: str | None = None, + rationale: str | None = None, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Apply a typed mutation, producing a NEW sealed profile and mutation manifest. + + Always sets ``production_prohibition=True``. Refuses overwrite of the + production profile path. + """ + if mutation_class not in SUPPORTED_MUTATION_CLASSES: + raise MutationError(f"unsupported mutation class: {mutation_class!r}") + + if supported_dimensions is not None and mutation_class not in supported_dimensions: + raise MutationError( + f"mutation {mutation_class!r} is not in adapter supported dimensions: " + f"{sorted(supported_dimensions)}" + ) + + if isinstance(profile, (str, Path)): + base = json.loads(Path(profile).read_text(encoding="utf-8")) + else: + base = dict(profile) + if not isinstance(base, dict): + raise MutationError("profile must be a JSON object") + require_valid_pcs_artifact(base, artifact_type="VerifierProfile.v1") + + params = dict(parameters or {}) + mutated = _apply_mutation_to_profile(base, mutation_class, params) + require_valid_pcs_artifact(mutated, artifact_type="VerifierProfile.v1") + + if mutated["integrity"]["artifact_digest"] == base["integrity"]["artifact_digest"]: + raise MutationError("mutation did not change profile digest (no-op refused)") + + effect = expected_effect or _DEFAULT_EFFECTS.get(mutation_class, _DEFAULT_EFFECTS["other"]) + mutation_id = f"mut-{mutation_class}-{mutated['integrity']['artifact_digest'][7:15]}" + manifest = build_mutation_manifest( + mutation_id=mutation_id, + base_profile_ref=profile_ref_from_profile(base), + mutated_profile_ref=profile_ref_from_profile(mutated), + mutation_class=mutation_class, + expected_effect=effect, + parameters=params or None, + supported_by_adapter=supported_dimensions is None or mutation_class in supported_dimensions, + rationale=rationale, + ) + + if out_path is not None: + out = Path(out_path) + prod = Path(production_profile_path) if production_profile_path is not None else None + _refuse_production_overwrite(out, prod) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(mutated, indent=2, sort_keys=True) + "\n", encoding="utf-8") + manifest_path = out.with_suffix(".mutation.json") + if prod is not None: + _refuse_production_overwrite(manifest_path, prod) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + return mutated, manifest diff --git a/ovk/assurance/pcs_export.py b/ovk/assurance/pcs_export.py new file mode 100644 index 0000000..9399f7d --- /dev/null +++ b/ovk/assurance/pcs_export.py @@ -0,0 +1,299 @@ +"""Map OVK assurance snapshots/results to sealed PCS VA artifacts.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Mapping + +from ovk.assurance.pcs_hash import attach_nested_integrity, sha256_digest +from ovk.assurance.snapshot import ConfigurationSnapshot +from ovk.assurance.source import producer_fields, resolve_source_commit, resolve_source_repo + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _profile_id(snapshot: ConfigurationSnapshot) -> str: + return f"vp-{snapshot.backend_id}-{snapshot.content_digest[7:15]}" + + +def snapshot_to_verifier_profile( + snapshot: ConfigurationSnapshot, + *, + source_commit: str | None = None, + source_repo: str | None = None, + created_at: str | None = None, + seal: bool = True, +) -> dict[str, Any]: + """Build a VerifierProfile.v1 dict from a ConfigurationSnapshot and seal it.""" + commit = source_commit or resolve_source_commit() + repo = source_repo or resolve_source_repo() + producer = producer_fields() + implementation_digest = sha256_digest( + { + "adapter_id": snapshot.adapter_id, + "adapter_version": snapshot.adapter_version, + "backend_id": snapshot.backend_id, + } + ) + profile: dict[str, Any] = { + "schema_version": "v1", + "artifact_type": "VerifierProfile.v1", + "verifier_profile_id": _profile_id(snapshot), + "created_at": created_at or snapshot.created_at or _utc_now_iso(), + **producer, + "source_repo": repo, + "source_commit": commit, + "implementation": { + "name": snapshot.implementation_name or snapshot.backend_id, + "version": snapshot.adapter_version, + "language": "python", + "entry_point": snapshot.entry_point or snapshot.adapter_id, + "implementation_digest": implementation_digest, + "normalization_implementation_version": "ovk.normalize.v1", + }, + "configuration": { + "config_digest": snapshot.config_digest, + "policy_digest": snapshot.policy_digest, + "model_digest": snapshot.model_digest, + "prompt_digest": snapshot.prompt_digest, + "resource_limit_digest": snapshot.resource_limit_digest, + "rubric_digest": snapshot.rubric_digest, + "test_suite_digest": snapshot.test_suite_digest, + "threshold_digest": snapshot.threshold_digest, + "ensemble_digest": snapshot.ensemble_digest, + }, + "mechanism": { + "mechanism_class": snapshot.mechanism_class or "other", + "determinism": snapshot.determinism, + "allows_abstention": snapshot.allows_abstention, + "description": f"OVK assurance snapshot for backend {snapshot.backend_id}", + }, + "claim_surface": { + "supported_claim_ids": list(snapshot.supported_claim_ids) + or [f"claim.{snapshot.backend_id}"], + "guarantee_class": snapshot.guarantee_class, + "out_of_scope_claim_ids": list(snapshot.out_of_scope_claim_ids), + }, + "applicability": { + "status": "active", + "valid_from": created_at or snapshot.created_at or _utc_now_iso(), + }, + "assumptions": list(snapshot.assumptions) or ["configuration digests bind material inputs"], + "known_blind_spots": list(snapshot.known_blind_spots) + or ["Profile binds digests only; does not assert checker correctness"], + "canonical_configuration": dict(snapshot.config), + "redacted_environment": dict(snapshot.redacted_environment), + } + if snapshot.decision_space: + profile["claim_surface"]["decision_space"] = list(snapshot.decision_space) + if snapshot.external_dependencies: + profile["external_dependencies"] = list(snapshot.external_dependencies) + if snapshot.timeout_ms is not None: + profile["execution_controls"] = {"timeout_ms": int(snapshot.timeout_ms)} + profile["limitations_notice"] = ( + "Profile binds configuration and digests only; it does not claim " + "checker correctness beyond recorded results." + ) + if seal: + return attach_nested_integrity(profile) + return profile + + +def build_verification_result( + *, + verification_result_id: str, + profile: Mapping[str, Any], + decision: str, + execution_status: str, + claim_ids: list[str], + raw_backend_output_digest: str, + normalized_result_digest: str, + check_groups: list[dict[str, Any]] | None = None, + resource_limits: dict[str, Any] | None = None, + guarantee_class: str | None = None, + declared_input_guarantee_class: str | None = None, + normalization_applied: bool = True, + normalizer_version: str = "ovk.normalize.v1", + invocation_ref: dict[str, Any] | None = None, + input_bundle_digest: str | None = None, + assumptions: list[dict[str, Any]] | None = None, + limits: list[dict[str, Any]] | None = None, + source_commit: str | None = None, + source_repo: str | None = None, + created_at: str | None = None, + seal: bool = True, +) -> dict[str, Any]: + """Build a VerificationResult.v1 dict and optionally seal it.""" + profile_digest = profile.get("integrity", {}).get("artifact_digest") + if not isinstance(profile_digest, str): + body = {k: v for k, v in profile.items() if k != "integrity"} + profile_digest = sha256_digest(body) + producer = producer_fields() + resolved_guarantee = guarantee_class or profile.get("claim_surface", {}).get( + "guarantee_class", "observational" + ) + result: dict[str, Any] = { + "schema_version": "v1", + "artifact_type": "VerificationResult.v1", + "verification_result_id": verification_result_id, + "created_at": created_at or _utc_now_iso(), + **producer, + "source_repo": source_repo or resolve_source_repo(), + "source_commit": source_commit or resolve_source_commit(), + "verifier_profile": { + "verifier_profile_id": profile["verifier_profile_id"], + "profile_digest": profile_digest, + }, + "claim_ids": list(claim_ids), + "raw_backend_output_digest": raw_backend_output_digest, + "normalized_result_digest": normalized_result_digest, + "normalization_applied": normalization_applied, + "normalizer_version": normalizer_version, + "check_groups": list(check_groups or []), + "resource_limits": dict(resource_limits or {}), + "execution_status": execution_status, + "decision": decision, + "guarantee_class": resolved_guarantee, + } + if declared_input_guarantee_class is not None: + result["declared_input_guarantee_class"] = declared_input_guarantee_class + if invocation_ref is not None: + result["invocation_ref"] = dict(invocation_ref) + if input_bundle_digest is not None: + result["input_bundle_digest"] = input_bundle_digest + if assumptions is not None: + result["assumptions"] = list(assumptions) + if limits is not None: + result["limits"] = list(limits) + if seal: + return attach_nested_integrity(result) + return result + + +def build_invocation_record_artifact( + invocation: Mapping[str, Any], + *, + seal: bool = True, +) -> dict[str, Any]: + """Ensure an invocation dict is a sealed VerifierInvocationRecord.v1.""" + payload = dict(invocation) + payload.setdefault("schema_version", "v1") + payload.setdefault("artifact_type", "VerifierInvocationRecord.v1") + payload.setdefault("canonicalization_version", "v1") + producer = producer_fields() + payload.setdefault("producer", producer["producer"]) + payload.setdefault("producer_version", producer["producer_version"]) + if seal: + return attach_nested_integrity(payload) + return payload + + +def build_replay_report( + *, + replay_id: str, + original_invocation_ref: dict[str, Any], + profile_ref: dict[str, Any], + replay_status: str, + determinism: str, + original_raw_digest: str, + replay_raw_digest: str, + original_normalized_digest: str, + replay_normalized_digest: str, + drift: dict[str, Any], + replay_invocation_ref: dict[str, Any] | None = None, + indeterminate_reason: str | None = None, + failure_reason: str | None = None, + created_at: str | None = None, + seal: bool = True, +) -> dict[str, Any]: + """Build a sealed VerifierReplayReport.v1.""" + producer = producer_fields() + report: dict[str, Any] = { + "schema_version": "v1", + "artifact_type": "VerifierReplayReport.v1", + "canonicalization_version": "v1", + "replay_id": replay_id, + "original_invocation_ref": dict(original_invocation_ref), + "profile_ref": dict(profile_ref), + "replay_status": replay_status, + "determinism": determinism, + "original_raw_digest": original_raw_digest, + "replay_raw_digest": replay_raw_digest, + "original_normalized_digest": original_normalized_digest, + "replay_normalized_digest": replay_normalized_digest, + "drift": dict(drift), + **producer, + "created_at": created_at or _utc_now_iso(), + } + if replay_invocation_ref is not None: + report["replay_invocation_ref"] = dict(replay_invocation_ref) + if indeterminate_reason is not None: + report["indeterminate_reason"] = indeterminate_reason + if failure_reason is not None: + report["failure_reason"] = failure_reason + if seal: + return attach_nested_integrity(report) + return report + + +def build_mutation_manifest( + *, + mutation_id: str, + base_profile_ref: dict[str, Any], + mutated_profile_ref: dict[str, Any], + mutation_class: str, + expected_effect: str, + parameters: dict[str, Any] | None = None, + supported_by_adapter: bool | None = None, + rationale: str | None = None, + created_at: str | None = None, + seal: bool = True, +) -> dict[str, Any]: + """Build a sealed VerifierMutationManifest.v1 (production_prohibition always true).""" + producer = producer_fields() + manifest: dict[str, Any] = { + "schema_version": "v1", + "artifact_type": "VerifierMutationManifest.v1", + "canonicalization_version": "v1", + "mutation_id": mutation_id, + "base_profile_ref": dict(base_profile_ref), + "mutated_profile_ref": dict(mutated_profile_ref), + "mutation_class": mutation_class, + "expected_effect": expected_effect, + "production_prohibition": True, + **producer, + "created_at": created_at or _utc_now_iso(), + } + if parameters is not None: + manifest["parameters"] = dict(parameters) + if supported_by_adapter is not None: + manifest["supported_by_adapter"] = supported_by_adapter + if rationale is not None: + manifest["rationale"] = rationale + if seal: + return attach_nested_integrity(manifest) + return manifest + + +def profile_ref_from_profile(profile: Mapping[str, Any]) -> dict[str, Any]: + digest = profile.get("integrity", {}).get("artifact_digest") + if not isinstance(digest, str): + raise ValueError("profile missing integrity.artifact_digest") + return { + "artifact_type": "VerifierProfile.v1", + "verifier_profile_id": profile["verifier_profile_id"], + "profile_digest": digest, + } + + +def invocation_ref_from_record(record: Mapping[str, Any]) -> dict[str, Any]: + """Build a PCS opaque invocation_ref (id + digest only; no artifact_type).""" + digest = record.get("integrity", {}).get("artifact_digest") + if not isinstance(digest, str): + raise ValueError("invocation missing integrity.artifact_digest") + return { + "invocation_id": record["invocation_id"], + "invocation_digest": digest, + } diff --git a/ovk/assurance/pcs_hash.py b/ovk/assurance/pcs_hash.py new file mode 100644 index 0000000..c31419d --- /dev/null +++ b/ovk/assurance/pcs_hash.py @@ -0,0 +1,129 @@ +"""PCS-compatible hashing helpers for verifier-assurance artifacts. + +PCS artifact sealing and profile configuration digests MUST use +``pcs_core.hash.canonical_hash``. Local non-PCS digests may fall back to a +compatible Canonical-JSON hasher when pcs-core is unavailable. +""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any + +from ovk.assurance.errors import PinError +from ovk.assurance.pin import ensure_pcs_on_path, resolve_pcs_root + +CANONICALIZATION_VERSION = "v1" + + +def _local_canonical_json_bytes(value: Any) -> bytes: + """Compatible Canonical JSON serializer used only for non-PCS local digests.""" + + def _normalize(item: Any) -> Any: + if isinstance(item, dict): + return {str(k): _normalize(item[k]) for k in sorted(item, key=lambda x: str(x))} + if isinstance(item, list): + return [_normalize(v) for v in item] + if isinstance(item, bool) or item is None: + return item + if isinstance(item, int) and not isinstance(item, bool): + return item + if isinstance(item, float): + raise PinError( + "float values are prohibited in Canonical JSON digests; " + "use a normalized decimal string instead" + ) + return item + + return json.dumps(_normalize(value), separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _try_pcs_canonical_hash(): + root = resolve_pcs_root() + if root is None: + return None + try: + ensure_pcs_on_path() + from pcs_core.hash import canonical_hash as pcs_canonical_hash + + return pcs_canonical_hash + except Exception: + return None + + +def pcs_canonical_hash(value: dict[str, Any]) -> str: + """Hash a PCS artifact body with pcs_core. Fail closed if unavailable.""" + hasher = _try_pcs_canonical_hash() + if hasher is None: + raise PinError( + "pcs_core.hash.canonical_hash is required to seal or validate PCS artifacts; " + "resolve the PCS pin (OVK_PCS_CORE_PATH / sibling pcs-core / installed package)" + ) + return hasher(value) + + +def sha256_digest(value: Any, *, require_pcs: bool = False) -> str: + """Return ``sha256:`` + 64 hex for a JSON-like value. + + When ``require_pcs`` is True, always use pcs_core (fail closed). + Otherwise prefer pcs_core when available; fall back to a local Canonical + JSON hasher for non-PCS local digests only. + """ + if isinstance(value, dict): + hasher = _try_pcs_canonical_hash() + if hasher is not None: + return hasher(value) + if require_pcs: + raise PinError( + "pcs_core.hash.canonical_hash is required; PCS pin unavailable" + ) + digest = hashlib.sha256(_local_canonical_json_bytes(value)).hexdigest() + return f"sha256:{digest}" + + # Non-dict values: hash Canonical JSON encoding of the value. + hasher = _try_pcs_canonical_hash() + if require_pcs and hasher is None: + raise PinError("pcs_core.hash.canonical_hash is required; PCS pin unavailable") + if hasher is not None and isinstance(value, (list, dict)): + # pcs_core.canonical_hash expects a dict; wrap scalars/lists. + if isinstance(value, list): + payload = {"_": value} + # Prefer raw local for lists to avoid wrapping skew — use local bytes. + digest = hashlib.sha256(_local_canonical_json_bytes(value)).hexdigest() + return f"sha256:{digest}" + digest = hashlib.sha256(_local_canonical_json_bytes(value)).hexdigest() + return f"sha256:{digest}" + + +def attach_nested_integrity(data: dict[str, Any]) -> dict[str, Any]: + """Seal a PCS artifact by hashing the body WITHOUT ``integrity``. + + Sets ``integrity: {canonicalization_version: "v1", artifact_digest}``. + Requires pcs_core (fail closed). + """ + body = {k: v for k, v in data.items() if k != "integrity"} + digest = pcs_canonical_hash(body) + out = dict(body) + out["integrity"] = { + "canonicalization_version": CANONICALIZATION_VERSION, + "artifact_digest": digest, + } + return out + + +def verify_nested_integrity(data: dict[str, Any]) -> str: + """Recompute and verify nested integrity; return the matching digest.""" + integrity = data.get("integrity") + if not isinstance(integrity, dict): + raise PinError("PCS artifact missing nested integrity") + expected = integrity.get("artifact_digest") + if not isinstance(expected, str) or not expected.startswith("sha256:"): + raise PinError("PCS artifact integrity.artifact_digest is missing or invalid") + body = {k: v for k, v in data.items() if k != "integrity"} + actual = pcs_canonical_hash(body) + if actual != expected: + raise PinError( + f"PCS artifact integrity mismatch: expected {expected}, recomputed {actual}" + ) + return actual diff --git a/ovk/assurance/pcs_validate.py b/ovk/assurance/pcs_validate.py new file mode 100644 index 0000000..9660eb9 --- /dev/null +++ b/ovk/assurance/pcs_validate.py @@ -0,0 +1,138 @@ +"""Validate PCS verifier-assurance artifacts against the pin schemas.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Any + +from jsonschema import Draft202012Validator +from referencing import Registry, Resource + +from ovk.assurance.errors import PinError +from ovk.assurance.pcs_hash import verify_nested_integrity +from ovk.assurance.pin import ( + ARTIFACT_SCHEMA_FILES, + ensure_pcs_on_path, + require_pcs_pin, + schema_path, + schemas_dir, +) + + +@dataclass(frozen=True) +class PcsValidationIssue: + path: str + message: str + + +@dataclass(frozen=True) +class PcsValidationReport: + valid: bool + issues: list[PcsValidationIssue] + + +def _load_json(path: Path) -> dict[str, Any]: + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise PinError(f"schema root must be an object: {path}") + return data + + +@lru_cache(maxsize=1) +def pcs_schema_registry() -> Registry: + """Build a jsonschema registry from the pinned PCS schemas directory. + + Only JSON Schema documents are registered (``*.schema.json`` and + ``*.defs.json``). Catalog / status JSON files under schemas/ are skipped. + """ + from referencing.jsonschema import DRAFT202012 + + root = schemas_dir() + registry = Registry() + paths = sorted(set(root.glob("*.schema.json")) | set(root.glob("*.defs.json"))) + for path in paths: + schema = _load_json(path) + if "$schema" not in schema and "$id" not in schema and "$defs" not in schema: + continue + resource = Resource.from_contents(schema, default_specification=DRAFT202012) + schema_id = str(schema.get("$id") or path.as_uri()) + registry = registry.with_resource(schema_id, resource) + # Also register by basename for relative $ref resolution. + registry = registry.with_resource(path.name, resource) + return registry + + +def validate_against_pin_schema( + artifact: dict[str, Any], + *, + artifact_type: str | None = None, + check_integrity: bool = True, +) -> PcsValidationReport: + """Validate *artifact* against the pin schema for its artifact type.""" + resolved_type = artifact_type or str(artifact.get("artifact_type") or "") + if not resolved_type: + return PcsValidationReport( + valid=False, + issues=[PcsValidationIssue(path="artifact_type", message="missing artifact_type")], + ) + if resolved_type not in ARTIFACT_SCHEMA_FILES: + raise PinError(f"unknown PCS assurance artifact type: {resolved_type!r}") + + schema = _load_json(schema_path(resolved_type)) + validator = Draft202012Validator(schema, registry=pcs_schema_registry()) + issues = [ + PcsValidationIssue( + path="/".join(str(part) for part in error.path) or "$", + message=error.message, + ) + for error in sorted(validator.iter_errors(artifact), key=lambda item: list(item.path)) + ] + + if check_integrity and not issues: + try: + verify_nested_integrity(artifact) + except PinError as exc: + issues.append(PcsValidationIssue(path="integrity", message=str(exc))) + + return PcsValidationReport(valid=not issues, issues=issues) + + +def require_valid_pcs_artifact( + artifact: dict[str, Any], + *, + artifact_type: str | None = None, + check_integrity: bool = True, +) -> None: + """Raise ``PinError`` when schema or integrity validation fails.""" + report = validate_against_pin_schema( + artifact, + artifact_type=artifact_type, + check_integrity=check_integrity, + ) + if report.valid: + return + detail = "; ".join(f"{issue.path}: {issue.message}" for issue in report.issues) + raise PinError(f"PCS artifact failed validation: {detail}") + + +def semantic_validate(artifact: dict[str, Any]) -> list[str]: + """Optionally run pcs_core semantic validation when importable. + + Returns a list of issue strings (empty when valid or when the semantic + validator is unavailable). Schema validation remains the hard gate. + """ + try: + ensure_pcs_on_path() + require_pcs_pin() + from pcs_core.verifier_assurance_validate import validate_va_semantics + except Exception: + return [] + + try: + issues = validate_va_semantics(artifact) + except Exception as exc: # pragma: no cover - defensive + return [str(exc)] + return [str(item) for item in (issues or [])] diff --git a/ovk/assurance/pin.py b/ovk/assurance/pin.py new file mode 100644 index 0000000..d011162 --- /dev/null +++ b/ovk/assurance/pin.py @@ -0,0 +1,187 @@ +"""PCS pin resolution for verifier-assurance schemas. + +Resolution order (documented in docs/PCS_PIN.md): + +1. ``OVK_PCS_CORE_PATH`` +2. ``PCS_CORE_PATH`` +3. Sibling ``../pcs-core`` relative to the OVK repository root +4. Installed ``pcs-core`` package (when available) + +Missing pin / missing required schemas / digest drift / unknown artifact types +fail closed. +""" + +from __future__ import annotations + +import hashlib +import importlib.util +import os +import sys +from collections.abc import Mapping +from functools import lru_cache +from pathlib import Path + +from ovk.assurance.errors import PinError +from ovk.paths import ovk_data_root + +# Authoritative OVK pin surface (all required at the documented pcs-core SHA). +ARTIFACT_SCHEMA_FILES: dict[str, str] = { + "VerifierProfile.v1": "VerifierProfile.v1.schema.json", + "VerificationResult.v1": "VerificationResult.v1.schema.json", + "VerifierInvocationRecord.v1": "VerifierInvocationRecord.v1.schema.json", + "VerifierReplayReport.v1": "VerifierReplayReport.v1.schema.json", + "VerifierMutationManifest.v1": "VerifierMutationManifest.v1.schema.json", +} + +DEFS_SCHEMA_FILE = "verifier_assurance.defs.json" + +# Digests for pcs-core commit fb588a41a7eab68064429e3c4dfb26c328b9863d (docs/PCS_PIN.md). +EXPECTED_SCHEMA_DIGESTS: dict[str, str] = { + "VerifierProfile.v1": ( + "sha256:a657a63eee47a00419f31008f0adee5559e37fdba2544831e8b297c0a2dbe9bd" + ), + "VerificationResult.v1": ( + "sha256:146534a7ebf8ee8cdaecaa57258c0ce11224f50aed1a71196bc7b72d2c5b6d17" + ), + "VerifierInvocationRecord.v1": ( + "sha256:3ee1384cd5fae5e08b87870100609a9a9b8cf2502b2c4d92de9dedc1f9ffbc3d" + ), + "VerifierReplayReport.v1": ( + "sha256:06660ef51c89385869306c2f1c7f1364bec129b783007cc7a8caa4322582bd3b" + ), + "VerifierMutationManifest.v1": ( + "sha256:b82952c1d41ddd151cd71440a5a38f7e768c468c3ff4ae11f3a80325d4cb4819" + ), + DEFS_SCHEMA_FILE: ( + "sha256:c417accb1b4bc08d6e6f0f98e71ee6e7c87a923d19c5054a18841e7e04eadabb" + ), +} + +PCS_PIN_COMMIT = "fb588a41a7eab68064429e3c4dfb26c328b9863d" + + +def _looks_like_pcs_root(path: Path) -> bool: + schemas = path / "schemas" + return schemas.is_dir() and (schemas / "VerifierProfile.v1.schema.json").is_file() + + +def _installed_pcs_root() -> Path | None: + spec = importlib.util.find_spec("pcs_core") + if spec is None or spec.origin is None: + return None + package_dir = Path(spec.origin).resolve().parent + # Editable / src layout: pcs-core/python/pcs_core -> pcs-core root + candidate = package_dir.parent.parent + if _looks_like_pcs_root(candidate): + return candidate + # Wheel may ship schemas next to the package + if _looks_like_pcs_root(package_dir): + return package_dir + if _looks_like_pcs_root(package_dir.parent): + return package_dir.parent + return None + + +def resolve_pcs_root() -> Path | None: + """Resolve the pcs-core checkout root, or None when unavailable.""" + for env_name in ("OVK_PCS_CORE_PATH", "PCS_CORE_PATH"): + raw = (os.environ.get(env_name) or "").strip() + if raw: + path = Path(raw).expanduser().resolve() + if _looks_like_pcs_root(path): + return path + return None + + ovk_root = ovk_data_root() + sibling = (ovk_root / ".." / "pcs-core").resolve() + if _looks_like_pcs_root(sibling): + return sibling + + return _installed_pcs_root() + + +def require_pcs_pin() -> Path: + """Return the resolved PCS root or raise ``PinError``.""" + root = resolve_pcs_root() + if root is None: + raise PinError( + "PCS pin unavailable: set OVK_PCS_CORE_PATH or PCS_CORE_PATH, " + "place a sibling ../pcs-core checkout, or install pcs-core" + ) + return root + + +def ensure_pcs_on_path() -> Path: + """Ensure ``pcs_core`` is importable by adding ``/python`` to sys.path.""" + root = require_pcs_pin() + python_root = root / "python" + if python_root.is_dir(): + path = str(python_root) + if path not in sys.path: + sys.path.insert(0, path) + return root + + +def schema_path(artifact_type: str) -> Path: + """Return the absolute schema path for a pinned VA artifact type.""" + filename = ARTIFACT_SCHEMA_FILES.get(artifact_type) + if filename is None: + raise PinError(f"unknown PCS assurance artifact type: {artifact_type!r}") + root = require_pcs_pin() + path = root / "schemas" / filename + if not path.is_file(): + raise PinError(f"PCS schema missing for {artifact_type}: {path}") + return path + + +def schemas_dir() -> Path: + """Return the PCS schemas directory from the resolved pin.""" + root = require_pcs_pin() + path = root / "schemas" + if not path.is_dir(): + raise PinError(f"PCS schemas directory missing: {path}") + return path + + +def file_digest(path: Path) -> str: + """Return ``sha256:`` + hex digest of file bytes.""" + digest = hashlib.sha256(path.read_bytes()).hexdigest() + return f"sha256:{digest}" + + +@lru_cache(maxsize=1) +def load_schema_digests() -> dict[str, str]: + """Load digests for pinned VA schema files (fail closed if pin incomplete).""" + root = require_pcs_pin() + schemas = root / "schemas" + digests: dict[str, str] = {} + for artifact_type, filename in ARTIFACT_SCHEMA_FILES.items(): + path = schemas / filename + if not path.is_file(): + raise PinError(f"PCS schema missing for {artifact_type}: {path}") + digests[artifact_type] = file_digest(path) + defs_path = schemas / DEFS_SCHEMA_FILE + if not defs_path.is_file(): + raise PinError(f"PCS defs schema missing: {defs_path}") + digests[DEFS_SCHEMA_FILE] = file_digest(defs_path) + return digests + + +def verify_pin_digests(*, expected: Mapping[str, str] | None = None) -> dict[str, str]: + """Fail closed when resolved schema digests drift from the documented pin table. + + Returns the verified digest map on success. + """ + table = dict(expected or EXPECTED_SCHEMA_DIGESTS) + actual = load_schema_digests() + mismatches: list[str] = [] + for key, want in sorted(table.items()): + got = actual.get(key) + if got != want: + mismatches.append(f"{key}: expected {want}, got {got!r}") + if mismatches: + raise PinError( + "PCS pin schema digest drift (commit " + f"{PCS_PIN_COMMIT}): " + "; ".join(mismatches) + ) + return actual diff --git a/ovk/assurance/redaction.py b/ovk/assurance/redaction.py new file mode 100644 index 0000000..e0493f4 --- /dev/null +++ b/ovk/assurance/redaction.py @@ -0,0 +1,132 @@ +"""Secret redaction for assurance configuration and environment exports.""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from ovk.assurance.pcs_hash import sha256_digest +from ovk.core.execution_budget import _SECRET_ENV_DENYLIST + +REDACTION_POLICY_ID = "ovk-redact-v1" + +_SECRET_NAME_RE = re.compile( + r"(?:^|_)(TOKEN|SECRET|PASSWORD|PASSWD|API[_-]?KEY|PRIVATE[_-]?KEY|" + r"CREDENTIAL|CONN(?:ECTION)?[_-]?STRING|JWT|BEARER|AUTH)(?:_|$)", + re.IGNORECASE, +) + +_SECRET_VALUE_RE = re.compile( + r"(?:" + r"ghp_[A-Za-z0-9]{20,}" + r"|github_pat_[A-Za-z0-9_]{20,}" + r"|sk-[A-Za-z0-9]{20,}" + r"|Bearer\s+[A-Za-z0-9\-._~+/]+=*" + r"|-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----" + r")", + re.IGNORECASE, +) + +_EXTRA_SECRET_KEY_TOKENS = ( + "TOKEN", + "SECRET", + "PASSWORD", + "PASSWD", + "API_KEY", + "APIKEY", + "PRIVATE_KEY", + "CREDENTIAL", + "CONNECTION_STRING", + "CONN_STRING", + "PGPASSWORD", + "MYSQL_PWD", + "DATABASE_URL", + "JWT", +) + + +def is_secret_key(key: str) -> bool: + """Return True when *key* matches the denylist or secret name patterns.""" + upper = key.upper() + if upper in _SECRET_ENV_DENYLIST: + return True + if upper in _EXTRA_SECRET_KEY_TOKENS: + return True + if _SECRET_NAME_RE.search(key): + return True + for token in _EXTRA_SECRET_KEY_TOKENS: + if token in upper: + return True + return False + + +def looks_like_secret_value(value: Any) -> bool: + """Return True when *value* matches high-confidence secret value patterns.""" + if not isinstance(value, str) or not value: + return False + return _SECRET_VALUE_RE.search(value) is not None + + +def redact_environment( + environment: Mapping[str, str] | None, +) -> dict[str, Any]: + """Redact secret-bearing environment entries. + + Returns a PCS-shaped redacted_environment object:: + + { + "redaction_policy_id": "ovk-redact-v1", + "entries": {...}, + "redacted_keys": [...], + "environment_digest": "sha256:..." + } + """ + entries: dict[str, str] = {} + redacted_keys: list[str] = [] + for key, value in sorted((environment or {}).items(), key=lambda item: item[0]): + text = str(value) + if is_secret_key(key) or looks_like_secret_value(text): + redacted_keys.append(key) + continue + entries[str(key)] = text + payload = { + "redaction_policy_id": REDACTION_POLICY_ID, + "entries": entries, + "redacted_keys": sorted(set(redacted_keys)), + } + payload["environment_digest"] = sha256_digest( + {"entries": entries, "redacted_keys": payload["redacted_keys"], "policy": REDACTION_POLICY_ID} + ) + return payload + + +def redact_mapping(data: Mapping[str, Any] | None) -> tuple[dict[str, Any], list[str]]: + """Return a copy of *data* with secret keys/values removed, plus redacted paths.""" + cleaned: dict[str, Any] = {} + redacted: list[str] = [] + for key, value in (data or {}).items(): + key_s = str(key) + if is_secret_key(key_s): + redacted.append(key_s) + continue + if isinstance(value, dict): + nested, nested_redacted = redact_mapping(value) + cleaned[key_s] = nested + redacted.extend(f"{key_s}.{item}" for item in nested_redacted) + elif isinstance(value, list): + nested_list: list[Any] = [] + for index, item in enumerate(value): + if isinstance(item, dict): + nested_item, nested_redacted = redact_mapping(item) + nested_list.append(nested_item) + redacted.extend(f"{key_s}[{index}].{path}" for path in nested_redacted) + elif looks_like_secret_value(item): + redacted.append(f"{key_s}[{index}]") + else: + nested_list.append(item) + cleaned[key_s] = nested_list + elif looks_like_secret_value(value): + redacted.append(key_s) + else: + cleaned[key_s] = value + return cleaned, sorted(set(redacted)) diff --git a/ovk/assurance/registry.py b/ovk/assurance/registry.py new file mode 100644 index 0000000..0a4304f --- /dev/null +++ b/ovk/assurance/registry.py @@ -0,0 +1,73 @@ +"""Verifier registry for assurance describe/lookup (ordinary backends remain ordinary).""" + +from __future__ import annotations + +from typing import Any + +from ovk.adapters.assurance import build_assurance_registry +from ovk.adapters.authorization import build_authorization_registry +from ovk.adapters.ci_secrets import build_ci_secrets_registry +from ovk.adapters.deployment import build_deployment_registry +from ovk.adapters.infrastructure import build_infrastructure_registry +from ovk.adapters.lane import build_default_lane_registry +from ovk.adapters.self_protection import build_self_protection_registry +from ovk.assurance.capability import is_assurance_capable +from ovk.assurance.errors import AssuranceError +from ovk.core.backend_registry import BackendRegistry, BackendRegistryError + + +def build_verifier_registry(*, include_lanes: bool = True, include_assurance: bool = True) -> BackendRegistry: + """Register known lane + native/deterministic + assurance backends for describe/lookup. + + Does not register test-only adapters. Ordinary backends remain non-assurance + unless they advertise a valid assurance section. Assurance-only backends are + registered for ``ovk verifier`` and are not selected by ordinary routing. + """ + registry = BackendRegistry() + builders = [ + build_self_protection_registry, + build_authorization_registry, + build_infrastructure_registry, + build_ci_secrets_registry, + build_deployment_registry, + ] + if include_lanes: + builders.append(build_default_lane_registry) + if include_assurance: + builders.append(build_assurance_registry) + + for builder in builders: + partial = builder() + for adapter in partial.all(): + if registry.get(adapter.backend_id) is not None: + continue + try: + registry.register(adapter) + except BackendRegistryError: + # Skip duplicates across domain/lane builders. + continue + return registry + + +def lookup_backend(backend: str, registry: BackendRegistry | None = None) -> Any: + """Return a registered backend adapter or raise AssuranceError.""" + reg = registry or build_verifier_registry() + adapter = reg.get(backend) + if adapter is None: + raise AssuranceError(f"backend not registered: {backend}") + return adapter + + +def describe_backend(backend: str, registry: BackendRegistry | None = None) -> dict[str, Any]: + """Return ordinary capability plus assurance summary for a backend.""" + adapter = lookup_backend(backend, registry=registry) + manifest = adapter.manifest() + payload = manifest.model_dump(mode="json", exclude_none=True) + payload["assurance_capable"] = is_assurance_capable(manifest) + if hasattr(adapter, "supported_mutation_dimensions") and callable(adapter.supported_mutation_dimensions): + payload["supported_mutation_dimensions"] = list(adapter.supported_mutation_dimensions()) + elif manifest.assurance is not None: + payload["supported_mutation_dimensions"] = list(manifest.assurance.mutation_dimensions) + else: + payload["supported_mutation_dimensions"] = [] + return payload diff --git a/ovk/assurance/replay.py b/ovk/assurance/replay.py new file mode 100644 index 0000000..1406fb4 --- /dev/null +++ b/ovk/assurance/replay.py @@ -0,0 +1,217 @@ +"""Assurance invocation replay (distinct from ordinary cache replay).""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Mapping + +from ovk.assurance.errors import ReplayError +from ovk.assurance.pcs_export import ( + build_replay_report, + invocation_ref_from_record, + profile_ref_from_profile, +) +from ovk.assurance.pcs_hash import sha256_digest +from ovk.assurance.pcs_validate import require_valid_pcs_artifact +from ovk.assurance.runner import run_assurance +from ovk.assurance.snapshot import ConfigurationSnapshot + + +def _load_json(path: Path) -> dict[str, Any]: + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ReplayError(f"expected JSON object: {path}") + return data + + +def replay_invocation( + invocation: Mapping[str, Any] | Path | str, + *, + adapter: Any, + evidence_dir: Path | str | None = None, + input_data: Mapping[str, Any] | None = None, + config: Mapping[str, Any] | None = None, + profile: Mapping[str, Any] | None = None, + claim_matched: bool = False, +) -> dict[str, Any]: + """Replay a prior invocation against an assurance-capable adapter. + + Fail closed on configuration drift when claiming ``matched``. Stochastic + backends cannot claim matched. + """ + if isinstance(invocation, (str, Path)): + original = _load_json(Path(invocation)) + else: + original = dict(invocation) + require_valid_pcs_artifact(original, artifact_type="VerifierInvocationRecord.v1") + + evidence_root = Path(evidence_dir) if evidence_dir is not None else None + sealed_profile = dict(profile) if profile is not None else None + if sealed_profile is None and evidence_root is not None: + profile_path = evidence_root / "verifier_profile.pcs.json" + if profile_path.is_file(): + sealed_profile = _load_json(profile_path) + + if sealed_profile is None: + raise ReplayError("replay requires a sealed verifier profile (pass profile= or evidence-dir)") + + require_valid_pcs_artifact(sealed_profile, artifact_type="VerifierProfile.v1") + + # Detect profile drift against invocation binding. + expected_profile_digest = original["profile_ref"]["profile_digest"] + actual_profile_digest = sealed_profile["integrity"]["artifact_digest"] + environment_drift: list[str] = [] + missing_dependencies: list[str] = [] + if expected_profile_digest != actual_profile_digest: + environment_drift.append( + f"profile_digest drift: invocation={expected_profile_digest} current={actual_profile_digest}" + ) + + determinism = str( + sealed_profile.get("mechanism", {}).get("determinism") + or getattr(getattr(adapter, "manifest")().assurance, "determinism", "deterministic") + ) + + # Resolve input for rerun. + resolved_input = dict(input_data or {}) + if not resolved_input and evidence_root is not None: + obligation_path = evidence_root / "compiled_obligation.json" + if obligation_path.is_file(): + obligation = _load_json(obligation_path) + if isinstance(obligation.get("input"), dict): + resolved_input = dict(obligation["input"]) + if not resolved_input: + raise ReplayError("replay requires input_data or compiled_obligation.json with input") + + # Detect missing external dependencies advertised on the profile. + for dep in sealed_profile.get("external_dependencies") or []: + if not isinstance(dep, dict): + continue + if dep.get("optional"): + continue + identity = str(dep.get("identity") or "") + kind = str(dep.get("kind") or "") + if kind == "binary" and identity: + from shutil import which + + if which(identity) is None: + missing_dependencies.append(identity) + + original_raw = original["raw_backend_result_digest"] + original_normalized = original["normalized_result_digest"] + + if missing_dependencies: + report = build_replay_report( + replay_id=f"replay-{original['invocation_id']}", + original_invocation_ref=invocation_ref_from_record(original), + profile_ref=profile_ref_from_profile(sealed_profile), + replay_status="indeterminate", + determinism=determinism, + original_raw_digest=original_raw, + replay_raw_digest=sha256_digest({"missing": missing_dependencies}), + original_normalized_digest=original_normalized, + replay_normalized_digest=sha256_digest({"missing": missing_dependencies}), + drift={ + "raw_digest_match": False, + "normalized_digest_match": False, + "missing_dependencies": missing_dependencies, + "environment_drift": environment_drift, + "notes": "missing required dependencies; replay not executed", + }, + indeterminate_reason="missing_checker", + ) + if claim_matched: + raise ReplayError("cannot claim matched when dependencies are missing") + return report + + if environment_drift and claim_matched: + raise ReplayError(f"configuration drift; refuse matched claim: {environment_drift}") + + if determinism == "stochastic": + report = build_replay_report( + replay_id=f"replay-{original['invocation_id']}", + original_invocation_ref=invocation_ref_from_record(original), + profile_ref=profile_ref_from_profile(sealed_profile), + replay_status="indeterminate", + determinism=determinism, + original_raw_digest=original_raw, + replay_raw_digest=original_raw, + original_normalized_digest=original_normalized, + replay_normalized_digest=original_normalized, + drift={ + "raw_digest_match": False, + "normalized_digest_match": False, + "missing_dependencies": [], + "environment_drift": environment_drift, + "notes": "stochastic verifier cannot claim bit-identical matched replay", + }, + indeterminate_reason="declared_nondeterminism", + ) + if claim_matched: + raise ReplayError("stochastic verifier cannot claim matched replay") + return report + + # Deterministic rerun + snapshot = None + if hasattr(adapter, "snapshot_config"): + snap = adapter.snapshot_config(config or sealed_profile.get("canonical_configuration") or {}) + snapshot = snap if isinstance(snap, ConfigurationSnapshot) else ConfigurationSnapshot.model_validate(snap) + + outcome = run_assurance( + adapter, + input_data=resolved_input, + config=config or sealed_profile.get("canonical_configuration") or {}, + profile=sealed_profile, + evidence_dir=None, + ) + replay_invocation = outcome.invocation + replay_raw = replay_invocation["raw_backend_result_digest"] + replay_normalized = replay_invocation["normalized_result_digest"] + raw_match = replay_raw == original_raw + normalized_match = replay_normalized == original_normalized + + if environment_drift: + status = "drifted" + elif raw_match and normalized_match: + status = "matched" + else: + status = "drifted" + + if claim_matched and status != "matched": + raise ReplayError( + f"replay drifted (raw_match={raw_match}, normalized_match={normalized_match}, " + f"env_drift={environment_drift}); refuse matched claim" + ) + + report = build_replay_report( + replay_id=f"replay-{original['invocation_id']}", + original_invocation_ref=invocation_ref_from_record(original), + profile_ref=profile_ref_from_profile(sealed_profile), + replay_status=status, + determinism=determinism, + original_raw_digest=original_raw, + replay_raw_digest=replay_raw, + original_normalized_digest=original_normalized, + replay_normalized_digest=replay_normalized, + drift={ + "raw_digest_match": raw_match, + "normalized_digest_match": normalized_match, + "missing_dependencies": missing_dependencies, + "environment_drift": environment_drift, + **( + {"notes": f"snapshot_digest={snapshot.content_digest}"} + if snapshot is not None + else {} + ), + }, + replay_invocation_ref=invocation_ref_from_record(replay_invocation), + ) + + if evidence_root is not None: + evidence_root.mkdir(parents=True, exist_ok=True) + (evidence_root / "replay_report.pcs.json").write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report diff --git a/ovk/assurance/runner.py b/ovk/assurance/runner.py new file mode 100644 index 0000000..201cb4e --- /dev/null +++ b/ovk/assurance/runner.py @@ -0,0 +1,280 @@ +"""Assurance-mode runner (does not touch ordinary ovk check / MCP paths).""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from ovk.assurance.capability import is_assurance_capable +from ovk.assurance.errors import AssuranceError +from ovk.assurance.evidence_pack import write_evidence_pack +from ovk.assurance.guarantee import clamp_guarantee_class +from ovk.assurance.indeterminate import ( + DECISION_ACCEPT, + DECISION_REJECT, + decision_for_indeterminate_reason, + execution_status_for_reason, + indeterminate_outcome, + indeterminate_reason_for_termination, +) +from ovk.assurance.invocation import build_invocation_record +from ovk.assurance.pcs_export import build_verification_result, snapshot_to_verifier_profile +from ovk.assurance.pcs_hash import sha256_digest +from ovk.assurance.snapshot import ConfigurationSnapshot, snapshot_from_adapter + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +@dataclass +class AssuranceRunOutcome: + """Result of an assurance-mode run including evidence pack paths.""" + + decision: str + execution_status: str + indeterminate_reason: str | None + profile: dict[str, Any] + result: dict[str, Any] + invocation: dict[str, Any] + snapshot: ConfigurationSnapshot + evidence_dir: Path | None = None + raw_result: dict[str, Any] = field(default_factory=dict) + normalized_result: dict[str, Any] = field(default_factory=dict) + + +def _require_assurance_adapter(adapter: Any) -> None: + if not hasattr(adapter, "manifest") or not callable(adapter.manifest): + raise AssuranceError("adapter must expose manifest()") + manifest = adapter.manifest() + if not is_assurance_capable(manifest): + raise AssuranceError( + f"backend {getattr(adapter, 'backend_id', '?')!r} is not assurance_capable; " + "snapshot-config/run require an assurance-capable adapter" + ) + if not hasattr(adapter, "snapshot_config") or not callable(adapter.snapshot_config): + raise AssuranceError( + f"backend {getattr(adapter, 'backend_id', '?')!r} claims assurance but " + "does not implement snapshot_config()" + ) + + +def _status_to_decision(status: str) -> str: + normalized = str(status or "").strip().lower() + if normalized in {"pass", "passed", "accept", "allow", "success"}: + return DECISION_ACCEPT + if normalized in {"fail", "failed", "reject", "deny"}: + return DECISION_REJECT + return decision_for_indeterminate_reason(indeterminate_reason_for_termination(normalized)) + + +def run_assurance( + adapter: Any, + *, + input_data: Mapping[str, Any], + config: Mapping[str, Any] | None = None, + environment: Mapping[str, str] | None = None, + evidence_dir: Path | str | None = None, + profile: Mapping[str, Any] | None = None, + cwd: str | None = None, +) -> AssuranceRunOutcome: + """Execute an assurance-capable adapter and optionally write an evidence pack. + + Ordinary ``ovk check`` / MCP paths are not invoked. + """ + _require_assurance_adapter(adapter) + manifest = adapter.manifest() + assurance = manifest.assurance + assert assurance is not None + + snapshot: ConfigurationSnapshot + if hasattr(adapter, "snapshot_config"): + snap = adapter.snapshot_config(config, environment=environment) + snapshot = snap if isinstance(snap, ConfigurationSnapshot) else ConfigurationSnapshot.model_validate(snap) + else: + snapshot = snapshot_from_adapter(adapter, config, environment=environment) + + sealed_profile = dict(profile) if profile is not None else snapshot_to_verifier_profile(snapshot) + + started = _utc_now_iso() + exit_kind = "exited" + exit_code: int | None = 0 + exit_message: str | None = None + stdout = "" + stderr = "" + raw_result: dict[str, Any] = {} + normalized_result: dict[str, Any] = {} + decision = decision_for_indeterminate_reason("other") + execution_status = "error" + indeterminate_reason: str | None = None + declared_guarantee = assurance.decision_semantics.guarantee_class + guarantee_class = declared_guarantee + command_argv = [snapshot.adapter_id, "assurance-run"] + + try: + if hasattr(adapter, "run_assurance") and callable(adapter.run_assurance): + outcome = adapter.run_assurance( + input_data=dict(input_data), + snapshot=snapshot, + config=dict(config or {}), + ) + if not isinstance(outcome, Mapping): + raise AssuranceError("run_assurance must return a mapping") + exit_kind = str(outcome.get("exit_kind") or "exited") + exit_code = outcome.get("exit_code") + exit_message = outcome.get("message") + stdout = str(outcome.get("stdout") or "") + stderr = str(outcome.get("stderr") or "") + raw_result = dict(outcome.get("raw_result") or {}) + normalized_result = dict(outcome.get("normalized_result") or {}) + indeterminate_reason = outcome.get("indeterminate_reason") + if outcome.get("decision"): + decision = str(outcome["decision"]) + elif indeterminate_reason: + decision = decision_for_indeterminate_reason(str(indeterminate_reason)) + else: + decision = _status_to_decision(str(outcome.get("status") or exit_kind)) + execution_status = str( + outcome.get("execution_status") + or ( + execution_status_for_reason(str(indeterminate_reason)) + if indeterminate_reason + else "completed" + ) + ) + if outcome.get("guarantee_class"): + guarantee_class = clamp_guarantee_class(declared_guarantee, str(outcome["guarantee_class"])) + if outcome.get("command_argv"): + command_argv = list(outcome["command_argv"]) + else: + raise AssuranceError( + f"backend {adapter.backend_id!r} does not implement run_assurance(); " + "assurance run requires an explicit assurance execution path" + ) + except AssuranceError: + raise + except FileNotFoundError as exc: + ind = indeterminate_outcome(reason="missing_checker", message=str(exc)) + decision = ind["decision"] + execution_status = ind["execution_status"] + indeterminate_reason = ind["indeterminate_reason"] + exit_kind = "missing_checker" + exit_code = None + exit_message = str(exc) + raw_result = {"error": "missing_checker", "message": str(exc)} + normalized_result = {"decision": decision, "indeterminate_reason": indeterminate_reason} + except TimeoutError as exc: + ind = indeterminate_outcome(reason="timeout", message=str(exc)) + decision = ind["decision"] + execution_status = ind["execution_status"] + indeterminate_reason = ind["indeterminate_reason"] + exit_kind = "timeout" + exit_code = None + exit_message = str(exc) + raw_result = {"error": "timeout", "message": str(exc)} + normalized_result = {"decision": decision, "indeterminate_reason": indeterminate_reason} + + # Hard invariant: missing checker / timeout never become accept. + if indeterminate_reason in {"missing_checker", "timeout"} and decision == DECISION_ACCEPT: + decision = decision_for_indeterminate_reason(indeterminate_reason) + + completed = _utc_now_iso() + claim_ids = list(assurance.decision_semantics.supported_claim_ids) or [ + f"claim.{snapshot.backend_id}" + ] + + check_status = "passed" if decision == DECISION_ACCEPT else ("failed" if decision == DECISION_REJECT else "skipped") + check_groups: list[dict[str, Any]] = [ + { + "kind": "authority", + "checks": [ + { + "check_id": f"{snapshot.backend_id}-predicate", + "mandatory": True, + "status": check_status if check_status != "skipped" else "skipped", + **({"reason_code": indeterminate_reason} if check_status == "skipped" and indeterminate_reason else {}), + } + ], + } + ] + if check_status == "skipped" and not indeterminate_reason: + check_groups[0]["checks"][0]["reason_code"] = "other" + + invocation = build_invocation_record( + snapshot=snapshot, + profile=sealed_profile, + input_data=input_data, + command_argv=command_argv, + cwd=cwd or os.getcwd(), + started_at=started, + completed_at=completed, + exit_kind=exit_kind, + exit_code=exit_code if isinstance(exit_code, int) else None, + exit_message=exit_message, + stdout=stdout, + stderr=stderr, + raw_result=raw_result, + normalized_result=normalized_result, + guarantee_class=guarantee_class, + indeterminate_reason=indeterminate_reason, + compiled_obligation={"backend_id": snapshot.backend_id, "input": dict(input_data)}, + timeout_ms=snapshot.timeout_ms, + ) + + result = build_verification_result( + verification_result_id=f"vr-{snapshot.backend_id}-{sha256_digest(dict(input_data))[7:15]}", + profile=sealed_profile, + decision=decision, + execution_status=execution_status, + claim_ids=claim_ids, + raw_backend_output_digest=invocation["raw_backend_result_digest"], + normalized_result_digest=invocation["normalized_result_digest"], + check_groups=check_groups, + resource_limits={"wall_time_ms": 0}, + guarantee_class=guarantee_class, + declared_input_guarantee_class=declared_guarantee, + invocation_ref={ + "invocation_id": invocation["invocation_id"], + "invocation_digest": invocation["integrity"]["artifact_digest"], + }, + input_bundle_digest=invocation["input_digest"], + created_at=completed, + ) + + evidence_path: Path | None = None + if evidence_dir is not None: + evidence_path = write_evidence_pack( + evidence_dir, + invocation=invocation, + profile=sealed_profile, + result=result, + compiled_obligation={"backend_id": snapshot.backend_id, "input": dict(input_data)}, + raw_result=raw_result, + normalized_result=normalized_result, + stdout=stdout, + stderr=stderr, + ) + + return AssuranceRunOutcome( + decision=decision, + execution_status=execution_status, + indeterminate_reason=indeterminate_reason, + profile=sealed_profile, + result=result, + invocation=invocation, + snapshot=snapshot, + evidence_dir=evidence_path, + raw_result=raw_result, + normalized_result=normalized_result, + ) + + +def load_json_mapping(path: Path | str) -> dict[str, Any]: + data = json.loads(Path(path).read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise AssuranceError(f"expected JSON object in {path}") + return data diff --git a/ovk/assurance/snapshot.py b/ovk/assurance/snapshot.py new file mode 100644 index 0000000..ecbe535 --- /dev/null +++ b/ovk/assurance/snapshot.py @@ -0,0 +1,246 @@ +"""Immutable configuration snapshots for verifier-assurance.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Mapping + +from pydantic import BaseModel, ConfigDict, Field + +from ovk.assurance.pcs_hash import sha256_digest +from ovk.assurance.redaction import redact_environment, redact_mapping + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +class ConfigurationSnapshot(BaseModel): + """Immutable configuration snapshot bound by content digests.""" + + model_config = ConfigDict(frozen=True) + + snapshot_id: str + backend_id: str + adapter_id: str + adapter_version: str + created_at: str + config: dict[str, Any] = Field(default_factory=dict) + config_digest: str + policy_digest: str | None = None + model_digest: str | None = None + prompt_digest: str | None = None + resource_limit_digest: str | None = None + rubric_digest: str | None = None + test_suite_digest: str | None = None + threshold_digest: str | None = None + ensemble_digest: str | None = None + redacted_environment: dict[str, Any] = Field(default_factory=dict) + content_digest: str + mechanism_class: str | None = None + determinism: str = "deterministic" + allows_abstention: bool = True + guarantee_class: str = "observational" + decision_space: list[str] = Field(default_factory=list) + supported_claim_ids: list[str] = Field(default_factory=list) + out_of_scope_claim_ids: list[str] = Field(default_factory=list) + assumptions: list[str] = Field(default_factory=list) + known_blind_spots: list[str] = Field(default_factory=list) + external_dependencies: list[dict[str, Any]] = Field(default_factory=list) + entry_point: str | None = None + implementation_name: str | None = None + timeout_ms: int | None = None + mutation_dimensions: list[str] = Field(default_factory=list) + extra: dict[str, Any] = Field(default_factory=dict) + + +def _optional_digest(value: Any | None) -> str | None: + if value is None: + return None + if isinstance(value, str) and value.startswith("sha256:"): + return value + return sha256_digest(value) + + +def build_configuration_snapshot( + *, + backend_id: str, + adapter_id: str, + adapter_version: str, + config: Mapping[str, Any] | None = None, + environment: Mapping[str, str] | None = None, + policy: Any | None = None, + model: Any | None = None, + prompt: Any | None = None, + resource_limits: Any | None = None, + rubric: Any | None = None, + test_suite: Any | None = None, + threshold: Any | None = None, + ensemble: Any | None = None, + mechanism_class: str | None = None, + determinism: str = "deterministic", + allows_abstention: bool = True, + guarantee_class: str = "observational", + decision_space: list[str] | None = None, + supported_claim_ids: list[str] | None = None, + out_of_scope_claim_ids: list[str] | None = None, + assumptions: list[str] | None = None, + known_blind_spots: list[str] | None = None, + external_dependencies: list[dict[str, Any]] | None = None, + entry_point: str | None = None, + implementation_name: str | None = None, + timeout_ms: int | None = None, + mutation_dimensions: list[str] | None = None, + extra: Mapping[str, Any] | None = None, + created_at: str | None = None, +) -> ConfigurationSnapshot: + """Build an immutable snapshot; material config changes yield a new digest.""" + cleaned_config, _ = redact_mapping(dict(config or {})) + # Prefer explicit timeout from config when present. + resolved_timeout = timeout_ms + if resolved_timeout is None and isinstance(cleaned_config.get("timeout_ms"), int): + resolved_timeout = int(cleaned_config["timeout_ms"]) + + config_digest = sha256_digest(cleaned_config) + env_block = redact_environment(environment) + + policy_digest = _optional_digest(policy if policy is not None else cleaned_config.get("policy")) + model_digest = _optional_digest(model if model is not None else cleaned_config.get("model")) + prompt_digest = _optional_digest(prompt if prompt is not None else cleaned_config.get("prompt")) + resource_limit_digest = _optional_digest( + resource_limits if resource_limits is not None else cleaned_config.get("resource_limits") + ) + if resource_limit_digest is None and resolved_timeout is not None: + resource_limit_digest = sha256_digest({"timeout_ms": resolved_timeout}) + rubric_digest = _optional_digest(rubric if rubric is not None else cleaned_config.get("rubric")) + test_suite_digest = _optional_digest( + test_suite if test_suite is not None else cleaned_config.get("test_suite") + ) + threshold_digest = _optional_digest( + threshold if threshold is not None else cleaned_config.get("threshold") + ) + ensemble_digest = _optional_digest( + ensemble if ensemble is not None else cleaned_config.get("ensemble") + ) + + body = { + "backend_id": backend_id, + "adapter_id": adapter_id, + "adapter_version": adapter_version, + "config": cleaned_config, + "config_digest": config_digest, + "policy_digest": policy_digest, + "model_digest": model_digest, + "prompt_digest": prompt_digest, + "resource_limit_digest": resource_limit_digest, + "rubric_digest": rubric_digest, + "test_suite_digest": test_suite_digest, + "threshold_digest": threshold_digest, + "ensemble_digest": ensemble_digest, + "redacted_environment": env_block, + "mechanism_class": mechanism_class, + "determinism": determinism, + "timeout_ms": resolved_timeout, + "extra": dict(extra or {}), + } + content = sha256_digest(body) + snapshot_id = f"snap-{content[7:23]}" + return ConfigurationSnapshot( + snapshot_id=snapshot_id, + backend_id=backend_id, + adapter_id=adapter_id, + adapter_version=adapter_version, + created_at=created_at or _utc_now_iso(), + config=cleaned_config, + config_digest=config_digest, + policy_digest=policy_digest, + model_digest=model_digest, + prompt_digest=prompt_digest, + resource_limit_digest=resource_limit_digest, + rubric_digest=rubric_digest, + test_suite_digest=test_suite_digest, + threshold_digest=threshold_digest, + ensemble_digest=ensemble_digest, + redacted_environment=env_block, + content_digest=content, + mechanism_class=mechanism_class, + determinism=determinism, + allows_abstention=allows_abstention, + guarantee_class=guarantee_class, + decision_space=list(decision_space or []), + supported_claim_ids=list(supported_claim_ids or []), + out_of_scope_claim_ids=list(out_of_scope_claim_ids or []), + assumptions=list(assumptions or []), + known_blind_spots=list(known_blind_spots or []), + external_dependencies=list(external_dependencies or []), + entry_point=entry_point, + implementation_name=implementation_name or backend_id, + timeout_ms=resolved_timeout, + mutation_dimensions=list(mutation_dimensions or []), + extra=dict(extra or {}), + ) + + +def snapshot_from_adapter( + adapter: Any, + config: Mapping[str, Any] | None = None, + *, + environment: Mapping[str, str] | None = None, +) -> ConfigurationSnapshot: + """Build a snapshot via the adapter's ``snapshot_config`` when available.""" + if hasattr(adapter, "snapshot_config") and callable(adapter.snapshot_config): + result = adapter.snapshot_config(config, environment=environment) + if isinstance(result, ConfigurationSnapshot): + return result + if isinstance(result, dict): + return ConfigurationSnapshot.model_validate(result) + raise TypeError("snapshot_config must return ConfigurationSnapshot or dict") + + manifest = adapter.manifest() + assurance = getattr(manifest, "assurance", None) + mechanism = None + determinism = "deterministic" + allows_abstention = True + guarantee_class = "observational" + decision_space: list[str] = [] + supported_claim_ids: list[str] = [] + out_of_scope: list[str] = [] + entry_point = None + implementation_name = None + mutation_dimensions: list[str] = [] + external_deps: list[dict[str, Any]] = [] + known_limits: list[str] = [] + if assurance is not None: + mechanism = assurance.mechanism_class + determinism = assurance.determinism + allows_abstention = assurance.abstention.allows_abstention + guarantee_class = assurance.decision_semantics.guarantee_class + decision_space = list(assurance.decision_semantics.decision_space) + supported_claim_ids = list(assurance.decision_semantics.supported_claim_ids) + out_of_scope = list(assurance.decision_semantics.out_of_scope_claim_ids) + entry_point = assurance.verifier_identity.entry_point + implementation_name = assurance.verifier_identity.implementation_name + mutation_dimensions = list(assurance.mutation_dimensions) + external_deps = [dep.model_dump(mode="json", exclude_none=True) for dep in assurance.external_dependencies] + known_limits = list(assurance.known_limits) + + return build_configuration_snapshot( + backend_id=str(adapter.backend_id), + adapter_id=str(adapter.adapter_id), + adapter_version=str(adapter.adapter_version), + config=config, + environment=environment, + mechanism_class=mechanism, + determinism=determinism, + allows_abstention=allows_abstention, + guarantee_class=guarantee_class, + decision_space=decision_space, + supported_claim_ids=supported_claim_ids, + out_of_scope_claim_ids=out_of_scope, + assumptions=list(getattr(manifest, "assumptions", []) or []), + known_blind_spots=known_limits or list(getattr(manifest, "limits", []) or []), + external_dependencies=external_deps, + entry_point=entry_point, + implementation_name=implementation_name, + mutation_dimensions=mutation_dimensions, + ) diff --git a/ovk/assurance/source.py b/ovk/assurance/source.py new file mode 100644 index 0000000..eaf163e --- /dev/null +++ b/ovk/assurance/source.py @@ -0,0 +1,54 @@ +"""Source commit and producer metadata helpers for PCS exports.""" + +from __future__ import annotations + +import os +import re +import subprocess +from pathlib import Path + +from ovk.assurance.errors import AssuranceError +from ovk.core.release_metadata import OVK_VERSION +from ovk.paths import ovk_data_root + +PRODUCER_NAME = "OVK" +DEFAULT_SOURCE_REPO = "https://github.com/fraware/open-verification-kernel" + +_GIT_SHA40_RE = re.compile(r"^[0-9a-f]{40}$") + + +def resolve_source_repo() -> str: + """Return source_repo URI from env or the documented default.""" + return (os.environ.get("OVK_SOURCE_REPO") or DEFAULT_SOURCE_REPO).strip() + + +def resolve_source_commit(repo_root: Path | None = None) -> str: + """Resolve git HEAD (40 hex) of the OVK repo. Fail closed — never all-zero.""" + root = repo_root or ovk_data_root() + try: + completed = subprocess.run( + ["git", "-C", str(root), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ) + except (OSError, subprocess.CalledProcessError) as exc: + raise AssuranceError( + f"unable to resolve OVK source_commit from git HEAD under {root}: {exc}" + ) from exc + commit = completed.stdout.strip().lower() + if not _GIT_SHA40_RE.fullmatch(commit): + raise AssuranceError( + f"git HEAD must be a 40-hex commit SHA, got {commit!r} (all-zero placeholder forbidden)" + ) + if commit == "0" * 40: + raise AssuranceError("all-zero source_commit placeholder is forbidden") + return commit + + +def producer_fields() -> dict[str, str]: + """Return producer / producer_version for PCS artifacts.""" + return { + "producer": PRODUCER_NAME, + "producer_version": OVK_VERSION, + } diff --git a/ovk/cli.py b/ovk/cli.py index 8122f1b..0ee9dd2 100644 --- a/ovk/cli.py +++ b/ovk/cli.py @@ -54,6 +54,10 @@ template_app = typer.Typer(help="Verification intent template commands") app.add_typer(template_app, name="template") +from ovk.assurance.cli import verifier_app + +app.add_typer(verifier_app, name="verifier") + def _finish_lane( bundle: EvidenceBundle, diff --git a/ovk/core/backend_registry.py b/ovk/core/backend_registry.py index adc31c1..c4bdabc 100644 --- a/ovk/core/backend_registry.py +++ b/ovk/core/backend_registry.py @@ -24,6 +24,60 @@ class BackendRegistryError(ValueError): """Raised when registry registration or lookup invariants are violated.""" +def _validate_assurance_section(manifest: BackendCapabilityManifest) -> None: + """Validate optional assurance claims for internal consistency. + + Missing ``assurance`` remains valid (ordinary-only). When present and + ``assurance_capable`` is True, snapshot+replay must be supported, failure + behavior must be indeterminate, and decision_space must be non-empty. + """ + section = manifest.assurance + if section is None: + return + if not section.assurance_capable: + return + + if not section.configuration_snapshot_support.supported: + raise BackendRegistryError( + f"adapter capability {manifest.capability_id!r} claims assurance_capable " + "but configuration_snapshot_support.supported is false" + ) + if not section.replay_support.supported: + raise BackendRegistryError( + f"adapter capability {manifest.capability_id!r} claims assurance_capable " + "but replay_support.supported is false" + ) + if not section.decision_semantics.decision_space: + raise BackendRegistryError( + f"adapter capability {manifest.capability_id!r} claims assurance_capable " + "but decision_semantics.decision_space is empty" + ) + if not section.evidence_channels: + raise BackendRegistryError( + f"adapter capability {manifest.capability_id!r} claims assurance_capable " + "but evidence_channels is empty" + ) + failure = section.failure_behavior + for field_name in ( + "missing_checker", + "timeout", + "parser_failure", + "unsupported_input", + "external_service_error", + ): + if getattr(failure, field_name) != "indeterminate": + raise BackendRegistryError( + f"adapter capability {manifest.capability_id!r} assurance failure_behavior." + f"{field_name} must be 'indeterminate'" + ) + identity = section.verifier_identity + if not identity.verifier_id.strip() or not identity.implementation_name.strip(): + raise BackendRegistryError( + f"adapter capability {manifest.capability_id!r} assurance verifier_identity " + "must declare non-empty verifier_id and implementation_name" + ) + + def _validate_capability_manifest(manifest: BackendCapabilityManifest) -> None: """Validate a typed capability manifest against the JSON schema.""" schema_file = schema_path("verification.capability.schema.json") @@ -37,6 +91,7 @@ def _validate_capability_manifest(manifest: BackendCapabilityManifest) -> None: f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" for issue in report.issues ) raise BackendRegistryError(f"capability manifest failed schema validation: {issues}") + _validate_assurance_section(manifest) def _require_adapter_identity(adapter: BackendAdapter) -> tuple[str, str, str]: diff --git a/ovk/core/execution_models.py b/ovk/core/execution_models.py index ea1384b..12d021c 100644 --- a/ovk/core/execution_models.py +++ b/ovk/core/execution_models.py @@ -167,6 +167,174 @@ class BackendGuaranteeDeclaration(BaseModel): meaning_of_unknown: str +# --------------------------------------------------------------------------- +# Optional verifier-assurance capability section (VA-01+) +# --------------------------------------------------------------------------- + +AssuranceDecision = Literal[ + "accept", + "reject", + "indeterminate_insufficient_evidence", + "indeterminate_execution_error", + "indeterminate_out_of_scope", + "indeterminate_configuration_drift", +] + +AssuranceGuaranteeClass = Literal[ + "observational", + "runtime_observed", + "empirically_measured", + "human_reviewed", + "certificate_checked", + "formally_checked", + "unchecked_advisory", +] + +AssuranceMechanismClass = Literal[ + "static_analysis", + "dynamic_analysis", + "policy_engine", + "test_suite", + "model_judge", + "human_review", + "formal_proof", + "replay", + "hybrid", + "other", +] + +AssuranceDeterminism = Literal["deterministic", "stochastic"] + +AssuranceEvidenceChannel = Literal[ + "stdout", + "stderr", + "raw_backend_result", + "normalized_result", + "proof", + "counterexample", + "test_report", + "state_diff", + "model_judgment", + "compiled_obligation", +] + +AssuranceMutationDimension = Literal[ + "remove_success_predicate", + "remove_process_predicate", + "remove_authority_predicate", + "change_threshold", + "reduce_test_subset", + "alter_timeout", + "suppress_error", + "change_rubric", + "change_prompt", + "ensemble_quorum", + "hidden_state_access", + "policy_bundle", + "abstention", + "other", +] + +AssuranceDependencyKind = Literal[ + "binary", + "container", + "library", + "service", + "dataset", + "model", + "toolchain", + "other", +] + +AssuranceFailureOutcome = Literal["indeterminate"] + + +class AssuranceVerifierIdentity(BaseModel): + """Identity block inside an optional assurance capability section.""" + + verifier_id: str + implementation_name: str + entry_point: str | None = None + pcs_profile_artifact_type: Literal["VerifierProfile.v1"] | None = None + + +class AssuranceDecisionSemantics(BaseModel): + """Decision vocabulary and guarantee class for assurance-capable adapters.""" + + decision_space: list[AssuranceDecision] + guarantee_class: AssuranceGuaranteeClass + supported_claim_ids: list[str] = Field(default_factory=list) + out_of_scope_claim_ids: list[str] = Field(default_factory=list) + + +class AssuranceReplaySupport(BaseModel): + """Whether invocation replay is supported for this adapter.""" + + supported: bool + compares_raw_digest: bool | None = None + compares_normalized_digest: bool | None = None + notes: str | None = None + + +class AssuranceSnapshotSupport(BaseModel): + """Whether configuration snapshots / PCS profiles are supported.""" + + supported: bool + exports_pcs_profile: bool | None = None + notes: str | None = None + + +class AssuranceAbstention(BaseModel): + """Abstention / indeterminate allowance declaration.""" + + allows_abstention: bool + notes: str | None = None + + +class AssuranceFailureBehavior(BaseModel): + """Fail-closed mapping from checker failures to indeterminate outcomes.""" + + missing_checker: AssuranceFailureOutcome = "indeterminate" + timeout: AssuranceFailureOutcome = "indeterminate" + parser_failure: AssuranceFailureOutcome = "indeterminate" + unsupported_input: AssuranceFailureOutcome = "indeterminate" + external_service_error: AssuranceFailureOutcome = "indeterminate" + + +class AssuranceExternalDependency(BaseModel): + """External dependency declared on an assurance capability section.""" + + dependency_id: str + kind: AssuranceDependencyKind + identity: str + optional: bool | None = None + + +class AssuranceCapabilitySection(BaseModel): + """Optional verifier-assurance capability declaration. + + Absent from a manifest means ordinary-only. When ``assurance_capable`` is + True, snapshot and replay support must be advertised and failure modes must + remain indeterminate (never silent accept). + """ + + assurance_capable: bool + verifier_identity: AssuranceVerifierIdentity + decision_semantics: AssuranceDecisionSemantics + mechanism_class: AssuranceMechanismClass + determinism: AssuranceDeterminism + evidence_channels: list[AssuranceEvidenceChannel] + replay_support: AssuranceReplaySupport + configuration_snapshot_support: AssuranceSnapshotSupport + mutation_dimensions: list[AssuranceMutationDimension] = Field(default_factory=list) + abstention: AssuranceAbstention + failure_behavior: AssuranceFailureBehavior + external_dependencies: list[AssuranceExternalDependency] = Field(default_factory=list) + known_limits: list[str] = Field(default_factory=list) + requires_authoritative_state: bool | None = None + requires_complete_trajectory: bool | None = None + + class BackendCapabilityManifest(BaseModel): """Typed capability declaration for a registered backend.""" @@ -182,6 +350,7 @@ class BackendCapabilityManifest(BaseModel): result_format: str | None = None counterexample_format: str | None = None timeout_behavior: TimeoutBehavior = "unknown" + assurance: AssuranceCapabilitySection | None = None class BackendEnvironmentFingerprint(BaseModel): diff --git a/ovk/core/release_metadata.py b/ovk/core/release_metadata.py index 99746fe..30d9dce 100644 --- a/ovk/core/release_metadata.py +++ b/ovk/core/release_metadata.py @@ -34,6 +34,12 @@ "ovk template apply", "ovk bench", "ovk pilot", + "ovk verifier describe", + "ovk verifier snapshot-config", + "ovk verifier run", + "ovk verifier validate-evidence", + "ovk verifier replay", + "ovk verifier mutate", ] diff --git a/pyproject.toml b/pyproject.toml index 2389899..9570f18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,11 +42,18 @@ solvers = [ mcp = [ "mcp>=1.0.0" ] +# Verifier-assurance depends on pcs-core schemas/validator. +# Pin identity (commit): docs/PCS_PIN.md — fb588a41a7eab68064429e3c4dfb26c328b9863d +# Sibling editable remains the local fallback: pip install -e "../pcs-core/python" +# After pcs-core publishes a versioned VA release, replace with: pcs-core== +assurance = [ + "pcs-core @ git+https://github.com/SentinelOps-CI/pcs-core@fb588a41a7eab68064429e3c4dfb26c328b9863d#subdirectory=python", +] backends-wave1 = [] backends-wave2 = [] all = [ "z3-solver>=4.13.0", - "mcp>=1.0.0" + "mcp>=1.0.0", ] [project.scripts] diff --git a/schemas/verification.capability.schema.json b/schemas/verification.capability.schema.json index f9831d8..89d3191 100644 --- a/schemas/verification.capability.schema.json +++ b/schemas/verification.capability.schema.json @@ -39,7 +39,254 @@ "limits": { "type": "array", "items": { "type": "string" } }, "result_format": { "type": "string" }, "counterexample_format": { "type": "string" }, - "timeout_behavior": { "type": "string", "enum": ["unknown", "error", "fail"] } + "timeout_behavior": { "type": "string", "enum": ["unknown", "error", "fail"] }, + "assurance": { + "type": "object", + "description": "Optional verifier-assurance capability declaration. Absent => ordinary-only.", + "required": [ + "assurance_capable", + "verifier_identity", + "decision_semantics", + "mechanism_class", + "determinism", + "evidence_channels", + "replay_support", + "configuration_snapshot_support", + "mutation_dimensions", + "abstention", + "failure_behavior", + "external_dependencies", + "known_limits" + ], + "additionalProperties": false, + "properties": { + "assurance_capable": { "type": "boolean" }, + "verifier_identity": { + "type": "object", + "required": ["verifier_id", "implementation_name"], + "additionalProperties": false, + "properties": { + "verifier_id": { "type": "string", "minLength": 1 }, + "implementation_name": { "type": "string", "minLength": 1 }, + "entry_point": { "type": "string", "minLength": 1 }, + "pcs_profile_artifact_type": { + "type": "string", + "const": "VerifierProfile.v1" + } + } + }, + "decision_semantics": { + "type": "object", + "required": ["decision_space", "guarantee_class"], + "additionalProperties": false, + "properties": { + "decision_space": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "accept", + "reject", + "indeterminate_insufficient_evidence", + "indeterminate_execution_error", + "indeterminate_out_of_scope", + "indeterminate_configuration_drift" + ] + } + }, + "guarantee_class": { + "type": "string", + "enum": [ + "observational", + "runtime_observed", + "empirically_measured", + "human_reviewed", + "certificate_checked", + "formally_checked", + "unchecked_advisory" + ] + }, + "supported_claim_ids": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "out_of_scope_claim_ids": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + } + } + }, + "mechanism_class": { + "type": "string", + "enum": [ + "static_analysis", + "dynamic_analysis", + "policy_engine", + "test_suite", + "model_judge", + "human_review", + "formal_proof", + "replay", + "hybrid", + "other" + ] + }, + "requires_authoritative_state": { "type": "boolean" }, + "requires_complete_trajectory": { "type": "boolean" }, + "determinism": { + "type": "string", + "enum": ["deterministic", "stochastic"] + }, + "evidence_channels": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "stdout", + "stderr", + "raw_backend_result", + "normalized_result", + "proof", + "counterexample", + "test_report", + "state_diff", + "model_judgment", + "compiled_obligation" + ] + } + }, + "replay_support": { + "type": "object", + "required": ["supported"], + "additionalProperties": false, + "properties": { + "supported": { "type": "boolean" }, + "compares_raw_digest": { "type": "boolean" }, + "compares_normalized_digest": { "type": "boolean" }, + "notes": { "type": "string", "minLength": 1 } + } + }, + "configuration_snapshot_support": { + "type": "object", + "required": ["supported"], + "additionalProperties": false, + "properties": { + "supported": { "type": "boolean" }, + "exports_pcs_profile": { "type": "boolean" }, + "notes": { "type": "string", "minLength": 1 } + } + }, + "mutation_dimensions": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "remove_success_predicate", + "remove_process_predicate", + "remove_authority_predicate", + "change_threshold", + "reduce_test_subset", + "alter_timeout", + "suppress_error", + "change_rubric", + "change_prompt", + "ensemble_quorum", + "hidden_state_access", + "policy_bundle", + "abstention", + "other" + ] + } + }, + "abstention": { + "type": "object", + "required": ["allows_abstention"], + "additionalProperties": false, + "properties": { + "allows_abstention": { "type": "boolean" }, + "notes": { "type": "string", "minLength": 1 } + } + }, + "failure_behavior": { + "type": "object", + "required": [ + "missing_checker", + "timeout", + "parser_failure", + "unsupported_input", + "external_service_error" + ], + "additionalProperties": false, + "properties": { + "missing_checker": { "type": "string", "const": "indeterminate" }, + "timeout": { "type": "string", "const": "indeterminate" }, + "parser_failure": { "type": "string", "const": "indeterminate" }, + "unsupported_input": { "type": "string", "const": "indeterminate" }, + "external_service_error": { "type": "string", "const": "indeterminate" } + } + }, + "external_dependencies": { + "type": "array", + "items": { + "type": "object", + "required": ["dependency_id", "kind", "identity"], + "additionalProperties": false, + "properties": { + "dependency_id": { "type": "string", "minLength": 1 }, + "kind": { + "type": "string", + "enum": [ + "binary", + "container", + "library", + "service", + "dataset", + "model", + "toolchain", + "other" + ] + }, + "identity": { "type": "string", "minLength": 1 }, + "optional": { "type": "boolean" } + } + } + }, + "known_limits": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + } + }, + "allOf": [ + { + "if": { + "properties": { "assurance_capable": { "const": true } }, + "required": ["assurance_capable"] + }, + "then": { + "properties": { + "configuration_snapshot_support": { + "properties": { "supported": { "const": true } }, + "required": ["supported"] + }, + "replay_support": { + "properties": { "supported": { "const": true } }, + "required": ["supported"] + }, + "failure_behavior": { "required": [ + "missing_checker", + "timeout", + "parser_failure", + "unsupported_input", + "external_service_error" + ] } + } + } + } + ] + } }, "additionalProperties": true } diff --git a/tests/assurance/__init__.py b/tests/assurance/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/assurance/fixtures/invalid_assurance_capable_without_snapshot.json b/tests/assurance/fixtures/invalid_assurance_capable_without_snapshot.json new file mode 100644 index 0000000..9764c38 --- /dev/null +++ b/tests/assurance/fixtures/invalid_assurance_capable_without_snapshot.json @@ -0,0 +1,51 @@ +{ + "capability_id": "assurance-fixture-invalid-v1", + "tool": { + "name": "fixture-backend", + "adapter": "fixture-adapter", + "adapter_version": "0.1.0", + "version": "0.1.0" + }, + "backend_class": "custom", + "supported_domains": ["assurance_test"], + "supported_property_kinds": ["exact_predicate"], + "guarantee": { + "type": "exact_predicate", + "meaning_of_pass": "pass", + "meaning_of_fail": "fail", + "meaning_of_unknown": "unknown" + }, + "assurance": { + "assurance_capable": true, + "verifier_identity": { + "verifier_id": "ovk.fixture.invalid", + "implementation_name": "FixtureAdapter" + }, + "decision_semantics": { + "decision_space": ["accept", "reject"], + "guarantee_class": "observational" + }, + "mechanism_class": "static_analysis", + "determinism": "deterministic", + "evidence_channels": ["raw_backend_result"], + "replay_support": { + "supported": true + }, + "configuration_snapshot_support": { + "supported": false + }, + "mutation_dimensions": [], + "abstention": { + "allows_abstention": true + }, + "failure_behavior": { + "missing_checker": "indeterminate", + "timeout": "indeterminate", + "parser_failure": "indeterminate", + "unsupported_input": "indeterminate", + "external_service_error": "indeterminate" + }, + "external_dependencies": [], + "known_limits": [] + } +} diff --git a/tests/assurance/fixtures/opa/allow_read.rego b/tests/assurance/fixtures/opa/allow_read.rego new file mode 100644 index 0000000..db3d539 --- /dev/null +++ b/tests/assurance/fixtures/opa/allow_read.rego @@ -0,0 +1,12 @@ +package ovk.assurance + +default allow = false + +allow { + input.action == "read" +} + +violation[msg] { + not allow + msg := sprintf("action %v not allowed", [input.action]) +} diff --git a/tests/assurance/fixtures/pytest_suite/test_offline_suite.py b/tests/assurance/fixtures/pytest_suite/test_offline_suite.py new file mode 100644 index 0000000..09d66bd --- /dev/null +++ b/tests/assurance/fixtures/pytest_suite/test_offline_suite.py @@ -0,0 +1,9 @@ +"""Offline pytest suite fixture for VA-07 assurance adapter.""" + + +def test_fixture_passes() -> None: + assert 1 + 1 == 2 + + +def test_fixture_also_passes() -> None: + assert "ovk".upper() == "OVK" diff --git a/tests/assurance/fixtures/sql/after.sqlite b/tests/assurance/fixtures/sql/after.sqlite new file mode 100644 index 0000000..d8b2acd Binary files /dev/null and b/tests/assurance/fixtures/sql/after.sqlite differ diff --git a/tests/assurance/fixtures/sql/before.sqlite b/tests/assurance/fixtures/sql/before.sqlite new file mode 100644 index 0000000..45273d4 Binary files /dev/null and b/tests/assurance/fixtures/sql/before.sqlite differ diff --git a/tests/assurance/fixtures/valid_assurance_capability.json b/tests/assurance/fixtures/valid_assurance_capability.json new file mode 100644 index 0000000..0af7720 --- /dev/null +++ b/tests/assurance/fixtures/valid_assurance_capability.json @@ -0,0 +1,62 @@ +{ + "capability_id": "assurance-fixture-valid-v1", + "tool": { + "name": "fixture-backend", + "adapter": "fixture-adapter", + "adapter_version": "0.1.0", + "version": "0.1.0" + }, + "backend_class": "custom", + "supported_domains": ["assurance_test"], + "supported_property_kinds": ["exact_predicate"], + "guarantee": { + "type": "exact_predicate", + "meaning_of_pass": "pass", + "meaning_of_fail": "fail", + "meaning_of_unknown": "unknown" + }, + "assurance": { + "assurance_capable": true, + "verifier_identity": { + "verifier_id": "ovk.fixture.valid", + "implementation_name": "FixtureAdapter", + "entry_point": "fixture.entry", + "pcs_profile_artifact_type": "VerifierProfile.v1" + }, + "decision_semantics": { + "decision_space": [ + "accept", + "reject", + "indeterminate_execution_error" + ], + "guarantee_class": "observational", + "supported_claim_ids": ["claim.fixture"], + "out_of_scope_claim_ids": [] + }, + "mechanism_class": "static_analysis", + "determinism": "deterministic", + "evidence_channels": ["raw_backend_result", "normalized_result"], + "replay_support": { + "supported": true, + "compares_raw_digest": true, + "compares_normalized_digest": true + }, + "configuration_snapshot_support": { + "supported": true, + "exports_pcs_profile": true + }, + "mutation_dimensions": ["alter_timeout", "change_threshold"], + "abstention": { + "allows_abstention": true + }, + "failure_behavior": { + "missing_checker": "indeterminate", + "timeout": "indeterminate", + "parser_failure": "indeterminate", + "unsupported_input": "indeterminate", + "external_service_error": "indeterminate" + }, + "external_dependencies": [], + "known_limits": ["fixture only"] + } +} diff --git a/tests/assurance/support/__init__.py b/tests/assurance/support/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/assurance/support/digest_adapter.py b/tests/assurance/support/digest_adapter.py new file mode 100644 index 0000000..eb51a20 --- /dev/null +++ b/tests/assurance/support/digest_adapter.py @@ -0,0 +1,391 @@ +"""Real deterministic test-only assurance adapter (NOT production-registered). + +Exact predicate: an input field equals an expected digest, or a boolean +predicate over JSON. Implements snapshot_config, run_assurance, normalize +semantics, and mutations alter_timeout + change_threshold. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Mapping + +from ovk.assurance.indeterminate import ( + DECISION_ACCEPT, + DECISION_REJECT, + indeterminate_outcome, +) +from ovk.assurance.pcs_hash import sha256_digest +from ovk.assurance.snapshot import ConfigurationSnapshot, build_configuration_snapshot +from ovk.core.execution_models import ( + AbstractionCoverage, + AssuranceAbstention, + AssuranceCapabilitySection, + AssuranceDecisionSemantics, + AssuranceFailureBehavior, + AssuranceReplaySupport, + AssuranceSnapshotSupport, + AssuranceVerifierIdentity, + BackendCapabilityAssessment, + BackendCapabilityManifest, + BackendEnvironmentFingerprint, + BackendGuaranteeDeclaration, + BackendObligation, + BackendToolIdentity, + ExecutionBudget, + ExecutionContext, + HumanExplanation, + NormalizedBackendResult, + RawBackendExecution, + RoutingDecision, + VerificationObligation, + compute_backend_obligation_id, + compute_payload_digest, +) +from ovk.core.models import VerificationStatus + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +class DigestPredicateAdapter: + """Deterministic digest/boolean predicate checker for assurance tests only.""" + + backend_id = "test-digest-predicate" + adapter_id = "ovk-adapter-test-digest-predicate" + adapter_version = "0.1.0" + + def __init__( + self, + *, + expected_digest: str | None = None, + field: str = "digest", + predicate_field: str = "ok", + mode: str = "digest", + missing_checker: bool = False, + force_timeout: bool = False, + threshold: int = 1, + timeout_ms: int = 5_000, + ) -> None: + self.expected_digest = expected_digest + self.field = field + self.predicate_field = predicate_field + self.mode = mode # "digest" | "boolean" + self.missing_checker = missing_checker + self.force_timeout = force_timeout + self.threshold = threshold + self.timeout_ms = timeout_ms + + def manifest(self) -> BackendCapabilityManifest: + return BackendCapabilityManifest( + capability_id="test-digest-predicate-v1", + tool=BackendToolIdentity( + name=self.backend_id, + adapter=self.adapter_id, + adapter_version=self.adapter_version, + version=self.adapter_version, + ), + backend_class="custom", + guarantee=BackendGuaranteeDeclaration( + type="exact_predicate", + meaning_of_pass="Input satisfies the configured exact predicate.", + meaning_of_fail="Input violates the configured exact predicate.", + meaning_of_unknown="Checker missing, timed out, or input unsupported.", + ), + input_languages=["json"], + supported_domains=["assurance_test"], + supported_property_kinds=["exact_predicate"], + assumptions=["Test-only adapter; not for production assurance claims."], + limits=["Evaluates a single field predicate only."], + result_format="ovk.result.v1", + timeout_behavior="unknown", + assurance=AssuranceCapabilitySection( + assurance_capable=True, + verifier_identity=AssuranceVerifierIdentity( + verifier_id="ovk.test.digest_predicate", + implementation_name="DigestPredicateAdapter", + entry_point="tests.assurance.support.digest_adapter.DigestPredicateAdapter", + pcs_profile_artifact_type="VerifierProfile.v1", + ), + decision_semantics=AssuranceDecisionSemantics( + decision_space=[ + "accept", + "reject", + "indeterminate_execution_error", + "indeterminate_out_of_scope", + ], + guarantee_class="observational", + supported_claim_ids=["claim.test.digest_predicate"], + out_of_scope_claim_ids=["claim.formal.full_correctness"], + ), + mechanism_class="static_analysis", + determinism="deterministic", + evidence_channels=["raw_backend_result", "normalized_result", "compiled_obligation"], + replay_support=AssuranceReplaySupport( + supported=True, + compares_raw_digest=True, + compares_normalized_digest=True, + ), + configuration_snapshot_support=AssuranceSnapshotSupport( + supported=True, + exports_pcs_profile=True, + ), + mutation_dimensions=["alter_timeout", "change_threshold"], + abstention=AssuranceAbstention(allows_abstention=True), + failure_behavior=AssuranceFailureBehavior(), + external_dependencies=[], + known_limits=["test-only exact predicate"], + ), + ) + + def supported_mutation_dimensions(self) -> list[str]: + return ["alter_timeout", "change_threshold"] + + def snapshot_config( + self, + config: Mapping[str, Any] | None = None, + *, + environment: Mapping[str, str] | None = None, + ) -> ConfigurationSnapshot: + cfg = dict(config or {}) + cfg.setdefault("timeout_ms", self.timeout_ms) + cfg.setdefault("threshold", self.threshold) + cfg.setdefault("mode", self.mode) + cfg.setdefault("field", self.field) + if self.expected_digest is not None: + cfg.setdefault("expected_digest", self.expected_digest) + return build_configuration_snapshot( + backend_id=self.backend_id, + adapter_id=self.adapter_id, + adapter_version=self.adapter_version, + config=cfg, + environment=environment, + mechanism_class="static_analysis", + determinism="deterministic", + allows_abstention=True, + guarantee_class="observational", + decision_space=[ + "accept", + "reject", + "indeterminate_execution_error", + "indeterminate_out_of_scope", + ], + supported_claim_ids=["claim.test.digest_predicate"], + out_of_scope_claim_ids=["claim.formal.full_correctness"], + assumptions=["Test-only adapter"], + known_blind_spots=["single-field predicate"], + entry_point="tests.assurance.support.digest_adapter.DigestPredicateAdapter", + implementation_name="DigestPredicateAdapter", + timeout_ms=int(cfg["timeout_ms"]), + mutation_dimensions=self.supported_mutation_dimensions(), + threshold=cfg.get("threshold"), + ) + + def run_assurance( + self, + *, + input_data: Mapping[str, Any], + snapshot: ConfigurationSnapshot, + config: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + if self.missing_checker: + raise FileNotFoundError("digest-predicate-checker binary not found") + if self.force_timeout: + raise TimeoutError("digest-predicate-checker timed out") + + cfg = dict(snapshot.config) + if config: + cfg.update(dict(config)) + threshold = int(cfg.get("threshold", self.threshold)) + mode = str(cfg.get("mode", self.mode)) + field = str(cfg.get("field", self.field)) + + raw: dict[str, Any] = { + "mode": mode, + "field": field, + "threshold": threshold, + "input": dict(input_data), + } + + if mode == "boolean": + value = input_data.get(self.predicate_field) + if not isinstance(value, bool): + ind = indeterminate_outcome(reason="unsupported_input", message="boolean field missing") + normalized = { + "decision": ind["decision"], + "guarantee_class": "observational", + "status": "unknown", + } + return { + "exit_kind": "exited", + "exit_code": 2, + "status": "unknown", + "decision": ind["decision"], + "execution_status": ind["execution_status"], + "indeterminate_reason": ind["indeterminate_reason"], + "stdout": "", + "stderr": ind["message"], + "raw_result": raw, + "normalized_result": normalized, + "guarantee_class": "observational", + "command_argv": [self.adapter_id, "boolean-predicate"], + } + passed = value is True and threshold >= 1 + else: + expected = cfg.get("expected_digest", self.expected_digest) + actual = input_data.get(field) + raw["expected_digest"] = expected + raw["actual"] = actual + if not isinstance(expected, str) or not isinstance(actual, str): + ind = indeterminate_outcome(reason="unsupported_input", message="digest field missing") + normalized = { + "decision": ind["decision"], + "guarantee_class": "observational", + "status": "unknown", + } + return { + "exit_kind": "exited", + "exit_code": 2, + "status": "unknown", + "decision": ind["decision"], + "execution_status": ind["execution_status"], + "indeterminate_reason": ind["indeterminate_reason"], + "stdout": "", + "stderr": ind["message"], + "raw_result": raw, + "normalized_result": normalized, + "guarantee_class": "observational", + "command_argv": [self.adapter_id, "digest-predicate"], + } + passed = actual == expected and threshold >= 1 + + decision = DECISION_ACCEPT if passed else DECISION_REJECT + raw["passed"] = passed + normalized = { + "decision": decision, + "guarantee_class": "observational", + "status": "pass" if passed else "fail", + "predicate_digest": sha256_digest({"mode": mode, "field": field, "threshold": threshold}), + } + return { + "exit_kind": "exited", + "exit_code": 0 if passed else 1, + "status": "pass" if passed else "fail", + "decision": decision, + "execution_status": "completed", + "indeterminate_reason": None, + "stdout": "pass" if passed else "fail", + "stderr": "", + "raw_result": raw, + "normalized_result": normalized, + "guarantee_class": "observational", + "command_argv": [self.adapter_id, f"{mode}-predicate"], + } + + # --- BackendAdapter surface (required for temporary registry registration) --- + + def can_handle( + self, + obligation: VerificationObligation, + context: ExecutionContext, + ) -> BackendCapabilityAssessment: + return BackendCapabilityAssessment( + backend=self.backend_id, + support="supported", + score=1.0, + guarantee_type="exact_predicate", + material_requirements_met=True, + coverage_requirements_met=True, + native_available=True, + estimated_wall_time_seconds=0.1, + estimated_memory_mb=64, + reasons=["test digest predicate"], + ) + + def compile( + self, + obligation: VerificationObligation, + routing: RoutingDecision, + ) -> BackendObligation: + payload = {"abstraction": obligation.abstraction} + provisional = BackendObligation( + backend_obligation_id="pending", + obligation_id=obligation.obligation_id, + backend=self.backend_id, + adapter_id=self.adapter_id, + adapter_version=self.adapter_version, + input_language="json", + payload=payload, + payload_digest=compute_payload_digest(payload), + required=True, + timeout_seconds=5.0, + memory_mb=64, + ) + return provisional.model_copy( + update={"backend_obligation_id": compute_backend_obligation_id(provisional)} + ) + + def fingerprint(self, backend_obligation: BackendObligation) -> BackendEnvironmentFingerprint: + return BackendEnvironmentFingerprint( + backend=self.backend_id, + adapter_version=self.adapter_version, + environment_digest=sha256_digest({"backend": self.backend_id})[7:], + native_available=True, + ) + + def run( + self, + backend_obligation: BackendObligation, + budget: ExecutionBudget, + ) -> RawBackendExecution: + started = _utc_now_iso() + snap = self.snapshot_config({"timeout_ms": self.timeout_ms, "threshold": self.threshold}) + outcome = self.run_assurance( + input_data=dict(backend_obligation.payload.get("abstraction") or {}), + snapshot=snap, + ) + return RawBackendExecution( + backend=self.backend_id, + termination="completed", + native_execution=True, + raw_result=dict(outcome.get("raw_result") or {}), + stdout=str(outcome.get("stdout") or ""), + stderr=str(outcome.get("stderr") or ""), + exit_code=outcome.get("exit_code"), + started_at=started, + finished_at=_utc_now_iso(), + ) + + def normalize( + self, + raw: RawBackendExecution, + backend_obligation: BackendObligation, + ) -> NormalizedBackendResult: + status_raw = str((raw.raw_result or {}).get("passed")) + if status_raw == "True": + status = VerificationStatus.PASS + elif status_raw == "False": + status = VerificationStatus.FAIL + else: + status = VerificationStatus.UNKNOWN + # Never upgrade guarantee class beyond observational. + return NormalizedBackendResult( + attempt_id="test-digest", + backend=self.backend_id, + status=status, + guarantee_type="exact_predicate", + assumptions=["observational exact predicate"], + limits=["cannot upgrade to formally_checked"], + ) + + def explain(self, result: NormalizedBackendResult) -> HumanExplanation: + return HumanExplanation( + summary=f"digest predicate status={result.status}", + repair_hint="Adjust input digest or boolean predicate.", + failure_mode=None, + ) + + +# Silence unused import for AbstractionCoverage if needed by callers constructing obligations. +_ = AbstractionCoverage diff --git a/tests/assurance/support/make_sql_fixtures.py b/tests/assurance/support/make_sql_fixtures.py new file mode 100644 index 0000000..572c49f --- /dev/null +++ b/tests/assurance/support/make_sql_fixtures.py @@ -0,0 +1,44 @@ +"""Create offline SQLite fixtures for VA-10 tests.""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] / "fixtures" / "sql" + + +def main() -> None: + ROOT.mkdir(parents=True, exist_ok=True) + before = ROOT / "before.sqlite" + after = ROOT / "after.sqlite" + if before.is_file() and after.is_file(): + # Reuse existing fixtures (Windows may lock open sqlite handles). + print(f"reuse {before} and {after}") + return + + for path in (before, after): + if path.exists(): + try: + path.unlink() + except OSError: + print(f"could not unlink {path}; continuing") + return + + conn = sqlite3.connect(before) + conn.execute("CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT)") + conn.execute("INSERT INTO items(id, name) VALUES (1, 'a')") + conn.commit() + conn.close() + + conn = sqlite3.connect(after) + conn.execute("CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT)") + conn.execute("INSERT INTO items(id, name) VALUES (1, 'a')") + conn.execute("INSERT INTO items(id, name) VALUES (2, 'b')") + conn.commit() + conn.close() + print(f"wrote {before} and {after}") + + +if __name__ == "__main__": + main() diff --git a/tests/assurance/test_assurance_adapters.py b/tests/assurance/test_assurance_adapters.py new file mode 100644 index 0000000..53362b0 --- /dev/null +++ b/tests/assurance/test_assurance_adapters.py @@ -0,0 +1,209 @@ +"""Adapter-level tests for VA-06..VA-11 assurance backends.""" + +from __future__ import annotations + +from pathlib import Path +from shutil import which + +import pytest + +from ovk.adapters.assurance.auth_state import AuthoritativeStateAdapter +from ovk.adapters.assurance.lean_pfcore import LeanPfCoreAssuranceAdapter +from ovk.adapters.assurance.model_judge import ModelJudgeAdapter +from ovk.adapters.assurance.opa_policy import OpaPolicyAssuranceAdapter +from ovk.adapters.assurance.pytest_suite import PytestSuiteAdapter +from ovk.adapters.assurance.sql_diff import SqlStateDiffAdapter +from ovk.assurance.capability import is_assurance_capable +from ovk.assurance.indeterminate import DECISION_ACCEPT +from ovk.assurance.pin import resolve_pcs_root +from ovk.assurance.registry import build_verifier_registry, describe_backend +from ovk.assurance.runner import run_assurance +from ovk.adapters.lean.adapter import LeanAdapter + +FIXTURES = Path(__file__).resolve().parent / "fixtures" +PCS_AVAILABLE = resolve_pcs_root() is not None +requires_pcs = pytest.mark.skipif(not PCS_AVAILABLE, reason="PCS pin unavailable") +requires_opa = pytest.mark.skipif(which("opa") is None, reason="opa binary not installed") +requires_lean = pytest.mark.skipif(which("lean") is None, reason="lean binary not installed") + + +def test_assurance_backends_registered_and_capable() -> None: + registry = build_verifier_registry() + expected = { + "auth-state-predicate", + "pytest-suite", + "opa-policy", + "lean-pfcore", + "sql-state-diff", + "model-judge", + } + for backend_id in expected: + adapter = registry.get(backend_id) + assert adapter is not None, backend_id + assert is_assurance_capable(adapter.manifest()) + desc = describe_backend(backend_id, registry=registry) + assert desc["assurance_capable"] is True + + +def test_ordinary_lean_is_not_assurance_capable() -> None: + registry = build_verifier_registry() + assert registry.get("lean") is None + lean = LeanAdapter() + manifest = lean.capability_manifest + assert isinstance(manifest, dict) + assert not (manifest.get("assurance") or {}).get("assurance_capable") + + +@requires_pcs +def test_auth_state_accept_and_reject() -> None: + adapter = AuthoritativeStateAdapter() + state = {"status": "approved", "roles": ["reader", "admin"]} + outcome = run_assurance( + adapter, + input_data={ + "authoritative_state": state, + "predicates": [ + {"kind": "field_equals", "path": "status", "expected": "approved"}, + {"kind": "set_contains", "path": "roles", "member": "admin"}, + ], + }, + config={"timeout_ms": 1000}, + ) + assert outcome.decision == "accept" + + bad = run_assurance( + adapter, + input_data={ + "authoritative_state": state, + "predicates": [{"kind": "field_equals", "path": "status", "expected": "denied"}], + }, + config={"timeout_ms": 1000}, + ) + assert bad.decision == "reject" + assert bad.normalized_result.get("counterexamples") + + +@requires_pcs +def test_auth_state_missing_state_indeterminate() -> None: + adapter = AuthoritativeStateAdapter() + outcome = run_assurance(adapter, input_data={"predicates": [{"kind": "field_equals", "path": "x", "expected": 1}]}) + assert outcome.decision != DECISION_ACCEPT + assert outcome.indeterminate_reason == "missing_authoritative_state" + + +@requires_pcs +def test_pytest_suite_offline() -> None: + adapter = PytestSuiteAdapter() + suite = FIXTURES / "pytest_suite" + outcome = run_assurance( + adapter, + input_data={"suite_path": str(suite)}, + config={"timeout_ms": 60_000, "threshold": 0}, + ) + assert outcome.decision == "accept" + assert outcome.snapshot.guarantee_class == "runtime_observed" + assert outcome.normalized_result.get("guarantee_class") == "runtime_observed" + + +@requires_pcs +def test_opa_missing_is_indeterminate() -> None: + if which("opa") is not None: + pytest.skip("opa present; missing-checker path covered when absent") + adapter = OpaPolicyAssuranceAdapter() + policy = (FIXTURES / "opa" / "allow_read.rego").read_text(encoding="utf-8") + outcome = run_assurance( + adapter, + input_data={"policy": policy, "query": "data.ovk.assurance.violation", "input": {"action": "write"}}, + config={"timeout_ms": 5000}, + ) + assert outcome.decision != DECISION_ACCEPT + assert outcome.indeterminate_reason == "missing_checker" + + +@requires_pcs +@requires_opa +def test_opa_real_eval() -> None: + adapter = OpaPolicyAssuranceAdapter() + policy = (FIXTURES / "opa" / "allow_read.rego").read_text(encoding="utf-8") + accept = run_assurance( + adapter, + input_data={"policy": policy, "query": "data.ovk.assurance.violation", "input": {"action": "read"}}, + config={"timeout_ms": 10_000, "policy": policy, "query": "data.ovk.assurance.violation"}, + ) + # violation query empty => pass in run_opa_policy semantics + assert accept.decision in {"accept", "reject"} # depends on violation list emptiness + reject = run_assurance( + adapter, + input_data={"policy": policy, "query": "data.ovk.assurance.violation", "input": {"action": "write"}}, + config={"timeout_ms": 10_000, "policy": policy, "query": "data.ovk.assurance.violation"}, + ) + assert reject.decision == "reject" + + +@requires_pcs +@requires_lean +def test_lean_pfcore_real_source() -> None: + adapter = LeanPfCoreAssuranceAdapter() + outcome = run_assurance( + adapter, + input_data={"lean_source": "#check Nat\n#eval (2 : Nat)\n"}, + config={"timeout_ms": 60_000}, + ) + assert outcome.decision == "accept" + + +@requires_pcs +def test_lean_missing_indeterminate(monkeypatch: pytest.MonkeyPatch) -> None: + adapter = LeanPfCoreAssuranceAdapter() + monkeypatch.setattr("ovk.adapters.assurance.lean_pfcore.lean_available", lambda: False) + monkeypatch.setattr("ovk.adapters.assurance.lean_pfcore.which", lambda _name: None) + outcome = run_assurance( + adapter, + input_data={"lean_source": "#check Nat\n"}, + config={"timeout_ms": 1000}, + ) + assert outcome.decision != DECISION_ACCEPT + assert outcome.indeterminate_reason == "missing_checker" + + +@requires_pcs +def test_sql_state_diff() -> None: + # Ensure fixtures exist + from tests.assurance.support.make_sql_fixtures import main as make_sql + + make_sql() + adapter = SqlStateDiffAdapter() + before = FIXTURES / "sql" / "before.sqlite" + after = FIXTURES / "sql" / "after.sqlite" + outcome = run_assurance( + adapter, + input_data={ + "before_db": str(before), + "after_db": str(after), + "expect_changed_tables": ["items"], + }, + config={"timeout_ms": 5000}, + ) + assert outcome.decision == "accept" + assert "state_diff" in outcome.normalized_result + + +@requires_pcs +def test_model_judge_contract_fake_no_upgrade() -> None: + adapter = ModelJudgeAdapter() + outcome = run_assurance( + adapter, + input_data={ + "prompt": "Rate this change as safe.", + "judge_client": "contract_fake", + "rubric": {"threshold": "0.0"}, + }, + config={"timeout_ms": 1000, "judge_client": "contract_fake", "rubric": {"threshold": "0.0"}}, + ) + assert outcome.decision == "accept" + assert outcome.normalized_result.get("guarantee_class") == "empirically_measured" + # Cannot upgrade + assert outcome.result.get("claim_surface", {}).get("guarantee_class") in { + None, + "empirically_measured", + } or outcome.snapshot.guarantee_class == "empirically_measured" diff --git a/tests/assurance/test_capability_manifests.py b/tests/assurance/test_capability_manifests.py new file mode 100644 index 0000000..d6b982a --- /dev/null +++ b/tests/assurance/test_capability_manifests.py @@ -0,0 +1,70 @@ +"""Capability manifest validation for ordinary + assurance sections.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from ovk.adapters.lane import build_default_lane_registry +from ovk.assurance.capability import is_assurance_capable, validate_assurance_claim +from ovk.core.backend_registry import BackendRegistry, BackendRegistryError +from ovk.core.execution_models import BackendCapabilityManifest +from ovk.core.schema_validation import load_json, validate_against_schema +from ovk.paths import schema_path +from tests.assurance.support.digest_adapter import DigestPredicateAdapter + +FIXTURES = Path(__file__).parent / "fixtures" + + +def test_existing_lane_adapter_manifests_still_validate() -> None: + schema = load_json(schema_path("verification.capability.schema.json")) + registry = build_default_lane_registry() + for adapter in registry.all(): + payload = adapter.manifest().model_dump(mode="json", exclude_none=True) + report = validate_against_schema(payload, schema) + assert report.valid, f"{adapter.backend_id}: {report.issues}" + assert adapter.manifest().assurance is None + assert not is_assurance_capable(adapter.manifest()) + + +def test_valid_assurance_fixture_validates() -> None: + schema = load_json(schema_path("verification.capability.schema.json")) + payload = json.loads((FIXTURES / "valid_assurance_capability.json").read_text(encoding="utf-8")) + report = validate_against_schema(payload, schema) + assert report.valid, report.issues + model = BackendCapabilityManifest.model_validate(payload) + assert is_assurance_capable(model) + validate_assurance_claim(model) + + +def test_invalid_assurance_capable_without_snapshot_fails_schema_and_registry() -> None: + schema = load_json(schema_path("verification.capability.schema.json")) + payload = json.loads( + (FIXTURES / "invalid_assurance_capable_without_snapshot.json").read_text(encoding="utf-8") + ) + report = validate_against_schema(payload, schema) + assert not report.valid + + model = BackendCapabilityManifest.model_validate(payload) + with pytest.raises((ValueError, BackendRegistryError)): + validate_assurance_claim(model) + + adapter = DigestPredicateAdapter() + # Force a bad manifest via a wrapper. + class Broken(DigestPredicateAdapter): + def manifest(self) -> BackendCapabilityManifest: + return model + + registry = BackendRegistry() + with pytest.raises(BackendRegistryError): + registry.register(Broken()) + + +def test_digest_adapter_registers_with_assurance() -> None: + registry = BackendRegistry() + adapter = DigestPredicateAdapter(expected_digest="sha256:" + ("a" * 64)) + registry.register(adapter) + assert is_assurance_capable(adapter.manifest()) + assert registry.get(adapter.backend_id) is adapter diff --git a/tests/assurance/test_cedar_excluded.py b/tests/assurance/test_cedar_excluded.py new file mode 100644 index 0000000..abd891c --- /dev/null +++ b/tests/assurance/test_cedar_excluded.py @@ -0,0 +1,28 @@ +"""Cedar and ordinary lane adapters must remain non-assurance.""" + +from __future__ import annotations + +import json + +from ovk.adapters.assurance import build_assurance_registry +from ovk.adapters.lane import build_default_lane_registry +from ovk.assurance.capability import is_assurance_capable +from ovk.core.execution_models import BackendCapabilityManifest +from ovk.paths import resource_path + + +def test_cedar_is_not_assurance_capable() -> None: + payload = json.loads(resource_path("adapters", "cedar", "capability.json").read_text(encoding="utf-8")) + cedar = BackendCapabilityManifest.model_validate(payload) + assert not is_assurance_capable(cedar) + assert cedar.assurance is None + + +def test_ordinary_lane_registry_excludes_assurance_backends() -> None: + lane_ids = {adapter.backend_id for adapter in build_default_lane_registry().all()} + assurance_ids = {adapter.backend_id for adapter in build_assurance_registry().all()} + assert lane_ids.isdisjoint(assurance_ids) + assert "cedar" not in assurance_ids + assert "auth-state-predicate" not in lane_ids + assert "opa-policy" not in lane_ids + assert "pytest-suite" not in lane_ids diff --git a/tests/assurance/test_conformance_and_adjudication.py b/tests/assurance/test_conformance_and_adjudication.py new file mode 100644 index 0000000..3781f37 --- /dev/null +++ b/tests/assurance/test_conformance_and_adjudication.py @@ -0,0 +1,208 @@ +"""VA-12 conformance harness + VA-13 adjudication importer tests.""" + +from __future__ import annotations + +import json +from pathlib import Path +from shutil import which + +import pytest + +from ovk.adapters.assurance.auth_state import AuthoritativeStateAdapter +from ovk.adapters.assurance.model_judge import ModelJudgeAdapter +from ovk.adapters.assurance.pytest_suite import PytestSuiteAdapter +from ovk.adapters.assurance.sql_diff import SqlStateDiffAdapter +from ovk.assurance.adjudication import ( + AdjudicationImportError, + import_adjudication_reference, + refuse_labels_in_verifier_input, +) +from ovk.assurance.conformance import ConformanceCase, gate_assurance_adapters, write_example_pack +from ovk.assurance.pin import resolve_pcs_root +from ovk.assurance.runner import run_assurance +from tests.assurance.support.make_sql_fixtures import main as make_sql + +FIXTURES = Path(__file__).resolve().parent / "fixtures" +PCS_AVAILABLE = resolve_pcs_root() is not None +requires_pcs = pytest.mark.skipif(not PCS_AVAILABLE, reason="PCS pin unavailable") + + +@requires_pcs +def test_conformance_gate_core_offline_adapters(tmp_path: Path) -> None: + make_sql() + suite = FIXTURES / "pytest_suite" + before = FIXTURES / "sql" / "before.sqlite" + after = FIXTURES / "sql" / "after.sqlite" + + cases = [ + ConformanceCase( + adapter=AuthoritativeStateAdapter(), + input_data={ + "authoritative_state": {"status": "ok", "roles": ["a"]}, + "predicates": [ + {"kind": "field_equals", "path": "status", "expected": "ok"}, + {"kind": "set_contains", "path": "roles", "member": "a"}, + ], + }, + config={"timeout_ms": 1000}, + alt_config={"timeout_ms": 2000}, + unsupported_input={}, + expect_accept=True, + ), + ConformanceCase( + adapter=PytestSuiteAdapter(), + input_data={"suite_path": str(suite)}, + config={"timeout_ms": 60_000, "threshold": 0, "suite_path": str(suite)}, + alt_config={"timeout_ms": 60_001, "threshold": 0, "suite_path": str(suite)}, + unsupported_input={}, + mutation_class="reduce_test_subset", + mutation_parameters={"test_ids": ["test_fixture_passes"]}, + expect_accept=True, + ), + ConformanceCase( + adapter=SqlStateDiffAdapter(), + input_data={ + "before_db": str(before), + "after_db": str(after), + "expect_changed_tables": ["items"], + }, + config={"timeout_ms": 5000, "before_db": str(before), "after_db": str(after)}, + alt_config={"timeout_ms": 5001, "before_db": str(before), "after_db": str(after)}, + unsupported_input={}, + expect_accept=True, + ), + ConformanceCase( + adapter=ModelJudgeAdapter(), + input_data={"prompt": "hello", "rubric": {"threshold": "0.0"}, "judge_client": "contract_fake"}, + config={ + "timeout_ms": 1000, + "prompt": "hello", + "rubric": {"threshold": "0.0"}, + "judge_client": "contract_fake", + }, + alt_config={ + "timeout_ms": 1001, + "prompt": "hello", + "rubric": {"threshold": "0.0"}, + "judge_client": "contract_fake", + }, + unsupported_input={}, + mutation_class="change_prompt", + mutation_parameters={"prompt": "mutated prompt"}, + expect_accept=True, + ), + ] + + # OPA / Lean included when toolchains exist (real runs only). + if which("opa") is not None: + from ovk.adapters.assurance.opa_policy import OpaPolicyAssuranceAdapter + + policy = (FIXTURES / "opa" / "allow_read.rego").read_text(encoding="utf-8") + cases.append( + ConformanceCase( + adapter=OpaPolicyAssuranceAdapter(), + input_data={ + "policy": policy, + "query": "data.ovk.assurance.violation", + "input": {"action": "read"}, + }, + config={"timeout_ms": 10_000, "policy": policy, "query": "data.ovk.assurance.violation"}, + alt_config={"timeout_ms": 10_001, "policy": policy, "query": "data.ovk.assurance.violation"}, + unsupported_input={}, + mutation_class="policy_bundle", + mutation_parameters={"policy": policy + "\n# mutated\n"}, + expect_accept=True, + ) + ) + if which("lean") is not None: + from ovk.adapters.assurance.lean_pfcore import LeanPfCoreAssuranceAdapter + + cases.append( + ConformanceCase( + adapter=LeanPfCoreAssuranceAdapter(), + input_data={"lean_source": "#check Nat\n"}, + config={"timeout_ms": 60_000}, + alt_config={"timeout_ms": 60_001}, + unsupported_input={}, + expect_accept=True, + ) + ) + + report = gate_assurance_adapters(cases) + if not report["passed"]: + pytest.fail(json.dumps(report["failures"], indent=2)) + + # Example pack for auth-state + pack_dir = tmp_path / "examples" / "auth-state-predicate" + write_example_pack( + AuthoritativeStateAdapter(), + out_dir=pack_dir, + input_data={ + "authoritative_state": {"status": "ok"}, + "predicates": [{"kind": "field_equals", "path": "status", "expected": "ok"}], + }, + config={"timeout_ms": 1000}, + ) + assert (pack_dir / "invocation.json").is_file() + + +def test_adjudication_requires_freeze(tmp_path: Path) -> None: + marker = tmp_path / "freeze.json" + marker.write_text(json.dumps({"campaign_id": "c1", "frozen": False}), encoding="utf-8") + with pytest.raises(AdjudicationImportError): + import_adjudication_reference( + freeze_marker_path=marker, + adjudication_ref={"reference_id": "adj-1"}, + ) + + +def test_adjudication_import_and_audit(tmp_path: Path) -> None: + marker = tmp_path / "freeze.json" + marker.write_text( + json.dumps({"campaign_id": "campaign-42", "frozen": True, "frozen_at": "2026-07-24T00:00:00Z"}), + encoding="utf-8", + ) + audit = tmp_path / "audit.jsonl" + record = import_adjudication_reference( + freeze_marker_path=marker, + adjudication_ref={"reference_id": "adj-1", "artifact_digest": "sha256:" + ("ab" * 32)}, + audit_log_path=audit, + ) + assert record["campaign_id"] == "campaign-42" + assert record["label_isolation"]["hidden_labels_accessible"] is False + assert audit.is_file() + line = audit.read_text(encoding="utf-8").strip() + assert "ovk.assurance.adjudication_import.v1" in line + + +def test_adjudication_refuses_hidden_labels(tmp_path: Path) -> None: + marker = tmp_path / "freeze.json" + marker.write_text(json.dumps({"campaign_id": "c1", "frozen": True}), encoding="utf-8") + with pytest.raises(AdjudicationImportError): + import_adjudication_reference( + freeze_marker_path=marker, + adjudication_ref={"reference_id": "adj-1", "hidden_label": "SECRET"}, + ) + with pytest.raises(AdjudicationImportError): + refuse_labels_in_verifier_input({"hidden_labels": ["x"]}) + + +@requires_pcs +def test_verifier_run_refuses_embedded_labels() -> None: + with pytest.raises(AdjudicationImportError): + refuse_labels_in_verifier_input( + { + "authoritative_state": {"status": "ok"}, + "holdout_label": "should-not-leak", + } + ) + # Positive path still works without labels + outcome = run_assurance( + AuthoritativeStateAdapter(), + input_data={ + "authoritative_state": {"status": "ok"}, + "predicates": [{"kind": "field_equals", "path": "status", "expected": "ok"}], + }, + config={"timeout_ms": 1000}, + ) + assert outcome.decision == "accept" diff --git a/tests/assurance/test_invocation_evidence.py b/tests/assurance/test_invocation_evidence.py new file mode 100644 index 0000000..be8101a --- /dev/null +++ b/tests/assurance/test_invocation_evidence.py @@ -0,0 +1,70 @@ +"""Invocation + evidence pack tests.""" + +from __future__ import annotations + +import pytest + +from ovk.assurance.errors import AssuranceError +from ovk.assurance.evidence_pack import validate_evidence_dir +from ovk.assurance.guarantee import assert_no_guarantee_upgrade, clamp_guarantee_class +from ovk.assurance.indeterminate import DECISION_ACCEPT +from ovk.assurance.pin import resolve_pcs_root +from ovk.assurance.runner import run_assurance +from tests.assurance.support.digest_adapter import DigestPredicateAdapter + +PCS_AVAILABLE = resolve_pcs_root() is not None +requires_pcs = pytest.mark.skipif( + not PCS_AVAILABLE, + reason="PCS pin unavailable (set OVK_PCS_CORE_PATH or sibling ../pcs-core)", +) + + +@requires_pcs +def test_evidence_pack_layout_and_accept(tmp_path) -> None: + expected = "sha256:" + ("cd" * 32) + adapter = DigestPredicateAdapter(expected_digest=expected) + evidence_dir = tmp_path / "evidence" + outcome = run_assurance( + adapter, + input_data={"digest": expected}, + config={"timeout_ms": 1000, "threshold": 1}, + evidence_dir=evidence_dir, + ) + assert outcome.decision == "accept" + for name in ( + "invocation.json", + "verifier_profile.pcs.json", + "verification_result.pcs.json", + "compiled_obligation.json", + ): + assert (evidence_dir / name).is_file() + for name in ("raw", "normalized", "provenance"): + assert (evidence_dir / name).is_dir() + report = validate_evidence_dir(evidence_dir) + assert report["valid"] is True + + +@requires_pcs +def test_missing_checker_is_typed_indeterminate(tmp_path) -> None: + adapter = DigestPredicateAdapter( + expected_digest="sha256:" + ("11" * 32), + missing_checker=True, + ) + outcome = run_assurance( + adapter, + input_data={"digest": "sha256:" + ("11" * 32)}, + evidence_dir=tmp_path / "evidence", + ) + assert outcome.decision != DECISION_ACCEPT + assert outcome.indeterminate_reason == "missing_checker" + assert outcome.decision.startswith("indeterminate_") + assert outcome.result["decision"] != "accept" + assert outcome.result["execution_status"] != "completed" or outcome.indeterminate_reason + + +def test_normalize_cannot_upgrade_guarantee_class() -> None: + assert_no_guarantee_upgrade("observational", "observational") + with pytest.raises(AssuranceError): + assert_no_guarantee_upgrade("observational", "formally_checked") + assert clamp_guarantee_class("observational", "formally_checked") == "observational" + assert clamp_guarantee_class("formally_checked", "observational") == "observational" diff --git a/tests/assurance/test_mutation.py b/tests/assurance/test_mutation.py new file mode 100644 index 0000000..30fbbbe --- /dev/null +++ b/tests/assurance/test_mutation.py @@ -0,0 +1,68 @@ +"""Typed mutation tests.""" + +from __future__ import annotations + +import pytest + +from ovk.assurance.errors import MutationError +from ovk.assurance.mutation import mutate_profile +from ovk.assurance.pcs_export import snapshot_to_verifier_profile +from ovk.assurance.pin import resolve_pcs_root +from tests.assurance.support.digest_adapter import DigestPredicateAdapter + +PCS_AVAILABLE = resolve_pcs_root() is not None +requires_pcs = pytest.mark.skipif( + not PCS_AVAILABLE, + reason="PCS pin unavailable (set OVK_PCS_CORE_PATH or sibling ../pcs-core)", +) + + +@requires_pcs +def test_mutation_changes_digest_and_refuses_production_overwrite(tmp_path) -> None: + adapter = DigestPredicateAdapter(expected_digest="sha256:" + ("aa" * 32)) + snapshot = adapter.snapshot_config({"timeout_ms": 1000, "threshold": 1}) + profile = snapshot_to_verifier_profile(snapshot) + production = tmp_path / "production" / "profile.json" + production.parent.mkdir(parents=True) + production.write_text("{}", encoding="utf-8") + + mutated, manifest = mutate_profile( + profile, + mutation_class="alter_timeout", + parameters={"timeout_ms": 9000}, + out_path=tmp_path / "mutated.json", + production_profile_path=production, + supported_dimensions=adapter.supported_mutation_dimensions(), + ) + assert mutated["integrity"]["artifact_digest"] != profile["integrity"]["artifact_digest"] + assert manifest["production_prohibition"] is True + assert manifest["mutation_class"] == "alter_timeout" + + with pytest.raises(MutationError, match="production"): + mutate_profile( + profile, + mutation_class="alter_timeout", + parameters={"timeout_ms": 8000}, + out_path=production, + production_profile_path=production, + supported_dimensions=adapter.supported_mutation_dimensions(), + ) + + +@requires_pcs +def test_unsupported_mutation_fails() -> None: + adapter = DigestPredicateAdapter(expected_digest="sha256:" + ("bb" * 32)) + profile = snapshot_to_verifier_profile(adapter.snapshot_config({"timeout_ms": 1000})) + with pytest.raises(MutationError, match="not in adapter supported"): + mutate_profile( + profile, + mutation_class="change_prompt", + parameters={"prompt": "x"}, + supported_dimensions=adapter.supported_mutation_dimensions(), + ) + with pytest.raises(MutationError, match="unsupported mutation class"): + mutate_profile( + profile, + mutation_class="not_a_real_mutation", + parameters={}, + ) diff --git a/tests/assurance/test_pin_digests.py b/tests/assurance/test_pin_digests.py new file mode 100644 index 0000000..a62713c --- /dev/null +++ b/tests/assurance/test_pin_digests.py @@ -0,0 +1,34 @@ +"""PCS pin digest enforcement.""" + +from __future__ import annotations + +import pytest + +from ovk.assurance.errors import PinError +from ovk.assurance.pin import ( + EXPECTED_SCHEMA_DIGESTS, + PCS_PIN_COMMIT, + load_schema_digests, + resolve_pcs_root, + verify_pin_digests, +) + + +@pytest.mark.skipif(resolve_pcs_root() is None, reason="PCS pin unavailable") +def test_pin_digests_match_documented_table() -> None: + actual = verify_pin_digests() + assert actual["VerifierProfile.v1"] == EXPECTED_SCHEMA_DIGESTS["VerifierProfile.v1"] + assert actual["VerifierInvocationRecord.v1"] == EXPECTED_SCHEMA_DIGESTS[ + "VerifierInvocationRecord.v1" + ] + assert PCS_PIN_COMMIT.startswith("fb588a41") + + +@pytest.mark.skipif(resolve_pcs_root() is None, reason="PCS pin unavailable") +def test_pin_digest_drift_fails_closed() -> None: + bad = dict(EXPECTED_SCHEMA_DIGESTS) + bad["VerifierProfile.v1"] = "sha256:" + ("0" * 64) + with pytest.raises(PinError, match="digest drift"): + verify_pin_digests(expected=bad) + # Cache may retain good digests; ensure load still works. + assert load_schema_digests()["VerifierProfile.v1"].startswith("sha256:") diff --git a/tests/assurance/test_redaction.py b/tests/assurance/test_redaction.py new file mode 100644 index 0000000..ea118f5 --- /dev/null +++ b/tests/assurance/test_redaction.py @@ -0,0 +1,98 @@ +"""Adversarial secret redaction coverage for assurance exports.""" + +from __future__ import annotations + +from ovk.assurance.pcs_export import snapshot_to_verifier_profile +from ovk.assurance.redaction import ( + is_secret_key, + looks_like_secret_value, + redact_environment, + redact_mapping, +) +from ovk.assurance.snapshot import build_configuration_snapshot + + +def test_secret_keys_detected() -> None: + assert is_secret_key("GITHUB_TOKEN") + assert is_secret_key("MY_API_KEY") + assert is_secret_key("db_password") + assert is_secret_key("SERVICE_SECRET") + assert is_secret_key("TLS_PRIVATE_KEY") + assert is_secret_key("PGPASSWORD") + assert is_secret_key("DATABASE_URL") + assert is_secret_key("MYSQL_PWD") + assert is_secret_key("connection_string") + assert is_secret_key("JWT") + assert not is_secret_key("PATH") + assert not is_secret_key("CI") + assert not is_secret_key("HISTORY") + + +def test_secret_value_patterns() -> None: + assert looks_like_secret_value("ghp_abcdefghijklmnopqrstuvwxyz0123456789") + assert looks_like_secret_value("sk-abcdefghijklmnopqrstuvwxyz012345") + assert looks_like_secret_value("Authorization: Bearer abc.def.ghi") + assert looks_like_secret_value("-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----") + assert not looks_like_secret_value("ordinary note") + + +def test_redact_environment_strips_secrets_and_digests() -> None: + env = { + "PATH": "/usr/bin", + "CI": "true", + "GITHUB_TOKEN": "ghp_secret", + "OPENAI_API_KEY": "sk-secret", + "CUSTOM_PASSWORD": "hunter2", + "NOTE": "Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig", + } + redacted = redact_environment(env) + assert redacted["redaction_policy_id"] == "ovk-redact-v1" + assert "GITHUB_TOKEN" not in redacted["entries"] + assert "OPENAI_API_KEY" not in redacted["entries"] + assert "CUSTOM_PASSWORD" not in redacted["entries"] + assert "NOTE" not in redacted["entries"] + assert redacted["entries"]["PATH"] == "/usr/bin" + assert "GITHUB_TOKEN" in redacted["redacted_keys"] + assert redacted["environment_digest"].startswith("sha256:") + blob = str(redacted) + assert "ghp_secret" not in blob + assert "sk-secret" not in blob + assert "hunter2" not in blob + assert "Bearer eyJ" not in blob + + +def test_redact_mapping_recurses_lists_and_values() -> None: + cleaned, redacted_keys = redact_mapping( + { + "ok": 1, + "PASSWORD": "x", + "nested": {"TOKEN": "y"}, + "headers": [{"Authorization": "Bearer abc.def.ghi"}, {"ok": True}], + "note": "ghp_abcdefghijklmnopqrstuvwxyz0123456789", + } + ) + assert cleaned["ok"] == 1 + assert cleaned["nested"] == {} + assert cleaned["headers"] == [{}, {"ok": True}] + assert "note" not in cleaned + assert "PASSWORD" in redacted_keys + assert any("TOKEN" in item for item in redacted_keys) + + +def test_secrets_absent_from_snapshot_and_profile_export() -> None: + snapshot = build_configuration_snapshot( + backend_id="test-backend", + adapter_id="test-adapter", + adapter_version="0.1.0", + config={"timeout_ms": 1000, "API_TOKEN": "should-not-leak", "threshold": 1}, + environment={"PATH": "/bin", "GH_TOKEN": "secret-token", "CI": "1"}, + mechanism_class="static_analysis", + guarantee_class="observational", + supported_claim_ids=["claim.test"], + ) + assert "API_TOKEN" not in snapshot.config + assert "GH_TOKEN" not in snapshot.redacted_environment["entries"] + profile = snapshot_to_verifier_profile(snapshot) + serialized = str(profile) + assert "should-not-leak" not in serialized + assert "secret-token" not in serialized diff --git a/tests/assurance/test_replay.py b/tests/assurance/test_replay.py new file mode 100644 index 0000000..8ca3f37 --- /dev/null +++ b/tests/assurance/test_replay.py @@ -0,0 +1,77 @@ +"""Assurance replay: golden match and fail-closed drift.""" + +from __future__ import annotations + +import json + +import pytest + +from ovk.assurance.errors import ReplayError +from ovk.assurance.pin import resolve_pcs_root +from ovk.assurance.replay import replay_invocation +from ovk.assurance.runner import run_assurance +from tests.assurance.support.digest_adapter import DigestPredicateAdapter + +PCS_AVAILABLE = resolve_pcs_root() is not None +requires_pcs = pytest.mark.skipif( + not PCS_AVAILABLE, + reason="PCS pin unavailable (set OVK_PCS_CORE_PATH or sibling ../pcs-core)", +) + + +@requires_pcs +def test_replay_golden_match(tmp_path) -> None: + expected = "sha256:" + ("ee" * 32) + adapter = DigestPredicateAdapter(expected_digest=expected, timeout_ms=1500) + evidence_dir = tmp_path / "evidence" + outcome = run_assurance( + adapter, + input_data={"digest": expected}, + config={"timeout_ms": 1500, "threshold": 1, "expected_digest": expected}, + evidence_dir=evidence_dir, + ) + invocation_path = evidence_dir / "invocation.json" + report = replay_invocation( + invocation_path, + adapter=adapter, + evidence_dir=evidence_dir, + config={"timeout_ms": 1500, "threshold": 1, "expected_digest": expected}, + ) + assert report["replay_status"] == "matched" + assert report["drift"]["raw_digest_match"] is True + assert report["drift"]["normalized_digest_match"] is True + + +@requires_pcs +def test_replay_drift_fails_closed(tmp_path) -> None: + expected = "sha256:" + ("ff" * 32) + adapter = DigestPredicateAdapter(expected_digest=expected, timeout_ms=1500, threshold=1) + evidence_dir = tmp_path / "evidence" + outcome = run_assurance( + adapter, + input_data={"digest": expected}, + config={"timeout_ms": 1500, "threshold": 1, "expected_digest": expected}, + evidence_dir=evidence_dir, + ) + # Mutate stored raw digest to force drift detection on claim_matched + invocation = json.loads((evidence_dir / "invocation.json").read_text(encoding="utf-8")) + # Rerun with different threshold so digests differ, then claim matched. + drifted_adapter = DigestPredicateAdapter(expected_digest=expected, timeout_ms=1500, threshold=0) + with pytest.raises(ReplayError): + replay_invocation( + invocation, + adapter=drifted_adapter, + evidence_dir=evidence_dir, + config={"timeout_ms": 1500, "threshold": 0, "expected_digest": expected}, + claim_matched=True, + ) + + report = replay_invocation( + invocation, + adapter=drifted_adapter, + evidence_dir=evidence_dir, + config={"timeout_ms": 1500, "threshold": 0, "expected_digest": expected}, + claim_matched=False, + ) + assert report["replay_status"] == "drifted" + assert outcome.decision == "accept" diff --git a/tests/assurance/test_snapshot_export.py b/tests/assurance/test_snapshot_export.py new file mode 100644 index 0000000..f05490a --- /dev/null +++ b/tests/assurance/test_snapshot_export.py @@ -0,0 +1,75 @@ +"""Snapshot digests and PCS profile export.""" + +from __future__ import annotations + +import pytest + +from ovk.assurance.pcs_export import snapshot_to_verifier_profile +from ovk.assurance.pcs_validate import validate_against_pin_schema +from ovk.assurance.pin import resolve_pcs_root +from ovk.assurance.snapshot import build_configuration_snapshot +from tests.assurance.support.digest_adapter import DigestPredicateAdapter + +PCS_AVAILABLE = resolve_pcs_root() is not None +requires_pcs = pytest.mark.skipif( + not PCS_AVAILABLE, + reason="PCS pin unavailable (set OVK_PCS_CORE_PATH or sibling ../pcs-core)", +) + + +def test_snapshot_digest_stable_for_identical_config() -> None: + kwargs = { + "backend_id": "b", + "adapter_id": "a", + "adapter_version": "0.1.0", + "config": {"timeout_ms": 1000, "threshold": 1}, + "environment": {"CI": "true"}, + "mechanism_class": "static_analysis", + "created_at": "2026-07-24T12:00:00Z", + } + s1 = build_configuration_snapshot(**kwargs) + s2 = build_configuration_snapshot(**kwargs) + assert s1.config_digest == s2.config_digest + assert s1.content_digest == s2.content_digest + + +def test_material_change_changes_digest() -> None: + base = build_configuration_snapshot( + backend_id="b", + adapter_id="a", + adapter_version="0.1.0", + config={"timeout_ms": 1000, "threshold": 1}, + mechanism_class="static_analysis", + created_at="2026-07-24T12:00:00Z", + ) + changed = build_configuration_snapshot( + backend_id="b", + adapter_id="a", + adapter_version="0.1.0", + config={"timeout_ms": 2000, "threshold": 1}, + mechanism_class="static_analysis", + created_at="2026-07-24T12:00:00Z", + ) + assert base.config_digest != changed.config_digest + assert base.content_digest != changed.content_digest + + +@requires_pcs +def test_adapter_snapshot_exports_valid_pcs_profile() -> None: + adapter = DigestPredicateAdapter(expected_digest="sha256:" + ("ab" * 32)) + snapshot = adapter.snapshot_config({"timeout_ms": 2500, "threshold": 1}) + profile = snapshot_to_verifier_profile(snapshot) + assert profile["artifact_type"] == "VerifierProfile.v1" + assert profile["verifier_profile_id"] + assert profile["configuration"]["config_digest"].startswith("sha256:") + assert profile["configuration"]["policy_digest"] is None or isinstance( + profile["configuration"]["policy_digest"], str + ) + # Explicit null slots present + for key in ("policy_digest", "model_digest", "prompt_digest", "resource_limit_digest"): + assert key in profile["configuration"] + assert "integrity" in profile + assert profile["integrity"]["canonicalization_version"] == "v1" + assert profile["source_commit"] != "0" * 40 + report = validate_against_pin_schema(profile, artifact_type="VerifierProfile.v1") + assert report.valid, report.issues