From 3fdd494162406317e047f3fe9d4e4186ce959365 Mon Sep 17 00:00:00 2001 From: Bjorn Tipling Date: Fri, 4 Sep 2026 08:14:31 +0000 Subject: [PATCH 1/6] CXF-220: pre-1 judgment skills (design-access-model, source-openapi-spec) + park-vs-proceed eval measurement - Two new pre-1 skills (design-access-model, source-openapi-spec) with SOURCES.md pins; bundle v0.3.0 (7 skills); orchestrator routing split. - Fixture no-IAM surface: openapi-noiam.json + /noiam/openapi.json route + verify.sh checks 17-19 (all 19 assertions). - Runner pre1 kind: scenario kind/validation, P0-P4 gates, scoreRun branch with decision_verdict/decision_evidence, readPre1Artifact, pre1 main branch, baseline S0-first skip, tier0 canned-dir selection. - Two pre1 scenarios + canned replay sets; unit smokes in existing tests. Co-authored-by: c1-squire-dev[bot] --- evals/README.md | 37 ++- evals/fixture/README.md | 10 +- evals/fixture/openapi-noiam.json | 223 +++++++++++++++++ evals/fixture/server.ts | 16 +- evals/fixture/verify.sh | 24 +- evals/runner/agent.test.ts | 1 + evals/runner/agent.ts | 22 ++ evals/runner/baseline.test.ts | 25 ++ evals/runner/baseline.ts | 13 + evals/runner/driver.ts | 3 + .../canned-pre1-directory-proceed/pre1.json | 29 +++ .../score-input.json | 1 + .../transcript.json | 55 +++++ .../tier0/canned-pre1-noiam-park/pre1.json | 9 + .../canned-pre1-noiam-park/score-input.json | 1 + .../canned-pre1-noiam-park/transcript.json | 55 +++++ evals/runner/drivers/tier0/driver.test.ts | 95 +++++++- evals/runner/drivers/tier0/driver.ts | 57 ++++- evals/runner/record.ts | 3 + evals/runner/run.test.ts | 19 +- evals/runner/run.ts | 100 +++++++- evals/runner/scenario.test.ts | 128 +++++++++- evals/runner/scenario.ts | 228 +++++++++++++++--- evals/runner/score.test.ts | 99 ++++++++ evals/runner/score.ts | 42 +++- evals/runner/skills_bundle.test.ts | 53 +++- evals/runner/stages.test.ts | 180 +++++++++++++- evals/runner/stages.ts | 204 ++++++++++++++++ evals/scenarios/pre1-directory-proceed.json | 36 +++ evals/scenarios/pre1-noiam-park.json | 34 +++ evals/scenarios/tier1-directory-full.json | 2 +- evals/skills-bundle/README.md | 12 + evals/skills-bundle/bundle.json | 6 +- skills/README.md | 4 +- skills/author-in-app-connector/SKILL.md | 11 +- skills/design-access-model/SKILL.md | 90 +++++++ skills/design-access-model/SOURCES.md | 13 + skills/source-openapi-spec/SKILL.md | 81 +++++++ skills/source-openapi-spec/SOURCES.md | 12 + 39 files changed, 1935 insertions(+), 98 deletions(-) create mode 100644 evals/fixture/openapi-noiam.json create mode 100644 evals/runner/drivers/tier0/canned-pre1-directory-proceed/pre1.json create mode 100644 evals/runner/drivers/tier0/canned-pre1-directory-proceed/score-input.json create mode 100644 evals/runner/drivers/tier0/canned-pre1-directory-proceed/transcript.json create mode 100644 evals/runner/drivers/tier0/canned-pre1-noiam-park/pre1.json create mode 100644 evals/runner/drivers/tier0/canned-pre1-noiam-park/score-input.json create mode 100644 evals/runner/drivers/tier0/canned-pre1-noiam-park/transcript.json create mode 100644 evals/scenarios/pre1-directory-proceed.json create mode 100644 evals/scenarios/pre1-noiam-park.json create mode 100644 skills/design-access-model/SKILL.md create mode 100644 skills/design-access-model/SOURCES.md create mode 100644 skills/source-openapi-spec/SKILL.md create mode 100644 skills/source-openapi-spec/SOURCES.md diff --git a/evals/README.md b/evals/README.md index 4ad139e..6687f15 100644 --- a/evals/README.md +++ b/evals/README.md @@ -14,8 +14,8 @@ fixture and produces a scored JSONL record with the full S0–S11 stage funnel. | `evals/fixture/` | Deterministic Directory API fixture (zero-dependency `node:http`) | | `evals/runner/` | Runner + scorer (`run.ts` CLI, driver interfaces, stage gates) | | `evals/runner/drivers/` | Driver implementations — Tier-0 local/static driver; authoring contract in `drivers/README.md` | -| `evals/scenarios/` | Scenario definitions (`tier1-directory.json`, `tier1-directory-guide-only.json`, `tier1-directory-full.json`) | -| `evals/skills-bundle/` | Skill-bundle mount point (v0.2.0 manifest — five funnel skills in `skills/`) | +| `evals/scenarios/` | Scenario definitions (`tier1-directory.json`, `tier1-directory-guide-only.json`, `tier1-directory-full.json`, `pre1-directory-proceed.json`, `pre1-noiam-park.json`) | +| `evals/skills-bundle/` | Skill-bundle mount point (v0.3.0 manifest — seven skills in `skills/`) | | `evals/results/` | JSONL run records (gitignored; `.gitkeep` committed) | ## How to run @@ -24,7 +24,7 @@ fixture and produces a scored JSONL record with the full S0–S11 stage funnel. # start the fixture locally (port 18080) npm run eval:fixture -# verify the fixture (all 16 assertions, port 18081; requires curl + jq) +# verify the fixture (all 19 assertions, port 18081; requires curl + jq) npm run eval:verify # run the committed unit smokes (scorer/parser/stages/record/scenario/driver) @@ -113,6 +113,35 @@ never invented. | S10 | durable PASS evidence | score-input `evidence.result == "PASS"` | | S11 | handoff discipline | deploy + mint succeeded; all 10 handoff fields; zero tool calls after the mint except the handoff write; no token redemption | +## Pre-1 kind (P0–P4) + +A `kind: "pre1"` scenario runs the pre-1 judgment phase instead of the funnel: +the agent sources the provider's OpenAPI spec and designs the access model, +then writes a `pre1.json` artifact to the run channel's `pre1Path` and stops. +Pre-1 scenarios carry `providerBrief`, `expectedDecision` (`"proceed"` or +`"park"`), and exactly one of `expectedAccessModel` (proceed) or +`expectedParkEvidence` (park); the funnel-only fields (`seed`, `expected`, +`requiredSourceFiles`) must be absent. The Tier-0 driver replays canned +artifacts from `drivers/tier0/canned-/`. + +Pre-1 records are scored against the P0–P4 gate set: + +| Stage | Gate | Evidence | +|---|---|---| +| P0 | artifact written | `pre1.json` present with a `decision` of `"proceed"` or `"park"` | +| P1 | decision correctness | `decision === expectedDecision` — the separately-measured park-vs-proceed metric | +| P2 | access-model match (proceed) | resource-type `{id, traits}` pairs, entitlement slugs, and grant edges set-equal to the expected sets; `id_compatibility` non-empty; every `provisioning` entry justified with a boolean `provisionable` | +| P3 | sourcing provenance (proceed) | `spec_url`/`fetched_at`/`authority_rung` non-empty; `spec_bytes` a positive integer < 1048576 | +| P4 | park evidence (park) | all four `park_evidence` fields non-empty with `missing_paths` a non-empty array | + +The record's summary carries `decision_verdict` (`"proceed"`/`"park"` when P1 +passes, `"incorrect"` otherwise) and `decision_evidence` (the P1 row's +evidence) — the park-vs-proceed metric is measured from these fields. +`parity_verdict`/`hygiene_verdict` are `"PASS"` with evidence +`"not applicable (pre1 run)"`; `parity_tenant` is `"not_applicable"`; +`handoff_discipline_verdict` is `true`; `recovery_cycles` is 0. Pre-1 records +write no skipped rows (`skippedRows = []`). + ## Fixture traps The fixture (`evals/fixture/`) mirrors the documented failure modes: @@ -207,7 +236,7 @@ real-tenant driver and the tool surface are available. ## Non-goals -- The remaining five skills (design-access-model, source-openapi-spec, verify-connector-output, update-and-rollback, diagnose-authoring-failure) — later PRs; the fifth funnel skill ships in this PR. +- The remaining three skills (verify-connector-output, update-and-rollback, diagnose-authoring-failure) — later PRs. - Tier-2 real sandbox providers and the qualitative LLM-judge tier. - Operator-side activation E2E leg (redeeming the approval token) — those two fields are `skipped_human_boundary`. diff --git a/evals/fixture/README.md b/evals/fixture/README.md index 7292ab1..d3d7953 100644 --- a/evals/fixture/README.md +++ b/evals/fixture/README.md @@ -17,6 +17,7 @@ with a fixed seed so sync results are reproducible. | Method | Path | Auth | Pagination | |---|---|---|---| | GET | `/openapi.json` | none | — | +| GET | `/noiam/openapi.json` | none | — | | GET | `/v1/users` | basic | offset (`{items, offset, limit, total}`) | | GET | `/v1/groups` | basic | offset | | GET | `/v1/groups/{groupId}/members` | basic | offset | @@ -33,6 +34,9 @@ every non-final page (absent on the final page). Auth failures return `401` with `WWW-Authenticate` (`Basic realm="fixture"` on `/v1`, `Bearer` on `/v2`). Unknown paths return `404`; wrong methods `405`. +`/noiam/openapi.json` serves the no-IAM surface (whoami + business endpoints; +no member listing/roles/groups/keys) used by the pre1 park scenario. + ## Seed - 23 users (`user-001`..`user-023`), `active: false` for `user-007` and @@ -80,9 +84,9 @@ bash evals/fixture/verify.sh npm run eval:verify ``` -`verify.sh` runs the server on port 18081 (never 18080) and asserts all 16 -checks: every endpoint, both auth variants, both pagination variants, and all -three traps. +`verify.sh` runs the server on port 18081 (never 18080) and asserts all 19 +checks: every endpoint, both auth variants, both pagination variants, all +three traps, and the no-IAM surface. ## Container diff --git a/evals/fixture/openapi-noiam.json b/evals/fixture/openapi-noiam.json new file mode 100644 index 0000000..6f2d5bc --- /dev/null +++ b/evals/fixture/openapi-noiam.json @@ -0,0 +1,223 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Acme Analytics API", + "version": "1.2.0", + "description": "Usage-analytics platform API for the connector-authoring eval harness. Exposes a whoami endpoint and business endpoints only. User and group management is available only in the Acme web console; there is no public API for member listing, role management, or API key inventory." + }, + "paths": { + "/v1/me": { + "get": { + "summary": "Whoami - the current authenticated identity", + "security": [ + { + "bearerAuth": [] + } + ], + "responses": { + "200": { + "description": "The current identity", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Identity" + } + } + } + }, + "401": { + "description": "Missing or invalid bearer auth" + } + } + } + }, + "/v1/projects": { + "get": { + "summary": "List projects", + "security": [ + { + "bearerAuth": [] + } + ], + "responses": { + "200": { + "description": "A list of projects", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Project" + } + } + } + } + }, + "401": { + "description": "Missing or invalid bearer auth" + } + } + } + }, + "/v1/projects/{projectId}/reports": { + "get": { + "summary": "List reports for a project", + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "projectId", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "A list of reports", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Report" + } + } + } + } + }, + "401": { + "description": "Missing or invalid bearer auth" + }, + "404": { + "description": "Unknown project" + } + } + } + }, + "/v1/invoices": { + "get": { + "summary": "List invoices", + "security": [ + { + "bearerAuth": [] + } + ], + "responses": { + "200": { + "description": "A list of invoices", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Invoice" + } + } + } + } + }, + "401": { + "description": "Missing or invalid bearer auth" + } + } + } + } + }, + "components": { + "securitySchemes": { + "bearerAuth": { + "type": "http", + "scheme": "bearer" + } + }, + "schemas": { + "Identity": { + "type": "object", + "required": [ + "id", + "email" + ], + "properties": { + "id": { + "type": "string" + }, + "email": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "Project": { + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "Report": { + "type": "object", + "required": [ + "id", + "projectId", + "name" + ], + "properties": { + "id": { + "type": "string" + }, + "projectId": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "Invoice": { + "type": "object", + "required": [ + "id", + "amount" + ], + "properties": { + "id": { + "type": "string" + }, + "amount": { + "type": "number" + }, + "status": { + "type": "string" + } + } + }, + "Error": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + } + } + } + } +} diff --git a/evals/fixture/server.ts b/evals/fixture/server.ts index 14aaf06..fabacd6 100644 --- a/evals/fixture/server.ts +++ b/evals/fixture/server.ts @@ -20,6 +20,7 @@ import { } from "./data.ts" const OPENAPI = readFileSync(new URL("./openapi.json", import.meta.url), "utf8") +const OPENAPI_NOIAM = readFileSync(new URL("./openapi-noiam.json", import.meta.url), "utf8") // --- args: --port (default 18080), --host (default 127.0.0.1) --- let port = 18080 @@ -158,7 +159,7 @@ const server = createServer(async (req: IncomingMessage, res: ServerResponse) => const log = (status: number) => stdout.write(`${method} ${path} ${status}\n`) try { - // --- openapi.json: unauthenticated --- +// --- openapi.json: unauthenticated --- if (path === "/openapi.json") { if (method !== "GET") { log(405) @@ -171,6 +172,19 @@ const server = createServer(async (req: IncomingMessage, res: ServerResponse) => return } + // --- noiam/openapi.json: unauthenticated (pre1 park scenario) --- + if (path === "/noiam/openapi.json") { + if (method !== "GET") { + log(405) + sendEmpty(res, 405) + return + } + log(200) + res.writeHead(200, {"Content-Type": "application/json"}) + res.end(OPENAPI_NOIAM) + return + } + // --- /v1/* : basic auth + offset pagination --- if (path.startsWith("/v1/")) { const creds = parseBasicAuth(req) diff --git a/evals/fixture/verify.sh b/evals/fixture/verify.sh index 4e8a4fd..2e8ffcf 100755 --- a/evals/fixture/verify.sh +++ b/evals/fixture/verify.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # verify.sh — self-check for the Directory API fixture (evals/fixture). -# Runs the server on port 18081 (never 18080) and asserts all 16 checks: -# every endpoint, both auth variants, both pagination variants, and all three -# traps. Requires curl + jq. Kills the server on exit (trap). +# Runs the server on port 18081 (never 18080) and asserts all 19 checks: +# every endpoint, both auth variants, both pagination variants, all three +# traps, and the no-IAM surface. Requires curl + jq. Kills the server on +# exit (trap). set -euo pipefail PORT=18081 @@ -160,4 +161,19 @@ CODE="$(curl -sS -o /dev/null -w '%{http_code}' $BASIC_AUTH -X POST "$BASE/v1/us [ "$CODE" = "405" ] || fail "POST /v1/users != 405 (got $CODE)" echo "ok: wrong method -> 405" -echo "== all 16 fixture assertions passed ==" +# (17) GET /noiam/openapi.json -> 200 +CODE="$(curl -sS -o /dev/null -w '%{http_code}' "$BASE/noiam/openapi.json")" +[ "$CODE" = "200" ] || fail "/noiam/openapi.json != 200 (got $CODE)" +echo "ok: /noiam/openapi.json -> 200" + +# (18) the no-IAM spec serves the whoami path +NOIAM="$(curl -sS "$BASE/noiam/openapi.json")" +jq -e '.paths["/v1/me"]' <<<"$NOIAM" >/dev/null || fail "noiam openapi.json missing /v1/me path" +echo "ok: noiam openapi.json serves /v1/me" + +# (19) the no-IAM spec has NO member-listing paths +jq -e '.paths["/v1/users"] | not' <<<"$NOIAM" >/dev/null || fail "noiam openapi.json has /v1/users (must be absent)" +jq -e '.paths["/v1/groups"] | not' <<<"$NOIAM" >/dev/null || fail "noiam openapi.json has /v1/groups (must be absent)" +echo "ok: noiam openapi.json has no /v1/users or /v1/groups" + +echo "== all 19 fixture assertions passed ==" diff --git a/evals/runner/agent.test.ts b/evals/runner/agent.test.ts index 9c9aecd..7f30096 100644 --- a/evals/runner/agent.test.ts +++ b/evals/runner/agent.test.ts @@ -38,6 +38,7 @@ const CHANNEL: RunChannel = { handoffPath: "/tmp/evals-run/handoff.json", scoreInputPath: "/tmp/evals-run/score-input.json", transcriptPath: "/tmp/evals-run/transcript.json", + pre1Path: "/tmp/evals-run/pre1.json", handoffInstructions: 'Write it with driver.write_file: args {path: "", content: ""}.', completionInstructions: 'Then terminate the run with driver.complete_run: args {summary: "handoff written; funnel complete to human-activation boundary"}.', } diff --git a/evals/runner/agent.ts b/evals/runner/agent.ts index 2d1660a..632ea58 100644 --- a/evals/runner/agent.ts +++ b/evals/runner/agent.ts @@ -15,6 +15,28 @@ function skillBundleSection(scenario: Scenario): string { } export function buildPrompt(scenario: Scenario, runId: string, baseUrl: string, channel: RunChannel): string { + if (scenario.kind === "pre1") { + return `You are an eval agent performing the pre-1 judgment phase for a net-new connector provider. Your goal: source the provider's OpenAPI spec and design the access model, then write the pre1.json artifact and stop. + +(1) ROLE + GOAL +Run the two pre-1 skills in order: source-openapi-spec (source the spec, run the IAM go/no-go gate) then design-access-model (design the access model). Write the pre1.json artifact to ${channel.pre1Path} and stop. + +(2) PROVIDER BRIEF +${scenario.providerBrief} + +(3) SPEC +- Spec URL: ${baseUrl}${scenario.fixture.openapiPath} +- Fetch it with bash: curl -sS ${baseUrl}${scenario.fixture.openapiPath} (the fixture serves it unauthenticated). + +(4) SKILL BUNDLE +${skillBundleSection(scenario)} + +(5) OUTPUT CONTRACT +Write pre1.json via driver.write_file: args {path: "${channel.pre1Path}", content: ""}. Schema: {decision: "proceed"|"park", access_model: {resource_types: [{id, traits}], entitlements: [{slug, display_name, grantable_principals, stable_id_shape}], grants: [{resource_type, entitlement, principal_type}], id_compatibility: [...], provisioning: [{resource_type, provisionable, justification}]}, sourcing: {spec_url, fetched_at, authority_rung, spec_bytes}, park_evidence: {spec_version_checked, missing_paths, vendor_doc, revisit_trigger}}. Include only the sections your decision requires: proceed -> access_model + sourcing; park -> park_evidence. + +(6) COMPLETION +Then terminate the run with driver.complete_run: args {summary: "pre1 artifact written"}.` + } const creds = scenario.fixture.basicAuth return `You are an eval agent implementing a read-only Directory API connector and completing the 12-step in-app authoring funnel. Your goal: implement the connector source files, walk the funnel to the human-activation boundary, and stop there. diff --git a/evals/runner/baseline.test.ts b/evals/runner/baseline.test.ts index 645394b..af59e8f 100644 --- a/evals/runner/baseline.test.ts +++ b/evals/runner/baseline.test.ts @@ -340,3 +340,28 @@ test("(j) duplicate stage row exits 1", async () => { rmSync(dir, {recursive: true, force: true}) } }) + +test("(k) a pre1 record (stage rows begin with P0) is skipped with a warning, never fatal", async () => { + const dir = tmpDir() + try { + // A normal funnel record so the baseline has a matrix record to process. + writeFileSync(join(dir, "a.jsonl"), allPassRecord("evals-tier1-directory-20260902-120000-000", "none")) + // A pre1-shaped record with mode "none" (in the matrix) whose stage rows + // begin with P0 — must be skipped, not validated, not fatal. + const pre1Lines = [metaLine("evals-pre1-directory-proceed-20260902-120000-000", "none")] + for (const stage of ["P0", "P1", "P2", "P3"]) { + pre1Lines.push(stageRow(stage, true, true)) + } + pre1Lines.push(summaryLine(1.0)) + writeFileSync(join(dir, "b.jsonl"), pre1Lines.join("\n") + "\n") + const {code, stderr} = await runBaseline(dir) + assert.equal(code, 0) + assert.ok(stderr.includes("does not begin with the S0 funnel row"), `stderr=${stderr}`) + assert.ok(stderr.includes("b.jsonl")) + const b = readBaseline(dir) + const none = b.modes as Record> + assert.deepEqual(none.none.run_ids, ["evals-tier1-directory-20260902-120000-000"]) + } finally { + rmSync(dir, {recursive: true, force: true}) + } +}) diff --git a/evals/runner/baseline.ts b/evals/runner/baseline.ts index 7ae2e90..284cfa1 100644 --- a/evals/runner/baseline.ts +++ b/evals/runner/baseline.ts @@ -74,6 +74,19 @@ function parseRecord(file: string, lines: string[]): RunRecord | null { fail(file, 1, "meta field started_at missing or not an ISO timestamp") } + // Pre-1 records (stage rows begin with P0, not S0) can land in + // evals/results/ locally without breaking baseline generation: skip them + // with a one-line stderr warning, never fatal. Funnel-shaped records + // validate exactly as before. + const firstRowRaw = parseLine(file, 2, lines[1]) + if (typeof firstRowRaw === "object" && firstRowRaw !== null && !Array.isArray(firstRowRaw)) { + const firstRow = firstRowRaw as Record + if (firstRow.stage !== "S0") { + console.error(`WARNING: skipping ${file}: record does not begin with the S0 funnel row`) + return null + } + } + // The last line must be the summary. const lastIdx = lines.length - 1 const summaryRaw = parseLine(file, lastIdx + 1, lines[lastIdx]) diff --git a/evals/runner/driver.ts b/evals/runner/driver.ts index b0e728a..c6a5719 100644 --- a/evals/runner/driver.ts +++ b/evals/runner/driver.ts @@ -60,6 +60,7 @@ export interface RunChannel { handoffPath: string scoreInputPath: string transcriptPath: string + pre1Path: string handoffInstructions: string completionInstructions: string } @@ -71,6 +72,8 @@ export interface AgentRunRequest { channel: RunChannel timeoutMs: number model: string + // The scenario id (pre-1 runs use it to select the canned replay set). + scenarioId?: string // Declared scenario reasoning-effort pin; driver-interpreted — a driver // that can set agent reasoning effort applies it. Tier-0 ignores it. reasoningEffort?: "high" | "medium" | "low" diff --git a/evals/runner/drivers/tier0/canned-pre1-directory-proceed/pre1.json b/evals/runner/drivers/tier0/canned-pre1-directory-proceed/pre1.json new file mode 100644 index 0000000..1b13887 --- /dev/null +++ b/evals/runner/drivers/tier0/canned-pre1-directory-proceed/pre1.json @@ -0,0 +1,29 @@ +{ + "decision": "proceed", + "access_model": { + "resource_types": [ + {"id": "user", "traits": ["TRAIT_USER"]}, + {"id": "group", "traits": ["TRAIT_GROUP"]} + ], + "entitlements": [ + {"slug": "member", "display_name": "Member", "grantable_principals": ["user"], "stable_id_shape": "group:{id}:member"} + ], + "grants": [ + {"resource_type": "group", "entitlement": "member", "principal_type": "user"} + ], + "id_compatibility": [ + {"resource_type": "user", "id_shape": "user:{id}", "stable": true}, + {"resource_type": "group", "id_shape": "group:{id}", "stable": true} + ], + "provisioning": [ + {"resource_type": "user", "provisionable": false, "justification": "because the API lacks a user create/update endpoint"}, + {"resource_type": "group", "provisionable": false, "justification": "because the API lacks a group create/update endpoint"} + ] + }, + "sourcing": { + "spec_url": "http://127.0.0.1:18080/openapi.json", + "fetched_at": "2026-09-04T00:00:00.000Z", + "authority_rung": "official published spec at a stable URL", + "spec_bytes": 12297 + } +} diff --git a/evals/runner/drivers/tier0/canned-pre1-directory-proceed/score-input.json b/evals/runner/drivers/tier0/canned-pre1-directory-proceed/score-input.json new file mode 100644 index 0000000..4609451 --- /dev/null +++ b/evals/runner/drivers/tier0/canned-pre1-directory-proceed/score-input.json @@ -0,0 +1 @@ +{"run_id": "pre1-canned", "draft": {"required_source_files": {}, "source_files": [], "config_schema": {"fields": []}, "runtime_schema": {"fields": []}}, "connector_config": {}, "evidence": {}, "build_run": {}, "tenant_counts": {"users": null, "groups": null, "memberships": null}, "resource_ids": {"users": [], "groups": []}} diff --git a/evals/runner/drivers/tier0/canned-pre1-directory-proceed/transcript.json b/evals/runner/drivers/tier0/canned-pre1-directory-proceed/transcript.json new file mode 100644 index 0000000..1c0b67d --- /dev/null +++ b/evals/runner/drivers/tier0/canned-pre1-directory-proceed/transcript.json @@ -0,0 +1,55 @@ +[ + { + "type": "tool_call", + "message": "bash", + "data": { + "input": { + "i": "fetch spec", + "command": "curl -sS http://127.0.0.1:18080/openapi.json -o /tmp/spec.json && wc -c /tmp/spec.json" + } + } + }, + { + "type": "tool_result", + "message": "12297 /tmp/spec.json", + "data": { + "tool_name": "bash", + "is_error": false + } + }, + { + "type": "tool_call", + "message": "driver.write_file", + "data": { + "input": { + "path": "/pre1.json", + "content": "{\"decision\":\"proceed\",\"access_model\":{\"resource_types\":[{\"id\":\"user\",\"traits\":[\"TRAIT_USER\"]},{\"id\":\"group\",\"traits\":[\"TRAIT_GROUP\"]}],\"entitlements\":[{\"slug\":\"member\",\"display_name\":\"Member\",\"grantable_principals\":[\"user\"],\"stable_id_shape\":\"group:{id}:member\"}],\"grants\":[{\"resource_type\":\"group\",\"entitlement\":\"member\",\"principal_type\":\"user\"}],\"id_compatibility\":[{\"resource_type\":\"user\",\"id_shape\":\"user:{id}\",\"stable\":true},{\"resource_type\":\"group\",\"id_shape\":\"group:{id}\",\"stable\":true}],\"provisioning\":[{\"resource_type\":\"user\",\"provisionable\":false,\"justification\":\"because the API lacks a user create/update endpoint\"},{\"resource_type\":\"group\",\"provisionable\":false,\"justification\":\"because the API lacks a group create/update endpoint\"}]},\"sourcing\":{\"spec_url\":\"http://127.0.0.1:18080/openapi.json\",\"fetched_at\":\"2026-09-04T00:00:00.000Z\",\"authority_rung\":\"official published spec at a stable URL\",\"spec_bytes\":12297}}" + } + } + }, + { + "type": "tool_result", + "message": "written", + "data": { + "tool_name": "driver.write_file", + "is_error": false + } + }, + { + "type": "tool_call", + "message": "driver.complete_run", + "data": { + "input": { + "summary": "pre1 artifact written" + } + } + }, + { + "type": "tool_result", + "message": "done", + "data": { + "tool_name": "driver.complete_run", + "is_error": false + } + } +] diff --git a/evals/runner/drivers/tier0/canned-pre1-noiam-park/pre1.json b/evals/runner/drivers/tier0/canned-pre1-noiam-park/pre1.json new file mode 100644 index 0000000..0156a88 --- /dev/null +++ b/evals/runner/drivers/tier0/canned-pre1-noiam-park/pre1.json @@ -0,0 +1,9 @@ +{ + "decision": "park", + "park_evidence": { + "spec_version_checked": "1.2.0", + "missing_paths": ["/v1/users", "/v1/groups", "/v1/groups/{groupId}/members"], + "vendor_doc": "User and group management is available only in the Acme web console; there is no public API for member listing, role management, or API key inventory.", + "revisit_trigger": "Acme ships a members or teams API (e.g. GET /v1/users or GET /v1/teams/{id}/members)" + } +} diff --git a/evals/runner/drivers/tier0/canned-pre1-noiam-park/score-input.json b/evals/runner/drivers/tier0/canned-pre1-noiam-park/score-input.json new file mode 100644 index 0000000..4609451 --- /dev/null +++ b/evals/runner/drivers/tier0/canned-pre1-noiam-park/score-input.json @@ -0,0 +1 @@ +{"run_id": "pre1-canned", "draft": {"required_source_files": {}, "source_files": [], "config_schema": {"fields": []}, "runtime_schema": {"fields": []}}, "connector_config": {}, "evidence": {}, "build_run": {}, "tenant_counts": {"users": null, "groups": null, "memberships": null}, "resource_ids": {"users": [], "groups": []}} diff --git a/evals/runner/drivers/tier0/canned-pre1-noiam-park/transcript.json b/evals/runner/drivers/tier0/canned-pre1-noiam-park/transcript.json new file mode 100644 index 0000000..d45712d --- /dev/null +++ b/evals/runner/drivers/tier0/canned-pre1-noiam-park/transcript.json @@ -0,0 +1,55 @@ +[ + { + "type": "tool_call", + "message": "bash", + "data": { + "input": { + "i": "fetch spec", + "command": "curl -sS http://127.0.0.1:18080/noiam/openapi.json -o /tmp/spec.json && wc -c /tmp/spec.json" + } + } + }, + { + "type": "tool_result", + "message": "12297 /tmp/spec.json", + "data": { + "tool_name": "bash", + "is_error": false + } + }, + { + "type": "tool_call", + "message": "driver.write_file", + "data": { + "input": { + "path": "/pre1.json", + "content": "{\"decision\":\"park\",\"park_evidence\":{\"spec_version_checked\":\"1.2.0\",\"missing_paths\":[\"/v1/users\",\"/v1/groups\",\"/v1/groups/{groupId}/members\"],\"vendor_doc\":\"User and group management is available only in the Acme web console; there is no public API for member listing, role management, or API key inventory.\",\"revisit_trigger\":\"Acme ships a members or teams API (e.g. GET /v1/users or GET /v1/teams/{id}/members)\"}}" + } + } + }, + { + "type": "tool_result", + "message": "written", + "data": { + "tool_name": "driver.write_file", + "is_error": false + } + }, + { + "type": "tool_call", + "message": "driver.complete_run", + "data": { + "input": { + "summary": "pre1 artifact written" + } + } + }, + { + "type": "tool_result", + "message": "done", + "data": { + "tool_name": "driver.complete_run", + "is_error": false + } + } +] diff --git a/evals/runner/drivers/tier0/driver.test.ts b/evals/runner/drivers/tier0/driver.test.ts index 77af01e..f93d5ad 100644 --- a/evals/runner/drivers/tier0/driver.test.ts +++ b/evals/runner/drivers/tier0/driver.test.ts @@ -9,7 +9,7 @@ import {join} from "node:path" import {tier0, TIER0_TOOL_SURFACE} from "./driver.ts" import {scoreRun} from "../../score.ts" import {writeRecord, type RunMeta, type SummaryLine} from "../../record.ts" -import {SKIPPED_STAGES, type ScoreInput, type StageCtx} from "../../stages.ts" +import {SKIPPED_STAGES, type Pre1Artifact, type ScoreInput, type StageCtx} from "../../stages.ts" import {FUNNEL_TOOLS, type RunChannel} from "../../driver.ts" import type {Scenario} from "../../scenario.ts" @@ -46,7 +46,7 @@ test("tier0 provisioner readiness passes against the live local fixture", async try { await tier0.provisioner.checkReadiness(handle) assert.ok(handle.toolSurface.length >= 16) - assert.ok(SCENARIO.readinessTools.every((t) => handle.toolSurface.includes(t))) + assert.ok((SCENARIO.readinessTools ?? []).every((t) => handle.toolSurface.includes(t))) // The declared surface carries the full funnel, so the runner's // funnel_tools_present derivation (FUNNEL_TOOLS ⊆ toolSurface) is true. assert.ok(FUNNEL_TOOLS.every((t) => handle.toolSurface.includes(t))) @@ -63,6 +63,7 @@ test("tier0 agent driver replays a canned run that scores a schema-valid 16-line handoffPath: join(dir, "handoff.json"), scoreInputPath: join(dir, "score-input.json"), transcriptPath: join(dir, "transcript.json"), + pre1Path: join(dir, "pre1.json"), handoffInstructions: "", completionInstructions: "", } @@ -151,10 +152,94 @@ test("canned artifacts are grep-clean", () => { "be" + "ad", "CXF" + "-", ] - for (const name of ["transcript.json", "handoff.json", "score-input.json"]) { - const content = readFileSync(join(import.meta.dirname, name), "utf8") + // Walk the tier0 dir including the canned-* subdirectories. + const names = ["transcript.json", "handoff.json", "score-input.json", "pre1.json"] + const walk = (dir: string): string[] => { + const out: string[] = [] + for (const entry of readdirSync(dir, {withFileTypes: true})) { + const p = join(dir, entry.name) + if (entry.isDirectory()) out.push(...walk(p)) + else if (names.includes(entry.name)) out.push(p) + } + return out + } + const files = walk(import.meta.dirname) + assert.ok(files.length >= 6, `expected the fixed set + two canned sets, got ${files.join(", ")}`) + for (const file of files) { + const content = readFileSync(file, "utf8") for (const token of forbidden) { - assert.ok(!content.includes(token), `${name} contains forbidden token ${token}`) + assert.ok(!content.includes(token), `${file} contains forbidden token ${token}`) } } }) + +// --- pre-1 replays --- + +import {loadScenario} from "../../scenario.ts" +import type {ScoreResult} from "../../score.ts" + +function pre1Channel(dir: string): RunChannel { + return { + runDir: dir, + handoffPath: join(dir, "handoff.json"), + scoreInputPath: join(dir, "score-input.json"), + transcriptPath: join(dir, "transcript.json"), + pre1Path: join(dir, "pre1.json"), + handoffInstructions: "", + completionInstructions: "", + } +} + +async function replayPre1(scenarioPath: string, scenarioId: string): Promise<{scored: ScoreResult; pre1: Pre1Artifact}> { + const dir = mkdtempSync(join(tmpdir(), "tier0-pre1-")) + try { + const channel = pre1Channel(dir) + const result = await tier0.agentDriver.runAgent({ + kind: "agent", + prompt: "p", + toolSurface: TIER0_TOOL_SURFACE, + channel, + timeoutMs: 60_000, + model: "m", + ref: "", + scenarioId, + }) + assert.equal(result.timedOut, false) + const pre1 = JSON.parse(readFileSync(channel.pre1Path, "utf8")) as Pre1Artifact + const scenario = loadScenario(scenarioPath) + const ctx: StageCtx = { + transcript: result.transcript, + handoff: {}, + scoreInput: { + run_id: "pre1-canned", + draft: {required_source_files: {}, source_files: [], config_schema: {fields: []}, runtime_schema: {fields: []}}, + connector_config: {}, + evidence: {}, + build_run: {}, + tenant_counts: {users: null, groups: null, memberships: null}, + resource_ids: {users: [], groups: []}, + }, + handoffPath: channel.pre1Path, + kind: "pre1", + pre1, + expected: {decision: scenario.expectedDecision!, accessModel: scenario.expectedAccessModel, parkEvidence: scenario.expectedParkEvidence}, + } + return {scored: scoreRun(ctx), pre1} + } finally { + rmSync(dir, {recursive: true, force: true}) + } +} + +test("a pre1-directory-proceed replay writes pre1.json and scores P0-P3 pass", async () => { + const {scored} = await replayPre1("evals/scenarios/pre1-directory-proceed.json", "pre1-directory-proceed") + assert.equal(scored.decision_verdict, "proceed") + assert.deepEqual(scored.stageRows.map((r) => r.stage), ["P0", "P1", "P2", "P3"]) + assert.ok(scored.stageRows.every((r) => r.pass)) +}) + +test("a pre1-noiam-park replay writes pre1.json and scores P0/P1/P4 pass", async () => { + const {scored} = await replayPre1("evals/scenarios/pre1-noiam-park.json", "pre1-noiam-park") + assert.equal(scored.decision_verdict, "park") + assert.deepEqual(scored.stageRows.map((r) => r.stage), ["P0", "P1", "P4"]) + assert.ok(scored.stageRows.every((r) => r.pass)) +}) diff --git a/evals/runner/drivers/tier0/driver.ts b/evals/runner/drivers/tier0/driver.ts index f1fea57..db32bc6 100644 --- a/evals/runner/drivers/tier0/driver.ts +++ b/evals/runner/drivers/tier0/driver.ts @@ -1,6 +1,6 @@ // drivers/tier0/driver.ts — Tier-0 local/static driver (no credentials). import {spawn, type ChildProcess} from "node:child_process" -import {readFileSync, writeFileSync} from "node:fs" +import {existsSync, readFileSync, writeFileSync} from "node:fs" import {join} from "node:path" import {cwd} from "node:process" @@ -86,6 +86,23 @@ async function waitForFixture(baseUrl: string, expectedUsers: number): Promise for HTTP 200 (same 30 s deadline + 2 s probe +// timeout as waitForFixture). +async function waitForSpec(baseUrl: string, openapiPath: string): Promise { + const deadline = Date.now() + 30_000 + while (Date.now() < deadline) { + try { + const res = await fetch(baseUrl + openapiPath, {signal: AbortSignal.timeout(2_000)}) + if (res.status === 200) return + } catch { + /* server not up yet — retry */ + } + await new Promise((r) => setTimeout(r, 500)) + } + throw new ReadinessError(`spec not reachable at ${baseUrl}${openapiPath} within 30 s`) +} + const provisioner: Provisioner = { provision: async (ctx) => { const child = spawn(process.execPath, ["--experimental-strip-types", FIXTURE_SCRIPT, "--port", "0", "--host", "127.0.0.1"], {cwd: cwd(), stdio: ["ignore", "pipe", "ignore"]}) @@ -107,8 +124,19 @@ const provisioner: Provisioner = { child.kill() throw err } - const baseUrl = "http://127.0.0.1:" + port - return {baseUrl, credentials: {username: "connector@example.com", password: "fixture-token"}, toolSurface: TIER0_TOOL_SURFACE, meta: {child, port, expectedUsers: ctx.scenario.seed.users}} +const baseUrl = "http://127.0.0.1:" + port + return { + baseUrl, + credentials: {username: "connector@example.com", password: "fixture-token"}, + toolSurface: TIER0_TOOL_SURFACE, + meta: { + child, + port, + expectedUsers: ctx.scenario.seed?.users, + pre1: ctx.scenario.kind === "pre1", + openapiPath: ctx.scenario.fixture.openapiPath, + }, + } }, checkReadiness: async (handle) => { // Fail fast if the fixture child already exited (spawn error or crash) — @@ -117,6 +145,10 @@ const provisioner: Provisioner = { if (child && (child.exitCode !== null || child.signalCode !== null)) { throw new ReadinessError(`fixture exited before readiness (code ${child.exitCode ?? "signal " + child.signalCode})`) } + if (handle.meta?.pre1 === true) { + await waitForSpec(handle.baseUrl, (handle.meta?.openapiPath as string | undefined) ?? "/openapi.json") + return + } await waitForFixture(handle.baseUrl, (handle.meta?.expectedUsers as number | undefined) ?? 23) }, teardown: async (handle) => { @@ -136,7 +168,15 @@ const agentDriver: AgentDriver = { runAgent: async (req) => { const startedAt = Date.now() if (req.kind === "agent") { - const raw = JSON.parse(readCanned("transcript.json")) as Record[] + // Canned-dir selection: a scenarioId with a matching canned- dir + // next to the fixed set selects that set (pre-1 replays); otherwise the + // fixed funnel set is the default. + const cannedDir = + req.scenarioId !== undefined && existsSync(join(CANNED_DIR, `canned-${req.scenarioId}`)) + ? join(CANNED_DIR, `canned-${req.scenarioId}`) + : CANNED_DIR + const readCannedFrom = (name: string) => readFileSync(join(cannedDir, name), "utf8") + const raw = JSON.parse(readCannedFrom("transcript.json")) as Record[] // Deep copy: the substitution must never mutate the canned file. const events = JSON.parse(JSON.stringify(raw)) as Record[] for (const ev of events) { @@ -144,7 +184,14 @@ const agentDriver: AgentDriver = { if (input && typeof input.path === "string") input.path = input.path.replaceAll("", req.channel.runDir) } writeFileSync(req.channel.transcriptPath, JSON.stringify(events, null, 2)) - writeFileSync(req.channel.handoffPath, readCanned("handoff.json")) + if (existsSync(join(cannedDir, "pre1.json"))) { + writeFileSync(req.channel.pre1Path, readCannedFrom("pre1.json")) + } else { + writeFileSync(req.channel.handoffPath, readCannedFrom("handoff.json")) + } + if (existsSync(join(cannedDir, "score-input.json"))) { + writeFileSync(req.channel.scoreInputPath, readCannedFrom("score-input.json")) + } return {transcript: parseStream(events), timedOut: false, wallTimeMs: Date.now() - startedAt} } writeFileSync(req.channel.scoreInputPath, readCanned("score-input.json")) diff --git a/evals/runner/record.ts b/evals/runner/record.ts index 5626e1c..9c6941a 100644 --- a/evals/runner/record.ts +++ b/evals/runner/record.ts @@ -33,6 +33,9 @@ export interface SummaryLine { turns: number tokens_in: number | null tokens_out: number | null + // Pre-1 only: the separately-measured park-vs-proceed metric. + decision_verdict?: "proceed" | "park" | "incorrect" + decision_evidence?: string } // Run meta (locked D15): harness is inherited (no override); reasoning_effort diff --git a/evals/runner/run.test.ts b/evals/runner/run.test.ts index 0ea64d4..a215461 100644 --- a/evals/runner/run.test.ts +++ b/evals/runner/run.test.ts @@ -78,7 +78,7 @@ function makeDriver(provisioner: Provisioner): Driver { } function fullSurface(): string[] { - return [...FUNNEL_TOOLS, ...SCENARIO.readinessTools] + return [...FUNNEL_TOOLS, ...(SCENARIO.readinessTools ?? [])] } function makeChannel(dir: string): RunChannel { @@ -87,6 +87,7 @@ function makeChannel(dir: string): RunChannel { handoffPath: join(dir, "handoff.json"), scoreInputPath: join(dir, "score-input.json"), transcriptPath: join(dir, "transcript.json"), + pre1Path: join(dir, "pre1.json"), handoffInstructions: "", completionInstructions: "", } @@ -224,7 +225,7 @@ test("provisionWithRetry throws after 3 failed attempts, tearing down each handl }) test("provisionWithRetry throws ReadinessError when a readiness tool is missing from the declared surface", async () => { - const handle: TenantHandle = {baseUrl: "http://x", credentials: {}, toolSurface: SCENARIO.readinessTools.slice(0, 4)} + const handle: TenantHandle = {baseUrl: "http://x", credentials: {}, toolSurface: (SCENARIO.readinessTools ?? []).slice(0, 4)} const driver = makeDriver({ provision: async () => handle, checkReadiness: async () => {}, @@ -237,7 +238,7 @@ test("provisionWithRetry throws ReadinessError when a readiness tool is missing }) test("funnel_tools_present is false when the declared surface lacks funnel tools", async () => { - const handle: TenantHandle = {baseUrl: "http://x", credentials: {}, toolSurface: SCENARIO.readinessTools} + const handle: TenantHandle = {baseUrl: "http://x", credentials: {}, toolSurface: SCENARIO.readinessTools ?? []} const driver = makeDriver({ provision: async () => handle, checkReadiness: async () => {}, @@ -258,7 +259,7 @@ test("collectScoreInput returns the normalized score-input on success", async () return {transcript: emptyStream(), timedOut: false, wallTimeMs: 0} }, } - const {scoreInput} = await collectScoreInput(driver, SCENARIO, "r", channel, join(dir, "handoff-sanitized.json"), {}, SCENARIO.readinessTools, false, "", 1) + const {scoreInput} = await collectScoreInput(driver, SCENARIO, "r", channel, join(dir, "handoff-sanitized.json"), {}, SCENARIO.readinessTools ?? [], false, "", 1) assert.equal(scoreInput.evidence.result, "PASS") } finally { rmSync(dir, {recursive: true, force: true}) @@ -279,7 +280,7 @@ test("collectScoreInput retries once on a transient failure", async () => { return {transcript: emptyStream(), timedOut: false, wallTimeMs: 0} }, } - const {scoreInput} = await collectScoreInput(driver, SCENARIO, "r", channel, join(dir, "handoff-sanitized.json"), {}, SCENARIO.readinessTools, false, "", 1) + const {scoreInput} = await collectScoreInput(driver, SCENARIO, "r", channel, join(dir, "handoff-sanitized.json"), {}, SCENARIO.readinessTools ?? [], false, "", 1) assert.equal(calls, 2) assert.equal(scoreInput.evidence.result, "PASS") } finally { @@ -308,7 +309,7 @@ test("collectScoreInput rethrows after 2 failures when the handoff is complete", deployment_instance_id: "di", activation_url: "https://x", } - await assert.rejects(collectScoreInput(driver, SCENARIO, "r", channel, join(dir, "handoff-sanitized.json"), fullHandoff, SCENARIO.readinessTools, true, "", 1), /collector down/) + await assert.rejects(collectScoreInput(driver, SCENARIO, "r", channel, join(dir, "handoff-sanitized.json"), fullHandoff, SCENARIO.readinessTools ?? [], true, "", 1), /collector down/) } finally { rmSync(dir, {recursive: true, force: true}) } @@ -335,7 +336,7 @@ test("collectScoreInput rethrows after 2 failures on a partial handoff", async ( deployment_instance_id: "", activation_url: "", } - await assert.rejects(collectScoreInput(driver, SCENARIO, "r", channel, join(dir, "handoff-sanitized.json"), partialHandoff, SCENARIO.readinessTools, false, "", 1), /collector down/) + await assert.rejects(collectScoreInput(driver, SCENARIO, "r", channel, join(dir, "handoff-sanitized.json"), partialHandoff, SCENARIO.readinessTools ?? [], false, "", 1), /collector down/) } finally { rmSync(dir, {recursive: true, force: true}) } @@ -350,7 +351,7 @@ test("collectScoreInput returns a null score-input on the stalled path (absent h throw new Error("collector down") }, } - const {scoreInput} = await collectScoreInput(driver, SCENARIO, "r", channel, join(dir, "handoff-sanitized.json"), {}, SCENARIO.readinessTools, false, "", 1) + const {scoreInput} = await collectScoreInput(driver, SCENARIO, "r", channel, join(dir, "handoff-sanitized.json"), {}, SCENARIO.readinessTools ?? [], false, "", 1) assert.deepEqual(scoreInput.draft.required_source_files, {}) assert.equal(scoreInput.tenant_counts.users, null) } finally { @@ -418,7 +419,7 @@ test("collectScoreInput redacts a malformed score-input parse error", async () = activation_url: "https://x", } await assert.rejects( - collectScoreInput(driver, SCENARIO, "r", channel, join(dir, "handoff-sanitized.json"), fullHandoff, SCENARIO.readinessTools, true, "", 1), + collectScoreInput(driver, SCENARIO, "r", channel, join(dir, "handoff-sanitized.json"), fullHandoff, SCENARIO.readinessTools ?? [], true, "", 1), (err: unknown) => err instanceof Error && err.message.includes("unreadable score-input") && !err.message.includes("super-secret-value"), ) } finally { diff --git a/evals/runner/run.ts b/evals/runner/run.ts index 80ee4e8..d3e483e 100644 --- a/evals/runner/run.ts +++ b/evals/runner/run.ts @@ -11,7 +11,7 @@ import {loadScenario, type Scenario} from "./scenario.ts" import {FUNNEL_TOOLS, ReadinessError, type AgentDriver, type AgentRunResult, type Driver, type RunChannel, type TenantHandle} from "./driver.ts" import {buildPrompt} from "./agent.ts" import {buildCollectorPrompt, normalizeScoreInput} from "./collect.ts" -import {SKIPPED_STAGES, STAGES, handoffEmpty, sanitizeHandoff, type Handoff, type ScoreInput, type StageCtx} from "./stages.ts" +import {SKIPPED_STAGES, STAGES, handoffEmpty, sanitizeHandoff, type Handoff, type Pre1Artifact, type ScoreInput, type StageCtx} from "./stages.ts" import {scoreRun} from "./score.ts" import {buildRunMeta, writeRecord, type RunMeta, type SummaryLine} from "./record.ts" import {tier0} from "./drivers/tier0/driver.ts" @@ -158,7 +158,41 @@ export async function readHandoff(handoffPath: string): Promise } } } - stderr.write(`WARNING: handoff read failed after 3 attempts: ${(lastErr as Error).message} — scoring as stalled\n`) +stderr.write(`WARNING: handoff read failed after 3 attempts: ${(lastErr as Error).message} — scoring as stalled\n`) + return null +} + +// The pre1 artifact is agent-written and untrusted: same bounded 3-attempt +// retry + 64 MiB size cap + JSON-parse + non-object-returns-null semantics as +// readHandoff. Returns null on missing/oversized/malformed/non-object (P0 +// then fails). +export async function readPre1Artifact(path: string): Promise { + let lastErr: unknown + for (let attempt = 1; attempt <= 3; attempt++) { + try { + if (statSync(path).size > MAX_HANDOFF_BYTES) { + stderr.write(`WARNING: pre1 artifact at ${path} exceeds ${MAX_HANDOFF_BYTES} bytes — scoring as absent\n`) + return null + } + const content = readFileSync(path, "utf8") + if (content.length === 0) return null + let parsed: unknown + try { + parsed = JSON.parse(content) + } catch { + stderr.write(`WARNING: pre1 artifact at ${path} is not valid JSON — scoring as absent\n`) + return null + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null + return parsed as Pre1Artifact + } catch (err) { + lastErr = err + if (attempt < 3) { + await new Promise((resolve) => setTimeout(resolve, 2000)) + } + } + } +stderr.write(`WARNING: pre1 artifact read failed after 3 attempts: ${(lastErr as Error).message} — scoring as absent\n`) return null } @@ -171,11 +205,12 @@ function buildChannel(out: string, runId: string): RunChannel { // and strict-equality-compared in the S11 gate — they must not be // cwd-dependent for a private driver whose agent runs elsewhere. const runDir = join(resolve(out), runId) - return { +return { runDir, handoffPath: join(runDir, "handoff.json"), scoreInputPath: join(runDir, "score-input.json"), transcriptPath: join(runDir, "transcript.json"), + pre1Path: join(runDir, "pre1.json"), handoffInstructions: "", completionInstructions: "", } @@ -198,7 +233,7 @@ export async function provisionWithRetry( const h = await driver.provisioner.provision({scenario, runId, ref}) handle = h await driver.provisioner.checkReadiness(h) - const missing = scenario.readinessTools.filter((t) => !h.toolSurface.includes(t)) + const missing = (scenario.readinessTools ?? []).filter((t) => !h.toolSurface.includes(t)) if (missing.length > 0) throw new ReadinessError("missing readiness tools: " + missing.join(", ")) const funnelToolsPresent = FUNNEL_TOOLS.every((t) => h.toolSurface.includes(t)) return {handle: h, funnelToolsPresent} @@ -327,7 +362,7 @@ async function main(): Promise { try { const prompt = buildPrompt(scenario, runId, handle.baseUrl, channel) - const result = await driver.agentDriver.runAgent({ +const result = await driver.agentDriver.runAgent({ kind: "agent", prompt, toolSurface: handle.toolSurface, @@ -336,6 +371,7 @@ async function main(): Promise { model: scenario.model, reasoningEffort: scenario.reasoningEffort, ref: cli.ref, + scenarioId: scenario.id, }) const {transcript, timedOut, wallTimeMs} = result if (timedOut) { @@ -345,10 +381,62 @@ async function main(): Promise { // infrastructure outage, not an agent outcome — fail loudly with no // record rather than scoring an all-fail funnel. A genuine zero-tool-call // stall (no collectionFailed signal) stays a scored exit-0 outcome. - if (isCollectionFailure(result)) { +if (isCollectionFailure(result)) { throw new Error("agent driver reported a stream collection failure with an empty transcript — no record written") } + // Pre-1 runs: read the pre1 artifact, skip the collector leg entirely, + // and score the P0..P4 gate set. The empty ScoreInput literal is the + // same shape the stalled-path fallback in collectScoreInput uses. + if (scenario.kind === "pre1") { + const pre1 = await readPre1Artifact(channel.pre1Path) + const scoreInput: ScoreInput = { + run_id: runId, + draft: {required_source_files: {}, source_files: [], config_schema: {fields: []}, runtime_schema: {fields: []}}, + connector_config: {}, + evidence: {}, + build_run: {}, + tenant_counts: {users: null, groups: null, memberships: null}, + resource_ids: {users: [], groups: []}, + } + const ctx: StageCtx = { + transcript, + handoff: {}, + scoreInput, + handoffPath: channel.pre1Path, + kind: "pre1", + pre1, + expected: {decision: scenario.expectedDecision!, accessModel: scenario.expectedAccessModel, parkEvidence: scenario.expectedParkEvidence}, + } + const scored = scoreRun(ctx) + const stageRows = scored.stageRows + const meta: RunMeta = buildRunMeta(runId, scenario, driver.name, startedAt, wallTimeMs, funnelToolsPresent) + const summary: SummaryLine = { + summary: true, + funnel: stageRows.filter((r) => r.pass).map((r) => r.stage), + first_pass_rate: scored.first_pass_rate, + recovery_cycles: scored.recovery_cycles, + parity_verdict: scored.parity_verdict, + parity_evidence: scored.parity_evidence, + parity_tenant: scored.parity_tenant, + parity_tenant_evidence: scored.parity_tenant_evidence, + hygiene_verdict: scored.hygiene_verdict, + hygiene_evidence: scored.hygiene_evidence, + handoff_discipline_verdict: scored.handoff_discipline_verdict, + tool_calls: transcript.toolCalls.length, + turns: transcript.turns, + tokens_in: transcript.tokensIn, + tokens_out: transcript.tokensOut, + decision_verdict: scored.decision_verdict, + decision_evidence: scored.decision_evidence, + } + const recordPath = writeRecord(runId, scenario, meta, stageRows, [], summary, cli.out) + const passList = stageRows.filter((r) => r.pass).map((r) => r.stage).join(",") + stdout.write(`record: ${recordPath}\n`) + stdout.write(`summary: funnel=[${passList}] first_pass_rate=${summary.first_pass_rate.toFixed(2)} decision=${summary.decision_verdict ?? "n/a"} tool_calls=${summary.tool_calls} turns=${summary.turns} tokens_in=${summary.tokens_in} tokens_out=${summary.tokens_out}\n`) + return 0 + } + // Handoff: missing/incomplete -> L18 stalled-agent path. const handoff = (await readHandoff(channel.handoffPath)) ?? {} const handoffOk = handoffComplete(handoff) diff --git a/evals/runner/scenario.test.ts b/evals/runner/scenario.test.ts index 7b4bb71..66d14c9 100644 --- a/evals/runner/scenario.test.ts +++ b/evals/runner/scenario.test.ts @@ -14,20 +14,20 @@ test("loadScenario loads the real tier1-directory.json", () => { assert.equal(s.fixture.basicAuth.username, "connector@example.com") assert.equal(s.fixture.basicAuth.password, "fixture-token") assert.equal(s.fixture.bearerToken, "fixture-token") - assert.equal(s.seed.users, 23) - assert.equal(s.seed.groups, 5) - assert.equal(s.seed.memberships, 23) - assert.equal(s.seed.nullTitleUsers, 3) - assert.equal(s.seed.unscopedSubset, 3) - assert.equal(s.seed.disabledUsers, 2) - assert.equal(s.expected.users, 23) - assert.equal(s.expected.groups, 5) - assert.equal(s.expected.memberships, 23) + assert.equal(s.seed!.users, 23) + assert.equal(s.seed!.groups, 5) + assert.equal(s.seed!.memberships, 23) + assert.equal(s.seed!.nullTitleUsers, 3) + assert.equal(s.seed!.unscopedSubset, 3) + assert.equal(s.seed!.disabledUsers, 2) + assert.equal(s.expected!.users, 23) + assert.equal(s.expected!.groups, 5) + assert.equal(s.expected!.memberships, 23) assert.equal(s.skillBundle.mode, "none") assert.equal(s.model, "together/deepseek-ai/DeepSeek-V4-Flash-0731") assert.equal(s.reasoningEffort, "high") - assert.equal(s.requiredSourceFiles.length, 4) - assert.equal(s.readinessTools.length, 5) + assert.equal(s.requiredSourceFiles!.length, 4) + assert.equal(s.readinessTools!.length, 5) }) test("loadScenario loads the real tier1-directory-guide-only.json", () => { @@ -76,3 +76,109 @@ test("loadScenario rejects an unsafe scenario id (path traversal)", () => { rmSync(dir, {recursive: true, force: true}) } }) + +// --- pre1 kind --- + +function pre1Base(): Record { + return JSON.parse(readFileSync("evals/scenarios/pre1-directory-proceed.json", "utf8")) as Record +} + +function writePre1(dir: string, name: string, data: Record): string { + const p = join(dir, name) + writeFileSync(p, JSON.stringify(data)) + return p +} + +test("loadScenario loads the pre1-directory-proceed scenario (kind pre1, proceed)", () => { + const s = loadScenario("evals/scenarios/pre1-directory-proceed.json") + assert.equal(s.kind, "pre1") + assert.equal(s.expectedDecision, "proceed") + assert.ok(s.expectedAccessModel) + const ids = s.expectedAccessModel.resource_types.map((rt) => rt.id) + assert.ok(ids.includes("user")) + assert.ok(ids.includes("group")) + assert.equal(s.seed, undefined) + assert.equal(s.requiredSourceFiles, undefined) + assert.equal(s.skillBundle.mode, "full") + assert.equal(s.skillBundle.version, "0.3.0") +}) + +test("loadScenario loads the pre1-noiam-park scenario (kind pre1, park)", () => { + const s = loadScenario("evals/scenarios/pre1-noiam-park.json") + assert.equal(s.kind, "pre1") + assert.equal(s.expectedDecision, "park") + assert.ok(s.expectedParkEvidence) + assert.ok(s.expectedParkEvidence.missing_paths.length > 0) + assert.equal(s.seed, undefined) +}) + +test("loadScenario rejects a pre1 scenario carrying a funnel-only seed", () => { + const dir = mkdtempSync(join(tmpdir(), "evals-scenario-")) + try { + const bad = pre1Base() + bad.seed = {users: 1, groups: 1, memberships: 1, nullTitleUsers: 0, unscopedSubset: 0, disabledUsers: 0} + assert.throws(() => loadScenario(writePre1(dir, "bad.json", bad)), /funnel-only/) + } finally { + rmSync(dir, {recursive: true, force: true}) + } +}) + +test("loadScenario rejects a pre1 scenario with BOTH expected halves", () => { + const dir = mkdtempSync(join(tmpdir(), "evals-scenario-")) + try { + const bad = pre1Base() + bad.expectedParkEvidence = { + spec_version_checked: "1.0.0", + missing_paths: ["/v1/users"], + vendor_doc: "console only", + revisit_trigger: "ships an API", + } + assert.throws(() => loadScenario(writePre1(dir, "bad.json", bad)), /exactly one/) + } finally { + rmSync(dir, {recursive: true, force: true}) + } +}) + +test("loadScenario rejects a pre1 scenario with NEITHER expected half", () => { + const dir = mkdtempSync(join(tmpdir(), "evals-scenario-")) + try { + const bad = pre1Base() + delete bad.expectedAccessModel + assert.throws(() => loadScenario(writePre1(dir, "bad.json", bad)), /exactly one/) + } finally { + rmSync(dir, {recursive: true, force: true}) + } +}) + +test("loadScenario rejects an invalid pre1 expectedDecision", () => { + const dir = mkdtempSync(join(tmpdir(), "evals-scenario-")) + try { + const bad = pre1Base() + bad.expectedDecision = "maybe" + assert.throws(() => loadScenario(writePre1(dir, "bad.json", bad)), /expectedDecision/) + } finally { + rmSync(dir, {recursive: true, force: true}) + } +}) + +test("loadScenario rejects a pre1 scenario missing providerBrief", () => { + const dir = mkdtempSync(join(tmpdir(), "evals-scenario-")) + try { + const bad = pre1Base() + delete bad.providerBrief + assert.throws(() => loadScenario(writePre1(dir, "bad.json", bad)), /providerBrief/) + } finally { + rmSync(dir, {recursive: true, force: true}) + } +}) + +test("the existing funnel scenarios still load with kind funnel", () => { + for (const p of ["evals/scenarios/tier1-directory.json", "evals/scenarios/tier1-directory-guide-only.json", "evals/scenarios/tier1-directory-full.json"]) { + const s = loadScenario(p) + assert.equal(s.kind, "funnel") + assert.ok(s.seed) + assert.ok(s.expected) + assert.equal(s.requiredSourceFiles?.length, 4) + assert.equal(s.readinessTools?.length, 5) + } +}) diff --git a/evals/runner/scenario.ts b/evals/runner/scenario.ts index eb72c6e..95c6dce 100644 --- a/evals/runner/scenario.ts +++ b/evals/runner/scenario.ts @@ -31,17 +31,41 @@ export interface SkillBundleConfig { version: string } +export interface ExpectedAccessModel { + resource_types: {id: string; traits: string[]}[] + entitlements: {slug: string}[] + grants: {resource_type: string; entitlement: string; principal_type: string}[] +} + +export interface ExpectedParkEvidence { + spec_version_checked: string + missing_paths: string[] + vendor_doc: string + revisit_trigger: string +} + export interface Scenario { id: string name: string fixture: FixtureConfig - seed: SeedConfig - expected: ExpectedConfig + // kind is optional in the interface so the existing SCENARIO literals in + // the test files compile unchanged; loadScenario always returns it + // (default "funnel"). + kind?: "funnel" | "pre1" + // Funnel-only fields are optional in the interface for the same reason; + // loadScenario returns them for funnel and omits them for pre1. + seed?: SeedConfig + expected?: ExpectedConfig skillBundle: SkillBundleConfig model: string reasoningEffort: "high" | "medium" | "low" - requiredSourceFiles: string[] - readinessTools: string[] + requiredSourceFiles?: string[] + readinessTools?: string[] + // Pre-1 fields (kind === "pre1" only). + providerBrief?: string + expectedDecision?: "proceed" | "park" + expectedAccessModel?: ExpectedAccessModel + expectedParkEvidence?: ExpectedParkEvidence } // Canonical record guard for the evals/runner package (no shared type-guard @@ -111,23 +135,119 @@ const id = requireString(data, "id", "scenario") username: requireString(fixture.basicAuth, "username", "fixture.basicAuth"), password: requireString(fixture.basicAuth, "password", "fixture.basicAuth"), } - const bearerToken = requireString(fixture, "bearerToken", "fixture") +const bearerToken = requireString(fixture, "bearerToken", "fixture") - if (!isRecord(data.seed)) throw new Error("scenario field seed missing or not an object") - const seed = { - users: requireNumber(data.seed, "users", "seed"), - groups: requireNumber(data.seed, "groups", "seed"), - memberships: requireNumber(data.seed, "memberships", "seed"), - nullTitleUsers: requireNumber(data.seed, "nullTitleUsers", "seed"), - unscopedSubset: requireNumber(data.seed, "unscopedSubset", "seed"), - disabledUsers: requireNumber(data.seed, "disabledUsers", "seed"), + const kind = data.kind === undefined ? "funnel" : data.kind + if (kind !== "funnel" && kind !== "pre1") { + throw new Error('scenario field kind must be "funnel" or "pre1"') } - if (!isRecord(data.expected)) throw new Error("scenario field expected missing or not an object") - const expected = { - users: requireNumber(data.expected, "users", "expected"), - groups: requireNumber(data.expected, "groups", "expected"), - memberships: requireNumber(data.expected, "memberships", "expected"), + if (kind === "funnel") { + // Funnel validation — VERBATIM (same field checks, same error messages, + // same order; funnel behavior is byte-identical). + if (!isRecord(data.seed)) throw new Error("scenario field seed missing or not an object") + const seed = { + users: requireNumber(data.seed, "users", "seed"), + groups: requireNumber(data.seed, "groups", "seed"), + memberships: requireNumber(data.seed, "memberships", "seed"), + nullTitleUsers: requireNumber(data.seed, "nullTitleUsers", "seed"), + unscopedSubset: requireNumber(data.seed, "unscopedSubset", "seed"), + disabledUsers: requireNumber(data.seed, "disabledUsers", "seed"), + } + + if (!isRecord(data.expected)) throw new Error("scenario field expected missing or not an object") + const expected = { + users: requireNumber(data.expected, "users", "expected"), + groups: requireNumber(data.expected, "groups", "expected"), + memberships: requireNumber(data.expected, "memberships", "expected"), + } + + if (!isRecord(data.skillBundle)) throw new Error("scenario field skillBundle missing or not an object") + const mode = data.skillBundle.mode + if (mode !== "none" && mode !== "guide-only" && mode !== "full") { + throw new Error('scenario field skillBundle.mode must be "none", "guide-only", or "full"') + } + const skillBundle = { + mode: mode as "none" | "guide-only" | "full", + version: requireString(data.skillBundle, "version", "skillBundle"), + } + if (skillBundle.mode === "full") { + // Fail fast when the scenario's bundle version drifts from the mounted + // bundle: run records must never be stamped with a version that does not + // match the skills actually mounted (bundle.json is the single source). + // Resolve relative to this module so the check is not cwd-dependent, and + // wrap read/parse errors like the scenario file's own. + const bundlePath = fileURLToPath(new URL("../skills-bundle/bundle.json", import.meta.url)) + let bundleRaw: string + try { + bundleRaw = readFileSync(bundlePath, "utf8") + } catch (err) { + throw new Error(`cannot read skill bundle ${bundlePath}: ${(err as Error).message}`) + } + let bundle: {version?: unknown} + try { + bundle = JSON.parse(bundleRaw) as {version?: unknown} + } catch (err) { + throw new Error(`skill bundle ${bundlePath} is not valid JSON: ${(err as Error).message}`) + } + if (typeof bundle.version !== "string" || bundle.version !== skillBundle.version) { + throw new Error( + `scenario skillBundle.version ${skillBundle.version} does not match evals/skills-bundle/bundle.json version ${bundle.version}`, + ) + } + } + + const reasoningEffort = data.reasoningEffort + if (reasoningEffort !== "high" && reasoningEffort !== "medium" && reasoningEffort !== "low") { + throw new Error('scenario field reasoningEffort must be "high", "medium", or "low"') + } + + const model = requireString(data, "model", "scenario") + const requiredSourceFiles = requireStringArray(data, "requiredSourceFiles", "scenario") + if (requiredSourceFiles.length !== 4) { + throw new Error("scenario field requiredSourceFiles must have exactly 4 entries") + } + const readinessTools = requireStringArray(data, "readinessTools", "scenario") + if (readinessTools.length !== 5) { + throw new Error("scenario field readinessTools must have exactly 5 entries") + } + + return { + id, + name, + fixture: {port, baseUrl, auth, openapiPath, basicAuth, bearerToken}, + seed, + expected, + skillBundle, + model, + reasoningEffort: reasoningEffort as "high" | "medium" | "low", + requiredSourceFiles, + readinessTools, + kind: "funnel", + } + } + + // --- pre1 branch --- + const providerBrief = requireString(data, "providerBrief", "scenario") + const expectedDecision = data.expectedDecision + if (expectedDecision !== "proceed" && expectedDecision !== "park") { + throw new Error('scenario field expectedDecision must be "proceed" or "park"') + } + const hasAccessModel = data.expectedAccessModel !== undefined + const hasParkEvidence = data.expectedParkEvidence !== undefined + if (hasAccessModel === hasParkEvidence) { + throw new Error("scenario field expectedAccessModel/expectedParkEvidence: exactly one must be present for kind pre1") + } + for (const key of ["seed", "expected", "requiredSourceFiles"]) { + if (data[key] !== undefined) { + throw new Error(`scenario field ${key} is funnel-only and must be absent for kind pre1`) + } + } + // readinessTools is optional for pre1: absent -> [], present tolerated and + // validated as a string array. + let readinessTools: string[] = [] + if (data.readinessTools !== undefined) { + readinessTools = requireStringArray(data, "readinessTools", "scenario") } if (!isRecord(data.skillBundle)) throw new Error("scenario field skillBundle missing or not an object") @@ -140,11 +260,6 @@ const id = requireString(data, "id", "scenario") version: requireString(data.skillBundle, "version", "skillBundle"), } if (skillBundle.mode === "full") { - // Fail fast when the scenario's bundle version drifts from the mounted - // bundle: run records must never be stamped with a version that does not - // match the skills actually mounted (bundle.json is the single source). - // Resolve relative to this module so the check is not cwd-dependent, and - // wrap read/parse errors like the scenario file's own. const bundlePath = fileURLToPath(new URL("../skills-bundle/bundle.json", import.meta.url)) let bundleRaw: string try { @@ -171,25 +286,74 @@ const id = requireString(data, "id", "scenario") } const model = requireString(data, "model", "scenario") - const requiredSourceFiles = requireStringArray(data, "requiredSourceFiles", "scenario") - if (requiredSourceFiles.length !== 4) { - throw new Error("scenario field requiredSourceFiles must have exactly 4 entries") + + // Validate the expected-shape halves (defensive: a malformed scenario must + // fail at load time, not at score time). + let expectedAccessModel: ExpectedAccessModel | undefined + let expectedParkEvidence: ExpectedParkEvidence | undefined + if (hasAccessModel) { + const am = data.expectedAccessModel as Record + if (!isRecord(am)) throw new Error("scenario field expectedAccessModel missing or not an object") + if (!Array.isArray(am.resource_types) || am.resource_types.length === 0) { + throw new Error("scenario field expectedAccessModel.resource_types must be a non-empty array") + } + for (const rt of am.resource_types) { + if (!isRecord(rt)) throw new Error("scenario field expectedAccessModel.resource_types entry must be an object") + if (typeof rt.id !== "string" || rt.id.length === 0) { + throw new Error("scenario field expectedAccessModel.resource_types entry id missing or not a non-empty string") + } + if (!Array.isArray(rt.traits) || rt.traits.length === 0 || rt.traits.some((t) => typeof t !== "string" || t.length === 0)) { + throw new Error("scenario field expectedAccessModel.resource_types entry traits missing or not a non-empty string array") + } + } + if (!Array.isArray(am.entitlements) || am.entitlements.length === 0) { + throw new Error("scenario field expectedAccessModel.entitlements must be a non-empty array") + } + for (const ent of am.entitlements) { + if (!isRecord(ent)) throw new Error("scenario field expectedAccessModel.entitlements entry must be an object") + if (typeof ent.slug !== "string" || ent.slug.length === 0) { + throw new Error("scenario field expectedAccessModel.entitlements entry slug missing or not a non-empty string") + } + } + if (!Array.isArray(am.grants) || am.grants.length === 0) { + throw new Error("scenario field expectedAccessModel.grants must be a non-empty array") + } + for (const g of am.grants) { + if (!isRecord(g)) throw new Error("scenario field expectedAccessModel.grants entry must be an object") + for (const key of ["resource_type", "entitlement", "principal_type"]) { + if (typeof g[key] !== "string" || (g[key] as string).length === 0) { + throw new Error(`scenario field expectedAccessModel.grants entry ${key} missing or not a non-empty string`) + } + } + } + expectedAccessModel = am as unknown as ExpectedAccessModel } -const readinessTools = requireStringArray(data, "readinessTools", "scenario") - if (readinessTools.length !== 5) { - throw new Error("scenario field readinessTools must have exactly 5 entries") + if (hasParkEvidence) { + const pe = data.expectedParkEvidence as Record + if (!isRecord(pe)) throw new Error("scenario field expectedParkEvidence missing or not an object") + for (const key of ["spec_version_checked", "vendor_doc", "revisit_trigger"]) { + if (typeof pe[key] !== "string" || (pe[key] as string).length === 0) { + throw new Error(`scenario field expectedParkEvidence.${key} missing or not a non-empty string`) + } + } + if (!Array.isArray(pe.missing_paths) || pe.missing_paths.length === 0 || pe.missing_paths.some((p) => typeof p !== "string" || p.length === 0)) { + throw new Error("scenario field expectedParkEvidence.missing_paths missing or not a non-empty string array") + } + expectedParkEvidence = pe as unknown as ExpectedParkEvidence } return { id, name, fixture: {port, baseUrl, auth, openapiPath, basicAuth, bearerToken}, - seed, - expected, skillBundle, model, reasoningEffort: reasoningEffort as "high" | "medium" | "low", - requiredSourceFiles, readinessTools, + kind: "pre1", + providerBrief, + expectedDecision: expectedDecision as "proceed" | "park", + expectedAccessModel, + expectedParkEvidence, } } diff --git a/evals/runner/score.test.ts b/evals/runner/score.test.ts index 62cd96b..59de5c4 100644 --- a/evals/runner/score.test.ts +++ b/evals/runner/score.test.ts @@ -213,3 +213,102 @@ test("parity fails when the connector source is unavailable", () => { const result = scoreRun(ctx({scoreInput: noSource})) assert.equal(result.parity_verdict, "FAIL") }) + +// --- pre-1 scoring --- + +import {PRE1_STAGES, type Pre1Artifact} from "./stages.ts" + +function pre1Artifact(overrides: Partial): Pre1Artifact { + return { + decision: "proceed", + access_model: { + resource_types: [ + {id: "user", traits: ["TRAIT_USER"]}, + {id: "group", traits: ["TRAIT_GROUP"]}, + ], + entitlements: [{slug: "member"}], + grants: [{resource_type: "group", entitlement: "member", principal_type: "user"}], + id_compatibility: [{resource_type: "user", id_shape: "user:{id}", stable: true}], + provisioning: [ + {resource_type: "user", provisionable: false, justification: "because the API lacks a user create/update endpoint"}, + ], + }, + sourcing: { + spec_url: "http://127.0.0.1:18080/openapi.json", + fetched_at: "2026-09-04T00:00:00.000Z", + authority_rung: "official published spec at a stable URL", + spec_bytes: 12297, + }, + ...overrides, + } +} + +function pre1Ctx(overrides: Partial): StageCtx { + return { + transcript: cleanTranscript(), + handoff: {}, + scoreInput: goodScoreInput(), + handoffPath: "/tmp/evals-run/pre1.json", + kind: "pre1", + pre1: pre1Artifact({}), + expected: { + decision: "proceed", + accessModel: { + resource_types: [ + {id: "user", traits: ["TRAIT_USER"]}, + {id: "group", traits: ["TRAIT_GROUP"]}, + ], + entitlements: [{slug: "member"}], + grants: [{resource_type: "group", entitlement: "member", principal_type: "user"}], + }, + }, + ...overrides, + } +} + +test("a correct proceed pre1 run scores decision_verdict proceed with P0-P3 passing", () => { + const result = scoreRun(pre1Ctx({})) + assert.equal(result.decision_verdict, "proceed") + assert.ok(result.decision_evidence?.includes("decision=proceed")) + assert.deepEqual(result.stageRows.map((r) => r.stage), ["P0", "P1", "P2", "P3"]) + assert.ok(result.stageRows.every((r) => r.pass)) + assert.deepEqual(result.funnel, ["P0", "P1", "P2", "P3"]) + assert.equal(result.first_pass_rate, 1.0) + assert.equal(result.parity_verdict, "PASS") + assert.equal(result.hygiene_verdict, "PASS") + assert.equal(result.handoff_discipline_verdict, true) + assert.equal(result.recovery_cycles, 0) +}) + +test("a correct park pre1 run scores decision_verdict park with P0/P1/P4 passing", () => { + const pre1 = pre1Artifact({ + decision: "park", + park_evidence: { + spec_version_checked: "1.2.0", + missing_paths: ["/v1/users", "/v1/groups"], + vendor_doc: "console only", + revisit_trigger: "ships an API", + }, + }) + const result = scoreRun(pre1Ctx({pre1, expected: {decision: "park", parkEvidence: {spec_version_checked: "1.2.0", missing_paths: ["/v1/users"], vendor_doc: "console only", revisit_trigger: "ships an API"}}})) + assert.equal(result.decision_verdict, "park") + assert.deepEqual(result.stageRows.map((r) => r.stage), ["P0", "P1", "P4"]) + assert.ok(result.stageRows.every((r) => r.pass)) + assert.deepEqual(result.funnel, ["P0", "P1", "P4"]) +}) + +test("a wrong pre1 decision scores decision_verdict incorrect with P1 failing", () => { + const pre1 = pre1Artifact({decision: "park"}) + const result = scoreRun(pre1Ctx({pre1})) + assert.equal(result.decision_verdict, "incorrect") + const p1 = result.stageRows.find((r) => r.stage === "P1") + assert.equal(p1?.pass, false) +}) + +test("a funnel ctx produces no decision_verdict field", () => { + const result = scoreRun(ctx({})) + assert.equal("decision_verdict" in result, false) + assert.equal("decision_evidence" in result, false) + assert.equal(result.stageRows.length, STAGES.length) + assert.deepEqual(result.funnel, ["S0", "S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8", "S9", "S10", "S11"]) +}) diff --git a/evals/runner/score.ts b/evals/runner/score.ts index 0c42cec..581395d 100644 --- a/evals/runner/score.ts +++ b/evals/runner/score.ts @@ -1,5 +1,5 @@ // score.ts — deterministic scorer. -import {SKIPPED_STAGES, STAGES, handoffEmpty, type StageCtx} from "./stages.ts" +import {PRE1_STAGES, SKIPPED_STAGES, STAGES, handoffEmpty, type StageCtx} from "./stages.ts" export interface StageRow { stage: string @@ -22,6 +22,9 @@ export interface ScoreResult { recovery_cycles: number first_pass_rate: number funnel: string[] + // Pre-1 only: the separately-measured park-vs-proceed metric. + decision_verdict?: "proceed" | "park" | "incorrect" + decision_evidence?: string } const MAX_FILE_BYTES = 12 * 1024 * 1024 @@ -161,6 +164,43 @@ function computeHygiene(ctx: StageCtx): {verdict: "PASS" | "FAIL"; evidence: str } export function scoreRun(ctx: StageCtx): ScoreResult { + // Pre-1 runs score the P0..P4 gate set (PRE1_STAGES) instead of the funnel. + // The park-vs-proceed metric is the P1 row: decision_verdict is the + // expected decision when P1 passes, "incorrect" otherwise. + if (ctx.kind === "pre1") { + const rows = PRE1_STAGES[ctx.expected?.decision ?? "proceed"] + const stageRows: StageRow[] = rows.map((s) => { + const pass = s.check(ctx) + return { + stage: s.stage, + gate: s.gate, + pass, + first_pass: pass, + attempts: pass ? 1 : 0, + evidence: s.evidence(ctx), + } + }) + const p1 = stageRows.find((r) => r.stage === "P1") + const decisionVerdict: "proceed" | "park" | "incorrect" = + p1 !== undefined && p1.pass && ctx.expected?.decision !== undefined ? ctx.expected.decision : "incorrect" + const passes = stageRows.filter((r) => r.pass).length + return { + stageRows, + parity_verdict: "PASS", + parity_evidence: "not applicable (pre1 run)", + parity_tenant: "not_applicable", + parity_tenant_evidence: "", + hygiene_verdict: "PASS", + hygiene_evidence: "not applicable (pre1 run)", + handoff_discipline_verdict: true, + recovery_cycles: 0, + first_pass_rate: stageRows.length > 0 ? passes / stageRows.length : 0, + funnel: stageRows.filter((r) => r.pass).map((r) => r.stage), + decision_verdict: decisionVerdict, + decision_evidence: p1?.evidence ?? "", + } + } + // L18 stalled-agent path: when the handoff is COMPLETELY absent, S1..S10 // are force-failed with the locked evidence (the agent never reached them — // the handoff is the funnel's ledger). A PARTIAL handoff is scored from diff --git a/evals/runner/skills_bundle.test.ts b/evals/runner/skills_bundle.test.ts index 32aa369..e727218 100644 --- a/evals/runner/skills_bundle.test.ts +++ b/evals/runner/skills_bundle.test.ts @@ -14,8 +14,19 @@ import {loadScenario} from "./scenario.ts" const execFileAsync = promisify(execFile) const RUN = "evals/runner/run.ts" const BUNDLE = "evals/skills-bundle/bundle.json" -const SKILLS = ["author-in-app-connector", "read-authoring-contract", "write-connector-source", "build-and-test", "deploy-and-activate"] -const VERSION = "0.2.0" +const SKILLS = ["author-in-app-connector", "read-authoring-contract", "write-connector-source", "build-and-test", "deploy-and-activate", "design-access-model", "source-openapi-spec"] +const VERSION = "0.3.0" +// The plan's locked versions are intentionally non-uniform: the two new +// pre-1 skills ship at 0.1.0 while the funnel skills keep their versions. +const SKILL_VERSIONS: Record = { + "author-in-app-connector": "0.2.1", + "read-authoring-contract": "0.2.0", + "write-connector-source": "0.2.0", + "build-and-test": "0.2.0", + "deploy-and-activate": "0.2.0", + "design-access-model": "0.1.0", + "source-openapi-spec": "0.1.0", +} function readBundle(): {version: string; skills: {name: string; path: string}[]} { return JSON.parse(readFileSync(BUNDLE, "utf8")) as {version: string; skills: {name: string; path: string}[]} @@ -43,15 +54,17 @@ test("(a) each SKILL.md exists with the locked frontmatter contract", () => { assert.ok(fm.description, `${name}: frontmatter description missing`) assert.ok(fm.description.includes("Use when"), `${name}: description must carry the trigger sentence`) assert.ok(fm.description.includes("Do not use when"), `${name}: description must carry the anti-trigger sentence`) - assert.equal(fm.version, bundle.version, `${name}: frontmatter version must equal bundle.json version`) - assert.equal(fm.version, VERSION) + // The plan's locked versions are intentionally non-uniform (the two new + // pre-1 skills ship at 0.1.0); each skill's frontmatter must match its + // locked per-skill version. + assert.equal(fm.version, SKILL_VERSIONS[name], `${name}: frontmatter version must equal the locked per-skill version`) } }) test("(b) every bundle.json path resolves to an existing file", () => { const bundle = readBundle() - assert.equal(bundle.version, VERSION) - assert.equal(bundle.skills.length, 5) +assert.equal(bundle.version, VERSION) + assert.equal(bundle.skills.length, 7) assert.deepEqual(bundle.skills.map((s) => s.name), SKILLS) const skillsRoot = resolve("skills") for (const skill of bundle.skills) { @@ -89,7 +102,7 @@ const SKILL_LITERALS: Record = { "Do not call `c1_connector_service_force_sync` during the funnel run", "Do not call `c1_connector_authoring_list_revision_summaries` during the funnel run", ], - "write-connector-source": [ +"write-connector-source": [ "Do not call fetch", "is_secret: true", "Do not use the secret: spelling", @@ -108,6 +121,26 @@ const SKILL_LITERALS: Record = { "user.id", "config(\"base-url\")", ], + "design-access-model": [ + "TRAIT_USER", + "TRAIT_GROUP", + "WithExternalID is DEPRECATED", + "because the API lacks", + "source-openapi-spec", + "write-connector-source", + "id_compatibility", + "provisioning", + ], + "source-openapi-spec": [ + "262144", + "1048576", + "wc -c", + "missing_paths", + "revisit_trigger", + "vendor_doc", + "Parking with evidence", + "authority ladder", + ], } test("(c) each SKILL.md carries the locked section markers, content literals, ASCII-only bodies, and stays <= 200 lines", () => { @@ -127,7 +160,7 @@ test("(c) each SKILL.md carries the locked section markers, content literals, AS test("(d) the full-mode scenario parses with mode full and the two pinned scenarios keep their locked modes", () => { const full = loadScenario("evals/scenarios/tier1-directory-full.json") assert.equal(full.skillBundle.mode, "full") - assert.equal(full.skillBundle.version, "0.2.0") + assert.equal(full.skillBundle.version, "0.3.0") assert.equal(full.id, "tier1-directory-full") const none = loadScenario("evals/scenarios/tier1-directory.json") assert.equal(none.skillBundle.mode, "none") @@ -175,8 +208,8 @@ test("(e) CLI end-to-end: full-mode Tier-0 run exits 0 and the record meta carri assert.equal(records.length, 1, `expected exactly one record, got ${records.join(", ")}`) const lines = readFileSync(join(dir, records[0]), "utf8").trim().split("\n") const meta = JSON.parse(lines[0]) as Record - assert.equal(meta.skill_bundle_mode, "full") - assert.equal(meta.skill_bundle_version, "0.2.0") +assert.equal(meta.skill_bundle_mode, "full") + assert.equal(meta.skill_bundle_version, "0.3.0") } finally { rmSync(dir, {recursive: true, force: true}) } diff --git a/evals/runner/stages.test.ts b/evals/runner/stages.test.ts index 1d07f7c..b5a4c9a 100644 --- a/evals/runner/stages.test.ts +++ b/evals/runner/stages.test.ts @@ -1,7 +1,7 @@ // stages.test.ts — unit smoke for the S0..S11 stage gates (locked D2/L23). import {test} from "node:test" import assert from "node:assert/strict" -import {STAGES, sanitizeHandoffValue, type Handoff, type ScoreInput, type StageCtx} from "./stages.ts" +import {PRE1_STAGES, STAGES, sanitizeHandoffValue, type Handoff, type Pre1Artifact, type ScoreInput, type StageCtx} from "./stages.ts" import {parseStream, type ParsedStream} from "./stream.ts" const HANDOFF_PATH = "/tmp/evals-run/handoff.json" @@ -282,3 +282,181 @@ test("S1 fails on an empty catalog_id while S4/S6/S7/S9 still pass on present fi assert.equal(check("S7", c), true) assert.equal(check("S9", c), true) }) + +// --- pre-1 gates (P0..P4) --- + +function pre1Artifact(overrides: Partial): Pre1Artifact { + return { + decision: "proceed", + access_model: { + resource_types: [ + {id: "user", traits: ["TRAIT_USER"]}, + {id: "group", traits: ["TRAIT_GROUP"]}, + ], + entitlements: [{slug: "member"}], + grants: [{resource_type: "group", entitlement: "member", principal_type: "user"}], + id_compatibility: [{resource_type: "user", id_shape: "user:{id}", stable: true}], + provisioning: [ + {resource_type: "user", provisionable: false, justification: "because the API lacks a user create/update endpoint"}, + ], + }, + sourcing: { + spec_url: "http://127.0.0.1:18080/openapi.json", + fetched_at: "2026-09-04T00:00:00.000Z", + authority_rung: "official published spec at a stable URL", + spec_bytes: 12297, + }, + ...overrides, + } +} + +function pre1Ctx(overrides: Partial): StageCtx { + return { + transcript: cleanTranscript(), + handoff: {}, + scoreInput: goodScoreInput(), + handoffPath: "/tmp/evals-run/pre1.json", + kind: "pre1", + pre1: pre1Artifact({}), + expected: { + decision: "proceed", + accessModel: { + resource_types: [ + {id: "user", traits: ["TRAIT_USER"]}, + {id: "group", traits: ["TRAIT_GROUP"]}, + ], + entitlements: [{slug: "member"}], + grants: [{resource_type: "group", entitlement: "member", principal_type: "user"}], + }, + }, + ...overrides, + } +} + +function pcheck(stage: string, c: StageCtx): boolean { + const s = PRE1_STAGES.proceed.find((x) => x.stage === stage) ?? PRE1_STAGES.park.find((x) => x.stage === stage) + assert.ok(s, `pre1 stage ${stage} exists`) + return s.check(c) +} + +test("P0 fails on a null artifact and passes on a valid one", () => { + assert.equal(pcheck("P0", pre1Ctx({pre1: null})), false) + assert.equal(pcheck("P0", pre1Ctx({})), true) +}) + +test("P1 passes when the decision matches and fails when it differs", () => { + assert.equal(pcheck("P1", pre1Ctx({})), true) + assert.equal(pcheck("P1", pre1Ctx({pre1: pre1Artifact({decision: "park"})})), false) +}) + +test("P2 passes on a matching access model", () => { + assert.equal(pcheck("P2", pre1Ctx({})), true) +}) + +test("P2 fails on a wrong resource-type id", () => { + const pre1 = pre1Artifact({}) + pre1.access_model!.resource_types = [{id: "user", traits: ["TRAIT_USER"]}, {id: "team", traits: ["TRAIT_GROUP"]}] + assert.equal(pcheck("P2", pre1Ctx({pre1})), false) +}) + +test("P2 fails on a wrong trait", () => { + const pre1 = pre1Artifact({}) + pre1.access_model!.resource_types = [{id: "user", traits: ["TRAIT_USER"]}, {id: "group", traits: ["TRAIT_ROLE"]}] + assert.equal(pcheck("P2", pre1Ctx({pre1})), false) +}) + +test("P2 fails on a missing entitlement slug", () => { + const pre1 = pre1Artifact({}) + pre1.access_model!.entitlements = [{slug: "owner"}] + assert.equal(pcheck("P2", pre1Ctx({pre1})), false) +}) + +test("P2 fails on a missing grant edge", () => { + const pre1 = pre1Artifact({}) + pre1.access_model!.grants = [] + assert.equal(pcheck("P2", pre1Ctx({pre1})), false) +}) + +test("P2 fails on an empty id_compatibility", () => { + const pre1 = pre1Artifact({}) + pre1.access_model!.id_compatibility = [] + assert.equal(pcheck("P2", pre1Ctx({pre1})), false) +}) + +test("P2 fails on an empty provisioning justification", () => { + const pre1 = pre1Artifact({}) + pre1.access_model!.provisioning = [{resource_type: "user", provisionable: false, justification: ""}] + assert.equal(pcheck("P2", pre1Ctx({pre1})), false) +}) + +test("P2 fails on a non-boolean provisionable", () => { + const pre1 = pre1Artifact({}) + pre1.access_model!.provisioning = [{resource_type: "user", provisionable: "yes" as unknown as boolean, justification: "because the API lacks a user create/update endpoint"}] + assert.equal(pcheck("P2", pre1Ctx({pre1})), false) +}) + +test("P3 passes on valid provenance", () => { + assert.equal(pcheck("P3", pre1Ctx({})), true) +}) + +test("P3 fails when spec_bytes is at or above the cap", () => { + const pre1 = pre1Artifact({}) + pre1.sourcing!.spec_bytes = 1048576 + assert.equal(pcheck("P3", pre1Ctx({pre1})), false) +}) + +test("P3 fails when spec_bytes is 0 or negative", () => { + for (const n of [0, -1]) { + const pre1 = pre1Artifact({}) + pre1.sourcing!.spec_bytes = n + assert.equal(pcheck("P3", pre1Ctx({pre1})), false, `spec_bytes ${n} should fail P3`) + } +}) + +test("P3 fails when spec_url is empty", () => { + const pre1 = pre1Artifact({}) + pre1.sourcing!.spec_url = "" + assert.equal(pcheck("P3", pre1Ctx({pre1})), false) +}) + +test("P4 passes on complete park evidence", () => { + const pre1 = pre1Artifact({ + decision: "park", + park_evidence: { + spec_version_checked: "1.2.0", + missing_paths: ["/v1/users", "/v1/groups"], + vendor_doc: "console only", + revisit_trigger: "ships an API", + }, + }) + assert.equal(pcheck("P4", pre1Ctx({pre1})), true) +}) + +test("P4 fails when missing_paths is empty", () => { + const pre1 = pre1Artifact({ + decision: "park", + park_evidence: { + spec_version_checked: "1.2.0", + missing_paths: [], + vendor_doc: "console only", + revisit_trigger: "ships an API", + }, + }) + assert.equal(pcheck("P4", pre1Ctx({pre1})), false) +}) + +test("P4 fails when any park-evidence field is empty", () => { + for (const key of ["spec_version_checked", "vendor_doc", "revisit_trigger"]) { + const pre1 = pre1Artifact({ + decision: "park", + park_evidence: { + spec_version_checked: "1.2.0", + missing_paths: ["/v1/users"], + vendor_doc: "console only", + revisit_trigger: "ships an API", + }, + }) + pre1.park_evidence![key as "spec_version_checked" | "vendor_doc" | "revisit_trigger"] = "" + assert.equal(pcheck("P4", pre1Ctx({pre1})), false, `${key} empty should fail P4`) + } +}) diff --git a/evals/runner/stages.ts b/evals/runner/stages.ts index 5ca8b14..6338686 100644 --- a/evals/runner/stages.ts +++ b/evals/runner/stages.ts @@ -1,6 +1,8 @@ // stages.ts — S0..S11 stage gate definitions. // Every check is a pure function over the locked evidence contract. import type {ParsedStream, ToolCallRecord} from "./stream.ts" +import {isRecord} from "./scenario.ts" +import type {ExpectedAccessModel, ExpectedParkEvidence} from "./scenario.ts" export interface Handoff { catalog_id?: string @@ -36,6 +38,40 @@ export interface StageCtx { handoff: Handoff scoreInput: ScoreInput handoffPath: string + // Pre-1 fields (all optional — the existing test ctx() helpers compile + // unchanged). kind === "pre1" selects the PRE1_STAGES gate set. + kind?: "funnel" | "pre1" + pre1?: Pre1Artifact | null + expected?: {decision: "proceed" | "park"; accessModel?: ExpectedAccessModel; parkEvidence?: ExpectedParkEvidence} +} + +export interface Pre1AccessModel { + resource_types: {id: string; traits: string[]}[] + entitlements: {slug: string; display_name?: string; grantable_principals?: string[]; stable_id_shape?: string}[] + grants: {resource_type: string; entitlement: string; principal_type: string}[] + id_compatibility: unknown[] + provisioning: {resource_type: string; provisionable: boolean; justification: string}[] +} + +export interface Pre1Sourcing { + spec_url: string + fetched_at: string + authority_rung: string + spec_bytes: number +} + +export interface Pre1ParkEvidence { + spec_version_checked: string + missing_paths: string[] + vendor_doc: string + revisit_trigger: string +} + +export interface Pre1Artifact { + decision: "proceed" | "park" + access_model?: Pre1AccessModel + sourcing?: Pre1Sourcing + park_evidence?: Pre1ParkEvidence } export interface Stage { @@ -318,3 +354,171 @@ export const SKIPPED_STAGES: {stage: string; gate: string}[] = [ {stage: "S11b", gate: "REVISION_STATUS_ACTIVE"}, {stage: "S11c", gate: "SYNC_STATUS_DONE"}, ] + +// --- pre-1 gates (P0..P4) --- +// The pre1 artifact is UNTRUSTED (agent-written). Every check type-checks it +// defensively and never throws; a malformed artifact fails its gate. + +function nonEmptyString(v: unknown): v is string { + return typeof v === "string" && v.length > 0 +} + +// Canonicalize BOTH sides identically: {id, traits} with traits sorted, then +// set-compare. Extra fields on either side are dropped by the canonical form. +function canonicalResourceType(rt: {id: string; traits: string[]}): string { + return JSON.stringify({id: rt.id, traits: [...rt.traits].sort()}) +} + +function canonicalGrant(g: {resource_type: string; entitlement: string; principal_type: string}): string { + return JSON.stringify({resource_type: g.resource_type, entitlement: g.entitlement, principal_type: g.principal_type}) +} + +function setEqual(a: string[], b: string[]): boolean { + if (a.length !== b.length) return false + const sortedA = [...a].sort() + const sortedB = [...b].sort() + return sortedA.every((v, i) => v === sortedB[i]) +} + +// Defensive access-model dimension checks shared by P2's check and evidence. +function accessModelDimensions(ctx: StageCtx): { + rtMatch: boolean + entMatch: boolean + grantMatch: boolean + idcCount: number + provisioning: "all justified" | "missing justification" | "non-boolean provisionable" | "absent" +} { + const pre1 = ctx.pre1 + const expected = ctx.expected?.accessModel + if (pre1 === null || pre1 === undefined || !isRecord(pre1.access_model) || expected === undefined) { + return {rtMatch: false, entMatch: false, grantMatch: false, idcCount: 0, provisioning: "absent"} + } + const am = pre1.access_model + const rtMatch = + Array.isArray(am.resource_types) && + setEqual( + am.resource_types + .filter((rt) => isRecord(rt) && typeof rt.id === "string" && Array.isArray(rt.traits) && rt.traits.every((t) => typeof t === "string")) + .map((rt) => canonicalResourceType(rt as {id: string; traits: string[]})), + expected.resource_types.map(canonicalResourceType), + ) + const entMatch = + Array.isArray(am.entitlements) && + setEqual( + am.entitlements.filter((e) => isRecord(e) && typeof e.slug === "string").map((e) => (e as {slug: string}).slug), + expected.entitlements.map((e) => e.slug), + ) + const grantMatch = + Array.isArray(am.grants) && + setEqual( + am.grants + .filter((g) => isRecord(g) && typeof g.resource_type === "string" && typeof g.entitlement === "string" && typeof g.principal_type === "string") + .map((g) => canonicalGrant(g as {resource_type: string; entitlement: string; principal_type: string})), + expected.grants.map(canonicalGrant), + ) + const idcCount = Array.isArray(am.id_compatibility) ? am.id_compatibility.length : 0 + let provisioning: "all justified" | "missing justification" | "non-boolean provisionable" | "absent" = "absent" + if (Array.isArray(am.provisioning)) { + if (am.provisioning.length === 0) { + provisioning = "missing justification" + } else if (am.provisioning.some((p) => !isRecord(p) || typeof p.provisionable !== "boolean")) { + provisioning = "non-boolean provisionable" + } else if (am.provisioning.some((p) => !nonEmptyString((p as Record).justification))) { + provisioning = "missing justification" + } else { + provisioning = "all justified" + } + } + return {rtMatch, entMatch, grantMatch, idcCount, provisioning} +} + +const P0: Stage = { + stage: "P0", + gate: "artifact written", + check: (ctx) => { + const pre1 = ctx.pre1 + return pre1 !== null && pre1 !== undefined && (pre1.decision === "proceed" || pre1.decision === "park") + }, + evidence: (ctx) => { + const pre1 = ctx.pre1 + const present = pre1 !== null && pre1 !== undefined ? "yes" : "no" + const decision = pre1 !== null && pre1 !== undefined && typeof pre1.decision === "string" ? pre1.decision : "none" + return `pre1.json present=${present}, decision=${decision}` + }, +} + +const P1: Stage = { + stage: "P1", + gate: "decision correctness", + check: (ctx) => { + const pre1 = ctx.pre1 + if (pre1 === null || pre1 === undefined) return false + return pre1.decision === ctx.expected?.decision + }, + evidence: (ctx) => { + const pre1 = ctx.pre1 + const decision = pre1 !== null && pre1 !== undefined && typeof pre1.decision === "string" ? pre1.decision : "none" + const expected = ctx.expected?.decision ?? "none" + return `decision=${decision}, expected=${expected}` + }, +} + +const P2: Stage = { + stage: "P2", + gate: "access-model match", + check: (ctx) => { + const d = accessModelDimensions(ctx) + return d.rtMatch && d.entMatch && d.grantMatch && d.idcCount > 0 && d.provisioning === "all justified" + }, + evidence: (ctx) => { + const d = accessModelDimensions(ctx) + return `resource_types=${d.rtMatch ? "match" : "mismatch"}, entitlements=${d.entMatch ? "match" : "mismatch"}, grants=${d.grantMatch ? "match" : "mismatch"}, id_compatibility=${d.idcCount}, provisioning=${d.provisioning}` + }, +} + +const P3: Stage = { + stage: "P3", + gate: "sourcing provenance", + check: (ctx) => { + const pre1 = ctx.pre1 + if (pre1 === null || pre1 === undefined || !isRecord(pre1.sourcing)) return false + const s = pre1.sourcing + if (!nonEmptyString(s.spec_url) || !nonEmptyString(s.fetched_at) || !nonEmptyString(s.authority_rung)) return false + return typeof s.spec_bytes === "number" && Number.isFinite(s.spec_bytes) && Number.isInteger(s.spec_bytes) && s.spec_bytes > 0 && s.spec_bytes < 1048576 + }, + evidence: (ctx) => { + const pre1 = ctx.pre1 + const s = isRecord(pre1?.sourcing) ? pre1.sourcing : null + const specUrl = s !== null && nonEmptyString(s.spec_url) ? "set" : "EMPTY" + const fetchedAt = s !== null && nonEmptyString(s.fetched_at) ? "set" : "EMPTY" + const authorityRung = s !== null && nonEmptyString(s.authority_rung) ? "set" : "EMPTY" + const specBytes = s !== null && typeof s.spec_bytes === "number" ? String(s.spec_bytes) : "none" + return `spec_url=${specUrl}, fetched_at=${fetchedAt}, authority_rung=${authorityRung}, spec_bytes=${specBytes}` + }, +} + +const P4: Stage = { + stage: "P4", + gate: "park evidence", + check: (ctx) => { + const pre1 = ctx.pre1 + if (pre1 === null || pre1 === undefined || !isRecord(pre1.park_evidence)) return false + const pe = pre1.park_evidence + if (!nonEmptyString(pe.spec_version_checked) || !nonEmptyString(pe.vendor_doc) || !nonEmptyString(pe.revisit_trigger)) return false + return Array.isArray(pe.missing_paths) && pe.missing_paths.length > 0 && pe.missing_paths.every((p) => typeof p === "string" && p.length > 0) + }, + evidence: (ctx) => { + const pre1 = ctx.pre1 + const pe = isRecord(pre1?.park_evidence) ? pre1.park_evidence : null + const specVersion = pe !== null && nonEmptyString(pe.spec_version_checked) ? "set" : "EMPTY" + const vendorDoc = pe !== null && nonEmptyString(pe.vendor_doc) ? "set" : "EMPTY" + const revisitTrigger = pe !== null && nonEmptyString(pe.revisit_trigger) ? "set" : "EMPTY" + const missingPaths = pe !== null && Array.isArray(pe.missing_paths) ? pe.missing_paths.filter((p) => typeof p === "string" && p.length > 0).length : 0 + return `spec_version_checked=${specVersion}, missing_paths=${missingPaths}, vendor_doc=${vendorDoc}, revisit_trigger=${revisitTrigger}` + }, +} + +export const PRE1_STAGES: Record<"proceed" | "park", Stage[]> = { + proceed: [P0, P1, P2, P3], + park: [P0, P1, P4], +} diff --git a/evals/scenarios/pre1-directory-proceed.json b/evals/scenarios/pre1-directory-proceed.json new file mode 100644 index 0000000..52fa477 --- /dev/null +++ b/evals/scenarios/pre1-directory-proceed.json @@ -0,0 +1,36 @@ +{ + "id": "pre1-directory-proceed", + "name": "Pre-1: Directory API access model + spec sourcing (proceed)", + "kind": "pre1", + "fixture": { + "port": 18080, + "baseUrl": "http://127.0.0.1:18080", + "auth": "basic", + "openapiPath": "/openapi.json", + "basicAuth": { + "username": "connector@example.com", + "password": "fixture-token" + }, + "bearerToken": "fixture-token" + }, + "providerBrief": "A directory service (the Directory API) exposes users, groups, and group memberships over REST. GET /v1/users lists users (basic auth, offset pagination, page object {items, offset, limit, total}); GET /v1/groups lists groups; GET /v1/groups/{groupId}/members lists a group's members; POST and DELETE /v1/groups/{groupId}/members grant and revoke memberships. The account_id scoping param is required on user listing. user.title is nullable. Design the access model and source the spec.", + "expectedDecision": "proceed", + "expectedAccessModel": { + "resource_types": [ + {"id": "user", "traits": ["TRAIT_USER"]}, + {"id": "group", "traits": ["TRAIT_GROUP"]} + ], + "entitlements": [ + {"slug": "member"} + ], + "grants": [ + {"resource_type": "group", "entitlement": "member", "principal_type": "user"} + ] + }, + "skillBundle": { + "mode": "full", + "version": "0.3.0" + }, + "model": "together/deepseek-ai/DeepSeek-V4-Flash-0731", + "reasoningEffort": "high" +} diff --git a/evals/scenarios/pre1-noiam-park.json b/evals/scenarios/pre1-noiam-park.json new file mode 100644 index 0000000..c90bada --- /dev/null +++ b/evals/scenarios/pre1-noiam-park.json @@ -0,0 +1,34 @@ +{ + "id": "pre1-noiam-park", + "name": "Pre-1: Acme Analytics spec sourcing (park)", + "kind": "pre1", + "fixture": { + "port": 18080, + "baseUrl": "http://127.0.0.1:18080", + "auth": "basic", + "openapiPath": "/noiam/openapi.json", + "basicAuth": { + "username": "connector@example.com", + "password": "fixture-token" + }, + "bearerToken": "fixture-token" + }, + "providerBrief": "Acme Analytics is a usage-analytics platform. Its public API exposes a whoami endpoint (GET /v1/me) and business endpoints (projects, reports, invoices). User and group management is available only in the Acme web console - there is no public API for member listing, roles, or key inventory. Source the spec and run the IAM go/no-go gate.", + "expectedDecision": "park", + "expectedParkEvidence": { + "spec_version_checked": "1.2.0", + "missing_paths": [ + "/v1/users", + "/v1/groups", + "/v1/groups/{groupId}/members" + ], + "vendor_doc": "User and group management is available only in the Acme web console; there is no public API for member listing, role management, or API key inventory.", + "revisit_trigger": "Acme ships a members or teams API (e.g. GET /v1/users or GET /v1/teams/{id}/members)" + }, + "skillBundle": { + "mode": "full", + "version": "0.3.0" + }, + "model": "together/deepseek-ai/DeepSeek-V4-Flash-0731", + "reasoningEffort": "high" +} diff --git a/evals/scenarios/tier1-directory-full.json b/evals/scenarios/tier1-directory-full.json index a2f0777..f5f111e 100644 --- a/evals/scenarios/tier1-directory-full.json +++ b/evals/scenarios/tier1-directory-full.json @@ -27,7 +27,7 @@ }, "skillBundle": { "mode": "full", - "version": "0.2.0" + "version": "0.3.0" }, "model": "together/deepseek-ai/DeepSeek-V4-Flash-0731", "reasoningEffort": "high", diff --git a/evals/skills-bundle/README.md b/evals/skills-bundle/README.md index 16fbf52..0fe1f6b 100644 --- a/evals/skills-bundle/README.md +++ b/evals/skills-bundle/README.md @@ -2,6 +2,18 @@ This directory is the skill-bundle mount point for the eval harness. +## v0.3.0 — the seven skills + +The bundle ships seven skills: the five funnel skills plus two new pre-1 +judgment skills at `0.1.0`: + +- `design-access-model` — pre-1: access-model design for net-new providers + (resource types, traits, entitlements, grants, provisioning scope). +- `source-openapi-spec` — pre-1: OpenAPI spec sourcing + the IAM go/no-go + gate, with park-with-evidence as a success outcome. + +The five funnel skills are unchanged; the bundle version is `0.3.0`. + ## v0.2.0 — the five funnel skills The bundle ships five skills, authored against the v0.0.26 DSL contract and diff --git a/evals/skills-bundle/bundle.json b/evals/skills-bundle/bundle.json index 8b96e69..b54b616 100644 --- a/evals/skills-bundle/bundle.json +++ b/evals/skills-bundle/bundle.json @@ -1,10 +1,12 @@ { - "version": "0.2.0", + "version": "0.3.0", "skills": [ {"name": "author-in-app-connector", "path": "../../skills/author-in-app-connector/SKILL.md"}, {"name": "read-authoring-contract", "path": "../../skills/read-authoring-contract/SKILL.md"}, {"name": "write-connector-source", "path": "../../skills/write-connector-source/SKILL.md"}, {"name": "build-and-test", "path": "../../skills/build-and-test/SKILL.md"}, - {"name": "deploy-and-activate", "path": "../../skills/deploy-and-activate/SKILL.md"} + {"name": "deploy-and-activate", "path": "../../skills/deploy-and-activate/SKILL.md"}, + {"name": "design-access-model", "path": "../../skills/design-access-model/SKILL.md"}, + {"name": "source-openapi-spec", "path": "../../skills/source-openapi-spec/SKILL.md"} ] } diff --git a/skills/README.md b/skills/README.md index 4324f31..e58368c 100644 --- a/skills/README.md +++ b/skills/README.md @@ -1,6 +1,6 @@ # Agent skills -The five funnel skills shipped in this batch, authored against +The seven skills shipped in this batch, authored against the v0.0.26 DSL contract and the 23-tool tenant MCP surface. Each skill's `SOURCES.md` names the pinned sources with their SHAs. @@ -11,6 +11,8 @@ the v0.0.26 DSL contract and the 23-tool tenant MCP surface. Each skill's | `write-connector-source` | Source authoring before S2 — the four-file source contract | MCP-served guide, `authoring.proto`, lifecycle doc, `runtime-gotchas.md`, shopify Makefile | | `build-and-test` | Stages 2–5, 9–10 — upload, build, draft test | MCP-served guide, `authoring.proto`, lifecycle doc | | `deploy-and-activate` | Stages 6–8, 11 — app, provision, configure, deploy, mint, handoff | MCP-served guide, `authoring.proto`, lifecycle doc | +| `design-access-model` | Pre-1 — access-model design for net-new providers | baton-admin `design-baton-access-model` @ `6fe6886f…` | +| `source-openapi-spec` | Pre-1 — OpenAPI spec sourcing + IAM go/no-go | claude-marketplace `source-openapi-spec` @ `0cc5ac2a…` | The eval bundle (`evals/skills-bundle/bundle.json`) is a manifest pointing into this directory; the skill bodies live here as the single source of diff --git a/skills/author-in-app-connector/SKILL.md b/skills/author-in-app-connector/SKILL.md index 9207bdc..9938795 100644 --- a/skills/author-in-app-connector/SKILL.md +++ b/skills/author-in-app-connector/SKILL.md @@ -1,7 +1,7 @@ --- name: author-in-app-connector description: Use when driving the full in-app connector authoring funnel end to end, from guide read through the human-activation handoff. Do not use when you are already mid-funnel and only need one stage's procedure - invoke the stage skill directly. -version: 0.2.0 +version: 0.2.1 --- # author-in-app-connector @@ -90,11 +90,18 @@ produces it: | write (source authoring, before S2) | `write-connector-source` | | 2-5, 9-10 | `build-and-test` | | 6-8, 11 | `deploy-and-activate` | -| design/spec/verify/update/diagnose (not yet shipped) | follow the served guide | +| pre-1 design (net-new provider) | `design-access-model` | +| pre-1 spec sourcing | `source-openapi-spec` | +| verify/update/diagnose (not yet shipped) | follow the served guide | Direct invocation of a stage skill is allowed; the router alone is not trusted to recover mid-funnel. +## Pre-S1 checklist + +Net-new provider: before S1, run both pre-1 skills - `source-openapi-spec` +(source the spec, IAM go/no-go) then `design-access-model` (access model). + ## Human boundary After S11's deploy + mint, present `activation_url` to a human tenant OWNER diff --git a/skills/design-access-model/SKILL.md b/skills/design-access-model/SKILL.md new file mode 100644 index 0000000..11ea4ca --- /dev/null +++ b/skills/design-access-model/SKILL.md @@ -0,0 +1,90 @@ +--- +name: design-access-model +description: Use when designing the access model (resource types, traits, entitlements, grants, provisioning scope) for a net-new provider before authoring connector source. Do not use when you are already mid-funnel or authoring source - invoke write-connector-source instead. +version: 0.1.0 +--- + +# design-access-model + +Use after `source-openapi-spec` (the spec is sourced and the IAM go/no-go gate passed) and before `write-connector-source`. + +## Workflow + +1. Map upstream entities to resource types and traits (table a). +2. Define stable resource IDs (stable-ID rules). +3. Define entitlements (slug, display name, grantable principals, stable ID shape). +4. Define grants (source map). +5. Decide sync-only vs provisionable (table b) with the mandatory justification. +6. Decide base-url config vs literal (table c). +7. Emit the output contract. + +## Decision table (a) - trait selection + +| Condition (upstream entity) | Decision (TRAIT_* const from `@baton/types`) | Mandatory justification sentence | +|---|---|---| +| Human user accounts (people with login credentials) | `TRAIT_USER` | "because the upstream entity is a human user account" | +| Groups / teams / directories of principals | `TRAIT_GROUP` | "because the upstream entity is a group of principals" | +| Roles / permission sets / policies | `TRAIT_ROLE` | "because the upstream entity is a role or permission set" | +| OAuth applications / API clients (app-user) | `TRAIT_APP` | "because the upstream entity is an application identity" | +| Service accounts / non-human identities | `TRAIT_APP` (or `TRAIT_SECRET` for static-credential NHIs) | "because the upstream entity is a non-human identity" | + +## Decision table (b) - sync-only vs provisionable + +| Condition (API surface) | Decision | Mandatory justification sentence | +|---|---|---| +| API exposes grant/revoke write ops (e.g. POST/DELETE membership endpoints) | provisionable (grant/revoke) | "because the API exposes the write operation" | +| API exposes user/group create/update/delete | provisionable (account provisioning) | "because the API exposes the write operation" | +| API lacks the write op for the resource or membership | sync-only | "because the API lacks " | + +## Decision table (c) - base-url config vs literal + +| Condition | Decision | Mandatory justification sentence | +|---|---|---| +| Multi-tenant / customer-owned base URL (per-tenant subdomain or region) | required config field `base-url` | "because the base URL is customer-owned and varies per tenant" | +| Single fixed endpoint (one global URL) | literal in source | "because the API has a single fixed endpoint" | + +## Stable-ID rules + +Never use display names, emails, or mutable slugs as IDs. Use the API's +immutable object id; composite IDs join stable parts. + +## Handoffs + +- Spec missing or the IAM go/no-go gate failed -> `source-openapi-spec`. +- Model done -> `write-connector-source`. + +## Output Contract + +Emit the `access_model` half of the pre1.json artifact: + +- `resource_types`: list of `{id, traits}` pairs (one per upstream entity). +- `entitlements`: list of `{slug, display_name, grantable_principals, stable_id_shape}`. +- `grants`: list of `{resource_type, entitlement, principal_type}` edges. +- `id_compatibility`: table of `{resource_type, id_shape, stable}` rows. +- `provisioning`: list of `{resource_type, provisionable, justification}` rows. + +## Eval-alignment contract + +- `id_compatibility` must be non-empty. +- Every `provisioning` entry must carry a non-empty `justification`. + +## Exit criteria + +- Every upstream entity mapped to a resource type with a trait from table (a). +- Every entitlement has a slug, display name, grantable principals, and stable ID shape. +- Every grant edge has a source in the API. +- Every provisioning decision carries the mandatory justification sentence. +- The base-url decision is justified from table (c). + +## Anti-patterns + +- `WithExternalID is DEPRECATED - never required`. +- Do not use display names, emails, or mutable slugs as stable IDs. +- Do not add every available endpoint to the connector model. +- Do not mark a resource provisionable when the API lacks the write op. + +## Blocker protocol + +If the same validation or runtime error remains unchanged after 2 failed fix +cycles on the same error, stop and report the exact error text instead of +guessing further. diff --git a/skills/design-access-model/SOURCES.md b/skills/design-access-model/SOURCES.md new file mode 100644 index 0000000..d500510 --- /dev/null +++ b/skills/design-access-model/SOURCES.md @@ -0,0 +1,13 @@ +# Sources — design-access-model + +Authored against the pinned sources below (decision 13: nothing written from +model memory). The three decision tables, the stable-ID rules, and the +`WithExternalID` deprecation are the locked intent of this batch (plan +decisions 2-3); the port source below is what this skill quotes. + +| Source | Pin / SHA | What this skill quotes | +|---|---|---| +| baton-admin `design-baton-access-model` | `6fe6886f607ed0d2e48a616c30e7ce4bffc32489` | The design port source: resource-type/trait mapping, stable-ID rules (never display names/emails/mutable slugs), entitlement definitions, grant source map, provisioning scope note, ID compatibility table, handoff discipline. | +| claude-marketplace `source-openapi-spec` | `0cc5ac2a2dbe60b430444c59e53016da2c72b3d1` | The IAM go/no-go + park-with-evidence contract this skill's handoff to `source-openapi-spec` carries. | +| c1 `docs/in-app-connector-authoring.md` | `16e0e0fbf0c999e3942f1ea0f8aef95c65e6fbc3` | The bundle caps (262144-byte source, 1048576-byte bundle) and the lifecycle contract the pre-1 judgment phase feeds. | +| In-repo SDK declarations + examples | connector-authoring `7dc673ac86616503acee3d45f98348370dc155cd` | The `baton/types.d.ts` TRAIT_* consts (`TRAIT_USER`, `TRAIT_GROUP`, `TRAIT_ROLE`, `TRAIT_APP`, `TRAIT_SECRET`) and the `examples/` access-model shapes. | diff --git a/skills/source-openapi-spec/SKILL.md b/skills/source-openapi-spec/SKILL.md new file mode 100644 index 0000000..e194dab --- /dev/null +++ b/skills/source-openapi-spec/SKILL.md @@ -0,0 +1,81 @@ +--- +name: source-openapi-spec +description: Use when sourcing the OpenAPI spec for a net-new provider and running the IAM go/no-go gate before any access-model design. Do not use when the spec is already sourced - invoke design-access-model instead. +version: 0.1.0 +--- + +# source-openapi-spec + +An in-app connector is spec-driven: the sourced OpenAPI document is the +contract the access model and the source are built from. Sourcing it wrong +ships wrong request shapes; skipping the IAM go/no-go gate ships a shell +connector with no governance value. + +## Step 1 - Find candidates, rank by the authority ladder + +From most to least authoritative: + +1. **Official published spec at a stable URL** (e.g. the provider's + `https:///openapi.json`). Vendor verbatim; record URL + fetch date. +2. **Runtime-generated spec from a PINNED release** - for OSS/self-hosted + products whose server generates its own spec. Install the exact release + and dump it; record the exact command + version as provenance so it is + reproducible on upgrade. +3. **A live hosted docs/Swagger instance** - cross-check only. It runs + whatever unpinned version that deployment happens to run. +4. **Hand-authored minimal spec** - only when nothing machine-readable + exists; every operation needs cited provenance from docs/SDK/source. +5. **NEVER: a checked-in spec snapshot in the provider's repo** without + verifying freshness. + +A provider having Terraform providers or SDKs is NOT evidence of an IAM API - +inspect what they actually call. + +## Step 2 - Verify the management surface exists (go/no-go gate) + +Load the candidate spec and check the paths programmatically - do not skim +docs pages. Required for a meaningful connector: a member/user LISTING at +minimum; membership add/remove, role data, and key inventory determine the +provisioning story. Verify each claim someone made from docs or memory against +the spec. + +**If the IAM surface is missing** (members are console-only, the only identity +endpoint is a whoami, SCIM is roadmap): recommend PARKING the connector. Write +the evidence down - exact spec version/commit checked, the paths that don't +exist, the vendor doc stating the limitation, and the concrete trigger to +revisit. Parking with evidence is a success outcome; a whoami-shell connector +in the catalog is not. + +## Step 3 - Weigh the spec (in-app twist) + +After sourcing, weigh the document: `wc -c` it and record the bytes. The +esbuild source cap is 262144 bytes and the bundle-plus-embedded-specs cap is +1048576 bytes. A spec whose selected operations push the bundle toward the cap +is trimmed by selection, never by editing the vendored document. + +## Output Contract + +Emit the `sourcing` and `park_evidence` halves of the pre1.json artifact: + +- `sourcing`: `{spec_url, fetched_at, authority_rung, spec_bytes}`. +- `park_evidence`: `{spec_version_checked, missing_paths, vendor_doc, revisit_trigger}`. + +## Exit criteria + +- The spec is sourced from the highest authority-ladder rung available. +- The IAM go/no-go gate ran programmatically against the spec's paths. +- `spec_bytes` is recorded from `wc -c` and is under 1048576. +- A park decision carries all four `park_evidence` fields. + +## Anti-patterns + +- Do not source from docs or memory - verify paths programmatically. +- Do not ship a whoami-shell connector when the IAM surface is missing. +- Do not edit the vendored document to fit the bundle caps - trim by selection. +- Do not vendor an unverified checked-in spec snapshot. + +## Blocker protocol + +If the same validation or runtime error remains unchanged after 2 failed fix +cycles on the same error, stop and report the exact error text instead of +guessing further. diff --git a/skills/source-openapi-spec/SOURCES.md b/skills/source-openapi-spec/SOURCES.md new file mode 100644 index 0000000..58811d4 --- /dev/null +++ b/skills/source-openapi-spec/SOURCES.md @@ -0,0 +1,12 @@ +# Sources — source-openapi-spec + +Authored against the pinned sources below (decision 13: nothing written from +model memory). The 5-rung authority ladder, the programmatic IAM go/no-go +gate, and the park-with-evidence contract are the locked intent of this batch +(plan decisions 4-5); the port source below is what this skill quotes. + +| Source | Pin / SHA | What this skill quotes | +|---|---|---| +| claude-marketplace `source-openapi-spec` | `0cc5ac2a2dbe60b430444c59e53016da2c72b3d1` | The sourcing port source: the 5-rung authority ladder, the programmatic IAM go/no-go gate (member/user LISTING at minimum), and park-with-evidence as a success outcome. | +| c1 `docs/in-app-connector-authoring.md` | `16e0e0fbf0c999e3942f1ea0f8aef95c65e6fbc3` | The bundle caps — the 262144-byte source limit and the 1048576-byte bundle limit (lines 727-728) — and the lifecycle contract the pre-1 judgment phase feeds. | +| In-repo SDK declarations + examples | connector-authoring `7dc673ac86616503acee3d45f98348370dc155cd` | The `baton/*.d.ts` module surface and the `examples/` spec-driven connector shapes the sourced spec feeds. | From 35828bfabf24fa336105c2186188e2e53474a6e1 Mon Sep 17 00:00:00 2001 From: Bjorn Tipling Date: Fri, 4 Sep 2026 08:40:25 +0000 Subject: [PATCH 2/6] =?UTF-8?q?CXF-220:=20review=20fixes=20=E2=80=94=20pre?= =?UTF-8?q?1=20production-path=20tests,=20P2=20malformed-entry=20gate,=20c?= =?UTF-8?q?anned=20fidelity,=20hygiene?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial review round 1 (A1-A6, P1-P4) found the pre1 production path (readPre1Artifact + main() pre1 branch + pre1 prompt) had zero committed test coverage — a regression there silently corrupts the park-vs-proceed metric. Fixes: - run.test.ts: readPre1Artifact unit cases (oversized/malformed/non-object -> null, valid -> parsed) + two CLI-level pre1 Tier-0 replays asserting the written record carries decision_verdict and the P-stage rows. - agent.test.ts: pre1 prompt case (pre1Path, both skills, output contract). - stages.ts: P2 gate now fails on malformed extra entries instead of filtering them out before the set-compare (gate contract: a malformed artifact fails its gate); stages.test.ts locks it. - run.ts: extract shared readBoundedJson used by readHandoff and readPre1Artifact (the two had already drifted); placeholder scrub stays in the handoff caller. - canned-pre1-noiam-park/transcript.json: wc -c 12297 -> 4997 (the real size of openapi-noiam.json; the proceed set's byte count was copied). - Re-indent ~9 statements dedented to column 0 across run.ts, driver.ts, scenario.ts, skills_bundle.test.ts, server.ts. Validation: typecheck, eval:test (133 pass), eval:verify (19 assertions), and the three eval:run Tier-0 replays (proceed P0-P3, park P0/P1/P4, funnel S0-S11) all exit 0. Co-authored-by: c1-squire-dev[bot] --- evals/fixture/server.ts | 2 +- evals/runner/agent.test.ts | 39 ++++++++ .../canned-pre1-noiam-park/transcript.json | 2 +- evals/runner/drivers/tier0/driver.ts | 2 +- evals/runner/run.test.ts | 74 +++++++++++++- evals/runner/run.ts | 98 ++++++++----------- evals/runner/scenario.ts | 2 +- evals/runner/skills_bundle.test.ts | 6 +- evals/runner/stages.test.ts | 10 ++ evals/runner/stages.ts | 17 ++-- 10 files changed, 181 insertions(+), 71 deletions(-) diff --git a/evals/fixture/server.ts b/evals/fixture/server.ts index fabacd6..95dd02a 100644 --- a/evals/fixture/server.ts +++ b/evals/fixture/server.ts @@ -159,7 +159,7 @@ const server = createServer(async (req: IncomingMessage, res: ServerResponse) => const log = (status: number) => stdout.write(`${method} ${path} ${status}\n`) try { -// --- openapi.json: unauthenticated --- + // --- openapi.json: unauthenticated --- if (path === "/openapi.json") { if (method !== "GET") { log(405) diff --git a/evals/runner/agent.test.ts b/evals/runner/agent.test.ts index 7f30096..9e64b4c 100644 --- a/evals/runner/agent.test.ts +++ b/evals/runner/agent.test.ts @@ -64,3 +64,42 @@ test("buildPrompt skill-bundle modes render", () => { const full = buildPrompt({...SCENARIO, skillBundle: {mode: "full", version: "1.2.3"}}, "r", "http://x", CHANNEL) assert.ok(full.includes("Skill bundle (version 1.2.3)")) }) + +test("buildPrompt pre1 branch names the pre1Path, both skills, and the output contract", () => { + const pre1: Scenario = { + id: "pre1-directory-proceed", + name: "Pre-1: Directory API access model + spec sourcing (proceed)", + fixture: { + port: 18080, + baseUrl: "http://127.0.0.1:18080", + auth: "basic", + openapiPath: "/openapi.json", + basicAuth: {username: "connector@example.com", password: "fixture-token"}, + bearerToken: "fixture-token", + }, + skillBundle: {mode: "full", version: "0.3.0"}, + model: "together/deepseek-ai/DeepSeek-V4-Flash-0731", + reasoningEffort: "high", + kind: "pre1", + providerBrief: "A directory service exposes users, groups, and memberships over REST.", + expectedDecision: "proceed", + expectedAccessModel: { + resource_types: [{id: "user", traits: ["TRAIT_USER"]}], + entitlements: [{slug: "member"}], + grants: [{resource_type: "group", entitlement: "member", principal_type: "user"}], + }, + } + const prompt = buildPrompt(pre1, "r", "http://127.0.0.1:18080", CHANNEL) + // The pre1 artifact target, the two pre-1 skills, and the completion verb + // must all be present — a regression that drops any of them silently + // breaks real pre1 runs (agents write malformed artifacts or never stop). + assert.ok(prompt.includes("/tmp/evals-run/pre1.json")) + assert.ok(prompt.includes("source-openapi-spec")) + assert.ok(prompt.includes("design-access-model")) + assert.ok(prompt.includes("driver.write_file")) + assert.ok(prompt.includes("driver.complete_run")) + // The output contract names the schema halves. + assert.ok(prompt.includes("park_evidence")) + assert.ok(prompt.includes("spec_bytes")) + assert.ok(prompt.includes("missing_paths")) +}) diff --git a/evals/runner/drivers/tier0/canned-pre1-noiam-park/transcript.json b/evals/runner/drivers/tier0/canned-pre1-noiam-park/transcript.json index d45712d..496112c 100644 --- a/evals/runner/drivers/tier0/canned-pre1-noiam-park/transcript.json +++ b/evals/runner/drivers/tier0/canned-pre1-noiam-park/transcript.json @@ -11,7 +11,7 @@ }, { "type": "tool_result", - "message": "12297 /tmp/spec.json", + "message": "4997 /tmp/spec.json", "data": { "tool_name": "bash", "is_error": false diff --git a/evals/runner/drivers/tier0/driver.ts b/evals/runner/drivers/tier0/driver.ts index db32bc6..f3c8618 100644 --- a/evals/runner/drivers/tier0/driver.ts +++ b/evals/runner/drivers/tier0/driver.ts @@ -124,7 +124,7 @@ const provisioner: Provisioner = { child.kill() throw err } -const baseUrl = "http://127.0.0.1:" + port + const baseUrl = "http://127.0.0.1:" + port return { baseUrl, credentials: {username: "connector@example.com", password: "fixture-token"}, diff --git a/evals/runner/run.test.ts b/evals/runner/run.test.ts index a215461..343bb16 100644 --- a/evals/runner/run.test.ts +++ b/evals/runner/run.test.ts @@ -11,10 +11,10 @@ import assert from "node:assert/strict" import {execFile} from "node:child_process" import {promisify} from "node:util" import {buildRunMeta} from "./record.ts" -import {mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync} from "node:fs" +import {mkdtempSync, openSync, closeSync, ftruncateSync, readFileSync, readdirSync, rmSync, writeFileSync} from "node:fs" import {tmpdir} from "node:os" import {join} from "node:path" -import {collectScoreInput, isCollectionFailure, provisionWithRetry, readHandoff} from "./run.ts" +import {collectScoreInput, isCollectionFailure, provisionWithRetry, readHandoff, readPre1Artifact} from "./run.ts" import {FUNNEL_TOOLS, ReadinessError, type AgentDriver, type Driver, type Provisioner, type RunChannel, type TenantHandle} from "./driver.ts" import type {ParsedStream} from "./stream.ts" import type {Scenario} from "./scenario.ts" @@ -396,6 +396,76 @@ test("readHandoff treats a malformed handoff as a stall without surfacing the pa } }) +test("readPre1Artifact returns null on oversized, malformed, and non-object artifacts", async () => { + const dir = mkdtempSync(join(tmpdir(), "pre1-read-")) + try { + // Oversized: a file larger than the 64 MiB cap must be treated as absent. + const big = join(dir, "big.json") + const fd = openSync(big, "w") + ftruncateSync(fd, 64 * 1024 * 1024 + 1) + closeSync(fd) + assert.equal(await readPre1Artifact(big), null) + // Malformed JSON (the agent wrote garbage) — null, never a throw. + const bad = join(dir, "bad.json") + writeFileSync(bad, '{"decision": "proceed",') + assert.equal(await readPre1Artifact(bad), null) + // Non-object (array) — null. + const arr = join(dir, "arr.json") + writeFileSync(arr, "[1,2,3]") + assert.equal(await readPre1Artifact(arr), null) + } finally { + rmSync(dir, {recursive: true, force: true}) + } +}) + +test("readPre1Artifact parses a valid artifact", async () => { + const dir = mkdtempSync(join(tmpdir(), "pre1-read-")) + try { + const path = join(dir, "pre1.json") + writeFileSync(path, JSON.stringify({decision: "proceed", access_model: {resource_types: []}})) + const artifact = await readPre1Artifact(path) + assert.equal(artifact?.decision, "proceed") + } finally { + rmSync(dir, {recursive: true, force: true}) + } +}) + +test("a pre1-directory-proceed Tier-0 run writes a record with decision_verdict proceed", async () => { + const dir = mkdtempSync(join(tmpdir(), "run-pre1-")) + try { + const {code, stdout} = await runCli(["--scenario", "evals/scenarios/pre1-directory-proceed.json", "--driver", "tier0", "--out", dir], 120_000) + assert.equal(code, 0, `run.ts exited ${code}; stdout=${stdout}`) + assert.ok(stdout.includes("record:"), `expected a record line, got stdout=${stdout}`) + const records = readdirSync(dir).filter((f) => f.endsWith(".jsonl")) + assert.equal(records.length, 1, `expected exactly one record, got ${records.join(", ")}`) + const lines = readFileSync(join(dir, records[0]), "utf8").trim().split("\n") + const stages = lines.slice(1, -1).map((l) => (JSON.parse(l) as {stage: string}).stage) + assert.deepEqual(stages, ["P0", "P1", "P2", "P3"]) + const summary = JSON.parse(lines[lines.length - 1]) as Record + assert.equal(summary.decision_verdict, "proceed") + assert.ok((summary.decision_evidence as string).includes("decision=proceed")) + } finally { + rmSync(dir, {recursive: true, force: true}) + } +}) + +test("a pre1-noiam-park Tier-0 run writes a record with decision_verdict park", async () => { + const dir = mkdtempSync(join(tmpdir(), "run-pre1-")) + try { + const {code, stdout} = await runCli(["--scenario", "evals/scenarios/pre1-noiam-park.json", "--driver", "tier0", "--out", dir], 120_000) + assert.equal(code, 0, `run.ts exited ${code}; stdout=${stdout}`) + const records = readdirSync(dir).filter((f) => f.endsWith(".jsonl")) + assert.equal(records.length, 1, `expected exactly one record, got ${records.join(", ")}`) + const lines = readFileSync(join(dir, records[0]), "utf8").trim().split("\n") + const stages = lines.slice(1, -1).map((l) => (JSON.parse(l) as {stage: string}).stage) + assert.deepEqual(stages, ["P0", "P1", "P4"]) + const summary = JSON.parse(lines[lines.length - 1]) as Record + assert.equal(summary.decision_verdict, "park") + } finally { + rmSync(dir, {recursive: true, force: true}) + } +}) + test("collectScoreInput redacts a malformed score-input parse error", async () => { const dir = mkdtempSync(join(tmpdir(), "collect-")) try { diff --git a/evals/runner/run.ts b/evals/runner/run.ts index d3e483e..afab9e6 100644 --- a/evals/runner/run.ts +++ b/evals/runner/run.ts @@ -115,85 +115,73 @@ function isPlaceholder(v: string): boolean { const MAX_HANDOFF_BYTES = 64 * 1024 * 1024 const MAX_SCORE_INPUT_BYTES = 64 * 1024 * 1024 -export async function readHandoff(handoffPath: string): Promise { - // Bounded retry: a transient read failure must not be misread as a stalled - // agent (locked L18). A written-but-empty file is a genuine stall; a - // missing file is retried (a private driver's transport may lag the write). +// Shared bounded read for agent-written artifacts (the handoff and the pre1 +// artifact are equally untrusted): cap the read at 64 MiB (the removed +// gateway path bounded every such read via execFile maxBuffer) so a runaway +// agent cannot buffer a multi-gigabyte file into the runner; retry transient +// failures 3 times (a private driver's transport may lag the write); return +// null on missing/oversized/malformed/non-object. The caller applies +// artifact-specific post-processing (e.g. the handoff's placeholder scrub). +async function readBoundedJson(path: string, label: string, absent: string): Promise { let lastErr: unknown for (let attempt = 1; attempt <= 3; attempt++) { try { - if (statSync(handoffPath).size > MAX_HANDOFF_BYTES) { - stderr.write(`WARNING: handoff at ${handoffPath} exceeds ${MAX_HANDOFF_BYTES} bytes — scoring as stalled\n`) + if (statSync(path).size > MAX_HANDOFF_BYTES) { + stderr.write(`WARNING: ${label} at ${path} exceeds ${MAX_HANDOFF_BYTES} bytes — scoring as ${absent}\n`) return null } - const content = readFileSync(handoffPath, "utf8") + const content = readFileSync(path, "utf8") if (content.length === 0) return null let parsed: unknown try { parsed = JSON.parse(content) } catch { - // A malformed handoff is a genuine stall (the agent wrote garbage) — - // return null immediately. Never surface the parse error: JSON.parse - // messages embed a snippet of the offending content, which can carry - // agent-written values. The warning is content-free so an operator - // can still distinguish "agent wrote garbage" from "agent never - // wrote the handoff". - stderr.write(`WARNING: handoff at ${handoffPath} is not valid JSON — scoring as stalled\n`) + // A malformed artifact is a genuine failure (the agent wrote + // garbage) — return null immediately. Never surface the parse error: + // JSON.parse messages embed a snippet of the offending content, + // which can carry agent-written values. The warning is content-free + // so an operator can still distinguish "agent wrote garbage" from + // "agent never wrote the artifact". + stderr.write(`WARNING: ${label} at ${path} is not valid JSON — scoring as ${absent}\n`) return null } if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null - const handoff = parsed as Handoff - for (const f of HANDOFF_FIELDS) { - const v = handoff[f] - if (typeof v === "string" && isPlaceholder(v)) handoff[f] = "" - } - return handoff + return parsed } catch (err) { // ENOENT is retried too: a private driver's transport may lag the - // handoff write (the old read retried every failure). Only a - // written-but-empty file is an immediate stall. + // write (the old read retried every failure). Only a written-but-empty + // file is an immediate failure. lastErr = err if (attempt < 3) { await new Promise((resolve) => setTimeout(resolve, 2000)) } } } -stderr.write(`WARNING: handoff read failed after 3 attempts: ${(lastErr as Error).message} — scoring as stalled\n`) + stderr.write(`WARNING: ${label} read failed after 3 attempts: ${(lastErr as Error).message} — scoring as ${absent}\n`) return null } +export async function readHandoff(handoffPath: string): Promise { + // Bounded retry: a transient read failure must not be misread as a stalled + // agent (locked L18). A written-but-empty file is a genuine stall; a + // missing file is retried (a private driver's transport may lag the write). + const parsed = await readBoundedJson(handoffPath, "handoff", "stalled") + if (parsed === null) return null + const handoff = parsed as Handoff + for (const f of HANDOFF_FIELDS) { + const v = handoff[f] + if (typeof v === "string" && isPlaceholder(v)) handoff[f] = "" + } + return handoff +} + // The pre1 artifact is agent-written and untrusted: same bounded 3-attempt // retry + 64 MiB size cap + JSON-parse + non-object-returns-null semantics as -// readHandoff. Returns null on missing/oversized/malformed/non-object (P0 -// then fails). +// readHandoff (shared readBoundedJson). Returns null on +// missing/oversized/malformed/non-object (P0 then fails). export async function readPre1Artifact(path: string): Promise { - let lastErr: unknown - for (let attempt = 1; attempt <= 3; attempt++) { - try { - if (statSync(path).size > MAX_HANDOFF_BYTES) { - stderr.write(`WARNING: pre1 artifact at ${path} exceeds ${MAX_HANDOFF_BYTES} bytes — scoring as absent\n`) - return null - } - const content = readFileSync(path, "utf8") - if (content.length === 0) return null - let parsed: unknown - try { - parsed = JSON.parse(content) - } catch { - stderr.write(`WARNING: pre1 artifact at ${path} is not valid JSON — scoring as absent\n`) - return null - } - if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null - return parsed as Pre1Artifact - } catch (err) { - lastErr = err - if (attempt < 3) { - await new Promise((resolve) => setTimeout(resolve, 2000)) - } - } - } -stderr.write(`WARNING: pre1 artifact read failed after 3 attempts: ${(lastErr as Error).message} — scoring as absent\n`) - return null + const parsed = await readBoundedJson(path, "pre1 artifact", "absent") + return parsed as Pre1Artifact | null } // L18 stalled-agent path is applied inside scoreRun (a completely absent @@ -205,7 +193,7 @@ function buildChannel(out: string, runId: string): RunChannel { // and strict-equality-compared in the S11 gate — they must not be // cwd-dependent for a private driver whose agent runs elsewhere. const runDir = join(resolve(out), runId) -return { + return { runDir, handoffPath: join(runDir, "handoff.json"), scoreInputPath: join(runDir, "score-input.json"), @@ -362,7 +350,7 @@ async function main(): Promise { try { const prompt = buildPrompt(scenario, runId, handle.baseUrl, channel) -const result = await driver.agentDriver.runAgent({ + const result = await driver.agentDriver.runAgent({ kind: "agent", prompt, toolSurface: handle.toolSurface, @@ -381,7 +369,7 @@ const result = await driver.agentDriver.runAgent({ // infrastructure outage, not an agent outcome — fail loudly with no // record rather than scoring an all-fail funnel. A genuine zero-tool-call // stall (no collectionFailed signal) stays a scored exit-0 outcome. -if (isCollectionFailure(result)) { + if (isCollectionFailure(result)) { throw new Error("agent driver reported a stream collection failure with an empty transcript — no record written") } diff --git a/evals/runner/scenario.ts b/evals/runner/scenario.ts index 95c6dce..35d2909 100644 --- a/evals/runner/scenario.ts +++ b/evals/runner/scenario.ts @@ -135,7 +135,7 @@ const id = requireString(data, "id", "scenario") username: requireString(fixture.basicAuth, "username", "fixture.basicAuth"), password: requireString(fixture.basicAuth, "password", "fixture.basicAuth"), } -const bearerToken = requireString(fixture, "bearerToken", "fixture") + const bearerToken = requireString(fixture, "bearerToken", "fixture") const kind = data.kind === undefined ? "funnel" : data.kind if (kind !== "funnel" && kind !== "pre1") { diff --git a/evals/runner/skills_bundle.test.ts b/evals/runner/skills_bundle.test.ts index e727218..9f04642 100644 --- a/evals/runner/skills_bundle.test.ts +++ b/evals/runner/skills_bundle.test.ts @@ -63,7 +63,7 @@ test("(a) each SKILL.md exists with the locked frontmatter contract", () => { test("(b) every bundle.json path resolves to an existing file", () => { const bundle = readBundle() -assert.equal(bundle.version, VERSION) + assert.equal(bundle.version, VERSION) assert.equal(bundle.skills.length, 7) assert.deepEqual(bundle.skills.map((s) => s.name), SKILLS) const skillsRoot = resolve("skills") @@ -102,7 +102,7 @@ const SKILL_LITERALS: Record = { "Do not call `c1_connector_service_force_sync` during the funnel run", "Do not call `c1_connector_authoring_list_revision_summaries` during the funnel run", ], -"write-connector-source": [ + "write-connector-source": [ "Do not call fetch", "is_secret: true", "Do not use the secret: spelling", @@ -208,7 +208,7 @@ test("(e) CLI end-to-end: full-mode Tier-0 run exits 0 and the record meta carri assert.equal(records.length, 1, `expected exactly one record, got ${records.join(", ")}`) const lines = readFileSync(join(dir, records[0]), "utf8").trim().split("\n") const meta = JSON.parse(lines[0]) as Record -assert.equal(meta.skill_bundle_mode, "full") + assert.equal(meta.skill_bundle_mode, "full") assert.equal(meta.skill_bundle_version, "0.3.0") } finally { rmSync(dir, {recursive: true, force: true}) diff --git a/evals/runner/stages.test.ts b/evals/runner/stages.test.ts index b5a4c9a..13d261a 100644 --- a/evals/runner/stages.test.ts +++ b/evals/runner/stages.test.ts @@ -395,6 +395,16 @@ test("P2 fails on a non-boolean provisionable", () => { assert.equal(pcheck("P2", pre1Ctx({pre1})), false) }) +test("P2 fails on a malformed extra resource-type entry", () => { + const pre1 = pre1Artifact({}) + pre1.access_model!.resource_types = [ + {id: "user", traits: ["TRAIT_USER"]}, + {id: "group", traits: ["TRAIT_GROUP"]}, + {id: "team", traits: "x" as unknown as string[]}, + ] + assert.equal(pcheck("P2", pre1Ctx({pre1})), false) +}) + test("P3 passes on valid provenance", () => { assert.equal(pcheck("P3", pre1Ctx({})), true) }) diff --git a/evals/runner/stages.ts b/evals/runner/stages.ts index 6338686..b4b53a5 100644 --- a/evals/runner/stages.ts +++ b/evals/runner/stages.ts @@ -394,26 +394,29 @@ function accessModelDimensions(ctx: StageCtx): { return {rtMatch: false, entMatch: false, grantMatch: false, idcCount: 0, provisioning: "absent"} } const am = pre1.access_model + // Every entry must be shape-valid AND set-equal to expected: a malformed + // extra entry (e.g. {id: "team", traits: "x"}) fails the gate rather than + // being filtered out before the set-compare (the gate's contract: a + // malformed artifact fails its gate). const rtMatch = Array.isArray(am.resource_types) && + am.resource_types.every((rt) => isRecord(rt) && typeof rt.id === "string" && Array.isArray(rt.traits) && rt.traits.every((t) => typeof t === "string")) && setEqual( - am.resource_types - .filter((rt) => isRecord(rt) && typeof rt.id === "string" && Array.isArray(rt.traits) && rt.traits.every((t) => typeof t === "string")) - .map((rt) => canonicalResourceType(rt as {id: string; traits: string[]})), + am.resource_types.map((rt) => canonicalResourceType(rt as {id: string; traits: string[]})), expected.resource_types.map(canonicalResourceType), ) const entMatch = Array.isArray(am.entitlements) && + am.entitlements.every((e) => isRecord(e) && typeof e.slug === "string") && setEqual( - am.entitlements.filter((e) => isRecord(e) && typeof e.slug === "string").map((e) => (e as {slug: string}).slug), + am.entitlements.map((e) => (e as {slug: string}).slug), expected.entitlements.map((e) => e.slug), ) const grantMatch = Array.isArray(am.grants) && + am.grants.every((g) => isRecord(g) && typeof g.resource_type === "string" && typeof g.entitlement === "string" && typeof g.principal_type === "string") && setEqual( - am.grants - .filter((g) => isRecord(g) && typeof g.resource_type === "string" && typeof g.entitlement === "string" && typeof g.principal_type === "string") - .map((g) => canonicalGrant(g as {resource_type: string; entitlement: string; principal_type: string})), + am.grants.map((g) => canonicalGrant(g as {resource_type: string; entitlement: string; principal_type: string})), expected.grants.map(canonicalGrant), ) const idcCount = Array.isArray(am.id_compatibility) ? am.id_compatibility.length : 0 From 1847660ddf83ce4e918acd6406d53b02f572b115 Mon Sep 17 00:00:00 2001 From: Bjorn Tipling Date: Fri, 4 Sep 2026 09:42:37 +0000 Subject: [PATCH 3/6] =?UTF-8?q?CXF-220:=20address=20pr-review=20suggestion?= =?UTF-8?q?s=20=E2=80=94=20channel-driven=20pre1=20instructions,=20canned-?= =?UTF-8?q?dir=20guard,=20funnel=20score-input=20ENOENT,=20bundle=20versio?= =?UTF-8?q?n=20guard,=20pre1=20metric=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: c1-squire-dev[bot] --- evals/README.md | 6 ++++++ evals/runner/agent.test.ts | 1 + evals/runner/agent.ts | 5 +---- evals/runner/driver.ts | 6 +++++- evals/runner/drivers/tier0/driver.test.ts | 2 ++ evals/runner/drivers/tier0/driver.ts | 18 +++++++++++++++--- evals/runner/run.test.ts | 3 ++- evals/runner/run.ts | 5 ++++- evals/runner/skills_bundle.test.ts | 5 ++++- 9 files changed, 40 insertions(+), 11 deletions(-) diff --git a/evals/README.md b/evals/README.md index 6687f15..cb85a52 100644 --- a/evals/README.md +++ b/evals/README.md @@ -142,6 +142,12 @@ evidence) — the park-vs-proceed metric is measured from these fields. `handoff_discipline_verdict` is `true`; `recovery_cycles` is 0. Pre-1 records write no skipped rows (`skippedRows = []`). +Pre-1 `first_pass_rate` is passes over the gate count for the expected +decision — 4 for proceed runs (P0–P3), 3 for park runs (P0/P1/P4) — so it is +comparable only between runs of the same expected decision, not across the +two pre1 scenarios. `first_pass`/`attempts` on pre-1 stage rows mirror `pass` +(1/0) rather than transcript-derived attempts, unlike the funnel path. + ## Fixture traps The fixture (`evals/fixture/`) mirrors the documented failure modes: diff --git a/evals/runner/agent.test.ts b/evals/runner/agent.test.ts index 9e64b4c..3665551 100644 --- a/evals/runner/agent.test.ts +++ b/evals/runner/agent.test.ts @@ -41,6 +41,7 @@ const CHANNEL: RunChannel = { pre1Path: "/tmp/evals-run/pre1.json", handoffInstructions: 'Write it with driver.write_file: args {path: "", content: ""}.', completionInstructions: 'Then terminate the run with driver.complete_run: args {summary: "handoff written; funnel complete to human-activation boundary"}.', + pre1Instructions: 'Write it with driver.write_file: args {path: "", content: ""}. Then terminate the run with driver.complete_run: args {summary: "pre1 artifact written"}.', } test("buildPrompt carries the stop rule, handoff path, and credentials", () => { diff --git a/evals/runner/agent.ts b/evals/runner/agent.ts index 632ea58..5cb2d31 100644 --- a/evals/runner/agent.ts +++ b/evals/runner/agent.ts @@ -32,10 +32,7 @@ ${scenario.providerBrief} ${skillBundleSection(scenario)} (5) OUTPUT CONTRACT -Write pre1.json via driver.write_file: args {path: "${channel.pre1Path}", content: ""}. Schema: {decision: "proceed"|"park", access_model: {resource_types: [{id, traits}], entitlements: [{slug, display_name, grantable_principals, stable_id_shape}], grants: [{resource_type, entitlement, principal_type}], id_compatibility: [...], provisioning: [{resource_type, provisionable, justification}]}, sourcing: {spec_url, fetched_at, authority_rung, spec_bytes}, park_evidence: {spec_version_checked, missing_paths, vendor_doc, revisit_trigger}}. Include only the sections your decision requires: proceed -> access_model + sourcing; park -> park_evidence. - -(6) COMPLETION -Then terminate the run with driver.complete_run: args {summary: "pre1 artifact written"}.` +${channel.pre1Instructions} Schema: {decision: "proceed"|"park", access_model: {resource_types: [{id, traits}], entitlements: [{slug, display_name, grantable_principals, stable_id_shape}], grants: [{resource_type, entitlement, principal_type}], id_compatibility: [...], provisioning: [{resource_type, provisionable, justification}]}, sourcing: {spec_url, fetched_at, authority_rung, spec_bytes}, park_evidence: {spec_version_checked, missing_paths, vendor_doc, revisit_trigger}}. Include only the sections your decision requires: proceed -> access_model + sourcing; park -> park_evidence.` } const creds = scenario.fixture.basicAuth return `You are an eval agent implementing a read-only Directory API connector and completing the 12-step in-app authoring funnel. Your goal: implement the connector source files, walk the funnel to the human-activation boundary, and stop there. diff --git a/evals/runner/driver.ts b/evals/runner/driver.ts index c6a5719..f086ceb 100644 --- a/evals/runner/driver.ts +++ b/evals/runner/driver.ts @@ -63,6 +63,7 @@ export interface RunChannel { pre1Path: string handoffInstructions: string completionInstructions: string + pre1Instructions: string } export interface AgentRunRequest { @@ -74,6 +75,9 @@ export interface AgentRunRequest { model: string // The scenario id (pre-1 runs use it to select the canned replay set). scenarioId?: string + // The scenario kind (pre-1 runs use it to fail loudly when the canned + // replay set is missing instead of silently replaying the funnel set). + scenarioKind?: "funnel" | "pre1" // Declared scenario reasoning-effort pin; driver-interpreted — a driver // that can set agent reasoning effort applies it. Tier-0 ignores it. reasoningEffort?: "high" | "medium" | "low" @@ -102,5 +106,5 @@ export interface Driver { name: string provisioner: Provisioner agentDriver: AgentDriver - channelInstructions(channel: RunChannel): {handoffInstructions: string; completionInstructions: string} + channelInstructions(channel: RunChannel): {handoffInstructions: string; completionInstructions: string; pre1Instructions: string} } diff --git a/evals/runner/drivers/tier0/driver.test.ts b/evals/runner/drivers/tier0/driver.test.ts index f93d5ad..6127a50 100644 --- a/evals/runner/drivers/tier0/driver.test.ts +++ b/evals/runner/drivers/tier0/driver.test.ts @@ -66,6 +66,7 @@ test("tier0 agent driver replays a canned run that scores a schema-valid 16-line pre1Path: join(dir, "pre1.json"), handoffInstructions: "", completionInstructions: "", + pre1Instructions: "", } const result = await tier0.agentDriver.runAgent({kind: "agent", prompt: "p", toolSurface: TIER0_TOOL_SURFACE, channel, timeoutMs: 60_000, model: "m", ref: ""}) assert.equal(result.timedOut, false) @@ -187,6 +188,7 @@ function pre1Channel(dir: string): RunChannel { pre1Path: join(dir, "pre1.json"), handoffInstructions: "", completionInstructions: "", + pre1Instructions: "", } } diff --git a/evals/runner/drivers/tier0/driver.ts b/evals/runner/drivers/tier0/driver.ts index f3c8618..2cba148 100644 --- a/evals/runner/drivers/tier0/driver.ts +++ b/evals/runner/drivers/tier0/driver.ts @@ -175,6 +175,13 @@ const agentDriver: AgentDriver = { req.scenarioId !== undefined && existsSync(join(CANNED_DIR, `canned-${req.scenarioId}`)) ? join(CANNED_DIR, `canned-${req.scenarioId}`) : CANNED_DIR + // A pre1 run with no canned- dir must fail loudly: silently + // replaying the funnel set would write handoff.json instead of + // pre1.json and produce an exit-0 record with every P gate failed — + // indistinguishable from an agent failure. + if (req.scenarioKind === "pre1" && cannedDir === CANNED_DIR) { + throw new Error(`tier0: pre1 scenario ${req.scenarioId ?? ""} has no canned- replay directory — refusing to replay the funnel canned set`) + } const readCannedFrom = (name: string) => readFileSync(join(cannedDir, name), "utf8") const raw = JSON.parse(readCannedFrom("transcript.json")) as Record[] // Deep copy: the substitution must never mutate the canned file. @@ -186,12 +193,16 @@ const agentDriver: AgentDriver = { writeFileSync(req.channel.transcriptPath, JSON.stringify(events, null, 2)) if (existsSync(join(cannedDir, "pre1.json"))) { writeFileSync(req.channel.pre1Path, readCannedFrom("pre1.json")) + // Pre-1 runs skip the collector leg, so the agent leg writes the + // score-input itself. The funnel agent leg must NOT pre-write it: + // collectScoreInput relies on the ENOENT to detect a collector that + // returned without writing its output. + if (existsSync(join(cannedDir, "score-input.json"))) { + writeFileSync(req.channel.scoreInputPath, readCannedFrom("score-input.json")) + } } else { writeFileSync(req.channel.handoffPath, readCannedFrom("handoff.json")) } - if (existsSync(join(cannedDir, "score-input.json"))) { - writeFileSync(req.channel.scoreInputPath, readCannedFrom("score-input.json")) - } return {transcript: parseStream(events), timedOut: false, wallTimeMs: Date.now() - startedAt} } writeFileSync(req.channel.scoreInputPath, readCanned("score-input.json")) @@ -206,5 +217,6 @@ export const tier0: Driver = { channelInstructions: (channel) => ({ handoffInstructions: "Write it with driver.write_file: args {path: \"" + channel.handoffPath + "\", content: \"\"}.", completionInstructions: "Then terminate the run with driver.complete_run: args {summary: \"handoff written; funnel complete to human-activation boundary\"}.", + pre1Instructions: "Write it with driver.write_file: args {path: \"" + channel.pre1Path + "\", content: \"\"}. Then terminate the run with driver.complete_run: args {summary: \"pre1 artifact written\"}.", }), } diff --git a/evals/runner/run.test.ts b/evals/runner/run.test.ts index 343bb16..1d12057 100644 --- a/evals/runner/run.test.ts +++ b/evals/runner/run.test.ts @@ -73,7 +73,7 @@ function makeDriver(provisioner: Provisioner): Driver { agentDriver: { runAgent: async () => ({transcript: emptyStream(), timedOut: false, wallTimeMs: 0}), }, - channelInstructions: () => ({handoffInstructions: "", completionInstructions: ""}), + channelInstructions: () => ({handoffInstructions: "", completionInstructions: "", pre1Instructions: ""}), } } @@ -90,6 +90,7 @@ function makeChannel(dir: string): RunChannel { pre1Path: join(dir, "pre1.json"), handoffInstructions: "", completionInstructions: "", + pre1Instructions: "", } } diff --git a/evals/runner/run.ts b/evals/runner/run.ts index afab9e6..30ae019 100644 --- a/evals/runner/run.ts +++ b/evals/runner/run.ts @@ -201,6 +201,7 @@ function buildChannel(out: string, runId: string): RunChannel { pre1Path: join(runDir, "pre1.json"), handoffInstructions: "", completionInstructions: "", + pre1Instructions: "", } } @@ -340,9 +341,10 @@ async function main(): Promise { const runId = runIdFor(scenario, new Date()) const startedAt = new Date().toISOString() const channel = buildChannel(cli.out, runId) - const {handoffInstructions, completionInstructions} = driver.channelInstructions(channel) + const {handoffInstructions, completionInstructions, pre1Instructions} = driver.channelInstructions(channel) channel.handoffInstructions = handoffInstructions channel.completionInstructions = completionInstructions + channel.pre1Instructions = pre1Instructions mkdirSync(channel.runDir, {recursive: true}) // Provision + readiness retry loop: 3 attempts, teardown between. @@ -360,6 +362,7 @@ async function main(): Promise { reasoningEffort: scenario.reasoningEffort, ref: cli.ref, scenarioId: scenario.id, + scenarioKind: scenario.kind, }) const {transcript, timedOut, wallTimeMs} = result if (timedOut) { diff --git a/evals/runner/skills_bundle.test.ts b/evals/runner/skills_bundle.test.ts index 9f04642..2734152 100644 --- a/evals/runner/skills_bundle.test.ts +++ b/evals/runner/skills_bundle.test.ts @@ -56,7 +56,10 @@ test("(a) each SKILL.md exists with the locked frontmatter contract", () => { assert.ok(fm.description.includes("Do not use when"), `${name}: description must carry the anti-trigger sentence`) // The plan's locked versions are intentionally non-uniform (the two new // pre-1 skills ship at 0.1.0); each skill's frontmatter must match its - // locked per-skill version. + // locked per-skill version. Every bundled skill must have a locked + // version entry — adding a skill to the bundle without recording its + // version fails here. + assert.ok(SKILL_VERSIONS[name] !== undefined, `${name}: no locked SKILL_VERSIONS entry`) assert.equal(fm.version, SKILL_VERSIONS[name], `${name}: frontmatter version must equal the locked per-skill version`) } }) From 7a598014c482e49194ff75e2b3620a020b95d1f4 Mon Sep 17 00:00:00 2001 From: Bjorn Tipling Date: Fri, 4 Sep 2026 09:51:39 +0000 Subject: [PATCH 4/6] =?UTF-8?q?CXF-220:=20address=20pr-review=20round=202?= =?UTF-8?q?=20=E2=80=94=20pre1Instructions=20contract=20comment,=20tier0?= =?UTF-8?q?=20guard=20tests,=20drop=20dead=20readBundle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: c1-squire-dev[bot] --- evals/runner/driver.ts | 4 ++++ evals/runner/drivers/tier0/driver.test.ts | 29 ++++++++++++++++++++++- evals/runner/skills_bundle.test.ts | 1 - 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/evals/runner/driver.ts b/evals/runner/driver.ts index f086ceb..2fb54e8 100644 --- a/evals/runner/driver.ts +++ b/evals/runner/driver.ts @@ -63,6 +63,10 @@ export interface RunChannel { pre1Path: string handoffInstructions: string completionInstructions: string + // Pre-1 runs: the driver-supplied instruction carrying BOTH the artifact + // write (with the pre1Path) and the complete_run termination verb — a + // driver must supply both halves or the pre-1 agent has no stop + // instruction. pre1Instructions: string } diff --git a/evals/runner/drivers/tier0/driver.test.ts b/evals/runner/drivers/tier0/driver.test.ts index 6127a50..ba46391 100644 --- a/evals/runner/drivers/tier0/driver.test.ts +++ b/evals/runner/drivers/tier0/driver.test.ts @@ -3,7 +3,7 @@ import {test} from "node:test" import assert from "node:assert/strict" import {execFile} from "node:child_process" import {promisify} from "node:util" -import {mkdtempSync, readFileSync, readdirSync, rmSync} from "node:fs" +import {existsSync, mkdtempSync, readFileSync, readdirSync, rmSync} from "node:fs" import {tmpdir} from "node:os" import {join} from "node:path" import {tier0, TIER0_TOOL_SURFACE} from "./driver.ts" @@ -74,6 +74,9 @@ test("tier0 agent driver replays a canned run that scores a schema-valid 16-line assert.equal(result.transcript.stageAttempts["S11"], 1) const handoff = JSON.parse(readFileSync(channel.handoffPath, "utf8")) as Record assert.equal(Object.keys(handoff).length, 10) + // The funnel agent leg must NOT pre-write score-input.json — collectScoreInput + // relies on the ENOENT to detect a collector that returned without writing. + assert.equal(existsSync(channel.scoreInputPath), false) // The collector leg writes score-input.json into the run channel. await tier0.agentDriver.runAgent({kind: "collector", prompt: "p", toolSurface: TIER0_TOOL_SURFACE, channel, timeoutMs: 60_000, model: "m", ref: ""}) const scoreInput = JSON.parse(readFileSync(channel.scoreInputPath, "utf8")) as ScoreInput @@ -205,6 +208,7 @@ async function replayPre1(scenarioPath: string, scenarioId: string): Promise<{sc model: "m", ref: "", scenarioId, + scenarioKind: "pre1", }) assert.equal(result.timedOut, false) const pre1 = JSON.parse(readFileSync(channel.pre1Path, "utf8")) as Pre1Artifact @@ -245,3 +249,26 @@ test("a pre1-noiam-park replay writes pre1.json and scores P0/P1/P4 pass", async assert.deepEqual(scored.stageRows.map((r) => r.stage), ["P0", "P1", "P4"]) assert.ok(scored.stageRows.every((r) => r.pass)) }) + +test("a pre1 run with no canned- dir rejects loudly instead of replaying the funnel set", async () => { + const dir = mkdtempSync(join(tmpdir(), "tier0-pre1-missing-")) + try { + const channel = pre1Channel(dir) + await assert.rejects( + tier0.agentDriver.runAgent({ + kind: "agent", + prompt: "p", + toolSurface: TIER0_TOOL_SURFACE, + channel, + timeoutMs: 60_000, + model: "m", + ref: "", + scenarioId: "pre1-no-such-canned-dir", + scenarioKind: "pre1", + }), + /refusing to replay the funnel canned set/, + ) + } finally { + rmSync(dir, {recursive: true, force: true}) + } +}) diff --git a/evals/runner/skills_bundle.test.ts b/evals/runner/skills_bundle.test.ts index 2734152..bdccb95 100644 --- a/evals/runner/skills_bundle.test.ts +++ b/evals/runner/skills_bundle.test.ts @@ -46,7 +46,6 @@ function parseFrontmatter(file: string): Record { } test("(a) each SKILL.md exists with the locked frontmatter contract", () => { - const bundle = readBundle() for (const name of SKILLS) { const file = readFileSync(join("skills", name, "SKILL.md"), "utf8") const fm = parseFrontmatter(file) From 2823a2970dd64353f338227b2b97a57dd8b7ee61 Mon Sep 17 00:00:00 2001 From: Bjorn Tipling Date: Fri, 4 Sep 2026 10:00:47 +0000 Subject: [PATCH 5/6] =?UTF-8?q?CXF-220:=20address=20pr-review=20round=203?= =?UTF-8?q?=20=E2=80=94=20narrow=20baseline=20skip=20to=20pre1=20shape,=20?= =?UTF-8?q?drop=20dead=20pre1=20score-input=20write,=20single-source=20ski?= =?UTF-8?q?llBundle=20validation,=20document=20driver=20contract?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: c1-squire-dev[bot] --- evals/runner/baseline.ts | 8 +- evals/runner/drivers/README.md | 16 ++- .../score-input.json | 1 - .../canned-pre1-noiam-park/score-input.json | 1 - evals/runner/drivers/tier0/driver.ts | 7 -- evals/runner/scenario.ts | 107 +++++++----------- 6 files changed, 60 insertions(+), 80 deletions(-) delete mode 100644 evals/runner/drivers/tier0/canned-pre1-directory-proceed/score-input.json delete mode 100644 evals/runner/drivers/tier0/canned-pre1-noiam-park/score-input.json diff --git a/evals/runner/baseline.ts b/evals/runner/baseline.ts index 284cfa1..fd561d6 100644 --- a/evals/runner/baseline.ts +++ b/evals/runner/baseline.ts @@ -76,12 +76,14 @@ function parseRecord(file: string, lines: string[]): RunRecord | null { // Pre-1 records (stage rows begin with P0, not S0) can land in // evals/results/ locally without breaking baseline generation: skip them - // with a one-line stderr warning, never fatal. Funnel-shaped records - // validate exactly as before. + // with a one-line stderr warning, never fatal. The predicate is narrowed to + // the pre-1 shape (P-led stages) so a corrupt funnel record that dropped + // its S0 row still fails the canonical-row check below instead of being + // silently skipped. const firstRowRaw = parseLine(file, 2, lines[1]) if (typeof firstRowRaw === "object" && firstRowRaw !== null && !Array.isArray(firstRowRaw)) { const firstRow = firstRowRaw as Record - if (firstRow.stage !== "S0") { + if (typeof firstRow.stage === "string" && firstRow.stage.startsWith("P")) { console.error(`WARNING: skipping ${file}: record does not begin with the S0 funnel row`) return null } diff --git a/evals/runner/drivers/README.md b/evals/runner/drivers/README.md index b97a769..f778569 100644 --- a/evals/runner/drivers/README.md +++ b/evals/runner/drivers/README.md @@ -19,10 +19,17 @@ through its registry (`--driver `, default `tier0`). score an empty transcript as an all-fail outcome). The driver MUST honor `req.timeoutMs` and return `timedOut: true` when the run exceeds it (the runner scores the partial stream on timeout). `req.ref` is the git ref - under test, driver-interpreted. + under test, driver-interpreted. `req.scenarioId` names the scenario and + `req.scenarioKind` is `"funnel"` or `"pre1"` — a pre-1 run writes the + artifact to `channel.pre1Path` (never `channel.handoffPath`) and skips the + collector leg. - **RunChannel** — the runner-owned local file contract: `runDir`, - `handoffPath`, `scoreInputPath`, `transcriptPath`, plus the driver-supplied - `handoffInstructions`/`completionInstructions` prompt text. + `handoffPath`, `scoreInputPath`, `transcriptPath`, `pre1Path`, plus the + driver-supplied `handoffInstructions`/`completionInstructions`/ + `pre1Instructions` prompt text. `pre1Instructions` must carry BOTH the + artifact write (with the `pre1Path`) and the `complete_run` termination + verb — a driver that supplies only the write half leaves a pre-1 agent + with no stop instruction. ## Reserved control names @@ -46,6 +53,9 @@ The driver must: 3. Persist the raw transcript to `channel.transcriptPath` for the record. 4. Return the parsed transcript in `AgentRunResult.transcript` (the runner scores that stream; it never re-reads the raw transcript). +5. For `scenarioKind === "pre1"`: write the agent's `pre1.json` artifact to + `channel.pre1Path` (not `channel.handoffPath`) and skip the collector leg + — the runner reads the artifact itself and scores the P0–P4 gate set. ## Readiness gate diff --git a/evals/runner/drivers/tier0/canned-pre1-directory-proceed/score-input.json b/evals/runner/drivers/tier0/canned-pre1-directory-proceed/score-input.json deleted file mode 100644 index 4609451..0000000 --- a/evals/runner/drivers/tier0/canned-pre1-directory-proceed/score-input.json +++ /dev/null @@ -1 +0,0 @@ -{"run_id": "pre1-canned", "draft": {"required_source_files": {}, "source_files": [], "config_schema": {"fields": []}, "runtime_schema": {"fields": []}}, "connector_config": {}, "evidence": {}, "build_run": {}, "tenant_counts": {"users": null, "groups": null, "memberships": null}, "resource_ids": {"users": [], "groups": []}} diff --git a/evals/runner/drivers/tier0/canned-pre1-noiam-park/score-input.json b/evals/runner/drivers/tier0/canned-pre1-noiam-park/score-input.json deleted file mode 100644 index 4609451..0000000 --- a/evals/runner/drivers/tier0/canned-pre1-noiam-park/score-input.json +++ /dev/null @@ -1 +0,0 @@ -{"run_id": "pre1-canned", "draft": {"required_source_files": {}, "source_files": [], "config_schema": {"fields": []}, "runtime_schema": {"fields": []}}, "connector_config": {}, "evidence": {}, "build_run": {}, "tenant_counts": {"users": null, "groups": null, "memberships": null}, "resource_ids": {"users": [], "groups": []}} diff --git a/evals/runner/drivers/tier0/driver.ts b/evals/runner/drivers/tier0/driver.ts index 2cba148..0125a48 100644 --- a/evals/runner/drivers/tier0/driver.ts +++ b/evals/runner/drivers/tier0/driver.ts @@ -193,13 +193,6 @@ const agentDriver: AgentDriver = { writeFileSync(req.channel.transcriptPath, JSON.stringify(events, null, 2)) if (existsSync(join(cannedDir, "pre1.json"))) { writeFileSync(req.channel.pre1Path, readCannedFrom("pre1.json")) - // Pre-1 runs skip the collector leg, so the agent leg writes the - // score-input itself. The funnel agent leg must NOT pre-write it: - // collectScoreInput relies on the ENOENT to detect a collector that - // returned without writing its output. - if (existsSync(join(cannedDir, "score-input.json"))) { - writeFileSync(req.channel.scoreInputPath, readCannedFrom("score-input.json")) - } } else { writeFileSync(req.channel.handoffPath, readCannedFrom("handoff.json")) } diff --git a/evals/runner/scenario.ts b/evals/runner/scenario.ts index 35d2909..4332f28 100644 --- a/evals/runner/scenario.ts +++ b/evals/runner/scenario.ts @@ -98,6 +98,46 @@ function requireStringArray(obj: Record, key: string, where: st return v as string[] } +// Single source for skillBundle validation + the bundle-version drift check — +// shared by the funnel and pre-1 branches so the pin check cannot diverge. +function parseSkillBundle(data: Record): SkillBundleConfig { + if (!isRecord(data.skillBundle)) throw new Error("scenario field skillBundle missing or not an object") + const mode = data.skillBundle.mode + if (mode !== "none" && mode !== "guide-only" && mode !== "full") { + throw new Error('scenario field skillBundle.mode must be "none", "guide-only", or "full"') + } + const skillBundle = { + mode: mode as "none" | "guide-only" | "full", + version: requireString(data.skillBundle, "version", "skillBundle"), + } + if (skillBundle.mode === "full") { + // Fail fast when the scenario's bundle version drifts from the mounted + // bundle: run records must never be stamped with a version that does not + // match the skills actually mounted (bundle.json is the single source). + // Resolve relative to this module so the check is not cwd-dependent, and + // wrap read/parse errors like the scenario file's own. + const bundlePath = fileURLToPath(new URL("../skills-bundle/bundle.json", import.meta.url)) + let bundleRaw: string + try { + bundleRaw = readFileSync(bundlePath, "utf8") + } catch (err) { + throw new Error(`cannot read skill bundle ${bundlePath}: ${(err as Error).message}`) + } + let bundle: {version?: unknown} + try { + bundle = JSON.parse(bundleRaw) as {version?: unknown} + } catch (err) { + throw new Error(`skill bundle ${bundlePath} is not valid JSON: ${(err as Error).message}`) + } + if (typeof bundle.version !== "string" || bundle.version !== skillBundle.version) { + throw new Error( + `scenario skillBundle.version ${skillBundle.version} does not match evals/skills-bundle/bundle.json version ${bundle.version}`, + ) + } + } + return skillBundle +} + export function loadScenario(path: string): Scenario { let raw: string try { @@ -162,41 +202,7 @@ const id = requireString(data, "id", "scenario") memberships: requireNumber(data.expected, "memberships", "expected"), } - if (!isRecord(data.skillBundle)) throw new Error("scenario field skillBundle missing or not an object") - const mode = data.skillBundle.mode - if (mode !== "none" && mode !== "guide-only" && mode !== "full") { - throw new Error('scenario field skillBundle.mode must be "none", "guide-only", or "full"') - } - const skillBundle = { - mode: mode as "none" | "guide-only" | "full", - version: requireString(data.skillBundle, "version", "skillBundle"), - } - if (skillBundle.mode === "full") { - // Fail fast when the scenario's bundle version drifts from the mounted - // bundle: run records must never be stamped with a version that does not - // match the skills actually mounted (bundle.json is the single source). - // Resolve relative to this module so the check is not cwd-dependent, and - // wrap read/parse errors like the scenario file's own. - const bundlePath = fileURLToPath(new URL("../skills-bundle/bundle.json", import.meta.url)) - let bundleRaw: string - try { - bundleRaw = readFileSync(bundlePath, "utf8") - } catch (err) { - throw new Error(`cannot read skill bundle ${bundlePath}: ${(err as Error).message}`) - } - let bundle: {version?: unknown} - try { - bundle = JSON.parse(bundleRaw) as {version?: unknown} - } catch (err) { - throw new Error(`skill bundle ${bundlePath} is not valid JSON: ${(err as Error).message}`) - } - if (typeof bundle.version !== "string" || bundle.version !== skillBundle.version) { - throw new Error( - `scenario skillBundle.version ${skillBundle.version} does not match evals/skills-bundle/bundle.json version ${bundle.version}`, - ) - } - } - + const skillBundle = parseSkillBundle(data) const reasoningEffort = data.reasoningEffort if (reasoningEffort !== "high" && reasoningEffort !== "medium" && reasoningEffort !== "low") { throw new Error('scenario field reasoningEffort must be "high", "medium", or "low"') @@ -250,36 +256,7 @@ const id = requireString(data, "id", "scenario") readinessTools = requireStringArray(data, "readinessTools", "scenario") } - if (!isRecord(data.skillBundle)) throw new Error("scenario field skillBundle missing or not an object") - const mode = data.skillBundle.mode - if (mode !== "none" && mode !== "guide-only" && mode !== "full") { - throw new Error('scenario field skillBundle.mode must be "none", "guide-only", or "full"') - } - const skillBundle = { - mode: mode as "none" | "guide-only" | "full", - version: requireString(data.skillBundle, "version", "skillBundle"), - } - if (skillBundle.mode === "full") { - const bundlePath = fileURLToPath(new URL("../skills-bundle/bundle.json", import.meta.url)) - let bundleRaw: string - try { - bundleRaw = readFileSync(bundlePath, "utf8") - } catch (err) { - throw new Error(`cannot read skill bundle ${bundlePath}: ${(err as Error).message}`) - } - let bundle: {version?: unknown} - try { - bundle = JSON.parse(bundleRaw) as {version?: unknown} - } catch (err) { - throw new Error(`skill bundle ${bundlePath} is not valid JSON: ${(err as Error).message}`) - } - if (typeof bundle.version !== "string" || bundle.version !== skillBundle.version) { - throw new Error( - `scenario skillBundle.version ${skillBundle.version} does not match evals/skills-bundle/bundle.json version ${bundle.version}`, - ) - } - } - + const skillBundle = parseSkillBundle(data) const reasoningEffort = data.reasoningEffort if (reasoningEffort !== "high" && reasoningEffort !== "medium" && reasoningEffort !== "low") { throw new Error('scenario field reasoningEffort must be "high", "medium", or "low"') From e71eca3e3dfcd2a9a22e0d6ac67e2372f69802eb Mon Sep 17 00:00:00 2001 From: Bjorn Tipling Date: Fri, 4 Sep 2026 16:26:12 +0000 Subject: [PATCH 6/6] CXF-220: enforce expected park evidence in P4, make bundle.json the per-skill version source - P4 now compares the artifact's spec_version_checked against the scenario's expectedParkEvidence and requires every expected missing path to be present (superset allowed), so a park run with generic boilerplate evidence no longer scores a perfect P0/P1/P4. - bundle.json entries carry the per-skill version; the bundle test asserts each SKILL.md frontmatter version against its bundle entry instead of a test-local SKILL_VERSIONS constant, keeping bundle.json the single source for version provenance. Co-authored-by: c1-squire-dev[bot] --- evals/runner/skills_bundle.test.ts | 30 +++++-------- evals/runner/stages.test.ts | 72 ++++++++++++++++++++++++++++-- evals/runner/stages.ts | 9 +++- evals/skills-bundle/bundle.json | 14 +++--- 4 files changed, 94 insertions(+), 31 deletions(-) diff --git a/evals/runner/skills_bundle.test.ts b/evals/runner/skills_bundle.test.ts index bdccb95..735bb16 100644 --- a/evals/runner/skills_bundle.test.ts +++ b/evals/runner/skills_bundle.test.ts @@ -16,20 +16,9 @@ const RUN = "evals/runner/run.ts" const BUNDLE = "evals/skills-bundle/bundle.json" const SKILLS = ["author-in-app-connector", "read-authoring-contract", "write-connector-source", "build-and-test", "deploy-and-activate", "design-access-model", "source-openapi-spec"] const VERSION = "0.3.0" -// The plan's locked versions are intentionally non-uniform: the two new -// pre-1 skills ship at 0.1.0 while the funnel skills keep their versions. -const SKILL_VERSIONS: Record = { - "author-in-app-connector": "0.2.1", - "read-authoring-contract": "0.2.0", - "write-connector-source": "0.2.0", - "build-and-test": "0.2.0", - "deploy-and-activate": "0.2.0", - "design-access-model": "0.1.0", - "source-openapi-spec": "0.1.0", -} -function readBundle(): {version: string; skills: {name: string; path: string}[]} { - return JSON.parse(readFileSync(BUNDLE, "utf8")) as {version: string; skills: {name: string; path: string}[]} +function readBundle(): {version: string; skills: {name: string; version: string; path: string}[]} { + return JSON.parse(readFileSync(BUNDLE, "utf8")) as {version: string; skills: {name: string; version: string; path: string}[]} } // The repo has no YAML dependency: parse the frontmatter block (between the @@ -46,6 +35,8 @@ function parseFrontmatter(file: string): Record { } test("(a) each SKILL.md exists with the locked frontmatter contract", () => { + const bundle = readBundle() + const bundleVersions: Record = Object.fromEntries(bundle.skills.map((s) => [s.name, s.version])) for (const name of SKILLS) { const file = readFileSync(join("skills", name, "SKILL.md"), "utf8") const fm = parseFrontmatter(file) @@ -53,13 +44,12 @@ test("(a) each SKILL.md exists with the locked frontmatter contract", () => { assert.ok(fm.description, `${name}: frontmatter description missing`) assert.ok(fm.description.includes("Use when"), `${name}: description must carry the trigger sentence`) assert.ok(fm.description.includes("Do not use when"), `${name}: description must carry the anti-trigger sentence`) - // The plan's locked versions are intentionally non-uniform (the two new - // pre-1 skills ship at 0.1.0); each skill's frontmatter must match its - // locked per-skill version. Every bundled skill must have a locked - // version entry — adding a skill to the bundle without recording its - // version fails here. - assert.ok(SKILL_VERSIONS[name] !== undefined, `${name}: no locked SKILL_VERSIONS entry`) - assert.equal(fm.version, SKILL_VERSIONS[name], `${name}: frontmatter version must equal the locked per-skill version`) + // bundle.json is the single source for per-skill versions: every bundled + // skill must carry a version entry, and the SKILL.md frontmatter must + // match it — bumping a skill body without bumping its bundle entry fails. + const entryVersion = bundleVersions[name] + assert.ok(entryVersion !== undefined, `${name}: no version entry in bundle.json`) + assert.equal(fm.version, entryVersion, `${name}: frontmatter version must equal the bundle.json entry version`) } }) diff --git a/evals/runner/stages.test.ts b/evals/runner/stages.test.ts index 13d261a..f2200d9 100644 --- a/evals/runner/stages.test.ts +++ b/evals/runner/stages.test.ts @@ -3,6 +3,7 @@ import {test} from "node:test" import assert from "node:assert/strict" import {PRE1_STAGES, STAGES, sanitizeHandoffValue, type Handoff, type Pre1Artifact, type ScoreInput, type StageCtx} from "./stages.ts" import {parseStream, type ParsedStream} from "./stream.ts" +import type {ExpectedParkEvidence} from "./scenario.ts" const HANDOFF_PATH = "/tmp/evals-run/handoff.json" @@ -429,6 +430,19 @@ test("P3 fails when spec_url is empty", () => { assert.equal(pcheck("P3", pre1Ctx({pre1})), false) }) +function parkExpected(overrides: Partial = {}): {decision: "park"; parkEvidence: ExpectedParkEvidence} { + return { + decision: "park", + parkEvidence: { + spec_version_checked: "1.2.0", + missing_paths: ["/v1/users", "/v1/groups"], + vendor_doc: "console only", + revisit_trigger: "ships an API", + ...overrides, + }, + } +} + test("P4 passes on complete park evidence", () => { const pre1 = pre1Artifact({ decision: "park", @@ -439,7 +453,7 @@ test("P4 passes on complete park evidence", () => { revisit_trigger: "ships an API", }, }) - assert.equal(pcheck("P4", pre1Ctx({pre1})), true) + assert.equal(pcheck("P4", pre1Ctx({pre1, expected: parkExpected()})), true) }) test("P4 fails when missing_paths is empty", () => { @@ -452,7 +466,7 @@ test("P4 fails when missing_paths is empty", () => { revisit_trigger: "ships an API", }, }) - assert.equal(pcheck("P4", pre1Ctx({pre1})), false) + assert.equal(pcheck("P4", pre1Ctx({pre1, expected: parkExpected()})), false) }) test("P4 fails when any park-evidence field is empty", () => { @@ -467,6 +481,58 @@ test("P4 fails when any park-evidence field is empty", () => { }, }) pre1.park_evidence![key as "spec_version_checked" | "vendor_doc" | "revisit_trigger"] = "" - assert.equal(pcheck("P4", pre1Ctx({pre1})), false, `${key} empty should fail P4`) + assert.equal(pcheck("P4", pre1Ctx({pre1, expected: parkExpected()})), false, `${key} empty should fail P4`) } }) + +test("P4 fails when the checked spec version differs from the expected", () => { + const pre1 = pre1Artifact({ + decision: "park", + park_evidence: { + spec_version_checked: "2.0.0", + missing_paths: ["/v1/users", "/v1/groups"], + vendor_doc: "console only", + revisit_trigger: "ships an API", + }, + }) + assert.equal(pcheck("P4", pre1Ctx({pre1, expected: parkExpected()})), false) +}) + +test("P4 fails when an expected missing path is absent from the evidence", () => { + const pre1 = pre1Artifact({ + decision: "park", + park_evidence: { + spec_version_checked: "1.2.0", + missing_paths: ["/v1/users"], + vendor_doc: "console only", + revisit_trigger: "ships an API", + }, + }) + assert.equal(pcheck("P4", pre1Ctx({pre1, expected: parkExpected()})), false) +}) + +test("P4 passes when the evidence lists extra missing paths beyond the expected", () => { + const pre1 = pre1Artifact({ + decision: "park", + park_evidence: { + spec_version_checked: "1.2.0", + missing_paths: ["/v1/users", "/v1/groups", "/v1/teams"], + vendor_doc: "console only", + revisit_trigger: "ships an API", + }, + }) + assert.equal(pcheck("P4", pre1Ctx({pre1, expected: parkExpected()})), true) +}) + +test("P4 fails when the scenario carries no expected park evidence", () => { + const pre1 = pre1Artifact({ + decision: "park", + park_evidence: { + spec_version_checked: "1.2.0", + missing_paths: ["/v1/users", "/v1/groups"], + vendor_doc: "console only", + revisit_trigger: "ships an API", + }, + }) + assert.equal(pcheck("P4", pre1Ctx({pre1, expected: {decision: "park"}})), false) +}) diff --git a/evals/runner/stages.ts b/evals/runner/stages.ts index b4b53a5..57997b9 100644 --- a/evals/runner/stages.ts +++ b/evals/runner/stages.ts @@ -508,7 +508,14 @@ const P4: Stage = { if (pre1 === null || pre1 === undefined || !isRecord(pre1.park_evidence)) return false const pe = pre1.park_evidence if (!nonEmptyString(pe.spec_version_checked) || !nonEmptyString(pe.vendor_doc) || !nonEmptyString(pe.revisit_trigger)) return false - return Array.isArray(pe.missing_paths) && pe.missing_paths.length > 0 && pe.missing_paths.every((p) => typeof p === "string" && p.length > 0) + if (!Array.isArray(pe.missing_paths) || pe.missing_paths.length === 0 || !pe.missing_paths.every((p) => typeof p === "string" && p.length > 0)) return false + // The scenario's expected park evidence is enforced, not just documented: + // the checked spec version must match exactly and every expected missing + // path must be present (the artifact may list more). + const expected = ctx.expected?.parkEvidence + if (expected === undefined) return false + if (pe.spec_version_checked !== expected.spec_version_checked) return false + return expected.missing_paths.every((p) => pe.missing_paths.includes(p)) }, evidence: (ctx) => { const pre1 = ctx.pre1 diff --git a/evals/skills-bundle/bundle.json b/evals/skills-bundle/bundle.json index b54b616..1815d6b 100644 --- a/evals/skills-bundle/bundle.json +++ b/evals/skills-bundle/bundle.json @@ -1,12 +1,12 @@ { "version": "0.3.0", "skills": [ - {"name": "author-in-app-connector", "path": "../../skills/author-in-app-connector/SKILL.md"}, - {"name": "read-authoring-contract", "path": "../../skills/read-authoring-contract/SKILL.md"}, - {"name": "write-connector-source", "path": "../../skills/write-connector-source/SKILL.md"}, - {"name": "build-and-test", "path": "../../skills/build-and-test/SKILL.md"}, - {"name": "deploy-and-activate", "path": "../../skills/deploy-and-activate/SKILL.md"}, - {"name": "design-access-model", "path": "../../skills/design-access-model/SKILL.md"}, - {"name": "source-openapi-spec", "path": "../../skills/source-openapi-spec/SKILL.md"} + {"name": "author-in-app-connector", "version": "0.2.1", "path": "../../skills/author-in-app-connector/SKILL.md"}, + {"name": "read-authoring-contract", "version": "0.2.0", "path": "../../skills/read-authoring-contract/SKILL.md"}, + {"name": "write-connector-source", "version": "0.2.0", "path": "../../skills/write-connector-source/SKILL.md"}, + {"name": "build-and-test", "version": "0.2.0", "path": "../../skills/build-and-test/SKILL.md"}, + {"name": "deploy-and-activate", "version": "0.2.0", "path": "../../skills/deploy-and-activate/SKILL.md"}, + {"name": "design-access-model", "version": "0.1.0", "path": "../../skills/design-access-model/SKILL.md"}, + {"name": "source-openapi-spec", "version": "0.1.0", "path": "../../skills/source-openapi-spec/SKILL.md"} ] }