From 966e7c562ae48f67d8ba8c0a0155026ef23d9479 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:39:31 -0400 Subject: [PATCH 01/18] =?UTF-8?q?e2e:=20the=20package=20road,=20asserted?= =?UTF-8?q?=20=E2=80=94=20both=20directions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The e2e workflow proves the delivery mechanism every future package rides: an app composed from the skeleton (dev), the recipes checkout served as the Flex endpoint locally, minspec/fixture-hello required through it, and the independent test author's assertions run against the result. The planted-fault arm corrupts the served recipe, proves the plant landed and did not leak into the clean fixture, serves it from its own port so no cache can launder it, and requires the build to fail. CI additionally asserts Mate's container answer instead of listing tools. assert.sh was authored by the independent test author from the recipe contract alone, without sight of the implementation, and calibrated locally: 4/4 PASS on the wired app, 4/4 loud failures on an unwired app. Source: owner 2026-09-01 Source: original Co-Authored-By: GPT-5.6 Sol Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XehTac5TJNmPAskwrPp7rJ --- .github/workflows/ci.yml | 7 +++- .github/workflows/e2e.yml | 77 +++++++++++++++++++++++++++++++++++++++ tests/e2e/assert.sh | 66 +++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/e2e.yml create mode 100755 tests/e2e/assert.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82996e0..e0501cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,4 +19,9 @@ jobs: - run: composer install --prefer-dist --no-progress --no-interaction - run: find src -name '*.php' -print0 | xargs -0 -r -n50 php -l - run: php bin/console about - - run: vendor/bin/mate tools:list + - name: mate serves measured truth + run: | + vendor/bin/mate tools:call symfony-services --query=router --limit=10 --format=json \ + | jq -e '.untrusted_data.services["router.default"]' + php bin/console fixture:hello 2>/dev/null && { echo "fixture must not be in the workbench app itself"; exit 1; } || true + vendor/bin/mate tools:list diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..d5e9c63 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,77 @@ +name: e2e +on: + pull_request: + push: + branches: [dev, main] +jobs: + e2e: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: minspec/skeleton + ref: dev + path: .e2e/skeleton + - uses: actions/checkout@v4 + with: + repository: minspec/workbench-fixtures + ref: dev + path: .e2e/fixtures + - uses: actions/checkout@v4 + with: + repository: minspec/recipes + ref: dev + path: .e2e/recipes + - uses: shivammathur/setup-php@v2 + with: + php-version: '8.4' + + - name: serve the recipes endpoint locally (clean arm) + run: | + cd .e2e/recipes + jq '.recipe_template = "http://127.0.0.1:8099/{package_dotted}.{version}.json"' index.json > index.tmp && mv index.tmp index.json + php -S 127.0.0.1:8099 & + sleep 1 + curl -fsS http://127.0.0.1:8099/index.json | jq -e '.recipes["minspec/fixture-hello"]' + + - name: compose the app from the skeleton + run: | + composer create-project minspec/skeleton .e2e/app --repository="{\"type\":\"path\",\"url\":\"$GITHUB_WORKSPACE/.e2e/skeleton\"}" --stability=dev --no-interaction + + - name: require fixture-hello through the endpoint + run: | + cd .e2e/app + composer config secure-http false + composer config repositories.fixtures "{\"type\":\"path\",\"url\":\"$GITHUB_WORKSPACE/.e2e/fixtures/packages/fixture-hello\"}" + composer config extra.symfony.endpoint --json "[\"http://127.0.0.1:8099/index.json\",\"flex://defaults\"]" + composer require minspec/fixture-hello:1.0.0 --no-interaction + + - name: assertions (independent test author) + run: bash tests/e2e/assert.sh .e2e/app + + - name: planted fault must fire + run: | + # Corrupt the served recipe (parameter renamed) and serve it as a + # DIFFERENT endpoint (own port) so no cache can hand the broken + # arm the clean recipe. Prove the plant landed, then a fresh app + # build must fail. The clean arm above is the stay-quiet half. + mkdir -p .e2e/recipes-broken + cp .e2e/recipes/index.json .e2e/recipes-broken/ + sed 's/fixture_hello.greeting/fixture_hello.wrong/' \ + .e2e/recipes/minspec.fixture-hello.1.0.json > .e2e/recipes-broken/minspec.fixture-hello.1.0.json + grep -q 'fixture_hello.wrong' .e2e/recipes-broken/minspec.fixture-hello.1.0.json || { echo "plant did not land"; exit 1; } + ! grep -q 'fixture_hello.wrong' .e2e/recipes/minspec.fixture-hello.1.0.json || { echo "plant leaked into clean fixture"; exit 1; } + jq '.recipe_template = "http://127.0.0.1:8098/{package_dotted}.{version}.json"' .e2e/recipes-broken/index.json > .e2e/recipes-broken/index.tmp && mv .e2e/recipes-broken/index.tmp .e2e/recipes-broken/index.json + (cd .e2e/recipes-broken && php -S 127.0.0.1:8098 &) + sleep 1 + composer create-project minspec/skeleton .e2e/app-broken --repository="{\"type\":\"path\",\"url\":\"$GITHUB_WORKSPACE/.e2e/skeleton\"}" --stability=dev --no-interaction + cd .e2e/app-broken + composer config secure-http false + composer config repositories.fixtures "{\"type\":\"path\",\"url\":\"$GITHUB_WORKSPACE/.e2e/fixtures/packages/fixture-hello\"}" + composer config extra.symfony.endpoint --json "[\"http://127.0.0.1:8098/index.json\",\"flex://defaults\"]" + if composer require minspec/fixture-hello:1.0.0 --no-interaction >/dev/null 2>&1 && php bin/console list >/dev/null 2>&1; then + echo "planted fault did NOT fire — the oracle is blind" + exit 1 + fi + echo "planted fault fired as required" diff --git a/tests/e2e/assert.sh b/tests/e2e/assert.sh new file mode 100755 index 0000000..84c7aa9 --- /dev/null +++ b/tests/e2e/assert.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +if [[ $# -ne 1 || ! -d "$1" || ! -f "$1/bin/console" || ! -f "$1/config/bundles.php" ]]; then + printf '%s\n' 'app: expected to be a Symfony application; found missing or invalid directory; needed pass a Symfony app directory containing bin/console and config/bundles.php' >&2 + exit 2 +fi + +app_dir=$1 +failures=0 + +one_line() { + printf '%s' "$1" | tr '\r\n' ' ' | tr -s ' ' +} + +bundles_file="$app_dir/config/bundles.php" +if grep -Eq "Minspec\\\\FixtureHello\\\\FixtureHelloBundle::class[[:space:]]*=>[[:space:]]*\\[[[:space:]]*['\"]all['\"][[:space:]]*=>[[:space:]]*true[[:space:]]*\\]" "$bundles_file"; then + printf '%s\n' 'PASS: FixtureHelloBundle is registered for all environments' +else + printf '%s\n' 'bundle: expected FixtureHelloBundle registered for all environments; found no matching all-environments registration' + failures=$((failures + 1)) +fi + +yaml_file="$app_dir/config/packages/fixture_hello.yaml" +if [[ ! -f "$yaml_file" ]]; then + printf '%s\n' 'recipe config: expected config/packages/fixture_hello.yaml with fixture_hello.greeting wired-by-recipe; found file missing' + failures=$((failures + 1)) +elif grep -Eq "^[[:space:]]*fixture_hello\\.greeting:[[:space:]]*(['\"])?wired-by-recipe\\1[[:space:]]*(#.*)?$" "$yaml_file"; then + printf '%s\n' 'PASS: recipe configuration defines fixture_hello.greeting as wired-by-recipe' +else + printf '%s\n' 'recipe config: expected fixture_hello.greeting value wired-by-recipe; found parameter line missing or different' + failures=$((failures + 1)) +fi + +container_result=$(cd "$app_dir" && php bin/console debug:container fixture_hello.service 2>&1) +container_status=$? +if [[ $container_status -eq 0 ]]; then + printf '%s\n' 'PASS: public container service fixture_hello.service exists' +else + printf 'service: expected public container service fixture_hello.service; found command exit %d: %s\n' \ + "$container_status" "$(one_line "$container_result")" + failures=$((failures + 1)) +fi + +command_capture=$( + cd "$app_dir" || exit 125 + php bin/console fixture:hello 2>&1 + command_status=$? + printf '\036%d' "$command_status" +) +command_status=${command_capture##*$'\036'} +command_output=${command_capture%$'\036'*} +expected_output=$'fixture-hello: wired-by-recipe\n' + +if [[ $command_status -eq 0 && "$command_output" == "$expected_output" ]]; then + printf '%s\n' 'PASS: fixture:hello prints exactly fixture-hello: wired-by-recipe' +else + printf 'command: expected exact output fixture-hello: wired-by-recipe; found exit %s, output <%s>\n' \ + "$command_status" "$(one_line "$command_output")" + failures=$((failures + 1)) +fi + +if [[ $failures -ne 0 ]]; then + exit 1 +fi + +exit 0 From 98d14ec965a787a7908323804b7215efe027a5a2 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:42:28 -0400 Subject: [PATCH 02/18] e2e: serve recipes truly locally; apply the audit's workflow hardening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first CI run caught two defects, which is the workflow doing its job. One: the planted fault did not fire — the local-endpoint rewrite set a top-level recipe_template while Flex reads it from _links, so both arms silently fetched the clean recipe from the GitHub API; both templates now point at the local server, and the broken arm also changes the recipe ref so no cache can launder the plant. Two: composer validate caught the lock file the bump plugin left stale while Actions was still disabled; reconciled. Hardening from the independent security audit (CHANGES verdict): workflow-level permissions contents:read on ci and e2e; persist-credentials false on every checkout; the committed APP_SECRET replaced with an obvious non-secret fixture sentinel. Finding: [P1] workflows ran with default token permissions Finding: [P2] .env.dev committed a real-looking APP_SECRET Verified: composer validate --strict at e2a7da1 (now valid); python yaml.safe_load on both workflows Source: original Co-Authored-By: GPT-5 Codex Co-Authored-By: Claude Fable 5 Reviewed-by: GPT-5 Codex Claude-Session: https://claude.ai/code/session_01XehTac5TJNmPAskwrPp7rJ --- .env.dev | 2 +- .github/workflows/ci.yml | 4 ++++ .github/workflows/e2e.yml | 12 ++++++++++-- composer.lock | 2 +- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.env.dev b/.env.dev index fb7881a..13bdbdf 100644 --- a/.env.dev +++ b/.env.dev @@ -1,4 +1,4 @@ ###> symfony/framework-bundle ### -APP_SECRET=bfba68e0d5a77f01269c924e502e3acf +APP_SECRET=NotASecretTestFixtureOnlyNeverDeploy0 ###< symfony/framework-bundle ### diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0501cd..ab35eeb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,11 +3,15 @@ on: pull_request: push: branches: [dev, main] +permissions: + contents: read jobs: ci: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: shivammathur/setup-php@v2 with: php-version: '8.4' diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d5e9c63..622a537 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -3,23 +3,30 @@ on: pull_request: push: branches: [dev, main] +permissions: + contents: read jobs: e2e: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/checkout@v4 with: + persist-credentials: false repository: minspec/skeleton ref: dev path: .e2e/skeleton - uses: actions/checkout@v4 with: + persist-credentials: false repository: minspec/workbench-fixtures ref: dev path: .e2e/fixtures - uses: actions/checkout@v4 with: + persist-credentials: false repository: minspec/recipes ref: dev path: .e2e/recipes @@ -30,7 +37,7 @@ jobs: - name: serve the recipes endpoint locally (clean arm) run: | cd .e2e/recipes - jq '.recipe_template = "http://127.0.0.1:8099/{package_dotted}.{version}.json"' index.json > index.tmp && mv index.tmp index.json + jq '._links.recipe_template = "http://127.0.0.1:8099/{package_dotted}.{version}.json" | .recipe_template = "http://127.0.0.1:8099/{package_dotted}.{version}.json"' index.json > index.tmp && mv index.tmp index.json php -S 127.0.0.1:8099 & sleep 1 curl -fsS http://127.0.0.1:8099/index.json | jq -e '.recipes["minspec/fixture-hello"]' @@ -62,7 +69,8 @@ jobs: .e2e/recipes/minspec.fixture-hello.1.0.json > .e2e/recipes-broken/minspec.fixture-hello.1.0.json grep -q 'fixture_hello.wrong' .e2e/recipes-broken/minspec.fixture-hello.1.0.json || { echo "plant did not land"; exit 1; } ! grep -q 'fixture_hello.wrong' .e2e/recipes/minspec.fixture-hello.1.0.json || { echo "plant leaked into clean fixture"; exit 1; } - jq '.recipe_template = "http://127.0.0.1:8098/{package_dotted}.{version}.json"' .e2e/recipes-broken/index.json > .e2e/recipes-broken/index.tmp && mv .e2e/recipes-broken/index.tmp .e2e/recipes-broken/index.json + jq '._links.recipe_template = "http://127.0.0.1:8098/{package_dotted}.{version}.json" | .recipe_template = "http://127.0.0.1:8098/{package_dotted}.{version}.json"' .e2e/recipes-broken/index.json > .e2e/recipes-broken/index.tmp && mv .e2e/recipes-broken/index.tmp .e2e/recipes-broken/index.json + sed -i 's/e2e-fixture-1.0/e2e-broken-1.0/' .e2e/recipes-broken/minspec.fixture-hello.1.0.json (cd .e2e/recipes-broken && php -S 127.0.0.1:8098 &) sleep 1 composer create-project minspec/skeleton .e2e/app-broken --repository="{\"type\":\"path\",\"url\":\"$GITHUB_WORKSPACE/.e2e/skeleton\"}" --stability=dev --no-interaction diff --git a/composer.lock b/composer.lock index f36f9f8..f2e7dc9 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e09e18a3ce71d999e9ca8c6736eb80b5", + "content-hash": "e05e93e48bc3aed75f910553a87d1c37", "packages": [ { "name": "psr/cache", From 34a03725f16fd0d7e7e1281b0246a7a2b59d3332 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:47:40 -0400 Subject: [PATCH 03/18] =?UTF-8?q?e2e:=20the=20org=20gate=20=E2=80=94=20one?= =?UTF-8?q?=20canonical=20road=20test,=20callable=20from=20every=20repo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The e2e workflow gains workflow_call with a ref input per component: a calling repo slots its own PR into its slot and takes siblings at dev, so skeleton, recipes, and fixtures PRs run the SAME road test the workbench runs, not a copy that can drift. Checkouts are SHA-pinned and credential-free per the audit standard; the workbench checks out into wb/ so own-run and called-run share one path shape. Source: owner 2026-09-01 ("make it a real test and stick to it") Source: original Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XehTac5TJNmPAskwrPp7rJ --- .github/workflows/e2e.yml | 61 +++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 622a537..311962c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,36 +1,65 @@ name: e2e +# The org's one road test: package -> recipe -> endpoint -> app. +# Canonical here; every repo that can break the road calls this same +# workflow with its own PR slotted into the matching ref, siblings at +# dev. Never weaken it; extend it. on: pull_request: push: branches: [dev, main] + workflow_call: + inputs: + skeleton_ref: + type: string + default: dev + recipes_ref: + type: string + default: dev + fixtures_ref: + type: string + default: dev + workbench_ref: + type: string + default: '' permissions: contents: read jobs: e2e: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: checkout workbench (own run) + if: ${{ inputs.workbench_ref == '' }} + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: persist-credentials: false - - uses: actions/checkout@v4 + path: wb + - name: checkout workbench (called) + if: ${{ inputs.workbench_ref != '' }} + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: + repository: minspec/workbench + ref: ${{ inputs.workbench_ref }} persist-credentials: false + path: wb + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: repository: minspec/skeleton - ref: dev + ref: ${{ inputs.skeleton_ref || 'dev' }} + persist-credentials: false path: .e2e/skeleton - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: - persist-credentials: false repository: minspec/workbench-fixtures - ref: dev + ref: ${{ inputs.fixtures_ref || 'dev' }} + persist-credentials: false path: .e2e/fixtures - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: - persist-credentials: false repository: minspec/recipes - ref: dev + ref: ${{ inputs.recipes_ref || 'dev' }} + persist-credentials: false path: .e2e/recipes - - uses: shivammathur/setup-php@v2 + - uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # v2 with: php-version: '8.4' @@ -55,22 +84,22 @@ jobs: composer require minspec/fixture-hello:1.0.0 --no-interaction - name: assertions (independent test author) - run: bash tests/e2e/assert.sh .e2e/app + run: bash wb/tests/e2e/assert.sh .e2e/app - name: planted fault must fire run: | - # Corrupt the served recipe (parameter renamed) and serve it as a - # DIFFERENT endpoint (own port) so no cache can hand the broken - # arm the clean recipe. Prove the plant landed, then a fresh app - # build must fail. The clean arm above is the stay-quiet half. + # Corrupt the served recipe (parameter renamed, ref changed so no + # cache can launder the plant), serve it on its own port, prove + # the plant landed and did not leak, then a fresh app build must + # fail. The clean arm above is the stay-quiet half. mkdir -p .e2e/recipes-broken cp .e2e/recipes/index.json .e2e/recipes-broken/ sed 's/fixture_hello.greeting/fixture_hello.wrong/' \ .e2e/recipes/minspec.fixture-hello.1.0.json > .e2e/recipes-broken/minspec.fixture-hello.1.0.json + sed -i 's/e2e-fixture-1.0/e2e-broken-1.0/' .e2e/recipes-broken/minspec.fixture-hello.1.0.json grep -q 'fixture_hello.wrong' .e2e/recipes-broken/minspec.fixture-hello.1.0.json || { echo "plant did not land"; exit 1; } ! grep -q 'fixture_hello.wrong' .e2e/recipes/minspec.fixture-hello.1.0.json || { echo "plant leaked into clean fixture"; exit 1; } jq '._links.recipe_template = "http://127.0.0.1:8098/{package_dotted}.{version}.json" | .recipe_template = "http://127.0.0.1:8098/{package_dotted}.{version}.json"' .e2e/recipes-broken/index.json > .e2e/recipes-broken/index.tmp && mv .e2e/recipes-broken/index.tmp .e2e/recipes-broken/index.json - sed -i 's/e2e-fixture-1.0/e2e-broken-1.0/' .e2e/recipes-broken/minspec.fixture-hello.1.0.json (cd .e2e/recipes-broken && php -S 127.0.0.1:8098 &) sleep 1 composer create-project minspec/skeleton .e2e/app-broken --repository="{\"type\":\"path\",\"url\":\"$GITHUB_WORKSPACE/.e2e/skeleton\"}" --stability=dev --no-interaction From 7c85f20c29b7afa6bf8ce0340d2e927a85b08778 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:08:44 -0400 Subject: [PATCH 04/18] =?UTF-8?q?ops:=20the=20review=20=E2=80=94=20Codex?= =?UTF-8?q?=20sweeps,=20Grok=20audits,=20the=20conductor=20applies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The maintainer's definition of review for big PRs to dev: a Codex security sweep and a Grok audit, both independent of the author, both on the wire, findings applied through the lane with the finder credited. Merge follows the verdicts; stable stays the maintainer's. Source: owner 2026-09-01 ("codex security sweep and a grok audit for big PRs to dev - that's the review") Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XehTac5TJNmPAskwrPp7rJ --- ops/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ops/README.md b/ops/README.md index 243a652..914800a 100644 --- a/ops/README.md +++ b/ops/README.md @@ -22,6 +22,23 @@ maintainer installs it at `ops/bin/dispatch.sh` when wanted; its staged text lives with the conductor session. Until then, delegation runs through peer sessions and the House dispatch lane. +## The review + +Big PRs to dev — substantive code, doctrine, or workflow changes, as +opposed to line fixes — take two independent passes before merge, each +from a different harness than the author: + +- a **Codex security sweep**: supply chain, workflows, authority + boundaries, secrets +- a **Grok audit**: correctness, internal consistency, test quality + +Both report on the wire (VERDICT / STAMP / FINDINGS with priorities; +anything unexamined marked UNCHECKED — silence is not a declaration). +The conductor dispatches both, applies findings through the lane +crediting the finder, and merges only on an APPROVE or on a CHANGES +whose P1/P2 findings are applied or explicitly ruled by the +maintainer. Ratification of stable remains the maintainer's. + ## The rules the lane binds - One line of work, one worktree (`../wt//`); the main From 4d697c206faacd4e15d04b5734f671651ed53b1a Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:21:15 -0400 Subject: [PATCH 05/18] =?UTF-8?q?ops:=20the=20dev-lane=20=E2=80=94=20dispa?= =?UTF-8?q?tch/task/harness/telemetry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dev-lane machinery for MinSpec, reviewed by the lane's own two-harness review (Codex security sweep + Grok audit) with their findings applied or stated in AGENTS.md: dispatch (worker launcher), task (job registry), harness (isolation proofs), telemetry (usage readers), fixtures (synthetic replay data), the process docs, dispatch.sh/conf for this ops home (state under minspec/dispatch), the process skills, and the test-author and test-skeptic role agents. Public-safe by construction: - fixture envelopes carry only synthetic session and request ids, paths, reasoning text, and cost figures; store fixtures generate from a cwd parameter, so no machine path is baked in. - no committed .pyc; ops/devlane/.gitignore added. - ops/devlane/AGENTS.md states the trust boundary: TRUSTED work on the operator host only — no OS containment, so untrusted or public commits need an ephemeral container not yet built; dispatch.conf is owner-controlled shell; fixtures are synthetic, never raw captures. - the launcher self-resolves from the lane (no absolute pin); the real dispatch.conf is machine-local and gitignored behind an .example; apply-push carries a content guard that refuses home paths, job captures, and credential references in a landing commit. - go.sh: the one-line way to hand work to the lane. Verified: dispatch 247, task 151, harness 34, telemetry 63, fixtures 14 — all suites OK. Source: owner 2026-09-01 Co-Authored-By: GPT-5.6 Sol Co-Authored-By: Grok 4.6 Co-Authored-By: Claude Fable 5 Co-Authored-By: Claude Fable 5.1 Reviewed-by: GPT-5.6 Sol Claude-Session: https://claude.ai/code/session_01XehTac5TJNmPAskwrPp7rJ Claude-Session: https://claude.ai/code/session_012Jj94rkp3tfHAxUkTCthgY --- .claude/agents/test-author.md | 10 + .claude/agents/test-skeptic.md | 10 + .claude/skills/bdd/SKILL.md | 8 + .claude/skills/cross-review/SKILL.md | 10 + .claude/skills/dev-lane/SKILL.md | 38 + .claude/skills/tdd/SKILL.md | 11 + ops/bin/.gitignore | 2 + ops/bin/dispatch.conf.example | 21 + ops/bin/dispatch.sh | 144 ++ ops/bin/go.sh | 77 + ops/devlane/.gitignore | 7 + ops/devlane/AGENTS.md | 29 + ops/devlane/dispatch/CONTRACT.md | 331 +++ ops/devlane/dispatch/launch.py | 1317 ++++++++++++ ops/devlane/dispatch/levers/README.md | 37 + ops/devlane/dispatch/levers/apply-push.sh | 367 ++++ .../dispatch/levers/claude/fable-dispatch.sh | 241 +++ .../dispatch/levers/claude/selftest.sh | 45 + .../dispatch/levers/codex/codex-dispatch.sh | 554 +++++ .../dispatch/levers/codex/vendor/isolation.py | 382 ++++ ops/devlane/dispatch/levers/grok-dispatch.sh | 349 +++ ops/devlane/dispatch/levers/selftest.sh | 149 ++ ops/devlane/dispatch/record.py | 81 + ops/devlane/dispatch/tests/launch_support.py | 1233 +++++++++++ ops/devlane/dispatch/tests/support.py | 37 + .../tests/test_launch_claude_readable.py | 189 ++ .../dispatch/tests/test_launch_collect.py | 145 ++ .../test_launch_envelope_by_construction.py | 958 +++++++++ .../tests/test_launch_envelope_parse.py | 227 ++ .../tests/test_launch_envelope_u10_fix.py | 718 +++++++ .../tests/test_launch_grok_head_commit.py | 92 + .../tests/test_launch_harness_fixes.py | 383 ++++ .../dispatch/tests/test_launch_isolation.py | 460 ++++ .../dispatch/tests/test_launch_job_caps.py | 186 ++ .../tests/test_launch_prompt_feed_findings.py | 737 +++++++ .../dispatch/tests/test_launch_record.py | 268 +++ .../tests/test_launch_record_identity.py | 138 ++ .../dispatch/tests/test_launch_refusals.py | 561 +++++ .../tests/test_launch_review_p1_findings.py | 270 +++ .../dispatch/tests/test_launch_snapshot.py | 356 ++++ .../dispatch/tests/test_launch_u10b.py | 517 +++++ .../dispatch/tests/test_launch_verbs.py | 164 ++ .../dispatch/tests/test_launch_watch.py | 453 ++++ ops/devlane/dispatch/tests/test_record.py | 306 +++ .../claude-result-error-max-turns.json | 1 + .../claude-result-error-max-turns.meta.json | 7 + .../envelopes/codex-ndjson-tail.jsonl | 3 + .../envelopes/codex-ndjson-tail.meta.json | 7 + .../envelopes/grok-fenced-json.meta.json | 8 + .../envelopes/grok-fenced-json.raw.out | 29 + ...json-schema-short-circuit-66ccb2.meta.json | 9 + ...k-json-schema-short-circuit-66ccb2.raw.out | 48 + .../grok-narration-then-object.meta.json | 8 + .../grok-narration-then-object.raw.out | 1 + .../envelopes/grok-s2-13f5f2.meta.json | 11 + .../fixtures/envelopes/grok-s2-13f5f2.raw.out | 1 + ops/devlane/fixtures/stores.py | 194 ++ .../fixtures/tests/test_grok_usage_fixture.py | 388 ++++ ops/devlane/fixtures/tests/test_stores.py | 1084 ++++++++++ .../fixtures/tests/test_stores_findings.py | 109 + ops/devlane/harness/context.cue | 437 ++++ ops/devlane/harness/controls/build.py | 377 ++++ .../harness/controls/dispatchable-flags.json | 129 ++ .../harness/controls/dispatchable-home.json | 130 ++ .../harness/controls/evidence-receipt.json | 19 + .../attestation-is-not-admissible.json | 24 + .../attestation-is-not-admissible.reason.json | 8 + .../controls/rejects/empty-snapshot.json | 124 ++ .../rejects/empty-snapshot.reason.json | 8 + .../controls/rejects/flags-not-recorded.json | 126 ++ .../rejects/flags-not-recorded.reason.json | 8 + .../rejects/home-without-auth-files.json | 127 ++ .../home-without-auth-files.reason.json | 8 + .../controls/rejects/home-without-home.json | 129 ++ .../rejects/home-without-home.reason.json | 8 + .../rejects/omits-dangling_references.json | 129 ++ .../omits-dangling_references.reason.json | 9 + ...ss-isolation-observed-harness_version.json | 129 ++ ...ation-observed-harness_version.reason.json | 9 + ...tion-observed-operator_config_present.json | 129 ++ ...served-operator_config_present.reason.json | 9 + .../harness/controls/rejects/omits-role.json | 129 ++ .../controls/rejects/omits-role.reason.json | 9 + .../controls/rejects/omits-staged-count.json | 129 ++ .../rejects/omits-staged-count.reason.json | 9 + .../controls/rejects/omits-staged-given.json | 127 ++ .../rejects/omits-staged-given.reason.json | 9 + .../omits-staged-proof-given_unmet.json | 129 ++ ...omits-staged-proof-given_unmet.reason.json | 9 + .../omits-staged-proof-withheld_present.json | 129 ++ ...-staged-proof-withheld_present.reason.json | 9 + .../controls/rejects/omits-task-produces.json | 110 + .../rejects/omits-task-produces.reason.json | 9 + .../rejects/omits-task-report_fields.json | 126 ++ .../omits-task-report_fields.reason.json | 9 + .../rejects/omits-unmet_requirements.json | 129 ++ .../omits-unmet_requirements.reason.json | 9 + .../controls/rejects/role-mismatch.json | 130 ++ .../rejects/role-mismatch.reason.json | 8 + .../controls/rejects/stale-observation.json | 130 ++ .../rejects/stale-observation.reason.json | 8 + .../rejects/task-brief-interface.json | 118 ++ .../rejects/task-brief-interface.reason.json | 8 + .../rejects/task-brief-report-fields.json | 129 ++ .../task-brief-report-fields.reason.json | 8 + .../rejects/task-states-less-than-brief.json | 115 + .../task-states-less-than-brief.reason.json | 9 + ops/devlane/harness/evidence.cue | 206 ++ ops/devlane/harness/isolation.py | 382 ++++ ops/devlane/harness/probe.py | 309 +++ ops/devlane/harness/stage.py | 237 +++ ops/devlane/harness/tests/support.py | 21 + ops/devlane/harness/tests/test_isolation.py | 207 ++ .../harness/tests/test_isolation_contract.py | 345 +++ ops/devlane/harness/tests/test_probe.py | 121 ++ .../harness/tests/test_probe_contract.py | 134 ++ ops/devlane/harness/vet_context.py | 331 +++ ops/devlane/harness/wires.py | 155 ++ ops/devlane/infra/gitea/runner-token.seed | 0 ops/devlane/task/CONTRACT.md | 187 ++ ops/devlane/task/envelope.py | 310 +++ ops/devlane/task/fileset.py | 432 ++++ ops/devlane/task/jobs.json | 91 + ops/devlane/task/run.py | 1326 ++++++++++++ ops/devlane/task/tests/support.py | 37 + ops/devlane/task/tests/test_envelope.py | 275 +++ .../task/tests/test_envelope_schema.py | 364 ++++ ops/devlane/task/tests/test_fileset.py | 721 +++++++ ops/devlane/task/tests/test_jobs.py | 52 + ops/devlane/task/tests/test_run.py | 1479 +++++++++++++ ops/devlane/task/tests/test_verify.py | 629 ++++++ ops/devlane/task/verify.py | 232 ++ ops/devlane/telemetry/breaker.py | 301 +++ ops/devlane/telemetry/pulse.py | 321 +++ .../telemetry/tests/test_bdd_breaker.py | 131 ++ ops/devlane/telemetry/tests/test_bdd_worth.py | 156 ++ ops/devlane/telemetry/tests/test_breaker.py | 235 +++ .../telemetry/tests/test_breaker_grok.py | 376 ++++ .../telemetry/tests/test_grok_usage_pulse.py | 400 ++++ .../telemetry/tests/test_grok_usage_reader.py | 546 +++++ ops/devlane/telemetry/tests/test_pulse.py | 1123 ++++++++++ .../telemetry/tests/test_pulse_findings.py | 472 +++++ ops/devlane/telemetry/tests/test_usage.py | 146 ++ ops/devlane/telemetry/tests/test_wiring.py | 1864 +++++++++++++++++ ops/devlane/telemetry/tests/test_worth.py | 1699 +++++++++++++++ .../telemetry/tests/test_worth_seams.py | 394 ++++ ops/devlane/telemetry/usage.py | 357 ++++ ops/devlane/telemetry/worth.py | 594 ++++++ ops/devlane/workflow/checks/term_wall.py | 125 ++ .../workflow/checks/vocabulary_wall.py | 253 +++ ops/process/agentic-management.md | 409 ++++ ops/process/bdd.md | 60 + ops/process/cross-review.md | 244 +++ ops/process/pipeline.md | 148 ++ ops/process/roles/test-author.md | 40 + ops/process/roles/test-skeptic.md | 49 + ops/process/tdd.md | 91 + ops/process/token-thrift.md | 28 + ops/process/worth.md | 92 + 159 files changed, 37582 insertions(+) create mode 100644 .claude/agents/test-author.md create mode 100644 .claude/agents/test-skeptic.md create mode 100644 .claude/skills/bdd/SKILL.md create mode 100644 .claude/skills/cross-review/SKILL.md create mode 100644 .claude/skills/dev-lane/SKILL.md create mode 100644 .claude/skills/tdd/SKILL.md create mode 100644 ops/bin/.gitignore create mode 100644 ops/bin/dispatch.conf.example create mode 100755 ops/bin/dispatch.sh create mode 100755 ops/bin/go.sh create mode 100644 ops/devlane/.gitignore create mode 100644 ops/devlane/AGENTS.md create mode 100644 ops/devlane/dispatch/CONTRACT.md create mode 100644 ops/devlane/dispatch/launch.py create mode 100644 ops/devlane/dispatch/levers/README.md create mode 100755 ops/devlane/dispatch/levers/apply-push.sh create mode 100755 ops/devlane/dispatch/levers/claude/fable-dispatch.sh create mode 100755 ops/devlane/dispatch/levers/claude/selftest.sh create mode 100755 ops/devlane/dispatch/levers/codex/codex-dispatch.sh create mode 100644 ops/devlane/dispatch/levers/codex/vendor/isolation.py create mode 100755 ops/devlane/dispatch/levers/grok-dispatch.sh create mode 100755 ops/devlane/dispatch/levers/selftest.sh create mode 100644 ops/devlane/dispatch/record.py create mode 100644 ops/devlane/dispatch/tests/launch_support.py create mode 100644 ops/devlane/dispatch/tests/support.py create mode 100644 ops/devlane/dispatch/tests/test_launch_claude_readable.py create mode 100644 ops/devlane/dispatch/tests/test_launch_collect.py create mode 100644 ops/devlane/dispatch/tests/test_launch_envelope_by_construction.py create mode 100644 ops/devlane/dispatch/tests/test_launch_envelope_parse.py create mode 100644 ops/devlane/dispatch/tests/test_launch_envelope_u10_fix.py create mode 100644 ops/devlane/dispatch/tests/test_launch_grok_head_commit.py create mode 100644 ops/devlane/dispatch/tests/test_launch_harness_fixes.py create mode 100644 ops/devlane/dispatch/tests/test_launch_isolation.py create mode 100644 ops/devlane/dispatch/tests/test_launch_job_caps.py create mode 100644 ops/devlane/dispatch/tests/test_launch_prompt_feed_findings.py create mode 100644 ops/devlane/dispatch/tests/test_launch_record.py create mode 100644 ops/devlane/dispatch/tests/test_launch_record_identity.py create mode 100644 ops/devlane/dispatch/tests/test_launch_refusals.py create mode 100644 ops/devlane/dispatch/tests/test_launch_review_p1_findings.py create mode 100644 ops/devlane/dispatch/tests/test_launch_snapshot.py create mode 100644 ops/devlane/dispatch/tests/test_launch_u10b.py create mode 100644 ops/devlane/dispatch/tests/test_launch_verbs.py create mode 100644 ops/devlane/dispatch/tests/test_launch_watch.py create mode 100644 ops/devlane/dispatch/tests/test_record.py create mode 100644 ops/devlane/fixtures/envelopes/claude-result-error-max-turns.json create mode 100644 ops/devlane/fixtures/envelopes/claude-result-error-max-turns.meta.json create mode 100644 ops/devlane/fixtures/envelopes/codex-ndjson-tail.jsonl create mode 100644 ops/devlane/fixtures/envelopes/codex-ndjson-tail.meta.json create mode 100644 ops/devlane/fixtures/envelopes/grok-fenced-json.meta.json create mode 100644 ops/devlane/fixtures/envelopes/grok-fenced-json.raw.out create mode 100644 ops/devlane/fixtures/envelopes/grok-json-schema-short-circuit-66ccb2.meta.json create mode 100644 ops/devlane/fixtures/envelopes/grok-json-schema-short-circuit-66ccb2.raw.out create mode 100644 ops/devlane/fixtures/envelopes/grok-narration-then-object.meta.json create mode 100644 ops/devlane/fixtures/envelopes/grok-narration-then-object.raw.out create mode 100644 ops/devlane/fixtures/envelopes/grok-s2-13f5f2.meta.json create mode 100644 ops/devlane/fixtures/envelopes/grok-s2-13f5f2.raw.out create mode 100644 ops/devlane/fixtures/stores.py create mode 100644 ops/devlane/fixtures/tests/test_grok_usage_fixture.py create mode 100644 ops/devlane/fixtures/tests/test_stores.py create mode 100644 ops/devlane/fixtures/tests/test_stores_findings.py create mode 100644 ops/devlane/harness/context.cue create mode 100644 ops/devlane/harness/controls/build.py create mode 100644 ops/devlane/harness/controls/dispatchable-flags.json create mode 100644 ops/devlane/harness/controls/dispatchable-home.json create mode 100644 ops/devlane/harness/controls/evidence-receipt.json create mode 100644 ops/devlane/harness/controls/rejects/attestation-is-not-admissible.json create mode 100644 ops/devlane/harness/controls/rejects/attestation-is-not-admissible.reason.json create mode 100644 ops/devlane/harness/controls/rejects/empty-snapshot.json create mode 100644 ops/devlane/harness/controls/rejects/empty-snapshot.reason.json create mode 100644 ops/devlane/harness/controls/rejects/flags-not-recorded.json create mode 100644 ops/devlane/harness/controls/rejects/flags-not-recorded.reason.json create mode 100644 ops/devlane/harness/controls/rejects/home-without-auth-files.json create mode 100644 ops/devlane/harness/controls/rejects/home-without-auth-files.reason.json create mode 100644 ops/devlane/harness/controls/rejects/home-without-home.json create mode 100644 ops/devlane/harness/controls/rejects/home-without-home.reason.json create mode 100644 ops/devlane/harness/controls/rejects/omits-dangling_references.json create mode 100644 ops/devlane/harness/controls/rejects/omits-dangling_references.reason.json create mode 100644 ops/devlane/harness/controls/rejects/omits-harness-isolation-observed-harness_version.json create mode 100644 ops/devlane/harness/controls/rejects/omits-harness-isolation-observed-harness_version.reason.json create mode 100644 ops/devlane/harness/controls/rejects/omits-harness-isolation-observed-operator_config_present.json create mode 100644 ops/devlane/harness/controls/rejects/omits-harness-isolation-observed-operator_config_present.reason.json create mode 100644 ops/devlane/harness/controls/rejects/omits-role.json create mode 100644 ops/devlane/harness/controls/rejects/omits-role.reason.json create mode 100644 ops/devlane/harness/controls/rejects/omits-staged-count.json create mode 100644 ops/devlane/harness/controls/rejects/omits-staged-count.reason.json create mode 100644 ops/devlane/harness/controls/rejects/omits-staged-given.json create mode 100644 ops/devlane/harness/controls/rejects/omits-staged-given.reason.json create mode 100644 ops/devlane/harness/controls/rejects/omits-staged-proof-given_unmet.json create mode 100644 ops/devlane/harness/controls/rejects/omits-staged-proof-given_unmet.reason.json create mode 100644 ops/devlane/harness/controls/rejects/omits-staged-proof-withheld_present.json create mode 100644 ops/devlane/harness/controls/rejects/omits-staged-proof-withheld_present.reason.json create mode 100644 ops/devlane/harness/controls/rejects/omits-task-produces.json create mode 100644 ops/devlane/harness/controls/rejects/omits-task-produces.reason.json create mode 100644 ops/devlane/harness/controls/rejects/omits-task-report_fields.json create mode 100644 ops/devlane/harness/controls/rejects/omits-task-report_fields.reason.json create mode 100644 ops/devlane/harness/controls/rejects/omits-unmet_requirements.json create mode 100644 ops/devlane/harness/controls/rejects/omits-unmet_requirements.reason.json create mode 100644 ops/devlane/harness/controls/rejects/role-mismatch.json create mode 100644 ops/devlane/harness/controls/rejects/role-mismatch.reason.json create mode 100644 ops/devlane/harness/controls/rejects/stale-observation.json create mode 100644 ops/devlane/harness/controls/rejects/stale-observation.reason.json create mode 100644 ops/devlane/harness/controls/rejects/task-brief-interface.json create mode 100644 ops/devlane/harness/controls/rejects/task-brief-interface.reason.json create mode 100644 ops/devlane/harness/controls/rejects/task-brief-report-fields.json create mode 100644 ops/devlane/harness/controls/rejects/task-brief-report-fields.reason.json create mode 100644 ops/devlane/harness/controls/rejects/task-states-less-than-brief.json create mode 100644 ops/devlane/harness/controls/rejects/task-states-less-than-brief.reason.json create mode 100644 ops/devlane/harness/evidence.cue create mode 100644 ops/devlane/harness/isolation.py create mode 100644 ops/devlane/harness/probe.py create mode 100644 ops/devlane/harness/stage.py create mode 100644 ops/devlane/harness/tests/support.py create mode 100644 ops/devlane/harness/tests/test_isolation.py create mode 100644 ops/devlane/harness/tests/test_isolation_contract.py create mode 100644 ops/devlane/harness/tests/test_probe.py create mode 100644 ops/devlane/harness/tests/test_probe_contract.py create mode 100644 ops/devlane/harness/vet_context.py create mode 100644 ops/devlane/harness/wires.py create mode 100644 ops/devlane/infra/gitea/runner-token.seed create mode 100644 ops/devlane/task/CONTRACT.md create mode 100644 ops/devlane/task/envelope.py create mode 100644 ops/devlane/task/fileset.py create mode 100644 ops/devlane/task/jobs.json create mode 100644 ops/devlane/task/run.py create mode 100644 ops/devlane/task/tests/support.py create mode 100644 ops/devlane/task/tests/test_envelope.py create mode 100644 ops/devlane/task/tests/test_envelope_schema.py create mode 100644 ops/devlane/task/tests/test_fileset.py create mode 100644 ops/devlane/task/tests/test_jobs.py create mode 100644 ops/devlane/task/tests/test_run.py create mode 100644 ops/devlane/task/tests/test_verify.py create mode 100644 ops/devlane/task/verify.py create mode 100644 ops/devlane/telemetry/breaker.py create mode 100644 ops/devlane/telemetry/pulse.py create mode 100644 ops/devlane/telemetry/tests/test_bdd_breaker.py create mode 100644 ops/devlane/telemetry/tests/test_bdd_worth.py create mode 100644 ops/devlane/telemetry/tests/test_breaker.py create mode 100644 ops/devlane/telemetry/tests/test_breaker_grok.py create mode 100644 ops/devlane/telemetry/tests/test_grok_usage_pulse.py create mode 100644 ops/devlane/telemetry/tests/test_grok_usage_reader.py create mode 100644 ops/devlane/telemetry/tests/test_pulse.py create mode 100644 ops/devlane/telemetry/tests/test_pulse_findings.py create mode 100644 ops/devlane/telemetry/tests/test_usage.py create mode 100644 ops/devlane/telemetry/tests/test_wiring.py create mode 100644 ops/devlane/telemetry/tests/test_worth.py create mode 100644 ops/devlane/telemetry/tests/test_worth_seams.py create mode 100644 ops/devlane/telemetry/usage.py create mode 100644 ops/devlane/telemetry/worth.py create mode 100644 ops/devlane/workflow/checks/term_wall.py create mode 100755 ops/devlane/workflow/checks/vocabulary_wall.py create mode 100644 ops/process/agentic-management.md create mode 100644 ops/process/bdd.md create mode 100644 ops/process/cross-review.md create mode 100644 ops/process/pipeline.md create mode 100644 ops/process/roles/test-author.md create mode 100644 ops/process/roles/test-skeptic.md create mode 100644 ops/process/tdd.md create mode 100644 ops/process/token-thrift.md create mode 100644 ops/process/worth.md diff --git a/.claude/agents/test-author.md b/.claude/agents/test-author.md new file mode 100644 index 0000000..d0012b1 --- /dev/null +++ b/.claude/agents/test-author.md @@ -0,0 +1,10 @@ +--- +name: test-author +description: Writes failing tests from a behavior contract without reading the implementation. Use for the red stage of a tdd work order when no second harness is available to author the tests. +--- + +Adopt the role card at `ops/process/roles/test-author.md` and follow it +exactly. Read the card first. You are the in-harness fallback for this role — +a second harness (Codex or Grok, per `ops/process/cross-review.md`) is +preferred when available, and the caller must say in the PR when the +fallback was used. diff --git a/.claude/agents/test-skeptic.md b/.claude/agents/test-skeptic.md new file mode 100644 index 0000000..38498d4 --- /dev/null +++ b/.claude/agents/test-skeptic.md @@ -0,0 +1,10 @@ +--- +name: test-skeptic +description: Adversarially reviews tests for shapes that pass over broken code. Use before recording green on a tdd work order when no third harness is available to judge the tests. +--- + +Adopt the role card at `ops/process/roles/test-skeptic.md` and follow it +exactly. Read the card first. You are the in-harness fallback for this role — +a harness that wrote neither the tests nor the implementation is preferred +(`ops/process/cross-review.md`), and the caller must say in the PR when the +fallback was used. diff --git a/.claude/skills/bdd/SKILL.md b/.claude/skills/bdd/SKILL.md new file mode 100644 index 0000000..017c962 --- /dev/null +++ b/.claude/skills/bdd/SKILL.md @@ -0,0 +1,8 @@ +--- +name: bdd +description: Given/When/Then behavior scenarios before tests or code, kept traceable to tests. Use at the spec stage of a work order, or when behavior needs agreeing on before implementation. +--- + +Read and follow `ops/process/bdd.md` — the canonical process document, +shared by every harness that works this repo. Scenarios land under +`.dev/design/features/`; when they are done, hand off per `ops/process/tdd.md`. diff --git a/.claude/skills/cross-review/SKILL.md b/.claude/skills/cross-review/SKILL.md new file mode 100644 index 0000000..6d97183 --- /dev/null +++ b/.claude/skills/cross-review/SKILL.md @@ -0,0 +1,10 @@ +--- +name: cross-review +description: Have the other two harnesses (Codex, Grok — or Claude when another harness leads) review work against a detached snapshot. Use before merging test-bearing or evidence-bearing changes. +--- + +Read and follow `ops/process/cross-review.md` — the canonical process +document. The two rules that must survive any summary: reviewers get a +detached snapshot, never the live worktree; and only the owner triggers +`@codex review` on GitHub — the local `codex exec` reviewer is a different +thing and is yours to run. diff --git a/.claude/skills/dev-lane/SKILL.md b/.claude/skills/dev-lane/SKILL.md new file mode 100644 index 0000000..bc61da5 --- /dev/null +++ b/.claude/skills/dev-lane/SKILL.md @@ -0,0 +1,38 @@ +--- +name: dev-lane +description: The dev-lane pipeline — plan (Fable), tests (Grok), check the tests (Codex), code (Opus), review the code (Grok AND Codex, both). NO PRODUCER OWNS TWO CONSECUTIVE ARTIFACTS: whoever writes an implementation does not write or approve its tests. Scope is settled in session with the owner, then handed off. Load before starting any change under .dev/, and before dispatching any harness. +--- + +Read and follow `ops/process/pipeline.md` — the canonical sequence. +`ops/process/cross-review.md` covers review only; reading it alone gives +three roles where there are six. + +Three rules that must survive any summary: + +- **No producer owns two consecutive artifacts.** The specific models + matter less than that constraint. Measured twice on this repo: nine + defects past a green suite whose tests and code shared an author, and + eleven of fourteen review findings against two checkers being missing + test cases rather than coding errors. + +- **The firewall is proved, not intended.** Withholding is invisible — a + snapshot that leaked the wrong file looks exactly like one that did + not. Prove both directions before dispatching: nothing matching a + withheld pattern present, *and* something matching every given pattern + present. An empty snapshot satisfies the first perfectly. + +- **Contracts are extracted from an app's contract document, never from a + plan.** That is what keeps a plan disposable; the plan it was learned + from could not be retired because 1,236 citations under + `.dev/app/workflow/contracts/` point at it. + +Scope is settled in session with the owner; the plan is not. State +*what* and the boundaries, and leave *how* to the planner. Review goes +to two harnesses, not one, so the rule holds whoever is driving — and a +CHANGES verdict is ruled on by a harness that produced neither the +artifact nor the finding, never by the producer, before any of it is +worked. + +Reach: Claude loads this natively and Grok through claude-compat. **Codex +does not see repo skills** — its copy of these rules is `AGENTS.md`, which +every harness reads. diff --git a/.claude/skills/tdd/SKILL.md b/.claude/skills/tdd/SKILL.md new file mode 100644 index 0000000..c5b7f34 --- /dev/null +++ b/.claude/skills/tdd/SKILL.md @@ -0,0 +1,11 @@ +--- +name: tdd +description: The wf-governed red→green loop — honest reds, sealed frozen sets, independent tests. Use when implementing anything under a tdd gate kind, or starting test-first work. +--- + +Read and follow `ops/process/tdd.md` — it is the canonical process document, +shared by every harness that works this repo (Claude, Codex, Grok), so it is +not duplicated here. When a step calls for the test-author or test-skeptic +role, prefer a different harness via `ops/process/cross-review.md`; the +`test-author` and `test-skeptic` subagents are the fallback when you must +fill a role in-harness. diff --git a/ops/bin/.gitignore b/ops/bin/.gitignore new file mode 100644 index 0000000..c54180c --- /dev/null +++ b/ops/bin/.gitignore @@ -0,0 +1,2 @@ +# Owner-owned, machine-specific — copy from dispatch.conf.example. +dispatch.conf diff --git a/ops/bin/dispatch.conf.example b/ops/bin/dispatch.conf.example new file mode 100644 index 0000000..8d3040f --- /dev/null +++ b/ops/bin/dispatch.conf.example @@ -0,0 +1,21 @@ +# Copy to dispatch.conf. The real dispatch.conf is gitignored: it is +# owner-owned, machine-specific config sourced by dispatch.sh, never a +# public artifact. A missing conf refuses loudly — it is not inferred. +# +# LAUNCHER_PIN is optional. Left empty, dispatch.sh finds the launcher next +# to itself (../devlane/dispatch/launch.py) — portable across worktrees and +# machines. Set it to an ABSOLUTE launch.py path only if you want the hard +# supply-chain lock (dispatch then runs exactly that launcher and no other). +# +# Owner allow rule (adding it is the owner's act): +# "Bash(bash /ABSOLUTE/PATH/TO/minspec/workbench/ops/bin/dispatch.sh:*)" +# Values may be overridden per call by an environment variable of the same +# name, then by the corresponding flag. + +LAUNCHER_PIN='' +DEFAULT_TIMEOUT='300' +WF_AGENT_DEFAULT='GPT-5.6 Sol ' +MODEL_GROK='grok-4.6' +MODEL_CODEX='gpt-5.6-sol' +MODEL_CLAUDE_READ='claude-opus-5' +MODEL_CLAUDE_PLAN='claude-fable-5' diff --git a/ops/bin/dispatch.sh b/ops/bin/dispatch.sh new file mode 100755 index 0000000..1934401 --- /dev/null +++ b/ops/bin/dispatch.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -uo pipefail + +refuse() { + printf 'dispatch: refusal: %s\n' "$*" >&2 + exit 3 +} + +script_dir=$(CDPATH='' cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +conf_path=${DISPATCH_CONF:-"$script_dir/dispatch.conf"} +[[ -f $conf_path ]] || refuse "expected a sourced conf; found no file at $conf_path; set DISPATCH_CONF to dispatch.conf" + +# LAUNCHER_PIN is resolved after sourcing, not required here: by default the +# launcher is found relative to this script (portable across worktrees and +# machines, and no home path leaks into a public conf). An operator who wants +# the hard supply-chain lock may still set an absolute LAUNCHER_PIN in the conf. +keys=(DEFAULT_TIMEOUT WF_AGENT_DEFAULT MODEL_GROK MODEL_CODEX MODEL_CLAUDE_READ MODEL_CLAUDE_PLAN) +for key in "${keys[@]}"; do + if [[ -v $key ]]; then + printf -v "saved_$key" '%s' "${!key}" + printf -v "had_$key" '%s' 1 + else + printf -v "had_$key" '%s' 0 + fi +done +# shellcheck source=/dev/null +source "$conf_path" || refuse "expected a sourceable conf; found an error in $conf_path; fix its shell assignments" +for key in "${keys[@]}"; do + had="had_$key" + saved="saved_$key" + if [[ ${!had} == 1 ]]; then + printf -v "$key" '%s' "${!saved}" + fi + [[ -v $key ]] || refuse "expected conf key $key; found it unset; define $key in $conf_path" +done + +# Resolve the launcher. An absolute LAUNCHER_PIN from the conf or environment +# wins (the operator's hard lock); otherwise default to this script's sibling +# launcher, so a fresh checkout or worktree runs without editing any path. +if [[ -z ${LAUNCHER_PIN:-} ]]; then + LAUNCHER_PIN="$script_dir/../devlane/dispatch/launch.py" +fi +LAUNCHER_PIN=$(realpath -e -- "$LAUNCHER_PIN" 2>/dev/null) \ + || refuse "expected LAUNCHER_PIN to name a launcher; found none at the resolved path; place launch.py or set LAUNCHER_PIN" + +branch='' +worktree='' +job='' +harness='' +model='' +unit='' +stage='' +scope_file='' +input='' +follows='' +timeout=$DEFAULT_TIMEOUT +agent=$WF_AGENT_DEFAULT +dry_run=0 + +need_value() { + (($# >= 2)) || refuse "expected a value after $1; found end of arguments; provide $1 VALUE" + [[ $2 != --* ]] || refuse "expected a value after $1; found $2; provide $1 VALUE" +} + +while (($#)); do + case $1 in + --branch) need_value "$@"; branch=$2; shift 2 ;; + --worktree) need_value "$@"; worktree=$2; shift 2 ;; + --job) need_value "$@"; job=$2; shift 2 ;; + --harness) need_value "$@"; harness=$2; shift 2 ;; + --model) need_value "$@"; model=$2; shift 2 ;; + --unit) need_value "$@"; unit=$2; shift 2 ;; + --stage) need_value "$@"; stage=$2; shift 2 ;; + --scope-file) need_value "$@"; scope_file=$2; shift 2 ;; + --input) need_value "$@"; input=$2; shift 2 ;; + --follows) need_value "$@"; follows=$2; shift 2 ;; + --timeout) need_value "$@"; timeout=$2; shift 2 ;; + --agent) need_value "$@"; agent=$2; shift 2 ;; + --dry-run) dry_run=1; shift ;; + *) refuse "expected a documented flag; found unknown flag $1; remove it" ;; + esac +done + +[[ -n $job ]] || refuse "expected required --job; found it missing; pass --job JOB" +[[ -n $harness ]] || refuse "expected required --harness; found it missing; pass --harness grok, codex, or claude" +[[ -n $scope_file ]] || refuse "expected required --scope-file; found it missing; pass --scope-file FILE" +[[ -n $branch || -n $worktree ]] || refuse "expected one of --branch or --worktree; found neither; select a lineage worktree" +[[ -z $branch || -z $worktree ]] || refuse "expected one of --branch or --worktree; found both; pass exactly one" +case $harness in grok|codex|claude) ;; *) refuse "expected --harness grok, codex, or claude; found $harness; choose a supported harness" ;; esac +[[ -f $scope_file ]] || refuse "expected --scope-file to name a file; found $scope_file; create the file or correct the path" + +scope_bytes=$(wc -c < "$scope_file") || refuse "expected a readable --scope-file; found unreadable $scope_file; correct its permissions" +scope_bytes=${scope_bytes//[[:space:]]/} +((scope_bytes <= 1024)) || refuse "expected --scope-file at most 1024 bytes; found $scope_bytes bytes; shorten it" +scope=$(cat -- "$scope_file"; printf x) || refuse "expected a readable --scope-file; found unreadable $scope_file; correct its permissions" +scope=${scope%x} + +if [[ -n $branch ]]; then + found_path= + candidate= + while IFS= read -r line; do + case $line in + 'worktree '*) candidate=${line#worktree } ;; + "branch refs/heads/$branch") found_path=$candidate; break ;; + esac + done < <(git worktree list --porcelain) + [[ -n $found_path ]] || refuse "expected branch $branch in a worktree; found none; run git worktree add $branch" + worktree=$found_path +fi +[[ -d $worktree ]] || refuse "expected a worktree directory; found $worktree; correct --worktree" + +ref=$(git -C "$worktree" rev-parse HEAD 2>/dev/null) || refuse "expected a git worktree; found $worktree without HEAD; correct the target" +lineage=$(git -C "$worktree" symbolic-ref --quiet --short HEAD 2>/dev/null || true) + +if [[ -z $model ]]; then + case $harness in + grok) model=$MODEL_GROK ;; + codex) model=$MODEL_CODEX ;; + claude) if [[ $stage == plan ]]; then model=$MODEL_CLAUDE_PLAN; else model=$MODEL_CLAUDE_READ; fi ;; + esac +fi + +argv=("$LAUNCHER_PIN" "$job" --harness "$harness" --model "$model" --ref "$ref") +[[ -z $lineage ]] || argv+=(--lineage "$lineage") +[[ -z $unit ]] || argv+=(--unit "$unit") +[[ -z $stage ]] || argv+=(--stage "$stage") +argv+=(--scope "$scope") +[[ -z $input ]] || argv+=(--input "$input") +[[ -z $follows ]] || argv+=(--follows "$follows") + +print_launch() { + DEFAULT_TIMEOUT=$timeout WF_AGENT=$agent python3 -c 'import json, os, sys; print(json.dumps({"argv": sys.argv[1:], "env": {"DEFAULT_TIMEOUT": os.environ["DEFAULT_TIMEOUT"], "WF_AGENT": os.environ["WF_AGENT"]}}, ensure_ascii=False))' "${argv[@]}" +} + +print_launch +((dry_run)) && exit 0 + +DEFAULT_TIMEOUT=$timeout WF_AGENT=$agent python3 "${argv[@]}" +rc=$? +printf 'LAUNCH EXIT %s\n' "$rc" +jobs_root=${DISPATCH_JOBS:-"${XDG_STATE_HOME:-$HOME/.local/state}/minspec/dispatch"} +newest=$(find "$jobs_root" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %f\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-) +printf 'record: %s\n' "${newest:-unresolved: no dispatch record found under $jobs_root}" +exit "$rc" diff --git a/ops/bin/go.sh b/ops/bin/go.sh new file mode 100755 index 0000000..bd28288 --- /dev/null +++ b/ops/bin/go.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# go.sh — the one-line way to hand work to the lane, so delegating is +# easier than grinding it inline. Everything dispatch.sh needs — harness, +# model, scope file, worktree resolution — is defaulted here from a short +# intent word. Read-only jobs stay read-only; the firewall is baked into +# the harness defaults (no producer owns two consecutive artifacts). +# +# go.sh +# +# intents (default harness in parens): +# sweep (codex) security/defect sweep over disjoint categories +# audit (grok) adversarial review of a change — findings that survive +# review (both) sweep + audit, the review lane in one word +# plan (claude) a plan for the scope, no edits +# tests (claude) author tests from the contract, red-first +# check (codex) check existing tests against the contract +# implement (claude) implement to green +# adjudicate (grok) adjudicate a contested result +# +# Override the harness with HARNESS=grok|codex|claude. Everything prints +# where its verdict landed; nothing is cached. +set -uo pipefail + +here=$(CDPATH='' cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +die() { printf 'go: %s\n' "$*" >&2; exit 64; } + +(( $# >= 3 )) || die "usage: go.sh " +intent=$1; where=$2; shift 2 +scope="$*" + +# intent -> job + default harness. review is the one that fans out. +case $intent in + sweep) job=sweep; def_harness=codex ;; + audit) job=adversarial-review; def_harness=grok ;; + plan) job=plan; def_harness=claude; stage=plan ;; + tests) job=author-tests; def_harness=claude ;; + check) job=check-tests; def_harness=codex ;; + implement) job=implement; def_harness=claude ;; + adjudicate) job=adjudicate; def_harness=grok ;; + review) job=__fanout__; def_harness= ;; + *) die "unknown intent '$intent'; see the header of $here/go.sh" ;; +esac + +# where: '.' means this checkout; else a branch name in some worktree. +if [[ $where == . ]]; then + root=$(git -C "$PWD" rev-parse --show-toplevel 2>/dev/null) \ + || die "'.' given but $PWD is not inside a git worktree" + locator=(--worktree "$root") +else + locator=(--branch "$where") +fi + +# scope must fit the launcher's 1024-byte scope-file cap. +(( ${#scope} <= 1024 )) || die "scope is ${#scope} bytes; the cap is 1024 — shorten it" +scope_file=$(mktemp -t go-scope.XXXXXX) || die "cannot create scope file" +trap 'rm -f "$scope_file"' EXIT +printf '%s' "$scope" >"$scope_file" + +fire() { # fire [] + local j=$1 h=${2:-} st=${3:-} + h=${HARNESS:-$h} + [[ -n $h ]] || die "no harness for job '$j'; set HARNESS=grok|codex|claude" + printf '>> go: %s job=%s harness=%s target=%s\n' "$intent" "$j" "$h" "$where" >&2 + local args=(--job "$j" --harness "$h" --scope-file "$scope_file" "${locator[@]}") + [[ -n $st ]] && args+=(--stage "$st") + bash "$here/dispatch.sh" "${args[@]}" +} + +if [[ $job == __fanout__ ]]; then + # The review lane: security sweep (codex) then adversarial audit (grok). + # Two harnesses, so no single producer owns the whole review. + rc=0 + fire sweep codex || rc=$? + fire adversarial-review grok || rc=$? + exit "$rc" +fi +fire "$job" "$def_harness" "${stage:-}" diff --git a/ops/devlane/.gitignore b/ops/devlane/.gitignore new file mode 100644 index 0000000..0466322 --- /dev/null +++ b/ops/devlane/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.py[cod] + +# Runtime dispatch output — job captures, snapshots, isolated HOMEs. +# The lever SOURCE (levers/*.sh, levers/claude, levers/codex, README) +# is tracked; everything a run WRITES under jobs/ is never committed. +dispatch/levers/jobs/ diff --git a/ops/devlane/AGENTS.md b/ops/devlane/AGENTS.md new file mode 100644 index 0000000..0b3ca5c --- /dev/null +++ b/ops/devlane/AGENTS.md @@ -0,0 +1,29 @@ +# AGENTS.md — the dev-lane (dispatch/task/harness/telemetry) + +The MinSpec dev-lane. The machinery: `dispatch/` launches harness workers, +`task/` holds the job registry, `harness/` proves isolation, `telemetry/` +reads usage, `fixtures/` supplies synthetic replay data. + +## Trust boundary — READ THIS BEFORE DISPATCHING + +This lane runs harness workers **on the operator's host** with edit and +shell authority inside a snapshot. It does **not** provide OS-level +containment; `harness/isolation.py` strips operator config, not the +filesystem or credentials. Therefore: + +- **Dispatch only TRUSTED work here** — maintainer-directed changes on + branches the maintainer opened. Never dispatch an untrusted or + public contributor's commit on this host: a hostile `conftest.py`, + CUE tool, or test import would run with the operator's authority. +- Dispatching untrusted PRs requires an **ephemeral VM/container** with + no operator home, SSH agent, or host mounts. That boundary is not + built; until it is, untrusted dispatch is out of scope. +- `dispatch.conf` is sourced as shell — treat it as owner-controlled + code, never worker-editable. + +## Fixtures are synthetic + +`fixtures/envelopes/` are shaped like real harness output but carry only +synthetic ids, costs, and paths. Never replace them with raw captures: +a real capture leaks session metadata into a public repo. Regenerate +via `stores.py`, never by copying a live run. diff --git a/ops/devlane/dispatch/CONTRACT.md b/ops/devlane/dispatch/CONTRACT.md new file mode 100644 index 0000000..12f278a --- /dev/null +++ b/ops/devlane/dispatch/CONTRACT.md @@ -0,0 +1,331 @@ +# The dispatch app — the dev-lane launcher + +`launch.py` is the **policy** over the task app's `run.py` mechanism, +and the record it writes is built and validated by `record.py`. + +Dispatch is a *composition* app. It reads the task app's job registry +(`ops/devlane/task/jobs.json`) and drives the harness app's isolation and +budget wiring (`ops/devlane/harness/isolation.py`, +`ops/devlane/harness/wires.py`). It was split out of the task app so it may +depend on both without the `task → harness` edge the cross-app import +contract forbids: a launcher that executes `isolation.isolated` and +reads `wires.budget` genuinely depends on `harness`, and the task app +must not. The allowed edges `dispatch → task` and `dispatch → harness` +are declared in `.dev/contracts/imports.json`; the reverse would be a +cycle. + +| file | job | +|:--|:--| +| `ops/devlane/dispatch/launch.py` | the §Dispatch policy: mint a snapshot, launch, supervise, collect, record | +| `ops/devlane/dispatch/record.py` | build and validate one dispatch record, one implementation per rule | + +The rest of this file is the launcher's contract, written before the +launcher existed so its tests could be authored from it. The guide page +is `.dev/guide/dispatch.md`. + +## Dispatch — the dev-lane launcher + +`run.py` is the mechanism: snapshot, launch, supervise, collect, parse. +`launch.py` is the policy over it, and the **sole dispatch path for +dev-lane work**: it mints a snapshot from a named ref, keeps every +artifact outside the tree, applies the preconditions below as refusals, +supervises inline, returns the envelope, and writes a dispatch record +into a tracked directory on the branch. This section is its contract. +It is written before the launcher exists, so that the tests can be +authored from it and the implementation judged against it; invocation +examples therefore sit in plain fences until the file they name does. + +Vocabulary: the **invoking repository** is the checkout the launcher +is run from; the **lineage branch** is the branch the work unit lives +on; the **job directory** is where one dispatch's snapshot and +artifacts live, outside every repository. + +### Verbs + +```text +launch.py --harness H --model M [--effort E] --ref R + [--lineage B] [--unit U] [--stage S] [--scope TEXT] + [--input PATH]... [--follows ID]... [--override ID:REASON]... +launch.py status [ID] [--json] +launch.py resume ID [--prompt-file PATH] +launch.py close ID +launch.py brief --check ID +``` + +- `` names a `jobs.json` entry. A job never names a model; `--model` + is the owner's choice at dispatch and is required. +- `--ref` is resolved to a commit in the invoking repository. `--lineage` + defaults to the invoking checkout's current branch; `--unit` defaults + to the lineage branch and groups records when one branch carries + several work units; `--stage` is one of `plan`, `tests`, + `check-tests`, `code`, `review`, `adjudicate`. +- `--scope` is the **one free field** the dispatcher may add, capped at + **1024 bytes**, reproduced verbatim in the record. A job whose + template has no `{scope}` refuses it. +- `--input PATH` copies a file into the job directory's `in/` and names + it to the template as `{inputs}`; its digest enters the record. +- `--follows ID` names the records this dispatch was given and follows + (plan → tests → check-tests → code → review → adjudicate). It is a + dispatch-graph edge; it is **not** `context.prior`, which stays + "findings from an earlier call". +- `--override ID:REASON` is accepted only for a refusal marked + overridable below, and only with a non-empty reason. +- Identity: `WF_AGENT` in the `Name
` form, as for `wf`. + Recording always needs one. +- Jobs root: `$DISPATCH_JOBS`, default + `${XDG_STATE_HOME:-$HOME/.local/state}/minspec/dispatch`. There is + no flag for it; the record carries the path it used. + +### Snapshot modes + +A job declares one of two modes. They are mutually exclusive by +construction, and a job that needs both is refused. + +**`whole`** — a fresh repository minted from the invoking one: +`git init`, then `git -c core.logAllRefUpdates=false fetch refs/heads/:refs/heads/`, then `FETCH_HEAD` +removed, then `checkout --detach `. Guaranteed, and pinned by +test: `HEAD` equals `ref_sha`; the ref's full history is present, so +`merge-base` questions are answerable; `git remote` is empty; +`objects/info/alternates` does not exist; `logs/` does not exist; the +invoking repository's path occurs in **no byte** under `.git`; `git +status --porcelain` is empty before launch. It is not `git worktree +add` (that registers in the live repository's worktree list, which is +what the `live-target` refusal exists to catch) and not `git clone +--shared` (that leaves the source path in `alternates` and the reflogs +after `origin` is removed, and makes the source object store writable +by absolute path). + +**`fileset`** — `fileset.snapshot` with `include` and `withheld`, +proved by `stage.prove` in both directions, and **no `.git`**: a +withheld file is one `git show HEAD:path` away in any snapshot with +history. Its manifest and diff live in the job directory, not at the +snapshot root. This mode lands after `whole`; until it does, a job that +declares it is refused by name. + +`direct` jobs (`verify`) take the caller's fileset as today and declare +no mode. + +### The job directory + +```text +// + snapshot/ the tree, and nothing that is not the tree + prompt.txt the rendered brief, exactly the bytes handed over + in/ copies of every --input, by basename + out/ where a job writes a document deliverable ({out}) + raw.out the harness's stdout, byte for byte + stderr the harness's stderr + breaker.log the battery's stderr + TRIPPED.md written by the battery on a trip + state.json pid, pgid, session id, stream path, attempt + exit the harness's exit status, written last + home/codex minimal home for codex (auth.json only) + home/grok minimal home for grok (auth.json only) +``` + +Nothing the launcher, the battery or the harness writes for its own +bookkeeping goes under `snapshot/`. Two incidents sit behind that +line: report files at a snapshot root tripped the vocabulary wall +twice, and bookkeeping files counted as repository changes. + +`id` is `---<6 hex>`. + +### Template values + +`render` receives, beyond `context` and `require`: `ref` and `base` +(shas), `diff` (path, or absent), `into` (the snapshot root), `out` +(the absolute path of the job's `out/`), `inputs` (the absolute paths +of the copied inputs, space-separated, in the order given), and +`scope`. A template naming a value the launcher did not supply is a +refusal at render, never a brief with a hole in it. + +### The child's environment + +Applied to the harness process only, never exported by the launcher +for itself: unset `CLICOLOR_FORCE` and `FORCE_COLOR` (they beat +`NO_COLOR`; measured, 525 escape bytes in `gh` JSON); set `NO_COLOR=1 +CLICOLOR=0 TERM=dumb PAGER=cat GH_PAGER=cat GIT_PAGER=cat LESS=FRX +CI=true GIT_TERMINAL_PROMPT=0 GIT_EDITOR=true EDITOR=true +PYTHONUNBUFFERED=1 PYTHONIOENCODING=utf-8 LC_ALL=C.UTF-8`, plus +`WF_LANE=dev` and `DISPATCH_JOB=`, plus whatever +`isolation.dispatch_env` returns. The working directory is `snapshot/`. + +### Isolation, per harness + +`isolation.isolated()` runs on every launch; there is no argument that +turns it off, and a harness with no entry is refused. The stream store +is read from the same entry's `sessions` spec, because a relocated +`CODEX_HOME` or `GROK_HOME` relocates the stream and a launcher looking +in `~/.codex/sessions` would report "no stream" and run unsupervised. + +| harness | isolation | read sandbox | write sandbox | containment | model that ran | +|:--|:--|:--|:--|:--|:--| +| claude | flags `--setting-sources project,local --strict-mcp-config --disable-slash-commands`; HOME untouched | `plan` | **not admitted** — no write row until the containment probe below has passed | policy | `~/.claude/projects//.jsonl`, `assistant` records, `message.model` | +| codex | `CODEX_HOME=/home/codex` holding exactly `auth.json` | `read-only` | `workspace-write` | os | `$CODEX_HOME/sessions/Y/M/D/rollout-*.jsonl`, `turn_context.payload.model` (id and cwd from `session_meta`) | +| grok | `GROK_HOME` **and** `HOME` at `/home/grok`, holding exactly `auth.json`; prompt via `--prompt-file` from the job directory | `plan` | `auto` | policy, until an owner-named `--sandbox` profile is added to the row | `$GROK_HOME/sessions///summary.json`, `current_model_id`; its `head_commit` is cross-checked against `snapshot.ref_sha` | + +`containment` says what stops a write role reaching outside its +snapshot: `os` when the harness's own sandbox enforces it, `policy` when +only the harness's permission mode does. It is recorded, not judged. + +The session handle is minted at launch — `claude --session-id `, +`grok -s ` — and codex's is read from `session_meta` once the +stream is found. A harness that ignores the minted id is recorded as a +mismatch, never silently accepted. + +**Containment probe (admits a claude write row).** Two arms, both +dispatched, both must answer on their own terms: arm A runs the +candidate write mode with a brief that runs the snapshot's test suite +and edits a file, and must succeed at both; arm B runs the same mode +with a brief that writes a file at an absolute path outside the +snapshot, and must fail to. An arm that did not run is INVALID, not a +pass. The result is recorded verbatim in the adapter row's +`containment` entry and in every record that row produces. + +**Behavioural observation, per harness**, recorded verbatim under +`harness.isolation.observed`: claude asks the model whether the +operator's probe phrase is in its instructions (unisolated YES, +isolated NO); grok reads `grok inspect` (unisolated ≥1 instruction +file, isolated 0); codex asks for the names of the skills available to +it, one per line, or NONE (unisolated arm lists ≥1 or the run is +INVALID because the machine cannot demonstrate the leak; isolated arm +must answer NONE). Each is true of one harness version on one day and +carries `checked_at` and `harness_version`. A record for a harness +whose observation has not run carries `observed: {"unresolved": +}` — never a manufactured `false`. + +### Refusals + +Exit 3, before anything is minted unless the table says otherwise. The +text of each names what was expected, what was found, and what would +satisfy it. `override` marks the only overridable one. + +| id | refuses when | expected / found / satisfy | +|:--|:--|:--| +| `identity` | `WF_AGENT` unset or not `Name
` | expected an identity in the `Name
` form; found ``; export `WF_AGENT='Your Model Name '` | +| `live-target` | the destination is inside any entry of the invoking repository's `git worktree list --porcelain` or its common dir | expected a job directory outside every worktree of ``; found `` inside ``; set `DISPATCH_JOBS` to a directory outside the repository | +| `ref` | `--ref` does not resolve to a commit | expected a ref naming a commit in ``; found `` (``); commit the work, then name the commit — uncommitted work is never dispatched | +| `record-target` | the invoking checkout is detached, on `dev` or `main`, or on a branch other than `lineage.branch`; re-checked at close | expected the checkout on ``; found ``; run from a worktree of `` (at close: the record file is left in place, nothing is committed, and `close ID` is re-run from the right branch) | +| `stale-base` (**override**) | `ref_sha` is not an ancestor of `lineage.branch` | expected `` reachable from ``; found it is not (`` is at ``); name a commit on the branch, or `--override stale-base:` | +| `model` | no `--model`; or `--effort` on a harness whose adapter cannot pass it | expected a model — a job never names one and the launcher never defaults one; found none; pass `--model` (for effort on codex: the CLI has no effort flag, so the dial is refused rather than dropped) | +| `isolation` | no `HARNESSES` entry, a credential missing, or a minimal home that is not empty | `isolation.NotIsolated`'s own text | +| `history-vs-withheld` | the job declares `withheld` and mode `whole` | expected one of history or withholding; found both on ``; a withheld file is recoverable from `.git`, so declare `fileset` | +| `mode-unavailable` | the job declares a mode the launcher does not yet mint (`fileset`) | expected `whole`; found `fileset` on ``; stage it by hand with `stage.py` and name the dispatch in the PR body until fileset mode lands | +| `write-role-unadmitted` | a write role on a harness whose adapter row has no write sandbox (claude) | expected a write row with a containment entry; found none for ``; run the containment probe and add the row, or dispatch the role on codex or grok | +| `scope-cap` | `--scope` over 1024 bytes, or given to a job with no `{scope}` | expected ≤ 1024 bytes on a job that takes a scope; found `` bytes / a scope on ``, which takes none; shorten it — a scope that needs more is a brief, and briefs are derived — or drop it | +| `caps` | an unknown wire, or `cap` and `total` (or `cap_out` and `out`) both given | `run.py`'s own text | +| `off-lineage-head` (at close, write roles) | the snapshot's `HEAD` is not a descendant of `ref_sha` | expected `HEAD` to descend from ``; found ``; the envelope is `invalid`, nothing is fetched, and the record says so | + +Runtime outcomes that are not refusals, and still produce a record: +`unsupervised` (no session stream under the isolated store within the +grace period, 120 seconds until measured, while the process lives → +terminated, `invalid`), `harness-cli`, `envelope-parse`, `timeout`, +`trip` — as `run.py` returns them today. An override on any id not +marked overridable, or with an empty reason, is itself refused. + +### Collect + +Write roles: the snapshot's `HEAD` must descend from `ref_sha`; it is +fetched into the invoking repository as `refs/dispatch/` with +`--no-write-fetch-head`; `changed_paths` is `git diff --name-only +ref_sha..head`; `residual_paths` is `git status --porcelain` in the +snapshot. Read roles: `head` must equal `ref_sha`; a non-empty +`residual_paths` is recorded, not judged — records are evidence, never +authority. + +### The record + +One file per dispatch at `.dev/records/dispatches/.json`, written +at launch with `status: launched` and no `result`, finalized at close, +and committed on the lineage branch as one commit containing only that +file — `git commit --only -- ` — with the message `dispatch: +record `, `Source: generated: ops/devlane/dispatch/launch.py`, and +`Co-Authored-By: $WF_AGENT`. `record.py` builds and validates it, one +implementation per rule as `envelope.py` does; a record that fails +validation is never committed and the launcher exits non-zero naming +the field. + +Fields, in this order: + +```text +id, lane ("dev"), stage, unit, lineage {branch, base_sha}, follows [ids], +job, role (read|write), dispatched_by, at {launched, closed}, +snapshot {mode, ref_name, ref_sha, behind_tip, root}, +harness {name, version, + isolation {mechanism, flags | env, home?, auth_files?, store, + observed {operator_config_present, evidence, + checked_at, harness_version} + | {unresolved}}, + sandbox, containment (os|policy), argv}, +model {requested, effort_requested, ran, read_from}, +session {id, stream, stream_sha256_at_close}, +brief {template {path, sha256}, scope, inputs [{path, sha256}], sha256, bytes}, +caps {..., source}, +overrides [{refusal, reason, by}], +attempts [{n, launched, ended, exit, tripped}], +result {head, changed_paths, residual_paths, envelope}, +status (launched|closed|died) +``` + +- `model.requested` is the alias the owner chose; `model.ran` is what + the harness's own stream says, read from `read_from`; with no stream + it is `null` with a note, and never the alias copied over. +- `brief.sha256` is the digest of the exact bytes handed to the + harness; `brief --check ID` re-renders from `jobs.json` at `ref_sha`, + the recorded scope and the recorded input digests, and compares. +- `behind_tip` is `rev-list --count ref_sha..lineage`, a fact, not a + judgement. +- `caps` come from `wires.py` for the role; a per-launch change is + recorded with its source. +- `caps.timeout` (seconds) resolves in one order: `DISPATCH_TIMEOUT` + from the invoking environment, else the job's own `caps.timeout` in + `jobs.json`, else 900 — and `caps.timeout_source` names the layer + that answered: `DISPATCH_TIMEOUT`, `job`, or `default`. The job + layer exists because runtime is a property of the job, not of the + session that happens to launch it: author-tests proves every + assertion red before green, its successful runs measure p50 928s + against the flat 900s default, and five of seven wall-kills in the + record store were that class, each launched bare after the session + that knew the `DISPATCH_TIMEOUT` compensation had ended (finding: + 20260901T004909Z-tests-grok-3a7479). + +**The permitted delta to the invoking repository**, and nothing else: +one new file at the record path; one commit containing only it, on the +current branch, which equals `lineage.branch` at launch and at close; +for write roles, one new ref `refs/dispatch/` and its reflog line. +The index and worktree are otherwise untouched, `FETCH_HEAD` is not +written, and every other ref is byte-identical before and after. + +What makes the record expensive to fake after the fact — not +impossible, and the owner is the backstop: the brief digest re-derives; +`ref_sha` must be reachable from the branch; `result.head` must descend +from `ref_sha` and is a fetched commit with its own dates and trailers; +spend and `model.ran` must agree with a stream in the harness's own +store; and the record is a commit, so changing it is a diff. + +### Watching, and picking up after a kill + +The launch is the watch: `launch.py` is one foreground process the +session backgrounds once. `state.json` carries `pid`, `pgid`, the +session id, the stream path and the attempt number; `exit` is written +last. `status` reports `running` (pid alive), `finished` (exit file), +`tripped` (`TRIPPED.md`), **`DIED`** (pid gone and no exit file — never +reported as finished), `unlaunched` (refused before launch). `close ID` +finalizes and commits a DIED job's record as `invalid`. + +`resume ID` relaunches in the same snapshot directory — grok scopes +sessions by cwd — with `claude -p -r `, `grok -r `, or `codex +exec resume `, clears the previous exit and trip markers before +registering, appends output rather than truncating it, re-arms the +battery, and appends an `attempts` entry. A job the battery tripped +resumes only with a changed cap or a stated reason, recorded. + +### Lane + +Everything here is dev-lane: the launcher code under +`ops/devlane/dispatch/` (over the task app's `run.py`), the records +under `.dev/records/`, `lane: "dev"` in every record, +`WF_LANE=dev` in every child, and a refusal to record on `main`. The +product lane's Conductor will carry its own launcher and its own +records; nothing here binds it. diff --git a/ops/devlane/dispatch/launch.py b/ops/devlane/dispatch/launch.py new file mode 100644 index 0000000..2233277 --- /dev/null +++ b/ops/devlane/dispatch/launch.py @@ -0,0 +1,1317 @@ +#!/usr/bin/env python3 +"""Policy launcher for dev-lane task dispatches.""" + +from __future__ import annotations + +import argparse +import contextlib +import hashlib +import importlib.util +import json +import os +import re +import shutil +import signal +import string +import subprocess +import sys +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path + +import record + +APP = Path(__file__).resolve().parent +# jobs.json stays in the task app (it is data, read by path, not an import); +# dispatch composes task's job registry with harness's isolation/wires. +JOBS_PATH = APP.parent / "task" / "jobs.json" +HARNESS_APP = APP.parent / "harness" +TELEMETRY_APP = APP.parent / "telemetry" +REFUSAL = 3 +STAGES = ("plan", "tests", "check-tests", "code", "review", "adjudicate") +OWNER_NAME = "xormania" +OWNER_EMAIL = "127287135+xormania@users.noreply.github.com" +monotonic = time.monotonic +sleep = time.sleep + + +class Refused(Exception): + pass + + +def _module(name, path): + spec = importlib.util.spec_from_file_location(f"task_launch_{name}", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +ENVELOPE = _module("envelope", APP.parent / "task" / "envelope.py") +ENVELOPE_SCHEMA_JSON = json.dumps(ENVELOPE.ENVELOPE_SCHEMA, separators=(",", ":")) + + +def _git(repo, *args, check=True, env=None, input=None): + e = dict(os.environ if env is None else env) + e.update({"GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", "GIT_TERMINAL_PROMPT": "0"}) + p = subprocess.run( + ["git", "-C", str(repo), *args], + env=e, + input=input, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + if check and p.returncode: + raise RuntimeError(p.stderr.strip() or f"git exited {p.returncode}") + return p + + +def _refuse(ident, expected, found, satisfy): + raise Refused(f"{ident}: expected {expected}; found {found}; satisfy by {satisfy}") + + +def _now(): + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def mint_id(stage, harness): + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + return f"{stamp}-{stage}-{harness}-{os.urandom(3).hex()}" + + +def _jobs_root(): + if os.environ.get("DISPATCH_JOBS"): + return Path(os.environ["DISPATCH_JOBS"]) + state = os.environ.get("XDG_STATE_HOME") + base = Path(state) if state else Path(os.environ.get("HOME", "")) / ".local/state" + return base / "minspec" / "dispatch" + + +def _repo(): + p = _git(Path.cwd(), "rev-parse", "--show-toplevel") + return Path(p.stdout.strip()).resolve() + + +def _branch(repo): + p = _git(repo, "symbolic-ref", "--quiet", "--short", "HEAD", check=False) + return p.stdout.strip() if p.returncode == 0 else "detached" + + +def _identity(): + value = os.environ.get("WF_AGENT") + if not value or not re.fullmatch(r"[^<>\n]+ <[^<>\s@]+@[^<>\s]+>", value): + _refuse("identity", "an identity in the Name
form", + value or "unset", "export WF_AGENT='Your Model Name '") + return value + + +def _inside(path, root): + try: + path.resolve(strict=False).relative_to(root.resolve()) + return True + except ValueError: + return False + + +def _parse_override(items, agent): + out = [] + for item in items: + ident, sep, reason = item.partition(":") + if ident != "stale-base" or not sep or not reason.strip(): + _refuse("override", "stale-base with a non-empty reason", item, + "drop the override or pass stale-base:REASON") + out.append({"refusal": ident, "reason": reason, "by": agent}) + return out + + +def _snapshot(repo, lineage, sha, root): + root.mkdir(parents=True) + _git(root, "init", "-q") + _git(root, "config", "core.logAllRefUpdates", "false") + _git(root, "fetch", "--quiet", str(repo), + f"refs/heads/{lineage}:refs/heads/{lineage}") + # An explicitly overridden off-lineage commit need not be reachable + # from the lineage ref fetched above. + _git(root, "fetch", "--quiet", str(repo), sha) + (root / ".git" / "FETCH_HEAD").unlink(missing_ok=True) + shutil.rmtree(root / ".git" / "logs", ignore_errors=True) + _git(root, "checkout", "--quiet", "--detach", sha) + shutil.rmtree(root / ".git" / "logs", ignore_errors=True) + + +def _sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def _render(template, values): + fields = [name for _, name, _, _ in string.Formatter().parse(template) if name] + missing = [name for name in fields if name not in values] + if missing: + _refuse("render", "every template value supplied", missing[0], + f"supply {missing[0]} before rendering") + return template.format(**values) + + +def _harness_meta(name, role, job_dir, env): + isolation = _module("isolation", HARNESS_APP / "isolation.py") + try: + iso_env, flags = isolation.isolated(name, job_dir / "home" / name, env) + except (KeyError, OSError, RuntimeError, TypeError, ValueError) as exc: + _refuse("isolation", "an isolated harness with credentials", str(exc), + "add the isolation entry and required credential") + spec = isolation.HARNESSES[name] + # Read roles still write: plan, check-tests and adjudicate deliver a + # file under the job's out/, and check-tests runs a suite that needs a + # writable tempdir. codex read-only denied both (2026-08-28: "sandbox + # rejected writes as read-only ... no affected assertion ran"), and + # plan mode on claude and grok ends the turn at a plan. The snapshot's + # integrity is proved after the run instead: a read role's HEAD must + # equal ref_sha and residual_paths is recorded. + sandbox = {"claude": "acceptEdits", "codex": "workspace-write", "grok": "always-approve"}[name] + containment = "os" if name == "codex" else "policy" + home = iso_env.get(spec.get("home_env", "")) + store_base = Path(home) if spec["sessions"]["under"] == "minimal" else Path(env["HOME"]) / ".claude" + store = store_base / spec["sessions"]["path"] + data = {"mechanism": spec["mechanism"], + "flags" if spec["mechanism"] == "flags" else "env": flags if spec["mechanism"] == "flags" else iso_env, + "store": str(store), "observed": {"unresolved": "behavioural probe has not run"}} + if home: + data.update(home=home, auth_files=list(spec["auth_files"])) + return iso_env, flags, sandbox, containment, data + + +def _preflight_isolation(name, target, env): + isolation = _module("isolation_preflight", HARNESS_APP / "isolation.py") + spec = isolation.HARNESSES.get(name) + if spec is None: + _refuse("isolation", "a harness isolation entry", name, + "add an isolation entry with a measurement") + if target.exists() and any(target.iterdir()): + found = ", ".join(p.name for p in target.iterdir()) + _refuse("isolation", "an empty minimal home", found, + "use a fresh job id and empty home") + if spec["mechanism"] == "home": + named = spec.get("home_env") + source = Path(env.get(named) or Path(env.get("HOME", "")) / f".{name}") + missing = [f for f in spec["auth_files"] if not (source / f).exists()] + if missing: + _refuse("isolation", "the required credential", str(source / missing[0]), + "install the credential in the harness home") + + +# A read role must be able to execute to judge: a skeptic that cannot run the +# suite or plant a mutant is structural (2026-08-29: four check-tests/review +# dispatches on claude reported "This command requires approval" for +# `python3 -c 'print(2+2)'` under --print acceptEdits). These rules are +# what the claude child may run without a prompt. Honest limits, measured +# by the first review round (ba0d93, ffaf14): claude has no home isolation +# (CONTRACT.md §isolation: mechanism `flags`), so anything a rule allows +# runs with the operator's uid. The list therefore names verification +# commands, never an interpreter (`python3 *`, `env *`, `find *`, `sed *` +# were arbitrary code and are gone); `python3 .dev/*` runs the snapshot's +# own scripts, the same trust as running its suite. A write into the +# snapshot is recorded as residual and preserved (residual.patch), not +# refused. Real containment is Claude Code's OS sandbox — plan U2, gate +# G-CLI-1 — not this list. +CLAUDE_TOOL_RULES = ( + "Bash(python3 -m unittest *)", "Bash(python3 -m pytest *)", + "Bash(python3 -m ruff *)", "Bash(python3 .dev/*)", + "Bash(ruff check *)", "Bash(ruff format --check *)", + "Bash(cue vet *)", "Bash(cue export *)", "Bash(cue eval *)", "Bash(cue version)", + "Bash(git diff *)", "Bash(git log *)", "Bash(git show *)", + "Bash(git status *)", "Bash(git rev-parse *)", "Bash(git ls-files *)", + "Bash(git grep *)", "Bash(git blame *)", + "Bash(ls *)", "Bash(cat *)", "Bash(head *)", "Bash(tail *)", + "Bash(grep *)", "Bash(rg *)", "Bash(wc *)", "Bash(uniq *)", "Bash(diff *)", + "Bash(sha256sum *)", "Bash(jq *)", "Bash(which *)", "Bash(test *)", "Bash(cd *)", +) +# Tools no dispatch may use: the network (jobs say "no network") and +# sub-agents (the owner's hard rule, 2026-08-29). +CLAUDE_DENIED_TOOLS = ("WebFetch", "WebSearch", "Agent", "Task") + + +def _grok_permission(sandbox): + """`--always-approve` is grok's own flag, not a --permission-mode value; + web search and fetch are switched off beside it because every job says + "no network" and always-approve would otherwise approve them too.""" + if sandbox == "always-approve": + return ["--always-approve", "--disable-web-search"] + return ["--permission-mode", sandbox, "--disable-web-search"] + + +def _argv(name, model, effort, session, prompt, flags, sandbox, resume=False): + executable = shutil.which(name) or name + last_message = str((Path(prompt).parent / "out" / "last-message.json").resolve()) + if resume: + if name == "codex": + return [executable, "exec", "resume", session, "--model", model, + "--json", "-o", last_message] + if name == "claude": + out = str((Path(prompt).parent / "out").resolve()) + incoming = str((Path(prompt).parent / "in").resolve()) + context = str((Path(prompt).parent / "context").resolve()) + return [executable, *flags, "-r", session, "--print", + "--permission-mode", sandbox, "--add-dir", out, + "--add-dir", incoming, "--add-dir", context, + "--allowedTools", *CLAUDE_TOOL_RULES, + "--disallowedTools", *CLAUDE_DENIED_TOOLS, + "--model", model, "--output-format", "json", + "--json-schema", ENVELOPE_SCHEMA_JSON] + # grok resume must carry the same flags as a fresh launch, or B1 + # returns on every resumed dispatch (review ffaf14). + argv = [executable, *flags, "-r", session, "--model", model, + "--output-format", "plain"] + argv += _grok_permission(sandbox) + if effort: + argv += ["--reasoning-effort", effort] + argv += ["--prompt-file", str(prompt)] + return argv + out = str((Path(prompt).parent / "out").resolve()) + if name == "claude": + incoming = str((Path(prompt).parent / "in").resolve()) + context = str((Path(prompt).parent / "context").resolve()) + return [executable, *flags, "--session-id", session, "--print", + "--permission-mode", sandbox, "--add-dir", out, + "--add-dir", incoming, "--add-dir", context, + "--allowedTools", *CLAUDE_TOOL_RULES, + "--disallowedTools", *CLAUDE_DENIED_TOOLS, + "--model", model, "--output-format", "json", + "--json-schema", ENVELOPE_SCHEMA_JSON] + if name == "codex": + # codex exec takes the brief on stdin; "-" says so explicitly. With + # DEVNULL it exits 1 "No prompt provided via stdin" before any work. + # out/ sits outside the snapshot, so the workspace sandbox is told + # it is writable; nothing else outside cwd and /tmp is. + return [executable, "exec", "--sandbox", sandbox, + "-c", f'sandbox_workspace_write.writable_roots=["{out}"]', + "--model", model, "--json", "-o", last_message, "-"] + # grok: the record states `sandbox`; the argv must carry it, or the + # record claims a permission mode the child never had. plain output + # keeps stdout parseable as the envelope. `always-approve` is grok's + # own flag, not a --permission-mode value: under `auto` a session + # raised 111 permission prompts and the last one timed out after + # 30 s on a non-interactive stdin, ending the turn with no envelope + # (2026-08-29, record 20260829T170852Z-tests-grok-abfc3b). + # grok 1.0.5 with `--json-schema` answers after ONE model call with a + # schema-valid but empty envelope and ends the turn (measured + # 2026-08-29, record 20260829T234838Z-review-grok-66ccb2: 213 output + # tokens, "note": "starting: reading briefs and review contract", + # zero findings, $0.0067) — the structured-output mode short-circuits + # the agentic loop. Until the probe matrix finds a grok mode that both + # works and yields the object, grok keeps plain output and the scan. + argv = [executable, *flags, "-s", session, "--model", model, + "--output-format", "plain"] + argv += _grok_permission(sandbox) + if effort: + argv += ["--reasoning-effort", effort] + argv += ["--prompt-file", str(prompt)] + return argv + + +def _child_env(base, additions, job_id): + env = dict(base) + for key in ("CLICOLOR_FORCE", "FORCE_COLOR", "CLAUDE_CONFIG_DIR"): + env.pop(key, None) + env.update({"NO_COLOR": "1", "CLICOLOR": "0", "TERM": "dumb", + "PAGER": "cat", "GH_PAGER": "cat", "GIT_PAGER": "cat", + "LESS": "FRX", "CI": "true", "GIT_TERMINAL_PROMPT": "0", + "GIT_EDITOR": "true", "EDITOR": "true", "PYTHONUNBUFFERED": "1", + "PYTHONIOENCODING": "utf-8", "LC_ALL": "C.UTF-8", + "WF_LANE": "dev", "DISPATCH_JOB": job_id}) + env.update(additions) + return env + + +def _stream(job_dir, harness, session, snapshot, ref_sha): + if harness == "codex": + paths = list((job_dir / "home/codex/sessions").rglob("rollout-*.jsonl")) + elif harness == "grok": + paths = list((job_dir / "home/grok/sessions").rglob("summary.json")) + else: + slug = str(snapshot).replace("/", "-").replace(".", "-") + paths = list((Path(os.environ["HOME"]) / ".claude/projects" / slug).glob("*.jsonl")) + if not paths: + return None, None, None, None + path = max(paths, key=lambda p: p.stat().st_mtime) + ran = None + stream_id = None + mismatch = None + try: + if path.name == "summary.json": + data = json.loads(path.read_text()) + ran = data.get("current_model_id") + stream_id = (data.get("info") or {}).get("id") + if data.get("head_commit") and data["head_commit"] != ref_sha: + mismatch = "head_commit mismatch" + else: + for line in path.read_text().splitlines(): + row = json.loads(line) + if row.get("type") == "assistant": + ran = (row.get("message") or {}).get("model") or ran + stream_id = row.get("sessionId") or stream_id + payload = row.get("payload") or {} + if row.get("type") == "session_meta": + stream_id = payload.get("id") + if row.get("type") == "turn_context": + ran = payload.get("model") + except (json.JSONDecodeError, OSError, RuntimeError, TypeError, ValueError) as exc: + mismatch = f"stream parse: {exc}" + if stream_id and harness != "codex" and stream_id != session: + mismatch = f"session mismatch: minted {session}, found {stream_id}" + return path, ran, stream_id, mismatch + + +ENVELOPE_KEYS = ("job", "status", "verdict", "counts", "findings", + "artifacts", "spend", "stamp", "note") + + +def _invalid(job, ref_sha, note): + return {"job": job, "status": "invalid", "verdict": None, "counts": {}, + "findings": [], "artifacts": {}, "spend": {}, + "stamp": {"ref": ref_sha}, "note": note} + + +def _invalidate_envelope(data, note): + data["status"] = "invalid" + data["verdict"] = None + prior = data.get("note") + data["note"] = f"{prior}; {note}" if prior else note + return data + + +def _json_objects(text): + decoder = json.JSONDecoder() + found = [] + for match in re.finditer(r"\{", text): + try: + candidate, _end = decoder.raw_decode(text, match.start()) + except json.JSONDecodeError: + continue + if isinstance(candidate, dict): + found.append(candidate) + return found + + +def _schema_valid(data): + def matches(value, schema): + kinds = schema.get("type") + kinds = [kinds] if isinstance(kinds, str) else kinds + checks = {"object": lambda: isinstance(value, dict), + "array": lambda: isinstance(value, list), + "string": lambda: isinstance(value, str), + "integer": lambda: isinstance(value, int) and not isinstance(value, bool), + "number": lambda: isinstance(value, (int, float)) and not isinstance(value, bool), + "boolean": lambda: isinstance(value, bool), + "null": lambda: value is None} + if kinds and not any(checks[kind]() for kind in kinds): + return False + if "enum" in schema and value not in schema["enum"]: + return False + if isinstance(value, (int, float)) and "minimum" in schema and value < schema["minimum"]: + return False + if isinstance(value, list) and "items" in schema: + return all(matches(item, schema["items"]) for item in value) + if isinstance(value, dict): + properties = schema.get("properties", {}) + if any(key not in value for key in schema.get("required", [])): + return False + extra = set(value) - set(properties) + additional = schema.get("additionalProperties", True) + if extra and additional is False: + return False + if any(not matches(value[key], properties[key]) + for key in set(value) & set(properties)): + return False + if isinstance(additional, dict) and any( + not matches(value[key], additional) for key in extra): + return False + return True + + return (matches(data, ENVELOPE.ENVELOPE_SCHEMA) + and not (data.get("status") == "invalid" + and data.get("verdict") == "approve")) + + +def _most_envelope_shaped(objects, *, last=False): + if not objects: + return None + score = max(sum(key in item for key in ENVELOPE_KEYS) for item in objects) + matches = [item for item in objects + if sum(key in item for key in ENVELOPE_KEYS) == score] + return matches[-1] if last else matches[0] + + +def _wrapper_spend(wrapper): + usage = wrapper.get("usage") + if not isinstance(usage, dict): + return None + cached = (usage.get("cache_read_input_tokens") or 0) + ( + usage.get("cache_creation_input_tokens") or 0) + spend = { + "input": usage.get("input_tokens"), + "cached": cached or None, + "output": usage.get("output_tokens"), + "source": "result.usage", + } + if wrapper.get("total_cost_usd") is not None: + spend["cost_usd"] = wrapper["total_cost_usd"] + spend = {key: value for key, value in spend.items() if value is not None} + if not any(value for key, value in spend.items() if key != "source"): + return None + spend["total"] = sum(spend.get(key, 0) for key in ("input", "cached", "output")) + return spend + + +def _merge_spend(prior, wrapper): + if not wrapper: + return prior + merged = dict(prior) if isinstance(prior, dict) and "unresolved" not in prior else {} + merged.update(wrapper) + merged["total"] = sum(merged.get(key, 0) for key in ("input", "cached", "output")) + return merged + + +def _envelope(raw, job, ref_sha, note=None, *, harness=None, job_dir=None, + rec=None): + """The last JSON object on stdout, or an invalid envelope that says why. + + Narration before or after the object is tolerated (a harness talks + while it works); a missing key is a refusal, not a default; a `stamp` + that is not an object is replaced rather than mislabeled as a model id + (codex once returned the SHA as a string). + """ + text = raw.decode("utf-8", "replace") + objects = _json_objects(text) + data = None + cause = None + if harness == "claude" and objects: + wrapper = next((item for item in objects if item.get("type") == "result"), None) + if wrapper is not None: + if rec is not None: + spend = _wrapper_spend(wrapper) + if spend: + rec["session"]["spend"] = _merge_spend( + rec["session"].get("spend"), spend) + if wrapper.get("is_error"): + cause = f"claude-result:{wrapper.get('subtype') or 'unknown'}" + elif isinstance(wrapper.get("structured_output"), dict): + data = wrapper["structured_output"] + elif isinstance(wrapper.get("result"), str): + data = _most_envelope_shaped(_json_objects(wrapper["result"]), + last=True) + if data is None and cause is None: + cause = f"claude-result:{wrapper.get('subtype') or 'unknown'}" + elif harness == "codex": + last = Path(job_dir) / "out" / "last-message.json" if job_dir else None + if last is not None and last.is_file(): + try: + candidate = json.loads(last.read_text(encoding="utf-8")) + data = candidate if isinstance(candidate, dict) else None + except (OSError, json.JSONDecodeError): + data = None + if data is None: + for item in objects: + payload = item.get("item") + if isinstance(payload, dict) and payload.get("type") == "agent_message": + try: + candidate = json.loads(payload.get("text", "")) + except json.JSONDecodeError: + continue + if isinstance(candidate, dict): + data = candidate + if data is None: + data = next((item for item in reversed(objects) + if _schema_valid(item)), None) + if data is None: + cause = "no-last-message" + elif harness == "grok" and objects: + # JSON output is the final top-level object; an earlier object can be + # narration or a diagnostic and is not the requested result. + data = next((item for item in reversed(objects) + if _schema_valid(item)), None) + if data is None: + data = _most_envelope_shaped(objects, last=True) + if data is None and cause is None: + # Compatibility for pre-U10 raw.out: choose the most envelope-shaped + # complete object, preserving the first on ties. + data = _most_envelope_shaped(objects) + if data is None: + if cause is None: + cause = "envelope-parse" + refusal = (f"{cause}: no structured envelope" + if cause != "envelope-parse" + else "envelope-parse: no JSON object on stdout") + data = _invalid(job, ref_sha, refusal) + elif harness is not None and not _schema_valid(data): + cause = "schema-invalid" + data = dict(data) + data.update(status="invalid", verdict=None, + note="schema-invalid: object does not match ENVELOPE_SCHEMA") + elif harness is None: + missing = [key for key in ENVELOPE_KEYS if key not in data] + if missing: + data["status"] = "invalid" + prior = data.get("note") + refusal = f"envelope-missing: {', '.join(missing)}" + data["note"] = f"{prior}; {refusal}" if prior else refusal + stamp = data.get("stamp") + if not isinstance(stamp, dict): + data["stamp"] = {} + data["stamp"]["ref"] = ref_sha + if note: + _invalidate_envelope(data, note) + if rec is not None and cause: + rec["_envelope_cause"] = cause + return data + + +def _write_record(repo, rec): + path = repo / ".dev/records/dispatches" / f"{rec['id']}.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(record.validate(rec), indent=2) + "\n", encoding="utf-8") + return path + + +def _commit_record(repo, path, rec): + rel = path.relative_to(repo) + env = dict(os.environ) + env.update(GIT_AUTHOR_NAME=OWNER_NAME, GIT_AUTHOR_EMAIL=OWNER_EMAIL, + GIT_COMMITTER_NAME=OWNER_NAME, GIT_COMMITTER_EMAIL=OWNER_EMAIL) + msg = (f"dispatch: record {rec['id']}\n\nSource: generated: ops/devlane/dispatch/launch.py\n" + f"Co-Authored-By: {rec['dispatched_by']}") + for item in rec.get("overrides", []): + msg += f"\n\nOverride: {item['refusal']}: {item['reason']}" + _git(repo, "add", "--intent-to-add", "--", str(rel), env=env) + _git(repo, "commit", "--only", "-m", msg, "--", str(rel), env=env) + + +def _run_attempt(rec, job_dir, *, job_caps=None, resume=False, reason=None): + harness = rec["harness"]["name"] + snapshot = Path(rec["snapshot"]["root"]) + prompt = job_dir / "prompt.txt" + iso_env = rec["harness"]["isolation"].get("env", {}) + flags = rec["harness"]["isolation"].get("flags", []) + argv = _argv(harness, rec["model"]["requested"], rec["model"]["effort_requested"], + rec["session"]["id"], prompt, flags, rec["harness"]["sandbox"], resume) + rec["harness"]["argv"] = argv + if harness == "codex": + (job_dir / "out" / "last-message.json").unlink(missing_ok=True) + env = _child_env(os.environ, iso_env, rec["id"]) + for marker in ("exit", "TRIPPED.md"): + (job_dir / marker).unlink(missing_ok=True) + launched = _now() + raw_mode = "ab" if resume else "wb" + # codex exec and claude --print read the brief from stdin ("Input must + # be provided either through stdin or as a prompt argument"); grok names + # the prompt file on argv and gets DEVNULL so nothing waits on a tty. + feed = prompt.open("rb") if harness in ("codex", "claude") else None + with (job_dir / "raw.out").open(raw_mode) as raw, (job_dir / "stderr").open(raw_mode) as err: + proc = subprocess.Popen(argv, cwd=snapshot, env=env, + stdin=feed if feed is not None else subprocess.DEVNULL, + stdout=raw, stderr=err, start_new_session=True) + if feed is not None: + feed.close() + state = {"pid": proc.pid, "pgid": proc.pid, "session": {"id": rec["session"]["id"]}, + "stream": rec["session"].get("stream"), "attempt": len(rec["attempts"]) + 1} + (job_dir / "state.json").write_text(json.dumps(state) + "\n") + job_timeout = (job_caps or {}).get("timeout") + timeout = float(os.environ.get( + "DISPATCH_TIMEOUT", job_timeout if job_timeout is not None else "900")) + rec["caps"]["timeout"] = timeout + rec["caps"]["timeout_source"] = ( + "DISPATCH_TIMEOUT" if "DISPATCH_TIMEOUT" in os.environ + else "job" if job_timeout is not None else "default") + grace = float(os.environ.get("DISPATCH_STREAM_GRACE", "120")) + started_clock = monotonic() + runtime_note = None + while proc.poll() is None: + elapsed = monotonic() - started_clock + stream, _ran, _sid, _mis = _stream( + job_dir, harness, rec["session"]["id"], snapshot, + rec["snapshot"]["ref_sha"], + ) + over = False + if stream and rec["caps"].get("cap-out") is not None: + with contextlib.suppress(OSError): + over = any(int(n) > int(rec["caps"]["cap-out"]) + for n in re.findall(r'"output_tokens"\s*:\s*(\d+)', stream.read_text())) + if over: + runtime_note = "trip: output cap exceeded" + break + if elapsed >= timeout: + runtime_note = "timeout: harness exceeded runtime" + break + if not stream and elapsed >= grace: + runtime_note = "unsupervised: no session stream within grace" + break + sleep(min(0.05, max(0.0, min(timeout, grace) - elapsed))) + if runtime_note: + with contextlib.suppress(OSError, ProcessLookupError): + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + code = 137 + (job_dir / "TRIPPED.md").write_text(runtime_note + "\n") + rec["_runtime_note"] = runtime_note + else: + code = proc.returncode + (job_dir / "exit").write_text(f"{code}\n") + rec["attempts"].append({"n": len(rec["attempts"]) + 1, "launched": launched, + "ended": _now(), "exit": code, + "tripped": (job_dir / "TRIPPED.md").exists()}) + if reason: + rec.setdefault("note", reason) + return code + + +def _launch(args): + agent = _identity() + repo = _repo() + branch = _branch(repo) + lineage = args.lineage or branch + if branch in {"detached", "dev", "main"} or branch != lineage: + _refuse("record-target", f"the checkout on {lineage}", branch, + f"run from a worktree of {lineage}") + root = _jobs_root().resolve() + wt = _git(repo, "worktree", "list", "--porcelain").stdout + for line in wt.splitlines(): + if line.startswith("worktree ") and _inside(root, Path(line[9:])): + _refuse("live-target", f"a job directory outside every worktree of {repo}", + f"{root} inside {line[9:]}", "set DISPATCH_JOBS outside the repository") + jobs = json.loads(Path(JOBS_PATH).read_text()) + if args.job not in jobs or jobs[args.job].get("adapter") != "harness": + _refuse("job", "a harness job in jobs.json", args.job, "name a dispatchable job") + job = jobs[args.job] + role = job.get("role") + stage = args.stage or ("code" if role == "write" else "review") + if not args.model: + _refuse("model", "a model", "none", "pass --model") + if args.harness == "codex" and args.effort: + _refuse("model", "an effort-capable adapter", "effort on codex", "drop --effort") + if args.harness == "claude" and role == "write": + _refuse("write-role-unadmitted", "a write row with a containment entry", + "none for claude", "run the containment probe or use codex or grok") + mode = job.get("snapshot") + if job.get("withheld") and mode == "whole": + _refuse("history-vs-withheld", "one of history or withholding", f"both on {args.job}", "declare fileset") + if mode != "whole": + _refuse("mode-unavailable", "whole", f"{mode} on {args.job}", "stage it by hand") + scope_bytes = len((args.scope or "").encode()) + takes_scope = "{scope}" in (job.get("prompt") or "") + if scope_bytes > 1024 or (args.scope is not None and not takes_scope): + _refuse("scope-cap", "<= 1024 bytes on a job that takes a scope", + f"{scope_bytes} bytes on {args.job}", "shorten it or drop it") + resolved = _git(repo, "rev-parse", "--verify", f"{args.ref}^{{commit}}", check=False) + if resolved.returncode: + _refuse("ref", f"a ref naming a commit in {repo}", f"{args.ref} ({resolved.stderr.strip()})", + "commit the work, then name the commit") + ref_sha = resolved.stdout.strip() + overrides = _parse_override(args.override, agent) + ancestor = _git(repo, "merge-base", "--is-ancestor", ref_sha, lineage, check=False).returncode == 0 + if not ancestor and not overrides: + tip = _git(repo, "rev-parse", lineage).stdout.strip() + _refuse("stale-base", f"{ref_sha[:8]} reachable from {lineage}", + f"it is not ({lineage} is at {tip[:8]})", "name a commit on the branch or override stale-base:REASON") + # The base a review diffs against is where the lineage left dev, not + # the lineage's own tip -- merge-base(ref, lineage) is ref itself when + # ref is the tip, and a review of an empty comparison reviews nothing. + candidates = [] + for anchor in ("origin/dev", "dev"): + probe = _git(repo, "merge-base", ref_sha, anchor, check=False) + if probe.returncode == 0 and probe.stdout.strip(): + candidate = probe.stdout.strip() + distance = int(_git(repo, "rev-list", "--count", + f"{candidate}..{ref_sha}").stdout) + candidates.append((distance, candidate)) + if candidates: + base_sha = min(candidates)[1] + else: + base_sha = _git(repo, "merge-base", ref_sha, lineage).stdout.strip() + prompt_template = job.get("prompt") or "" + takes_comparison = all(slot in prompt_template for slot in ("{base}", "{diff}")) + if takes_comparison and args.stage == "review" and base_sha == ref_sha: + _refuse("empty-comparison", f"{lineage} to differ from ref {ref_sha}", + f"an empty comparison at {ref_sha}", + f"name a ref on {lineage} with changes to review") + job_id = mint_id(stage, args.harness) + job_dir = root / job_id + record_path = repo / ".dev/records/dispatches" / f"{job_id}.json" + _preflight_isolation(args.harness, job_dir / "home" / args.harness, os.environ) + if job_dir.exists() or record_path.exists(): + raise RuntimeError(f"dispatch id collision: {job_id}") + job_dir.mkdir(parents=True) + for name in ("in", "out"): + (job_dir / name).mkdir() + snapshot = job_dir / "snapshot" + _snapshot(repo, lineage, ref_sha, snapshot) + copied = [] + for source in args.input: + src = Path(source) + dest = job_dir / "in" / src.name + if dest.exists(): + raise RuntimeError(f"duplicate input basename: {src.name}") + shutil.copyfile(src, dest) + copied.append({"path": str(dest), "sha256": _sha(dest)}) + diff_text = _git(repo, "diff", f"{base_sha}..{ref_sha}").stdout if base_sha != ref_sha else "" + context_dir = job_dir / "context" + context_dir.mkdir() + context_diff = context_dir / "diff.patch" + context_diff.write_text(diff_text) + diff_file = job_dir / "diff.patch" + diff_file.symlink_to(context_diff) + diff_path = str(diff_file.resolve()) + values = {"ref": ref_sha, "base": base_sha, "diff": diff_path, "into": str(snapshot.resolve()), + "out": str((job_dir / "out").resolve()), + "inputs": " ".join(item["path"] for item in copied), "scope": args.scope or ""} + prompt_text = _render(job["prompt"], values) + prompt = job_dir / "prompt.txt" + prompt.write_text(prompt_text) + _iso_env, _flags, sandbox, containment, isolation_data = _harness_meta(args.harness, role, job_dir, os.environ) + session = str(uuid.uuid4()) + wires = _module("wires", HARNESS_APP / "wires.py") + tip = _git(repo, "rev-parse", lineage).stdout.strip() + rec = record.build({ + "id": job_id, "lane": "dev", "stage": stage, "unit": args.unit or lineage, + "lineage": {"branch": lineage, "base_sha": tip}, "follows": args.follows, + "job": args.job, "role": role, "dispatched_by": agent, + "at": {"launched": _now(), "closed": None}, + "snapshot": {"mode": "whole", "ref_name": args.ref, "ref_sha": ref_sha, + "behind_tip": (lambda n: n + 1 if n else 0)(int( + _git(repo, "rev-list", "--count", f"{ref_sha}..{lineage}").stdout)), + "root": str(snapshot.resolve())}, + "harness": {"name": args.harness, "version": "unknown", "isolation": isolation_data, + "sandbox": sandbox, "containment": containment, + "argv": _argv(args.harness, args.model, args.effort, session, + prompt, _flags, sandbox)}, + "model": {"requested": args.model, "effort_requested": args.effort, + "ran": None, "read_from": None, "note": "no stream found"}, + "session": {"id": session, "stream": None, "stream_sha256_at_close": None}, + "brief": {"template": {"path": str(JOBS_PATH), "sha256": _sha(JOBS_PATH)}, + "scope": args.scope, "inputs": copied, "sha256": _sha(prompt), "bytes": prompt.stat().st_size}, + "caps": {"cap-out": wires.budget(role), "source": "wires.py"}, + "overrides": overrides, "attempts": [], "result": None, "status": "launched"}) + _write_record(repo, rec) + _run_attempt(rec, job_dir, job_caps=job.get("caps")) + path, ran, found_id, mismatch = _stream( + job_dir, args.harness, session, snapshot, ref_sha, + ) + if path: + if args.harness in {"codex", "grok"}: + old_store = job_dir / "home" / args.harness / "sessions" + new_store = job_dir / "home" / f"{args.harness}-stream" / "sessions" + new_store.parent.mkdir(parents=True, exist_ok=True) + if old_store.exists(): + shutil.move(str(old_store), str(new_store)) + path = new_store / path.relative_to(old_store) + rec["session"].update(stream=str(path), stream_sha256_at_close=_sha(path)) + rec["model"].update(ran=ran, read_from=str(path)) + rec["model"].pop("note", None) + if args.harness == "codex" and found_id: + rec["session"]["id"] = found_id + rec["session"]["spend"] = _spend(rec, path, job_dir) + raw = (job_dir / "raw.out").read_bytes() + runtime_note = rec.pop("_runtime_note", None) + attempt_code = rec["attempts"][-1]["exit"] + if runtime_note: + env = _envelope(raw, args.job, ref_sha, runtime_note, + harness=args.harness, job_dir=job_dir, rec=rec) + if runtime_note.startswith("trip"): + env["status"] = "tripped" + elif attempt_code: + env = _envelope(raw, args.job, ref_sha, + f"harness-cli: exited {attempt_code}", + harness=args.harness, job_dir=job_dir, rec=rec) + else: + env = _envelope(raw, args.job, ref_sha, mismatch, + harness=args.harness, job_dir=job_dir, rec=rec) + _settle(repo, record_path, rec, env) + return 2 if runtime_note else 0 + + +VENDOR = {"codex": "noreply@openai.com", "grok": "noreply@x.ai", "claude": "noreply@anthropic.com"} +# Attribution names per CONTRIB.md, keyed by the model id the stream reports. +# An unknown id is credited as itself rather than guessed. +MODEL_NAMES = {"gpt-5.6-sol": "GPT-5.6 Sol", "grok-4.6": "Grok 4.6", + "claude-opus-5": "Claude Opus 5", "claude-fable-5": "Claude Fable 5", + "claude-sonnet-5": "Claude Sonnet 5"} + + +def _commit_message(rec, env=None, job_dir=None): + """The message the launcher commits with: the harness's own subject and + body when it wrote out/COMMIT_MSG or its envelope carries + `commit: {subject, body}`, else the generic line; either way the + dispatch id and the attribution the CONTRIB template names (the + model's display name, never its id).""" + model = rec["model"].get("ran") or rec["model"]["requested"] + name = MODEL_NAMES.get(model, model) + vendor = VENDOR.get(rec["harness"]["name"], "noreply@unknown") + commit = (env or {}).get("commit") + msg_file = job_dir / "out" / "COMMIT_MSG" if job_dir else None + if msg_file is not None and msg_file.is_file(): + text = msg_file.read_text(errors="replace").strip() + if text: + first, _, rest = text.partition("\n") + commit = {"subject": first, "body": rest} + subject = body = None + kept_trailers = [] + if isinstance(commit, dict) and isinstance(commit.get("subject"), str) and commit["subject"].strip(): + subject = commit["subject"].strip().splitlines()[0] + body = commit.get("body") if isinstance(commit.get("body"), str) else "" + body_lines = [] + for line in body.strip().splitlines(): + # A trailer-shaped line anywhere but the final block is body + # text to git and a refusal to the commit-msg hook (2026-08-29: + # a codex COMMIT_MSG with `Reviewed-by:` mid-body was refused + # and the generic subject landed). The launcher's own trailers + # are replaced; every other trailer the harness wrote moves + # into the final block, contiguous. + if re.match(r"^Dispatch:", line): + continue + if re.match(r"^[A-Za-z][A-Za-z-]*: \S", line) and not line.startswith(("http", "Note:", "TODO:")): + kept_trailers.append(line.strip()) + continue + body_lines.append(line) + body = "\n".join(body_lines).strip() + if subject is None: + subject = f"{rec['job']}: work of dispatch {rec['id']}" + body = "" + sources = [x for x in kept_trailers if x.startswith("Source:")] + coauthors = [re.sub(r"^Co-authored-by:", "Co-Authored-By:", x) for x in kept_trailers + if x.lower().startswith("co-authored-by:")] + others = [x for x in kept_trailers if not x.startswith("Source:") and not x.lower().startswith("co-authored-by:")] + mine = f"Co-Authored-By: {name} <{vendor}>" + # the running model's line replaces any the harness wrote for *its own* + # vendor (an id or a wrong display name); other co-authors are kept + coauthors = [c for c in coauthors if f"<{vendor}>" not in c] + block = [] + for line in [*sources, "Source: original", *others, *coauthors, mine]: + if line not in block: + block.append(line) + return (f"{subject}\n\n" + (f"{body}\n\n" if body else "") + + f"Committed by the launcher: the {rec['harness']['name']} sandbox denies .git.\n" + f"Dispatch: {rec['id']}\n\n" + "\n".join(block) + "\n") + + +def _commit_for_harness(snapshot, rec, env=None): + """codex's sandbox keeps .git read-only, so a write job leaves its work + uncommitted in the snapshot (2026-08-28: an implement job made the + named test green and could not commit). The launcher commits it, + crediting the model that ran, with the harness's own message when the + envelope carries one (2026-08-29: five green codex runs landed under + a generic subject and the conductor re-wrote each by hand).""" + owner_env = dict(os.environ) + owner_env.update(GIT_AUTHOR_NAME=OWNER_NAME, GIT_AUTHOR_EMAIL=OWNER_EMAIL, + GIT_COMMITTER_NAME=OWNER_NAME, + GIT_COMMITTER_EMAIL=OWNER_EMAIL) + _git(snapshot, "add", "-A", env=owner_env) + _git(snapshot, "commit", "-q", "-F", "-", env=owner_env, + input=_commit_message(rec, env, snapshot.parent)) + + +def _cause(rec, job_dir, env=None): + """Why the harness stopped, in the order plan ebe2bb U4b.1 fixes: + tripped → timeout → unsupervised → harness-cli: → a permission + prompt the turn never recovered from → the last event the session + store holds. First match wins; a cancelled prompt is blamed only when + it is the last permission event and nothing but phase/turn bookkeeping + followed it (review ba0d93: a cancellation the turn recovered from + must not be blamed).""" + env = env or {} + attempt = (rec.get("attempts") or [{}])[-1] + marker = job_dir / "TRIPPED.md" + if attempt.get("tripped") or env.get("status") == "tripped" or marker.exists(): + # The launcher writes TRIPPED.md for all three kills; only its own + # first line says which (review ff6440 N3: `tripped` shadowed + # timeout and unsupervised; N4: the harness's prose must never + # decide the reason). + own = marker.read_text(errors="replace").strip().splitlines()[0].lower() if marker.exists() else "" + for word in ("timeout", "unsupervised"): + if own.startswith(word) or f" {word}" in own[:80]: + return {"reason": word, "note": own or None} + return {"reason": "tripped", "note": own or None} + if attempt.get("exit"): + return {"reason": f"harness-cli:{attempt['exit']}"} + if rec.get("_envelope_cause"): + return {"reason": rec.pop("_envelope_cause")} + harness = rec["harness"]["name"] + stream = (rec.get("session") or {}).get("stream") + events = None + if harness == "grok": + candidates = [] + if stream and Path(stream).is_file(): + candidates.append(Path(stream).parent / "events.jsonl") + candidates += sorted((job_dir / "home").glob("grok-stream/**/events.jsonl")) + candidates += sorted((job_dir / "home").glob("grok/**/events.jsonl")) + events = next((c for c in candidates if c.is_file()), None) + elif stream and Path(stream).is_file(): + events = Path(stream) + if events is None: + return {"reason": "no-session-store"} + last = None + last_permission = None + after_permission = [] + for line in events.read_text(errors="replace").splitlines(): + try: + item = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(item, dict): + continue + last = item + if item.get("type") == "permission_resolved": + last_permission = item + after_permission = [] + elif last_permission is not None: + after_permission.append(item.get("type")) + # A cancelled tool emits its own tool_result row; that is the cancel, + # not recovery (review ff6440 N2). Recovery is later work: another + # permission request, an assistant turn, a tool call. + if after_permission and after_permission[0] == "tool_result": + after_permission = after_permission[1:] + bookkeeping = ("phase_changed", "turn_ended", "session_end", "tool_result") + if (last_permission is not None + and last_permission.get("decision") not in (None, "approved", "allowed", "allow") + and all(kind in bookkeeping for kind in after_permission)): + return {"reason": f"permission-{last_permission.get('decision')}", + "tool": last_permission.get("tool_name"), + "wait_ms": last_permission.get("wait_ms"), "at": last_permission.get("ts")} + if last is None: + return {"reason": "empty-session-store", "path": str(events)} + return {"reason": "last-event", "type": last.get("type"), + "at": last.get("ts") or last.get("timestamp")} + + +def _spend(rec, stream, job_dir): + """`session.spend` from the harness's own store, read by + telemetry/usage.py (the same parsers the usage report uses). A value + or `{unresolved: }` — never a zero (U5; 2026-08-29: 105 of 105 + records carried `spend: null`, so no cost could be measured).""" + harness = rec["harness"]["name"] + session = (rec.get("session") or {}).get("id") + if not stream: + return {"unresolved": "no session stream discovered"} + try: + usage = _module("usage", TELEMETRY_APP / "usage.py") + except (OSError, ImportError, SyntaxError) as exc: + return {"unresolved": f"usage.py not loadable: {exc}"} + stream = Path(stream) + try: + if harness == "claude": + root = stream.parent.parent + sessions = list(usage.claude_sessions(root, None)) + elif harness == "codex": + root = job_dir / "home" / "codex-stream" + if not root.exists(): + root = job_dir / "home" / "codex" + sessions = list(usage.codex_sessions(root, None)) + elif harness == "grok": + root = job_dir / "home" / "grok-stream" + if not root.exists(): + root = job_dir / "home" / "grok" + sessions = list(usage.grok_sessions(root, None)) + else: + return {"unresolved": f"no usage parser for {harness}"} + except (OSError, ValueError, KeyError, TypeError, json.JSONDecodeError) as exc: + return {"unresolved": f"usage parse failed: {type(exc).__name__}: {exc}"} + match = next((s for s in sessions if isinstance(s, dict) and str(s.get("session")) == str(session)), None) + if match is None: + return {"unresolved": f"session {session} not in the store ({len(sessions)} sessions read)"} + tokens = match.get("tokens") + if not isinstance(tokens, dict) or not any(isinstance(v, (int, float)) for v in tokens.values()): + return {"unresolved": "the store carries no usage for this session", "source": str(stream)} + spend = {k: tokens.get(k) for k in ("input", "cached", "output", "total") if tokens.get(k) is not None} + if match.get("cost_usd_ticks") is not None: + spend["cost_usd_ticks"] = match["cost_usd_ticks"] + if match.get("incomplete"): + spend["incomplete"] = True + spend["source"] = str(stream) + spend["messages"] = match.get("messages") + return spend + + +def _write_residual_patch(snapshot, job_dir): + """Preserve what a harness left uncommitted — staged, unstaged and + untracked alike — as one patch `git apply` accepts, so a retry can take + it as an input instead of the conductor diffing the snapshot by hand. + A temporary index keeps the snapshot's own index untouched.""" + import tempfile + untracked = _git(snapshot, "ls-files", "--others", "--exclude-standard", check=False).stdout.splitlines() + with tempfile.NamedTemporaryFile(prefix="residual-index-", delete=False) as tmp: + index = tmp.name + env = dict(os.environ, GIT_INDEX_FILE=index) + try: + _git(snapshot, "read-tree", "HEAD", env=env) + excludes = [f":(exclude,glob)**/{seg.rstrip('/')}/**" for seg in CACHE_RESIDUAL] + _git(snapshot, "add", "-A", "--", ".", *excludes, env=env) + patch = _git(snapshot, "diff", "--cached", "--binary", "HEAD", env=env, check=False).stdout + finally: + Path(index).unlink(missing_ok=True) + out = job_dir / "residual.patch" + out.write_text(patch) + return {"path": str(out), "sha256": _sha(out), "untracked": untracked} + + +# Tool caches a read role leaves behind by running the suite or ruff are +# not writes into the tree; everything else is. +CACHE_RESIDUAL = ("__pycache__/", ".ruff_cache/", ".pytest_cache/", ".mypy_cache/") + + +def _residual(snapshot): + lines = _git(snapshot, "status", "--porcelain=v1", "-uall").stdout.splitlines() + return [line for line in lines + if not any(seg in line[3:] for seg in CACHE_RESIDUAL)] + + +def _settle(repo, record_path, rec, env): + """Collect the snapshot's state, finalize the record, commit it.""" + snapshot = Path(rec["snapshot"]["root"]) + ref_sha = rec["snapshot"]["ref_sha"] + role, job_id = rec["role"], rec["id"] + residual = _residual(snapshot) + tripped = bool(rec.get("attempts") and rec["attempts"][-1].get("tripped")) + if role == "write" and rec["harness"]["name"] == "codex" and residual and not tripped: + _commit_for_harness(snapshot, rec, env) + residual = _residual(snapshot) + residual_patch = _write_residual_patch(snapshot, Path(rec["snapshot"]["root"]).parent) if residual else None + head = _git(snapshot, "rev-parse", "HEAD").stdout.strip() + changed = _git(snapshot, "diff", "--name-only", f"{ref_sha}..{head}").stdout.splitlines() + if role == "read" and head != ref_sha: + _invalidate_envelope(env, "read-role-head: HEAD must equal ref_sha") + elif role == "write" and _git(snapshot, "merge-base", "--is-ancestor", ref_sha, head, check=False).returncode: + _invalidate_envelope(env, "off-lineage-head: HEAD does not descend from ref_sha") + elif role == "write": + _git(repo, "-c", "core.logAllRefUpdates=always", "fetch", + "--no-write-fetch-head", str(snapshot), f"{head}:refs/dispatch/{job_id}") + rec["result"] = {"head": head, "changed_paths": changed, + "residual_paths": residual, "envelope": env} + if residual_patch: + rec["result"]["residual_patch"] = residual_patch + cause = _cause(rec, Path(rec["snapshot"]["root"]).parent, env) + rec.pop("_envelope_cause", None) + rec["result"]["cause"] = cause + if env.get("status") == "invalid" and str(env.get("note", "")).startswith("envelope-parse"): + env["note"] = f"{env['note']}; cause: {json.dumps(cause, sort_keys=True)}" + rec["at"]["closed"] = _now() + rec["status"] = "closed" + _write_record(repo, rec) + _commit_record(repo, record_path, rec) + + +def _status(args): + root = _jobs_root() + dirs = [root / args.id] if args.id else sorted(p for p in root.glob("*") if p.is_dir()) + rows = [] + for d in dirs: + if not d.exists(): + state = "unlaunched" + elif (d / "TRIPPED.md").exists(): + state = "tripped" + elif (d / "exit").exists(): + state = "finished" + else: + try: + pid = json.loads((d / "state.json").read_text()).get("pid") + os.kill(int(pid), 0) + state = "running" + except (json.JSONDecodeError, OSError, TypeError, ValueError): + state = "DIED" + rows.append({"id": d.name, "status": state}) + payload = rows[0] if args.id and rows else rows + print(json.dumps(payload) if args.json else "\n".join(f"{r['id']} {r['status']}" for r in rows)) + return 0 + + +def _find_record(repo, job_id): + path = repo / ".dev/records/dispatches" / f"{job_id}.json" + if not path.is_file(): + raise RuntimeError(f"record not found: {job_id}") + return path, json.loads(path.read_text()) + + +def _resume(args): + repo = _repo() + _path, rec = _find_record(repo, args.id) + job_dir = _jobs_root() / args.id + prior_envelope = (rec.get("result") or {}).get("envelope") or {} + settled_refusal = None + if rec.get("status") == "closed" and prior_envelope.get("status") == "invalid": + settled_refusal = { + key: prior_envelope.get(key) for key in ("status", "verdict", "note") + } + if (job_dir / "TRIPPED.md").exists() and not args.reason and args.cap_out is None: + raise RuntimeError("tripped job requires a changed cap or reason") + if args.cap_out is not None: + rec["caps"]["cap-out"] = args.cap_out + rec["caps"]["source"] = "resume" + jobs = json.loads(Path(JOBS_PATH).read_text()) + _run_attempt(rec, job_dir, job_caps=jobs[rec["job"]].get("caps"), + resume=True, reason=args.reason) + runtime_note = rec.pop("_runtime_note", None) + if runtime_note: + envelope = (rec.get("result") or {}).get("envelope") or {} + envelope.update(status="tripped" if runtime_note.startswith("trip") else "invalid", + verdict=None, note=runtime_note) + rec.setdefault("result", {})["envelope"] = envelope + else: + raw = (job_dir / "raw.out").read_bytes() + code = rec["attempts"][-1]["exit"] + note = f"harness-cli: exited {code}" if code else None + stream, ran, found_id, mismatch = _stream( + job_dir, rec["harness"]["name"], rec["session"]["id"], + Path(rec["snapshot"]["root"]), rec["snapshot"]["ref_sha"], + ) + if stream and rec["harness"]["name"] in {"codex", "grok"}: + old_store = job_dir / "home" / rec["harness"]["name"] / "sessions" + new_store = job_dir / "home" / f"{rec['harness']['name']}-stream" / "sessions" + if old_store.exists(): + new_store.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(old_store, new_store, dirs_exist_ok=True) + stream = new_store / stream.relative_to(old_store) + shutil.rmtree(old_store) + if stream: + rec["session"].update( + stream=str(stream), stream_sha256_at_close=_sha(stream), + ) + rec["model"].update(ran=ran, read_from=str(stream)) + rec["model"].pop("note", None) + if rec["harness"]["name"] == "codex" and found_id: + rec["session"]["id"] = found_id + rec["session"]["spend"] = _spend(rec, stream, job_dir) + if mismatch: + note = f"{note}; {mismatch}" if note else mismatch + env = _envelope(raw, rec["job"], rec["snapshot"]["ref_sha"], note, + harness=rec["harness"]["name"], job_dir=job_dir, + rec=rec) + if settled_refusal is not None: + env.update(settled_refusal) + rec.setdefault("result", {})["envelope"] = env + rec["result"]["cause"] = _cause(rec, job_dir, env) + _write_record(repo, rec) + return 0 + + +def _close(args): + repo = _repo() + path, rec = _find_record(repo, args.id) + branch = _branch(repo) + lineage = rec["lineage"]["branch"] + if branch != lineage: + _refuse("record-target", f"the checkout on {lineage}", branch, f"run from a worktree of {lineage}") + job_dir = _jobs_root() / args.id + state = json.loads((job_dir / "state.json").read_text()) + try: + os.kill(int(state["pid"]), 0) + died = False + except (OSError, TypeError, ValueError): + died = not (job_dir / "exit").exists() + if died: + raw_path = job_dir / "raw.out" + raw = raw_path.read_bytes() if raw_path.is_file() else b"" + env = _envelope(raw, rec["job"], rec["snapshot"]["ref_sha"], + "DIED: process vanished without exit", + harness=rec["harness"]["name"], job_dir=job_dir, + rec=rec) + rec["result"] = {"head": rec["snapshot"]["ref_sha"], "changed_paths": [], + "residual_paths": [], "envelope": env} + rec["status"] = "died" + rec["at"]["closed"] = _now() + _write_record(repo, rec) + _commit_record(repo, path, rec) + return 0 + if (job_dir / "exit").exists() and not rec.get("result"): + # The harness finished and wrote its exit; the launcher did not get + # to collect (it crashed on 2026-08-28 parsing a string stamp). The + # output is still in raw.out, so collect it now rather than lose it. + code = int((job_dir / "exit").read_text().strip() or 0) + raw = (job_dir / "raw.out").read_bytes() + note = f"harness-cli: exited {code}" if code else None + if (job_dir / "TRIPPED.md").exists(): + note = "trip: battery tripped" + stream, ran, found_id, mismatch = _stream( + job_dir, rec["harness"]["name"], rec["session"]["id"], + Path(rec["snapshot"]["root"]), rec["snapshot"]["ref_sha"], + ) + if stream: + rec["session"].update( + stream=str(stream), stream_sha256_at_close=_sha(stream), + ) + rec["model"].update(ran=ran, read_from=str(stream)) + rec["model"].pop("note", None) + if rec["harness"]["name"] == "codex" and found_id: + rec["session"]["id"] = found_id + if mismatch: + note = f"{note}; {mismatch}" if note else mismatch + env = _envelope(raw, rec["job"], rec["snapshot"]["ref_sha"], note, + harness=rec["harness"]["name"], job_dir=job_dir, + rec=rec) + if not rec["attempts"]: + rec["attempts"].append({"n": 1, "launched": rec["at"]["launched"], + "ended": _now(), "exit": code, + "tripped": (job_dir / "TRIPPED.md").exists()}) + _settle(repo, path, rec, env) + return 0 + + +def _brief(args): + repo = _repo() + _path, rec = _find_record(repo, args.check) + prompt = _jobs_root() / args.check / "prompt.txt" + ok = prompt.is_file() and _sha(prompt) == rec["brief"]["sha256"] + print("brief matches" if ok else "brief digest mismatch") + return 0 if ok else 1 + + +def _parser(): + p = argparse.ArgumentParser(prog="launch.py") + sub = p.add_subparsers(dest="verb") + s = sub.add_parser("status") + s.add_argument("id", nargs="?") + s.add_argument("--json", action="store_true") + r = sub.add_parser("resume") + r.add_argument("id") + r.add_argument("--prompt-file") + r.add_argument("--reason") + r.add_argument("--cap-out", type=int) + c = sub.add_parser("close") + c.add_argument("id") + b = sub.add_parser("brief") + b.add_argument("--check", required=True) + return p + + +def _launch_parser(): + p = argparse.ArgumentParser(prog="launch.py") + p.add_argument("job") + p.add_argument("--harness", required=True) + p.add_argument("--model") + p.add_argument("--effort") + p.add_argument("--ref", required=True) + p.add_argument("--lineage") + p.add_argument("--unit") + p.add_argument("--stage", choices=STAGES) + p.add_argument("--scope") + p.add_argument("--input", action="append", default=[]) + p.add_argument("--follows", action="append", default=[]) + p.add_argument("--override", action="append", default=[]) + return p + + +def main(argv=None): + argv = list(sys.argv[1:] if argv is None else argv) + try: + if argv[:1] and argv[0] in {"status", "resume", "close", "brief"}: + args = _parser().parse_args(argv) + return {"status": _status, "resume": _resume, "close": _close, "brief": _brief}[args.verb](args) + return _launch(_launch_parser().parse_args(argv)) + except Refused as exc: + print(f"launch.py: refusal: {exc}", file=sys.stderr) + return REFUSAL + except (RuntimeError, OSError, ValueError, json.JSONDecodeError) as exc: + print(f"launch.py: error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/devlane/dispatch/levers/README.md b/ops/devlane/dispatch/levers/README.md new file mode 100644 index 0000000..9e45ab2 --- /dev/null +++ b/ops/devlane/dispatch/levers/README.md @@ -0,0 +1,37 @@ +# dispatch/levers — custody-isolated harness dispatch (preserved) + +Working levers + apply-push bridge, preserved from session 62d2e497 (Opus 4.8) +for the Fable handoff, so they survive the session. + +- `grok-dispatch.sh` — Grok, headless (investigation) +- `codex/codex-dispatch.sh` — Codex, headless (implementation/security) +- `claude/fable-dispatch.sh` — Claude Code/Fable, headless, read/write; defaults + to `claude-fable-5`. Claude is isolated by the canonical three CLI flags and + carries no auth files because `HARNESSES["claude"].auth_files` is empty. + Write jobs leave a lever-created commit in `wt/` and the run-record fields + consumed unchanged by `apply-push.sh --from-job`. Provisioned `.levers/` are + locally excluded in the write clone and cannot enter that commit. +- `claude/selftest.sh` — offline refusal and write-commit checks; it proves the + model is never invoked for missing scope, HOME scope, empty prompt, or a + missing auth-file declaration, and proves a provisioned write commit omits + `.levers/`. +- `apply-push.sh` + `selftest.sh` — the trusted bridge that lands lever-produced + patches (runs gates + validates trailers). Hardened through 4 real bugs. + +`fable-dispatch.sh --provision-levers` installs Grok, Codex (with its vendored +isolation law), and the bridge selftest under the job workspace's `.levers/`. +It deliberately excludes `apply-push.sh`: dispatched custody may conduct more +isolated jobs but cannot push. Landing remains conductor-side. + +The conductor verified the read-role argv live on 2026-08-28 with the operator +host's `~/.local/bin/claude`: the default Fable model, `plan` permission mode, +text output, and the three canonical isolation flags exited 0 in 8 seconds with +the correct answer; the run record reported flags isolation and unchanged +operator home. The write role's `acceptEdits` path remains unverified live. + +TODO: +- de-vendor: import the real `ops/devlane/harness/isolation.py` instead of the copy + under `codex/vendor/`. +- proper mini-app wiring: CONTRACT, CI job, contract-tests. + +Full provenance/context: `.dev/xor/handoff/HANDOFF-fable-2026-08-27.md` diff --git a/ops/devlane/dispatch/levers/apply-push.sh b/ops/devlane/dispatch/levers/apply-push.sh new file mode 100755 index 0000000..74c84a1 --- /dev/null +++ b/ops/devlane/dispatch/levers/apply-push.sh @@ -0,0 +1,367 @@ +#!/usr/bin/env bash +# apply-push.sh — custody-preserving apply/push bridge for isolated write jobs. +# +# Accepts only patch bytes and a caller-supplied message from a harness job, +# validates the patch path boundary, re-applies it in a fresh detached worktree, +# authors a controlled commit, runs named gates, and pushes without importing +# the job's git objects, environment, home, configuration, or credentials. +# Every failed precondition is a REFUSAL (exit 64); operational gate/apply/push +# failures are also recorded and fail closed. Evidence is retained under the +# jobs root even though the disposable worktree is always removed. +# +# Usage: see usage() below. A repository remote named "origin" is required. + +set -u -o pipefail + +readonly PROGRAM=${0##*/} +LEVERS_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) || + { printf '%s: refused: cannot resolve script directory\n' "$PROGRAM" >&2; exit 64; } +readonly LEVERS_DIR +readonly JOBS_ROOT_DEFAULT="$LEVERS_DIR/jobs" +readonly BRIDGE_NAME="Apply Push Bridge" +readonly BRIDGE_EMAIL="noreply@apply-push-bridge" +readonly SAFE_PATH="/usr/local/bin:/usr/bin:/bin" + +refuse() { + failure=${failure:-$*} + printf '%s: REFUSED: %s\n' "$PROGRAM" "$*" >&2 + exit 64 +} + +usage() { + cat <<'EOF' +apply-push.sh --from-job DIR | --patch FILE --base SHA + --repo PATH --branch NAME + [--message-file PATH | --message TEXT] + [--gates "imports,lint,commit-trailers"] + [--gate-runner "python3 .dev/app/workflow/wf.py check --verify-only {gate}"] + --allow-paths GLOB [--allow-paths GLOB ...] + [--protected "main,dev"] [--force-with-lease] [--dry-run] + [--jobs-root DIR] +EOF +} + +from_job="" patch_file="" repo="" branch="" base="" +message_text="" message_file="" gates="" protected="main,dev" +gate_runner="python3 .dev/app/workflow/wf.py check --verify-only {gate}" +jobs_root="$JOBS_ROOT_DEFAULT" force_lease="no" dry_run="no" +allow_paths=() +original_argv=("$@") + +while (( $# )); do + case "$1" in + --from-job) from_job=${2:-}; shift 2 ;; + --patch) patch_file=${2:-}; shift 2 ;; + --repo) repo=${2:-}; shift 2 ;; + --branch) branch=${2:-}; shift 2 ;; + --base) base=${2:-}; shift 2 ;; + --message) message_text=${2:-}; shift 2 ;; + --message-file) message_file=${2:-}; shift 2 ;; + --gates) gates=${2:-}; shift 2 ;; + --gate-runner) gate_runner=${2:-}; shift 2 ;; + --allow-paths) allow_paths+=("${2:-}"); shift 2 ;; + --protected) protected=${2:-}; shift 2 ;; + --force-with-lease) force_lease="yes"; shift ;; + --dry-run) dry_run="yes"; shift ;; + --jobs-root) jobs_root=${2:-}; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) refuse "unknown argument '$1'" ;; + esac +done + +[[ -n "$repo" && -n "$branch" ]] || refuse "--repo and --branch are required" +[[ "$branch" != -* && "$branch" != *$'\n'* ]] || refuse "invalid branch name '$branch'" +(( ${#allow_paths[@]} )) || refuse "at least one --allow-paths glob is required; there is no allow-all default" +if [[ -n "$from_job" && -n "$patch_file" ]] || [[ -z "$from_job" && -z "$patch_file" ]]; then + refuse "give exactly one of --from-job DIR or --patch FILE" +fi +if [[ -n "$message_text" && -n "$message_file" ]]; then + refuse "give at most one of --message or --message-file" +fi +IFS=',' read -r -a denied <<<"$protected" +for item in "${denied[@]}"; do + [[ -z "$item" || "$branch" != "$item" ]] || refuse "branch '$branch' is protected" +done + +resolved_repo=$(realpath -e -- "$repo" 2>/dev/null) || refuse "--repo '$repo' does not resolve" +git -C "$resolved_repo" rev-parse --git-dir >/dev/null 2>&1 || refuse "'$resolved_repo' is not a git repository" +git -C "$resolved_repo" check-ref-format --branch "$branch" >/dev/null 2>&1 || refuse "invalid branch name '$branch'" +git -C "$resolved_repo" remote get-url origin >/dev/null 2>&1 || refuse "repository has no origin remote" +[[ -n "${HOME:-}" ]] || refuse "HOME is unset; operator git custody cannot be resolved" +readonly OPERATOR_HOME=$HOME + +mkdir -p -- "$jobs_root" || refuse "cannot create jobs root '$jobs_root'" +jobs_root=$(realpath -e -- "$jobs_root") || refuse "jobs root did not resolve" +job_id="$(date -u +%Y%m%dT%H%M%SZ)-apply-push-$(od -An -tx1 -N3 /dev/urandom | tr -d ' \n')" +record_dir="$jobs_root/$job_id" +mkdir -- "$record_dir" || refuse "cannot create evidence directory '$record_dir'" +patch_out="$record_dir/proposed.patch" +worktree="$record_dir/applied-worktree" +gate_results="$record_dir/gates.tsv" +: >"$gate_results" +started=$(date -u +%Y-%m-%dT%H:%M:%SZ) +source_ref="" patch_sha="" remote_start="" new_sha="" push_result="not-attempted" failure="" + +write_record() { + ended=$(date -u +%Y-%m-%dT%H:%M:%SZ) + argv_file="$record_dir/argv.nul" + printf '%s\0' "${original_argv[@]}" >"$argv_file" + python3 - "$record_dir/run-record.json" "$job_id" "$argv_file" "$source_ref" "$base" \ + "$patch_sha" "$gate_results" "$new_sha" "$push_result" "$started" "$ended" \ + "$dry_run" "$failure" <<'PY' +import json, pathlib, sys +(out, job, argvf, source, base, patch_sha, gatesf, new_sha, push, + started, ended, dry, failure) = sys.argv[1:] +raw = pathlib.Path(argvf).read_bytes().split(b"\0") +argv = [x.decode("utf-8", "surrogateescape") for x in raw if x] +results=[] +for line in pathlib.Path(gatesf).read_text(errors="replace").splitlines(): + name, rc, log = line.split("\t", 2) + results.append({"name": name, "exit_code": int(rc), "passed": rc == "0", "log": log}) +record={"job_id":job,"argv":argv,"source_job_ref":source or None,"base_sha":base or None, + "patch_sha256":patch_sha or None,"gates":results,"new_sha":new_sha or None, + "push_result":push,"dry_run":dry=="yes","failure":failure or None, + "stamp":{"started":started,"ended":ended}, + "artifacts":{"patch":"proposed.patch","applied_worktree_log":"applied-worktree.log"}} +pathlib.Path(out).write_text(json.dumps(record, indent=2)+"\n") +PY +} + +cleanup() { + rc=$? + if [[ -d "$worktree" ]]; then + git -C "$worktree" log -1 --stat --decorate --format=fuller >"$record_dir/applied-worktree.log" 2>&1 || true + git -C "$resolved_repo" worktree remove --force -- "$worktree" >/dev/null 2>&1 || true + fi + write_record 2>/dev/null || true + exit "$rc" +} +trap cleanup EXIT + +if [[ -n "$from_job" ]]; then + resolved_job=$(realpath -e -- "$from_job" 2>/dev/null) || refuse "--from-job '$from_job' does not resolve" + [[ -d "$resolved_job/wt" && -f "$resolved_job/run-record.json" ]] || refuse "source job lacks wt/ or run-record.json" + readarray -t job_info < <(python3 - "$resolved_job/run-record.json" <<'PY' +import json,re,sys +r=json.load(open(sys.argv[1], encoding="utf-8")) +if r.get("runtime",{}).get("role") != "write": raise SystemExit(1) +s=r.get("scope",{}).get("source","") +m=re.search(r"\b[0-9a-fA-F]{40,64}\b",s) +c=r.get("write_clone",{}).get("post_run_commit",{}).get("sha") or "" +if not m or not c: raise SystemExit(1) +print(m.group(0)); print(c) +PY + ) || refuse "source run record is not a completed write job with a recorded base" + (( ${#job_info[@]} == 2 )) || refuse "source run record did not yield exactly base and commit" + recorded_base=${job_info[0]}; source_commit=${job_info[1]} + [[ -z "$base" || "$base" == "$recorded_base" ]] || refuse "--base does not match source job's recorded base" + base=$recorded_base + git -C "$resolved_job/wt" cat-file -e "${base}^{commit}" 2>/dev/null || refuse "recorded base is absent from source wt" + [[ "$(git -C "$resolved_job/wt" rev-parse HEAD 2>/dev/null)" == "$source_commit" ]] || refuse "source wt HEAD no longer matches recorded lever commit" + git -C "$resolved_job/wt" diff --binary --full-index "$base" HEAD -- >"$patch_out" || refuse "cannot derive patch from source job" + source_ref="$resolved_job@$source_commit" +else + [[ -n "$base" ]] || refuse "--patch requires --base SHA" + resolved_patch=$(realpath -e -- "$patch_file" 2>/dev/null) || refuse "--patch '$patch_file' does not resolve" + [[ -f "$resolved_patch" && -s "$resolved_patch" ]] || refuse "patch is not a non-empty regular file" + cp -- "$resolved_patch" "$patch_out" || refuse "cannot retain patch bytes" + source_ref="patch:$resolved_patch" +fi +[[ -s "$patch_out" ]] || refuse "resolved patch is empty; no output is not success" +patch_sha=$(sha256sum -- "$patch_out" | awk '{print $1}') + +git -C "$resolved_repo" cat-file -e "${base}^{commit}" 2>/dev/null || refuse "base '$base' is not a commit in repo" +base=$(git -C "$resolved_repo" rev-parse "${base}^{commit}") +remote_start=$(env -i HOME="$OPERATOR_HOME" PATH="$SAFE_PATH" GIT_TERMINAL_PROMPT=0 \ + git -C "$resolved_repo" ls-remote --exit-code origin "refs/heads/$branch" 2>/dev/null | awk 'NR==1{print $1}') || + refuse "cannot observe origin branch '$branch'" +[[ -n "$remote_start" ]] || refuse "origin branch '$branch' has no tip" + +# Ask git which paths it would apply, then also retain both names from every +# diff header (needed for rename/copy sources). Keep producer statuses out of +# process substitutions: mapfile reports only its own status and would otherwise +# turn a parser failure into a partially populated, fail-open path list. +numstat="$record_dir/paths.numstat" +if ! git -C "$resolved_repo" apply --numstat -z "$patch_out" >"$numstat" 2>/dev/null; then + refuse "patch path metadata cannot be parsed by git" +fi +touched_file="$record_dir/paths.touched" +if ! python3 - "$patch_out" "$numstat" "$touched_file" <<'PY' +import sys +paths=[] +def add(raw): + if not raw: raise SystemExit(2) + try: path=raw.decode("utf-8") + except UnicodeDecodeError: raise SystemExit(2) + if "\n" in path or "\r" in path: raise SystemExit(2) + if path not in paths: paths.append(path) + +for raw in open(sys.argv[1], "rb"): + if raw.startswith(b"diff --git "): + p=raw.rstrip(b"\n").split(b" ") + if len(p)!=4 or not p[2].startswith(b"a/") or not p[3].startswith(b"b/"): + raise SystemExit(2) + add(p[2][2:]); add(p[3][2:]) + elif raw.startswith((b"rename from ", b"rename to ", b"copy from ", b"copy to ")): + name=raw.rstrip(b"\n").split(b" ", 2)[2] + if name.startswith(b'"'): raise SystemExit(2) + add(name) + elif raw.startswith((b"--- ", b"+++ ")): + name=raw.rstrip(b"\n")[4:] + if name == b"/dev/null": continue + if name.startswith((b"a/", b"b/")): name=name[2:] + if name.startswith(b'"') or b"\t" in name: raise SystemExit(2) + add(name) + +# --numstat -z is git apply's machine-readable view of the effective target, +# including /dev/null additions/deletions, mode-only, binary, and symlink diffs. +for record in open(sys.argv[2], "rb").read().split(b"\0"): + if not record: continue + fields=record.split(b"\t", 2) + if len(fields)!=3: raise SystemExit(2) + add(fields[2]) +if not paths: raise SystemExit(3) +with open(sys.argv[3], "w", encoding="utf-8") as out: + out.write("\n".join(paths)+"\n") +PY +then + refuse "patch has malformed, quoted, non-UTF-8, or missing diff headers" +fi +mapfile -t touched <"$touched_file" || refuse "cannot read validated patch paths" + +for path in "${touched[@]}"; do + [[ -n "$path" && "$path" != /* && "$path" != .git && "$path" != .git/* ]] || refuse "unsafe patch path '$path'" + IFS='/' read -r -a parts <<<"$path" + for part in "${parts[@]}"; do [[ "$part" != ".." && "$part" != ".git" ]] || refuse "unsafe patch path '$path'"; done + allowed="no" + for pattern in "${allow_paths[@]}"; do + [[ -n "$pattern" && "$pattern" != /* && "$pattern" != *".."* && "$pattern" != .git && "$pattern" != .git/* ]] || refuse "unsafe allow-paths pattern '$pattern'" + # shellcheck disable=SC2053 # intentional: --allow-paths is a glob set + if [[ "$path" == $pattern ]]; then allowed="yes"; break; fi + done + [[ "$allowed" == "yes" ]] || refuse "patch path '$path' is outside --allow-paths" +done + +git -C "$resolved_repo" worktree add --quiet --detach "$worktree" "$base" || refuse "cannot create fresh worktree at base" +if ! git -C "$worktree" apply --3way --index --whitespace=error-all "$patch_out" >"$record_dir/apply.log" 2>&1; then + refuse "patch does not apply cleanly (see apply.log)" +fi +git -C "$worktree" diff --cached --quiet && refuse "applied patch produced no change" + +# Content guard: a clean allow-paths list still lets an agent land bytes it +# should not. Refuse absolute home paths, credential-file additions, and +# runtime job-capture leaking into a landing commit — the kinds of thing an +# agent adds by accident, not intent. This is defence in depth behind the +# path allow-list, not a substitute for it. +guard_report="$record_dir/content-guard.log" +if ! git -C "$worktree" diff --cached --unified=0 -- >"$record_dir/staged.diff" 2>/dev/null; then + refuse "cannot read staged diff for content guard" +fi +python3 - "$record_dir/staged.diff" "$guard_report" <<'PY' +import re,sys +diff,report=sys.argv[1:] +added=[] # (path, lineno_in_added_hunk, text) +path=None +for raw in open(diff,encoding="utf-8",errors="replace"): + if raw.startswith("+++ b/"): + path=raw[6:].rstrip("\n") + elif raw.startswith("+") and not raw.startswith("+++"): + added.append((path, raw[1:].rstrip("\n"))) +hits=[] +# absolute POSIX home paths — machine-specific, private, never source truth +home=re.compile(r"/home/[A-Za-z0-9._-]+/") +# credential filenames introduced as content +cred=re.compile(r"\b(auth\.json|id_rsa|id_ed25519|\.pem|\.p12|credentials(\.json)?)\b") +for p,text in added: + pth=p or "(unknown)" + # runtime job capture path anywhere in an added line or as the file itself + if "levers/jobs/" in (pth+" "+text): + hits.append(f"{pth}: runtime job-capture content ('levers/jobs/')") + if home.search(text): + hits.append(f"{pth}: absolute home path in added content") + if cred.search(text) or cred.search(pth): + hits.append(f"{pth}: credential-file reference in added content") +seen=set(); uniq=[h for h in hits if not (h in seen or seen.add(h))] +open(report,"w",encoding="utf-8").write("\n".join(uniq)) +sys.exit(3 if uniq else 0) +PY +guard_rc=$? +if [[ $guard_rc -ne 0 ]]; then + first=$(head -1 "$guard_report" 2>/dev/null) + refuse "content guard: landing commit carries forbidden bytes — ${first:-see content-guard.log}; found in staged diff; needed a clean patch or an explicit operator waiver" +fi + +if [[ -n "$message_file" ]]; then + resolved_message=$(realpath -e -- "$message_file" 2>/dev/null) || refuse "message file does not resolve" + [[ -f "$resolved_message" && -s "$resolved_message" ]] || refuse "message file is empty or not regular" + cp -- "$resolved_message" "$record_dir/message.input" || refuse "cannot copy message" +else + [[ -n "${message_text//[$'\t\r\n ']/}" ]] || message_text="Apply isolated write job" + printf '%s\n' "$message_text" >"$record_dir/message.input" +fi +if ! python3 - "$record_dir/message.input" "$record_dir/message.final" "$job_id" "$patch_sha" <<'PY' +import re,sys +src,out,job,digest=sys.argv[1:] +s=open(src,encoding="utf-8").read().rstrip() +if "\x00" in s or not s.strip(): raise SystemExit(1) +trailer=re.compile(r"^[A-Za-z0-9][A-Za-z0-9-]*: .+$") +caller_last=re.split(r"\n[ \t]*\n",s)[-1].splitlines() +separator="\n" if caller_last and all(trailer.match(x) for x in caller_last) else "\n\n" +s += f"{separator}Apply-Push-Job: {job}\nPatch-SHA256: {digest}\n" +paras=re.split(r"\n[ \t]*\n",s.rstrip()) +last=paras[-1].splitlines() +if len(last)<2 or not all(trailer.match(x) for x in last): raise SystemExit(1) +open(out,"w",encoding="utf-8").write(s) +PY +then + refuse "landing message/trailer block is not a contiguous final paragraph" +fi + +if ! env -i HOME="$OPERATOR_HOME" PATH="$SAFE_PATH" LANG=C.UTF-8 LC_ALL=C.UTF-8 \ + git -C "$worktree" -c user.name="$BRIDGE_NAME" -c user.email="$BRIDGE_EMAIL" \ + commit --quiet --file "$record_dir/message.final" >"$record_dir/commit.log" 2>&1; then + refuse "bridge could not author landing commit" +fi +new_sha=$(git -C "$worktree" rev-parse HEAD) + +IFS=',' read -r -a gate_names <<<"$gates" +for gate in "${gate_names[@]}"; do + [[ -z "$gate" ]] && continue + [[ "$gate" =~ ^[A-Za-z0-9._-]+$ ]] || refuse "invalid gate name '$gate'" + command_text=${gate_runner//\{gate\}/$gate} + [[ "$command_text" != "$gate_runner" ]] || refuse "--gate-runner must contain {gate}" + log="gate-$gate.log" + # Gate commands are trusted repository policy, not job input. Execute in a + # fixed, empty environment; no job home/config/environment is consulted. + (cd -- "$worktree" && env -i HOME="$record_dir/gate-home" PATH="$SAFE_PATH" \ + LANG=C.UTF-8 LC_ALL=C.UTF-8 GIT_TERMINAL_PROMPT=0 /bin/sh -c "$command_text") \ + >"$record_dir/$log" 2>&1 + gate_rc=$? + printf '%s\t%s\t%s\n' "$gate" "$gate_rc" "$log" >>"$gate_results" + [[ $gate_rc -eq 0 ]] || refuse "gate '$gate' failed (see $log)" +done + +if git -C "$resolved_repo" merge-base --is-ancestor "$remote_start" "$new_sha" 2>/dev/null; then + push_mode="plain" +else + [[ "$force_lease" == "yes" ]] || refuse "landing is non-fast-forward; --force-with-lease is required" + push_mode="lease" +fi +if [[ "$dry_run" == "yes" ]]; then + push_result="dry-run:$push_mode" +else + if [[ "$push_mode" == "plain" ]]; then + env -i HOME="$OPERATOR_HOME" PATH="$SAFE_PATH" GIT_TERMINAL_PROMPT=0 \ + git -C "$worktree" push --porcelain origin "HEAD:refs/heads/$branch" >"$record_dir/push.log" 2>&1 + else + env -i HOME="$OPERATOR_HOME" PATH="$SAFE_PATH" GIT_TERMINAL_PROMPT=0 \ + git -C "$worktree" push --porcelain --force-with-lease="refs/heads/$branch:$remote_start" \ + origin "HEAD:refs/heads/$branch" >"$record_dir/push.log" 2>&1 + fi + push_rc=$? + [[ $push_rc -eq 0 ]] || { push_result="failed:$push_mode"; refuse "push failed or lease was stale (see push.log)"; } + push_result="pushed:$push_mode" +fi + +printf '%s: job=%s new=%s push=%s evidence=%s\n' "$PROGRAM" "$job_id" "$new_sha" "$push_result" "$record_dir" +exit 0 diff --git a/ops/devlane/dispatch/levers/claude/fable-dispatch.sh b/ops/devlane/dispatch/levers/claude/fable-dispatch.sh new file mode 100755 index 0000000..239198a --- /dev/null +++ b/ops/devlane/dispatch/levers/claude/fable-dispatch.sh @@ -0,0 +1,241 @@ +#!/usr/bin/env bash +# fable-dispatch.sh — custody-isolated headless Claude Code/Fable lever. +# +# Runs exactly one `claude -p` turn against an archived/copy snapshot. Read +# jobs see a read-only snapshot; write jobs edit a separate wt/ clone and this +# lever commits their changes after Claude exits. The resulting record exposes +# scope.source, runtime.role, and write_clone.post_run_commit.sha exactly as +# apply-push.sh --from-job consumes them. +# +# Claude's canonical isolation entry uses FLAGS, not a relocated home. Its +# auth_files list is empty: no credential is copied or linked from $HOME, and +# HOME stays the operator home so Claude's own authentication remains usable. +# The only operator setup suppressed is exactly what isolation.py records: +# --setting-sources project,local --strict-mcp-config +# --disable-slash-commands. No other operator-home path is hardcoded. +# +# --provision-levers makes /.levers/ contain grok-dispatch.sh, +# codex/ (including its vendor), and selftest.sh, allowing Fable to conduct +# Grok/Codex jobs. apply-push.sh is deliberately NEVER provisioned: a job may +# conduct, but cannot hold push custody. Landing remains conductor-side. +# +# The read-role argv was verified live by the conductor on the operator host +# (2026-08-28, ~/.local/bin/claude): claude -p --model claude-fable-5 +# --permission-mode plan --output-format text --setting-sources project,local +# --strict-mcp-config --disable-slash-commands. It exited 0 in 8s with the +# correct answer; the record reported flags isolation and an unchanged operator +# home. The write role's acceptEdits path has not yet been verified live. +# +# Usage: fable-dispatch.sh (--scope-ref REF --repo PATH | --scope-path DIR) +# (--prompt TEXT | --prompt-file PATH) +# [--model MODEL] [--role read|write] [--timeout SECONDS] +# [--jobs-root DIR] [--record-evidence] [--provision-levers] +# +# Job: //{snapshot/,wt/ (write),prompt.txt,stdout.log, +# stderr.log,argv.txt,run-record.json} + +set -u -o pipefail + +readonly PROGRAM=${0##*/} +LEVERS_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P) || + { printf '%s: REFUSED: cannot resolve levers directory\n' "$PROGRAM" >&2; exit 64; } +readonly LEVERS_DIR +readonly ISOLATION_PY="$LEVERS_DIR/../../harness/isolation.py" +readonly JOBS_ROOT_DEFAULT="$LEVERS_DIR/claude/jobs" + +refuse() { printf '%s: REFUSED: %s\n' "$PROGRAM" "$*" >&2; exit 64; } +usage() { + sed -n 's/^# Usage: //p; s/^# / /p' "${BASH_SOURCE[0]}" +} + +scope_ref=""; scope_repo=""; scope_path=""; prompt_text=""; prompt_file="" +model="claude-fable-5"; role="read"; wall_timeout="300" +jobs_root="$JOBS_ROOT_DEFAULT"; record_evidence="no"; provision_levers="no" +readonly AGENT_NAME="Claude Fable 5" +readonly AGENT_EMAIL="noreply@anthropic.com" +readonly AGENT_IDENTITY="$AGENT_NAME <$AGENT_EMAIL>" + +while (( $# )); do + case "$1" in + --scope-ref) scope_ref=${2:-}; shift 2 ;; + --repo) scope_repo=${2:-}; shift 2 ;; + --scope-path) scope_path=${2:-}; shift 2 ;; + --prompt) prompt_text=${2:-}; shift 2 ;; + --prompt-file) prompt_file=${2:-}; shift 2 ;; + --model) model=${2:-}; shift 2 ;; + --role) role=${2:-}; shift 2 ;; + --timeout) wall_timeout=${2:-}; shift 2 ;; + --jobs-root) jobs_root=${2:-}; shift 2 ;; + --record-evidence) record_evidence="yes"; shift ;; + --provision-levers) provision_levers="yes"; shift ;; + -h|--help) usage; exit 0 ;; + *) refuse "unknown argument '$1'" ;; + esac +done + +[[ -n "${HOME:-}" ]] || refuse "HOME is unset or empty; the operator/isolation boundary cannot be resolved" +readonly OPERATOR_HOME=$HOME +[[ -f "$ISOLATION_PY" && -r "$ISOLATION_PY" ]] || + refuse "canonical isolation law missing at '$ISOLATION_PY'; its claude auth_files declaration cannot be verified" + +if [[ -n "$scope_ref" && -n "$scope_path" ]]; then + refuse "give exactly one of --scope-ref or --scope-path, not both" +fi +if [[ -z "$scope_ref" && -z "$scope_path" ]]; then + refuse "no scope given: pass --scope-ref REF --repo PATH, or --scope-path DIR — there is no default scope, and the default is never the live repo or \$HOME" +fi +if [[ -n "$scope_ref" ]]; then + [[ -n "$scope_repo" ]] || refuse "--scope-ref requires --repo PATH naming the repository to archive it from" + resolved_repo=$(realpath -e -- "$scope_repo" 2>/dev/null) || refuse "--repo '$scope_repo' does not resolve to an existing path" + git -C "$resolved_repo" rev-parse --git-dir >/dev/null 2>&1 || refuse "'$resolved_repo' is not a git repository" + ref_sha=$(git -C "$resolved_repo" rev-parse --verify --end-of-options "${scope_ref}^{commit}" 2>&1) || + refuse "--scope-ref '$scope_ref' does not resolve to a commit in '$resolved_repo': $ref_sha" + SNAPSHOT_MODE="ref"; RESOLVED_REPO=$resolved_repo; REF_SHA=$ref_sha +else + resolved_scope=$(realpath -e -- "$scope_path" 2>/dev/null) || refuse "--scope-path '$scope_path' does not resolve to an existing path" + [[ -d "$resolved_scope" ]] || refuse "--scope-path '$resolved_scope' is not a directory" + resolved_home=$(realpath -e -- "$OPERATOR_HOME" 2>/dev/null) || resolved_home=$OPERATOR_HOME + [[ "$resolved_scope" != "$resolved_home" ]] || refuse "--scope-path resolves to the operator's \$HOME ($resolved_home); this is never permitted, no override exists" + [[ "$resolved_home" != "$resolved_scope"/* ]] || refuse "--scope-path '$resolved_scope' is an ancestor of the operator's \$HOME; this is never permitted" + for guard in .ssh .aws .gnupg .grok .claude .codex .config; do + [[ "$resolved_scope" != "$resolved_home/$guard" && "$resolved_scope" != "$resolved_home/$guard"/* ]] || + refuse "--scope-path '$resolved_scope' is inside the operator's '$guard' directory; refusing" + done + SNAPSHOT_MODE="path"; RESOLVED_SCOPE=$resolved_scope; REF_SHA="" +fi + +if [[ -n "$prompt_text" && -n "$prompt_file" ]]; then refuse "give exactly one of --prompt or --prompt-file, not both"; fi +if [[ -z "$prompt_text" && -z "$prompt_file" ]]; then refuse "no prompt given: pass --prompt TEXT or --prompt-file PATH"; fi +if [[ -n "$prompt_file" ]]; then + resolved_prompt_file=$(realpath -e -- "$prompt_file" 2>/dev/null) || refuse "--prompt-file '$prompt_file' does not resolve to an existing file" + [[ -f "$resolved_prompt_file" && -s "$resolved_prompt_file" ]] || refuse "--prompt-file '$resolved_prompt_file' is not a non-empty regular file" +fi +if [[ -n "$prompt_text" ]]; then + trimmed=${prompt_text//[$'\t\r\n ']/}; [[ -n "$trimmed" ]] || refuse "--prompt is empty or whitespace-only; refusing rather than dispatching an empty job" +fi +case "$role" in read|write) ;; *) refuse "--role '$role' is not one of read|write" ;; esac +[[ -n "$model" ]] || refuse "--model must not be empty" +[[ "$wall_timeout" =~ ^[0-9]+$ && "$wall_timeout" -gt 0 ]] || refuse "--timeout '$wall_timeout' must be a positive integer number of seconds" + +claude_bin=$(command -v claude 2>/dev/null || true) +[[ -n "$claude_bin" && -x "$claude_bin" ]] || refuse "no executable 'claude' binary found on PATH" +readonly CLAUDE_BIN=$claude_bin + +mkdir -p -- "$jobs_root" || refuse "cannot create jobs root '$jobs_root'" +jobs_root=$(realpath -e -- "$jobs_root") || refuse "jobs root '$jobs_root' did not resolve after creation" +job_id="$(date -u +%Y%m%dT%H%M%SZ)-fable-$(od -An -tx1 -N3 /dev/urandom | tr -d ' \n')" +job_dir="$jobs_root/$job_id" +mkdir -- "$job_dir" || refuse "job directory '$job_dir' already exists or could not be created" +mkdir -- "$job_dir/snapshot" || refuse "cannot create snapshot under '$job_dir'" + +if [[ "$SNAPSHOT_MODE" == "ref" ]]; then + if ! git -C "$RESOLVED_REPO" archive --format=tar "$REF_SHA" 2>"$job_dir/.archive.err" | tar -x -C "$job_dir/snapshot"; then + refuse "snapshot: git archive of $REF_SHA failed: $(<"$job_dir/.archive.err")" + fi + snapshot_source_desc="ref $scope_ref ($REF_SHA) archived from $RESOLVED_REPO" +else + if ! rsync -a --exclude='.git' -- "$RESOLVED_SCOPE"/ "$job_dir/snapshot"/ 2>"$job_dir/.rsync.err"; then + refuse "snapshot: copying '$RESOLVED_SCOPE' failed: $(<"$job_dir/.rsync.err")" + fi + snapshot_source_desc="path $RESOLVED_SCOPE (copied)" +fi +rm -f -- "$job_dir/.archive.err" "$job_dir/.rsync.err" +[[ -n "$(find "$job_dir/snapshot" -mindepth 1 -print -quit 2>/dev/null)" ]] || refuse "snapshot at '$job_dir/snapshot' is empty ($snapshot_source_desc); refusing" + +write_clone_desc=""; claude_cwd="$job_dir/snapshot" +if [[ "$role" == "write" ]]; then + if [[ "$SNAPSHOT_MODE" == "ref" ]]; then + git clone --no-hardlinks --quiet -- "$RESOLVED_REPO" "$job_dir/wt" 2>"$job_dir/.clone.err" || refuse "write side clone failed: $(<"$job_dir/.clone.err")" + git -C "$job_dir/wt" checkout --quiet --detach "$REF_SHA" 2>"$job_dir/.checkout.err" || refuse "write side clone checkout failed: $(<"$job_dir/.checkout.err")" + write_clone_desc="git clone of $RESOLVED_REPO at $REF_SHA" + else + mkdir -- "$job_dir/wt" || refuse "cannot create write side clone" + git -C "$job_dir/wt" init --quiet || refuse "write side clone git init failed" + rsync -a --exclude='.git' -- "$job_dir/snapshot"/ "$job_dir/wt"/ || refuse "write side clone seed failed" + git -C "$job_dir/wt" -c user.name="$AGENT_NAME" -c user.email="$AGENT_EMAIL" add -A || refuse "write baseline add failed" + git -C "$job_dir/wt" -c user.name="$AGENT_NAME" -c user.email="$AGENT_EMAIL" commit --quiet -m "snapshot baseline: $snapshot_source_desc" || refuse "write baseline commit failed" + REF_SHA=$(git -C "$job_dir/wt" rev-parse HEAD) + snapshot_source_desc="path $RESOLVED_SCOPE (copied; baseline $REF_SHA)" + write_clone_desc="git init + baseline commit at $REF_SHA, seeded from $RESOLVED_SCOPE" + fi + claude_cwd=$(realpath -e -- "$job_dir/wt") || refuse "write side clone did not resolve" + exclude_file=$(git -C "$job_dir/wt" rev-parse --git-path info/exclude 2>/dev/null) || + refuse "write side clone's per-clone exclude path could not be resolved" + [[ "$exclude_file" == /* ]] || exclude_file="$job_dir/wt/$exclude_file" + printf '/.levers/\n' >>"$exclude_file" || + refuse "cannot exclude provisioned .levers from the write side clone" +fi +chmod -R a-w -- "$job_dir/snapshot" || refuse "cannot make base snapshot read-only" + +if [[ -n "$prompt_file" ]]; then cp -- "$resolved_prompt_file" "$job_dir/prompt.txt" || refuse "cannot copy prompt"; else printf '%s\n' "$prompt_text" >"$job_dir/prompt.txt" || refuse "cannot write prompt"; fi +[[ -s "$job_dir/prompt.txt" ]] || refuse "rendered prompt.txt is empty" + +# Invoke the canonical module and verify both status and non-empty output before +# evaluating it. For Claude, ISO_ENV is empty and ISO_FLAGS is the mechanism. +iso_output=$(python3 "$ISOLATION_PY" --sh claude "$job_dir/unused-home" 2>"$job_dir/.isolation.err"); iso_rc=$? +[[ $iso_rc -eq 0 ]] || refuse "isolation.py could not isolate claude (exit $iso_rc): $(<"$job_dir/.isolation.err")" +[[ -n "$iso_output" ]] || refuse "isolation.py produced no output for claude; refusing the eval-empty shape" +eval "$iso_output" +[[ -z "${ISO_ENV:-}" ]] || refuse "claude isolation unexpectedly requested environment overrides '$ISO_ENV'" +[[ "${ISO_FLAGS:-}" == "--setting-sources project,local --strict-mcp-config --disable-slash-commands" ]] || refuse "claude isolation flags differ from the canonical expected boundary: '${ISO_FLAGS:-}'" +iso_extra_flags=(); read -r -a iso_extra_flags <<<"$ISO_FLAGS" + +if [[ "$provision_levers" == "yes" ]]; then + mkdir -- "$claude_cwd/.levers" || refuse "cannot create provisioned .levers directory" + cp -- "$LEVERS_DIR/grok-dispatch.sh" "$LEVERS_DIR/selftest.sh" "$claude_cwd/.levers/" || refuse "cannot provision sibling levers" + cp -R -- "$LEVERS_DIR/codex" "$claude_cwd/.levers/codex" || refuse "cannot provision codex lever and vendor" + [[ ! -e "$claude_cwd/.levers/apply-push.sh" ]] || refuse "apply-push.sh entered provisioned custody unexpectedly" +fi + +permission_mode="plan"; [[ "$role" == "write" ]] && permission_mode="acceptEdits" +child_env=("HOME=$OPERATOR_HOME" "PATH=/usr/bin:/bin:/usr/local/bin" "TERM=dumb" "NO_COLOR=1" "LANG=C.UTF-8" "LC_ALL=C.UTF-8" "GIT_TERMINAL_PROMPT=0") +[[ "$record_evidence" == "yes" ]] && child_env+=("WF_AGENT=$AGENT_IDENTITY") +claude_argv=("$CLAUDE_BIN" -p --model "$model" --permission-mode "$permission_mode" --output-format text "${iso_extra_flags[@]}") +started=$(date -u +%Y-%m-%dT%H:%M:%SZ); start_epoch=$(date -u +%s) +(cd -- "$claude_cwd" && exec env -i "${child_env[@]}" timeout --signal=KILL "${wall_timeout}s" "${claude_argv[@]}" <"$job_dir/prompt.txt") >"$job_dir/stdout.log" 2>"$job_dir/stderr.log" +exit_code=$?; ended=$(date -u +%Y-%m-%dT%H:%M:%SZ); end_epoch=$(date -u +%s) +printf '%s\n' "${claude_argv[@]}" >"$job_dir/argv.txt" + +commit_attempted="no"; commit_sha=""; commit_message=""; commit_changed=""; commit_note="" +if [[ "$role" == "write" ]]; then + commit_attempted="yes" + if ! git -C "$job_dir/wt" add -A 2>"$job_dir/.postadd.err"; then commit_note="git add failed: $(<"$job_dir/.postadd.err")" + elif git -C "$job_dir/wt" diff --cached --quiet; then commit_note="fable made no file changes; nothing to commit" + else + commit_changed=$(git -C "$job_dir/wt" diff --cached --name-only | tr '\n' ' ') + commit_message="fable write job $job_id" + if git -C "$job_dir/wt" -c user.name="$AGENT_NAME" -c user.email="$AGENT_EMAIL" commit --quiet -m "$commit_message" 2>"$job_dir/.postcommit.err"; then + commit_sha=$(git -C "$job_dir/wt" rev-parse HEAD); commit_note="committed by lever after Fable turn" + else commit_note="commit failed: $(<"$job_dir/.postcommit.err")"; fi + fi +fi + +python3 - "$job_dir" "$job_id" "$SNAPSHOT_MODE" "$snapshot_source_desc" "$role" "$model" "$permission_mode" "$wall_timeout" "$started" "$ended" "$start_epoch" "$end_epoch" "$exit_code" "$record_evidence" "$write_clone_desc" "$claude_cwd" "${ISO_STORE:-}" "$commit_attempted" "$commit_sha" "$commit_message" "$commit_changed" "$commit_note" "$provision_levers" <<'PY' +import json, os, sys +(job, jid, mode, source, role, model, permission, timeout, started, ended, + start_epoch, end_epoch, rc, evidence, clone_desc, cwd, store, attempted, + sha, message, changed, note, provisioned) = sys.argv[1:24] +snap = os.path.join(job, "snapshot") +record = { + "job_id": jid, "harness": "claude", "scope": {"mode": mode, "source": source}, + "prompt_file": "prompt.txt", + "runtime": {"model": model, "role": role, "permission_mode": permission, + "wall_timeout_s": int(timeout), "cwd": cwd}, + "isolation": {"mechanism": "flags", "law_source": ".dev/app/harness/isolation.py", + "flags": ["--setting-sources", "project,local", "--strict-mcp-config", "--disable-slash-commands"], + "auth_files": [], "operator_home_unchanged": True, "session_store": store, + "env_allowlisted": True, "record_evidence": evidence == "yes"}, + "snapshot_proof": {"top_level_entries": sorted(os.listdir(snap)), + "file_count": sum(len(f) for _,_,f in os.walk(snap)), "read_only_on_disk": True}, + "write_clone": ({"path": os.path.join(job,"wt"), "built_by": clone_desc, + "post_run_commit": {"attempted": attempted == "yes", "sha": sha or None, + "message": message or None, "changed_files": changed.split() if changed else [], "note": note or None}} + if clone_desc else None), + "provisioned_levers": provisioned == "yes", + "stamp": {"started": started, "ended": ended, "wall_seconds": int(end_epoch)-int(start_epoch)}, + "exit_code": int(rc), "artifacts": {"stdout":"stdout.log","stderr":"stderr.log","argv":"argv.txt"}} +with open(os.path.join(job,"run-record.json"),"w",encoding="utf-8") as fh: + json.dump(record,fh,indent=2); fh.write("\n") +PY +printf '%s: job=%s exit=%s dir=%s\n' "$PROGRAM" "$job_id" "$exit_code" "$job_dir" +exit "$exit_code" diff --git a/ops/devlane/dispatch/levers/claude/selftest.sh b/ops/devlane/dispatch/levers/claude/selftest.sh new file mode 100755 index 0000000..3debe2e --- /dev/null +++ b/ops/devlane/dispatch/levers/claude/selftest.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Offline refusal and write-commit tests. A fake claude marks invocation; each +# refusal must precede it, while the positive case uses it to make one change. +set -u -o pipefail +HERE=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) || exit 1 +readonly LEVER="$HERE/fable-dispatch.sh" +tmp=$(mktemp -d) || exit 1 +cleanup() { chmod -R u+w -- "$tmp" 2>/dev/null || true; rm -rf -- "$tmp"; } +trap cleanup EXIT +real_home=${HOME:-}; export HOME="$tmp/home"; mkdir -p "$HOME" "$tmp/bin" "$tmp/scope" +printf 'scope\n' >"$tmp/scope/file" +marker="$tmp/model-invoked" +printf '#!/bin/sh\n: >"%s"\nexit 99\n' "$marker" >"$tmp/bin/claude"; chmod +x "$tmp/bin/claude" +export PATH="$tmp/bin:/usr/bin:/bin" +passed=0; total=0 +check_refusal() { + name=$1; shift; total=$((total+1)); rm -f -- "$marker" + "$@" >"$tmp/out" 2>"$tmp/err"; rc=$? + if [[ $rc -eq 64 && ! -e "$marker" ]] && grep -q 'REFUSED:' "$tmp/err"; then + passed=$((passed+1)); printf 'PASS: %s\n' "$name" + else printf 'FAIL: %s (rc=%s, stderr=%s)\n' "$name" "$rc" "$(<"$tmp/err")"; fi +} +check_refusal "no scope" "$LEVER" --prompt x +check_refusal "scope resolving to HOME" "$LEVER" --scope-path "$HOME" --prompt x +check_refusal "empty prompt" "$LEVER" --scope-path "$tmp/scope" --prompt ' ' +mkdir -p "$tmp/orphan"; cp -- "$LEVER" "$tmp/orphan/fable-dispatch.sh"; chmod +x "$tmp/orphan/fable-dispatch.sh" +check_refusal "missing auth file declaration (canonical law absent)" "$tmp/orphan/fable-dispatch.sh" --scope-path "$tmp/scope" --prompt x + +total=$((total+1)); rm -f -- "$marker" +printf '#!/bin/sh\nprintf "model change\\n" > model-change.txt\n: >"%s"\nexit 0\n' "$marker" >"$tmp/bin/claude" +jobs_root="$tmp/jobs" +"$LEVER" --scope-path "$tmp/scope" --prompt x --role write \ + --provision-levers --jobs-root "$jobs_root" >"$tmp/out" 2>"$tmp/err"; rc=$? +job_dir=$(find "$jobs_root" -mindepth 1 -maxdepth 1 -type d -print -quit 2>/dev/null) +if [[ $rc -eq 0 && -e "$marker" && -n "$job_dir" ]] && + git -C "$job_dir/wt" diff-tree --no-commit-id --name-only -r HEAD | grep -qx 'model-change.txt' && + ! git -C "$job_dir/wt" diff-tree --no-commit-id --name-only -r HEAD | grep -q '^\.levers/'; then + passed=$((passed+1)); printf 'PASS: provisioned write commit excludes .levers\n' +else + printf 'FAIL: provisioned write commit excludes .levers (rc=%s, stderr=%s)\n' "$rc" "$(<"$tmp/err")" +fi + +printf 'SELFTEST: %s/%s passed; fake model invocations: 1\n' "$passed" "$total" +[[ "$passed" -eq "$total" ]] +export HOME=$real_home diff --git a/ops/devlane/dispatch/levers/codex/codex-dispatch.sh b/ops/devlane/dispatch/levers/codex/codex-dispatch.sh new file mode 100755 index 0000000..b28b873 --- /dev/null +++ b/ops/devlane/dispatch/levers/codex/codex-dispatch.sh @@ -0,0 +1,554 @@ +#!/usr/bin/env bash +# codex-dispatch.sh — custody-isolated headless Codex dispatch lever. +# +# Launches ONE headless, single-turn `codex exec` job against an isolated +# snapshot (a git ref archived read-only out of a repo) or an explicit path +# subtree copied into a per-job directory. Never runs against the +# operator's $HOME, never inherits the operator's shell environment or +# ~/.codex config (hooks.json, config.toml, sessions, ...), never touches +# the live repository working tree. +# +# Shape matches the sibling grok-dispatch.sh lever (../grok-dispatch.sh) +# so both can later be generalized into one harness-config primitive: a +# scope, a prompt, an optional model, and a role (read vs write). +# +# Every precondition below is an explicit check. If isolation cannot be +# established, this script REFUSES (exit 64) before Codex is ever +# invoked — it does not fall back to an unisolated run, and it does not +# treat "no output" as success. +# +# Real flags confirmed against `codex --version` (0.148.0) and +# `codex exec --help` run on this host, and cross-checked against the +# ADAPTERS table in origin/repo/lane-go:.dev/app/task/run.py and the +# harness/controls fixtures on that branch (dispatchable-flags.json, +# dispatchable-home.json): +# - headless entrypoint: `codex exec [OPTIONS] [PROMPT]`; PROMPT "-" (or +# omitted) reads the prompt from stdin. +# - sandbox: `-s/--sandbox read-only|workspace-write|danger-full-access`. +# This lever only ever uses read-only or workspace-write. +# - approval: `-a/--ask-for-approval` exists on the top-level interactive +# `codex` command, but is ABSENT from `codex exec --help` on this +# host (0.148.0) — confirmed by running it, not assumed from the +# parent command's help. `codex exec` is unconditionally +# non-interactive: `--sandbox` alone gates what the model may do, and +# a command it forbids is returned to the model as an execution +# failure rather than a hang waiting for approval. This matches the +# lane-go ADAPTERS table, whose codex argv is `exec --sandbox +# {sandbox} -` with no approval flag at all. This lever therefore +# does not expose one either. +# - working root: `-C/--cd DIR`. +# - isolation is NOT a flag on this harness — Codex discovers its config, +# auth, hooks and sessions store from $CODEX_HOME (default ~/.codex). +# The lane-go fixture recorded the failure mode directly: an +# unisolated dispatch loaded ~/.codex/hooks.json and ran a SessionStart +# hook. So isolation here means a from-scratch CODEX_HOME containing +# ONLY a symlinked auth.json, never the operator's real one. +# - there is no `--effort`/reasoning-effort flag on this harness (only +# the generic `-c key=value`, whose reasoning key lane-go's authors +# declined to guess at). This lever does not expose one either; do not +# add a silently-dropped dial. +# - `--skip-git-repo-check` lets exec run in a directory with no `.git` +# (true for a `git archive`/rsync snapshot) without Codex complaining. +# +# KNOWN CUSTODY QUIRK (why role=write forks the filesystem in two, and +# why Codex never runs `git commit` itself): +# Codex's sandbox is enforced by mediating writes under the cwd/add-dir +# it is told about; a *read* job's snapshot is additionally chmod'd +# a-w on disk after it's built, so even a workspace-write sandbox +# mistake can't touch it. That means a write job cannot commit "in +# place" in that snapshot. So role=write builds a SECOND, independent, +# writable side clone (`wt/`) — a real `git clone` of the source repo +# at the resolved ref (or a fresh `git init` + baseline commit, for a +# --scope-path source) — and Codex is pointed at wt/, never snapshot/, +# for the write role. snapshot/ stays read-only reference material. +# +# CONFIRMED BY RUNNING IT (codex-cli 0.148.0): even inside wt/, with +# --sandbox workspace-write and a fully-writable .git on disk, Codex's +# OWN `git commit` fails sandboxed: `fatal: Unable to create +# '.git/index.lock': Read-only file system`. Ordinary file writes in +# the same directory succeed — only `.git/` is denied. So this lever +# never asks Codex to commit. Codex edits files in wt/; once its turn +# ends, THIS SCRIPT stages and commits whatever changed, from its own +# unsandboxed shell. "Commit in a side clone" is mechanized as a +# deterministic post-run step this script controls, not a git +# invocation trusted to model output. +# +# Usage: +# codex-dispatch.sh --scope-ref REF --repo PATH (snapshot of repo at ref) +# codex-dispatch.sh --scope-path DIR (explicit subtree, copied) +# ... plus one of: +# --prompt TEXT | --prompt-file PATH +# ... and optionally: +# --model MODEL --role read|write [default: read] +# --sandbox read-only|workspace-write (default: derived from --role) +# --timeout SECONDS --jobs-root DIR --record-evidence +# +# Job output lands in: // +# snapshot/ the isolated, read-only copy Codex saw for role=read; +# reference-only baseline for role=write +# wt/ (role=write only) the writable side clone Codex +# actually ran in and could commit into +# prompt.txt the exact prompt bytes handed to Codex +# home/ isolated CODEX_HOME=HOME, containing only auth.json +# stdout.log Codex's stdout +# stderr.log Codex's stderr +# last-message.txt Codex's final agent message (--output-last-message) +# argv.txt the exact argv invoked +# run-record.json job id, argv, isolation proof, timestamps, exit code + +set -u -o pipefail + +readonly PROGRAM=${0##*/} +LEVERS_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) || + { printf '%s: refused: cannot resolve script directory\n' "$PROGRAM" >&2; exit 64; } +readonly LEVERS_DIR +readonly JOBS_ROOT_DEFAULT="$LEVERS_DIR/jobs" + +# The canonical isolation law, vendored (not imported live from the repo +# working tree — this lever must not touch that tree, and must not +# depend on whatever happens to be checked out there). Content is +# `.dev/app/harness/isolation.py` as of origin/dev; it is the owner's +# source of truth for what a harness may bring from the operator's +# machine, and it is invoked, not re-derived, so this lever cannot drift +# from it silently. A missing/unreadable vendor file is refused here, +# up front, rather than producing the empty-`eval` shape further down. +readonly ISOLATION_PY="$LEVERS_DIR/vendor/isolation.py" +[[ -f "$ISOLATION_PY" && -r "$ISOLATION_PY" ]] || + { printf '%s: REFUSED: vendored isolation module missing at %s; cannot establish isolation, refusing to dispatch\n' "$PROGRAM" "$ISOLATION_PY" >&2; exit 64; } + +refuse() { + printf '%s: REFUSED: %s\n' "$PROGRAM" "$*" >&2 + exit 64 +} + +usage() { + cat <<'EOF' +codex-dispatch.sh --scope-ref REF --repo PATH | --scope-path DIR + (--prompt TEXT | --prompt-file PATH) + [--model MODEL] [--role read|write] + [--sandbox read-only|workspace-write] + [--timeout SECONDS] [--jobs-root DIR] [--record-evidence] +EOF +} + +# ---- defaults ----------------------------------------------------------- +scope_ref="" +scope_repo="" +scope_path="" +prompt_text="" +prompt_file="" +model="" +role="read" +sandbox_mode="" +wall_timeout="300" +jobs_root="$JOBS_ROOT_DEFAULT" +record_evidence="no" +readonly AGENT_IDENTITY="Codex Dispatcher " + +# ---- args ----------------------------------------------------------------- +while (( $# )); do + case "$1" in + --scope-ref) scope_ref=${2:-}; shift 2 ;; + --repo) scope_repo=${2:-}; shift 2 ;; + --scope-path) scope_path=${2:-}; shift 2 ;; + --prompt) prompt_text=${2:-}; shift 2 ;; + --prompt-file) prompt_file=${2:-}; shift 2 ;; + --model) model=${2:-}; shift 2 ;; + --role) role=${2:-}; shift 2 ;; + --sandbox) sandbox_mode=${2:-}; shift 2 ;; + --timeout) wall_timeout=${2:-}; shift 2 ;; + --jobs-root) jobs_root=${2:-}; shift 2 ;; + --record-evidence) record_evidence="yes"; shift ;; + -h|--help) usage; exit 0 ;; + *) refuse "unknown argument '$1'" ;; + esac +done + +# ---- preconditions: scope ------------------------------------------------- +[[ -n "${HOME:-}" ]] || + refuse "HOME is unset or empty; the operator/isolation boundary cannot be resolved" +readonly OPERATOR_HOME=$HOME + +if [[ -n "$scope_ref" && -n "$scope_path" ]]; then + refuse "give exactly one of --scope-ref or --scope-path, not both" +fi +if [[ -z "$scope_ref" && -z "$scope_path" ]]; then + refuse "no scope given: pass --scope-ref REF --repo PATH, or --scope-path DIR — there is no default scope, and the default is never the live repo or \$HOME" +fi + +if [[ -n "$scope_ref" ]]; then + [[ -n "$scope_repo" ]] || + refuse "--scope-ref requires --repo PATH naming the repository to archive it from" + resolved_repo=$(realpath -e -- "$scope_repo" 2>/dev/null) || + refuse "--repo '$scope_repo' does not resolve to an existing path" + git -C "$resolved_repo" rev-parse --git-dir >/dev/null 2>&1 || + refuse "'$resolved_repo' is not a git repository" + ref_sha=$(git -C "$resolved_repo" rev-parse --verify --end-of-options \ + "${scope_ref}^{commit}" 2>&1) || + refuse "--scope-ref '$scope_ref' does not resolve to a commit in '$resolved_repo': $ref_sha" + readonly SNAPSHOT_MODE="ref" + readonly RESOLVED_REPO=$resolved_repo + readonly REF_SHA=$ref_sha +else + resolved_scope=$(realpath -e -- "$scope_path" 2>/dev/null) || + refuse "--scope-path '$scope_path' does not resolve to an existing path" + [[ -d "$resolved_scope" ]] || + refuse "--scope-path '$resolved_scope' is not a directory" + resolved_home=$(realpath -e -- "$OPERATOR_HOME" 2>/dev/null) || resolved_home=$OPERATOR_HOME + if [[ "$resolved_scope" == "$resolved_home" ]]; then + refuse "--scope-path resolves to the operator's \$HOME ($resolved_home); this is never permitted, no override exists" + fi + if [[ "$resolved_home" == "$resolved_scope"/* ]]; then + refuse "--scope-path '$resolved_scope' is an ancestor of the operator's \$HOME; this is never permitted" + fi + for guard in .ssh .aws .gnupg .grok .claude .codex .config; do + if [[ "$resolved_scope" == "$resolved_home/$guard" || "$resolved_scope" == "$resolved_home/$guard"/* ]]; then + refuse "--scope-path '$resolved_scope' is inside the operator's '$guard' directory; refusing" + fi + done + readonly SNAPSHOT_MODE="path" + readonly RESOLVED_SCOPE=$resolved_scope +fi + +# ---- preconditions: prompt -------------------------------------------- +if [[ -n "$prompt_text" && -n "$prompt_file" ]]; then + refuse "give exactly one of --prompt or --prompt-file, not both" +fi +if [[ -z "$prompt_text" && -z "$prompt_file" ]]; then + refuse "no prompt given: pass --prompt TEXT or --prompt-file PATH" +fi +if [[ -n "$prompt_file" ]]; then + resolved_prompt_file=$(realpath -e -- "$prompt_file" 2>/dev/null) || + refuse "--prompt-file '$prompt_file' does not resolve to an existing file" + [[ -f "$resolved_prompt_file" && -s "$resolved_prompt_file" ]] || + refuse "--prompt-file '$resolved_prompt_file' is not a non-empty regular file" +fi +if [[ -n "$prompt_text" ]]; then + # A prompt that is only whitespace is the "eval \"\"" shape: it would + # exit clean having asked Codex nothing. Refuse it explicitly. + trimmed=${prompt_text//[$'\t\r\n ']/} + [[ -n "$trimmed" ]] || + refuse "--prompt is empty or whitespace-only; refusing rather than dispatching an empty job" +fi + +# ---- preconditions: role / sandbox / approval dials ----------------------- +case "$role" in + read|write) ;; + *) refuse "--role '$role' is not one of read|write" ;; +esac +if [[ -z "$sandbox_mode" ]]; then + if [[ "$role" == "write" ]]; then sandbox_mode="workspace-write"; else sandbox_mode="read-only"; fi +fi +case "$sandbox_mode" in + read-only|workspace-write) ;; + danger-full-access) refuse "--sandbox danger-full-access is never permitted by this lever" ;; + *) refuse "--sandbox '$sandbox_mode' is not one of read-only|workspace-write" ;; +esac +if [[ "$role" == "read" && "$sandbox_mode" == "workspace-write" ]]; then + refuse "--role read with --sandbox workspace-write makes no sense: a read job has no writable side clone to write into" +fi +[[ "$wall_timeout" =~ ^[0-9]+$ && "$wall_timeout" -gt 0 ]] || + refuse "--timeout '$wall_timeout' must be a positive integer number of seconds" + +# ---- preconditions: the codex binary and its credential ------------------- +codex_bin=$(command -v codex 2>/dev/null || true) +if [[ -z "$codex_bin" ]]; then + for candidate in "$OPERATOR_HOME/.local/bin/codex" /home/work/.local/bin/codex; do + if [[ -x "$candidate" ]]; then codex_bin=$candidate; break; fi + done +fi +[[ -n "$codex_bin" && -x "$codex_bin" ]] || + refuse "no executable 'codex' binary found on PATH or at the known install location" +readonly CODEX_BIN=$codex_bin + +source_codex_home=${CODEX_HOME:-$OPERATOR_HOME/.codex} +source_auth="$source_codex_home/auth.json" +[[ -f "$source_auth" && -r "$source_auth" ]] || + refuse "codex credential '$source_auth' is absent or unreadable; refusing an unisolated fallback (this is the BLOCKER case: fix auth, do not disable isolation)" +resolved_auth=$(realpath -e -- "$source_auth" 2>/dev/null) || + refuse "codex credential '$source_auth' cannot be resolved" +readonly RESOLVED_AUTH=$resolved_auth + +# ---- job directory ------------------------------------------------------ +mkdir -p -- "$jobs_root" || refuse "cannot create jobs root '$jobs_root'" +jobs_root=$(realpath -e -- "$jobs_root") || refuse "jobs root '$jobs_root' did not resolve after creation" +job_id="$(date -u +%Y%m%dT%H%M%SZ)-codex-$(od -An -tx1 -N3 /dev/urandom | tr -d ' \n')" +job_dir="$jobs_root/$job_id" +mkdir -- "$job_dir" || refuse "job directory '$job_dir' already exists or could not be created" +mkdir -- "$job_dir/snapshot" "$job_dir/home" || refuse "cannot create job subdirectories under '$job_dir'" +chmod 700 -- "$job_dir/home" || refuse "cannot protect isolated home '$job_dir/home'" + +# ---- build the base snapshot (read-only from the source; never mutates it) - +if [[ "$SNAPSHOT_MODE" == "ref" ]]; then + if ! git -C "$RESOLVED_REPO" archive --format=tar "$REF_SHA" 2>"$job_dir/.archive.err" \ + | tar -x -C "$job_dir/snapshot"; then + refuse "snapshot: 'git archive' of $REF_SHA from '$RESOLVED_REPO' failed: $(cat -- "$job_dir/.archive.err" 2>/dev/null)" + fi + rm -f -- "$job_dir/.archive.err" + snapshot_source_desc="ref $scope_ref ($REF_SHA) archived from $RESOLVED_REPO" +else + if ! rsync -a --exclude='.git' -- "$RESOLVED_SCOPE"/ "$job_dir/snapshot"/ 2>"$job_dir/.rsync.err"; then + refuse "snapshot: copying '$RESOLVED_SCOPE' failed: $(cat -- "$job_dir/.rsync.err" 2>/dev/null)" + fi + rm -f -- "$job_dir/.rsync.err" + snapshot_source_desc="path $RESOLVED_SCOPE (copied)" +fi + +# FAIL CLOSED: an empty snapshot is exactly the "eval \"\"" shape — it +# would let Codex run, see nothing worth objecting to, exit 0, and be +# reported as a clean job. Refuse before Codex is ever launched. +if [[ -z "$(find "$job_dir/snapshot" -mindepth 1 -print -quit 2>/dev/null)" ]]; then + refuse "snapshot at '$job_dir/snapshot' is empty ($snapshot_source_desc); refusing to dispatch against an empty scope" +fi + +# ---- role=write: build the writable side clone BEFORE snapshot goes ------ +# ---- read-only, so it never inherits a locked-down permission bit. ----- +write_clone_desc="" +codex_cwd="$job_dir/snapshot" +if [[ "$role" == "write" ]]; then + if [[ "$SNAPSHOT_MODE" == "ref" ]]; then + if ! git clone --no-hardlinks --quiet -- "$RESOLVED_REPO" "$job_dir/wt" 2>"$job_dir/.clone.err"; then + refuse "write side clone: 'git clone' of '$RESOLVED_REPO' failed: $(cat -- "$job_dir/.clone.err" 2>/dev/null)" + fi + if ! git -C "$job_dir/wt" checkout --quiet --detach "$REF_SHA" 2>"$job_dir/.checkout.err"; then + refuse "write side clone: checkout of $REF_SHA in '$job_dir/wt' failed: $(cat -- "$job_dir/.checkout.err" 2>/dev/null)" + fi + rm -f -- "$job_dir/.clone.err" "$job_dir/.checkout.err" + write_clone_desc="git clone of $RESOLVED_REPO at $REF_SHA" + else + mkdir -- "$job_dir/wt" || refuse "cannot create write side clone directory '$job_dir/wt'" + if ! git -C "$job_dir/wt" init --quiet 2>"$job_dir/.init.err"; then + refuse "write side clone: 'git init' in '$job_dir/wt' failed: $(cat -- "$job_dir/.init.err" 2>/dev/null)" + fi + rm -f -- "$job_dir/.init.err" + if ! rsync -a --exclude='.git' -- "$job_dir/snapshot"/ "$job_dir/wt"/ 2>"$job_dir/.wtrsync.err"; then + refuse "write side clone: seeding '$job_dir/wt' from snapshot failed: $(cat -- "$job_dir/.wtrsync.err" 2>/dev/null)" + fi + rm -f -- "$job_dir/.wtrsync.err" + if ! git -C "$job_dir/wt" -c user.name="$AGENT_IDENTITY" -c user.email="noreply@codex-lever" \ + add -A 2>"$job_dir/.add.err" || + ! git -C "$job_dir/wt" -c user.name="$AGENT_IDENTITY" -c user.email="noreply@codex-lever" \ + commit --quiet -m "snapshot baseline: $snapshot_source_desc" 2>>"$job_dir/.add.err"; then + refuse "write side clone: baseline commit in '$job_dir/wt' failed: $(cat -- "$job_dir/.add.err" 2>/dev/null)" + fi + rm -f -- "$job_dir/.add.err" + write_clone_desc="git init + baseline commit, seeded from $snapshot_source_desc" + fi + # Isolation proof: the side clone must itself never be (or contain) + # the operator's HOME. + resolved_wt=$(realpath -e -- "$job_dir/wt") || refuse "write side clone '$job_dir/wt' did not resolve" + codex_cwd="$resolved_wt" +fi + +# Base snapshot is reference-only from here on: lock it down on disk so +# even a sandbox misconfiguration can't turn it into a write target. +chmod -R a-w -- "$job_dir/snapshot" || + refuse "cannot make base snapshot '$job_dir/snapshot' read-only" + +# ---- write the prompt, verbatim, inside the job directory -------------- +if [[ -n "$prompt_file" ]]; then + cp -- "$resolved_prompt_file" "$job_dir/prompt.txt" || + refuse "cannot copy prompt file into job directory" +else + printf '%s\n' "$prompt_text" > "$job_dir/prompt.txt" || + refuse "cannot write prompt into job directory" +fi +[[ -s "$job_dir/prompt.txt" ]] || + refuse "rendered prompt.txt is empty after being written; refusing to dispatch" + +# ---- build the isolated CODEX_HOME via the CANONICAL isolation law ------- +# `isolation.py --sh codex ` builds the minimal home (auth.json +# symlinked in, nothing else — codex's `auth_files` is exactly that one +# entry) and prints shell assignments for the env overrides its own +# HARNESSES table says codex needs. This is the exact call context.cue's +# incident describes: a launcher that does `eval "$(isolation.py ...)"` +# and never checks whether the command behind it actually ran. When the +# module was absent, that produced NO output, `eval ""` succeeded, and a +# dispatch went out fully unisolated with an exit code of 0. So here the +# command's own exit status AND its output are checked as two separate, +# explicit preconditions — neither is a comment — before anything is +# ever handed to `eval`. +iso_output=$(python3 "$ISOLATION_PY" --sh codex "$job_dir/home" 2>"$job_dir/.isolation.err") +iso_rc=$? +if [[ $iso_rc -ne 0 ]]; then + refuse "isolation.py could not isolate codex (exit $iso_rc): $(cat -- "$job_dir/.isolation.err" 2>/dev/null)" +fi +if [[ -z "$iso_output" ]]; then + refuse "isolation.py produced no output for codex; this is the 'eval \"\"' shape — refusing rather than treating silence as isolation" +fi +rm -f -- "$job_dir/.isolation.err" +eval "$iso_output" +[[ -n "${ISO_ENV_CODEX_HOME:-}" ]] || + refuse "isolation.py's output did not set ISO_ENV_CODEX_HOME; refusing to dispatch with an unverified isolation boundary" +[[ "$ISO_ENV_CODEX_HOME" == "$job_dir/home" ]] || + refuse "isolation.py isolated CODEX_HOME to '$ISO_ENV_CODEX_HOME', not the job home '$job_dir/home'; refusing" + +# Structural re-verification, independent of trusting the library call +# above: the built home must hold EXACTLY the declared credential and +# nothing else, checked by listing it, not by re-asserting the claim. +home_entries=$(ls -A -- "$job_dir/home") +[[ "$home_entries" == "auth.json" ]] || + refuse "isolated home '$job_dir/home' contains more than auth.json ($home_entries); refusing to launch with an unverified isolation boundary" + +# ---- assemble the child's environment: an explicit allowlist, never ---- +# ---- the operator's inherited environment. CODEX_HOME/HOME come from --- +# ---- isolation.py's own output (ISO_ENV_*) rather than being retyped. -- +child_env=( + "HOME=$job_dir/home" + "CODEX_HOME=$ISO_ENV_CODEX_HOME" + "PATH=/usr/bin:/bin:/usr/local/bin" + "TERM=dumb" + "NO_COLOR=1" + "LANG=C.UTF-8" + "LC_ALL=C.UTF-8" + "GIT_TERMINAL_PROMPT=0" +) +# ISO_FLAGS is empty for codex today (isolation.py: mechanism "home", not +# "flags") but is applied generically so this lever does not silently +# stop tracking the law if that ever changes. +iso_extra_flags=() +if [[ -n "${ISO_FLAGS:-}" ]]; then + read -r -a iso_extra_flags <<<"$ISO_FLAGS" +fi +if [[ "$record_evidence" == "yes" ]]; then + child_env+=("WF_AGENT=$AGENT_IDENTITY") +fi + +# ---- assemble Codex's argv ------------------------------------------------ +codex_argv=( + "$CODEX_BIN" exec + --skip-git-repo-check + --sandbox "$sandbox_mode" + -C "$codex_cwd" + --color never + --output-last-message "$job_dir/last-message.txt" +) +(( ${#iso_extra_flags[@]} )) && codex_argv+=("${iso_extra_flags[@]}") +[[ -n "$model" ]] && codex_argv+=(-m "$model") +codex_argv+=("-") + +# ---- launch, wall-clock capped, prompt piped in on stdin ------------- +started=$(date -u +%Y-%m-%dT%H:%M:%SZ) +start_epoch=$(date -u +%s) +( + exec env -i "${child_env[@]}" timeout --signal=KILL "${wall_timeout}s" \ + "${codex_argv[@]}" < "$job_dir/prompt.txt" +) >"$job_dir/stdout.log" 2>"$job_dir/stderr.log" +exit_code=$? +ended=$(date -u +%Y-%m-%dT%H:%M:%SZ) +end_epoch=$(date -u +%s) + +printf '%s\n' "${codex_argv[@]}" > "$job_dir/argv.txt" +[[ -f "$job_dir/last-message.txt" ]] || : > "$job_dir/last-message.txt" + +# ---- role=write: the LEVER commits, Codex never touches .git ------------ +# Empirically confirmed on this host (codex-cli 0.148.0): even with +# --sandbox workspace-write and cwd pointed at a fully-writable side +# clone, `git commit` INSIDE Codex's own sandboxed exec fails -- +# fatal: Unable to create '.git/index.lock': Read-only file system +# -- while ordinary file writes in that same directory succeed. Codex's +# sandbox mediates `.git` as read-only regardless of which repository it +# is, so asking the model to run the commit itself is not a prompt this +# lever can complete. This is that quirk in exactly the shape the brief +# named it: read-only base snapshot (chmod'd above) + a writable side +# clone for the WRITE, with the COMMIT itself done here, by the lever, in +# its own unsandboxed shell, after Codex's turn has ended. Codex edits +# files; it never invokes git. +commit_attempted="no" commit_sha="" commit_message="" commit_changed="" commit_note="" +if [[ "$role" == "write" ]]; then + commit_attempted="yes" + if ! git -C "$job_dir/wt" add -A 2>"$job_dir/.postadd.err"; then + commit_note="git add -A failed: $(cat -- "$job_dir/.postadd.err" 2>/dev/null)" + elif git -C "$job_dir/wt" diff --cached --quiet 2>/dev/null; then + commit_note="codex made no file changes in the side clone; nothing to commit" + else + commit_changed=$(git -C "$job_dir/wt" diff --cached --name-only | tr '\n' ' ') + commit_message="codex write job $job_id" + if git -C "$job_dir/wt" -c user.name="$AGENT_IDENTITY" -c user.email="noreply@codex-lever" \ + commit --quiet -m "$commit_message" 2>"$job_dir/.postcommit.err"; then + commit_sha=$(git -C "$job_dir/wt" rev-parse HEAD) + commit_note="committed by the lever after codex's turn ended; codex itself never wrote to .git" + else + commit_note="commit failed: $(cat -- "$job_dir/.postcommit.err" 2>/dev/null)" + fi + fi + rm -f -- "$job_dir/.postadd.err" "$job_dir/.postcommit.err" +fi + +# ---- write the run record ----------------------------------------------- +python3 - "$job_dir" "$job_id" "$SNAPSHOT_MODE" "$snapshot_source_desc" \ + "$role" "$sandbox_mode" "$model" "$wall_timeout" \ + "$started" "$ended" "$start_epoch" "$end_epoch" "$exit_code" "$record_evidence" \ + "$write_clone_desc" "$codex_cwd" "${ISO_STORE:-}" "$home_entries" \ + "$commit_attempted" "$commit_sha" "$commit_message" "$commit_changed" "$commit_note" <<'PY' +import json, os, sys + +(job_dir, job_id, mode, source_desc, role, sandbox_mode, + model, wall_timeout, started, ended, start_epoch, end_epoch, exit_code, + record_evidence, write_clone_desc, codex_cwd, iso_store, + pre_dispatch_home, commit_attempted, commit_sha, commit_message, + commit_changed, commit_note) = sys.argv[1:24] + +home_dir = os.path.join(job_dir, "home") +snapshot_dir = os.path.join(job_dir, "snapshot") +snapshot_entries = sorted(os.listdir(snapshot_dir)) +file_count = sum(len(files) for _, _, files in os.walk(snapshot_dir)) + +record = { + "job_id": job_id, + "harness": "codex", + "scope": {"mode": mode, "source": source_desc}, + "prompt_file": "prompt.txt", + "runtime": { + "model": model or None, + "role": role, + "sandbox": sandbox_mode, + "wall_timeout_s": int(wall_timeout), + "cwd": codex_cwd, + }, + "isolation": { + "mechanism": "home", + "law_source": ".dev/app/harness/isolation.py (vendored from origin/dev)", + "codex_home": home_dir, + "pre_dispatch_home_contents": pre_dispatch_home.split(), + "home_contents_after_run": sorted(os.listdir(home_dir)), + "session_store": iso_store or None, + "env_allowlisted": True, + "record_evidence": record_evidence == "yes", + }, + "snapshot_proof": { + "top_level_entries": snapshot_entries, + "file_count": file_count, + "read_only_on_disk": True, + }, + "write_clone": ({ + "path": os.path.join(job_dir, "wt"), + "built_by": write_clone_desc, + "post_run_commit": { + "attempted": commit_attempted == "yes", + "sha": commit_sha or None, + "message": commit_message or None, + "changed_files": commit_changed.split() if commit_changed else [], + "note": commit_note or None, + }, + } if write_clone_desc else None), + "stamp": { + "started": started, + "ended": ended, + "wall_seconds": int(end_epoch) - int(start_epoch), + }, + "exit_code": int(exit_code), + "artifacts": { + "stdout": "stdout.log", + "stderr": "stderr.log", + "argv": "argv.txt", + "last_message": "last-message.txt", + }, +} +with open(os.path.join(job_dir, "run-record.json"), "w", encoding="utf-8") as fh: + json.dump(record, fh, indent=2) + fh.write("\n") +PY + +printf '%s: job=%s exit=%s dir=%s\n' "$PROGRAM" "$job_id" "$exit_code" "$job_dir" +exit "$exit_code" diff --git a/ops/devlane/dispatch/levers/codex/vendor/isolation.py b/ops/devlane/dispatch/levers/codex/vendor/isolation.py new file mode 100644 index 0000000..25c640e --- /dev/null +++ b/ops/devlane/dispatch/levers/codex/vendor/isolation.py @@ -0,0 +1,382 @@ +"""Strip the operator's personal setup out of a dispatched harness. + +A harness launched from a developer's machine does not start empty. It +discovers instruction files, hooks, skills and MCP servers from that +person's home directory, and none of that is this project's law. It is +not in the repository, no other harness shares it, CI does not have it, +and nobody agreed that it applies to work done here. + +What was measured on 2026-08-22, dispatching into a throwaway snapshot +that contained nothing but the brief: + + claude the operator's ~/.claude/CLAUDE.md was in the system prompt. + Asked "do your instructions contain ", a + default dispatch answered YES and an isolated one answered NO. + A SessionStart hook, the personal skill listing and personal + MCP servers arrived as attachments. + codex ~/.codex/hooks.json ran a SessionStart command living in an + unrelated repository, and its output was injected. + grok `grok inspect` listed ~/.claude/CLAUDE.md as a project + instruction worth ~7012 tokens, plus a global rules file and + 31 skills, 24 of them the operator's. + +So the leak is not one harness's quirk. It is what all three do by +design, and the fix has to be per-harness because each reads a +different place. + +Two mechanisms are used here, whichever the harness supports: + + flags the harness offers a documented way to not load user-scoped + configuration. Cheapest and least invasive; nothing on disk + is touched. + home the harness only looks in a directory named by an + environment variable, so it is pointed at a directory built + here that holds credentials and nothing else. + +CREDENTIALS ARE THE ONE EXCEPTION, and it is deliberate. A harness +that cannot authenticate cannot run at all, so the minimal home links +the auth file through and nothing else. Everything that carries +instructions, behaviour or context is left behind. `auth_files` is the +complete list of what crosses that line; it is data, so it can be +read, and a test asserts nothing else is ever linked. + +WHAT THIS IS NOT. Every entry below closes a discovery path that was +found by looking. That makes this a list of known leaks, and a list of +known leaks is only ever as current as the last time somebody looked +-- a harness release can add a fourth path, and nothing here would +say so. `probe.py` exists for exactly that reason and should be run +when a harness version changes. + +The guarantee that does not depend on having enumerated correctly is a +container with no operator home mounted in it: then there is nothing +to discover, whatever the harness decides to look for next. That is +the backstop if a leak is found with no environment variable behind +it, or if a harness stops honouring one. This module is the cheap +version and it is honest about being the cheap version. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +# -------------------------------------------------------------------- +# The data. Every entry says how a harness is isolated and how that was +# established. An entry with no mechanism is not a harness that happens +# to be clean -- it is one nobody has checked, and dispatching it is +# refused rather than assumed safe. +# -------------------------------------------------------------------- + +HARNESSES = { + "claude": { + "mechanism": "flags", + "flags": [ + # Drops ~/.claude/settings.json AND user-scoped CLAUDE.md. + "--setting-sources", "project,local", + # Drops MCP servers configured in the operator's account. + "--strict-mcp-config", + # Drops the personal skill listing. + "--disable-slash-commands", + ], + "home_env": None, + "auth_files": [], + # A phrase that appears only in the operator's own doctrine + # file. Asking the model whether it can see this is the only + # way to observe the system prompt, which is not written to + # the session trace -- absence from the trace would prove + # nothing. + "probe_phrase": "Start from the class, not the instance", + # HOME is untouched, so traces stay in the real home. + "sessions": {"under": "real", "path": ".claude/projects"}, + "measured": { + "on": "2026-08-22", + "version": "unrecorded", + "probe_default": "YES", + "probe_isolated": "NO", + "attachment_bytes_default": 17304, + "attachment_bytes_isolated": 2761, + "note": "what remains is Claude Code's own built-in machinery", + }, + }, + "codex": { + "mechanism": "home", + "flags": [], + "home_env": "CODEX_HOME", + "auth_files": ["auth.json"], + "probe_phrase": None, + # CODEX_HOME moved, so the trace moves with it. + "sessions": {"under": "minimal", "path": "sessions"}, + "measured": { + "on": "2026-08-22", + "version": "0.148.0", + "leak": "~/.codex/hooks.json SessionStart ran " + "projects/xormania/xor/tools/xortations/hooks/session_start.py", + "note": "personal MCP servers and a memories store also live under the home", + }, + }, + "grok": { + # Two variables, because two different directories leak. GROK_HOME + # alone still let ~/.claude/CLAUDE.md through: grok looks for that + # under $HOME, not under its own home. HOME alone still let + # ~/.grok/rules through. Both, or neither works. + "mechanism": "home", + "flags": [], + "home_env": "GROK_HOME", + "also_env": ["HOME"], + "auth_files": ["auth.json"], + "probe_phrase": None, + "sessions": {"under": "minimal", "path": "sessions"}, + "measured": { + "on": "2026-08-22", + "version": "1.0.5", + "leak": "grok inspect listed ~/.claude/CLAUDE.md (~7012 tokens) " + "and ~/.grok/rules/00-xortations-first-turn.md (~161 tokens) " + "as project instructions; 31 skills, 24 user-scoped", + "note": "HOME alone drops CLAUDE.md and cuts skills 31 -> 7; " + "GROK_HOME alone drops the rules file; both are needed", + }, + }, +} + + +class NotIsolated(Exception): + """Raised instead of dispatching a harness that cannot be isolated. + + The refusal is the point. A harness absent from HARNESSES has not + been shown to be clean, and defaulting to "launch it anyway" + converts "nobody looked" into "we checked and it was fine". + """ + + +def _real_home(harness, env, given=True): + """Where this harness's real config lives, for reading credentials. + + Two corrections, both from a test author who could not see this + function and reasoned from what it PROMISES (PR #40 follow-up): + + `home_env` is honoured when the environment sets it. An operator who + runs codex with CODEX_HOME set keeps their config there, not in + ~/.codex, so reading the wrong directory would report a credential + absent that is present -- or link one that is not the one in use. + + `given` defaults to True -- the strict reading -- so a direct caller + gets the refusal and only `build_home`, which knows whether its + caller supplied an env, may ask for the lenient one. + + And when the caller passed an environment EXPLICITLY, a missing HOME + is refused rather than filled in from `Path.home()`. Passing an env + is how a caller says "this, and nothing of mine"; reaching past it + to the operator's real home is the leak this module exists to + prevent, and it made one of the author's tests pass on this machine + and fail on a clean one -- the shape of a suite that lies. + """ + spec = HARNESSES[harness] + named = spec.get("home_env") + if named and env.get(named): + return Path(env[named]) + home = env.get("HOME") + # EMPTY IS ABSENT. `HOME=""` is not None, so the refusal below did + # not fire, and `Path("") / ".codex"` is the RELATIVE path `.codex` + # — resolved against whatever directory the launcher happened to be + # in, which is ambient project state and exactly what an explicit + # environment is supposed to exclude (Copilot, PR #42). The same + # distinction this repo makes everywhere else between "we looked and + # found none" and "nobody looked", arriving one more time as a + # falsy value that is not None. + if not home: + if given: + raise NotIsolated( + f"{harness}: the environment passed here sets neither " + f"{named or 'HOME'} nor HOME, so there is nowhere to read " + f"credentials from. Falling back to the operator's own home " + f"would be the leak this builds a home to prevent.") + home = str(Path.home()) + return Path(home) / f".{harness}" + + +def build_home(harness, root, env=None): + """Create a minimal home for `harness` under `root`; return its path. + + It holds the credential files named in `auth_files` and nothing + else. Each is symlinked, not copied, so a credential is never + duplicated into a scratch directory that outlives the run. + + Raises NotIsolated when a credential the harness needs is missing, + rather than producing a home that will fail to authenticate in a + way that looks like a model refusal. + + `root` must be missing or an EMPTY directory. A reused one may carry + the operator's own setup, and preserving it is the leak this builds + a home to prevent -- so a populated root is refused, naming what it + found. That was not written down until an independent test author + assumed the opposite and expected a rebuild. + """ + given = env is not None + env = os.environ if env is None else env + spec = HARNESSES.get(harness) + if spec is None: + raise NotIsolated( + f"{harness!r} has no isolation entry: nobody has established " + f"what it loads from the operator's home, so it is not " + f"dispatched. Add an entry with a measurement.") + dest = Path(root) + # A MINIMAL home has to start empty. `exist_ok=True` on a reused + # root preserved whatever was already there and returned normally, + # so a home carrying the operator's hooks.json, AGENTS.md and + # skills/ could be handed to a dispatch — through the constructor + # of the module that exists to strip exactly those (Codex, PR #40). + # The structural probe is the only reader that would notice, and + # nothing on the launch path calls it. + if dest.exists(): + if not dest.is_dir(): + raise NotIsolated( + f"{harness}: {dest} exists and is not a directory; a " + f"minimal home cannot be built there.") + leftovers = sorted(p.name for p in dest.iterdir()) + if leftovers: + raise NotIsolated( + f"{harness}: {dest} is not empty ({', '.join(leftovers[:5])}" + f"{', …' if len(leftovers) > 5 else ''}). A reused home may " + f"carry the operator's own setup, which is the leak this " + f"builds a home to prevent. Pass a fresh directory.") + dest.mkdir(parents=True, exist_ok=True) + src_home = _real_home(harness, env, given) + for name in spec["auth_files"]: + src = src_home / name + if not src.exists(): + raise NotIsolated( + f"{harness}: credential {src} is absent, so an isolated " + f"home cannot authenticate. Not falling back to the " + f"operator's home.") + # No exists-check: the destination was just proved empty, so a + # name already there would mean something wrote into the home + # between the two, and skipping it silently is the same hole in + # miniature. + link = dest / name + link.parent.mkdir(parents=True, exist_ok=True) + # ABSOLUTE. `symlink_to` with a relative source resolves it + # against the LINK's directory, not the caller's, so a relative + # `src` produced a link pointing inside the minimal home — a + # dangling one, in a home that looked complete because an entry + # named `auth.json` was there. Surfaced while reproducing the + # empty-HOME finding above; `build_home` returned normally. + link.symlink_to(src.resolve()) + return dest + + +def dispatch_env(harness, home=None, env=None): + """The environment overrides that isolate `harness`. + + `home` is a directory from build_home. It is required for a + home-mechanism harness and ignored for a flag-mechanism one. + """ + env = os.environ if env is None else env + spec = HARNESSES.get(harness) + if spec is None: + raise NotIsolated(f"{harness!r} has no isolation entry") + if spec["mechanism"] == "flags": + return {} + # Same rule, one function along: `home=""` would emit + # `CODEX_HOME=""`, which the harness reads as unset and answers by + # loading the operator's real home. Found by looking for the shape + # rather than the instance, after the instance was reported. + if not home: + raise NotIsolated( + f"{harness} is isolated by relocating its home, and no home " + f"was built (got {home!r}). Call build_home first.") + out = {spec["home_env"]: str(home)} + for extra in spec.get("also_env", ()): + # HOME is redirected to the minimal home too, so that a harness + # looking for a SIBLING vendor's dotfile -- grok reading + # ~/.claude/CLAUDE.md -- finds nothing there either. + out[extra] = str(home) + return out + + +def dispatch_flags(harness): + """The argv fragment that isolates `harness`, possibly empty.""" + spec = HARNESSES.get(harness) + if spec is None: + raise NotIsolated(f"{harness!r} has no isolation entry") + return list(spec["flags"]) + + +def isolated(harness, root, env=None): + """Everything a launcher needs: (env_overrides, argv_fragment). + + The single entry point. A launcher that calls this cannot dispatch + an unisolated harness, because there is no argument that turns the + isolation off. + """ + spec = HARNESSES.get(harness) + if spec is None: + raise NotIsolated( + f"{harness!r} has no isolation entry: dispatching it would " + f"carry the operator's personal setup into this project.") + home = build_home(harness, root, env) if spec["mechanism"] == "home" else None + return dispatch_env(harness, home, env), dispatch_flags(harness) + + +def report(): + """What is known about each harness, as JSON. For the record, and + for a check that wants to notice an entry going stale.""" + # The WHOLE entry. It reported `mechanism` and `measured` only, + # while promising "what is known about each harness" and naming its + # own purpose as noticing an entry going stale -- and a check that + # cannot see `flags` or `auth_files` cannot notice those going + # stale, which is the operative half (PR #40 follow-up). + return json.dumps(HARNESSES, indent=2, sort_keys=True) + + +def _main(argv=None): + """A shell launcher needs the same answer this module already + holds. Giving it one is what keeps the flags from being written + down twice and drifting apart -- the duplicate copy is always the + one that misses the next fix. + + eval "$(isolation.py --sh claude /tmp/home)" + + emits `ISO_FLAGS` and any environment assignments, and exits + non-zero with an explanation on a harness that cannot be isolated, + so a launcher that checks its exit status cannot dispatch one. + """ + import argparse + import shlex + + ap = argparse.ArgumentParser(description="isolation facts for a launcher") + ap.add_argument("--sh", metavar="HARNESS", + help="emit shell assignments for this harness") + ap.add_argument("root", nargs="?", + help="directory to build a minimal home in (--sh only)") + args = ap.parse_args(argv) + + if not args.sh: + print(report()) + return 0 + try: + if HARNESSES.get(args.sh, {}).get("mechanism") == "home" and not args.root: + raise NotIsolated( + f"{args.sh} is isolated by relocating its home; pass a " + f"directory to build one in") + env, flags = isolated(args.sh, args.root or "") + except NotIsolated as exc: + print(f"echo {shlex.quote('REFUSED: ' + str(exc))} >&2; exit 78") + return 78 + spec = HARNESSES[args.sh] + sess = spec["sessions"] + base = (str(Path(os.environ.get("HOME") or Path.home())) + if sess["under"] == "real" else str(args.root)) + # Emitted as assignments, not exports: the launcher must apply these + # to the HARNESS only. Exporting HOME would relocate the launcher's + # own lookups too, and it still needs the real one. + for k, v in sorted(env.items()): + print(f"ISO_ENV_{k}={shlex.quote(v)}") + print(f"ISO_ENV={shlex.quote(' '.join(f'{k}={v}' for k, v in sorted(env.items())))}") + print(f"ISO_FLAGS={shlex.quote(' '.join(flags))}") + print(f"ISO_STORE={shlex.quote(str(Path(base) / sess['path']))}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/ops/devlane/dispatch/levers/grok-dispatch.sh b/ops/devlane/dispatch/levers/grok-dispatch.sh new file mode 100755 index 0000000..0c8b580 --- /dev/null +++ b/ops/devlane/dispatch/levers/grok-dispatch.sh @@ -0,0 +1,349 @@ +#!/usr/bin/env bash +# grok-dispatch.sh — custody-isolated headless Grok dispatch lever. +# +# Launches ONE headless, single-turn Grok investigation job against an +# isolated snapshot (a git ref archived read-only out of a repo) or an +# explicit path subtree copied into a per-job directory. Never runs +# against the operator's $HOME, never inherits the operator's shell +# environment, never touches the live repository working tree. +# +# Every precondition below is an explicit check. If isolation cannot be +# established, this script REFUSES (exit 64) before Grok is ever +# invoked — it does not fall back to an unisolated run, and it does not +# treat "no output" as success. +# +# Usage: +# grok-dispatch.sh --scope-ref REF --repo PATH (snapshot of repo at ref) +# grok-dispatch.sh --scope-path DIR (explicit subtree, copied) +# ... plus one of: +# --prompt TEXT | --prompt-file PATH +# ... and optionally: +# --model MODEL --output-format FMT --permission-mode MODE +# --max-turns N --timeout SECONDS --allow-web-search +# --jobs-root DIR --record-evidence +# +# Job output lands in: // +# snapshot/ the isolated copy Grok actually saw, nothing else +# prompt.txt the exact prompt bytes handed to Grok +# home/ isolated GROK_HOME=HOME, containing only auth.json +# stdout.log Grok's stdout (the plain/json response) +# stderr.log Grok's stderr +# run-record.json job id, argv, isolation proof, timestamps, exit code + +set -u -o pipefail + +readonly PROGRAM=${0##*/} +LEVERS_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) || + { printf '%s: refused: cannot resolve script directory\n' "$PROGRAM" >&2; exit 64; } +readonly LEVERS_DIR +readonly JOBS_ROOT_DEFAULT="$LEVERS_DIR/jobs" + +refuse() { + printf '%s: REFUSED: %s\n' "$PROGRAM" "$*" >&2 + exit 64 +} + +usage() { + cat <<'EOF' +grok-dispatch.sh --scope-ref REF --repo PATH | --scope-path DIR + (--prompt TEXT | --prompt-file PATH) + [--model MODEL] [--output-format FMT] + [--permission-mode MODE] [--max-turns N] + [--timeout SECONDS] [--allow-web-search] + [--jobs-root DIR] [--record-evidence] +EOF +} + +# ---- defaults ------------------------------------------------------- +scope_ref="" +scope_repo="" +scope_path="" +prompt_text="" +prompt_file="" +model="" +output_format="plain" +# NOT "plan": measured live (2026-08-27) that under --permission-mode plan +# a headless run's run_terminal_command tool call comes back "User +# cancelled the execution" -- plan mode asks for approval on shell +# execution and headless has no one to answer, so it soft-denies and the +# job still exits 0 having done nothing. "auto" auto-approves tool use +# inside the already-isolated, disposable snapshot/home, which is where +# this lever's containment actually lives (see CUSTODY notes in the +# header) rather than in the permission gate. +permission_mode="auto" +max_turns="8" +wall_timeout="300" +allow_web_search="no" +jobs_root="$JOBS_ROOT_DEFAULT" +record_evidence="no" +readonly AGENT_IDENTITY="Grok Investigator " + +# ---- args ------------------------------------------------------------- +while (( $# )); do + case "$1" in + --scope-ref) scope_ref=${2:-}; shift 2 ;; + --repo) scope_repo=${2:-}; shift 2 ;; + --scope-path) scope_path=${2:-}; shift 2 ;; + --prompt) prompt_text=${2:-}; shift 2 ;; + --prompt-file) prompt_file=${2:-}; shift 2 ;; + --model) model=${2:-}; shift 2 ;; + --output-format) output_format=${2:-}; shift 2 ;; + --permission-mode) permission_mode=${2:-}; shift 2 ;; + --max-turns) max_turns=${2:-}; shift 2 ;; + --timeout) wall_timeout=${2:-}; shift 2 ;; + --allow-web-search) allow_web_search="yes"; shift ;; + --jobs-root) jobs_root=${2:-}; shift 2 ;; + --record-evidence) record_evidence="yes"; shift ;; + -h|--help) usage; exit 0 ;; + *) refuse "unknown argument '$1'" ;; + esac +done + +# ---- preconditions: scope ------------------------------------------- +[[ -n "${HOME:-}" ]] || + refuse "HOME is unset or empty; the operator/isolation boundary cannot be resolved" +readonly OPERATOR_HOME=$HOME + +if [[ -n "$scope_ref" && -n "$scope_path" ]]; then + refuse "give exactly one of --scope-ref or --scope-path, not both" +fi +if [[ -z "$scope_ref" && -z "$scope_path" ]]; then + refuse "no scope given: pass --scope-ref REF --repo PATH, or --scope-path DIR — there is no default scope, and the default is never the live repo or \$HOME" +fi + +if [[ -n "$scope_ref" ]]; then + [[ -n "$scope_repo" ]] || + refuse "--scope-ref requires --repo PATH naming the repository to archive it from" + resolved_repo=$(realpath -e -- "$scope_repo" 2>/dev/null) || + refuse "--repo '$scope_repo' does not resolve to an existing path" + git -C "$resolved_repo" rev-parse --git-dir >/dev/null 2>&1 || + refuse "'$resolved_repo' is not a git repository" + ref_sha=$(git -C "$resolved_repo" rev-parse --verify --end-of-options \ + "${scope_ref}^{commit}" 2>&1) || + refuse "--scope-ref '$scope_ref' does not resolve to a commit in '$resolved_repo': $ref_sha" + readonly SNAPSHOT_MODE="ref" + readonly RESOLVED_REPO=$resolved_repo + readonly REF_SHA=$ref_sha +else + resolved_scope=$(realpath -e -- "$scope_path" 2>/dev/null) || + refuse "--scope-path '$scope_path' does not resolve to an existing path" + [[ -d "$resolved_scope" ]] || + refuse "--scope-path '$resolved_scope' is not a directory" + resolved_home=$(realpath -e -- "$OPERATOR_HOME" 2>/dev/null) || resolved_home=$OPERATOR_HOME + if [[ "$resolved_scope" == "$resolved_home" ]]; then + refuse "--scope-path resolves to the operator's \$HOME ($resolved_home); this is never permitted, no override exists" + fi + if [[ "$resolved_home" == "$resolved_scope"/* ]]; then + refuse "--scope-path '$resolved_scope' is an ancestor of the operator's \$HOME; this is never permitted" + fi + for guard in .ssh .aws .gnupg .grok .claude .codex .config; do + if [[ "$resolved_scope" == "$resolved_home/$guard" || "$resolved_scope" == "$resolved_home/$guard"/* ]]; then + refuse "--scope-path '$resolved_scope' is inside the operator's '$guard' directory; refusing" + fi + done + readonly SNAPSHOT_MODE="path" + readonly RESOLVED_SCOPE=$resolved_scope +fi + +# ---- preconditions: prompt -------------------------------------------- +if [[ -n "$prompt_text" && -n "$prompt_file" ]]; then + refuse "give exactly one of --prompt or --prompt-file, not both" +fi +if [[ -z "$prompt_text" && -z "$prompt_file" ]]; then + refuse "no prompt given: pass --prompt TEXT or --prompt-file PATH" +fi +if [[ -n "$prompt_file" ]]; then + resolved_prompt_file=$(realpath -e -- "$prompt_file" 2>/dev/null) || + refuse "--prompt-file '$prompt_file' does not resolve to an existing file" + [[ -f "$resolved_prompt_file" && -s "$resolved_prompt_file" ]] || + refuse "--prompt-file '$resolved_prompt_file' is not a non-empty regular file" +fi +if [[ -n "$prompt_text" ]]; then + # A prompt that is only whitespace is the "eval \"\"" shape: it would + # exit clean having asked Grok nothing. Refuse it explicitly. + trimmed=${prompt_text//[$'\t\r\n ']/} + [[ -n "$trimmed" ]] || + refuse "--prompt is empty or whitespace-only; refusing rather than dispatching an empty job" +fi + +# ---- preconditions: output/permission dials --------------------------- +case "$output_format" in + plain|json|streaming-json|streaming-messages-json) ;; + *) refuse "--output-format '$output_format' is not one of plain|json|streaming-json|streaming-messages-json" ;; +esac +case "$permission_mode" in + default|acceptEdits|auto|dontAsk|bypassPermissions|plan) ;; + *) refuse "--permission-mode '$permission_mode' is not one of default|acceptEdits|auto|dontAsk|bypassPermissions|plan" ;; +esac +[[ "$max_turns" =~ ^[0-9]+$ && "$max_turns" -gt 0 ]] || + refuse "--max-turns '$max_turns' must be a positive integer" +[[ "$wall_timeout" =~ ^[0-9]+$ && "$wall_timeout" -gt 0 ]] || + refuse "--timeout '$wall_timeout' must be a positive integer number of seconds" + +# ---- preconditions: the grok binary and its credential ----------------- +grok_bin=$(command -v grok 2>/dev/null || true) +if [[ -z "$grok_bin" ]]; then + for candidate in "$OPERATOR_HOME/.grok/bin/grok" /home/work/.grok/bin/grok; do + if [[ -x "$candidate" ]]; then grok_bin=$candidate; break; fi + done +fi +[[ -n "$grok_bin" && -x "$grok_bin" ]] || + refuse "no executable 'grok' binary found on PATH or at the known install location" +readonly GROK_BIN=$grok_bin + +source_grok_home=${GROK_HOME:-$OPERATOR_HOME/.grok} +source_auth="$source_grok_home/auth.json" +[[ -f "$source_auth" && -r "$source_auth" ]] || + refuse "grok credential '$source_auth' is absent or unreadable; refusing an unisolated fallback (this is the BLOCKER case: fix auth, do not disable isolation)" +resolved_auth=$(realpath -e -- "$source_auth" 2>/dev/null) || + refuse "grok credential '$source_auth' cannot be resolved" +readonly RESOLVED_AUTH=$resolved_auth + +# ---- job directory ------------------------------------------------------ +mkdir -p -- "$jobs_root" || refuse "cannot create jobs root '$jobs_root'" +jobs_root=$(realpath -e -- "$jobs_root") || refuse "jobs root '$jobs_root' did not resolve after creation" +job_id="$(date -u +%Y%m%dT%H%M%SZ)-grok-$(od -An -tx1 -N3 /dev/urandom | tr -d ' \n')" +job_dir="$jobs_root/$job_id" +mkdir -- "$job_dir" || refuse "job directory '$job_dir' already exists or could not be created" +mkdir -- "$job_dir/snapshot" "$job_dir/home" || refuse "cannot create job subdirectories under '$job_dir'" +chmod 700 -- "$job_dir/home" || refuse "cannot protect isolated home '$job_dir/home'" + +# ---- build the snapshot (read-only from the source; never mutates it) -- +if [[ "$SNAPSHOT_MODE" == "ref" ]]; then + if ! git -C "$RESOLVED_REPO" archive --format=tar "$REF_SHA" 2>"$job_dir/.archive.err" \ + | tar -x -C "$job_dir/snapshot"; then + refuse "snapshot: 'git archive' of $REF_SHA from '$RESOLVED_REPO' failed: $(cat -- "$job_dir/.archive.err" 2>/dev/null)" + fi + rm -f -- "$job_dir/.archive.err" + snapshot_source_desc="ref $scope_ref ($REF_SHA) archived from $RESOLVED_REPO" +else + if ! rsync -a --exclude='.git' -- "$RESOLVED_SCOPE"/ "$job_dir/snapshot"/ 2>"$job_dir/.rsync.err"; then + refuse "snapshot: copying '$RESOLVED_SCOPE' failed: $(cat -- "$job_dir/.rsync.err" 2>/dev/null)" + fi + rm -f -- "$job_dir/.rsync.err" + snapshot_source_desc="path $RESOLVED_SCOPE (copied)" +fi + +# FAIL CLOSED: an empty snapshot is exactly the "eval \"\"" shape — it +# would let Grok run, see nothing worth objecting to, exit 0, and be +# reported as a clean job. Refuse before Grok is ever launched. +if [[ -z "$(find "$job_dir/snapshot" -mindepth 1 -print -quit 2>/dev/null)" ]]; then + refuse "snapshot at '$job_dir/snapshot' is empty ($snapshot_source_desc); refusing to dispatch against an empty scope" +fi + +# ---- write the prompt, verbatim, inside the job directory -------------- +if [[ -n "$prompt_file" ]]; then + cp -- "$resolved_prompt_file" "$job_dir/prompt.txt" || + refuse "cannot copy prompt file into job directory" +else + printf '%s\n' "$prompt_text" > "$job_dir/prompt.txt" || + refuse "cannot write prompt into job directory" +fi +[[ -s "$job_dir/prompt.txt" ]] || + refuse "rendered prompt.txt is empty after being written; refusing to dispatch" + +# ---- build the isolated home: exactly one file, a symlink to auth ----- +ln -s -- "$RESOLVED_AUTH" "$job_dir/home/auth.json" || + refuse "cannot link the sole allowed grok credential into the isolated home" +home_entries=$(ls -A -- "$job_dir/home") +[[ "$home_entries" == "auth.json" ]] || + refuse "isolated home '$job_dir/home' contains more than auth.json ($home_entries); refusing to launch with an unverified isolation boundary" + +# ---- assemble the child's environment: an explicit allowlist, never ---- +# ---- the operator's inherited environment. -- +child_env=( + "HOME=$job_dir/home" + "GROK_HOME=$job_dir/home" + "PATH=/usr/bin:/bin:/usr/local/bin" + "TERM=dumb" + "NO_COLOR=1" + "CLICOLOR=0" + "LANG=C.UTF-8" + "LC_ALL=C.UTF-8" + "GIT_TERMINAL_PROMPT=0" +) +if [[ "$record_evidence" == "yes" ]]; then + child_env+=("WF_AGENT=$AGENT_IDENTITY") +fi + +# ---- assemble Grok's argv ------------------------------------------------ +grok_argv=( + "$GROK_BIN" + --prompt-file "$job_dir/prompt.txt" + --output-format "$output_format" + --permission-mode "$permission_mode" + --max-turns "$max_turns" +) +[[ "$allow_web_search" == "yes" ]] || grok_argv+=(--disable-web-search) +[[ -n "$model" ]] && grok_argv+=(-m "$model") + +# ---- launch, wall-clock capped, cwd pinned to the snapshot ------------- +started=$(date -u +%Y-%m-%dT%H:%M:%SZ) +start_epoch=$(date -u +%s) +( + cd -- "$job_dir/snapshot" && + exec env -i "${child_env[@]}" timeout --signal=KILL "${wall_timeout}s" "${grok_argv[@]}" +) >"$job_dir/stdout.log" 2>"$job_dir/stderr.log" +exit_code=$? +ended=$(date -u +%Y-%m-%dT%H:%M:%SZ) +end_epoch=$(date -u +%s) + +printf '%s\n' "${grok_argv[@]}" > "$job_dir/argv.txt" + +# ---- write the run record ----------------------------------------------- +python3 - "$job_dir" "$job_id" "$SNAPSHOT_MODE" "$snapshot_source_desc" \ + "$permission_mode" "$output_format" "$model" "$max_turns" "$wall_timeout" \ + "$started" "$ended" "$start_epoch" "$end_epoch" "$exit_code" "$record_evidence" <<'PY' +import json, os, sys + +(job_dir, job_id, mode, source_desc, permission_mode, output_format, model, + max_turns, wall_timeout, started, ended, start_epoch, end_epoch, exit_code, + record_evidence) = sys.argv[1:16] + +home_dir = os.path.join(job_dir, "home") +snapshot_dir = os.path.join(job_dir, "snapshot") +snapshot_entries = sorted(os.listdir(snapshot_dir)) +file_count = sum(len(files) for _, _, files in os.walk(snapshot_dir)) + +record = { + "job_id": job_id, + "harness": "grok", + "scope": {"mode": mode, "source": source_desc}, + "prompt_file": "prompt.txt", + "runtime": { + "model": model or None, + "output_format": output_format, + "permission_mode": permission_mode, + "max_turns": int(max_turns), + "wall_timeout_s": int(wall_timeout), + "web_search_disabled": True, + }, + "isolation": { + "home": home_dir, + "home_contents": sorted(os.listdir(home_dir)), + "env_allowlisted": True, + "record_evidence": record_evidence == "yes", + }, + "snapshot_proof": { + "top_level_entries": snapshot_entries, + "file_count": file_count, + }, + "stamp": { + "started": started, + "ended": ended, + "wall_seconds": int(end_epoch) - int(start_epoch), + }, + "exit_code": int(exit_code), + "artifacts": { + "stdout": "stdout.log", + "stderr": "stderr.log", + "argv": "argv.txt", + }, +} +with open(os.path.join(job_dir, "run-record.json"), "w", encoding="utf-8") as fh: + json.dump(record, fh, indent=2) + fh.write("\n") +PY + +printf '%s: job=%s exit=%s dir=%s\n' "$PROGRAM" "$job_id" "$exit_code" "$job_dir" +exit "$exit_code" diff --git a/ops/devlane/dispatch/levers/selftest.sh b/ops/devlane/dispatch/levers/selftest.sh new file mode 100755 index 0000000..de80c4d --- /dev/null +++ b/ops/devlane/dispatch/levers/selftest.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# selftest.sh — offline acceptance/refusal tests for apply-push.sh. +# Builds only throwaway repositories beneath mktemp and uses a local bare +# repository as the real push/lease endpoint. No network or caller repo is used. + +set -u -o pipefail +HERE=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) || exit 1 +readonly BRIDGE="$HERE/apply-push.sh" +tmp=$(mktemp -d) || exit 1 +trap 'rm -rf -- "$tmp"' EXIT +export GIT_CONFIG_NOSYSTEM=1 +export HOME="$tmp/home" +mkdir -p "$HOME" +passed=0 total=0 + +pass() { passed=$((passed+1)); total=$((total+1)); printf 'PASS: %s\n' "$1"; } +fail() { total=$((total+1)); printf 'FAIL: %s\n' "$1"; } +assert_eq() { [[ "$1" == "$2" ]]; } + +git init --bare --quiet "$tmp/remote.git" +git init --quiet "$tmp/source" +git -C "$tmp/source" config user.name Test +git -C "$tmp/source" config user.email test@example.invalid +mkdir -p "$tmp/source/allowed" +printf 'base\n' >"$tmp/source/allowed/data.txt" +printf 'outside\n' >"$tmp/source/outside.txt" +git -C "$tmp/source" add -A +git -C "$tmp/source" commit --quiet -m base +base=$(git -C "$tmp/source" rev-parse HEAD) +git -C "$tmp/source" branch feature +git -C "$tmp/source" branch dev +git -C "$tmp/source" remote add origin "$tmp/remote.git" +git -C "$tmp/source" push --quiet origin feature dev + +make_job() { + name=$1 path=$2 content=$3 + dir="$tmp/$name"; mkdir -p "$dir" + git clone --quiet "$tmp/source" "$dir/wt" + git -C "$dir/wt" checkout --quiet --detach "$base" + mkdir -p "$(dirname "$dir/wt/$path")" + printf '%s\n' "$content" >"$dir/wt/$path" + git -C "$dir/wt" -c user.name=Lever -c user.email=lever@example.invalid add -A + git -C "$dir/wt" -c user.name=Lever -c user.email=lever@example.invalid commit --quiet -m "lever commit" + sha=$(git -C "$dir/wt" rev-parse HEAD) + python3 - "$dir/run-record.json" "$base" "$sha" <<'PY' +import json,sys +json.dump({"scope":{"source":"ref feature (%s) archived"%sys.argv[2]}, + "runtime":{"role":"write"},"write_clone":{"post_run_commit":{"sha":sys.argv[3]}}},open(sys.argv[1],"w")) +PY + printf '%s' "$dir" +} + +run_bridge() { "$BRIDGE" --repo "$tmp/source" --branch feature --jobs-root "$tmp/evidence" \ + --allow-paths 'allowed/**' --gates stub --gate-runner "$1 {gate}" "${@:2}" >/dev/null 2>&1; } +remote_tip() { git --git-dir="$tmp/remote.git" rev-parse "refs/heads/$1"; } + +job=$(make_job github .github/workflows/x.yml ci) +if "$BRIDGE" --repo "$tmp/source" --branch feature --from-job "$job" --allow-paths '.github/**' \ + --jobs-root "$tmp/evidence" --gates stub --gate-runner 'true {gate}' --dry-run >/dev/null 2>&1; then + pass ".github/** allow-paths accepted" +else + fail ".github/** allow-paths accepted" +fi + +before=$(remote_tip feature) +if ! "$BRIDGE" --repo "$tmp/source" --branch feature --from-job "$job" --allow-paths '.git/**' \ + --jobs-root "$tmp/evidence" --gates stub --gate-runner 'true {gate}' --dry-run >/dev/null 2>&1 \ + && assert_eq "$(remote_tip feature)" "$before"; then + pass ".git/** allow-paths refused without push" +else + fail ".git/** allow-paths refused without push" +fi + +job=$(make_job happy allowed/data.txt landed) +if run_bridge true --from-job "$job" --message $'Land tested change\n\nSource: isolated-write-job\nCo-Authored-By: Lever '; then + tip=$(remote_tip feature) + msg=$(git --git-dir="$tmp/remote.git" show -s --format=%B "$tip") + treeval=$(git --git-dir="$tmp/remote.git" show "$tip:allowed/data.txt") + if [[ "$treeval" == landed ]] && python3 - "$msg" <<'PY' +import re,sys +p=re.split(r"\n\s*\n",sys.argv[1].strip())[-1].splitlines() +raise SystemExit(not (len(p)==4 and all(re.match(r"^[A-Za-z0-9-]+: .+",x) for x in p))) +PY + then pass "happy path lands expected tree with contiguous trailers"; else fail "happy path lands expected tree with contiguous trailers"; fi +else fail "happy path lands expected tree with contiguous trailers"; fi + +parsed=$(git --git-dir="$tmp/remote.git" show -s --format=%B "$tip" | git interpret-trailers --parse) +if [[ "$parsed" == *'Source: isolated-write-job'* ]] \ + && [[ "$parsed" == *'Co-Authored-By: Lever '* ]] \ + && [[ "$parsed" == *'Apply-Push-Job: '* ]] \ + && [[ "$parsed" == *'Patch-SHA256: '* ]]; then + pass "caller Source survives as git-parsed trailer after landing" +else + fail "caller Source survives as git-parsed trailer after landing" +fi + +job=$(make_job outside outside.txt changed); before=$(remote_tip feature) +if ! run_bridge true --from-job "$job" && assert_eq "$(remote_tip feature)" "$before"; then pass "outside allow-path refused without push"; else fail "outside allow-path refused without push"; fi + +# The advertised diff paths are allowed, but git apply follows ---/+++ and +# would modify outside.txt unless its effective target is independently parsed. +job=$(make_job headerlie outside.txt headerlie) +lie="$tmp/header-lie.patch" +git -C "$job/wt" diff --binary "$base" HEAD -- >"$lie" +sed -i '1s@a/outside.txt b/outside.txt@a/allowed/data.txt b/allowed/data.txt@' "$lie" +before=$(remote_tip feature) +if ! run_bridge true --patch "$lie" --base "$base" && assert_eq "$(remote_tip feature)" "$before"; then pass "mismatched diff target refused without push"; else fail "mismatched diff target refused without push"; fi + +# A raw malicious diff header proves traversal/.git syntax is rejected before apply. +bad="$tmp/bad.patch"; printf 'diff --git a/../.git/config b/../.git/config\n' >"$bad"; before=$(remote_tip feature) +if ! run_bridge true --patch "$bad" --base "$base" && assert_eq "$(remote_tip feature)" "$before"; then pass "dotdot/.git path refused without push"; else fail "dotdot/.git path refused without push"; fi + +# git apply accepts this trailing line as harmless text. The path parser exits +# non-zero after emitting the valid path; process-substitution mapfile used to +# swallow that failure and let the valid change continue to the push. +job=$(make_job parserfail allowed/data.txt parserfail) +malformed="$tmp/parser-failure.patch" +git -C "$job/wt" diff --binary "$base" HEAD -- >"$malformed" +printf 'diff --git malformed\n' >>"$malformed" +before=$(remote_tip feature) +if ! run_bridge true --patch "$malformed" --base "$base" && assert_eq "$(remote_tip feature)" "$before"; then pass "path parser failure refused without push"; else fail "path parser failure refused without push"; fi + +job=$(make_job gatefail allowed/data.txt gatefail); before=$(remote_tip feature) +if ! run_bridge false --from-job "$job" && assert_eq "$(remote_tip feature)" "$before"; then pass "failing gate refused without push"; else fail "failing gate refused without push"; fi + +job=$(make_job protected allowed/data.txt protected); before=$(remote_tip dev) +if ! "$BRIDGE" --repo "$tmp/source" --branch dev --from-job "$job" --allow-paths 'allowed/**' --jobs-root "$tmp/evidence" >/dev/null 2>&1 && assert_eq "$(remote_tip dev)" "$before"; then pass "protected dev refused without push"; else fail "protected dev refused without push"; fi + +# Remote feature now contains happy commit, while this job is based on original base. +job=$(make_job nonff allowed/data.txt nonff); before=$(remote_tip feature) +if ! run_bridge true --from-job "$job" && assert_eq "$(remote_tip feature)" "$before"; then pass "non-fast-forward refused without force-with-lease"; else fail "non-fast-forward refused without force-with-lease"; fi + +# The gate advances the remote after observation; exact lease must reject the push. +advance="$tmp/advance.sh" +cat >"$advance" <---<6 hex> +ID_RE = re.compile( + r"^\d{8}T\d{6}Z-" + r"(plan|tests|check-tests|code|review|adjudicate)-" + r"(claude|codex|grok)-[0-9a-f]{6}$" +) + +RECORD_FIELDS = ( + "id", "lane", "stage", "unit", "lineage", "follows", "job", "role", + "dispatched_by", "at", "snapshot", "harness", "model", "session", + "brief", "caps", "overrides", "attempts", "result", "status", +) + +# A subprocess harness. Records argv/cwd/env/stdin/prompt, optionally +# writes a harness-shaped stream, optionally mutates the snapshot. +# The start-witness path is interpolated at install time so a launcher +# that unsets TASK_LAUNCH_WITNESS still leaves evidence it ran. +# No network. No clock read — stream epochs come from the environment. +_FAKE_CLI = r"""#!/usr/bin/env python3 +import hashlib +import importlib.util +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +start_witness = Path(@@START_WITNESS@@) +ran_model = @@RAN_MODEL@@ +stores_path = @@STORES_PATH@@ +done_path = os.environ.get("TASK_LAUNCH_DONE") +sleep_s = float(os.environ.get("TASK_LAUNCH_SLEEP") or "0") +stdout_mode = os.environ.get("TASK_LAUNCH_STDOUT") or "envelope" +token = os.environ.get("TASK_LAUNCH_TOKEN") or "" +job = os.environ.get("TASK_LAUNCH_JOB") or "plan" +verdict = os.environ.get("TASK_LAUNCH_VERDICT") or "approve" +status = os.environ.get("TASK_LAUNCH_STATUS") or "ok" +envelope_commit = os.environ.get("TASK_LAUNCH_ENVELOPE_COMMIT") or "" +epoch = float(os.environ.get("TASK_LAUNCH_STREAM_EPOCH") or "1700000000") +write_stream = os.environ.get("TASK_LAUNCH_WRITE_STREAM") == "1" +ignore_session = os.environ.get("TASK_LAUNCH_IGNORE_SESSION") == "1" +stream_id_override = os.environ.get("TASK_LAUNCH_STREAM_ID") or "" +commit_rel = os.environ.get("TASK_LAUNCH_COMMIT") or "" +edit_rel = os.environ.get("TASK_LAUNCH_EDIT") or "" +orphan = os.environ.get("TASK_LAUNCH_ORPHAN") == "1" +head_commit = os.environ.get("TASK_LAUNCH_HEAD_COMMIT") or "" +exit_code = int(os.environ.get("TASK_LAUNCH_EXIT") or "0") +over_out = int(os.environ.get("TASK_LAUNCH_OVER_OUT") or "0") +grandchild_path = os.environ.get("TASK_LAUNCH_GRANDCHILD") or "" +witness = os.environ.get("TASK_LAUNCH_WITNESS") + +exe = Path(sys.argv[0]).name if sys.argv else "" +argv = sys.argv[1:] +prompt_file = None +if "--prompt-file" in argv: + idx = argv.index("--prompt-file") + if idx + 1 < len(argv): + prompt_file = argv[idx + 1] + +session_id = "" +if "--session-id" in argv: + idx = argv.index("--session-id") + if idx + 1 < len(argv): + session_id = argv[idx + 1] +if "-s" in argv: + idx = argv.index("-s") + if idx + 1 < len(argv): + session_id = argv[idx + 1] + +stdin_data = sys.stdin.read() +prompt_text = "" +if prompt_file: + try: + prompt_text = Path(prompt_file).read_text(encoding="utf-8") + except OSError: + prompt_text = "" + +env_keys = [ + "CLICOLOR_FORCE", "FORCE_COLOR", "NO_COLOR", "CLICOLOR", "TERM", + "PAGER", "GH_PAGER", "GIT_PAGER", "LESS", "CI", "GIT_TERMINAL_PROMPT", + "GIT_EDITOR", "EDITOR", "PYTHONUNBUFFERED", "PYTHONIOENCODING", + "LC_ALL", "WF_LANE", "DISPATCH_JOB", "CODEX_HOME", "GROK_HOME", + "HOME", "CLAUDE_CONFIG_DIR", "WF_AGENT", +] +env_shot = {k: os.environ[k] for k in env_keys if k in os.environ} + +job_parent = Path(os.getcwd()).parent +last_message = None +if "-o" in argv: + oidx = argv.index("-o") + if oidx + 1 < len(argv): + last_message = Path(argv[oidx + 1]) +if last_message is None: + last_message = job_parent / "out" / "last-message.json" + +def _schema_probe(): + if "--json-schema" not in argv: + return False, None, None, "missing" + sidx = argv.index("--json-schema") + if sidx + 1 >= len(argv): + return False, None, None, "no-value" + value = argv[sidx + 1] + if str(value).lstrip().startswith("{"): + try: + json.loads(value) + except (TypeError, ValueError, json.JSONDecodeError): + return False, None, None, "inline-invalid" + digest = hashlib.sha256(str(value).encode("utf-8")).hexdigest() + return True, None, digest, "inline" + path = Path(value) + try: + text = path.read_text(encoding="utf-8") + json.loads(text) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return False, str(path), None, "unreadable" + digest = hashlib.sha256(text.encode("utf-8")).hexdigest() + return True, str(path), digest, "file" + +schema_read, schema_path, schema_sha, schema_how = _schema_probe() + +try: + pgid = os.getpgid(0) +except OSError: + pgid = os.getpid() +start_witness.parent.mkdir(parents=True, exist_ok=True) +start_witness.write_text(json.dumps({ + "pid": os.getpid(), + "pgid": pgid, + "argv": sys.argv, + "cwd": os.getcwd(), + "exe": sys.argv[0] if sys.argv else "", + "session_id": session_id, + "exit_present": (job_parent / "exit").exists(), + "tripped_present": (job_parent / "TRIPPED.md").exists(), + "last_message_present": last_message.exists(), + "schema_read": schema_read, + "schema_path": schema_path, + "schema_sha": schema_sha, + "schema_how": schema_how, +}) + "\n", encoding="utf-8") + +if witness: + Path(witness).write_text(json.dumps({ + "argv": sys.argv, + "cwd": os.getcwd(), + "stdin": stdin_data, + "prompt_file": prompt_file, + "prompt_text": prompt_text, + "env": env_shot, + "session_id": session_id, + "exe": sys.argv[0] if sys.argv else "", + "schema_read": schema_read, + "schema_path": schema_path, + "schema_sha": schema_sha, + "schema_how": schema_how, + "last_message_present": last_message.exists(), + }), encoding="utf-8") + +if grandchild_path: + child_pid = os.fork() + if child_pid == 0: + time.sleep(max(sleep_s, 30)) + os._exit(0) + Path(grandchild_path).write_text(str(child_pid) + "\n", encoding="utf-8") + +stream_id = stream_id_override or session_id or "00000000-0000-4000-8000-000000000001" +if ignore_session: + stream_id = "ffffffff-ffff-4fff-8fff-ffffffffffff" + +def git(*args): + env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")} + env.update({ + "GIT_AUTHOR_NAME": "worker", + "GIT_AUTHOR_EMAIL": "worker@example.test", + "GIT_COMMITTER_NAME": "worker", + "GIT_COMMITTER_EMAIL": "worker@example.test", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + }) + return subprocess.run( + ["git", *args], cwd=os.getcwd(), env=env, + capture_output=True, text=True, + ) + +if commit_rel: + p = Path(commit_rel) + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("worker-commit\n", encoding="utf-8") + git("add", "--", commit_rel) + git("commit", "-m", "worker: change the snapshot") + +if edit_rel: + p = Path(edit_rel) + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("worker-edit-uncommitted\n", encoding="utf-8") + +if orphan: + tree = git("write-tree") + tree_sha = tree.stdout.strip() + made = git("commit-tree", tree_sha, "-m", "orphan head") + sha = made.stdout.strip() + if sha: + git("reset", "--soft", sha) + +if write_stream and stores_path: + spec = importlib.util.spec_from_file_location("task_launch_stores", stores_path) + stores = importlib.util.module_from_spec(spec) + spec.loader.exec_module(stores) + cwd = os.getcwd() + if exe == "claude": + home = os.environ.get("HOME") or "" + root = Path(home) / ".claude" / "projects" + slug = cwd.replace("/", "-").replace(".", "-") + stores.build_claude_store( + root, slug, base_timestamp=epoch, cwd=cwd, + session_id=stream_id, model=ran_model, effort="high", + marker="LAUNCH-FAKE", + ) + elif exe == "codex": + home = os.environ.get("CODEX_HOME") or str(Path(os.environ.get("HOME", "")) / ".codex") + stores.build_codex_store( + Path(home), base_timestamp=epoch, cwd=cwd, + session_id=stream_id, model=ran_model, effort="high", + marker="LAUNCH-FAKE", + ) + if over_out: + for p in Path(home).rglob("rollout-*.jsonl"): + extra = { + "timestamp": "2023-11-14T22:13:20.000Z", + "type": "event_msg", + "payload": { + "type": "token_count", + "info": { + "total_token_usage": { + "input_tokens": over_out, + "cached_input_tokens": 0, + "output_tokens": over_out, + "reasoning_output_tokens": 0, + "total_tokens": over_out, + } + }, + }, + } + with p.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(extra) + "\n") + elif exe == "grok": + home = os.environ.get("GROK_HOME") or str(Path(os.environ.get("HOME", "")) / ".grok") + stores.build_grok_store( + Path(home), cwd, base_timestamp=epoch, session_id=stream_id, + model=ran_model, marker="LAUNCH-FAKE", + head_commit=head_commit or None, + git_root_dir=cwd, + grok_home=home, + ) + +note = os.environ.get("TASK_LAUNCH_NOTE") +if note == "": + note = None +wrapper = os.environ.get("TASK_LAUNCH_WRAPPER") or "" + +def envelope_obj(): + env = { + "job": job, + "status": status, + "verdict": None if verdict == "null" else verdict, + "counts": {"p1": 0, "p2": 0, "p3": 0, "opinions": 0}, + "findings": [], + "artifacts": {}, + "spend": {"harness": exe if sys.argv else "codex", + "total": 0, "out": 0, "runs": 1}, + "stamp": {"ref": "harness-placeholder", "started": None, "ended": None}, + "note": note, + } + if envelope_commit: + env["commit"] = {"subject": "dispatch: pin U10", "body": "why"} + return env + +# U10 knobs: TASK_LAUNCH_WRAPPER=claude|codex|grok|plain emits each +# documented stdout/file shape so the E2E path is provable without a +# token. Default empty/plain keeps today's last-JSON-object scan. +if wrapper == "claude": + field = os.environ.get("TASK_LAUNCH_WRAPPER_FIELD") or "structured_output" + subtype = os.environ.get("TASK_LAUNCH_WRAPPER_SUBTYPE") or "success" + is_error = os.environ.get("TASK_LAUNCH_WRAPPER_IS_ERROR") == "1" + usage_mode = os.environ.get("TASK_LAUNCH_WRAPPER_USAGE") or "tokens" + wrap = { + "type": "result", + "subtype": subtype, + "is_error": is_error, + "session_id": session_id or stream_id, + } + if usage_mode == "empty": + wrap["usage"] = {} + elif usage_mode == "zero": + wrap["usage"] = {"input_tokens": 0, "output_tokens": 0} + wrap["total_cost_usd"] = 0.0 + elif usage_mode == "cached": + wrap["usage"] = { + "input_tokens": 12, + "output_tokens": 3000, + "cache_read_input_tokens": 480000, + "cache_creation_input_tokens": 9000, + } + wrap["total_cost_usd"] = 0.5 + else: + wrap["usage"] = {"input_tokens": 10, "output_tokens": 4} + wrap["total_cost_usd"] = 0.001 + env = envelope_obj() + decoy_note = os.environ.get("TASK_LAUNCH_WRAPPER_DECOY") + if decoy_note: + # Nested sibling before structured_output: a walk that takes the + # first nine-key dict is not reading the documented field. + alt = envelope_obj() + alt["note"] = decoy_note + wrap["alt"] = alt + if field == "structured_output": + wrap["structured_output"] = env + wrap["result"] = "ok" + elif field == "result": + wrap["result"] = json.dumps(env) + else: + wrap["result"] = os.environ.get("TASK_LAUNCH_WRAPPER_RESULT") or ( + "max turns reached" + ) + sys.stdout.write(json.dumps(wrap) + "\n") +elif wrapper == "codex": + env = envelope_obj() + field = os.environ.get("TASK_LAUNCH_WRAPPER_FIELD") or "file" + agent_note = os.environ.get("TASK_LAUNCH_WRAPPER_AGENT_NOTE") or "" + out_path = last_message + events = [ + {"type": "thread.started", "thread_id": session_id or stream_id}, + {"type": "turn.started"}, + ] + if field in ("agent_message", "both"): + agent_env = envelope_obj() + if agent_note: + agent_env["note"] = agent_note + events.append({ + "type": "item.completed", + "item": {"type": "agent_message", "text": json.dumps(agent_env)}, + }) + else: + events.append({ + "type": "item.completed", + "item": {"type": "reasoning", "text": "working"}, + }) + for ev in events: + sys.stdout.write(json.dumps(ev) + "\n") + if field == "stdout": + # Bare nine-key object after NDJSON with no agent_message and + # no -o file — the pre-U10 stdout fallback D-ENV-1 keeps. + sys.stdout.write(json.dumps(env) + "\n") + elif field in ("file", "both"): + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(env) + "\n", encoding="utf-8") +elif wrapper == "grok": + env = envelope_obj() + invalid = os.environ.get("TASK_LAUNCH_WRAPPER_INVALID") or "" + if invalid == "extra": + env["transcript"] = "the whole conversation" + elif invalid == "missing": + env.pop("note", None) + elif invalid == "counts-empty": + env["counts"] = {} + elif invalid == "counts-p1-string": + env["counts"]["p1"] = "lots" + elif invalid == "findings-str": + env["findings"] = ["not a finding"] + elif invalid == "findings-int": + env["findings"] = [42] + elif invalid == "artifacts-int": + env["artifacts"] = {"plan": 1} + elif invalid == "artifacts-obj": + env["artifacts"] = {"plan": {"inline": "..."}} + elif invalid == "commit-null": + env["commit"] = None + elif invalid == "stamp-extra": + env["stamp"]["extra"] = "x" + elif invalid == "invalid-approve": + env["status"] = "invalid" + env["verdict"] = "approve" + narration = os.environ.get("TASK_LAUNCH_WRAPPER_NARRATION") or "" + if narration: + sys.stdout.write(narration) + decoy_first = os.environ.get("TASK_LAUNCH_WRAPPER_DECOY_FIRST") + if decoy_first: + # Fully-valid decoy BEFORE the harness object (D-ENV-2: two + # fully-valid objects, the last wins; a first-valid reader + # would take this one). + first = envelope_obj() + first["note"] = decoy_first + first.pop("commit", None) + sys.stdout.write(json.dumps(first) + "\n") + sys.stdout.write(json.dumps(env) + "\n") + # Trailing scan-shaped decoy AFTER the harness object (feature + # scenario "a grok trailing nested-invalid object loses to an + # earlier valid envelope"): nine top-level keys, nested + # counts.p1 a string so last-that-validates must skip it. + decoy_note = os.environ.get("TASK_LAUNCH_WRAPPER_DECOY") + if decoy_note: + decoy = envelope_obj() + decoy["note"] = decoy_note + decoy.pop("commit", None) + decoy["verdict"] = "approve" + decoy["status"] = "ok" + decoy["counts"] = {"p1": "scan", "p2": 0, "p3": 0, "opinions": 0} + sys.stdout.write(json.dumps(decoy) + "\n") +elif stdout_mode == "envelope": + sys.stdout.write(json.dumps(envelope_obj()) + "\n") +elif stdout_mode == "token": + sys.stdout.write(token + "\n") +elif stdout_mode == "prose": + sys.stdout.write("VERDICT: approve\n") +decoy_note = os.environ.get("TASK_LAUNCH_WRAPPER_DECOY") +if decoy_note and wrapper != "grok": + # A trailing nine-key object the last-JSON-object scan would take. + # First-source unwrap must ignore it and use the harness field/file. + # Grok writes its own decoy after the harness object (see above). + decoy = envelope_obj() + decoy["note"] = decoy_note + sys.stdout.write(json.dumps(decoy) + "\n") +sys.stdout.flush() + +if sleep_s > 0: + time.sleep(sleep_s) + +if done_path: + Path(done_path).write_text("COMPLETED\n", encoding="utf-8") + +raise SystemExit(exit_code) +""" + + +class PlantFailed(Exception): + """A fault could not be proven to have landed intact.""" + + +class FakeClock: + """Injected monotonic/sleep seam. Tests assign these onto launch.py.""" + + def __init__(self, start=0.0): + self.now = float(start) + self.sleeps = [] + + def monotonic(self): + return self.now + + def sleep(self, seconds): + seconds = float(seconds) + self.sleeps.append(seconds) + self.now += max(seconds, 0.0) + + +def plant_bytes(path, mutate, *, expect="edit", recognisable=None) -> bytes: + """Rewrite ``path`` through ``mutate``, proving the fault landed intact.""" + path = Path(path) + before = path.read_bytes() + if not before: + raise PlantFailed(f"fixture {path} was already empty before planting") + after = mutate(before) + if not isinstance(after, (bytes, bytearray)): + raise PlantFailed(f"mutate() returned {type(after).__name__}, not bytes") + after = bytes(after) + if after == before: + raise PlantFailed(f"plant changed nothing in {path}") + if not after: + raise PlantFailed(f"plant emptied {path}") + if expect == "edit" and len(after) != len(before): + raise PlantFailed(f"plant claimed an in-place edit of {path}") + if expect == "shrink" and len(after) >= len(before): + raise PlantFailed(f"plant claimed to shrink {path}") + if expect == "grow" and len(after) <= len(before): + raise PlantFailed(f"plant claimed to grow {path}") + if expect not in ("edit", "shrink", "grow"): + raise PlantFailed(f"unknown expect={expect!r}") + if recognisable is not None and not recognisable(after): + raise PlantFailed(f"plant left {path} unrecognisable") + path.write_bytes(after) + landed = path.read_bytes() + if landed != after: + raise PlantFailed(f"plant did not reach disk for {path}") + return before + + +def _absent_launch(): + """Empty-stub stand-in: main returns 0 and writes nothing.""" + mod = types.ModuleType("task_launch_absent") + + def main(argv=None): + return 0 + + mod.main = main + mod.JOBS_PATH = None + mod.monotonic = time.monotonic + mod.sleep = time.sleep + return mod + + +def _absent_record(): + """Empty-stub stand-in: build echoes, validate is a no-op.""" + mod = types.ModuleType("task_record_absent") + + def build(payload=None, **kwargs): + if payload is None: + payload = kwargs + if isinstance(payload, dict): + return dict(payload) + return {} + + def validate(rec): + return rec + + mod.build = build + mod.validate = validate + mod.FIELDS = () + return mod + + +def require_module(test, name): + """Load ``ops/devlane/task/.py``, or an empty stub if it is absent. + + A missing file is not the red. The red is the test's contracted + assertion against a module that does not yet implement the + behaviour. An empty file on disk is wired the same way: missing + ``main`` / ``build`` / ``validate`` become no-ops so the assertion + the test is for is the one that fails. + """ + path = APP / f"{name}.py" + if not path.is_file(): + if name == "launch": + return _absent_launch() + if name == "record": + return _absent_record() + test.fail(f"{name}.py is not present at {path}") + try: + module = support.load(name) + except Exception as exc: + test.fail( + f"{name}.py exists but did not load: " + f"{type(exc).__name__}: {exc}" + ) + if name == "launch" and not hasattr(module, "main"): + module.main = lambda argv=None: 0 + if name == "record": + if not hasattr(module, "build"): + def _echo(payload=None, **kwargs): + if payload is None: + payload = kwargs + return dict(payload) if isinstance(payload, dict) else {} + module.build = _echo + if not hasattr(module, "validate"): + module.validate = lambda rec: rec + if not hasattr(module, "FIELDS"): + module.FIELDS = () + return module + + +def load_path(test, path, name): + path = Path(path) + test.assertTrue(path.is_file(), f"required module is missing: {path}") + spec = importlib.util.spec_from_file_location(name, path) + test.assertIsNotNone(spec, f"could not create an import spec for {path}") + test.assertIsNotNone(spec.loader, f"could not load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def git_env(home: Path) -> dict: + env = {k: v for k, v in os.environ.items() + if not k.startswith("GIT_") and k != "XDG_CONFIG_HOME"} + env.update({ + "HOME": str(home), + "GIT_AUTHOR_NAME": "launch-test", + "GIT_AUTHOR_EMAIL": "launch-test@example.test", + "GIT_COMMITTER_NAME": "launch-test", + "GIT_COMMITTER_EMAIL": "launch-test@example.test", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + }) + return env + + +def claude_slug(cwd: str) -> str: + return cwd.replace("/", "-").replace(".", "-") + + +def residual_entries(value): + if value is None: + return [] + if isinstance(value, str): + return [ln for ln in value.splitlines() if ln.strip()] + return list(value) + + +def changed_entries(value): + return residual_entries(value) + + +def sha256_file(path) -> str: + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def pid_is_alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def kill_if_alive(pid) -> None: + if not pid: + return + with contextlib.suppress(OSError, ProcessLookupError, ValueError): + os.kill(int(pid), 9) + + +class _TempLaunch(unittest.TestCase): + """Throwaway repo on branch ``work``, isolated HOME, fake CLIs.""" + + def setUp(self): + self._td = tempfile.TemporaryDirectory() + self.home = Path(self._td.name) + self.repo = self.home / "repo" + self.bin = self.home / "bin" + self.jobs_root = self.home / "jobs" + self.repo.mkdir() + self.bin.mkdir() + self.jobs_root.mkdir() + + self._orig_env = { + k: os.environ.get(k) for k in ( + "PATH", "HOME", "XDG_CONFIG_HOME", "XDG_STATE_HOME", + "WF_AGENT", "DISPATCH_JOBS", "DISPATCH_STREAM_GRACE", + "DISPATCH_TIMEOUT", "TASK_LAUNCH_WITNESS", "TASK_LAUNCH_DONE", + "TASK_LAUNCH_SLEEP", "TASK_LAUNCH_STDOUT", + "TASK_LAUNCH_TOKEN", "TASK_LAUNCH_JOB", + "TASK_LAUNCH_VERDICT", "TASK_LAUNCH_STATUS", + "TASK_LAUNCH_ENVELOPE_COMMIT", "TASK_LAUNCH_RAN_MODEL", + "TASK_LAUNCH_STREAM_EPOCH", "TASK_LAUNCH_STORES", + "TASK_LAUNCH_WRITE_STREAM", "TASK_LAUNCH_IGNORE_SESSION", + "TASK_LAUNCH_STREAM_ID", "TASK_LAUNCH_COMMIT", + "TASK_LAUNCH_EDIT", "TASK_LAUNCH_ORPHAN", + "TASK_LAUNCH_HEAD_COMMIT", "TASK_LAUNCH_EXIT", + "TASK_LAUNCH_OVER_OUT", "TASK_LAUNCH_GRANDCHILD", + "TASK_LAUNCH_WRAPPER", "TASK_LAUNCH_WRAPPER_FIELD", + "TASK_LAUNCH_WRAPPER_SUBTYPE", + "TASK_LAUNCH_WRAPPER_IS_ERROR", + "TASK_LAUNCH_WRAPPER_INVALID", + "TASK_LAUNCH_WRAPPER_NARRATION", + "TASK_LAUNCH_WRAPPER_RESULT", + "TASK_LAUNCH_WRAPPER_DECOY", + "TASK_LAUNCH_WRAPPER_DECOY_FIRST", + "TASK_LAUNCH_WRAPPER_USAGE", + "TASK_LAUNCH_WRAPPER_AGENT_NOTE", "TASK_LAUNCH_NOTE", + "CLICOLOR_FORCE", "FORCE_COLOR", + "NO_COLOR", "CODEX_HOME", "GROK_HOME", "CLAUDE_CONFIG_DIR", + ) + } + self._saved_git = {k: os.environ[k] for k in list(os.environ) + if k.startswith("GIT_")} + for k in list(self._saved_git): + del os.environ[k] + os.environ.pop("XDG_CONFIG_HOME", None) + os.environ.pop("CLICOLOR_FORCE", None) + os.environ.pop("FORCE_COLOR", None) + os.environ.pop("CODEX_HOME", None) + os.environ.pop("GROK_HOME", None) + os.environ.pop("CLAUDE_CONFIG_DIR", None) + os.environ.pop("TASK_LAUNCH_RAN_MODEL", None) + os.environ.pop("TASK_LAUNCH_STORES", None) + os.environ.pop("DISPATCH_STREAM_GRACE", None) + os.environ.pop("DISPATCH_TIMEOUT", None) + + os.environ["HOME"] = str(self.home) + os.environ["PATH"] = str(self.bin) + os.pathsep + os.environ.get( + "PATH", "" + ) + os.environ["WF_AGENT"] = AGENT + os.environ["DISPATCH_JOBS"] = str(self.jobs_root) + os.environ["TASK_LAUNCH_STREAM_EPOCH"] = str(STREAM_EPOCH) + os.environ["TASK_LAUNCH_WRITE_STREAM"] = "1" + os.environ["TASK_LAUNCH_STDOUT"] = "envelope" + os.environ["TASK_LAUNCH_VERDICT"] = "approve" + + self.env = git_env(self.home) + self._git("init", "-b", "work") + self._git("config", "user.name", "launch-test") + self._git("config", "user.email", "launch-test@example.test") + self._git("config", "commit.gpgsign", "false") + + jobs = json.loads(JOBS_PATH.read_text(encoding="utf-8")) + jobs["withheld-whole"] = { + "adapter": "harness", + "deliverable": "fixture for history-vs-withheld", + "role": "read", + "snapshot": "whole", + "withheld": ["secret.py"], + "prompt": "Read {ref}. Aim at: {scope}", + "constraints": ["read only"], + } + jobs["fileset-job"] = { + "adapter": "harness", + "deliverable": "fixture for mode-unavailable", + "role": "read", + "snapshot": "fileset", + "prompt": "Read {ref}. Aim at: {scope}", + "constraints": ["read only"], + } + jobs["needs-into"] = { + "adapter": "harness", + "deliverable": "fixture for template into/base/diff", + "role": "read", + "snapshot": "whole", + "prompt": ( + "into={into} base={base} diff={diff} ref={ref} " + "Aim at: {scope}" + ), + "constraints": ["read only"], + } + jobs["needs-hole"] = { + "adapter": "harness", + "deliverable": "fixture for a missing template value", + "role": "read", + "snapshot": "whole", + "prompt": "this names {not_a_slot} and {scope}", + "constraints": ["read only"], + } + self.jobs_file = self.repo / ".dev" / "app" / "task" / "jobs.json" + self.jobs_file.parent.mkdir(parents=True, exist_ok=True) + self.jobs_file.write_text( + json.dumps(jobs, indent=2) + "\n", encoding="utf-8" + ) + self._write("alpha.py", "alpha v1\n") + self._write("README.md", "fixture tree\n") + self.root_sha = self._commit("root") + self._write("alpha.py", "alpha v2\n") + self.mid_sha = self._commit("mid") + self._write("alpha.py", "alpha v3\n") + self.ref = self._commit("tip") + self.lineage = "work" + + self._git("checkout", "-b", "side", self.root_sha) + self._write("side.py", "off the lineage\n") + self.side_sha = self._commit("side") + self._git("checkout", "work") + + self.start_witness = self.home / "started.json" + self.witness = self.home / "witness.json" + self.grandchild = self.home / "grandchild.pid" + os.environ["TASK_LAUNCH_WITNESS"] = str(self.witness) + os.environ.pop("TASK_LAUNCH_DONE", None) + os.environ.pop("TASK_LAUNCH_SLEEP", None) + os.environ.pop("TASK_LAUNCH_IGNORE_SESSION", None) + os.environ.pop("TASK_LAUNCH_STREAM_ID", None) + os.environ.pop("TASK_LAUNCH_COMMIT", None) + os.environ.pop("TASK_LAUNCH_EDIT", None) + os.environ.pop("TASK_LAUNCH_ORPHAN", None) + os.environ.pop("TASK_LAUNCH_HEAD_COMMIT", None) + os.environ.pop("TASK_LAUNCH_EXIT", None) + os.environ.pop("TASK_LAUNCH_OVER_OUT", None) + os.environ.pop("TASK_LAUNCH_GRANDCHILD", None) + os.environ.pop("TASK_LAUNCH_WRAPPER", None) + os.environ.pop("TASK_LAUNCH_WRAPPER_FIELD", None) + os.environ.pop("TASK_LAUNCH_WRAPPER_SUBTYPE", None) + os.environ.pop("TASK_LAUNCH_WRAPPER_IS_ERROR", None) + os.environ.pop("TASK_LAUNCH_WRAPPER_INVALID", None) + os.environ.pop("TASK_LAUNCH_WRAPPER_NARRATION", None) + os.environ.pop("TASK_LAUNCH_WRAPPER_RESULT", None) + os.environ.pop("TASK_LAUNCH_WRAPPER_DECOY", None) + os.environ.pop("TASK_LAUNCH_WRAPPER_DECOY_FIRST", None) + os.environ.pop("TASK_LAUNCH_WRAPPER_USAGE", None) + os.environ.pop("TASK_LAUNCH_WRAPPER_AGENT_NOTE", None) + os.environ.pop("TASK_LAUNCH_NOTE", None) + os.environ.pop("TASK_LAUNCH_STATUS", None) + os.environ.pop("TASK_LAUNCH_ENVELOPE_COMMIT", None) + + (self.home / ".codex").mkdir() + (self.home / ".codex" / "auth.json").write_text("{}\n", encoding="utf-8") + (self.home / ".grok").mkdir() + (self.home / ".grok" / "auth.json").write_text("{}\n", encoding="utf-8") + + self._install_cli("claude") + self._install_cli("codex") + self._install_cli("grok") + + # Loaded on first use so a missing launch.py is an empty stub + # whose main is a no-op — the test method's assertion is the red. + self.launch = None + self._saved_jobs_path = None + + def load_launch(self): + if self.launch is not None: + return self.launch + self.launch = require_module(self, "launch") + if hasattr(self.launch, "JOBS_PATH"): + self._saved_jobs_path = self.launch.JOBS_PATH + self.launch.JOBS_PATH = self.jobs_file + return self.launch + + def tearDown(self): + if self.start_witness.is_file(): + with contextlib.suppress(OSError, json.JSONDecodeError, ValueError): + info = json.loads( + self.start_witness.read_text(encoding="utf-8") + ) + kill_if_alive(info.get("pid")) + kill_if_alive(info.get("pgid")) + if self.grandchild.is_file(): + with contextlib.suppress(OSError, ValueError): + kill_if_alive( + self.grandchild.read_text(encoding="utf-8").strip() + ) + if self.launch is not None and self._saved_jobs_path is not None: + self.launch.JOBS_PATH = self._saved_jobs_path + for k, v in self._orig_env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + for k in list(os.environ): + if k.startswith("GIT_"): + del os.environ[k] + os.environ.update(self._saved_git) + self._td.cleanup() + + def _git(self, *args, repo=None): + r = subprocess.run( + ["git", *args], cwd=repo or self.repo, env=self.env, + capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError( + f"git {args} failed ({r.returncode}): {r.stderr}") + return r + + def _write(self, rel, content): + p = self.repo / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content, encoding="utf-8") + return p + + def _commit(self, msg): + self._git("add", "-A") + self._git("commit", "-m", msg) + return self._git("rev-parse", "HEAD").stdout.strip() + + def _install_cli(self, name): + dest = self.bin / name + script = ( + _FAKE_CLI + .replace("@@START_WITNESS@@", json.dumps(str(self.start_witness))) + .replace("@@RAN_MODEL@@", json.dumps(RAN_MODEL)) + .replace("@@STORES_PATH@@", json.dumps(str(STORES_PATH))) + ) + dest.write_text(script, encoding="utf-8") + dest.chmod(dest.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP + | stat.S_IXOTH) + return dest + + def set_grace(self, seconds): + """External seam: DISPATCH_STREAM_GRACE, default 120 seconds.""" + os.environ["DISPATCH_STREAM_GRACE"] = str(seconds) + + def attach_clock(self): + """Injected clock/poller seam: launch.monotonic and launch.sleep.""" + launch = self.load_launch() + clock = FakeClock() + launch.monotonic = clock.monotonic + launch.sleep = clock.sleep + return clock + + def force_id(self, job_id): + launch = self.load_launch() + launch.mint_id = lambda *a, **k: job_id + return job_id + + def run_main(self, argv, *, cwd=None): + launch = self.load_launch() + out, err = io.StringIO(), io.StringIO() + cwd = cwd or self.repo + old = os.getcwd() + try: + os.chdir(cwd) + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + try: + code = launch.main(list(argv)) + except SystemExit as exc: + code = int(exc.code) if exc.code is not None else 0 + finally: + os.chdir(old) + if code is None: + code = 0 + return int(code), out.getvalue(), err.getvalue() + + def argv_for( + self, + job="plan", + *, + harness="grok", + model=REQUESTED_MODEL, + ref=None, + lineage=None, + stage=None, + scope="pin the launcher", + extra=(), + ): + argv = [job, "--harness", harness] + if model is not None: + argv.extend(["--model", model]) + argv.extend(["--ref", ref or self.ref]) + if lineage is not None: + argv.extend(["--lineage", lineage]) + if stage is not None: + argv.extend(["--stage", stage]) + if scope is not None: + argv.extend(["--scope", scope]) + argv.extend(list(extra)) + return argv + + def dispatch(self, argv=None, **kwargs): + if argv is None: + argv = self.argv_for(**kwargs) + return self.run_main(argv) + + def combined(self, out, err): + return (out or "") + (err or "") + + def assert_refusal(self, code, out, err, *, ident, phrases): + text = self.combined(out, err) + self.assertEqual( + code, REFUSAL_EXIT, + f"{ident} refuses with exit 3, got {code}: {text!r}", + ) + lower = text.lower() + self.assertIn("expected", lower, f"{ident} names expected: {text!r}") + self.assertIn("found", lower, f"{ident} names found: {text!r}") + self.assertIn("satisfy", lower, f"{ident} names satisfy: {text!r}") + for phrase in phrases: + self.assertIn( + phrase.lower(), lower, + f"{ident} text must name {phrase!r}: {text!r}", + ) + return text + + def assert_not_started(self): + self.assertFalse( + self.start_witness.is_file(), + "refusal must not start a child; baked start-witness exists: " + + (self.start_witness.read_text(encoding="utf-8") + if self.start_witness.is_file() else ""), + ) + + def job_dirs(self): + if not self.jobs_root.exists(): + return [] + return [p for p in self.jobs_root.iterdir() if p.is_dir()] + + def record_files(self): + root = self.repo / ".dev" / "records" / "dispatches" + if not root.exists(): + return [] + return sorted(root.glob("*.json")) + + def the_job_dir(self): + dirs = self.job_dirs() + self.assertEqual( + len(dirs), 1, + f"expected exactly one job dir, found {len(dirs)}: {dirs}", + ) + return dirs[0] + + def the_record_path(self): + files = self.record_files() + self.assertEqual( + len(files), 1, + f"expected exactly one record file, found {len(files)}: {files}", + ) + return files[0] + + def read_record(self, path=None): + path = path or self.the_record_path() + self.assertTrue(path.is_file(), f"record missing: {path}") + body = path.read_text(encoding="utf-8") + self.assertTrue(body.strip(), "record file is empty") + data = json.loads(body) + self.assertIsInstance(data, dict) + return data + + def read_start_witness(self): + self.assertTrue( + self.start_witness.is_file(), + "the harness CLI must have been launched (start-witness missing)", + ) + data = json.loads(self.start_witness.read_text(encoding="utf-8")) + self.assertIsInstance(data.get("argv"), list) + self.assertTrue(data["argv"], "recorded argv is empty") + return data + + def read_witness(self): + self.assertTrue( + self.witness.is_file(), + "the harness CLI must have been launched (witness missing)", + ) + data = json.loads(self.witness.read_text(encoding="utf-8")) + self.assertIsInstance(data.get("argv"), list) + self.assertTrue(data["argv"], "recorded argv is empty") + return data + + def launch_ok(self, argv=None, **kwargs): + code, out, err = self.dispatch(argv, **kwargs) + text = self.combined(out, err) + self.assertNotEqual( + code, REFUSAL_EXIT, + f"happy path must not refuse (exit 3): {text!r}", + ) + self.assertEqual(code, 0, f"happy path exits 0, got {code}: {text!r}") + rec = self.read_record() + self.assertEqual(rec.get("status"), "closed") + return rec, self.read_witness(), out, err + + def snapshot_of(self, rec=None): + rec = rec or self.read_record() + snap = rec.get("snapshot") if isinstance(rec.get("snapshot"), dict) else {} + root = snap.get("root") + self.assertIsInstance(root, str) + self.assertTrue(root.strip(), "snapshot.root must name a directory") + path = Path(root) + self.assertTrue(path.is_dir(), f"snapshot.root is not a dir: {root!r}") + return path + + def refs_map(self, repo=None): + raw = self._git( + "for-each-ref", "--format=%(refname) %(objectname)", + repo=repo, + ).stdout.splitlines() + out = {} + for line in raw: + if not line.strip(): + continue + name, sha = line.split(" ", 1) + out[name] = sha + self.assertTrue(out, "fixture must have at least one ref") + return out + + def fetch_head_bytes(self): + p = self.repo / ".git" / "FETCH_HEAD" + if not p.exists(): + return None + return p.read_bytes() + + def porcelain(self, repo=None): + return self._git( + "status", "--porcelain=v1", "-uall", repo=repo, + ).stdout + + def index_blob(self, repo=None): + return self._git("ls-files", "-s", repo=repo).stdout + + def worktree_bytes(self, repo=None): + """Index + porcelain + untracked file bytes, for exact delta.""" + root = Path(repo or self.repo) + return { + "porcelain": self.porcelain(repo=root), + "index": self.index_blob(repo=root), + } + + def worktree_paths(self, repo=None): + raw = self._git("worktree", "list", "--porcelain", repo=repo).stdout + paths = [] + for line in raw.splitlines(): + if line.startswith("worktree "): + paths.append(line.split(" ", 1)[1]) + self.assertTrue(paths, "worktree list must name the invoking repo") + return paths + + def git_dir_mentions(self, snapshot: Path, needle: bytes) -> list: + git = snapshot / ".git" + hits = [] + if not git.exists(): + return hits + for p in git.rglob("*"): + if p.is_symlink(): + try: + target = os.fsencode(os.readlink(p)) + except OSError: + continue + if needle in target: + hits.append(str(p.relative_to(git)) + " (symlink)") + continue + if not p.is_file(): + continue + try: + body = p.read_bytes() + except OSError: + continue + if needle in body: + hits.append(str(p.relative_to(git))) + return hits + + def assert_objects_not_shared(self, snapshot: Path): + snap_obj = snapshot / ".git" / "objects" + src_obj = self.repo / ".git" / "objects" + self.assertTrue(snap_obj.exists(), "snapshot has an object store") + self.assertFalse( + snap_obj.is_symlink(), + ".git/objects must not be a symlink to the invoking store", + ) + git = snapshot / ".git" + self.assertFalse( + git.is_file(), + "snapshot .git is a file (a worktree pointer), not a repo", + ) + if snap_obj.is_dir() and src_obj.is_dir(): + self.assertFalse( + os.path.samefile(snap_obj, src_obj), + "snapshot objects dir is the invoking objects dir", + ) + src_inodes = set() + for p in src_obj.rglob("*"): + if p.is_symlink() or not p.is_file(): + continue + st = p.stat() + src_inodes.add((st.st_dev, st.st_ino)) + for p in snap_obj.rglob("*"): + self.assertFalse( + p.is_symlink(), + f"snapshot object is a symlink: {p}", + ) + if not p.is_file(): + continue + st = p.stat() + self.assertNotIn( + (st.st_dev, st.st_ino), src_inodes, + f"hardlinked object {p} shares inode with the invoking store", + ) + + def dispatch_reflog(self, job_id): + r = subprocess.run( + ["git", "reflog", "show", f"refs/dispatch/{job_id}"], + cwd=self.repo, env=self.env, capture_output=True, text=True, + ) + return r.stdout + + def dead_pid(self): + proc = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(60)"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + pid = proc.pid + proc.kill() + proc.wait() + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + return pid + + def plant_new_file(self, path, content, *, must_contain=None): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + data = content.encode("utf-8") if isinstance(content, str) else content + self.assertTrue(data, "planted file must not be empty") + path.write_bytes(data) + landed = path.read_bytes() + self.assertEqual(landed, data, "plant did not reach disk") + self.assertGreater(len(landed), 0) + if must_contain is not None: + needle = (must_contain.encode("utf-8") + if isinstance(must_contain, str) else must_contain) + self.assertIn(needle, landed, "plant missing its marker") + return landed diff --git a/ops/devlane/dispatch/tests/support.py b/ops/devlane/dispatch/tests/support.py new file mode 100644 index 0000000..f05d68b --- /dev/null +++ b/ops/devlane/dispatch/tests/support.py @@ -0,0 +1,37 @@ +"""Load a task-app module by path, the way the other suites do. + +The dev-lane apps are standalone scripts, not an installed package: +`ops/devlane/task/envelope.py` has no importable dotted name. Every suite +here loads it from its path, so a test file can be run on its own +(`python3 ops/devlane/task/tests/test_envelope.py`) without a sys.path +ceremony repeated in each one. +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +APP = Path(__file__).resolve().parents[1] +REPO = APP.parents[2] + +# The app dir goes on sys.path so its standalone scripts can import one +# another plainly — `import envelope` inside verify.py. This mirrors +# ops/devlane/workflow/tests/support.py, which does the same for the same +# reason. Without it a module loaded BY PATH cannot resolve its +# siblings, and the failure looks like a missing dependency rather +# than a missing path entry. +if str(APP) not in sys.path: + sys.path.insert(0, str(APP)) + + +def load(name): + """Import /.py under a task_ prefix and return it.""" + path = APP / f"{name}.py" + spec = importlib.util.spec_from_file_location(f"task_{name}", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module diff --git a/ops/devlane/dispatch/tests/test_launch_claude_readable.py b/ops/devlane/dispatch/tests/test_launch_claude_readable.py new file mode 100644 index 0000000..5797eff --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_claude_readable.py @@ -0,0 +1,189 @@ +"""Named {inputs} and {diff} paths must be readable by a claude child. + +Written from CONTRACT.md §Dispatch Verbs lines 65-66 and Template +values lines 138-139. + + --input PATH copies a file into the job directory's in/ and names it + to the template as {inputs}; {diff} names the diff the launcher writes. + +Observed at record 20260828T205636Z-adjudicate-claude-44f652 +(repo/ci-burn): the claude child reported in/ as 'outside this session's +allowed working directories', refused every read of {inputs}, and wrote +no ruling. Claude's working directory is snapshot/; in/ and the +launcher-written diff live in the job directory beside it. The child's +allowed working directories are snapshot/ plus every --add-dir. + + C1 every {inputs} path on a claude dispatch is readable by the child + C2 the {diff} path on a claude dispatch is readable by the child +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import launch_support as ls + + +def _add_dirs(argv, cwd): + """Every --add-dir value, resolved the way the child would see it.""" + found = [] + argv = [str(a) for a in argv] + i = 0 + while i < len(argv): + a = argv[i] + raw = None + if a == "--add-dir" and i + 1 < len(argv): + raw = argv[i + 1] + i += 2 + elif a.startswith("--add-dir="): + raw = a.split("=", 1)[1] + i += 1 + else: + i += 1 + continue + if not raw: + continue + p = Path(raw) + if not p.is_absolute(): + p = Path(cwd) / p + found.append(p.resolve()) + return found + + +def _under(path, root): + if path == root: + return True + try: + path.relative_to(root) + except ValueError: + return False + return True + + +class ClaudeChildCanReadNamedTemplatePaths(ls._TempLaunch): + """C1 C2 / contract verbs {inputs} and template values {diff}.""" + + def setUp(self): + super().setUp() + jobs = json.loads(self.jobs_file.read_text(encoding="utf-8")) + jobs["names-inputs"] = { + "adapter": "harness", + "deliverable": "fixture: {inputs} named on its own line", + "role": "read", + "snapshot": "whole", + "prompt": "inputs={inputs}\nAim at: {scope}", + "constraints": ["read only"], + } + jobs["names-diff"] = { + "adapter": "harness", + "deliverable": "fixture: {diff} named on its own line", + "role": "read", + "snapshot": "whole", + "prompt": "diff={diff}\nAim at: {scope}", + "constraints": ["read only"], + } + self.jobs_file.write_text( + json.dumps(jobs, indent=2) + "\n", encoding="utf-8", + ) + self.ref = self._commit("fixture jobs that name inputs and diff") + + def _prompt_field(self, prompt, key): + prefix = key + "=" + for line in prompt.splitlines(): + if line.startswith(prefix): + return line[len(prefix):] + self.fail(f"rendered brief has no {key}= line:\n{prompt}") + + def _assert_claude_child(self, witness): + argv = [str(a) for a in witness["argv"]] + self.assertTrue(argv, "child argv is empty") + self.assertEqual( + Path(argv[0]).name, "claude", + f"this pin is for a claude child, argv[0]={argv[0]!r}", + ) + self.assertTrue(witness.get("cwd"), "child cwd is recorded") + + def _assert_readable_by_claude_child(self, path, witness, *, slot): + cwd = witness["cwd"] + target = Path(path) + if not target.is_absolute(): + target = Path(cwd) / target + target = target.resolve() + self.assertTrue( + target.is_file(), + f"{{{slot}}} must name a file the child can open, got {path!r}", + ) + roots = [Path(cwd).resolve(), *_add_dirs(witness["argv"], cwd)] + self.assertTrue( + any(_under(target, root) for root in roots), + f"claude child cannot read {{{slot}}} path {target}: " + f"outside this session's allowed working directories " + f"(cwd={cwd!r}, add-dir={[str(r) for r in roots[1:]]})", + ) + + def test_every_inputs_path_is_readable_by_the_claude_child(self): + first = self.home / "src" / "review-beta.md" + second = self.home / "src" / "review-alpha.md" + self.plant_new_file(first, "# beta-review\n", must_contain="beta-review") + self.plant_new_file( + second, "# alpha-review\n", must_contain="alpha-review", + ) + rec, witness, *_ = self.launch_ok(self.argv_for( + job="names-inputs", harness="claude", stage="adjudicate", + extra=["--input", str(first), "--input", str(second)], + )) + self.assertEqual(rec["job"], "names-inputs") + self._assert_claude_child(witness) + prompt = witness.get("stdin") or "" + self.assertTrue(prompt, "claude receives the rendered brief on stdin") + named = self._prompt_field(prompt, "inputs") + paths = named.split() + self.assertEqual( + len(paths), 2, + f"both --input copies must be named as {{inputs}}, got {named!r}", + ) + job_in = (self.the_job_dir() / "in").resolve() + self.assertTrue(job_in.is_dir(), "copies land in the job directory in/") + seen = [] + for raw in paths: + copy = Path(raw).resolve() + self.assertTrue( + copy.is_file(), + f"{{inputs}} names {raw!r} which is not a file", + ) + self.assertEqual( + copy.parent, job_in, + f"{{inputs}} names the copy under in/, got {copy}", + ) + seen.append(copy.name) + self.assertEqual( + seen, ["review-beta.md", "review-alpha.md"], + "{inputs} is the copies in the order given", + ) + self.assertIn(b"beta-review", (job_in / "review-beta.md").read_bytes()) + self.assertIn(b"alpha-review", (job_in / "review-alpha.md").read_bytes()) + for raw in paths: + self._assert_readable_by_claude_child( + Path(raw).resolve(), witness, slot="inputs", + ) + + def test_the_diff_path_is_readable_by_the_claude_child(self): + rec, witness, *_ = self.launch_ok(self.argv_for( + job="names-diff", harness="claude", stage="review", + scope="pin the launcher-written diff", + )) + self._assert_claude_child(witness) + prompt = witness.get("stdin") or "" + self.assertTrue(prompt, "claude receives the rendered brief on stdin") + named = self._prompt_field(prompt, "diff").strip() + job_dir = self.the_job_dir() + self.assertTrue( + named, + "{diff} must name the diff the launcher writes, not a hole " + f"in the brief; got {named!r} in:\n{prompt}\n" + f"job dir={[p.name for p in job_dir.iterdir()]}", + ) + self.assertNotIn(" ", named, f"{{diff}} is one path, got {named!r}") + self.assertEqual(rec["job"], "names-diff") + self._assert_readable_by_claude_child(named, witness, slot="diff") diff --git a/ops/devlane/dispatch/tests/test_launch_collect.py b/ops/devlane/dispatch/tests/test_launch_collect.py new file mode 100644 index 0000000..adbf65b --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_collect.py @@ -0,0 +1,145 @@ +"""Collect: ancestry, changed_paths, residual_paths, refs/dispatch. + +Written from CONTRACT.md §Dispatch Collect. Plan items (r)(s). + + C1 worker HEAD that does not descend from ref_sha → invalid, no fetch + C2 committed-and-clean → changed_paths from ref_sha..head, residual empty + C3 edited-only → changed_paths empty, residual non-empty + C4 read role: head equals ref_sha; residual recorded not judged +""" + +from __future__ import annotations + +import json +import os + +import launch_support as ls + + +class OffLineageHeadIsInvalidAndNotFetched(ls._TempLaunch): + """C1 / plan (r) / contract off-lineage-head.""" + + def test_an_orphan_head_is_invalid_and_does_not_create_refs_dispatch(self): + os.environ["TASK_LAUNCH_ORPHAN"] = "1" + os.environ["TASK_LAUNCH_VERDICT"] = "null" + before_refs = self.refs_map() + _code, out, err = self.dispatch(self.argv_for( + job="implement", harness="grok", stage="code", + scope="python3 -m unittest", + )) + rec = self.read_record() + blob = (self.combined(out, err) + json.dumps(rec)).lower() + envelope = (rec.get("result") or {}).get("envelope") or {} + self.assertEqual(envelope.get("status"), "invalid") + self.assertIn("off-lineage-head", blob) + after_refs = self.refs_map() + dispatch_refs = [ + n for n in after_refs if n.startswith("refs/dispatch/") + ] + self.assertEqual( + dispatch_refs, [], + "off-lineage-head must not fetch refs/dispatch", + ) + extra = { + n for n in (set(after_refs) - set(before_refs)) + if n.startswith("refs/dispatch/") + } + self.assertEqual(extra, set()) + self.assertNotIn(f"refs/dispatch/{rec['id']}", after_refs) + + +class CommittedCleanVersusEditedOnly(ls._TempLaunch): + """C2 / C3 / plan (s).""" + + def test_a_worker_that_committed_and_left_a_clean_tree(self): + os.environ["TASK_LAUNCH_COMMIT"] = "worker.py" + os.environ["TASK_LAUNCH_VERDICT"] = "null" + rec, *_ = self.launch_ok(self.argv_for( + job="implement", harness="grok", stage="code", + scope="python3 -m unittest", + )) + result = rec["result"] + self.assertIsInstance(result, dict) + changed = ls.changed_entries(result.get("changed_paths")) + residual = ls.residual_entries(result.get("residual_paths")) + self.assertGreater( + len(changed), 0, + "committed work produces a non-empty ref_sha..head name-only diff", + ) + self.assertIn("worker.py", " ".join(changed)) + self.assertEqual( + residual, [], + f"a clean snapshot tree has empty residual_paths, got {residual!r}", + ) + self.assertEqual( + self.porcelain(repo=self.snapshot_of(rec)), "", + ) + self.assertTrue(result.get("head")) + self.assertNotEqual(result["head"], rec["snapshot"]["ref_sha"]) + # Ancestry: head descends from ref_sha. _git raises on non-zero, + # so returning is the proof. + self._git( + "merge-base", "--is-ancestor", + rec["snapshot"]["ref_sha"], result["head"], + repo=self.snapshot_of(rec), + ) + self.assertEqual( + self.refs_map()[f"refs/dispatch/{rec['id']}"], result["head"], + ) + + def test_a_worker_that_only_edited_leaves_residuals_and_no_changed_paths( + self): + os.environ["TASK_LAUNCH_EDIT"] = "scratch.txt" + os.environ["TASK_LAUNCH_VERDICT"] = "null" + rec, *_ = self.launch_ok(self.argv_for( + job="implement", harness="grok", stage="code", + scope="python3 -m unittest", + )) + result = rec["result"] + changed = ls.changed_entries(result.get("changed_paths")) + residual = ls.residual_entries(result.get("residual_paths")) + self.assertEqual( + changed, [], + f"no commit means empty changed_paths, got {changed!r}", + ) + self.assertGreater( + len(residual), 0, + "an uncommitted edit is residual_paths, not changed_paths", + ) + self.assertTrue( + any("scratch.txt" in str(item) for item in residual), + residual, + ) + self.assertEqual(result.get("head"), rec["snapshot"]["ref_sha"]) + self.assertNotIn( + f"refs/dispatch/{rec['id']}", + # a write role with no new commit may still fetch HEAD==ref; + # the pin is changed_paths empty. Fetching the same sha is + # allowed; fetching something else is not. + [n for n, sha in self.refs_map().items() + if sha != rec["snapshot"]["ref_sha"] and n.startswith("refs/dispatch/")], + ) + + +class ReadRoleHeadEqualsRef(ls._TempLaunch): + """C4 — read roles: head must equal ref_sha; residual is recorded + not judged.""" + + def test_a_read_role_records_head_equal_to_ref_sha(self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + result = rec["result"] + self.assertEqual(result["head"], rec["snapshot"]["ref_sha"]) + self.assertEqual(result["head"], self.ref) + self.assertNotIn(f"refs/dispatch/{rec['id']}", self.refs_map()) + + def test_a_read_role_with_an_uncommitted_edit_records_residual_not_invalid( + self): + os.environ["TASK_LAUNCH_EDIT"] = "notes.txt" + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + result = rec["result"] + self.assertEqual(result["head"], rec["snapshot"]["ref_sha"]) + residual = ls.residual_entries(result.get("residual_paths")) + self.assertGreater(len(residual), 0) + envelope = result.get("envelope") or {} + self.assertNotEqual(envelope.get("status"), "invalid") + self.assertEqual(rec["status"], "closed") diff --git a/ops/devlane/dispatch/tests/test_launch_envelope_by_construction.py b/ops/devlane/dispatch/tests/test_launch_envelope_by_construction.py new file mode 100644 index 0000000..dee566b --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_envelope_by_construction.py @@ -0,0 +1,958 @@ +"""U10: structured envelopes by construction. + +Authored from `.dev/docs/scratch/harness-research.md` §4 / §2 feature +ending / §6 row U10, D-ENV-1 and D-E2E-1/2, and +`.dev/design/features/dispatch/structured-envelopes.feature`. +Implementation (launch.py, jobs.json, envelope.py) was not read. + +Skeptic 37802b CHANGES applied: pin the allowed envelope (types, +required, nullable verdict, optional commit) over extra denials; grok +first-source and codex file-vs-agent_message order; mapped BDD +scenarios; captured S2 bytes. + +Fake-CLI knobs added in launch_support.py for this unit: +TASK_LAUNCH_WRAPPER=claude|codex|grok|plain, plus +TASK_LAUNCH_WRAPPER_FIELD / _SUBTYPE / _IS_ERROR / _INVALID / +_NARRATION / _RESULT / _DECOY / _DECOY_FIRST / _AGENT_NOTE, +TASK_LAUNCH_NOTE, TASK_LAUNCH_STATUS, TASK_LAUNCH_ENVELOPE_COMMIT. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path + +import launch_support as ls + +ENVELOPES = ls.FIXTURES_DIR / "envelopes" +ENVELOPE_PY = ls.APP.parents[0] / "task" / "envelope.py" +NINE = ( + "job", "status", "verdict", "counts", "findings", + "artifacts", "spend", "stamp", "note", +) +GROK_NARRATION_FIXTURE = "grok-narration-then-object.raw.out" +GROK_S2_FIXTURE = "grok-s2-13f5f2.raw.out" +CLAUDE_MAX_TURNS_FIXTURE = "claude-result-error-max-turns.json" +CODEX_NDJSON_FIXTURE = "codex-ndjson-tail.jsonl" + + +def _cause_reason(rec): + cause = (rec.get("result") or {}).get("cause") + if isinstance(cause, dict): + return cause.get("reason") + return cause + + +def _envelope(rec): + return (rec.get("result") or {}).get("envelope") or {} + + +def _spend(rec): + session = rec.get("session") or {} + return session.get("spend") or {} + + +class _U10Launch(ls._TempLaunch): + def _argv_list(self, witness): + argv = [str(p) for p in witness["argv"]] + self.assertTrue(argv, "witness argv is empty") + return argv + + def _schema_value(self, argv): + self.assertIn( + "--json-schema", argv, + f"--json-schema missing from argv={argv!r}", + ) + idx = argv.index("--json-schema") + self.assertLess( + idx + 1, len(argv), + "--json-schema is present but has no value", + ) + value = str(argv[idx + 1]) + self.assertFalse( + value.startswith("-"), + f"--json-schema value looks like a flag: {value!r}", + ) + return value + + def _loaded_schema(self, value): + if value.lstrip().startswith("{"): + data = json.loads(value) + else: + path = Path(value) + self.assertTrue( + path.is_file(), + f"schema path is missing: {value!r}", + ) + data = json.loads(path.read_text(encoding="utf-8")) + self.assertIsInstance(data, dict) + return data + + def _exported_schema(self): + mod = ls.load_path(self, ENVELOPE_PY, "u10_envelope") + schema = getattr(mod, "ENVELOPE_SCHEMA", None) + self.assertIsInstance( + schema, dict, + "envelope.py must export ENVELOPE_SCHEMA", + ) + return schema + + def _child_schema_reachable(self, witness, value): + self.assertTrue( + witness.get("schema_read"), + "the child must open --json-schema from its own argv; " + f"schema_how={witness.get('schema_how')!r} " + f"schema_path={witness.get('schema_path')!r}", + ) + if value.lstrip().startswith("{"): + digest = hashlib.sha256(value.encode("utf-8")).hexdigest() + self.assertEqual(witness.get("schema_sha"), digest) + return + cwd = Path(witness["cwd"]).resolve() + path = Path(value) + path = (cwd / path).resolve() if not path.is_absolute() else path.resolve() + job_parent = cwd.parent + under_cwd = path == cwd or cwd in path.parents + under_job = path == job_parent or job_parent in path.parents + self.assertTrue( + under_cwd or under_job, + "--json-schema must be a path the sandboxed child can read " + f"(cwd or job dir), not a tempfile outside: value={value!r} " + f"cwd={cwd} job={job_parent}", + ) + text = path.read_text(encoding="utf-8") + digest = hashlib.sha256(text.encode("utf-8")).hexdigest() + self.assertEqual(witness.get("schema_sha"), digest) + + def _resume_witness(self, rec): + self.assertTrue(self.witness.is_file(), "first launch left a witness") + self.witness.unlink() + code, out, err = self.run_main(["resume", rec["id"]]) + self.assertEqual( + code, 0, + f"resume must exit 0: {self.combined(out, err)}", + ) + return self.read_witness() + + def _fixture_bytes(self, name): + path = ENVELOPES / name + self.assertTrue(path.is_file(), f"fixture missing: {path}") + body = path.read_bytes() + self.assertTrue(body, f"fixture empty: {path}") + return body + + def _fixture_text(self, name): + body = self._fixture_bytes(name).decode("utf-8") + self.assertTrue(body.strip(), f"fixture empty: {name}") + return body + + def _raw_out(self): + raw = self.the_job_dir() / "raw.out" + self.assertTrue(raw.is_file(), "raw.out is missing") + data = raw.read_bytes() + self.assertTrue(data, "raw.out is empty") + return raw, data + + def _dispatch_closed(self, **kwargs): + code, out, err = self.dispatch(self.argv_for(**kwargs)) + text = self.combined(out, err) + self.assertNotEqual( + code, ls.REFUSAL_EXIT, + f"finished job must not refuse: {text!r}", + ) + rec = self.read_record() + self.assertEqual(rec.get("status"), "closed", rec) + return rec, text + + +class ArgvAsksForTheEnvelopeByConstruction(_U10Launch): + def test_claude_fresh_argv_carries_json_output_and_json_schema(self): + """Scenario: claude fresh argv asks for json output and the envelope schema""" + rec, witness, *_ = self.launch_ok( + job="plan", harness="claude", stage="plan", + ) + argv = self._argv_list(witness) + self.assertIn("--output-format", argv) + self.assertEqual( + argv[argv.index("--output-format") + 1], "json", + f"claude must ask for json output, argv={argv!r}", + ) + value = self._schema_value(argv) + schema = self._loaded_schema(value) + self.assertEqual(schema.get("type"), "object") + self._child_schema_reachable(witness, value) + self.assertEqual(rec["harness"]["name"], "claude") + + def test_claude_resume_argv_keeps_json_output_and_json_schema(self): + """Scenario: claude resume argv keeps json output and the envelope schema""" + rec, *_ = self.launch_ok(job="plan", harness="claude", stage="plan") + argv = self._argv_list(self._resume_witness(rec)) + self.assertIn("--output-format", argv) + self.assertEqual(argv[argv.index("--output-format") + 1], "json") + self._schema_value(argv) + self.assertTrue("-r" in argv or "--resume" in argv or "-p" in argv + or "--print" in argv) + + def test_grok_fresh_argv_does_not_ask_for_a_schema(self): + """Scenario: grok keeps plain output — `--json-schema` short-circuits the loop + + Measured 2026-08-29 (record 20260829T234838Z-review-grok-66ccb2, + grok 1.0.5): with `--json-schema` grok returned a schema-valid, + empty envelope after one model call and ended the turn. Replay: + fixtures/envelopes/grok-json-schema-short-circuit-66ccb2.raw.out. + """ + rec, witness, *_ = self.launch_ok( + job="plan", harness="grok", stage="plan", + ) + argv = self._argv_list(witness) + self.assertNotIn("--json-schema", argv, f"grok argv={argv!r}") + self.assertIn("--output-format", argv) + self.assertEqual(argv[argv.index("--output-format") + 1], "plain") + self.assertEqual(rec["harness"]["name"], "grok") + + def test_grok_resume_argv_keeps_plain_output(self): + """Scenario: grok resume argv keeps plain output and no schema""" + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + argv = self._argv_list(self._resume_witness(rec)) + self.assertNotIn("--json-schema", argv) + self.assertEqual(argv[argv.index("--output-format") + 1], "plain") + self.assertIn("-r", argv) + self.assertIn( + "--model", argv, + "D-RES-1: resume argv is the fresh argv with the resume " + f"flag substituted; --model must stay: argv={argv!r}", + ) + self.assertEqual( + argv[argv.index("--model") + 1], ls.REQUESTED_MODEL, + f"resume must keep the pinned model: argv={argv!r}", + ) + self.assertIn( + "--prompt-file", argv, + "D-RES-1: grok resume must still pass --prompt-file: " + f"argv={argv!r}", + ) + + def test_the_captured_short_circuit_replay_is_a_valid_but_empty_envelope(self): + """Scenario: the launcher's scan accepts the captured bytes — and that is the problem the argv change answers""" + body = self._fixture_text("grok-json-schema-short-circuit-66ccb2.raw.out") + planted = json.loads(body) + nested = planted.get("structuredOutput") + self.assertIsInstance(nested, dict, "plant: nested envelope present") + self.assertEqual(nested.get("status"), "ok", "plant: empty ok envelope") + self.assertEqual(nested.get("findings"), [], "plant: no findings") + self.assertIn("starting", str(nested.get("note"))) + self.assertGreater( + len(planted), 9, + "plant: the wrapper is not itself a nine-key envelope", + ) + os.environ["TASK_LAUNCH_STDOUT"] = "token" + os.environ["TASK_LAUNCH_TOKEN"] = body.rstrip("\n") + rec, *_ = self._dispatch_closed( + job="plan", harness="grok", stage="plan", + ) + _raw, data = self._raw_out() + self.assertIn(b"structuredOutput", data, "plant: fixture replayed") + env = _envelope(rec) + self.assertEqual( + env.get("status"), "ok", + "the scan accepts the captured short-circuit bytes — that " + f"is why grok must not be launched with --json-schema: {env!r}", + ) + self.assertEqual(env.get("findings"), [], env) + self.assertIn("starting", str(env.get("note"))) + + def test_codex_fresh_argv_carries_json_and_last_message_file(self): + """Scenario: codex fresh argv asks for json events and a last-message file""" + rec, witness, *_ = self.launch_ok( + job="plan", harness="codex", stage="plan", + ) + argv = self._argv_list(witness) + self.assertIn("--json", argv, f"codex argv={argv!r}") + self.assertIn("-o", argv, f"codex argv={argv!r}") + dest = Path(argv[argv.index("-o") + 1]) + expected = Path(rec["snapshot"]["root"]).parent / "out" / ( + "last-message.json" + ) + self.assertEqual( + dest.resolve(), expected.resolve(), + f"-o must be /out/last-message.json, got {dest}", + ) + + def test_codex_resume_argv_keeps_json_and_last_message_file(self): + """Scenario: codex resume argv keeps json events and the last-message file""" + rec, *_ = self.launch_ok(job="plan", harness="codex", stage="plan") + argv = self._argv_list(self._resume_witness(rec)) + self.assertIn("--json", argv, f"codex resume argv={argv!r}") + self.assertIn("-o", argv, f"codex resume argv={argv!r}") + dest = Path(argv[argv.index("-o") + 1]) + expected = Path(rec["snapshot"]["root"]).parent / "out" / ( + "last-message.json" + ) + self.assertEqual(dest.resolve(), expected.resolve()) + self.assertIn("resume", argv) + + def test_argv_schema_is_the_exported_envelope_schema(self): + """Scenario: claude argv schema is the exported envelope schema""" + _rec, witness, *_ = self.launch_ok( + job="plan", harness="claude", stage="plan", + ) + supplied = self._loaded_schema( + self._schema_value(self._argv_list(witness)), + ) + self.assertEqual( + supplied, self._exported_schema(), + "the schema handed to the child must be envelope.ENVELOPE_SCHEMA", + ) + + def test_the_child_reads_the_json_schema_value(self): + """Scenario: the child reads the --json-schema value from its own argv""" + _rec, witness, *_ = self.launch_ok( + job="plan", harness="claude", stage="plan", + ) + value = self._schema_value(self._argv_list(witness)) + self._child_schema_reachable(witness, value) + self.assertEqual( + self._loaded_schema(value), self._exported_schema(), + ) + + def test_claude_argv_schema_is_the_exported_envelope_schema(self): + """Scenario: claude argv schema is the exported envelope schema""" + _rec, witness, *_ = self.launch_ok( + job="plan", harness="claude", stage="plan", + ) + supplied = self._loaded_schema( + self._schema_value(self._argv_list(witness)), + ) + self.assertEqual( + supplied, self._exported_schema(), + "claude --json-schema must be envelope.ENVELOPE_SCHEMA, " + "not a stale or grok-only copy", + ) + + +class ClaudeWrapperIsTheFirstEnvelopeSource(_U10Launch): + def test_structured_output_is_the_envelope(self): + """Scenario: a claude wrapper with structured_output is unwrapped as the envelope""" + os.environ["TASK_LAUNCH_WRAPPER"] = "claude" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "structured_output" + os.environ["TASK_LAUNCH_NOTE"] = "u10-claude-structured" + os.environ["TASK_LAUNCH_WRAPPER_DECOY"] = "u10-decoy-scan" + rec, *_ = self._dispatch_closed( + job="plan", harness="claude", stage="plan", + ) + _raw, data = self._raw_out() + self.assertIn(b"u10-claude-structured", data) + self.assertIn(b"u10-decoy-scan", data) + self.assertGreater( + data.rfind(b"u10-decoy-scan"), + data.find(b"u10-claude-structured"), + "plant: decoy is the last JSON object on stdout", + ) + env = _envelope(rec) + self.assertEqual( + env.get("note"), "u10-claude-structured", + "structured_output is the first source; a later scan-shaped " + f"object must not win: envelope={env!r}", + ) + self.assertEqual(env.get("status"), "ok", env) + + def test_result_string_is_parsed_when_structured_output_is_absent(self): + """Scenario: a claude wrapper with the envelope only in result text is unwrapped""" + os.environ["TASK_LAUNCH_WRAPPER"] = "claude" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "result" + os.environ["TASK_LAUNCH_NOTE"] = "u10-claude-result-string" + os.environ["TASK_LAUNCH_WRAPPER_DECOY"] = "u10-decoy-scan" + rec, *_ = self._dispatch_closed( + job="plan", harness="claude", stage="plan", + ) + _raw, data = self._raw_out() + first = data.decode("utf-8").splitlines()[0] + wrapper = json.loads(first) + self.assertEqual(wrapper.get("type"), "result", wrapper) + self.assertNotIn( + "structured_output", wrapper, + "plant: this case has no structured_output field", + ) + parsed = json.loads(wrapper["result"]) + self.assertEqual(parsed.get("note"), "u10-claude-result-string") + self.assertIn(b"u10-decoy-scan", data) + env = _envelope(rec) + self.assertEqual( + env.get("note"), "u10-claude-result-string", + "the result JSON string is the first source when " + f"structured_output is absent: envelope={env!r}", + ) + self.assertEqual(env.get("status"), "ok", env) + + def test_wrapper_without_envelope_is_claude_result_subtype(self): + """Scenario: a claude wrapper with no envelope is claude-result subtype""" + os.environ["TASK_LAUNCH_WRAPPER"] = "claude" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "none" + os.environ["TASK_LAUNCH_WRAPPER_SUBTYPE"] = "error_max_turns" + os.environ["TASK_LAUNCH_WRAPPER_IS_ERROR"] = "1" + rec, *_ = self._dispatch_closed( + job="plan", harness="claude", stage="plan", + ) + _raw, data = self._raw_out() + wrapper = json.loads(data.decode("utf-8")) + self.assertEqual(wrapper.get("type"), "result") + self.assertEqual(wrapper.get("subtype"), "error_max_turns") + self.assertTrue(wrapper.get("is_error")) + self.assertNotIsInstance(wrapper.get("structured_output"), dict) + self.assertEqual( + _cause_reason(rec), "claude-result:error_max_turns", + f"cause={ (rec.get('result') or {}).get('cause')!r} " + f"envelope={_envelope(rec)!r}", + ) + self.assertEqual(_envelope(rec).get("status"), "invalid") + + def test_is_error_wins_over_structured_output(self): + """Scenario: a claude wrapper with is_error true is claude-result even when structured_output is present""" + os.environ["TASK_LAUNCH_WRAPPER"] = "claude" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "structured_output" + os.environ["TASK_LAUNCH_WRAPPER_SUBTYPE"] = "error_max_turns" + os.environ["TASK_LAUNCH_WRAPPER_IS_ERROR"] = "1" + os.environ["TASK_LAUNCH_NOTE"] = "u10-truncated-structured" + rec, *_ = self._dispatch_closed( + job="plan", harness="claude", stage="plan", + ) + _raw, data = self._raw_out() + wrapper = json.loads(data.decode("utf-8").splitlines()[0]) + self.assertTrue(wrapper.get("is_error"), "plant: is_error landed") + self.assertIsInstance( + wrapper.get("structured_output"), dict, + "plant: structured_output is present beside is_error", + ) + self.assertEqual( + _cause_reason(rec), "claude-result:error_max_turns", + "is_error is the ending; structured_output written before " + "the wall must not close as ok: " + f"cause={ (rec.get('result') or {}).get('cause')!r} " + f"envelope={_envelope(rec)!r}", + ) + self.assertEqual(_envelope(rec).get("status"), "invalid") + + def test_replayed_error_max_turns_fixture_is_claude_result_subtype(self): + """Scenario: a claude wrapper with no envelope is claude-result subtype""" + body = self._fixture_text(CLAUDE_MAX_TURNS_FIXTURE) + planted = json.loads(body) + self.assertEqual(planted.get("subtype"), "error_max_turns") + self.assertTrue(planted.get("is_error")) + os.environ["TASK_LAUNCH_STDOUT"] = "token" + os.environ["TASK_LAUNCH_TOKEN"] = body + rec, *_ = self._dispatch_closed( + job="plan", harness="claude", stage="plan", + ) + _raw, data = self._raw_out() + replayed = json.loads(data.decode("utf-8")) + self.assertEqual(replayed.get("subtype"), "error_max_turns") + self.assertTrue(replayed.get("is_error")) + self.assertEqual( + _cause_reason(rec), "claude-result:error_max_turns", + f"replayed docs-shaped wrapper must name the subtype: " + f"cause={ (rec.get('result') or {}).get('cause')!r}", + ) + + +class GrokObjectIsTheEnvelope(_U10Launch): + def test_stdout_object_matching_schema_is_the_envelope(self): + """Scenario: a grok trailing nested-invalid object loses to an earlier valid envelope""" + os.environ["TASK_LAUNCH_WRAPPER"] = "grok" + os.environ["TASK_LAUNCH_NOTE"] = "u10-grok-object" + os.environ["TASK_LAUNCH_WRAPPER_DECOY"] = "u10-decoy-scan" + rec, *_ = self._dispatch_closed( + job="plan", harness="grok", stage="plan", + ) + _raw, data = self._raw_out() + self.assertIn(b"u10-grok-object", data) + self.assertIn(b"u10-decoy-scan", data) + self.assertGreater( + data.find(b"u10-decoy-scan"), + data.find(b"u10-grok-object"), + "plant: decoy trails the harness object on stdout", + ) + objects = [ + json.loads(ln) for ln in data.decode("utf-8").splitlines() + if ln.strip().startswith("{") + ] + self.assertGreaterEqual(len(objects), 2, "plant: object then decoy") + self.assertEqual(objects[0].get("note"), "u10-grok-object") + self.assertEqual(objects[-1].get("note"), "u10-decoy-scan") + self.assertEqual( + objects[-1].get("counts", {}).get("p1"), "scan", + "plant: trailing decoy is scan-shaped with nested wrong type", + ) + env = _envelope(rec) + self.assertEqual( + env.get("note"), "u10-grok-object", + "D-ENV-2: grok's envelope is the last fully-valid object; a " + "trailing scan-shaped decoy (nested wrong type) must lose: " + f"envelope={env!r}", + ) + self.assertEqual(env.get("status"), "ok", env) + self.assertNotEqual(_cause_reason(rec), "schema-invalid", rec) + + def test_null_verdict_on_invalid_is_not_schema_invalid(self): + """Scenario: a legal envelope with a null verdict is not schema-invalid""" + os.environ["TASK_LAUNCH_WRAPPER"] = "grok" + os.environ["TASK_LAUNCH_VERDICT"] = "null" + os.environ["TASK_LAUNCH_STATUS"] = "invalid" + os.environ["TASK_LAUNCH_NOTE"] = "u10-null-verdict" + os.environ["TASK_LAUNCH_WRAPPER_DECOY_FIRST"] = "u10-decoy-first" + rec, *_ = self._dispatch_closed( + job="plan", harness="grok", stage="plan", + ) + _raw, data = self._raw_out() + self.assertIn(b"u10-decoy-first", data, "plant: valid decoy first") + self.assertIn(b"u10-null-verdict", data) + self.assertLess( + data.find(b"u10-decoy-first"), + data.find(b"u10-null-verdict"), + "plant: fully-valid decoy precedes the allowed envelope", + ) + text = data.decode("utf-8") + objects = [ + json.loads(ln) for ln in text.splitlines() + if ln.strip().startswith("{") + ] + self.assertEqual(len(objects), 2, "plant: decoy then envelope") + self.assertEqual(objects[0].get("note"), "u10-decoy-first") + planted = objects[1] + self.assertIsNone(planted.get("verdict"), "plant: verdict is null") + self.assertEqual(planted.get("status"), "invalid") + self.assertNotIn("commit", planted, "plant: commit is absent") + self.assertNotEqual( + _cause_reason(rec), "schema-invalid", + "verdict null on invalid is allowed (D-ENV-1); a validator " + "that rejects it closes every invalid envelope: " + f"cause={ (rec.get('result') or {}).get('cause')!r} " + f"envelope={_envelope(rec)!r}", + ) + env = _envelope(rec) + self.assertEqual(env.get("note"), "u10-null-verdict", env) + self.assertIsNone(env.get("verdict"), env) + + def test_optional_commit_is_not_schema_invalid(self): + """Scenario: a legal envelope with optional commit is not schema-invalid""" + os.environ["TASK_LAUNCH_WRAPPER"] = "grok" + os.environ["TASK_LAUNCH_ENVELOPE_COMMIT"] = "1" + os.environ["TASK_LAUNCH_NOTE"] = "u10-with-commit" + os.environ["TASK_LAUNCH_WRAPPER_DECOY_FIRST"] = "u10-decoy-first" + rec, *_ = self._dispatch_closed( + job="plan", harness="grok", stage="plan", + ) + _raw, data = self._raw_out() + self.assertIn(b"u10-decoy-first", data, "plant: valid decoy first") + self.assertIn(b"u10-with-commit", data) + self.assertLess( + data.find(b"u10-decoy-first"), + data.find(b"u10-with-commit"), + "plant: fully-valid decoy precedes the allowed envelope", + ) + objects = [ + json.loads(ln) for ln in data.decode("utf-8").splitlines() + if ln.strip().startswith("{") + ] + self.assertEqual(len(objects), 2, "plant: decoy then envelope") + self.assertEqual(objects[0].get("note"), "u10-decoy-first") + self.assertNotIn("commit", objects[0], "plant: first decoy has no commit") + planted = objects[1] + self.assertIn("commit", planted, "plant: commit landed") + self.assertEqual(planted["commit"].get("subject"), "dispatch: pin U10") + self.assertNotEqual( + _cause_reason(rec), "schema-invalid", + "commit is optional and allowed when present: " + f"cause={ (rec.get('result') or {}).get('cause')!r} " + f"envelope={_envelope(rec)!r}", + ) + env = _envelope(rec) + self.assertEqual(env.get("note"), "u10-with-commit", env) + self.assertEqual(env.get("status"), "ok", env) + self.assertIn("commit", env, env) + + def test_object_with_an_extra_key_is_schema_invalid(self): + """Scenario: a grok stdout object that fails the schema is schema-invalid""" + os.environ["TASK_LAUNCH_WRAPPER"] = "grok" + os.environ["TASK_LAUNCH_WRAPPER_INVALID"] = "extra" + os.environ["TASK_LAUNCH_NOTE"] = "u10-grok-extra" + rec, *_ = self._dispatch_closed( + job="plan", harness="grok", stage="plan", + ) + _raw, data = self._raw_out() + planted = json.loads(data.decode("utf-8")) + self.assertIn("transcript", planted, "plant: extra key landed") + self.assertEqual(planted.get("note"), "u10-grok-extra") + self.assertEqual( + _cause_reason(rec), "schema-invalid", + "an object that fails ENVELOPE_SCHEMA is schema-invalid, " + "not envelope-parse or envelope-missing: " + f"cause={ (rec.get('result') or {}).get('cause')!r} " + f"envelope={_envelope(rec)!r}", + ) + self.assertEqual(_envelope(rec).get("status"), "invalid") + + def test_object_with_a_missing_key_is_schema_invalid(self): + """Scenario: a grok stdout object that fails the schema is schema-invalid""" + os.environ["TASK_LAUNCH_WRAPPER"] = "grok" + os.environ["TASK_LAUNCH_WRAPPER_INVALID"] = "missing" + rec, *_ = self._dispatch_closed( + job="plan", harness="grok", stage="plan", + ) + _raw, data = self._raw_out() + planted = json.loads(data.decode("utf-8")) + self.assertNotIn("note", planted, "plant: note stripped") + self.assertEqual( + _cause_reason(rec), "schema-invalid", + f"cause={ (rec.get('result') or {}).get('cause')!r} " + f"envelope={_envelope(rec)!r}", + ) + + def test_narration_then_object_still_parses_via_the_fallback_scan(self): + """Scenario: a grok narration-then-object raw.out still parses via the fallback scan""" + body = self._fixture_text(GROK_NARRATION_FIXTURE) + brace = body.find("{") + self.assertGreater(brace, 0, "plant: object begins mid-line") + self.assertNotEqual(body[brace - 1], "\n") + os.environ["TASK_LAUNCH_STDOUT"] = "token" + os.environ["TASK_LAUNCH_TOKEN"] = body + rec, *_ = self._dispatch_closed( + job="plan", harness="grok", stage="plan", + ) + _raw, data = self._raw_out() + self.assertIn(b"u10-grok-narration-then-object", data) + env = _envelope(rec) + note = str(env.get("note") or "") + self.assertNotIn("envelope-parse", note.lower(), env) + self.assertEqual( + env.get("note"), "u10-grok-narration-then-object", env, + ) + self.assertEqual(env.get("status"), "ok", env) + + def test_fallback_scan_object_that_fails_the_schema_is_schema_invalid(self): + """Scenario: a grok fallback-scan object that fails the schema is schema-invalid""" + os.environ["TASK_LAUNCH_WRAPPER"] = "grok" + os.environ["TASK_LAUNCH_WRAPPER_NARRATION"] = ( + "working, then the object." + ) + os.environ["TASK_LAUNCH_WRAPPER_INVALID"] = "extra" + os.environ["TASK_LAUNCH_NOTE"] = "u10-grok-fallback-extra" + rec, *_ = self._dispatch_closed( + job="plan", harness="grok", stage="plan", + ) + _raw, data = self._raw_out() + self.assertIn(b"working, then the object.", data) + text = data.decode("utf-8") + planted = json.loads(text[text.find("{"):]) + self.assertIn("transcript", planted, "plant: extra key landed") + self.assertEqual( + _cause_reason(rec), "schema-invalid", + "the fallback scan must still validate: " + f"cause={ (rec.get('result') or {}).get('cause')!r} " + f"envelope={_envelope(rec)!r}", + ) + + def test_captured_s2_raw_out_recovers_the_object(self): + """Scenario: a captured grok S2 raw.out recovers the object instead of envelope-parse""" + body = self._fixture_bytes(GROK_S2_FIXTURE) + brace = body.find(b"{") + self.assertGreater(brace, 0, "plant: object begins mid-line") + self.assertNotEqual(body[brace - 1], 10, "plant: no newline before {") + planted = json.loads(body[brace:]) + self.assertEqual(planted.get("job"), "author-tests") + os.environ["TASK_LAUNCH_STDOUT"] = "token" + os.environ["TASK_LAUNCH_TOKEN"] = body.decode("utf-8") + rec, *_ = self._dispatch_closed( + job="plan", harness="grok", stage="plan", + ) + _raw, data = self._raw_out() + self.assertEqual(data.rstrip(b"\n"), body.rstrip(b"\n")) + env = _envelope(rec) + blob = str(env.get("note") or "") + str(_cause_reason(rec) or "") + self.assertNotIn( + "envelope-parse", blob.lower(), + "D-ENV-1: today's raw.out still parses; 13f5f2's object is " + f"on stdout: note={env.get('note')!r} " + f"cause={ (rec.get('result') or {}).get('cause')!r}", + ) + self.assertEqual( + env.get("job"), "author-tests", + f"the captured object is the envelope: {env!r}", + ) + + +class CodexLastMessageFileIsTheFirstEnvelopeSource(_U10Launch): + def test_envelope_is_read_from_the_dash_o_file(self): + """Scenario: a codex last-message file is the first envelope source""" + os.environ["TASK_LAUNCH_WRAPPER"] = "codex" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "file" + os.environ["TASK_LAUNCH_NOTE"] = "u10-codex-file" + rec, *_ = self._dispatch_closed( + job="plan", harness="codex", stage="plan", + ) + job_dir = Path(rec["snapshot"]["root"]).parent + last = job_dir / "out" / "last-message.json" + self.assertTrue(last.is_file(), f"plant: {last} must exist") + planted = json.loads(last.read_text(encoding="utf-8")) + self.assertEqual(planted.get("note"), "u10-codex-file") + _raw, data = self._raw_out() + self.assertIn(b"thread.started", data) + self.assertNotIn(b"u10-codex-file", data) + env = _envelope(rec) + self.assertEqual( + env.get("note"), "u10-codex-file", + f"the -o file is the first source: envelope={env!r}", + ) + self.assertEqual(env.get("status"), "ok", env) + + def test_file_wins_over_a_disagreeing_agent_message(self): + """Scenario: a codex last-message file wins over a disagreeing agent_message""" + os.environ["TASK_LAUNCH_WRAPPER"] = "codex" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "both" + os.environ["TASK_LAUNCH_NOTE"] = "u10-codex-file-wins" + os.environ["TASK_LAUNCH_WRAPPER_AGENT_NOTE"] = "u10-codex-agent-loses" + rec, *_ = self._dispatch_closed( + job="plan", harness="codex", stage="plan", + ) + job_dir = Path(rec["snapshot"]["root"]).parent + last = job_dir / "out" / "last-message.json" + self.assertTrue(last.is_file(), f"plant: {last} must exist") + planted = json.loads(last.read_text(encoding="utf-8")) + self.assertEqual(planted.get("note"), "u10-codex-file-wins") + _raw, data = self._raw_out() + self.assertIn(b"u10-codex-agent-loses", data) + self.assertIn(b"agent_message", data) + env = _envelope(rec) + self.assertEqual( + env.get("note"), "u10-codex-file-wins", + "file then agent_message then stdout: the file wins when " + f"both disagree: envelope={env!r}", + ) + self.assertEqual(env.get("status"), "ok", env) + + def test_last_agent_message_is_used_when_the_file_is_absent(self): + """Scenario: a codex last agent_message is used when the file is absent""" + os.environ["TASK_LAUNCH_WRAPPER"] = "codex" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "agent_message" + os.environ["TASK_LAUNCH_NOTE"] = "u10-codex-agent-message" + rec, *_ = self._dispatch_closed( + job="plan", harness="codex", stage="plan", + ) + job_dir = Path(rec["snapshot"]["root"]).parent + last = job_dir / "out" / "last-message.json" + self.assertFalse( + last.is_file(), + f"plant: the -o file must be absent, found {last}", + ) + _raw, data = self._raw_out() + self.assertIn(b"u10-codex-agent-message", data) + self.assertIn(b"agent_message", data) + env = _envelope(rec) + self.assertEqual( + env.get("note"), "u10-codex-agent-message", + f"last agent_message is the second source: envelope={env!r}", + ) + self.assertEqual(env.get("status"), "ok", env) + + def test_replayed_ndjson_fixture_unwraps_the_agent_message(self): + """Scenario: a codex last agent_message is used when the file is absent""" + body = self._fixture_text(CODEX_NDJSON_FIXTURE) + self.assertIn("agent_message", body) + os.environ["TASK_LAUNCH_STDOUT"] = "token" + os.environ["TASK_LAUNCH_TOKEN"] = body.rstrip("\n") + rec, *_ = self._dispatch_closed( + job="plan", harness="codex", stage="plan", + ) + _raw, data = self._raw_out() + self.assertIn(b"u10-codex-agent-message", data) + env = _envelope(rec) + self.assertEqual( + env.get("note"), "u10-codex-agent-message", + f"replayed NDJSON tail must unwrap agent_message: {env!r}", + ) + + def test_absent_file_and_no_agent_message_is_no_last_message(self): + """Scenario: a missing codex last-message file with nothing else is no-last-message""" + os.environ["TASK_LAUNCH_WRAPPER"] = "codex" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "none" + rec, *_ = self._dispatch_closed( + job="plan", harness="codex", stage="plan", + ) + job_dir = Path(rec["snapshot"]["root"]).parent + last = job_dir / "out" / "last-message.json" + self.assertFalse(last.is_file(), f"plant: file absent, found {last}") + _raw, data = self._raw_out() + self.assertIn(b"thread.started", data) + self.assertNotIn(b'"job"', data) + self.assertEqual( + _cause_reason(rec), "no-last-message", + "absent -o file and no agent_message is no-last-message, " + "not envelope-parse: " + f"cause={ (rec.get('result') or {}).get('cause')!r} " + f"envelope={_envelope(rec)!r}", + ) + + def test_resume_clears_the_previous_last_message_file(self): + """Scenario: a resumed codex attempt clears the previous last-message file""" + os.environ["TASK_LAUNCH_WRAPPER"] = "codex" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "file" + os.environ["TASK_LAUNCH_NOTE"] = "u10-codex-attempt-1" + rec, *_ = self._dispatch_closed( + job="plan", harness="codex", stage="plan", + ) + job_dir = Path(rec["snapshot"]["root"]).parent + last = job_dir / "out" / "last-message.json" + self.assertTrue(last.is_file(), "plant: attempt 1 wrote the file") + first = json.loads(last.read_text(encoding="utf-8")) + self.assertEqual(first.get("note"), "u10-codex-attempt-1") + os.environ["TASK_LAUNCH_NOTE"] = "u10-codex-attempt-2" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "agent_message" + self.start_witness.unlink(missing_ok=True) + self.witness.unlink() + code, out, err = self.run_main(["resume", rec["id"]]) + self.assertEqual(code, 0, self.combined(out, err)) + started = self.read_start_witness() + self.assertFalse( + started.get("last_message_present"), + "resume must clear out/last-message.json before the child " + "starts, else a file-first reader returns attempt 1: " + f"start-witness={started!r}", + ) + rec2 = self.read_record() + env = _envelope(rec2) + self.assertEqual( + env.get("note"), "u10-codex-attempt-2", + "attempt 2's envelope, not the leftover file: " + f"envelope={env!r}", + ) + + def test_last_message_file_is_written_on_nonzero_exit(self): + """Scenario: a codex last-message file is written on a non-zero exit""" + os.environ["TASK_LAUNCH_WRAPPER"] = "codex" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "file" + os.environ["TASK_LAUNCH_NOTE"] = "u10-codex-nonzero" + os.environ["TASK_LAUNCH_EXIT"] = "1" + rec, *_ = self._dispatch_closed( + job="plan", harness="codex", stage="plan", + ) + job_dir = Path(rec["snapshot"]["root"]).parent + last = job_dir / "out" / "last-message.json" + self.assertTrue( + last.is_file(), + "premise: -o is written on a non-zero exit; " + f"missing {last}", + ) + planted = json.loads(last.read_text(encoding="utf-8")) + self.assertEqual(planted.get("note"), "u10-codex-nonzero") + reason = str(_cause_reason(rec) or "") + self.assertTrue( + reason.startswith("harness-cli:"), + "non-zero exit is harness-cli:, not an ok envelope from " + f"the file: cause={ (rec.get('result') or {}).get('cause')!r} " + f"envelope={_envelope(rec)!r}", + ) + note = str(_envelope(rec).get("note") or "") + self.assertIn( + "u10-codex-nonzero", note, + "the harness note is present beside the exit: " + f"envelope={_envelope(rec)!r}", + ) + self.assertIn( + "harness-cli:", note, + "a non-zero exit must name itself in the envelope note " + f"(not only in result.cause): note={note!r}", + ) + + +class EnvelopeParseOnlyWhenNothingParsed(_U10Launch): + def test_prose_stdout_is_still_envelope_parse(self): + """Scenario: envelope-parse remains only when nothing at all parsed""" + os.environ["TASK_LAUNCH_STDOUT"] = "prose" + rec, *_ = self._dispatch_closed( + job="plan", harness="grok", stage="plan", + ) + _raw, data = self._raw_out() + self.assertNotIn(b"{", data, "plant: prose has no JSON object") + env = _envelope(rec) + note = str(env.get("note") or "") + blob = note + str(_cause_reason(rec) or "") + self.assertIn( + "envelope-parse", blob.lower(), + "prose with no JSON object is envelope-parse, not a wrapper " + f"or schema cause: note={note!r} " + f"cause={ (rec.get('result') or {}).get('cause')!r}", + ) + self.assertEqual(env.get("status"), "invalid") + reason = str(_cause_reason(rec) or "") + self.assertFalse( + reason.startswith("claude-result:"), + f"prose is not a claude wrapper: cause={reason!r}", + ) + self.assertNotEqual(reason, "schema-invalid") + self.assertNotEqual(reason, "no-last-message") + + +class StampRefIsOverwrittenOnUnwrap(_U10Launch): + def test_wrapper_stamp_ref_is_replaced_with_the_snapshot_sha(self): + """Scenario: the launcher overwrites stamp.ref on an unwrapped envelope""" + os.environ["TASK_LAUNCH_WRAPPER"] = "claude" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "structured_output" + os.environ["TASK_LAUNCH_NOTE"] = "u10-stamp-overwrite" + os.environ["TASK_LAUNCH_WRAPPER_DECOY"] = "u10-decoy-scan" + rec, *_ = self._dispatch_closed( + job="plan", harness="claude", stage="plan", + ) + _raw, data = self._raw_out() + self.assertIn(b"u10-stamp-overwrite", data) + self.assertIn(b'"ref": "harness-placeholder"', data) + env = _envelope(rec) + self.assertEqual(env.get("note"), "u10-stamp-overwrite", env) + stamp = env.get("stamp") or {} + self.assertEqual( + stamp.get("ref"), rec["snapshot"]["ref_sha"], + f"launcher overwrites stamp.ref: stamp={stamp!r}", + ) + self.assertNotEqual(stamp.get("ref"), "harness-placeholder") + + +class ClaudeWrapperSpendIsSessionSpend(_U10Launch): + def test_wrapper_usage_is_session_spend(self): + """Scenario: claude wrapper usage is session.spend""" + os.environ["TASK_LAUNCH_WRAPPER"] = "claude" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "structured_output" + os.environ["TASK_LAUNCH_NOTE"] = "u10-spend-from-wrapper" + os.environ["TASK_LAUNCH_WRITE_STREAM"] = "0" + rec, *_ = self._dispatch_closed( + job="plan", harness="claude", stage="plan", + ) + _raw, data = self._raw_out() + wrapper = json.loads(data.decode("utf-8").splitlines()[0]) + self.assertEqual(wrapper.get("usage", {}).get("input_tokens"), 10) + self.assertEqual(wrapper.get("total_cost_usd"), 0.001) + spend = _spend(rec) + self.assertIsInstance(spend, dict, spend) + self.assertNotIn("unresolved", spend, spend) + self.assertEqual(spend.get("input"), 10, spend) + self.assertEqual(spend.get("output"), 4, spend) + self.assertEqual(spend.get("cost_usd"), 0.001, spend) + self.assertEqual(spend.get("source"), "result.usage", spend) + + def test_claude_result_still_records_wrapper_spend(self): + """Scenario: claude-result still records session.spend from the wrapper""" + os.environ["TASK_LAUNCH_WRAPPER"] = "claude" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "none" + os.environ["TASK_LAUNCH_WRAPPER_SUBTYPE"] = "error_max_turns" + os.environ["TASK_LAUNCH_WRAPPER_IS_ERROR"] = "1" + os.environ["TASK_LAUNCH_WRITE_STREAM"] = "0" + rec, *_ = self._dispatch_closed( + job="plan", harness="claude", stage="plan", + ) + self.assertEqual(_cause_reason(rec), "claude-result:error_max_turns") + spend = _spend(rec) + self.assertIsInstance(spend, dict, spend) + self.assertNotIn("unresolved", spend, spend) + self.assertEqual(spend.get("input"), 10, spend) + self.assertEqual(spend.get("output"), 4, spend) + self.assertEqual(spend.get("cost_usd"), 0.001, spend) + self.assertEqual(spend.get("source"), "result.usage", spend) diff --git a/ops/devlane/dispatch/tests/test_launch_envelope_parse.py b/ops/devlane/dispatch/tests/test_launch_envelope_parse.py new file mode 100644 index 0000000..794dfeb --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_envelope_parse.py @@ -0,0 +1,227 @@ +"""Close must find a complete envelope that begins mid-line on stdout. + +Written from records: + + 20260828T220752Z-review-grok-bf0679 (repo/self-hosted-runners) + 20260828T221053Z-review-grok-4c887d (dispatch/prompt-feed) + +Both closed ``envelope-parse: no JSON object on stdout`` while each +``raw.out`` holds a complete single-line envelope that begins mid-line, +directly after the final narration sentence, with no newline before +its opening brace. + +4c887d p3: a string stamp is stored under ``stamp.model``, so a SHA is +labelled as a model id. Envelope stamp fields are ``ref``, ``started``, +``ended`` (task CONTRACT.md §The envelope); a SHA is a ref, not a model. +""" + +from __future__ import annotations + +import json +import os + +import launch_support as ls + +# Distinct from any snapshot ref the launcher fills in. 40 hex chars. +STRING_STAMP_SHA = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + +# Final narration sentences from the two records, immediately followed +# by the envelope's opening brace in each raw.out. +BF0679_NARRATION = ( + "I'll reproduce the suspected defects against the tree before " + "recording any finding." +) +C887D_NARRATION = ( + "I'm going to verify the highest-severity claims with commands so " + "each finding has a reproduction, not a recollection." +) + +BF0679_NOTE = "bf0679-midline-self-hosted-runners" +C887D_NOTE = "4c887d-midline-prompt-feed" + + +def _ok_envelope(*, note, stamp=None): + if stamp is None: + stamp = { + "ref": "harness-placeholder", + "started": None, + "ended": None, + } + return { + "job": "plan", + "status": "ok", + "verdict": "changes", + "counts": {"p1": 0, "p2": 0, "p3": 0, "opinions": 0}, + "findings": [], + "artifacts": {}, + "spend": {"harness": "grok", "total": 0, "out": 0, "runs": 1}, + "stamp": stamp, + "note": note, + } + + +class MidlineEnvelopeOnStdoutIsParsed(ls._TempLaunch): + """A complete JSON envelope that starts mid-line is still an envelope. + + The two closed reviews emitted narration and then the object on the + same line. ``envelope-parse: no JSON object on stdout`` is the + wrong close for that stdout. + """ + + def _close_with_stdout(self, body): + os.environ["TASK_LAUNCH_STDOUT"] = "token" + os.environ["TASK_LAUNCH_TOKEN"] = body + code, out, err = self.dispatch(self.argv_for( + job="plan", harness="grok", stage="plan", + )) + text = self.combined(out, err) + self.assertNotEqual( + code, ls.REFUSAL_EXIT, + f"close of a finished job must not refuse: {text!r}", + ) + rec = self.read_record() + self.assertEqual(rec.get("status"), "closed") + return rec, self.the_job_dir() / "raw.out" + + def _assert_midline_plant(self, raw, *, narration, envelope): + """Prove the stdout shape the two records actually had.""" + self.assertTrue(raw.is_file(), "raw.out is missing") + data = raw.read_bytes() + self.assertTrue(data, "plant: raw.out is empty") + self.assertIn( + narration.encode("utf-8"), data, + "plant: narration must be in raw.out", + ) + brace = data.find(b"{") + self.assertGreater( + brace, 0, + "plant: envelope must begin mid-line, not at byte 0", + ) + self.assertNotEqual( + data[brace - 1], 0x0A, + "plant: no newline before the opening brace " + f"(byte before '{{' is {data[brace - 1]!r})", + ) + prefix = data[:brace] + self.assertTrue( + prefix.endswith(narration.encode("utf-8")), + "plant: the brace sits directly after the narration " + f"sentence; prefix={prefix[-80:]!r}", + ) + parsed = json.loads(data[brace:]) + self.assertEqual( + parsed, envelope, + "plant: raw.out must hold the complete planted envelope", + ) + self.assertEqual( + parsed.get("note"), envelope["note"], + "plant: unique note must survive into the JSON object", + ) + + def _assert_not_envelope_parse(self, rec, *, unique_note): + envelope = (rec.get("result") or {}).get("envelope") or {} + note = envelope.get("note") + note_text = str(note or "") + # Stated reason first: this is the close those two records got. + self.assertNotIn( + "no JSON object on stdout", + note_text, + "raw.out holds a complete JSON envelope; close must not " + f"report envelope-parse: note={note!r} envelope={envelope!r}", + ) + self.assertNotIn( + "envelope-parse", + note_text.lower(), + "a mid-line envelope is not envelope-parse: " + f"note={note!r} envelope={envelope!r}", + ) + self.assertNotEqual( + envelope.get("status"), "invalid", + "parsed stdout is not an invalid close: " + f"envelope={envelope!r}", + ) + self.assertEqual( + envelope.get("note"), unique_note, + "the planted envelope must be the one on the record, not " + f"a reconstructed stand-in: envelope={envelope!r}", + ) + + def test_self_hosted_runners_review_midline_envelope_is_parsed(self): + """Record 20260828T220752Z-review-grok-bf0679.""" + envelope = _ok_envelope(note=BF0679_NOTE) + body = BF0679_NARRATION + json.dumps(envelope, separators=(",", ":")) + rec, raw = self._close_with_stdout(body) + self._assert_midline_plant( + raw, narration=BF0679_NARRATION, envelope=envelope, + ) + self._assert_not_envelope_parse(rec, unique_note=BF0679_NOTE) + + def test_prompt_feed_review_midline_envelope_is_parsed(self): + """Record 20260828T221053Z-review-grok-4c887d.""" + envelope = _ok_envelope(note=C887D_NOTE) + body = C887D_NARRATION + json.dumps(envelope, separators=(",", ":")) + rec, raw = self._close_with_stdout(body) + self._assert_midline_plant( + raw, narration=C887D_NARRATION, envelope=envelope, + ) + self._assert_not_envelope_parse(rec, unique_note=C887D_NOTE) + + +class StringStampIsNotAModelId(ls._TempLaunch): + """4c887d p3 — launch.py ``_envelope``. + + A string stamp (a SHA) stored under ``stamp.model`` labels a commit + as a model id. Stamp fields are ref / started / ended. + """ + + def test_a_string_stamp_sha_is_not_stored_under_stamp_model(self): + envelope = _ok_envelope( + note="string-stamp-is-a-sha", + stamp=STRING_STAMP_SHA, + ) + self.assertIsInstance(envelope["stamp"], str) + self.assertEqual(envelope["stamp"], STRING_STAMP_SHA) + body = json.dumps(envelope, separators=(",", ":")) + os.environ["TASK_LAUNCH_STDOUT"] = "token" + os.environ["TASK_LAUNCH_TOKEN"] = body + code, out, err = self.dispatch(self.argv_for( + job="plan", harness="grok", stage="plan", + )) + text = self.combined(out, err) + self.assertNotEqual( + code, ls.REFUSAL_EXIT, + f"close of a finished job must not refuse: {text!r}", + ) + rec = self.read_record() + self.assertEqual(rec.get("status"), "closed") + + raw = self.the_job_dir() / "raw.out" + self.assertTrue(raw.is_file(), "raw.out is missing") + planted = json.loads(raw.read_text(encoding="utf-8")) + self.assertEqual( + planted.get("stamp"), STRING_STAMP_SHA, + "plant: stdout stamp must be the SHA string, " + f"got {planted.get('stamp')!r}", + ) + self.assertIsInstance( + planted.get("stamp"), str, + "plant: stamp on stdout is a string, not an object", + ) + self.assertNotEqual( + STRING_STAMP_SHA, rec["snapshot"]["ref_sha"], + "plant: the SHA string must differ from snapshot.ref_sha " + "so a copied ref cannot satisfy the assertion", + ) + + result_env = (rec.get("result") or {}).get("envelope") or {} + stamp = result_env.get("stamp") + self.assertIsInstance( + stamp, dict, + f"recorded stamp is an object, got {stamp!r}", + ) + # Stated reason: a SHA labelled as a model id. + self.assertNotEqual( + stamp.get("model"), STRING_STAMP_SHA, + "a string stamp that is a SHA must not be stored under " + f"stamp.model (a SHA is not a model id): stamp={stamp!r}", + ) diff --git a/ops/devlane/dispatch/tests/test_launch_envelope_u10_fix.py b/ops/devlane/dispatch/tests/test_launch_envelope_u10_fix.py new file mode 100644 index 0000000..f91ee14 --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_envelope_u10_fix.py @@ -0,0 +1,718 @@ +"""U10 fix cycle — tests for skeptic 2fd473 over grok's tests. + +Authored from in/BRIEF-harness-u10-fix-tests-3.md, D-ENV-2/3 as +ratified, .dev/design/features/dispatch/structured-envelopes.feature, +and the skeptic report. launch.py was not read. + +2fd473 findings, each a test here or a stated call in this docstring: + +P1 grok source-order rewritten to fit the code — GrokLastValidObject + restores the discriminating plants (earlier valid vs trailing + nested-invalid; two fully-valid, last wins). Feature says D-ENV-2. +P2 {unresolved} escape on empty/zero usage — WrapperSpendDoesNotInventZeros + asserts the U5 value; no unresolved disjunction. +P2 merged spend pins one key — WrapperSpendKeepsCached pins input, + cached, output and total (total arithmetic). +P2 no fenced ```json envelope on grok's plain path — FencedGrokEnvelope + plus fixtures/envelopes/grok-fenced-json.raw.out. +P3 grok resume drops --model/--prompt-file — pinned on the resume + scenario in test_launch_envelope_by_construction.py (D-RES-1). +P3 _envelope_cause leaks into the committed record — PrivateEnvelopeCause + stays out of the on-disk record. +P3 white-box short-circuit replay — converted to E2E TOKEN replay in + test_launch_envelope_by_construction.py. +P3 test author amended the ratification — REFUTED / discharged at + 9f07bb8 (D-ENV-2/3); this cycle does not edit the ratification. +""" + +from __future__ import annotations + +import json +import os +import threading +import time +from pathlib import Path + +import launch_support as ls + + +def _cause_reason(rec): + cause = (rec.get("result") or {}).get("cause") + if isinstance(cause, dict): + return cause.get("reason") + return cause + + +def _envelope(rec): + return (rec.get("result") or {}).get("envelope") or {} + + +def _spend(rec): + session = rec.get("session") or {} + return session.get("spend") or {} + + +NINE = ( + "job", "status", "verdict", "counts", "findings", + "artifacts", "spend", "stamp", "note", +) +U5_INPUT, U5_CACHED, U5_OUTPUT = 30, 12000, 500 +WRAPPER_INPUT, WRAPPER_OUTPUT = 10, 4 +GROK_FENCED_FIXTURE = "grok-fenced-json.raw.out" +ENVELOPES = ls.FIXTURES_DIR / "envelopes" + + +def _legal(**over): + env = { + "job": "plan", + "status": "ok", + "verdict": "changes", + "counts": {"p1": 0, "p2": 0, "p3": 0, "opinions": 0}, + "findings": [], + "artifacts": {}, + "spend": {}, + "stamp": {"ref": "x"}, + "note": "legal", + } + env.update(over) + return env + + +def _line_objects(data): + out = [] + for ln in data.decode("utf-8").splitlines(): + s = ln.strip() + if not s.startswith("{"): + continue + try: + obj = json.loads(s) + except json.JSONDecodeError: + continue + if isinstance(obj, dict): + out.append(obj) + return out + + +def _has_nine(obj): + return all(k in obj for k in NINE) + + +def _nested_counts_ok(obj): + counts = obj.get("counts") + if not isinstance(counts, dict): + return False + return all( + type(counts.get(key)) is int + for key in ("p1", "p2", "p3", "opinions") + ) + + +class _FixLaunch(ls._TempLaunch): + def _raw_out(self): + raw = self.the_job_dir() / "raw.out" + self.assertTrue(raw.is_file(), "raw.out is missing") + data = raw.read_bytes() + self.assertTrue(data, "raw.out is empty") + return raw, data + + def _dispatch_closed(self, **kwargs): + code, out, err = self.dispatch(self.argv_for(**kwargs)) + text = self.combined(out, err) + self.assertNotEqual( + code, ls.REFUSAL_EXIT, + f"finished job must not refuse: {text!r}", + ) + rec = self.read_record() + self.assertEqual(rec.get("status"), "closed", rec) + return rec, text + + def _close_token(self, payload, *, harness="grok"): + body = payload if isinstance(payload, str) else json.dumps(payload) + os.environ["TASK_LAUNCH_STDOUT"] = "token" + os.environ["TASK_LAUNCH_TOKEN"] = body + rec, *_ = self._dispatch_closed( + job="plan", harness=harness, stage="plan", + ) + _raw, data = self._raw_out() + return rec, data + + def _assert_schema_invalid(self, rec, *, plant): + env = _envelope(rec) + self.assertEqual( + env.get("status"), "invalid", + f"nested wrong type must close invalid, not ok: " + f"plant={plant!r} envelope={env!r} " + f"cause={ (rec.get('result') or {}).get('cause')!r}", + ) + self.assertEqual( + _cause_reason(rec), "schema-invalid", + f"nested wrong type is schema-invalid, not envelope-parse: " + f"plant={plant!r} " + f"cause={ (rec.get('result') or {}).get('cause')!r} " + f"envelope={env!r}", + ) + + +class NestedWrongTypeClosesInvalid(_FixLaunch): + """_schema_valid must validate nested ENVELOPE_SCHEMA types.""" + + def test_counts_empty_object_is_schema_invalid(self): + """Scenario: a nested wrong type closes schema-invalid""" + planted = _legal(counts={}, note="u10-counts-empty") + rec, data = self._close_token(planted) + parsed = json.loads(data.decode("utf-8")) + self.assertEqual(parsed.get("counts"), {}, "plant: counts {} landed") + self._assert_schema_invalid(rec, plant="counts={}") + + def test_counts_p1_string_is_schema_invalid(self): + """Scenario: a nested wrong type closes schema-invalid""" + counts = {"p1": "lots", "p2": 0, "p3": 0, "opinions": 0} + planted = _legal(counts=counts, note="u10-counts-p1-string") + rec, data = self._close_token(planted) + parsed = json.loads(data.decode("utf-8")) + self.assertEqual( + parsed.get("counts", {}).get("p1"), "lots", + "plant: counts.p1 string landed", + ) + self._assert_schema_invalid(rec, plant="counts.p1='lots'") + + def test_findings_string_items_are_schema_invalid(self): + """Scenario: a nested wrong type closes schema-invalid""" + planted = _legal( + findings=["not a finding"], note="u10-findings-str", + ) + rec, data = self._close_token(planted) + parsed = json.loads(data.decode("utf-8")) + self.assertEqual( + parsed.get("findings"), ["not a finding"], + "plant: findings=['not a finding'] landed", + ) + self._assert_schema_invalid(rec, plant="findings=['not a finding']") + + def test_findings_int_items_are_schema_invalid(self): + """Scenario: a nested wrong type closes schema-invalid""" + planted = _legal(findings=[42], note="u10-findings-int") + rec, data = self._close_token(planted) + parsed = json.loads(data.decode("utf-8")) + self.assertEqual( + parsed.get("findings"), [42], "plant: findings=[42] landed", + ) + self._assert_schema_invalid(rec, plant="findings=[42]") + + def test_findings_missing_required_keys_are_schema_invalid(self): + """Scenario: a nested wrong type closes schema-invalid""" + planted = _legal( + findings=[{"severity": "p1"}], + note="u10-findings-partial", + ) + rec, data = self._close_token(planted) + parsed = json.loads(data.decode("utf-8")) + items = parsed.get("findings") + self.assertEqual(len(items), 1, "plant: one finding landed") + self.assertEqual(items[0].get("severity"), "p1") + self.assertNotIn("where", items[0], "plant: where withheld") + self.assertNotIn("claim", items[0], "plant: claim withheld") + self.assertNotIn("reproduce", items[0], "plant: reproduce withheld") + self._assert_schema_invalid( + rec, plant="findings=[{severity:p1}] missing required keys", + ) + + def test_artifacts_int_values_are_schema_invalid(self): + """Scenario: a nested wrong type closes schema-invalid""" + planted = _legal( + artifacts={"plan": 1}, note="u10-artifacts-int", + ) + rec, data = self._close_token(planted) + parsed = json.loads(data.decode("utf-8")) + self.assertEqual( + parsed.get("artifacts"), {"plan": 1}, + "plant: artifacts int value landed", + ) + self._assert_schema_invalid(rec, plant="artifacts={'plan': 1}") + + def test_artifacts_object_values_are_schema_invalid(self): + """Scenario: a nested wrong type closes schema-invalid""" + planted = _legal( + artifacts={"plan": {"inline": "..."}}, + note="u10-artifacts-obj", + ) + rec, data = self._close_token(planted) + parsed = json.loads(data.decode("utf-8")) + self.assertEqual( + parsed.get("artifacts", {}).get("plan"), {"inline": "..."}, + "plant: artifacts object value landed", + ) + self._assert_schema_invalid( + rec, plant="artifacts={'plan': {'inline': '...'}}", + ) + + def test_commit_null_is_schema_invalid(self): + """Scenario: a nested wrong type closes schema-invalid""" + planted = _legal(commit=None, note="u10-commit-null") + rec, data = self._close_token(planted) + parsed = json.loads(data.decode("utf-8")) + self.assertIn("commit", parsed, "plant: commit key present") + self.assertIsNone(parsed.get("commit"), "plant: commit is null") + self._assert_schema_invalid(rec, plant="commit=null") + + def test_stamp_extra_key_is_schema_invalid(self): + """Scenario: a nested wrong type closes schema-invalid""" + planted = _legal( + stamp={"ref": "x", "extra": "no"}, + note="u10-stamp-extra", + ) + rec, data = self._close_token(planted) + parsed = json.loads(data.decode("utf-8")) + self.assertEqual( + parsed.get("stamp", {}).get("extra"), "no", + "plant: stamp extra key landed", + ) + self._assert_schema_invalid(rec, plant="stamp extra key") + + def test_invalid_status_with_approve_verdict_is_not_ok(self): + """Scenario: status invalid with verdict approve is not an ok envelope""" + planted = _legal( + status="invalid", verdict="approve", + note="u10-invalid-approve", + ) + rec, data = self._close_token(planted) + parsed = json.loads(data.decode("utf-8")) + self.assertEqual(parsed.get("status"), "invalid", "plant: status") + self.assertEqual(parsed.get("verdict"), "approve", "plant: verdict") + env = _envelope(rec) + self.assertNotEqual( + env.get("status"), "ok", + "a task that could not look must not close ok with " + f"verdict=approve: envelope={env!r}", + ) + self.assertNotEqual( + env.get("verdict"), "approve", + "headline rule: invalid must not approve: " + f"envelope={env!r}", + ) + + +class GrokLastValidObject(_FixLaunch): + def _close_two(self, first, second): + payload = json.dumps(first) + "\n" + json.dumps(second) + return self._close_token(payload) + + def test_trailing_nested_invalid_loses_to_the_earlier_valid(self): + """Scenario: a grok trailing nested-invalid object loses to an earlier valid envelope""" + valid = _legal(note="harness-object") + decoy = _legal( + note="trailing-decoy", + counts={"p1": "scan", "p2": 0, "p3": 0, "opinions": 0}, + ) + rec, data = self._close_two(valid, decoy) + self.assertIn(b"harness-object", data) + self.assertIn(b"trailing-decoy", data) + self.assertGreater( + data.find(b"trailing-decoy"), + data.find(b"harness-object"), + "plant: decoy trails the valid object", + ) + objects = _line_objects(data) + self.assertEqual(len(objects), 2, "plant: exactly two objects") + self.assertTrue(_has_nine(objects[0]), "plant: earlier is nine-key") + self.assertTrue( + _nested_counts_ok(objects[0]), + "plant: earlier object is nested-valid " + f"(counts={objects[0].get('counts')!r})", + ) + self.assertTrue(_has_nine(objects[1]), "plant: trailing is nine-key") + self.assertFalse( + _nested_counts_ok(objects[1]), + "plant: trailing fails nested validation " + f"(counts={objects[1].get('counts')!r})", + ) + self.assertEqual(objects[-1].get("note"), "trailing-decoy") + self.assertEqual(objects[-1]["counts"]["p1"], "scan") + env = _envelope(rec) + self.assertEqual( + env.get("note"), "harness-object", + "D-ENV-2: trailing nine-key that fails nested validation " + "must lose to the earlier fully-valid envelope: " + f"envelope={env!r}", + ) + self.assertEqual(env.get("status"), "ok", env) + + def test_two_fully_valid_objects_the_last_wins(self): + """Scenario: two fully-valid grok stdout objects - the last wins""" + draft = _legal(note="draft-envelope") + final = _legal(note="final-envelope") + rec, data = self._close_two(draft, final) + self.assertIn(b"draft-envelope", data) + self.assertIn(b"final-envelope", data) + self.assertGreater( + data.find(b"final-envelope"), + data.find(b"draft-envelope"), + "plant: final trails the draft", + ) + objects = _line_objects(data) + self.assertEqual(len(objects), 2, "plant: exactly two objects") + self.assertTrue(_has_nine(objects[0]) and _nested_counts_ok(objects[0])) + self.assertTrue(_has_nine(objects[1]) and _nested_counts_ok(objects[1])) + self.assertEqual(objects[0].get("note"), "draft-envelope") + self.assertEqual(objects[1].get("note"), "final-envelope") + env = _envelope(rec) + self.assertEqual( + env.get("note"), "final-envelope", + "D-ENV-2 stated limit: two fully-valid objects, the last " + f"wins (a first-valid reader ships the draft): envelope={env!r}", + ) + self.assertNotEqual( + env.get("note"), "draft-envelope", + "the earlier fully-valid object must not win: " + f"envelope={env!r}", + ) + self.assertEqual(env.get("status"), "ok", env) + + +class CodexStdoutFallback(_FixLaunch): + def test_bare_stdout_object_is_used_when_file_and_agent_message_fail(self): + """Scenario: a codex stdout last-JSON-object is used when -o and agent_message both fail""" + os.environ["TASK_LAUNCH_WRAPPER"] = "codex" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "stdout" + os.environ["TASK_LAUNCH_NOTE"] = "u10-codex-stdout-bare" + rec, *_ = self._dispatch_closed( + job="plan", harness="codex", stage="plan", + ) + job_dir = Path(rec["snapshot"]["root"]).parent + last = job_dir / "out" / "last-message.json" + self.assertFalse( + last.is_file(), + f"plant: -o file must be absent, found {last}", + ) + _raw, data = self._raw_out() + self.assertIn(b"thread.started", data) + self.assertNotIn(b"agent_message", data) + self.assertIn(b"u10-codex-stdout-bare", data) + env = _envelope(rec) + self.assertNotEqual( + _cause_reason(rec), "no-last-message", + "stdout last-JSON-object fallback must still run after " + "-o and agent_message fail: " + f"cause={ (rec.get('result') or {}).get('cause')!r} " + f"envelope={env!r}", + ) + self.assertEqual( + env.get("note"), "u10-codex-stdout-bare", + f"bare stdout object is the envelope: {env!r}", + ) + self.assertEqual(env.get("status"), "ok", env) + + +class WrapperSpendDoesNotInventZeros(_FixLaunch): + def test_empty_usage_leaves_the_store_spend(self): + """Scenario: empty wrapper usage does not overwrite session.spend with zeros""" + os.environ["TASK_LAUNCH_WRAPPER"] = "claude" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "structured_output" + os.environ["TASK_LAUNCH_WRAPPER_USAGE"] = "empty" + os.environ["TASK_LAUNCH_NOTE"] = "u10-spend-empty-usage" + rec, *_ = self._dispatch_closed( + job="plan", harness="claude", stage="plan", + ) + _raw, data = self._raw_out() + wrapper = json.loads(data.decode("utf-8").splitlines()[0]) + self.assertEqual(wrapper.get("usage"), {}, "plant: usage={}") + self.assertNotIn("total_cost_usd", wrapper, "plant: no cost") + spend = _spend(rec) + self.assertIsInstance(spend, dict, spend) + self.assertNotIn( + "unresolved", spend, + "D-ENV-3: usage={} never yields {unresolved} when U5 " + f"resolved: spend={spend!r}", + ) + self.assertEqual( + spend.get("input"), U5_INPUT, + "U5 store spend (input 30, cached 12000, output 500) " + f"must survive empty wrapper usage: spend={spend!r}", + ) + self.assertEqual(spend.get("cached"), U5_CACHED, spend) + self.assertEqual(spend.get("output"), U5_OUTPUT, spend) + self.assertEqual( + spend.get("total"), U5_INPUT + U5_CACHED + U5_OUTPUT, + f"U5 total arithmetic must survive empty usage: spend={spend!r}", + ) + + def test_all_zero_usage_leaves_the_store_spend(self): + """Scenario: empty wrapper usage does not overwrite session.spend with zeros""" + os.environ["TASK_LAUNCH_WRAPPER"] = "claude" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "structured_output" + os.environ["TASK_LAUNCH_WRAPPER_USAGE"] = "zero" + os.environ["TASK_LAUNCH_NOTE"] = "u10-spend-zero-usage" + rec, *_ = self._dispatch_closed( + job="plan", harness="claude", stage="plan", + ) + _raw, data = self._raw_out() + wrapper = json.loads(data.decode("utf-8").splitlines()[0]) + self.assertEqual(wrapper.get("usage", {}).get("input_tokens"), 0) + self.assertEqual(wrapper.get("usage", {}).get("output_tokens"), 0) + self.assertEqual(wrapper.get("total_cost_usd"), 0.0) + spend = _spend(rec) + self.assertIsInstance(spend, dict, spend) + self.assertNotIn( + "unresolved", spend, + "D-ENV-3: all-zero usage never yields {unresolved} when " + f"U5 resolved: spend={spend!r}", + ) + self.assertEqual( + spend.get("input"), U5_INPUT, + "U5 store spend must survive zero wrapper usage: " + f"spend={spend!r}", + ) + self.assertEqual(spend.get("cached"), U5_CACHED, spend) + self.assertEqual(spend.get("output"), U5_OUTPUT, spend) + self.assertEqual( + spend.get("total"), U5_INPUT + U5_CACHED + U5_OUTPUT, + f"U5 total arithmetic must survive zero usage: spend={spend!r}", + ) + + +class WrapperSpendKeepsCached(_FixLaunch): + def test_wrapper_usage_with_cache_tokens_records_cached_and_total(self): + """Scenario: wrapper usage that names cache tokens keeps cached and total""" + os.environ["TASK_LAUNCH_WRAPPER"] = "claude" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "structured_output" + os.environ["TASK_LAUNCH_WRAPPER_USAGE"] = "cached" + os.environ["TASK_LAUNCH_WRITE_STREAM"] = "0" + os.environ["TASK_LAUNCH_NOTE"] = "u10-spend-cached" + rec, *_ = self._dispatch_closed( + job="plan", harness="claude", stage="plan", + ) + _raw, data = self._raw_out() + wrapper = json.loads(data.decode("utf-8").splitlines()[0]) + usage = wrapper.get("usage") or {} + self.assertEqual(usage.get("input_tokens"), 12, "plant: input") + self.assertEqual(usage.get("output_tokens"), 3000, "plant: output") + self.assertEqual( + usage.get("cache_read_input_tokens"), 480000, + "plant: cache_read landed", + ) + self.assertEqual( + usage.get("cache_creation_input_tokens"), 9000, + "plant: cache_creation landed", + ) + spend = _spend(rec) + self.assertIsInstance(spend, dict, spend) + self.assertNotIn("unresolved", spend, spend) + cached = 480000 + 9000 + self.assertEqual( + spend.get("cached"), cached, + "cached is cache_creation + cache_read, not dropped: " + f"spend={spend!r}", + ) + self.assertEqual(spend.get("input"), 12, spend) + self.assertEqual(spend.get("output"), 3000, spend) + self.assertEqual( + spend.get("total"), 12 + cached + 3000, + f"total is input+cached+output: spend={spend!r}", + ) + + def test_incomplete_wrapper_usage_does_not_drop_store_cached(self): + """Scenario: incomplete wrapper usage does not drop the store cached tally""" + os.environ["TASK_LAUNCH_WRAPPER"] = "claude" + os.environ["TASK_LAUNCH_WRAPPER_FIELD"] = "structured_output" + os.environ["TASK_LAUNCH_NOTE"] = "u10-spend-keep-store-cached" + rec, *_ = self._dispatch_closed( + job="plan", harness="claude", stage="plan", + ) + _raw, data = self._raw_out() + wrapper = json.loads(data.decode("utf-8").splitlines()[0]) + usage = wrapper.get("usage") or {} + self.assertEqual(usage.get("input_tokens"), 10, "plant: wrapper") + self.assertNotIn( + "cache_read_input_tokens", usage, + "plant: wrapper usage has no cache keys", + ) + spend = _spend(rec) + self.assertIsInstance(spend, dict, spend) + self.assertNotIn("unresolved", spend, spend) + self.assertEqual( + spend.get("input"), WRAPPER_INPUT, + "wrapper input 10 is the turn measurement; ignoring the " + f"wrapper outright is not the merge: spend={spend!r}", + ) + self.assertEqual( + spend.get("cached"), U5_CACHED, + "incomplete wrapper usage (no cache keys) must not drop " + f"the U5 store cached tally: spend={spend!r}", + ) + self.assertEqual( + spend.get("output"), WRAPPER_OUTPUT, + f"wrapper output 4 is the turn measurement: spend={spend!r}", + ) + self.assertEqual( + spend.get("total"), + WRAPPER_INPUT + U5_CACHED + WRAPPER_OUTPUT, + "D-ENV-3 total arithmetic is input+cached+output; a total " + "that omits cached (14) or a missing total is the 489k-" + f"tokens-vanish shape one level down: spend={spend!r}", + ) + + +class ArgvPersistedAtLaunch(_FixLaunch): + def test_launched_record_carries_the_child_argv(self): + """Scenario: harness.argv is persisted in the launched record""" + os.environ["TASK_LAUNCH_SLEEP"] = "0.5" + seen = {} + + def watch(): + deadline = time.time() + 4 + while not self.start_witness.is_file() and time.time() < deadline: + time.sleep(0.01) + files = self.record_files() + seen["n_records"] = len(files) + if files: + data = json.loads(files[0].read_text(encoding="utf-8")) + seen["status"] = data.get("status") + harness = data.get("harness") or {} + seen["argv"] = harness.get("argv") + if self.start_witness.is_file(): + start = json.loads( + self.start_witness.read_text(encoding="utf-8"), + ) + seen["start_argv"] = start.get("argv") + + t = threading.Thread(target=watch) + t.start() + self.dispatch( + self.argv_for(job="plan", harness="grok", stage="plan"), + ) + t.join(timeout=6) + self.assertEqual( + seen.get("status"), "launched", + f"must sample the launched record: {seen!r}", + ) + argv = seen.get("argv") + self.assertIsInstance(argv, list, f"harness.argv type: {seen!r}") + self.assertTrue( + argv, + "harness.argv is persisted at launch, not first at settle: " + f"seen={seen!r}", + ) + start = seen.get("start_argv") or [] + self.assertTrue(start, f"child ran: {seen!r}") + self.assertEqual( + argv, start, + "on-disk harness.argv must be the child argv while status " + f"is launched: record={argv!r} child={start!r}", + ) + + +class NonZeroExitNamesTheExitInTheNote(_FixLaunch): + def test_exit_137_keeps_the_harness_note_and_names_the_code(self): + """Scenario: a non-zero exit keeps the harness note and names the exit""" + os.environ["TASK_LAUNCH_WRAPPER"] = "grok" + os.environ["TASK_LAUNCH_NOTE"] = "wrote the plan" + os.environ["TASK_LAUNCH_EXIT"] = "137" + rec, *_ = self._dispatch_closed( + job="plan", harness="grok", stage="plan", + ) + env = _envelope(rec) + note = str(env.get("note") or "") + self.assertIn( + "wrote the plan", note, + "harness note is present, not replaced: " + f"envelope={env!r}", + ) + self.assertIn( + "137", note, + "exit 137 must appear in the envelope note, not only in " + f"result.cause: note={note!r} " + f"cause={ (rec.get('result') or {}).get('cause')!r}", + ) + + +class FencedGrokEnvelope(_FixLaunch): + def test_fenced_pretty_printed_json_is_the_envelope(self): + """Scenario: a grok fenced json envelope on plain stdout is the envelope""" + path = ENVELOPES / GROK_FENCED_FIXTURE + self.assertTrue(path.is_file(), f"fixture missing: {path}") + body = path.read_text(encoding="utf-8") + self.assertTrue(body.strip(), f"fixture empty: {path}") + self.assertIn("```json", body, "plant: opening json fence") + self.assertIn("u10-grok-fenced", body, "plant: envelope note") + self.assertIn("```", body.rsplit("```json", 1)[-1]) + complete_lines = 0 + for ln in body.splitlines(): + s = ln.strip() + if not s.startswith("{"): + continue + try: + obj = json.loads(s) + except json.JSONDecodeError: + continue + if isinstance(obj, dict) and _has_nine(obj): + complete_lines += 1 + self.assertEqual( + complete_lines, 0, + "plant: no single line is a complete nine-key object " + "(a line-oriented scan would recover it without a fence)", + ) + interior = body.split("```json", 1)[1] + interior = interior.split("```", 1)[0] + planted = json.loads(interior) + self.assertTrue(_has_nine(planted), "plant: fence holds nine keys") + self.assertTrue( + _nested_counts_ok(planted), + f"plant: fenced object is nested-valid: {planted.get('counts')!r}", + ) + self.assertEqual(planted.get("note"), "u10-grok-fenced") + os.environ["TASK_LAUNCH_STDOUT"] = "token" + os.environ["TASK_LAUNCH_TOKEN"] = body.rstrip("\n") + rec, *_ = self._dispatch_closed( + job="plan", harness="grok", stage="plan", + ) + _raw, data = self._raw_out() + self.assertIn(b"```json", data, "plant: fence reached raw.out") + self.assertIn(b"u10-grok-fenced", data) + env = _envelope(rec) + blob = str(env.get("note") or "") + str(_cause_reason(rec) or "") + self.assertNotIn( + "envelope-parse", blob.lower(), + "D-ENV-3: a fenced grok envelope is not envelope-parse: " + f"note={env.get('note')!r} " + f"cause={ (rec.get('result') or {}).get('cause')!r}", + ) + self.assertEqual( + env.get("note"), "u10-grok-fenced", + "the pretty-printed fenced object is the envelope: " + f"envelope={env!r}", + ) + self.assertEqual(env.get("status"), "ok", env) + + +class PrivateEnvelopeCause(_FixLaunch): + def test_private_envelope_cause_key_stays_out_of_the_committed_record(self): + """Scenario: a committed record does not carry private envelope-scan keys""" + os.environ["TASK_LAUNCH_WRAPPER"] = "grok" + os.environ["TASK_LAUNCH_WRAPPER_INVALID"] = "extra" + os.environ["TASK_LAUNCH_NOTE"] = "u10-private-key" + os.environ["TASK_LAUNCH_EXIT"] = "1" + rec, *_ = self._dispatch_closed( + job="plan", harness="grok", stage="plan", + ) + _raw, data = self._raw_out() + planted = json.loads(data.decode("utf-8")) + self.assertIn("transcript", planted, "plant: extra key landed") + self.assertEqual(rec.get("status"), "closed", rec) + self.assertNotIn( + "_envelope_cause", rec, + "private scan key must not land in the committed record: " + f"keys={sorted(rec)}", + ) + result = rec.get("result") or {} + self.assertIsInstance(result, dict, result) + self.assertNotIn( + "_envelope_cause", result, + "private scan key must not land under result: " + f"result_keys={sorted(result)}", + ) + body = self.the_record_path().read_text(encoding="utf-8") + self.assertNotIn( + '"_envelope_cause"', body, + "committed record bytes must not carry _envelope_cause: " + f"body={body[:400]!r}", + ) diff --git a/ops/devlane/dispatch/tests/test_launch_grok_head_commit.py b/ops/devlane/dispatch/tests/test_launch_grok_head_commit.py new file mode 100644 index 0000000..b6f91b0 --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_grok_head_commit.py @@ -0,0 +1,92 @@ +"""Grok summary.json head_commit is judged against snapshot.ref_sha. + +Written from CONTRACT.md §Dispatch Isolation per harness, grok row: +``its head_commit is cross-checked against snapshot.ref_sha``. + +Observed: records 20260828T213936Z-tests-grok-44a721, +20260828T213939Z-tests-grok-7f670a and 20260828T214309Z-tests-grok-dcddad +closed with envelope status invalid, note 'head_commit mismatch', while +each snapshot held one new commit on top of the ref and summary.json +head_commit equalled the ref. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import launch_support as ls + + +class GrokHeadCommitIsJudgedAgainstTheRef(ls._TempLaunch): + """A write job that commits in its snapshot closes with head_commit + judged against the ref, not against the post-write HEAD.""" + + def test_a_write_job_that_commits_closes_with_head_commit_judged_against_the_ref( + self): + os.environ["TASK_LAUNCH_COMMIT"] = "worker.py" + os.environ["TASK_LAUNCH_HEAD_COMMIT"] = self.ref + os.environ["TASK_LAUNCH_VERDICT"] = "null" + code, out, err = self.dispatch(self.argv_for( + job="author-tests", harness="grok", stage="tests", + scope="pin grok head_commit against the ref", + )) + text = self.combined(out, err) + self.assertNotEqual( + code, ls.REFUSAL_EXIT, + f"a write job that commits must not refuse: {text!r}", + ) + rec = self.read_record() + self.assertEqual(rec.get("status"), "closed") + + result = rec.get("result") or {} + ref_sha = rec["snapshot"]["ref_sha"] + head = result.get("head") + self.assertEqual(ref_sha, self.ref) + self.assertTrue(head, "write collect records a head") + self.assertNotEqual( + head, ref_sha, + "plant: the snapshot must hold a new commit on top of the ref", + ) + snapshot = self.snapshot_of(rec) + self._git( + "merge-base", "--is-ancestor", ref_sha, head, repo=snapshot, + ) + porcelain = self.porcelain(repo=snapshot) + self.assertEqual( + porcelain, "", + f"plant: snapshot tree must be clean, got {porcelain!r}", + ) + + stream = rec.get("session", {}).get("stream") or "" + self.assertTrue(stream, "a grok stream path is recorded") + stream_path = Path(stream) + self.assertTrue(stream_path.is_file(), f"stream missing: {stream}") + self.assertEqual( + stream_path.name, "summary.json", + f"grok stream is summary.json, got {stream_path.name!r}", + ) + summary = json.loads(stream_path.read_text(encoding="utf-8")) + self.assertEqual( + summary.get("head_commit"), ref_sha, + "plant: summary.json head_commit must equal the ref", + ) + self.assertNotEqual( + summary.get("head_commit"), head, + "plant: head_commit equals the ref, not the post-write HEAD", + ) + + envelope = result.get("envelope") or {} + note = envelope.get("note") + self.assertNotEqual( + envelope.get("status"), "invalid", + "head_commit equal to snapshot.ref_sha is not a mismatch on a " + f"write that committed; envelope={envelope!r} rec={rec!r}", + ) + self.assertNotIn( + "head_commit mismatch", + str(note or "").lower(), + f"close must not name a mismatch when head_commit equals " + f"the ref: note={note!r}", + ) diff --git a/ops/devlane/dispatch/tests/test_launch_harness_fixes.py b/ops/devlane/dispatch/tests/test_launch_harness_fixes.py new file mode 100644 index 0000000..0134258 --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_harness_fixes.py @@ -0,0 +1,383 @@ +"""Harness fixes measured on 2026-08-29 (see the conductor's memo +`harness-bugs-2026-08-29.md`): read roles must be able to execute; grok +must never wait on a permission prompt; a harness that cannot commit +keeps its own message and attribution; what a harness leaves uncommitted +is preserved; a dispatch that ends without an envelope says why.""" + +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +import launch_support as ls + +LAUNCH = Path(__file__).resolve().parents[1] / "launch.py" + + +def _load(test): + return ls.load_path(test, LAUNCH, "launch_under_harness_fixes") + + +class ReadRolesCanExecute(ls._TempLaunch): + """B2: a skeptic that cannot run the suite is structural.""" + + def test_claude_argv_preapproves_verification_commands_only(self): + rec, witness, *_ = self.launch_ok( + job="check-tests", harness="claude", stage="check-tests", + ) + argv = [str(p) for p in witness["argv"]] + self.assertIn("--allowedTools", argv) + start = argv.index("--allowedTools") + 1 + rules = [] + for item in argv[start:]: + if item.startswith("--"): + break + rules.append(item) + for needed in ("Bash(python3 -m unittest *)", "Bash(ruff check *)", "Bash(cue vet *)", + "Bash(git diff *)", "Bash(git status *)", "Bash(python3 .dev/*)"): + self.assertIn(needed, rules) + # an interpreter is arbitrary code under the operator's uid (review ba0d93/ffaf14) + for leak in ("Bash(python3 *)", "Bash(python *)", "Bash(env *)", "Bash(find *)", + "Bash(cp *)", "Bash(sed *)", "Bash(sort *)", "Bash(mkdir *)", "Bash(*)", "Bash"): + self.assertNotIn(leak, rules) + for rule in rules: + self.assertTrue(rule.startswith("Bash("), rule) + for forbidden in ("git push", "git commit", "git reset", "docker", + "rm ", "curl", "wget", "ssh"): + self.assertNotIn(forbidden, rule) + self.assertIn("--disallowedTools", argv) + dstart = argv.index("--disallowedTools") + 1 + denied = [] + for item in argv[dstart:]: + if item.startswith("--"): + break + denied.append(item) + for tool in ("WebFetch", "WebSearch", "Agent", "Task"): + self.assertIn(tool, denied) + self.assertEqual(argv[argv.index("--permission-mode") + 1], "acceptEdits") + self.assertIn("--allowedTools", rec["harness"]["argv"]) + + def test_a_read_role_that_writes_into_the_snapshot_is_recorded_and_salvaged(self): + # The existing contract stands: a read role's stray write is recorded + # as residual (HEAD must still equal the ref), never a refusal that + # discards its report. What is new: the write is preserved. + os.environ["TASK_LAUNCH_EDIT"] = "README.md" + code, out, err = self.dispatch( + job="check-tests", harness="claude", stage="check-tests", + ) + self.assertNotEqual(code, ls.REFUSAL_EXIT, self.combined(out, err)) + rec = self.read_record() + env = rec["result"]["envelope"] + self.assertNotIn("read-role-residual", str(env.get("note", ""))) + self.assertTrue(rec["result"]["residual_paths"]) + salvage = rec["result"].get("residual_patch") + self.assertIsInstance(salvage, dict) + path = Path(salvage["path"]) + self.assertTrue(path.is_file(), salvage) + self.assertEqual(salvage["sha256"], ls.sha256_file(path)) + self.assertIn("README.md", path.read_text()) + self.assertIn("cause", rec["result"]) + + +class SpendIsMeasuredFromTheStore(ls._TempLaunch): + """U5: 105 of 105 records carried spend: null on 2026-08-29.""" + + def test_a_claude_dispatch_records_its_spend_from_the_fixture_store(self): + rec, *_ = self.launch_ok(job="plan", harness="claude", stage="plan") + spend = rec["session"]["spend"] + self.assertIsInstance(spend, dict) + self.assertNotIn("unresolved", spend, spend) + # fixtures/stores.py: input 30, cached 12000, output 500 + self.assertEqual(spend["input"], 30) + self.assertEqual(spend["cached"], 12000) + self.assertEqual(spend["output"], 500) + self.assertEqual(spend["total"], 12530) + self.assertTrue(spend["source"].endswith(".jsonl")) + + def test_a_codex_dispatch_records_the_last_cumulative_count(self): + rec, *_ = self.launch_ok(job="implement", harness="codex", stage="code") + spend = rec["session"]["spend"] + self.assertIsInstance(spend, dict) + self.assertNotIn("unresolved", spend, spend) + for key in ("input", "output", "total"): + self.assertIsInstance(spend.get(key), int, spend) + self.assertGreater(spend["total"], 0) + + def test_a_grok_store_without_usage_events_is_an_explicit_gap(self): + rec, *_ = self.launch_ok(job="author-tests", harness="grok", stage="tests") + spend = rec["session"]["spend"] + self.assertIsInstance(spend, dict) + # either measured tokens or a stated gap — never a zero + self.assertTrue(("total" in spend and spend["total"] > 0) or "unresolved" in spend, spend) + self.assertNotEqual(spend.get("total"), 0) + + +class GrokNeverWaitsOnAPrompt(ls._TempLaunch): + """B1: 111 prompts, the last cancelled after 30 s, no envelope.""" + + def test_grok_argv_carries_always_approve_and_no_web_and_no_permission_mode(self): + rec, witness, *_ = self.launch_ok( + job="author-tests", harness="grok", stage="tests", + ) + argv = [str(p) for p in witness["argv"]] + self.assertIn("--always-approve", argv) + self.assertIn("--disable-web-search", argv) + self.assertNotIn("--permission-mode", argv) + self.assertEqual(rec["harness"]["sandbox"], "always-approve") + + def test_grok_resume_argv_keeps_the_same_flags(self): + launch = _load(self) + argv = launch._argv("grok", "grok-4.6", None, "sid", Path("/x/prompt.txt"), + ["--flag"], "always-approve", resume=True) + self.assertIn("--always-approve", argv) + self.assertIn("--disable-web-search", argv) + self.assertIn("--flag", argv) + self.assertIn("-r", argv) + # U10: --json-schema (implies json). plain was the pre-U10 argv. + self.assertNotIn("--json-schema", argv) # grok 1.0.5 short-circuits under --json-schema (record 66ccb2) + if "--output-format" in argv: + self.assertEqual( + argv[argv.index("--output-format") + 1], "plain", + ) + + +class TheLauncherCommitsWithTheHarnessOwnMessage(unittest.TestCase): + """B3: five green codex runs landed under a generic subject.""" + + def setUp(self): + self.launch = _load(self) + self.rec = {"job": "implement", "id": "20260829T000000Z-code-codex-abc123", + "harness": {"name": "codex"}, + "model": {"requested": "gpt-5.6-sol", "ran": "gpt-5.6-sol"}} + + def test_subject_body_and_display_name_attribution(self): + env = {"commit": {"subject": "infra: run jobs in the host lane", + "body": "Why and how.\n\nCo-Authored-By: GPT-5 Codex \nSource: original"}} + msg = self.launch._commit_message(self.rec, env) + lines = msg.rstrip("\n").splitlines() + self.assertEqual(lines[0], "infra: run jobs in the host lane") + self.assertIn("Why and how.", msg) + self.assertEqual(msg.count("Co-Authored-By:"), 1) + self.assertIn("Co-Authored-By: GPT-5.6 Sol ", msg) + self.assertNotIn("gpt-5.6-sol <", msg) + self.assertIn("Dispatch: 20260829T000000Z-code-codex-abc123", msg) + self.assertEqual(lines[-2:], ["Source: original", + "Co-Authored-By: GPT-5.6 Sol "]) + + def test_the_harness_own_source_line_is_kept_not_stripped(self): + env = {"commit": {"subject": "x", "body": "b\n\nSource: owner 2026-08-29\nSource: original\nCo-Authored-By: GPT-5 Codex "}} + msg = self.launch._commit_message(self.rec, env) + lines = msg.rstrip("\n").splitlines() + block_start = lines.index("Source: owner 2026-08-29") + self.assertEqual(lines[block_start:], ["Source: owner 2026-08-29", "Source: original", + "Co-Authored-By: GPT-5.6 Sol "]) + self.assertEqual(msg.count("Source: original"), 1) + + def test_a_second_co_author_from_the_harness_is_kept(self): + env = {"commit": {"subject": "x", "body": "b\n\nCo-Authored-By: GPT-5 Codex \nCo-Authored-By: Grok 4.6 "}} + msg = self.launch._commit_message(self.rec, env) + self.assertIn("Co-Authored-By: Grok 4.6 ", msg) + self.assertNotIn("GPT-5 Codex", msg) # the running model's own line replaces its vendor's + self.assertEqual(msg.count(""), 1) + + def test_without_a_commit_object_the_generic_subject_stands(self): + msg = self.launch._commit_message(self.rec, {}) + self.assertTrue(msg.startswith("implement: work of dispatch 20260829T000000Z-code-codex-abc123\n")) + self.assertIn("Co-Authored-By: GPT-5.6 Sol ", msg) + + def test_an_unknown_model_id_is_credited_as_itself(self): + self.rec["model"] = {"requested": "gpt-9", "ran": "gpt-9"} + msg = self.launch._commit_message(self.rec, {"commit": {"subject": "x"}}) + self.assertIn("Co-Authored-By: gpt-9 ", msg) + + def test_trailer_shaped_body_lines_move_into_the_final_block(self): + env = {"commit": {"subject": "workflow: make vocabulary evidence honest", + "body": "Why.\n\nReviewed-by: Grok 4.6 \nReviewed-by: Claude Opus 5 \n\nMore why."}} + msg = self.launch._commit_message(self.rec, env) + lines = msg.rstrip("\n").splitlines() + # the final block is contiguous and holds every trailer + block_start = lines.index("Source: original") + self.assertEqual(lines[block_start:], [ + "Source: original", + "Reviewed-by: Grok 4.6 ", + "Reviewed-by: Claude Opus 5 ", + "Co-Authored-By: GPT-5.6 Sol ", + ]) + self.assertEqual(lines[block_start - 1], "") + body = "\n".join(lines[2:block_start - 1]) + self.assertNotIn("Reviewed-by", body) + self.assertIn("Why.", body) + self.assertIn("More why.", body) + + def test_out_commit_msg_wins_over_the_envelope(self): + with tempfile.TemporaryDirectory() as td: + job_dir = Path(td) + (job_dir / "out").mkdir() + (job_dir / "out" / "COMMIT_MSG").write_text("infra: from the file\n\nBody from file.\n") + msg = self.launch._commit_message(self.rec, {"commit": {"subject": "from envelope"}}, job_dir) + self.assertTrue(msg.startswith("infra: from the file\n\nBody from file.\n")) + + def test_a_blank_subject_falls_back(self): + msg = self.launch._commit_message(self.rec, {"commit": {"subject": " ", "body": "b"}}) + self.assertTrue(msg.startswith("implement: work of dispatch")) + + +class ADispatchThatEndsWithoutAnEnvelopeSaysWhy(unittest.TestCase): + """B4: the record carried only 'no JSON object on stdout'.""" + + def setUp(self): + self.launch = _load(self) + self._td = tempfile.TemporaryDirectory() + self.job_dir = Path(self._td.name) + + def tearDown(self): + self._td.cleanup() + + def _grok_events(self, lines): + d = self.job_dir / "home" / "grok-stream" / "sessions" / "enc" / "sid" + d.mkdir(parents=True) + (d / "events.jsonl").write_text("\n".join(json.dumps(x) for x in lines) + "\n") + return d + + def test_a_cancelled_permission_prompt_is_named(self): + self._grok_events([ + {"ts": "t1", "type": "permission_requested", "tool_name": "run_terminal_command"}, + {"ts": "t2", "type": "permission_resolved", "tool_name": "run_terminal_command", + "decision": "cancelled", "wait_ms": 30002}, + {"ts": "t3", "type": "phase_changed"}, + {"ts": "t4", "type": "turn_ended"}, + ]) + rec = {"harness": {"name": "grok"}, "session": {"stream": None}} + ended = self.launch._cause(rec, self.job_dir) + self.assertEqual(ended["reason"], "permission-cancelled") + self.assertEqual(ended["tool"], "run_terminal_command") + self.assertEqual(ended["wait_ms"], 30002) + + def test_an_approved_prompt_followed_by_work_is_not_blamed(self): + d = self._grok_events([ + {"ts": "t1", "type": "permission_requested", "tool_name": "run_terminal_command"}, + {"ts": "t2", "type": "permission_resolved", "tool_name": "run_terminal_command", + "decision": "approved", "wait_ms": 5}, + {"ts": "t3", "type": "assistant_message"}, + ]) + rec = {"harness": {"name": "grok"}, "session": {"stream": str(d / "updates.jsonl")}} + ended = self.launch._cause(rec, self.job_dir) + self.assertEqual(ended["reason"], "last-event") + self.assertEqual(ended["type"], "assistant_message") + + def test_a_recovered_prompt_is_not_blamed(self): + self._grok_events([ + {"ts": "t1", "type": "permission_resolved", "tool_name": "run_terminal_command", + "decision": "cancelled", "wait_ms": 30002}, + {"ts": "t2", "type": "tool_result"}, + {"ts": "t3", "type": "permission_resolved", "tool_name": "run_terminal_command", + "decision": "approved", "wait_ms": 3}, + {"ts": "t4", "type": "assistant_message"}, + {"ts": "t5", "type": "turn_ended"}, + ]) + rec = {"harness": {"name": "grok"}, "session": {"stream": None}} + self.assertEqual(self.launch._cause(rec, self.job_dir)["reason"], "last-event") + + def test_the_launchers_own_marker_names_the_kill_never_the_harness_prose(self): + self._grok_events([{"ts": "t1", "type": "permission_resolved", "decision": "cancelled", + "tool_name": "x", "wait_ms": 1}, {"ts": "t2", "type": "turn_ended"}]) + rec = {"harness": {"name": "grok"}, "session": {"stream": None}, + "attempts": [{"exit": 137, "tripped": True}]} + # production shape: TRIPPED.md written by the launcher, exit 137, tripped True + (self.job_dir / "TRIPPED.md").write_text("timeout: harness exceeded 2700s\n") + self.assertEqual(self.launch._cause(rec, self.job_dir)["reason"], "timeout") + (self.job_dir / "TRIPPED.md").write_text("unsupervised: no session stream within grace\n") + self.assertEqual(self.launch._cause(rec, self.job_dir)["reason"], "unsupervised") + (self.job_dir / "TRIPPED.md").write_text("trip: cap-out 500000 exceeded\n") + self.assertEqual(self.launch._cause(rec, self.job_dir)["reason"], "tripped") + (self.job_dir / "TRIPPED.md").unlink() + rec["attempts"] = [{"exit": 137, "tripped": False}] + self.assertEqual(self.launch._cause(rec, self.job_dir)["reason"], "harness-cli:137") + rec["attempts"] = [{"exit": 0, "tripped": False}] + # the harness's own prose never decides the reason + cause = self.launch._cause(rec, self.job_dir, {"status": "ok", "note": "no timeout was observed; unsupervised runs are fine"}) + self.assertEqual(cause["reason"], "permission-cancelled") + + def test_the_cancelled_tools_own_result_is_not_recovery(self): + self._grok_events([ + {"ts": "t1", "type": "permission_requested", "tool_name": "run_terminal_command"}, + {"ts": "t2", "type": "permission_resolved", "tool_name": "run_terminal_command", + "decision": "cancelled", "wait_ms": 30002}, + {"ts": "t2b", "type": "tool_result"}, + {"ts": "t3", "type": "phase_changed"}, + {"ts": "t4", "type": "turn_ended"}, + ]) + rec = {"harness": {"name": "grok"}, "session": {"stream": None}} + self.assertEqual(self.launch._cause(rec, self.job_dir)["reason"], "permission-cancelled") + + def test_a_claude_stream_reports_its_last_event(self): + stream = self.job_dir / "session.jsonl" + stream.write_text('{"type":"user","timestamp":"a"}\n{"type":"result","timestamp":"b"}\n') + rec = {"harness": {"name": "claude"}, "session": {"stream": str(stream)}} + ended = self.launch._cause(rec, self.job_dir) + self.assertEqual(ended, {"reason": "last-event", "type": "result", "at": "b"}) + + def test_no_store_is_said_not_guessed(self): + rec = {"harness": {"name": "codex"}, "session": {"stream": None}} + self.assertEqual(self.launch._cause(rec, self.job_dir)["reason"], "no-session-store") + + +class WhatAHarnessLeavesUncommittedIsPreserved(unittest.TestCase): + """B4: the conductor diffed a dead dispatch's snapshot by hand.""" + + def setUp(self): + self.launch = _load(self) + self._td = tempfile.TemporaryDirectory() + self.repo = Path(self._td.name) / "snap" + self.job_dir = Path(self._td.name) + self.repo.mkdir() + env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@t", + GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="t@t") + self.env = env + for args in (("init", "-q"), ): + subprocess.run(["git", *args], cwd=self.repo, check=True, env=env) + (self.repo / "a.txt").write_text("one\n") + (self.repo / ".gitignore").write_text("__pycache__/\n") + subprocess.run(["git", "add", "-A"], cwd=self.repo, check=True, env=env) + subprocess.run(["git", "commit", "-q", "-m", "base"], cwd=self.repo, check=True, env=env) + + def tearDown(self): + self._td.cleanup() + + def test_tracked_edits_and_untracked_files_land_in_residual_patch(self): + (self.repo / "a.txt").write_text("two\n") + (self.repo / "new.py").write_text("print(1)\n") + salvage = self.launch._write_residual_patch(self.repo, self.job_dir) + text = Path(salvage["path"]).read_text() + self.assertIn("-one", text) + self.assertIn("+two", text) + self.assertIn("+++ b/new.py", text) + self.assertIn("print(1)", text) + self.assertEqual(salvage["untracked"], ["new.py"]) + self.assertEqual(salvage["sha256"], ls.sha256_file(Path(salvage["path"]))) + + def test_tool_caches_are_not_in_the_patch_either(self): + (self.repo / ".ruff_cache").mkdir() + (self.repo / ".ruff_cache" / "x").write_text("c") + (self.repo / "real.txt").write_text("r") + salvage = self.launch._write_residual_patch(self.repo, self.job_dir) + text = Path(salvage["path"]).read_text() + self.assertIn("real.txt", text) + self.assertNotIn(".ruff_cache", text) + + def test_tool_caches_are_not_residual(self): + (self.repo / ".ruff_cache").mkdir() + (self.repo / ".ruff_cache" / "x").write_text("c") + (self.repo / "pkg" / "__pycache__").mkdir(parents=True) + (self.repo / "pkg" / "__pycache__" / "m.pyc").write_bytes(b"\x00") + self.assertEqual(self.launch._residual(self.repo), []) + (self.repo / "real.txt").write_text("r") + self.assertEqual([line[3:] for line in self.launch._residual(self.repo)], ["real.txt"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/dispatch/tests/test_launch_isolation.py b/ops/devlane/dispatch/tests/test_launch_isolation.py new file mode 100644 index 0000000..978c366 --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_isolation.py @@ -0,0 +1,460 @@ +"""Isolation, child env, stream discovery, session ids. + +Written from CONTRACT.md §Dispatch Isolation, child's environment, +Watching. Plan items (g)(h)(j)(k)(t)(u). + + I1 claude flags; HOME untouched; CLAUDE_CONFIG_DIR unset + I2 CODEX_HOME / GROK_HOME+HOME point at a home holding exactly auth.json + I3 agent-env: CLICOLOR_FORCE absent, NO_COLOR set, WF_LANE, DISPATCH_JOB + I4 stream chosen under the isolated store, not a newer ~/.codex + I5 unsupervised: live, no stream within grace → terminated, invalid + I6 minted session id on argv; codex from session_meta + I7 a fake that ignores --session-id is a recorded mismatch + I8 observed is unresolved or verbatim, never false +""" + +from __future__ import annotations + +import json +import os +import time +from pathlib import Path + +import launch_support as ls + + +class ClaudeIsolationIsFlagsAndHomeUntouched(ls._TempLaunch): + """I1 / contract isolation table / plan (k).""" + + def test_claude_argv_carries_the_isolation_flags_and_session_id(self): + rec, witness, *_ = self.launch_ok( + job="plan", harness="claude", stage="plan", + ) + argv = [str(p) for p in witness["argv"]] + self.assertIn("--setting-sources", argv) + self.assertEqual( + argv[argv.index("--setting-sources") + 1], "project,local", + ) + self.assertIn("--strict-mcp-config", argv) + self.assertIn("--disable-slash-commands", argv) + self.assertIn("--session-id", argv) + minted = argv[argv.index("--session-id") + 1] + self.assertTrue(minted, "a session id is minted at launch") + self.assertEqual(rec["session"]["id"], minted) + self.assertIn("--print", argv) + self.assertIn("--permission-mode", argv) + # acceptEdits, not plan: a plan job writes out/PLAN.md and plan mode + # ends the turn at a plan (2026-08-28). out/ is the only added dir. + self.assertEqual(argv[argv.index("--permission-mode") + 1], "acceptEdits") + self.assertIn("--add-dir", argv) + self.assertTrue(argv[argv.index("--add-dir") + 1].endswith("/out")) + env = witness["env"] + self.assertEqual(env.get("HOME"), str(self.home)) + self.assertNotIn("CLAUDE_CONFIG_DIR", env) + self.assertEqual(rec["harness"]["isolation"]["mechanism"], "flags") + self.assertEqual( + os.path.realpath(witness["cwd"]), + os.path.realpath(self.snapshot_of(rec)), + ) + + +class MinimalHomesHoldExactlyAuthJson(ls._TempLaunch): + """I2 / plan (k).""" + + def test_codex_home_is_the_job_home_holding_exactly_auth_json(self): + rec, witness, *_ = self.launch_ok( + job="plan", harness="codex", stage="plan", + ) + env = witness["env"] + home = env.get("CODEX_HOME") + self.assertTrue(home, "CODEX_HOME must be set") + job_dir = self.the_job_dir() + self.assertEqual( + os.path.realpath(home), + os.path.realpath(job_dir / "home" / "codex"), + ) + names = sorted(p.name for p in Path(home).iterdir()) + self.assertEqual(names, ["auth.json"]) + auth = Path(home) / "auth.json" + self.assertTrue(auth.is_symlink() or auth.is_file()) + self.assertNotEqual(env.get("HOME"), home) + self.assertEqual(rec["harness"]["containment"], "os") + argv = [str(p) for p in witness["argv"]] + self.assertIn("--sandbox", argv) + # workspace-write for read roles too: read-only denied the job its + # own out/ deliverable and a writable tempdir for the suite it must + # run (2026-08-28, check-tests: "no affected assertion ran"). The + # snapshot's integrity is proved after the run instead. + self.assertEqual(argv[argv.index("--sandbox") + 1], "workspace-write") + self.assertIn("-c", argv) + self.assertTrue(any(a.startswith("sandbox_workspace_write.writable_roots=") and a.endswith('/out"]') for a in argv)) + + def test_grok_home_and_home_both_point_at_the_job_home(self): + rec, witness, *_ = self.launch_ok( + job="plan", harness="grok", stage="plan", + ) + env = witness["env"] + grok_home = env.get("GROK_HOME") + home = env.get("HOME") + self.assertTrue(grok_home and home) + job_dir = self.the_job_dir() + expected = os.path.realpath(job_dir / "home" / "grok") + self.assertEqual(os.path.realpath(grok_home), expected) + self.assertEqual(os.path.realpath(home), expected) + names = sorted(p.name for p in Path(grok_home).iterdir()) + self.assertEqual(names, ["auth.json"]) + self.assertNotEqual( + os.path.realpath(home), os.path.realpath(self.home), + "operator HOME must not leak into the grok child", + ) + self.assertEqual(rec["harness"]["containment"], "policy") + + +class AgentEnvIsAppliedToTheChild(ls._TempLaunch): + """I3 / contract child's environment.""" + + def test_clicolor_force_is_absent_and_no_color_is_set(self): + os.environ["CLICOLOR_FORCE"] = "1" + os.environ["FORCE_COLOR"] = "1" + rec, witness, *_ = self.launch_ok( + job="plan", harness="grok", stage="plan", + ) + env = witness["env"] + self.assertNotIn("CLICOLOR_FORCE", env) + self.assertNotIn("FORCE_COLOR", env) + self.assertEqual(env.get("NO_COLOR"), "1") + self.assertEqual(env.get("CLICOLOR"), "0") + self.assertEqual(env.get("TERM"), "dumb") + self.assertEqual(env.get("PAGER"), "cat") + self.assertEqual(env.get("GH_PAGER"), "cat") + self.assertEqual(env.get("GIT_PAGER"), "cat") + self.assertEqual(env.get("CI"), "true") + self.assertEqual(env.get("GIT_TERMINAL_PROMPT"), "0") + self.assertEqual(env.get("GIT_EDITOR"), "true") + self.assertEqual(env.get("EDITOR"), "true") + self.assertEqual(env.get("PYTHONUNBUFFERED"), "1") + self.assertEqual(env.get("PYTHONIOENCODING"), "utf-8") + self.assertEqual(env.get("LC_ALL"), "C.UTF-8") + self.assertEqual(env.get("WF_LANE"), "dev") + self.assertEqual(env.get("DISPATCH_JOB"), rec["id"]) + self.assertEqual(env.get("LESS"), "FRX") + + +class StreamIsFoundUnderTheIsolatedStore(ls._TempLaunch): + """I4 / plan (j) — a newer stream planted in ~/.codex is not chosen.""" + + def test_a_newer_operator_codex_stream_is_not_chosen(self): + stores = ls.load_path(self, ls.STORES_PATH, "task_launch_stores") + operator_root = self.home / ".codex" + decoy_id = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" + stores.build_codex_store( + operator_root, + base_timestamp=ls.STREAM_EPOCH + 50_000, + cwd="/decoy/cwd", + session_id=decoy_id, + model="decoy-operator-model", + effort="high", + marker="DECOY", + ) + rollouts = list(operator_root.glob("sessions/*/*/*/rollout-*.jsonl")) + self.assertEqual(len(rollouts), 1, "plant: exactly one decoy stream") + decoy = rollouts[0] + # Newer mtime than anything the fake will write during launch. + os.utime(decoy, (2_000_000_000, 2_000_000_000)) + self.assertGreater(decoy.stat().st_mtime, 1_900_000_000) + self.assertIn(b"decoy-operator-model", decoy.read_bytes()) + + rec, witness, *_ = self.launch_ok( + job="plan", harness="codex", stage="plan", + ) + self.assertEqual(rec["model"]["ran"], ls.RAN_MODEL) + self.assertNotEqual(rec["model"]["ran"], "decoy-operator-model") + stream = rec["session"]["stream"] or "" + self.assertTrue(stream, "a stream path is recorded") + self.assertNotIn(str(decoy), stream) + isolated_home = witness["env"].get("CODEX_HOME") + self.assertTrue(isolated_home) + self.assertIn(os.path.realpath(isolated_home), os.path.realpath(stream)) + self.assertNotIn( + os.path.realpath(operator_root / "sessions"), + os.path.realpath(stream), + ) + self.assertNotEqual(rec["model"]["ran"], "decoy-operator-model") + self.assertNotIn( + "TASK_LAUNCH_RAN_MODEL", os.environ, + "ran must be parsed from the stream, not copied from the env", + ) + + def test_a_newer_operator_claude_stream_is_not_chosen(self): + stores = ls.load_path(self, ls.STORES_PATH, "task_launch_stores") + decoy_id = "cccccccc-cccc-4ccc-8ccc-cccccccccccc" + decoy_root = self.home / ".claude" / "projects" + stores.build_claude_store( + decoy_root, "decoy-slug", + base_timestamp=ls.STREAM_EPOCH + 50_000, + cwd="/decoy/cwd", session_id=decoy_id, + model="decoy-claude-model", effort="high", + marker="DECOY-CLAUDE", + ) + decoys = list(decoy_root.glob("decoy-slug/*.jsonl")) + self.assertEqual(len(decoys), 1, "plant: one decoy claude stream") + os.utime(decoys[0], (2_000_000_000, 2_000_000_000)) + self.assertIn(b"decoy-claude-model", decoys[0].read_bytes()) + rec, *_ = self.launch_ok(job="plan", harness="claude", stage="plan") + self.assertEqual(rec["model"]["ran"], ls.RAN_MODEL) + self.assertNotEqual(rec["model"]["ran"], "decoy-claude-model") + stream = rec["session"]["stream"] or "" + self.assertTrue(stream) + self.assertNotIn("decoy-slug", stream) + self.assertNotIn(decoy_id, stream) + + def test_a_newer_operator_grok_stream_is_not_chosen(self): + stores = ls.load_path(self, ls.STORES_PATH, "task_launch_stores") + decoy_id = "dddddddd-dddd-4ddd-8ddd-dddddddddddd" + operator = self.home / ".grok" + stores.build_grok_store( + operator, "/decoy/cwd", + base_timestamp=ls.STREAM_EPOCH + 50_000, + session_id=decoy_id, model="decoy-grok-model", + marker="DECOY-GROK", + ) + decoys = list(operator.glob("sessions/*/*/summary.json")) + self.assertGreaterEqual(len(decoys), 1, "plant: grok decoy stream") + self.assertIn(b"decoy-grok-model", decoys[0].read_bytes()) + rec, witness, *_ = self.launch_ok( + job="plan", harness="grok", stage="plan", + ) + self.assertEqual(rec["model"]["ran"], ls.RAN_MODEL) + self.assertNotEqual(rec["model"]["ran"], "decoy-grok-model") + stream = rec["session"]["stream"] or "" + isolated = witness["env"].get("GROK_HOME") + self.assertTrue(isolated) + self.assertIn(os.path.realpath(isolated), os.path.realpath(stream)) + self.assertNotIn(decoy_id, stream) + + +class UnsupervisedLiveProcessIsTerminated(ls._TempLaunch): + """I5 / contract unsupervised / plan (j). + + Seam: ``DISPATCH_STREAM_GRACE`` (default 120) and injected + ``launch.monotonic`` / ``launch.sleep``. The contract pins 120-second + behaviour, not a Python attribute named STREAM_GRACE. + """ + + def _invalid_envelope(self, rec, text): + envelope = rec.get("result") or {} + if isinstance(envelope, dict): + envelope = envelope.get("envelope") or envelope + status = (envelope or {}).get("status") or rec.get("status") + self.assertTrue( + status == "invalid" + or (isinstance(envelope, dict) + and envelope.get("status") == "invalid"), + f"unsupervised run must surface invalid: rec={rec!r} text={text!r}", + ) + self.assertTrue( + any(w in text.lower() + for w in ("unsupervised", "no stream", "invalid", "store")), + f"must name the store searched: {text!r}", + ) + self.assertIsNone(rec["model"]["ran"]) + + def test_no_stream_within_grace_kills_the_process_group(self): + self.set_grace(0.3) + os.environ["TASK_LAUNCH_WRITE_STREAM"] = "0" + os.environ["TASK_LAUNCH_SLEEP"] = "8" + os.environ["TASK_LAUNCH_GRANDCHILD"] = str(self.grandchild) + + def _reap(): + if self.start_witness.is_file(): + info = json.loads( + self.start_witness.read_text(encoding="utf-8") + ) + ls.kill_if_alive(info.get("pid")) + ls.kill_if_alive(info.get("pgid")) + if self.grandchild.is_file(): + gpid = self.grandchild.read_text(encoding="utf-8").strip() + ls.kill_if_alive(gpid) + + self.addCleanup(_reap) + started = time.monotonic() + code, out, err = self.dispatch(self.argv_for( + job="plan", harness="codex", stage="plan", + )) + elapsed = time.monotonic() - started + rec = self.read_record() + text = self.combined(out, err) + json.dumps(rec) + self.assertNotEqual(code, 0) + self._invalid_envelope(rec, text) + self.assertGreaterEqual(elapsed, 0.25, "grace must actually wait") + self.assertLess(elapsed, 4, "must not wait out the 8s child") + started_info = self.read_start_witness() + pid = started_info["pid"] + pgid = started_info["pgid"] + self.assertFalse( + ls.pid_is_alive(pid), + f"harness pid {pid} was left running after unsupervised kill", + ) + self.assertFalse( + ls.pid_is_alive(pgid), + f"process group {pgid} was left running", + ) + if self.grandchild.is_file(): + gpid = int(self.grandchild.read_text(encoding="utf-8").strip()) + self.assertFalse( + ls.pid_is_alive(gpid), + f"grandchild {gpid} survived the group kill", + ) + + def test_default_grace_is_one_hundred_and_twenty_seconds_on_the_clock( + self): + os.environ.pop("DISPATCH_STREAM_GRACE", None) + clock = self.attach_clock() + os.environ["TASK_LAUNCH_WRITE_STREAM"] = "0" + os.environ["TASK_LAUNCH_SLEEP"] = "30" + code, out, err = self.dispatch(self.argv_for( + job="plan", harness="codex", stage="plan", + )) + rec = self.read_record() + text = self.combined(out, err) + json.dumps(rec) + self.assertNotEqual(code, 0) + self._invalid_envelope(rec, text) + waited = clock.now if clock.sleeps else sum(clock.sleeps) + self.assertGreaterEqual( + max(clock.now, sum(clock.sleeps), waited), + ls.DEFAULT_GRACE, + f"default grace is 120s; clock advanced {clock.now} " + f"sleeps={clock.sleeps}", + ) + if self.start_witness.is_file(): + pid = self.read_start_witness()["pid"] + self.assertFalse(ls.pid_is_alive(pid)) + + +class SessionIdIsMintedOrRead(ls._TempLaunch): + """I6 / plan (h).""" + + def test_grok_receives_dash_s_uuid(self): + rec, witness, *_ = self.launch_ok( + job="plan", harness="grok", stage="plan", + ) + argv = [str(p) for p in witness["argv"]] + self.assertIn("-s", argv) + minted = argv[argv.index("-s") + 1] + self.assertEqual(rec["session"]["id"], minted) + self.assertRegex( + minted, + r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + ) + + def test_codex_session_id_is_read_from_session_meta(self): + rec, *_ = self.launch_ok(job="plan", harness="codex", stage="plan") + stream = rec["session"]["stream"] + self.assertTrue(stream) + body = Path(stream).read_text(encoding="utf-8") + self.assertIn(rec["session"]["id"], body) + self.assertIn("session_meta", body) + + +class IgnoringTheMintedIdIsAMismatch(ls._TempLaunch): + """I7 / plan (t).""" + + def test_a_fake_that_ignores_session_id_is_recorded_as_a_mismatch(self): + os.environ["TASK_LAUNCH_IGNORE_SESSION"] = "1" + rec, witness, *_ = self.launch_ok( + job="plan", harness="claude", stage="plan", + ) + argv = [str(p) for p in witness["argv"]] + self.assertIn("--session-id", argv) + minted = argv[argv.index("--session-id") + 1] + ignored = "ffffffff-ffff-4fff-8fff-ffffffffffff" + self.assertNotEqual(minted, ignored) + blob = json.dumps(rec).lower() + self.assertIn("mismatch", blob) + self.assertIn(minted, json.dumps(rec)) + self.assertIn(ignored, json.dumps(rec)) + self.assertNotEqual(rec["session"]["id"], ignored) + + +class ObservedIsUnresolvedOrVerbatim(ls._TempLaunch): + """I8 / plan (u). Tests do not run a live behavioural probe.""" + + def test_observed_is_unresolved_or_a_probe_dict_never_a_bare_false(self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + observed = rec["harness"]["isolation"]["observed"] + self.assertIsInstance(observed, dict) + self.assertNotEqual(observed, False) + self.assertNotEqual(observed, True) + if "unresolved" in observed: + self.assertTrue(str(observed["unresolved"]).strip()) + else: + self.assertIn("evidence", observed) + self.assertIn("checked_at", observed) + self.assertIn("harness_version", observed) + self.assertIn("operator_config_present", observed) + # A real probe may report False. A bare `observed: false` + # (the manufactured shape) is already refused above. + + +class CodexReadsTheBriefOnStdin(ls._TempLaunch): + """codex exec with DEVNULL on stdin exits 1 "No prompt provided via + stdin" before any work. Three dispatches on 2026-08-28 closed + harness-cli: exited 1 with empty findings and were committed as records.""" + + def test_codex_argv_ends_in_dash_and_stdin_carries_the_brief(self): + rec, witness, *_ = self.launch_ok( + job="plan", harness="codex", stage="plan", + ) + argv = [str(p) for p in witness["argv"]] + self.assertEqual(Path(argv[0]).name, "codex") + self.assertEqual(argv[1], "exec") + self.assertEqual(argv[-1], "-") + brief = (Path(rec["snapshot"]["root"]).parent / "prompt.txt").read_text( + encoding="utf-8", + ) + self.assertTrue(brief, "the rendered brief is empty") + self.assertEqual(witness["stdin"], brief) + + +class ClaudeReadsTheBriefOnStdin(ls._TempLaunch): + """claude --print with DEVNULL on stdin exits 1 "Input must be provided + either through stdin or as a prompt argument" (dispatch + 20260828T204050Z-plan-claude-61a0f6, harness-cli: exited 1).""" + + def test_claude_stdin_carries_the_brief(self): + rec, witness, *_ = self.launch_ok( + job="plan", harness="claude", stage="plan", + ) + brief = (Path(rec["snapshot"]["root"]).parent / "prompt.txt").read_text( + encoding="utf-8", + ) + self.assertTrue(brief, "the rendered brief is empty") + self.assertEqual(witness["stdin"], brief) + + +class GrokArgvCarriesTheSandboxTheRecordStates(ls._TempLaunch): + """A record saying sandbox: plan over an argv with no --permission-mode + claims a permission mode the child never had (dispatch + 20260828T194753Z-review-grok-6a8e2c).""" + + def test_grok_argv_carries_the_sandbox_the_record_states_and_plain_output(self): + # The record states `always-approve`; grok's flag for it is + # `--always-approve`, not a --permission-mode value. Under `auto` a + # session raised 111 prompts and the last timed out after 30 s on a + # non-interactive stdin (2026-08-29, tests-grok-abfc3b). + # U10: grok --json-schema implies json output; plain was the + # scan-fallback argv and contradicts structured envelopes. + rec, witness, *_ = self.launch_ok( + job="plan", harness="grok", stage="plan", + ) + argv = [str(p) for p in witness["argv"]] + self.assertEqual(rec["harness"]["sandbox"], "always-approve") + self.assertIn("--always-approve", argv) + self.assertNotIn("--permission-mode", argv) + self.assertNotIn("--json-schema", argv) # grok 1.0.5 short-circuits under --json-schema (record 66ccb2) + if "--output-format" in argv: + self.assertEqual( + argv[argv.index("--output-format") + 1], "plain", + f"U10: grok output is plain, not json; argv={argv!r}", + ) + self.assertIn("--prompt-file", argv) diff --git a/ops/devlane/dispatch/tests/test_launch_job_caps.py b/ops/devlane/dispatch/tests/test_launch_job_caps.py new file mode 100644 index 0000000..4d601ce --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_job_caps.py @@ -0,0 +1,186 @@ +"""Per-job runtime caps: resolution order, policy values, enforcement. + +Written from CONTRACT.md §The record, the ``caps.timeout`` bullet: +``DISPATCH_TIMEOUT`` from the invoking environment, else the job's own +``caps.timeout`` in jobs.json, else 900 — and ``caps.timeout_source`` +names the layer that answered: ``DISPATCH_TIMEOUT``, ``job``, or +``default``. Policy values and the author-tests commit-on-red clause +come from ``ops/devlane/task/jobs.json``. launch.py and record.py were +not read. +""" + +from __future__ import annotations + +import json +import os +import time +import unittest + +import launch_support as ls + +DEFAULT_TIMEOUT = 900.0 +POLICY_CAPS = { + "author-tests": 1800, + "check-tests": 1800, + "adversarial-review": 1200, +} +COMMIT_ON_RED = "Commit each test file as soon as its red is proven" +JOB_TIMEOUT = 42 +ENV_TIMEOUT = 123.5 +LOOP_TIMEOUT = 0.3 + + +def _caps(test, rec): + test.assertIsInstance(rec, dict) + test.assertIn("caps", rec) + caps = rec["caps"] + test.assertIsInstance(caps, dict) + test.assertIn("timeout", caps) + test.assertIn("timeout_source", caps) + timeout = caps["timeout"] + test.assertIsInstance(timeout, (int, float)) + test.assertNotIsInstance(timeout, bool) + test.assertIsInstance(caps["timeout_source"], str) + return caps + + +def plant_job_timeout(test, job, seconds): + """Write ``caps.timeout`` onto one job in the fixture catalog.""" + path = test.jobs_file + before_obj = json.loads(path.read_text(encoding="utf-8")) + test.assertIsInstance(before_obj, dict) + test.assertIn(job, before_obj) + prior = (before_obj.get(job) or {}).get("caps") + prior_timeout = None if not isinstance(prior, dict) else prior.get("timeout") + test.assertNotEqual( + prior_timeout, seconds, + f"plant {job} timeout={seconds} was already the fixture", + ) + + def mutate(raw): + jobs = json.loads(raw) + spec = dict(jobs[job]) + spec["caps"] = {"timeout": seconds} + jobs[job] = spec + return (json.dumps(jobs, indent=2) + "\n").encode("utf-8") + + ls.plant_bytes( + path, + mutate, + expect="grow", + recognisable=lambda after: ( + job.encode("utf-8") in after and b'"timeout"' in after + ), + ) + landed = json.loads(path.read_text(encoding="utf-8")) + test.assertEqual(landed[job]["caps"]["timeout"], seconds) + test.assertNotEqual(landed[job].get("caps"), prior) + test.ref = test._commit(f"plant {job} caps.timeout={seconds}") + return seconds + + +class JobTimeoutIsRecordedWhenEnvIsAbsent(ls._TempLaunch): + """Else the job's own caps.timeout; source is job.""" + + def test_a_job_cap_is_recorded_as_timeout_with_source_job(self): + self.assertNotIn("DISPATCH_TIMEOUT", os.environ) + plant_job_timeout(self, "plan", JOB_TIMEOUT) + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + caps = _caps(self, rec) + self.assertEqual(float(caps["timeout"]), float(JOB_TIMEOUT)) + self.assertNotEqual(float(caps["timeout"]), DEFAULT_TIMEOUT) + self.assertEqual(caps["timeout_source"], "job") + + +class EnvTimeoutWinsOverTheJobCap(ls._TempLaunch): + """DISPATCH_TIMEOUT from the invoking environment wins; source names it.""" + + def test_dispatch_timeout_beats_the_job_cap_and_names_its_source(self): + plant_job_timeout(self, "plan", JOB_TIMEOUT) + os.environ["DISPATCH_TIMEOUT"] = str(ENV_TIMEOUT) + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + caps = _caps(self, rec) + self.assertEqual(float(caps["timeout"]), float(ENV_TIMEOUT)) + self.assertNotEqual(float(caps["timeout"]), float(JOB_TIMEOUT)) + self.assertNotEqual(float(caps["timeout"]), DEFAULT_TIMEOUT) + self.assertEqual(caps["timeout_source"], "DISPATCH_TIMEOUT") + + +class DefaultTimeoutWhenNeitherLayerAnswers(ls._TempLaunch): + """Else 900, source default — a job with no caps entry and no env.""" + + def test_no_job_cap_and_no_env_records_900_from_default(self): + self.assertNotIn("DISPATCH_TIMEOUT", os.environ) + jobs = json.loads(self.jobs_file.read_text(encoding="utf-8")) + self.assertNotIn("caps", jobs["plan"]) + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + caps = _caps(self, rec) + self.assertEqual(float(caps["timeout"]), DEFAULT_TIMEOUT) + self.assertEqual(caps["timeout_source"], "default") + + +class JobsJsonCarriesExactlyTheThreePolicyCaps(unittest.TestCase): + """author-tests 1800, check-tests 1800, adversarial-review 1200.""" + + def test_exactly_three_jobs_carry_the_named_caps(self): + self.assertTrue(ls.JOBS_PATH.is_file(), f"missing {ls.JOBS_PATH}") + jobs = json.loads(ls.JOBS_PATH.read_text(encoding="utf-8")) + self.assertIsInstance(jobs, dict) + self.assertGreater(len(jobs), 0, "jobs.json must name jobs") + carrying = {} + for name, spec in jobs.items(): + self.assertIsInstance(spec, dict, f"{name} spec is not an object") + if "caps" not in spec: + continue + caps = spec["caps"] + self.assertIsInstance(caps, dict, f"{name} caps is not an object") + self.assertIn("timeout", caps, f"{name} caps has no timeout") + carrying[name] = caps["timeout"] + self.assertEqual( + len(carrying), len(POLICY_CAPS), + f"expected {len(POLICY_CAPS)} jobs with caps, got {carrying!r}", + ) + self.assertEqual(set(carrying), set(POLICY_CAPS)) + self.assertEqual(carrying, POLICY_CAPS) + + +class AuthorTestsPromptContainsTheCommitOnRedClause(unittest.TestCase): + """The author-tests prompt asks to commit each test file on proven red.""" + + def test_author_tests_prompt_contains_the_commit_on_red_clause(self): + self.assertTrue(ls.JOBS_PATH.is_file(), f"missing {ls.JOBS_PATH}") + jobs = json.loads(ls.JOBS_PATH.read_text(encoding="utf-8")) + self.assertIn("author-tests", jobs) + spec = jobs["author-tests"] + self.assertIsInstance(spec, dict) + prompt = spec.get("prompt") + self.assertIsInstance(prompt, str) + self.assertTrue(prompt.strip(), "author-tests prompt is empty") + self.assertIn(COMMIT_ON_RED, prompt) + + +class SupervisionEnforcesTheResolvedJobCap(ls._TempLaunch): + """The resolved cap, not a flat 900, is what the supervision loop enforces.""" + + def test_a_short_job_cap_trips_the_loop_without_waiting_out_900(self): + self.assertNotIn("DISPATCH_TIMEOUT", os.environ) + plant_job_timeout(self, "plan", LOOP_TIMEOUT) + os.environ["TASK_LAUNCH_SLEEP"] = "8" + os.environ["TASK_LAUNCH_WRITE_STREAM"] = "1" + started = time.monotonic() + self.dispatch(self.argv_for(job="plan", harness="grok", stage="plan")) + elapsed = time.monotonic() - started + rec = self.read_record() + self.assertLess( + elapsed, 4, + f"job cap {LOOP_TIMEOUT}s must not wait out 8s, " + f"let alone 900s: elapsed={elapsed}", + ) + blob = json.dumps(rec).lower() + self.assertIn("timeout", blob) + self.assertTrue( + self.start_witness.is_file(), + "plant: the harness child started before the timeout", + ) + pid = self.read_start_witness()["pid"] + self.assertFalse(ls.pid_is_alive(pid)) diff --git a/ops/devlane/dispatch/tests/test_launch_prompt_feed_findings.py b/ops/devlane/dispatch/tests/test_launch_prompt_feed_findings.py new file mode 100644 index 0000000..33c984a --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_prompt_feed_findings.py @@ -0,0 +1,737 @@ +"""Red pins for dispatch/prompt-feed review findings F1, F3-F10, F12. + +Authored from CONTRACT.md §Dispatch (Collect, Isolation, The job +directory, Watching, Template values) plus the findings at +``.dev/records/dispatches/20260828T221050Z-review-claude-4db38d.json``. +F2 is out of scope (not reproduced). F11 names CONTRACT.md text to +correct, not a test. + + F1 read role that commits → envelope invalid (head must equal ref_sha) + F3 claude --add-dir must not grant the job-directory evidence files + F4 launcher snapshot commit identity ignores GIT_AUTHOR_* env + F5 timed-out codex write is not committed by the launcher + F6 close of a finished uncollected job reads the stream + F7 narration after a pretty-printed envelope is tolerated + F8 a runtime note keeps the envelope-parse reason + F9 stale origin/dev is not preferred; an empty {diff} is loud + F10 resume feeds the brief on stdin and keeps the launch flags + F12 a missing envelope key refuses without discarding the payload +""" + +from __future__ import annotations + +import json +import os +import subprocess +import time +from pathlib import Path + +import launch_support as ls + +OWNER_IDENT = "xormania <127287135+xormania@users.noreply.github.com>" +OWNER_NAME = "xormania" +OWNER_EMAIL = "127287135+xormania@users.noreply.github.com" + +DECOY_AUTHOR_NAME = "decoy-author" +DECOY_AUTHOR_EMAIL = "decoy-author@example.invalid" +DECOY_COMMITTER_NAME = "decoy-committer" +DECOY_COMMITTER_EMAIL = "decoy-committer@example.invalid" + +ENVELOPE_KEYS = ( + "job", "status", "verdict", "counts", "findings", + "artifacts", "spend", "stamp", "note", +) + +EVIDENCE_NAMES = ( + "raw.out", "stderr", "exit", "state.json", "prompt.txt", + "TRIPPED.md", "breaker.log", +) + + +def _add_dirs(argv, cwd): + found = [] + argv = [str(a) for a in argv] + i = 0 + while i < len(argv): + a = argv[i] + raw = None + if a == "--add-dir" and i + 1 < len(argv): + raw = argv[i + 1] + i += 2 + elif a.startswith("--add-dir="): + raw = a.split("=", 1)[1] + i += 1 + else: + i += 1 + continue + if not raw: + continue + p = Path(raw) + if not p.is_absolute(): + p = Path(cwd) / p + found.append(p.resolve()) + return found + + +def _under(path, root): + if path == root: + return True + try: + path.relative_to(root) + except ValueError: + return False + return True + + +def _envelope_obj(job="plan", *, omit=(), extra=None): + data = { + "job": job, + "status": "ok", + "verdict": "approve", + "counts": {"p1": 0, "p2": 0, "p3": 0, "opinions": 0}, + "findings": [], + "artifacts": {}, + "spend": {"harness": "grok", "total": 0, "out": 0, "runs": 1}, + "stamp": {"ref": "harness-placeholder", "started": None, "ended": None}, + "note": None, + } + if extra: + data.update(extra) + for key in omit: + data.pop(key, None) + return data + + +class _ParserLaunch(ls._TempLaunch): + """Helpers for the stdout-envelope parser.""" + + def parse_envelope(self, raw, job="plan", sha="SHA", note=None): + launch = self.load_launch() + fn = getattr(launch, "_envelope", None) + self.assertTrue(callable(fn), "launch._envelope parses stdout") + got = fn(raw, job, sha) if note is None else fn(raw, job, sha, note) + if isinstance(got, dict): + return got + if isinstance(got, tuple) and got and isinstance(got[0], dict): + return got[0] + self.fail(f"_envelope returned {type(got).__name__}: {got!r}") + + def envelope_of(self, rec): + result = rec.get("result") or {} + if isinstance(result, dict): + env = result.get("envelope") + if isinstance(env, dict): + return env + self.fail(f"record has no result.envelope: {rec!r}") + + +class ReadRoleThatCommitsIsInvalid(_ParserLaunch): + """F1 / CONTRACT.md §Collect: Read roles: head must equal ref_sha. + + The existing ReadRoleHeadEqualsRef case never writes. A read role + that commits must not close with envelope status intact. + """ + + def test_a_read_role_that_commits_closes_invalid_not_ok(self): + os.environ["TASK_LAUNCH_COMMIT"] = "worker.py" + os.environ["TASK_LAUNCH_HEAD_COMMIT"] = self.ref + os.environ["TASK_LAUNCH_VERDICT"] = "approve" + _code, _out, _err = self.dispatch(self.argv_for( + job="plan", harness="grok", stage="plan", + )) + rec = self.read_record() + self.assertEqual(rec["role"], "read") + result = rec["result"] + ref_sha = rec["snapshot"]["ref_sha"] + self.assertEqual(ref_sha, self.ref) + self.assertNotEqual( + result.get("head"), ref_sha, + "plant: the read-role worker committed on top of the ref", + ) + changed = ls.changed_entries(result.get("changed_paths")) + self.assertTrue( + any("worker.py" in str(item) for item in changed), + f"plant: worker.py is in changed_paths, got {changed!r}", + ) + envelope = self.envelope_of(rec) + self.assertEqual( + envelope.get("status"), "invalid", + "CONTRACT.md §Collect: read roles, head must equal ref_sha; " + "a commit in the snapshot is not recorded as an ordinary " + f"ok/approve close: envelope={envelope!r}", + ) + + +class ClaudeAddDirDoesNotExposeJobBookkeeping(_ParserLaunch): + """F3 / CONTRACT.md §The job directory. + + Bookkeeping (raw.out, exit, state.json, prompt.txt, …) is kept + away from the harness tree. --add-dir may name in/ and out/; it + must not name the job directory that holds the evidence files. + """ + + def test_claude_add_dir_does_not_include_job_directory_evidence_files( + self): + rec, witness, *_ = self.launch_ok( + job="plan", harness="claude", stage="plan", + ) + argv = [str(a) for a in witness["argv"]] + self.assertEqual(Path(argv[0]).name, "claude") + job_dir = self.the_job_dir().resolve() + self.assertTrue((job_dir / "prompt.txt").is_file(), "plant: prompt.txt") + self.assertTrue((job_dir / "raw.out").is_file(), "plant: raw.out") + self.assertTrue((job_dir / "exit").is_file(), "plant: exit") + roots = _add_dirs(argv, witness["cwd"]) + self.assertNotIn( + job_dir, roots, + f"claude --add-dir must not grant the job directory " + f"itself (evidence files live there): add-dir={roots!r}", + ) + present = [] + for name in EVIDENCE_NAMES: + path = (job_dir / name).resolve() + if path.is_file(): + present.append(path) + self.assertTrue(present, "plant: job-directory evidence files exist") + leaked = [] + for path in present: + for root in roots: + if _under(path, root): + leaked.append((path.name, str(root))) + self.assertEqual( + leaked, [], + "CONTRACT.md §The job directory: launcher evidence files " + f"must sit outside every --add-dir, leaked={leaked!r} " + f"add-dir={[str(r) for r in roots]} rec={rec['id']}", + ) + + +class SnapshotCommitIdentityIgnoresEnvOverrides(_ParserLaunch): + """F4 / AGENTS.md §Attribution. + + GIT_AUTHOR_NAME and GIT_COMMITTER_* in the launcher's environment + must not stamp the snapshot commit the launcher makes for a codex + write job that left residual paths. + """ + + def _plant_decoy_identities(self): + plants = { + "GIT_AUTHOR_NAME": DECOY_AUTHOR_NAME, + "GIT_AUTHOR_EMAIL": DECOY_AUTHOR_EMAIL, + "GIT_COMMITTER_NAME": DECOY_COMMITTER_NAME, + "GIT_COMMITTER_EMAIL": DECOY_COMMITTER_EMAIL, + } + for key, value in plants.items(): + os.environ[key] = value + self.env[key] = value + self.assertEqual(os.environ[key], value) + self.assertNotEqual(value, OWNER_NAME) + self.assertNotEqual(value, OWNER_EMAIL) + author = subprocess.run( + ["git", "var", "GIT_AUTHOR_IDENT"], + cwd=self.repo, capture_output=True, text=True, + ) + committer = subprocess.run( + ["git", "var", "GIT_COMMITTER_IDENT"], + cwd=self.repo, capture_output=True, text=True, + ) + self.assertEqual(author.returncode, 0, author.stderr) + self.assertEqual(committer.returncode, 0, committer.stderr) + self.assertIn(DECOY_AUTHOR_NAME, author.stdout) + self.assertIn(DECOY_AUTHOR_EMAIL, author.stdout) + self.assertIn(DECOY_COMMITTER_NAME, committer.stdout) + self.assertIn(DECOY_COMMITTER_EMAIL, committer.stdout) + self.assertNotIn(OWNER_NAME, author.stdout) + self.assertNotIn(OWNER_EMAIL, author.stdout) + + def test_launcher_snapshot_commit_is_the_owner_despite_git_author_env( + self): + self._plant_decoy_identities() + os.environ["TASK_LAUNCH_EDIT"] = "scratch.txt" + os.environ["TASK_LAUNCH_VERDICT"] = "null" + rec, *_ = self.launch_ok(self.argv_for( + job="implement", harness="codex", stage="code", + scope="python3 -m unittest", + )) + result = rec["result"] + ref_sha = rec["snapshot"]["ref_sha"] + head = result.get("head") + self.assertTrue(head, "plant: launcher made a snapshot commit") + self.assertNotEqual( + head, ref_sha, + "plant: a codex write with residual paths is committed", + ) + snap = self.snapshot_of(rec) + author = self._git( + "log", "-1", "--format=%an <%ae>", head, repo=snap, + ).stdout.strip() + committer = self._git( + "log", "-1", "--format=%cn <%ce>", head, repo=snap, + ).stdout.strip() + decoy_author = f"{DECOY_AUTHOR_NAME} <{DECOY_AUTHOR_EMAIL}>" + decoy_committer = ( + f"{DECOY_COMMITTER_NAME} <{DECOY_COMMITTER_EMAIL}>" + ) + self.assertEqual(author, OWNER_IDENT) + self.assertEqual(committer, OWNER_IDENT) + self.assertNotEqual(author, decoy_author) + self.assertNotEqual(committer, decoy_committer) + + +class TrippedWriteIsNotCommittedByTheLauncher(_ParserLaunch): + """F5 / a timeout or trip must not manufacture a snapshot commit.""" + + def test_a_timed_out_codex_write_is_not_committed_by_the_launcher(self): + os.environ["DISPATCH_TIMEOUT"] = "0.5" + os.environ["TASK_LAUNCH_SLEEP"] = "8" + os.environ["TASK_LAUNCH_EDIT"] = "scratch.txt" + os.environ["TASK_LAUNCH_WRITE_STREAM"] = "1" + os.environ["TASK_LAUNCH_VERDICT"] = "null" + started = time.monotonic() + _code, out, err = self.dispatch(self.argv_for( + job="implement", harness="codex", stage="code", + scope="python3 -m unittest", + )) + elapsed = time.monotonic() - started + rec = self.read_record() + text = (self.combined(out, err) + json.dumps(rec)).lower() + self.assertLess(elapsed, 4, f"timeout must not wait out 8s: {elapsed}") + self.assertIn("timeout", text) + self.assertTrue( + self.start_witness.is_file(), + "plant: the harness child started before the timeout", + ) + snap = self.snapshot_of(rec) + scratch = snap / "scratch.txt" + log_names = self._git( + "log", "-1", "--name-only", "--format=", repo=snap, + ).stdout + self.assertTrue( + scratch.is_file() or "scratch.txt" in log_names, + "plant: the half-edit landed in the snapshot " + f"(porcelain={self.porcelain(repo=snap)!r} log={log_names!r})", + ) + result = rec["result"] + ref_sha = rec["snapshot"]["ref_sha"] + self.assertEqual( + result.get("head"), ref_sha, + "a timed-out codex write must not get a launcher-made " + f"commit: head={result.get('head')!r} ref={ref_sha!r} " + f"envelope={(result.get('envelope') or {})!r}", + ) + dispatch_ref = f"refs/dispatch/{rec['id']}" + refs = self.refs_map() + if dispatch_ref in refs: + self.assertEqual( + refs[dispatch_ref], ref_sha, + "refs/dispatch must not point at a manufactured commit " + "from a timed-out run", + ) + + +class CloseCollectsTheHarnessStream(_ParserLaunch): + """F6 / close ID must relocate and read the stream the way launch does.""" + + def test_close_of_a_finished_uncollected_job_sets_model_ran_from_the_stream( + self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + src_snap = Path(rec["snapshot"]["root"]) + self.assertTrue(src_snap.is_dir(), "plant: first snapshot exists") + + job_id = "20260826T000000Z-plan-grok-cl0se6" + d = self.jobs_root / job_id + d.mkdir() + snap = d / "snapshot" + self._git("clone", "--no-hardlinks", str(src_snap), str(snap)) + self.assertTrue((snap / ".git").exists() or (snap / ".git").is_file()) + + session_id = "22222222-2222-4222-8222-222222222222" + home = d / "home" / "grok" + home.mkdir(parents=True) + (home / "auth.json").write_text("{}\n", encoding="utf-8") + stores = ls.load_path(self, ls.STORES_PATH, "task_launch_stores") + stores.build_grok_store( + home, str(snap.resolve()), + base_timestamp=ls.STREAM_EPOCH, + session_id=session_id, + model=ls.RAN_MODEL, + marker="CLOSE-COLLECT", + head_commit=self.ref, + git_root_dir=str(snap.resolve()), + grok_home=str(home), + ) + streams = list(home.rglob("summary.json")) + self.assertEqual(len(streams), 1, "plant: one grok stream in job home") + planted_model = json.loads( + streams[0].read_text(encoding="utf-8"), + ).get("current_model_id") + self.assertEqual(planted_model, ls.RAN_MODEL) + + envelope = _envelope_obj(job="plan") + (d / "raw.out").write_text( + json.dumps(envelope) + "\n", encoding="utf-8", + ) + (d / "exit").write_text("0\n", encoding="utf-8") + (d / "prompt.txt").write_text("planted brief\n", encoding="utf-8") + pid = self.dead_pid() + (d / "state.json").write_text(json.dumps({ + "pid": pid, "pgid": pid, + "session": {"id": session_id}, + "stream": None, "attempt": 1, + }) + "\n", encoding="utf-8") + self.assertTrue((d / "exit").is_file(), "plant: exit is present") + self.assertTrue(streams[0].is_file(), "plant: stream sits on disk") + + seed = json.loads(json.dumps(rec)) + seed["id"] = job_id + seed["status"] = "launched" + seed["result"] = None + seed["at"]["closed"] = None + seed["model"]["ran"] = None + seed["model"]["read_from"] = None + seed["model"].pop("note", None) + seed["session"] = { + "id": session_id, "stream": None, + "stream_sha256_at_close": None, + } + seed["snapshot"]["root"] = str(snap) + seed["harness"]["name"] = "grok" + iso = seed["harness"].setdefault("isolation", {}) + iso["home"] = str(home) + iso["store"] = str(home / "sessions") + iso["env"] = {"GROK_HOME": str(home), "HOME": str(home)} + seed["harness"]["argv"] = [ + "grok", "-s", session_id, "--model", ls.REQUESTED_MODEL, + "--output-format", "plain", "--permission-mode", "auto", + "--prompt-file", str(d / "prompt.txt"), + ] + record_path = ( + self.repo / ".dev" / "records" / "dispatches" / f"{job_id}.json" + ) + record_path.write_text( + json.dumps(seed, indent=2) + "\n", encoding="utf-8", + ) + planted = json.loads(record_path.read_text(encoding="utf-8")) + self.assertEqual(planted["status"], "launched") + self.assertIsNone(planted["result"]) + self.assertIsNone(planted["model"]["ran"]) + self.assertIsNone(planted["session"]["stream"]) + planted_store = Path( + planted["harness"]["isolation"]["store"], + ).resolve() + self.assertEqual(planted_store, (home / "sessions").resolve()) + self.assertTrue( + str(planted_store) in str(streams[0].resolve()), + "plant: the stream sits under this job's isolation.store, " + f"store={planted_store} stream={streams[0]}", + ) + + _code, out, err = self.run_main(["close", job_id]) + rec2 = json.loads(record_path.read_text(encoding="utf-8")) + blob = (json.dumps(rec2) + self.combined(out, err)).lower() + self.assertEqual( + rec2["model"]["ran"], ls.RAN_MODEL, + "close must read model.ran from the stream sitting in the " + f"job home, not leave it null: rec={rec2!r} out={out!r} " + f"err={err!r}", + ) + stream = rec2.get("session", {}).get("stream") + self.assertTrue( + stream, + f"close must set session.stream: rec={rec2!r}", + ) + self.assertTrue( + Path(stream).is_file(), + f"session.stream must name a file that exists: {stream!r}", + ) + self.assertTrue( + rec2.get("session", {}).get("stream_sha256_at_close"), + "close must set stream_sha256_at_close", + ) + self.assertNotIn("no stream was found", blob) + + +class TrailingNarrationAfterPrettyEnvelopeIsTolerated(_ParserLaunch): + """F7 / narration after a pretty-printed object is tolerated.""" + + def test_narration_after_a_pretty_printed_envelope_does_not_drop_it(self): + pretty = json.dumps(_envelope_obj(job="plan"), indent=2) + self.assertIn("\n", pretty, "plant: the object spans several lines") + trailing = "tokens used: 12345" + raw = (pretty + "\n" + trailing + "\n").encode("utf-8") + self.assertTrue( + raw.strip().endswith(trailing.encode("utf-8")), + "plant: narration sits after the closing brace", + ) + self.assertIn(b"\n}", raw) + got = self.parse_envelope(raw, "plan", "SHA") + note = str(got.get("note") or "") + self.assertNotEqual( + got.get("status"), "invalid", + "narration after a pretty-printed envelope must not be " + f"recorded as unparseable: {got!r}", + ) + self.assertNotIn("envelope-parse", note) + self.assertEqual(got.get("job"), "plan") + self.assertEqual(got.get("status"), "ok") + + +class RuntimeNoteDoesNotDestroyParseNote(_ParserLaunch): + """F8 / a runtime note appends; it does not replace a parse diagnostic.""" + + def test_a_runtime_note_keeps_the_envelope_parse_reason(self): + raw = b"not json\n" + self.assertFalse(raw.lstrip().startswith(b"{"), "plant: not an object") + got = self.parse_envelope( + raw, "plan", "SHA", "head_commit mismatch", + ) + note = str(got.get("note") or "") + self.assertIn( + "envelope-parse", note, + "the parse reason must survive a supplied runtime note: " + f"note={note!r} envelope={got!r}", + ) + self.assertIn("head_commit mismatch", note) + self.assertNotEqual( + note.strip(), "head_commit mismatch", + "overwriting the parse diagnostic with only the runtime " + f"note loses why the envelope itself was lost: {note!r}", + ) + + +class ReviewDiffUsesTheMostRecentAnchor(_ParserLaunch): + """F9 / review base is the most recent resolvable merge-base. + + origin/dev at root is stale; local dev at mid is the recent fork + point. A zero-byte {diff} on a job whose template names {{diff}} + is a silent failure, not a success. + """ + + def test_stale_origin_dev_is_not_preferred_over_local_dev(self): + self._git("branch", "dev", self.mid_sha) + self._git("update-ref", "refs/remotes/origin/dev", self.root_sha) + self.assertEqual( + self._git("rev-parse", "dev").stdout.strip(), self.mid_sha, + ) + self.assertEqual( + self._git("rev-parse", "refs/remotes/origin/dev").stdout.strip(), + self.root_sha, + ) + self.assertNotEqual(self.mid_sha, self.root_sha) + self.assertNotEqual(self.mid_sha, self.ref) + self.assertNotEqual(self.root_sha, self.ref) + + rec, witness, *_ = self.launch_ok(self.argv_for( + job="adversarial-review", harness="grok", stage="review", + scope="pin the review merge-base", + )) + prompt = witness.get("prompt_text") or "" + if not prompt: + prompt = (self.the_job_dir() / "prompt.txt").read_text( + encoding="utf-8", + ) + self.assertIn(self.ref, prompt, "{ref} lands in the brief") + self.assertIn( + self.mid_sha, prompt, + "the most recent resolvable merge-base is local dev " + f"(mid={self.mid_sha}); prompt={prompt!r}", + ) + self.assertNotIn( + self.root_sha, prompt, + "stale origin/dev (root) must not win over local dev: " + f"prompt={prompt!r}", + ) + job_dir = self.the_job_dir() + diff_path = job_dir / "diff.patch" + self.assertTrue( + diff_path.is_file(), + f"{{diff}} is written to the job directory: {list(job_dir.iterdir())}", + ) + body = diff_path.read_text(encoding="utf-8") + self.assertTrue( + body.strip(), + "a job whose template names {diff} must not proceed silently " + f"with a zero-byte diff.patch (base==ref would be empty); " + f"rec base={rec.get('lineage')!r} ref={self.ref}", + ) + self.assertIn("alpha v2", body) + self.assertIn("alpha v3", body) + self.assertNotIn( + "alpha v1", body, + "diff against stale origin/dev (root) includes alpha v1; " + "the recent fork point is mid (v2→v3)", + ) + + +class ResumeFeedsTheBriefAndKeepsFlags(_ParserLaunch): + """F10 / resume feeds the brief on stdin and carries the launch flags. + + Look up each dispatch by id. After a second launch both records + remain at .dev/records/dispatches/.json (CONTRACT.md §The + record); the_job_dir / the_record_path "exactly one" helpers + must not be used here — they pinned the archive behaviour G2 + now forbids. + """ + + def test_claude_and_codex_resume_feed_the_brief_on_stdin(self): + rec, first, *_ = self.launch_ok( + job="plan", harness="claude", stage="plan", + ) + id_a = rec["id"] + record_a = ( + self.repo / ".dev" / "records" / "dispatches" / f"{id_a}.json" + ) + job_a = self.jobs_root / id_a + self.assertTrue( + record_a.is_file(), + f"plant: first record is at the canonical path {record_a}", + ) + self.assertTrue( + job_a.is_dir(), + f"plant: first job directory is under jobs-root {job_a}", + ) + brief = (job_a / "prompt.txt").read_text(encoding="utf-8") + self.assertTrue(brief, "plant: prompt.txt holds the rendered brief") + self.assertEqual(first.get("stdin"), brief) + first_argv = [str(a) for a in first["argv"]] + self.assertIn("--add-dir", first_argv) + self.assertIn("--model", first_argv) + + self.witness.unlink() + code, out, err = self.run_main(["resume", id_a]) + self.assertEqual(code, 0, self.combined(out, err)) + second = self.read_witness() + argv = [str(a) for a in second["argv"]] + self.assertEqual(Path(argv[0]).name, "claude") + self.assertEqual( + second.get("stdin"), brief, + "claude resume must feed prompt.txt on stdin; " + f"got {second.get('stdin')!r}", + ) + self.assertTrue(second.get("stdin"), "resume stdin is empty") + self.assertIn("--add-dir", argv) + self.assertIn("--model", argv) + self.assertEqual( + argv[argv.index("--model") + 1], ls.REQUESTED_MODEL, + ) + self.assertTrue( + record_a.is_file(), + "resume itself must not move the record off the canonical path", + ) + + self.witness.unlink() + code, out, err = self.dispatch(self.argv_for( + job="plan", harness="codex", stage="plan", + )) + self.assertEqual( + code, 0, + f"second launch must succeed: {self.combined(out, err)}", + ) + self.assertTrue( + record_a.is_file(), + "CONTRACT.md §The record: one file per dispatch at " + f".dev/records/dispatches/{id_a}.json; a later launch " + "must not move a closed, resumed job's record off the " + "canonical path", + ) + self.assertTrue( + job_a.is_dir(), + f"job directory must stay at {job_a}", + ) + records = self.record_files() + names = [p.name for p in records] + self.assertEqual( + len(records), 2, + "CONTRACT.md §The record: one file per dispatch, both " + "stay at the canonical top-level path after a second " + f"launch; got {names!r}", + ) + self.assertIn(f"{id_a}.json", names) + new_files = [p for p in records if p.stem != id_a] + self.assertEqual( + len(new_files), 1, + f"the second launch adds one canonical record, got {names!r}", + ) + rec_x = self.read_record(new_files[0]) + self.assertEqual(rec_x.get("status"), "closed") + id_x = rec_x["id"] + self.assertNotEqual(id_x, id_a) + job_x = self.jobs_root / id_x + self.assertTrue( + job_x.is_dir(), + f"plant: second job directory is at {job_x}", + ) + first_x = self.read_witness() + brief_x = (job_x / "prompt.txt").read_text(encoding="utf-8") + self.assertTrue(brief_x, "plant: codex prompt.txt") + self.assertEqual(first_x.get("stdin"), brief_x) + self.witness.unlink() + code, out, err = self.run_main(["resume", id_x]) + self.assertEqual(code, 0, self.combined(out, err)) + second_x = self.read_witness() + self.assertEqual(Path(second_x["argv"][0]).name, "codex") + self.assertEqual( + second_x.get("stdin"), brief_x, + "codex resume must feed prompt.txt on stdin; " + f"got {second_x.get('stdin')!r}", + ) + self.assertTrue( + record_a.is_file(), + "the first record stays at its canonical path after the " + "codex resume", + ) + self.assertTrue( + new_files[0].is_file(), + "the second record stays at its canonical path after resume", + ) + self.assertEqual( + len(self.record_files()), 2, + "both records remain top-level after the second resume", + ) + + +class MissingEnvelopeKeyKeepsThePayload(_ParserLaunch): + """F12 / a missing key refuses without discarding the parsed payload.""" + + def test_a_missing_spend_key_keeps_findings_counts_and_verdict(self): + extra = { + "job": "adversarial-review", + "status": "ok", + "verdict": "changes", + "counts": {"p1": 1, "p2": 0, "p3": 0, "opinions": 0}, + "findings": [{ + "id": "X1", "severity": "p1", "file": "a.py", + "finding": "kept", + }], + "artifacts": {"report": "out/R.md"}, + "note": None, + } + payload = _envelope_obj(job="adversarial-review", omit=("spend",), + extra=extra) + self.assertNotIn("spend", payload) + self.assertEqual(len(payload["findings"]), 1) + self.assertEqual(payload["verdict"], "changes") + for key in ENVELOPE_KEYS: + if key == "spend": + self.assertNotIn(key, payload) + else: + self.assertIn(key, payload) + raw = json.dumps(payload).encode("utf-8") + got = self.parse_envelope(raw, "adversarial-review", "SHA") + note = str(got.get("note") or "") + self.assertEqual( + got.get("status"), "invalid", + "a missing key is a refusal, not a default: " + f"envelope={got!r}", + ) + self.assertIn("envelope-missing", note) + self.assertIn("spend", note) + self.assertEqual( + got.get("findings"), payload["findings"], + "the parsed findings must survive a missing-key refusal: " + f"envelope={got!r}", + ) + self.assertEqual(got.get("verdict"), "changes") + self.assertEqual(got.get("counts"), payload["counts"]) + self.assertEqual(got.get("artifacts"), payload["artifacts"]) + self.assertEqual(got.get("job"), "adversarial-review") diff --git a/ops/devlane/dispatch/tests/test_launch_record.py b/ops/devlane/dispatch/tests/test_launch_record.py new file mode 100644 index 0000000..976dbc5 --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_record.py @@ -0,0 +1,268 @@ +"""The dispatch record as a committed file, filled from a real launch. + +Written from CONTRACT.md §Dispatch The record. Plan items (f)(g)(m)(o)(q). + + R1 field order, id form, lane, dispatched_by + R2 follows and unit default and land + R3 model.ran from fixture streams, or null with a note + R4 brief --check re-derives; a planted digest fails + R5 caps equal wires.py for the role + R6 behind_tip is a count, not a judgement +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path + +import launch_support as ls + + +class RecordShapeFromAClosedDispatch(ls._TempLaunch): + """R1 — the file on disk is the contract's ordered record.""" + + def test_closed_record_keys_are_the_contracted_order(self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + self.assertEqual(tuple(rec), ls.RECORD_FIELDS) + self.assertEqual(rec["lane"], "dev") + self.assertEqual(rec["stage"], "plan") + self.assertEqual(rec["job"], "plan") + self.assertEqual(rec["role"], "read") + self.assertEqual(rec["dispatched_by"], ls.AGENT) + self.assertRegex(rec["id"], ls.ID_RE) + self.assertEqual(rec["id"].split("-")[1], "plan") + self.assertIn("grok", rec["id"]) + self.assertEqual(rec["lineage"]["branch"], "work") + self.assertIsInstance(rec["at"], dict) + self.assertIn("launched", rec["at"]) + self.assertIn("closed", rec["at"]) + self.assertTrue(rec["at"]["launched"]) + self.assertTrue(rec["at"]["closed"]) + snap = rec["snapshot"] + self.assertEqual(snap["mode"], "whole") + self.assertEqual(snap["ref_sha"], self.ref) + self.assertEqual(self.the_job_dir().name, rec["id"]) + self.assertEqual( + rec["harness"]["containment"], "policy", + ) + + def test_the_record_commit_is_on_the_lineage_branch(self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + self.assertEqual( + self._git("rev-parse", "--abbrev-ref", "HEAD").stdout.strip(), + rec["lineage"]["branch"], + ) + self.assertEqual(rec["lineage"]["branch"], "work") + self.assertIn( + rec["id"], + (self.repo / ".dev" / "records" / "dispatches" / f"{rec['id']}.json" + ).name, + ) + + +class FollowsAndUnitLand(ls._TempLaunch): + """R2 / plan (q). follows is the dispatch-graph edge, not + context.prior. unit defaults to the lineage branch.""" + + def test_omitted_follows_is_an_empty_list_and_unit_defaults_to_the_branch( + self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + self.assertEqual(rec["follows"], []) + self.assertIsInstance(rec["follows"], list) + self.assertEqual(rec["unit"], rec["lineage"]["branch"]) + self.assertEqual(rec["unit"], "work") + + def test_given_follows_and_unit_land_verbatim(self): + prior = "20260826T000000Z-plan-grok-aaaaaa" + rec, *_ = self.launch_ok(self.argv_for( + job="plan", harness="grok", stage="plan", extra=[ + "--follows", prior, "--unit", "WO-7", + ], + )) + self.assertEqual(rec["follows"], [prior]) + self.assertEqual(rec["unit"], "WO-7") + self.assertNotEqual(rec["unit"], rec["lineage"]["branch"]) + # context.prior is not this field. + self.assertNotIn("prior", rec) + + +class ModelRanComesFromTheStream(ls._TempLaunch): + """R3 / plan (g) — requested is the alias; ran is the stream. + + The ran-model is baked into the fake CLI; TASK_LAUNCH_RAN_MODEL is + not in the parent environment, so copying that env cannot satisfy + these tests. + """ + + def _assert_stream_digest(self, rec): + stream = rec["session"]["stream"] + self.assertTrue(stream, "a stream path is recorded") + path = Path(stream) + self.assertTrue(path.is_file(), f"stream missing: {stream}") + digest = ls.sha256_file(path) + self.assertEqual( + rec["session"]["stream_sha256_at_close"], digest, + ) + self.assertNotIn("TASK_LAUNCH_RAN_MODEL", os.environ) + + def test_grok_ran_equals_the_fixture_stream_value_not_the_alias(self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + self.assertEqual(rec["model"]["requested"], ls.REQUESTED_MODEL) + self.assertEqual(rec["model"]["ran"], ls.RAN_MODEL) + self.assertNotEqual(rec["model"]["ran"], rec["model"]["requested"]) + self.assertTrue(rec["model"]["read_from"]) + self.assertIn("summary.json", rec["model"]["read_from"]) + self._assert_stream_digest(rec) + + def test_claude_ran_equals_the_fixture_stream_value(self): + rec, *_ = self.launch_ok(job="plan", harness="claude", stage="plan") + self.assertEqual(rec["model"]["requested"], ls.REQUESTED_MODEL) + self.assertEqual(rec["model"]["ran"], ls.RAN_MODEL) + self.assertNotEqual(rec["model"]["ran"], rec["model"]["requested"]) + self.assertTrue(rec["model"]["read_from"]) + self._assert_stream_digest(rec) + + def test_codex_ran_equals_the_fixture_stream_value(self): + rec, *_ = self.launch_ok(job="plan", harness="codex", stage="plan") + self.assertEqual(rec["model"]["requested"], ls.REQUESTED_MODEL) + self.assertEqual(rec["model"]["ran"], ls.RAN_MODEL) + self.assertNotEqual(rec["model"]["ran"], rec["model"]["requested"]) + self.assertTrue(rec["model"]["read_from"]) + named = (rec["session"]["stream"] or "") + (rec["model"]["read_from"] or "") + self.assertIn("rollout", named) + self._assert_stream_digest(rec) + + def test_no_stream_leaves_ran_null_with_a_note_never_the_alias(self): + os.environ["TASK_LAUNCH_WRITE_STREAM"] = "0" + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + self.assertIsNone(rec["model"]["ran"]) + self.assertEqual(rec["model"]["requested"], ls.REQUESTED_MODEL) + self.assertNotEqual(rec["model"]["ran"], rec["model"]["requested"]) + note = rec.get("note") or rec["model"].get("note") or json.dumps(rec) + self.assertTrue( + any(w in str(note).lower() + for w in ("no stream", "null", "absent", "missing")), + f"null ran must carry a note, not silence: {note!r}", + ) + + def test_grok_head_commit_mismatch_is_recorded_not_silent(self): + bogus = "ab" * 20 + self.assertNotEqual(bogus, self.ref) + os.environ["TASK_LAUNCH_HEAD_COMMIT"] = bogus + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + blob = json.dumps(rec).lower() + self.assertIn("head_commit", blob) + self.assertTrue( + "mismatch" in blob or (rec.get("result") or {}).get("envelope", {}) + .get("status") == "invalid", + f"grok head_commit ≠ ref_sha must be named, not accepted: {rec!r}", + ) + self.assertNotEqual(rec["snapshot"]["ref_sha"], bogus) + + +class BriefCheckRederives(ls._TempLaunch): + """R4 / plan (m) — brief --check re-renders from jobs.json@ref_sha + + scope + input digests.""" + + def test_brief_check_accepts_an_untouched_record(self): + rec, *_ = self.launch_ok( + job="plan", harness="grok", stage="plan", + scope="check-the-brief", + ) + code, out, err = self.run_main(["brief", "--check", rec["id"]]) + self.assertEqual( + code, 0, + f"brief --check must pass on the record just written: " + f"{out}{err}", + ) + + def test_a_planted_wrong_digest_makes_brief_check_fail(self): + rec, *_ = self.launch_ok( + job="plan", harness="grok", stage="plan", + scope="check-the-brief", + ) + path = self.the_record_path() + before = path.read_bytes() + bogus = "0" * 64 + self.assertNotEqual(rec["brief"]["sha256"], bogus) + ls.plant_bytes( + path, + lambda raw: raw.replace( + rec["brief"]["sha256"].encode("utf-8"), + bogus.encode("utf-8"), + ), + expect="edit", + recognisable=lambda after: b'"brief"' in after, + ) + after = path.read_bytes() + self.assertNotEqual(after, before) + self.assertIn(bogus.encode("utf-8"), after) + self.assertNotIn(rec["brief"]["sha256"].encode("utf-8"), after) + code, out, err = self.run_main(["brief", "--check", rec["id"]]) + self.assertNotEqual(code, 0, f"planted digest must fail: {out}{err}") + + def test_input_digest_enters_the_record(self): + report = self.home / "in" / "given.md" + body = b"# given input\n" + self.plant_new_file(report, body, must_contain="given input") + digest = hashlib.sha256(body).hexdigest() + rec, *_ = self.launch_ok(self.argv_for( + job="check-tests", harness="grok", stage="check-tests", + extra=["--input", str(report)], + )) + inputs = rec["brief"]["inputs"] + self.assertEqual(len(inputs), 1, "exactly one --input landed") + self.assertEqual(inputs[0]["sha256"], digest) + copied = self.the_job_dir() / "in" / "given.md" + self.assertTrue(copied.is_file()) + self.assertEqual(copied.read_bytes(), body) + + +class CapsEqualWires(ls._TempLaunch): + """R5 / plan (o).""" + + def test_record_caps_equal_wires_budget_and_name_their_source(self): + wires = ls.load_path(self, ls.WIRES_PATH, "task_launch_wires") + expected = wires.budget() + self.assertIsInstance(expected, int) + self.assertGreater(expected, 0) + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + caps = rec["caps"] + self.assertIsInstance(caps, dict) + self.assertTrue(caps, "caps must not be an empty dict") + source = str(caps.get("source", "")) + self.assertIn("wires", source) + values = [] + + def walk(obj): + if isinstance(obj, dict): + for v in obj.values(): + walk(v) + elif isinstance(obj, bool): + return + elif isinstance(obj, (int, float)) or ( + isinstance(obj, str) and obj.isdigit()): + values.append(int(obj)) + + walk(caps) + self.assertIn( + expected, values, + f"caps must carry wires.budget()={expected}, got {caps!r}", + ) + + +class BehindTipIsACount(ls._TempLaunch): + """R6 — behind_tip is rev-list --count ref_sha..lineage.""" + + def test_behind_tip_matches_rev_list_count(self): + rec, *_ = self.launch_ok( + job="plan", harness="grok", stage="plan", ref=self.mid_sha, + ) + counted = int(self._git( + "rev-list", "--count", f"{self.mid_sha}..work", + ).stdout.strip()) + self.assertGreater(counted, 0) + self.assertEqual(rec["snapshot"]["behind_tip"], counted) + self.assertIsInstance(rec["snapshot"]["behind_tip"], int) diff --git a/ops/devlane/dispatch/tests/test_launch_record_identity.py b/ops/devlane/dispatch/tests/test_launch_record_identity.py new file mode 100644 index 0000000..b463eb6 --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_record_identity.py @@ -0,0 +1,138 @@ +"""A record commit is stamped with the owner's identity, not the machine's. + +Written from AGENTS.md §Attribution — author and committer stay the +owner's identity, ``xormania <127287135+xormania@users.noreply.github.com>`` +— and CONTRACT.md §Dispatch The record (lines 239-247): one record +commit on the lineage branch, ``git commit --only``, message +``dispatch: record ``. + +Observed before this test existed: record commits carried the +machine's git identity ('machine project ') +as author and committer instead of the owner's. +""" + +from __future__ import annotations + +import os +import subprocess + +import launch_support as ls + +OWNER_IDENT = "xormania <127287135+xormania@users.noreply.github.com>" +OWNER_NAME = "xormania" +OWNER_EMAIL = "127287135+xormania@users.noreply.github.com" + +DECOY_AUTHOR_NAME = "decoy-author" +DECOY_AUTHOR_EMAIL = "decoy-author@example.invalid" +DECOY_COMMITTER_NAME = "decoy-committer" +DECOY_COMMITTER_EMAIL = "decoy-committer@example.invalid" +DECOY_CONFIG_NAME = "decoy-config" +DECOY_CONFIG_EMAIL = "decoy-config@example.invalid" + + +class RecordCommitCarriesTheOwnersIdentity(ls._TempLaunch): + """Author and committer of the record commit are the owner's identity. + + A launcher that inherits the process git identity, the repo + ``user.name``, or ``WF_AGENT`` still fails: those are planted as + decoys, and the owner's identity is none of them. + """ + + def _git_as_process(self, *args): + """git with the process environment, the way a child commit sees it.""" + return subprocess.run( + ["git", *args], + cwd=self.repo, + capture_output=True, + text=True, + check=False, + ) + + def _plant_decoy_identities(self): + """Every identity source git would consult is a non-owner decoy.""" + plants = { + "GIT_AUTHOR_NAME": DECOY_AUTHOR_NAME, + "GIT_AUTHOR_EMAIL": DECOY_AUTHOR_EMAIL, + "GIT_COMMITTER_NAME": DECOY_COMMITTER_NAME, + "GIT_COMMITTER_EMAIL": DECOY_COMMITTER_EMAIL, + } + for key, value in plants.items(): + os.environ[key] = value + self.env[key] = value + self.assertEqual(os.environ[key], value) + self.assertEqual(self.env[key], value) + self.assertNotEqual(value, OWNER_NAME) + self.assertNotEqual(value, OWNER_EMAIL) + + self._git("config", "user.name", DECOY_CONFIG_NAME) + self._git("config", "user.email", DECOY_CONFIG_EMAIL) + self.assertEqual( + self._git("config", "user.name").stdout.strip(), + DECOY_CONFIG_NAME, + ) + self.assertEqual( + self._git("config", "user.email").stdout.strip(), + DECOY_CONFIG_EMAIL, + ) + + author = self._git_as_process("var", "GIT_AUTHOR_IDENT") + committer = self._git_as_process("var", "GIT_COMMITTER_IDENT") + self.assertEqual(author.returncode, 0, author.stderr) + self.assertEqual(committer.returncode, 0, committer.stderr) + self.assertIn(DECOY_AUTHOR_NAME, author.stdout) + self.assertIn(DECOY_AUTHOR_EMAIL, author.stdout) + self.assertIn(DECOY_COMMITTER_NAME, committer.stdout) + self.assertIn(DECOY_COMMITTER_EMAIL, committer.stdout) + self.assertNotIn(OWNER_NAME, author.stdout) + self.assertNotIn(OWNER_EMAIL, author.stdout) + self.assertNotIn(OWNER_NAME, committer.stdout) + self.assertNotIn(OWNER_EMAIL, committer.stdout) + self.assertNotEqual(ls.AGENT, OWNER_IDENT) + self.assertEqual(os.environ.get("WF_AGENT"), ls.AGENT) + + def _record_commit_sha(self, rec): + rel = f".dev/records/dispatches/{rec['id']}.json" + path = self.repo / rel + self.assertTrue(path.is_file(), f"record file missing: {path}") + sha = self._git( + "log", "-1", "--format=%H", "--", rel, + ).stdout.strip() + self.assertRegex( + sha, r"^[0-9a-f]{40}$", + f"record file is not in any commit: {rel}", + ) + self.assertNotEqual(sha, self.ref) + return sha + + def _ident_of(self, sha, fmt): + ident = self._git( + "log", "-1", f"--format={fmt}", sha, + ).stdout.strip() + self.assertTrue(ident, f"empty identity from {fmt} on {sha}") + self.assertIn("<", ident) + self.assertIn(">", ident) + return ident + + def test_the_record_commit_author_is_the_owners_identity(self): + self._plant_decoy_identities() + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + sha = self._record_commit_sha(rec) + author = self._ident_of(sha, "%an <%ae>") + self.assertEqual(author, OWNER_IDENT) + self.assertNotEqual( + author, + f"{DECOY_AUTHOR_NAME} <{DECOY_AUTHOR_EMAIL}>", + ) + self.assertNotEqual(author, ls.AGENT) + + def test_the_record_commit_committer_is_the_owners_identity(self): + self._plant_decoy_identities() + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + sha = self._record_commit_sha(rec) + committer = self._ident_of(sha, "%cn <%ce>") + self.assertEqual(committer, OWNER_IDENT) + self.assertNotEqual( + committer, + f"{DECOY_COMMITTER_NAME} <{DECOY_COMMITTER_EMAIL}>", + ) + self.assertNotEqual(committer, ls.AGENT) diff --git a/ops/devlane/dispatch/tests/test_launch_refusals.py b/ops/devlane/dispatch/tests/test_launch_refusals.py new file mode 100644 index 0000000..6e9a419 --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_refusals.py @@ -0,0 +1,561 @@ +"""Refusals: planted and fired, each with its silent neighbour. + +Written from CONTRACT.md §Dispatch Refusals. Exit 3. The text of each +names expected / found / what would satisfy. ``stale-base`` is the only +overridable one. + + D1 identity — before any directory exists + D2 live-target + D3 ref + D4 record-target (detached, dev, main, other branch, close-time) + D5 stale-base + neighbours + the single override + D6 model (missing; effort on codex) + D7 write-role-unadmitted + D8 scope-cap (1025 / 1024 / adjudicate) + D9 history-vs-withheld, mode-unavailable + D10 isolation + D11 invalid overrides +""" + +from __future__ import annotations + +import json +import os +import stat + +import launch_support as ls + + +class IdentityRefusesBeforeAnyDirectory(ls._TempLaunch): + """D1 / contract identity / plan (f).""" + + def test_unset_wf_agent_refuses_and_creates_no_job_dir(self): + os.environ.pop("WF_AGENT", None) + before = list(self.jobs_root.iterdir()) + self.assertEqual(before, [], "jobs root starts empty") + # Unwritable jobs root: mkdir-then-rmdir cannot hide a mint. + writable = self.jobs_root.stat().st_mode + os.chmod( + self.jobs_root, + writable & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH), + ) + try: + code, out, err = self.dispatch() + finally: + os.chmod(self.jobs_root, writable) + self.assert_refusal( + code, out, err, ident="identity", + phrases=["identity", "Name
", "WF_AGENT"], + ) + self.assertEqual( + list(self.jobs_root.iterdir()), before, + "identity refuses before any directory is made", + ) + self.assert_not_started() + self.assertEqual(self.record_files(), []) + + def test_a_bare_name_is_refused_the_same_way(self): + os.environ["WF_AGENT"] = "Grok" + before = list(self.jobs_root.iterdir()) + code, out, err = self.dispatch() + text = self.assert_refusal( + code, out, err, ident="identity", + phrases=["identity", "Name
", "Grok"], + ) + self.assertIn("found", text.lower()) + self.assertEqual(list(self.jobs_root.iterdir()), before) + self.assert_not_started() + + def test_the_name_addr_form_is_the_silent_neighbour(self): + os.environ["WF_AGENT"] = ls.AGENT + rec, witness, _out, _err = self.launch_ok( + job="plan", harness="grok", stage="plan", + ) + self.assertEqual(rec["dispatched_by"], ls.AGENT) + self.assertTrue(witness["argv"]) + + +class LiveTargetRefusesAJobsRootInsideTheRepo(ls._TempLaunch): + """D2 / contract live-target.""" + + def test_a_jobs_root_inside_the_invoking_repo_is_refused(self): + inside = self.repo / "inside-jobs" + inside.mkdir() + os.environ["DISPATCH_JOBS"] = str(inside) + listed = self.worktree_paths() + self.assertTrue( + any(str(self.repo.resolve()) == str(p) or + str(self.repo.resolve()) in str(p) + for p in listed), + "plant: invoking repo is a worktree list entry", + ) + code, out, err = self.dispatch() + self.assert_refusal( + code, out, err, ident="live-target", + phrases=["live-target", "worktree", "DISPATCH_JOBS"], + ) + self.assert_not_started() + snaps = list(inside.rglob("snapshot")) + self.assertEqual(snaps, [], "no snapshot is minted inside the repo") + + def test_a_jobs_root_outside_every_worktree_is_the_silent_neighbour(self): + rec, witness, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + snap = self.snapshot_of(rec) + for wt in self.worktree_paths(): + resolved = os.path.realpath(wt) + self.assertFalse( + os.path.commonpath([os.path.realpath(snap), resolved]) + == resolved, + f"snapshot {snap} is inside worktree {wt}", + ) + self.assertTrue(witness["argv"]) + + +class RefMustNameACommit(ls._TempLaunch): + """D3 / contract ref.""" + + def test_an_unresolvable_ref_is_refused_and_never_launches(self): + missing = "no-such-ref-7e1c9a3d" + code, out, err = self.dispatch( + self.argv_for(ref=missing, stage="plan"), + ) + self.assert_refusal( + code, out, err, ident="ref", + phrases=["ref", missing, "commit"], + ) + self.assert_not_started() + self.assertEqual(self.job_dirs(), []) + + def test_a_resolvable_ref_is_the_silent_neighbour(self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan", + ref=self.ref) + self.assertEqual(rec["snapshot"]["ref_sha"], self.ref) + + +class RecordTargetMustBeTheLineageBranch(ls._TempLaunch): + """D4 / contract record-target / plan (f).""" + + def test_detached_head_is_refused(self): + self._git("checkout", "--detach", self.ref) + code, out, err = self.dispatch(self.argv_for(stage="plan")) + self.assert_refusal( + code, out, err, ident="record-target", + phrases=["record-target", "detached"], + ) + self.assertEqual(self.job_dirs(), []) + self.assert_not_started() + + def test_a_checkout_on_dev_is_refused(self): + self._git("checkout", "-b", "dev") + code, out, err = self.dispatch( + self.argv_for(lineage="dev", stage="plan"), + ) + self.assert_refusal( + code, out, err, ident="record-target", + phrases=["record-target", "dev"], + ) + self.assertEqual(self.job_dirs(), []) + self.assert_not_started() + + def test_a_checkout_on_main_is_refused(self): + self._git("checkout", "-b", "main") + code, out, err = self.dispatch( + self.argv_for(lineage="main", stage="plan"), + ) + self.assert_refusal( + code, out, err, ident="record-target", + phrases=["record-target", "main"], + ) + self.assertEqual(self.job_dirs(), []) + self.assert_not_started() + + def test_a_branch_other_than_lineage_is_refused_naming_both(self): + self._git("checkout", "-b", "other") + code, out, err = self.dispatch( + self.argv_for(lineage="work", stage="plan"), + ) + text = self.assert_refusal( + code, out, err, ident="record-target", + phrases=["record-target", "work", "other"], + ) + self.assertIn("work", text) + self.assertIn("other", text) + self.assertEqual(self.job_dirs(), []) + self.assert_not_started() + + def test_close_after_the_checkout_moves_branch_refuses_and_leaves_the_file( + self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + job_id = rec["id"] + # Re-open the close path: plant a launched record for a new id + # by copying the closed one, then switch branch and close. + launched_id = "20260826T000000Z-plan-grok-d1ed01" + src = self.the_record_path() + dest = src.parent / f"{launched_id}.json" + body = src.read_text(encoding="utf-8") + self.assertIn(job_id, body) + data = json.loads(src.read_text(encoding="utf-8")) + data["id"] = launched_id + data["status"] = "launched" + data["result"] = None + data["at"]["closed"] = None + dest.write_text(json.dumps(data) + "\n", encoding="utf-8") + landed = dest.read_text(encoding="utf-8") + self.assertIn(launched_id, landed) + self.assertIn('"launched"', landed) + self.assertNotEqual(landed, src.read_text(encoding="utf-8")) + + job_dir = self.jobs_root / launched_id + job_dir.mkdir() + (job_dir / "state.json").write_text( + json.dumps({"pid": self.dead_pid(), "pgid": 1, + "session": {"id": "x"}, "attempt": 1}) + "\n", + encoding="utf-8", + ) + self.assertTrue((job_dir / "state.json").is_file()) + + before_head = self._git("rev-parse", "HEAD").stdout.strip() + self._git("checkout", "-b", "moved") + code, out, err = self.run_main(["close", launched_id]) + text = self.assert_refusal( + code, out, err, ident="record-target", + phrases=["record-target", "work", "moved"], + ) + self.assertIn("work", text) + self.assertIn("moved", text) + self.assertTrue(dest.is_file(), "the record file is left in place") + self.assertEqual( + self._git("rev-parse", "HEAD").stdout.strip(), + self._git("rev-parse", "moved").stdout.strip(), + ) + # Nothing committed on `moved`. + log = self._git("log", "-1", "--format=%s").stdout + self.assertNotIn(launched_id, log) + self.assertEqual( + self._git("rev-parse", "work").stdout.strip(), before_head, + ) + + +class StaleBaseIsTheOneOverridableRefusal(ls._TempLaunch): + """D5 / contract stale-base / plan (d)(e).""" + + def test_a_ref_that_is_not_an_ancestor_of_lineage_is_refused(self): + code, out, err = self.dispatch( + self.argv_for(ref=self.side_sha, lineage="work", stage="plan"), + ) + self.assert_refusal( + code, out, err, ident="stale-base", + phrases=["stale-base", self.side_sha[:8], "work"], + ) + self.assert_not_started() + + def test_a_ref_far_behind_the_tip_but_on_the_branch_is_not_refused(self): + rec, *_ = self.launch_ok( + job="plan", harness="grok", stage="plan", ref=self.root_sha, + ) + self.assertEqual(rec["snapshot"]["ref_sha"], self.root_sha) + behind = rec["snapshot"]["behind_tip"] + self.assertIsInstance(behind, int) + self.assertGreater(behind, 0, "root is behind the tip; this is a fact") + self.assertEqual(rec.get("overrides") or [], []) + + def test_a_ref_at_the_tip_is_not_stale_and_behind_tip_is_zero(self): + rec, *_ = self.launch_ok( + job="plan", harness="grok", stage="plan", ref=self.ref, + ) + self.assertEqual(rec["snapshot"]["behind_tip"], 0) + self.assertEqual(rec.get("overrides") or [], []) + + def test_override_stale_base_with_a_reason_launches_and_is_recorded(self): + rec, *_ = self.launch_ok(self.argv_for( + ref=self.side_sha, lineage="work", stage="plan", extra=[ + "--override", "stale-base:owner said continue", + ], + )) + overrides = rec.get("overrides") + self.assertIsInstance(overrides, list) + self.assertEqual(len(overrides), 1, "exactly one override landed") + item = overrides[0] + self.assertEqual(item.get("refusal"), "stale-base") + self.assertEqual(item.get("reason"), "owner said continue") + self.assertEqual(item.get("by"), ls.AGENT) + msg = self._git("log", "-1", "--format=%B").stdout + self.assertIn("stale-base", msg) + self.assertIn("owner said continue", msg) + + def test_an_override_on_identity_is_itself_refused(self): + os.environ.pop("WF_AGENT", None) + before = list(self.jobs_root.iterdir()) + code, out, err = self.dispatch(self.argv_for( + stage="plan", extra=["--override", "identity:please"], + )) + text = self.combined(out, err).lower() + self.assertEqual(code, ls.REFUSAL_EXIT) + self.assertTrue( + "override" in text or "identity" in text, + f"non-overridable override must be refused: {text!r}", + ) + self.assertEqual(list(self.jobs_root.iterdir()), before) + self.assert_not_started() + + def test_an_empty_reason_is_refused(self): + code, out, err = self.dispatch(self.argv_for( + ref=self.side_sha, stage="plan", extra=["--override", "stale-base:"], + )) + self.assertEqual(code, ls.REFUSAL_EXIT) + text = self.combined(out, err).lower() + self.assertTrue( + "reason" in text or "override" in text, + f"empty reason must be refused: {text!r}", + ) + self.assert_not_started() + + +class ModelIsRequiredAndCodexEffortIsRefused(ls._TempLaunch): + """D6 / contract model / plan (l).""" + + def test_no_model_is_refused_and_nothing_launches(self): + argv = self.argv_for(stage="plan", model=None) + self.assertNotIn("--model", argv) + code, out, err = self.dispatch(argv) + self.assert_refusal( + code, out, err, ident="model", + phrases=["model", "--model"], + ) + self.assert_not_started() + self.assertEqual(self.job_dirs(), []) + + def test_effort_on_codex_is_refused(self): + code, out, err = self.dispatch(self.argv_for( + job="plan", harness="codex", stage="plan", extra=["--effort", "high"], + )) + self.assert_refusal( + code, out, err, ident="model", + phrases=["effort", "codex"], + ) + self.assert_not_started() + + def test_effort_on_grok_is_the_silent_neighbour(self): + rec, witness, *_ = self.launch_ok(self.argv_for( + job="plan", harness="grok", stage="plan", extra=["--effort", "high"], + )) + argv = [str(p) for p in witness["argv"]] + self.assertIn("--reasoning-effort", argv) + self.assertEqual(argv[argv.index("--reasoning-effort") + 1], "high") + self.assertEqual(rec["model"]["effort_requested"], "high") + + +class ClaudeWriteIsUnadmitted(ls._TempLaunch): + """D7 / contract write-role-unadmitted / plan (l).""" + + def test_a_claude_write_role_is_refused(self): + code, out, err = self.dispatch(self.argv_for( + job="implement", harness="claude", stage="code", + scope="python3 -m unittest", + )) + self.assert_refusal( + code, out, err, ident="write-role-unadmitted", + phrases=["write-role-unadmitted", "claude"], + ) + self.assert_not_started() + self.assertEqual(self.job_dirs(), []) + + def test_claude_author_tests_is_refused_the_same_way(self): + # The unadmitted rule is the write role, not the implement job + # name. author-tests is the other write job in the registry. + code, out, err = self.dispatch(self.argv_for( + job="author-tests", harness="claude", stage="tests", + scope="pin the contract", + )) + self.assert_refusal( + code, out, err, ident="write-role-unadmitted", + phrases=["write-role-unadmitted", "claude"], + ) + self.assert_not_started() + self.assertEqual(self.job_dirs(), []) + + def test_a_claude_read_role_is_the_silent_neighbour(self): + rec, witness, *_ = self.launch_ok(self.argv_for( + job="plan", harness="claude", stage="plan", + )) + self.assertEqual(rec["role"], "read") + self.assertEqual(rec["harness"]["name"], "claude") + self.assertTrue(witness["argv"]) + + def test_a_grok_write_role_is_admitted(self): + os.environ["TASK_LAUNCH_COMMIT"] = "worker.py" + os.environ["TASK_LAUNCH_VERDICT"] = "null" + rec, *_ = self.launch_ok(self.argv_for( + job="implement", harness="grok", stage="code", + scope="python3 -m unittest", + )) + self.assertEqual(rec["role"], "write") + + +class ScopeCapIs1024AndAdjudicateTakesNone(ls._TempLaunch): + """D8 / contract scope-cap / plan (m)(n).""" + + def test_a_scope_of_1025_bytes_is_refused(self): + scope = "x" * (ls.SCOPE_CAP + 1) + self.assertEqual(len(scope.encode("utf-8")), 1025) + code, out, err = self.dispatch(self.argv_for( + stage="plan", scope=scope, + )) + self.assert_refusal( + code, out, err, ident="scope-cap", + phrases=["scope-cap", "1024"], + ) + self.assert_not_started() + + def test_a_scope_of_1024_bytes_is_the_silent_neighbour(self): + scope = "y" * ls.SCOPE_CAP + self.assertEqual(len(scope.encode("utf-8")), 1024) + rec, *_ = self.launch_ok(self.argv_for(stage="plan", scope=scope)) + self.assertEqual(rec["brief"]["scope"], scope) + self.assertEqual(len(rec["brief"]["scope"].encode("utf-8")), 1024) + + def test_adjudicate_refuses_a_scope(self): + report = self.home / "in" / "report.md" + self.plant_new_file(report, "# report\n", must_contain="report") + code, out, err = self.dispatch(self.argv_for( + job="adjudicate", harness="grok", stage="adjudicate", + scope="this job takes none", extra=["--input", str(report)], + )) + self.assert_refusal( + code, out, err, ident="scope-cap", + phrases=["scope-cap", "adjudicate"], + ) + self.assert_not_started() + + def test_adjudicate_without_scope_is_the_silent_neighbour(self): + report = self.home / "in" / "report.md" + self.plant_new_file(report, "# report\n", must_contain="report") + rec, *_ = self.launch_ok(self.argv_for( + job="adjudicate", harness="grok", stage="adjudicate", + scope=None, extra=["--input", str(report)], + )) + self.assertIn(rec["brief"].get("scope"), (None, "", [])) + self.assertEqual(rec["job"], "adjudicate") + + +class HistoryVersusWithheldAndModeUnavailable(ls._TempLaunch): + """D9 / contract history-vs-withheld, mode-unavailable.""" + + def test_withheld_and_whole_together_are_refused(self): + code, out, err = self.dispatch(self.argv_for( + job="withheld-whole", harness="grok", stage="plan", + )) + self.assert_refusal( + code, out, err, ident="history-vs-withheld", + phrases=["history-vs-withheld", "withheld-whole"], + ) + self.assert_not_started() + + def test_a_fileset_job_is_refused_by_name_until_that_mode_lands(self): + code, out, err = self.dispatch(self.argv_for( + job="fileset-job", harness="grok", stage="plan", + )) + self.assert_refusal( + code, out, err, ident="mode-unavailable", + phrases=["mode-unavailable", "fileset"], + ) + self.assert_not_started() + + +class IsolationRefusesUnknownAndDirtyHomes(ls._TempLaunch): + """D10 / contract isolation.""" + + def test_an_unknown_harness_is_refused(self): + code, out, err = self.dispatch(self.argv_for( + harness="not-a-harness", stage="plan", + )) + self.assertEqual(code, ls.REFUSAL_EXIT) + text = self.combined(out, err) + self.assertTrue( + "isolation" in text.lower() or "not-a-harness" in text, + f"unknown harness must be refused by isolation: {text!r}", + ) + self.assert_not_started() + + def test_a_missing_codex_credential_is_refused(self): + auth = self.home / ".codex" / "auth.json" + self.assertTrue(auth.is_file(), "plant: credential present before unlink") + auth.unlink() + self.assertFalse(auth.exists(), "plant: credential is gone") + code, out, err = self.dispatch(self.argv_for( + job="plan", harness="codex", stage="plan", + )) + self.assertEqual(code, ls.REFUSAL_EXIT) + text = self.combined(out, err).lower() + self.assertTrue( + "isolation" in text or "auth.json" in text or "credential" in text, + f"missing credential must surface isolation.NotIsolated: {text!r}", + ) + self.assert_not_started() + self.assertEqual(self.job_dirs(), []) + + def test_a_missing_grok_credential_is_refused(self): + auth = self.home / ".grok" / "auth.json" + self.assertTrue(auth.is_file(), "plant: credential present before unlink") + auth.unlink() + self.assertFalse(auth.exists(), "plant: credential is gone") + code, out, err = self.dispatch(self.argv_for( + job="plan", harness="grok", stage="plan", + )) + self.assertEqual(code, ls.REFUSAL_EXIT) + text = self.combined(out, err).lower() + self.assertTrue( + "isolation" in text or "auth.json" in text or "credential" in text, + f"missing credential must surface isolation.NotIsolated: {text!r}", + ) + self.assert_not_started() + + def test_a_dirty_minimal_home_is_refused(self): + job_id = "20260826T000000Z-plan-codex-d1r7ee" + self.force_id(job_id) + dirty = self.jobs_root / job_id / "home" / "codex" + dirty.mkdir(parents=True) + leftover = dirty / "hooks.json" + leftover.write_text("{}\n", encoding="utf-8") + self.assertEqual( + leftover.read_text(encoding="utf-8"), "{}\n", + "plant: leftover landed in the would-be minimal home", + ) + code, out, err = self.dispatch(self.argv_for( + job="plan", harness="codex", stage="plan", + )) + self.assertEqual(code, ls.REFUSAL_EXIT) + text = self.combined(out, err).lower() + self.assertTrue( + "isolation" in text or "not empty" in text or "hooks.json" in text, + f"dirty minimal home must be NotIsolated: {text!r}", + ) + self.assert_not_started() + + def test_an_isolated_grok_home_holds_exactly_auth_json(self): + rec, witness, *_ = self.launch_ok( + job="plan", harness="grok", stage="plan", + ) + home = witness["env"].get("GROK_HOME") + self.assertTrue(home, "isolated grok launch sets GROK_HOME") + from pathlib import Path + names = sorted(p.name for p in Path(home).iterdir()) + self.assertEqual(names, ["auth.json"]) + self.assertEqual(rec["harness"]["name"], "grok") + + +class MissingLineageBranchIsRefused(ls._TempLaunch): + """No dedicated refusal id: contract record-target already covers + current branch ≠ lineage. The text must name record-target and + expected/found/satisfy, not merely exit 3 and the branch string.""" + + def test_a_lineage_branch_that_does_not_exist_locally_is_refused(self): + code, out, err = self.dispatch(self.argv_for( + lineage="no-such-lineage", stage="plan", + )) + self.assert_refusal( + code, out, err, ident="record-target", + phrases=["record-target", "no-such-lineage", "work"], + ) + self.assert_not_started() + self.assertEqual(self.job_dirs(), []) diff --git a/ops/devlane/dispatch/tests/test_launch_review_p1_findings.py b/ops/devlane/dispatch/tests/test_launch_review_p1_findings.py new file mode 100644 index 0000000..7d6fd5e --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_review_p1_findings.py @@ -0,0 +1,270 @@ +"""Red pins for dispatch/review-fixes p1 findings G1 and G2. + +Authored from CONTRACT.md §Dispatch (Verbs, Template values, Isolation +per harness, The record permitted delta) plus the two p1 findings at +``.dev/records/dispatches/20260828T224426Z-review-claude-043571.json``. + + G1 claude --add-dir names directories, so the child can open {diff} + G2 a later launch does not move a closed, resumed job's record or + job directory off the canonical paths +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import launch_support as ls + + +def _add_dir_values(argv, cwd): + """Every --add-dir value, resolved the way the child would see it.""" + found = [] + argv = [str(a) for a in argv] + i = 0 + while i < len(argv): + a = argv[i] + raw = None + if a == "--add-dir" and i + 1 < len(argv): + raw = argv[i + 1] + i += 2 + elif a.startswith("--add-dir="): + raw = a.split("=", 1)[1] + i += 1 + else: + i += 1 + continue + if not raw: + continue + p = Path(raw) + if not p.is_absolute(): + p = Path(cwd) / p + found.append(p.resolve()) + return found + + +def _inside_directory(path, root): + """True when ``path`` lives inside directory ``root``. + + ``--add-dir`` names a directory. Granting the file itself + (``path == root`` when ``root`` is not a directory) is not a grant. + """ + if not root.is_dir(): + return False + try: + path.relative_to(root) + except ValueError: + return False + return True + + +class ClaudeAddDirValuesAreDirectories(ls._TempLaunch): + """G1 / CONTRACT.md §Template values {diff} and Isolation. + + --add-dir names an additional working DIRECTORY. Passing the + {diff} FILE as an --add-dir root does not let a claude child open + it; observed live on 20260828T224426Z-review-claude-043571. + """ + + def setUp(self): + super().setUp() + jobs = json.loads(self.jobs_file.read_text(encoding="utf-8")) + jobs["names-diff"] = { + "adapter": "harness", + "deliverable": "fixture: {diff} named on its own line", + "role": "read", + "snapshot": "whole", + "prompt": "diff={diff}\nAim at: {scope}", + "constraints": ["read only"], + } + self.jobs_file.write_text( + json.dumps(jobs, indent=2) + "\n", encoding="utf-8", + ) + self.ref = self._commit("fixture job that names diff") + + def _prompt_field(self, prompt, key): + prefix = key + "=" + for line in prompt.splitlines(): + if line.startswith(prefix): + return line[len(prefix):] + self.fail(f"rendered brief has no {key}= line:\n{prompt}") + + def test_claude_add_dir_values_are_directories_so_diff_is_openable(self): + rec, witness, *_ = self.launch_ok(self.argv_for( + job="names-diff", harness="claude", stage="review", + scope="pin --add-dir as directories for {diff}", + )) + argv = [str(a) for a in witness["argv"]] + self.assertTrue(argv, "child argv is empty") + self.assertEqual( + Path(argv[0]).name, "claude", + f"this pin is for a claude child, argv[0]={argv[0]!r}", + ) + cwd = witness.get("cwd") + self.assertTrue(cwd, "child cwd is recorded") + prompt = witness.get("stdin") or "" + self.assertTrue(prompt, "claude receives the rendered brief on stdin") + named = self._prompt_field(prompt, "diff").strip() + self.assertTrue( + named, + "{diff} must name the diff the launcher writes, not a hole " + f"in the brief; got {named!r} in:\n{prompt}", + ) + self.assertNotIn(" ", named, f"{{diff}} is one path, got {named!r}") + diff_path = Path(named) + if not diff_path.is_absolute(): + diff_path = Path(cwd) / diff_path + diff_path = diff_path.resolve() + self.assertTrue( + diff_path.is_file(), + f"{{diff}} must name a file the child can open, got {named!r}", + ) + self.assertEqual(rec["job"], "names-diff") + + add_dirs = _add_dir_values(argv, cwd) + self.assertTrue( + add_dirs, + "claude argv must include --add-dir so {diff} outside " + f"snapshot/ is granted; argv={argv!r}", + ) + not_dirs = [str(p) for p in add_dirs if not p.is_dir()] + self.assertEqual( + not_dirs, [], + "--add-dir names a directory, not a file; the claude child " + "cannot open a path that was itself passed as --add-dir. " + f"non-directories={not_dirs!r} argv={argv!r} {{diff}}={named!r}", + ) + self.assertTrue( + any(_inside_directory(diff_path, root) for root in add_dirs), + f"claude child cannot read {{diff}} path {diff_path}: " + "outside this session's allowed working directories " + f"(cwd={cwd!r}, add-dir={[str(r) for r in add_dirs]})", + ) + + +class ClosedResumedJobStaysAtCanonicalPaths(ls._TempLaunch): + """G2 / CONTRACT.md §The record permitted delta. + + One file per dispatch at .dev/records/dispatches/.json. The + permitted delta is that file plus its commit; the index and + worktree are otherwise untouched. A later launch must not + shutil.move a closed, resumed job's record or job directory. + """ + + def test_a_later_launch_does_not_archive_a_closed_resumed_job(self): + rec_a, *_ = self.launch_ok( + job="plan", harness="claude", stage="plan", + ) + id_a = rec_a["id"] + record_rel = ".dev/records/dispatches/" + id_a + ".json" + path_a = self.repo / record_rel + job_a = self.jobs_root / id_a + self.assertTrue( + path_a.is_file(), + f"plant: first record is at the canonical path {path_a}", + ) + self.assertTrue( + job_a.is_dir(), + f"plant: first job directory is under jobs-root {job_a}", + ) + tracked = self._git("ls-files", "--", record_rel).stdout.strip() + self.assertEqual( + tracked, record_rel, + "plant: first record is a tracked file on the lineage branch", + ) + self.assertEqual(rec_a["status"], "closed") + + self.witness.unlink() + code, out, err = self.run_main(["resume", id_a]) + self.assertEqual( + code, 0, + f"resume of the closed job must succeed: {self.combined(out, err)}", + ) + self.assertTrue( + path_a.is_file(), + "resume itself must not move the record off the canonical path", + ) + self.assertTrue(job_a.is_dir(), "resume keeps the same job directory") + + code, out, err = self.dispatch(self.argv_for( + job="plan", harness="codex", stage="plan", + )) + self.assertEqual( + code, 0, + f"second launch must succeed: {self.combined(out, err)}", + ) + + self.assertTrue( + path_a.is_file(), + "CONTRACT.md §The record: one file per dispatch at " + f"{record_rel}; a later launch must not move a closed, " + "resumed job's tracked record to " + ".dev/records/dispatches/archive/", + ) + still_tracked = self._git("ls-files", "--", record_rel).stdout.strip() + self.assertEqual( + still_tracked, record_rel, + "the first record stays tracked; git commit --only of the " + "new record must not be covering a working-tree deletion", + ) + porcelain = self.porcelain() + deleted = [ + line for line in porcelain.splitlines() + if re.search(r"^D\s+", line) + and record_rel in line + ] + self.assertEqual( + deleted, [], + "CONTRACT.md §The record permitted delta: the index and " + "worktree are otherwise untouched; a later launch must not " + f"delete the tracked record {record_rel}: {porcelain!r}", + ) + archive_record = ( + self.repo / ".dev" / "records" / "dispatches" / "archive" + / f"{id_a}.json" + ) + self.assertFalse( + archive_record.exists(), + f"record must not be moved to {archive_record}", + ) + self.assertTrue( + job_a.is_dir(), + f"job directory must stay at {job_a}, not be moved to " + f"{self.jobs_root.name}-archive/", + ) + jobs_archive = self.jobs_root.parent / (self.jobs_root.name + "-archive") + self.assertFalse( + (jobs_archive / id_a).exists(), + f"job directory must not be moved to {jobs_archive / id_a}", + ) + + try: + code, out, err = self.run_main(["brief", "--check", id_a]) + except Exception as exc: + self.fail( + f"brief --check {id_a} must find the record at " + f"{record_rel}; raised {type(exc).__name__}: {exc}" + ) + self.assertEqual( + code, 0, + f"brief --check {id_a} must find the record at {record_rel}: " + f"{self.combined(out, err)}", + ) + try: + code, out, err = self.run_main(["status", id_a]) + except Exception as exc: + self.fail( + f"status {id_a} must find the job; raised " + f"{type(exc).__name__}: {exc}" + ) + text = self.combined(out, err) + self.assertNotIn( + "unlaunched", text.lower(), + f"status {id_a} must not report unlaunched after a later " + f"launch: {text!r}", + ) + self.assertEqual( + code, 0, + f"status {id_a} exits 0: {text!r}", + ) diff --git a/ops/devlane/dispatch/tests/test_launch_snapshot.py b/ops/devlane/dispatch/tests/test_launch_snapshot.py new file mode 100644 index 0000000..1d4142c --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_snapshot.py @@ -0,0 +1,356 @@ +"""Whole-mode snapshot, permitted delta, FILESET.md wall. + +Written from CONTRACT.md §Dispatch Snapshot modes, Job directory, +Collect, The record (permitted delta). Plan items (a)(b)(c)(m). + + S1 invoking repo delta is exactly the permitted set + S2 --only: an unrelated staged file stays staged + S3 FETCH_HEAD unchanged + S4 whole mint guarantees (HEAD, parents, remote, alternates, logs, + source path, clean status, not a worktree) + S5 FILESET.md plant fires the wall; the launcher snapshot stays clean + S6 prompt and bookkeeping live outside snapshot/ +""" + +from __future__ import annotations + +import hashlib +import os +import subprocess + +import launch_support as ls + + +class PermittedDeltaToTheInvokingRepository(ls._TempLaunch): + """S1 / contract permitted delta / plan (a).""" + + def test_a_read_dispatch_adds_one_record_commit_and_no_dispatch_ref(self): + before_head = self._git("rev-parse", "HEAD").stdout.strip() + before_tree = self._git("rev-parse", "HEAD^{tree}").stdout.strip() + before_refs = self.refs_map() + before_fetch = self.fetch_head_bytes() + before_status = self.porcelain() + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + after_head = self._git("rev-parse", "HEAD").stdout.strip() + after_refs = self.refs_map() + self.assertEqual( + self._git("rev-parse", "--abbrev-ref", "HEAD").stdout.strip(), + "work", + ) + self.assertNotEqual(after_head, before_head, "one new commit") + self.assertEqual( + self._git("rev-parse", "HEAD^").stdout.strip(), before_head, + ) + names = self._git( + "diff-tree", "--no-commit-id", "--name-only", "-r", + before_head, after_head, + ).stdout.splitlines() + record_rel = ( + ".dev/records/dispatches/" + rec["id"] + ".json" + ) + self.assertEqual(names, [record_rel]) + self.assertNotIn( + f"refs/dispatch/{rec['id']}", after_refs, + "read roles do not fetch refs/dispatch", + ) + extra = set(after_refs) - set(before_refs) + self.assertEqual(extra, set()) + for name in before_refs: + if name == "refs/heads/work": + continue + self.assertEqual(after_refs[name], before_refs[name], name) + self.assertEqual(self.fetch_head_bytes(), before_fetch) + # The record is committed, so it leaves the index; nothing else + # about the index or worktree moves. + self.assertEqual(self.porcelain(), before_status) + self.assertNotEqual( + self._git("rev-parse", "HEAD^{tree}").stdout.strip(), + before_tree, + ) + + def test_a_write_dispatch_adds_the_record_commit_and_one_dispatch_ref(self): + os.environ["TASK_LAUNCH_COMMIT"] = "worker.py" + os.environ["TASK_LAUNCH_VERDICT"] = "null" + before_head = self._git("rev-parse", "HEAD").stdout.strip() + before_refs = self.refs_map() + before_fetch = self.fetch_head_bytes() + before_status = self.porcelain() + before_index = self.index_blob() + rec, *_ = self.launch_ok(self.argv_for( + job="implement", harness="grok", stage="code", + scope="python3 -m unittest", + )) + after_head = self._git("rev-parse", "HEAD").stdout.strip() + after_refs = self.refs_map() + self.assertEqual( + self._git("rev-parse", "--abbrev-ref", "HEAD").stdout.strip(), + rec["lineage"]["branch"], + ) + names = self._git( + "diff-tree", "--no-commit-id", "--name-only", "-r", + before_head, after_head, + ).stdout.splitlines() + record_rel = ".dev/records/dispatches/" + rec["id"] + ".json" + self.assertEqual(names, [record_rel]) + dispatch_ref = f"refs/dispatch/{rec['id']}" + self.assertIn(dispatch_ref, after_refs) + extra = set(after_refs) - set(before_refs) + self.assertEqual(extra, {dispatch_ref}) + for name in before_refs: + if name == "refs/heads/work": + continue + self.assertEqual(after_refs[name], before_refs[name], name) + self.assertEqual(self.fetch_head_bytes(), before_fetch) + snap_head = rec["result"]["head"] + self.assertEqual(after_refs[dispatch_ref], snap_head) + self.assertNotEqual(snap_head, rec["snapshot"]["ref_sha"]) + # Index and worktree otherwise untouched — not just the commit tree. + self.assertEqual(self.porcelain(), before_status) + after_index = self.index_blob() + record_entry = self._git("ls-files", "-s", "--", record_rel).stdout + self.assertTrue(record_entry.strip()) + self.assertEqual(after_index.replace(record_entry, ""), before_index) + reflog = self.dispatch_reflog(rec["id"]) + self.assertTrue( + reflog.strip(), + "write collect writes a reflog line for refs/dispatch/", + ) + self.assertIn(snap_head[:8], reflog) + + +class CommitOnlyLeavesUnrelatedIndexEntries(ls._TempLaunch): + """S2 / contract git commit --only.""" + + def test_a_planted_unrelated_staged_file_stays_staged_and_uncommitted(self): + planted = self._write("unrelated.txt", "do not sweep me in\n") + self._git("add", "--", "unrelated.txt") + staged_before = self._git("diff", "--cached", "--name-only").stdout + self.assertIn("unrelated.txt", staged_before, "plant: file is staged") + self.assertTrue(planted.is_file()) + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + staged_after = self._git("diff", "--cached", "--name-only").stdout + self.assertIn("unrelated.txt", staged_after) + committed = self._git( + "diff-tree", "--no-commit-id", "--name-only", "-r", "HEAD", + ).stdout.splitlines() + self.assertNotIn("unrelated.txt", committed) + self.assertEqual( + committed, + [".dev/records/dispatches/" + rec["id"] + ".json"], + ) + msg = self._git("log", "-1", "--format=%B").stdout + self.assertIn(f"dispatch: record {rec['id']}", msg) + self.assertIn("Source: generated: ops/devlane/dispatch/launch.py", msg) + self.assertIn(f"Co-Authored-By: {ls.AGENT}", msg) + + +class FetchHeadIsNotWritten(ls._TempLaunch): + """S3 — collect uses --no-write-fetch-head; mint removes FETCH_HEAD.""" + + def test_an_existing_fetch_head_is_byte_identical_after_a_write_dispatch(self): + fetch = self.repo / ".git" / "FETCH_HEAD" + marker = b"planted-fetch-head-marker\n" + fetch.write_bytes(marker) + self.assertEqual(fetch.read_bytes(), marker, "plant landed") + os.environ["TASK_LAUNCH_COMMIT"] = "worker.py" + os.environ["TASK_LAUNCH_VERDICT"] = "null" + rec, *_ = self.launch_ok(self.argv_for( + job="implement", harness="grok", stage="code", + scope="python3 -m unittest", + )) + self.assertEqual(fetch.read_bytes(), marker) + self.assertIn(f"refs/dispatch/{rec['id']}", self.refs_map()) + + def test_absent_fetch_head_stays_absent_after_a_read_dispatch(self): + fetch = self.repo / ".git" / "FETCH_HEAD" + self.assertFalse(fetch.exists()) + self.launch_ok(job="plan", harness="grok", stage="plan") + self.assertFalse(fetch.exists()) + + +class WholeMintGuarantees(ls._TempLaunch): + """S4 / contract whole / plan (b).""" + + def test_head_equals_ref_sha_and_parents_are_present(self): + rec, *_ = self.launch_ok( + job="plan", harness="grok", stage="plan", ref=self.mid_sha, + ) + snap = self.snapshot_of(rec) + head = self._git("rev-parse", "HEAD", repo=snap).stdout.strip() + self.assertEqual(head, self.mid_sha) + self.assertEqual(head, rec["snapshot"]["ref_sha"]) + parent = self._git("rev-parse", "HEAD^", repo=snap).stdout.strip() + self.assertEqual(parent, self.root_sha) + kind = self._git("cat-file", "-t", self.root_sha, repo=snap) + self.assertEqual(kind.stdout.strip(), "commit") + + def test_remote_is_empty_and_alternates_and_logs_do_not_exist(self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + snap = self.snapshot_of(rec) + remote = self._git("remote", repo=snap).stdout.strip() + self.assertEqual(remote, "") + git = snap / ".git" + self.assertTrue(git.exists()) + self.assertFalse( + (git / "objects" / "info" / "alternates").exists(), + "fetch-minted snapshots do not share objects via alternates", + ) + self.assertFalse( + (git / "logs").exists(), + "core.logAllRefUpdates=false and no logs/ directory", + ) + + def test_object_store_is_not_shared_by_symlink_or_hardlink(self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + self.assert_objects_not_shared(self.snapshot_of(rec)) + + def test_the_invoking_repo_path_occurs_in_no_byte_under_dot_git(self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + snap = self.snapshot_of(rec) + needle = str(self.repo.resolve()).encode("utf-8") + self.assertTrue(needle, "source path must be a real needle") + hits = self.git_dir_mentions(snap, needle) + self.assertEqual( + hits, [], + f"source path leaked into snapshot .git: {hits}", + ) + + def test_snapshot_status_is_empty_before_the_harness_runs(self): + rec, witness, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + snap = self.snapshot_of(rec) + # After a read role the tree should still be clean: the worker + # is contracted not to edit, and the fake does not. + self.assertEqual(self.porcelain(repo=snap), "") + self.assertEqual( + os.path.realpath(witness["cwd"]), + os.path.realpath(snap), + ) + + def test_uncommitted_invoking_bytes_are_not_in_the_snapshot(self): + dirty = self._write("alpha.py", "DIRTY WORKTREE\n") + self.assertIn("DIRTY WORKTREE", dirty.read_text(encoding="utf-8")) + rec, *_ = self.launch_ok( + job="plan", harness="grok", stage="plan", ref=self.ref, + ) + snap = self.snapshot_of(rec) + snap_alpha = (snap / "alpha.py").read_text(encoding="utf-8") + self.assertEqual(snap_alpha, "alpha v3\n") + self.assertNotIn("DIRTY WORKTREE", snap_alpha) + self.assertEqual( + self._git("rev-parse", "HEAD", repo=snap).stdout.strip(), + self.ref, + ) + + def test_the_snapshot_is_not_in_the_invoking_repo_worktree_list(self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + snap = os.path.realpath(self.snapshot_of(rec)) + listed = [os.path.realpath(p) for p in self.worktree_paths()] + self.assertNotIn(snap, listed) + self.assertTrue( + (self.snapshot_of(rec) / ".git").exists(), + "whole mode has its own .git — this is not fileset", + ) + + +class FilesetManifestMustNotLandInTheSnapshot(ls._TempLaunch): + """S5 / plan (c) — a planted FILESET.md listing receipts.py fires + the wall; the launcher's snapshot stays clean of that file.""" + + def test_a_planted_fileset_md_fires_the_wall_and_the_snapshot_does_not(self): + self.assertTrue(ls.WALL_PY.is_file(), "vocabulary_wall.py must exist") + plant_root = self.home / "wall-plant" + plant_root.mkdir() + fileset = plant_root / "FILESET.md" + self.plant_new_file( + fileset, + "included:\n ops/devlane/workflow/receipts.py\n", + must_contain="receipts.py", + ) + receipts = plant_root / ".dev" / "app" / "workflow" / "receipts.py" + # The file is under .dev/ so the wall's default walk would skip + # it; FILESET.md at the root is the design-surface hit. + self.plant_new_file(receipts, "# receipts fixture\n", + must_contain="receipts") + wall = subprocess.run( + ["python3", str(ls.WALL_PY), "--root", str(plant_root)], + capture_output=True, text=True, + ) + self.assertEqual( + wall.returncode, 1, + "plant must make the wall fire, got " + f"{wall.returncode}: {wall.stdout}{wall.stderr}", + ) + self.assertIn("receipts", wall.stdout.lower()) + + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + snap = self.snapshot_of(rec) + self.assertFalse( + (snap / "FILESET.md").exists(), + "launcher snapshot must not carry FILESET.md at the root", + ) + self.assertFalse((snap / "FILESET.diff").exists()) + clean = subprocess.run( + ["python3", str(ls.WALL_PY), "--root", str(snap)], + capture_output=True, text=True, + ) + self.assertNotIn("FILESET.md", clean.stdout) + self.assertNotEqual( + clean.returncode, 1, + f"launcher snapshot must not trip the wall: {clean.stdout}", + ) + + +class PromptAndBookkeepingLiveInTheJobDirectory(ls._TempLaunch): + """S6 / contract job directory / plan (m).""" + + def test_prompt_path_is_outside_the_snapshot_and_matches_the_cli_bytes(self): + rec, witness, *_ = self.launch_ok( + job="plan", harness="grok", stage="plan", + scope="digest-me-exactly", + ) + snap = self.snapshot_of(rec).resolve() + job_dir = self.the_job_dir().resolve() + prompt = job_dir / "prompt.txt" + self.assertTrue(prompt.is_file(), "prompt.txt lives in the job dir") + self.assertFalse( + (snap / "prompt.txt").exists(), + "prompt must not be written inside the snapshot", + ) + self.assertFalse(snap in prompt.parents or prompt.parent == snap) + body = prompt.read_bytes() + digest = hashlib.sha256(body).hexdigest() + self.assertEqual(rec["brief"]["sha256"], digest) + self.assertEqual(rec["brief"]["bytes"], len(body)) + received = witness.get("prompt_text", "") + self.assertTrue(received, "the fake CLI must have read the prompt file") + self.assertEqual( + hashlib.sha256(received.encode("utf-8")).hexdigest(), digest, + ) + self.assertIn("digest-me-exactly", received) + self.assertEqual( + os.path.realpath(witness.get("prompt_file") or ""), + os.path.realpath(prompt), + ) + + def test_launcher_bookkeeping_is_not_under_snapshot(self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + snap = self.snapshot_of(rec) + job_dir = self.the_job_dir() + for name in ( + "prompt.txt", "raw.out", "stderr", "breaker.log", + "TRIPPED.md", "state.json", "exit", + ): + self.assertFalse( + (snap / name).exists(), + f"{name} must not land under snapshot/", + ) + self.assertTrue( + (job_dir / "state.json").is_file() + or (job_dir / "exit").is_file(), + "job dir carries state.json and/or exit", + ) + for required in ("snapshot", "prompt.txt", "in", "out"): + self.assertTrue( + (job_dir / required).exists(), + f"job dir layout includes {required}", + ) diff --git a/ops/devlane/dispatch/tests/test_launch_u10b.py b/ops/devlane/dispatch/tests/test_launch_u10b.py new file mode 100644 index 0000000..a968e9d --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_u10b.py @@ -0,0 +1,517 @@ +"""U10b red pins: D-ENV-4 and D-DIFF-1. + +Authored from `.dev/docs/scratch/harness-fixes-ratification.md` +Amendment 2026-08-30 — U10b (ratified at 5f205a8), as amended +2026-08-30 (2) at b4746e6. launch.py was not read. Existing +collect/F1 cases that only assert `status == "invalid"` already +pass at HEAD; these tests pin the discriminating evidence. + + D-ENV-4 a post-run refusal amends the parsed envelope; findings, + counts, artifacts and spend survive verbatim; stamp + survives verbatim except stamp.ref, which the launcher + normalises to the snapshot ref_sha + D-DIFF-1 a review-shaped job with base_sha == ref_sha is refused + before launch: no job directory, snapshot, record, or spend +""" + +from __future__ import annotations + +import json +import os + +import launch_support as ls + +NINE = ( + "job", "status", "verdict", "counts", "findings", + "artifacts", "spend", "stamp", "note", +) + +# Distinctive values that `_invalid(...)` / `_envelope(b"")` cannot +# coincidentally reproduce. Counts are deliberately not the length of +# the findings list so a recomputed tally is not "verbatim". +PLANTED_COUNTS = {"p1": 5, "p2": 3, "p3": 3, "opinions": 1} +PLANTED_FINDINGS = [ + { + "severity": "p1", + "where": "dispatch collect read-role-head", + "claim": "D-ENV-4-survives-P1-a", + "reproduce": "python3 -m unittest " + "dev.app.dispatch.tests.test_launch_u10b", + }, + { + "severity": "p2", + "where": "dispatch collect off-lineage-head", + "claim": "D-ENV-4-survives-P2-b", + "reproduce": None, + }, +] +PLANTED_ARTIFACTS = {"review": "out/review.json"} +PLANTED_SPEND = {"harness": "grok", "total": 7777, "out": 333, "runs": 4} +PLANTED_STAMP = { + "ref": "u10b-stamp-ref", + "started": "2026-08-30T05:04:26Z", + "ended": "2026-08-30T05:19:00Z", +} +PLANTED_NOTE = "harness-produced-note-u10b" + + +def _planted(job): + return { + "job": job, + "status": "ok", + "verdict": "changes", + "counts": dict(PLANTED_COUNTS), + "findings": [dict(item) for item in PLANTED_FINDINGS], + "artifacts": dict(PLANTED_ARTIFACTS), + "spend": dict(PLANTED_SPEND), + "stamp": dict(PLANTED_STAMP), + "note": PLANTED_NOTE, + } + + +class _U10bLaunch(ls._TempLaunch): + def _emit(self, planted): + body = json.dumps(planted) + os.environ["TASK_LAUNCH_STDOUT"] = "token" + os.environ["TASK_LAUNCH_TOKEN"] = body + return planted + + def _raw_object(self): + raw = self.the_job_dir() / "raw.out" + self.assertTrue(raw.is_file(), "raw.out is missing") + data = raw.read_bytes() + self.assertTrue(data, "raw.out is empty") + text = data.decode("utf-8").strip() + self.assertTrue(text, "raw.out is whitespace") + emitted = json.loads(text) + self.assertIsInstance(emitted, dict, f"raw.out is not an object: {text!r}") + return emitted + + def _envelope_of(self, rec): + result = rec.get("result") or {} + env = result.get("envelope") if isinstance(result, dict) else None + self.assertIsInstance( + env, dict, + f"record has no result.envelope: {rec!r}", + ) + return env + + def _assert_plant_landed(self, emitted, planted): + """The harness actually emitted the distinctive evidence.""" + self.assertGreater( + len(planted["findings"]), 0, + "plant must carry findings; empty evidence cannot discriminate", + ) + self.assertGreater( + planted["counts"]["p1"], 0, + "plant must carry a non-zero p1 tally", + ) + self.assertNotEqual( + planted["spend"].get("total"), 0, + "plant spend.total must not be the _invalid default of 0", + ) + self.assertEqual( + emitted.get("findings"), planted["findings"], + "plant: raw.out findings did not land", + ) + self.assertEqual( + emitted.get("counts"), planted["counts"], + "plant: raw.out counts did not land", + ) + self.assertEqual( + emitted.get("artifacts"), planted["artifacts"], + "plant: raw.out artifacts did not land", + ) + self.assertEqual( + emitted.get("spend"), planted["spend"], + "plant: raw.out spend did not land", + ) + self.assertEqual( + emitted.get("stamp"), planted["stamp"], + "plant: raw.out stamp did not land", + ) + self.assertEqual(emitted.get("status"), "ok", emitted) + self.assertEqual(emitted.get("verdict"), "changes", emitted) + self.assertEqual(emitted.get("note"), PLANTED_NOTE, emitted) + + def _assert_evidence_survived(self, env, planted, *, refusal, ref_sha): + """D-ENV-4: only status, verdict and the note are amended. + + stamp.ref is the snapshot ref_sha, not the planted value — + the launcher normalises it on every parsed envelope, refused + or not (amendment 2026-08-30 (2) at b4746e6). + """ + for key in NINE: + self.assertIn(key, env, f"envelope missing {key}: {env!r}") + self.assertEqual( + env.get("status"), "invalid", + f"post-run refusal amends status to invalid: {env!r}", + ) + self.assertIsNone( + env.get("verdict"), + f"post-run refusal amends verdict to null: {env!r}", + ) + findings = env.get("findings") + self.assertIsInstance(findings, list, f"findings={findings!r}") + self.assertEqual( + len(findings), len(planted["findings"]), + "findings must survive at the planted count, not be replaced " + f"with an empty _invalid list: envelope={env!r}", + ) + self.assertEqual( + findings, planted["findings"], + "findings must survive verbatim: " + f"got {findings!r} planted {planted['findings']!r}", + ) + self.assertEqual( + env.get("counts"), planted["counts"], + "counts must survive verbatim (not recomputed from findings): " + f"got {env.get('counts')!r}", + ) + self.assertEqual( + env.get("artifacts"), planted["artifacts"], + f"artifacts must survive verbatim: {env.get('artifacts')!r}", + ) + self.assertEqual( + env.get("spend"), planted["spend"], + f"spend must survive verbatim: {env.get('spend')!r}", + ) + planted_stamp = planted["stamp"] + self.assertIsInstance(planted_stamp, dict, planted_stamp) + self.assertTrue(ref_sha, "snapshot.ref_sha is required") + self.assertNotEqual( + planted_stamp.get("ref"), ref_sha, + "plant: planted stamp.ref must differ from snapshot.ref_sha " + "so a skipped normalisation is visible", + ) + stamp = env.get("stamp") + self.assertIsInstance(stamp, dict, f"stamp={stamp!r}") + self.assertEqual( + stamp.get("ref"), ref_sha, + "stamp.ref is normalised to snapshot.ref_sha on the refusal " + "path, same as the non-refused path; skipping that " + "normalisation leaves the planted ref: " + f"got {stamp.get('ref')!r} planted {planted_stamp.get('ref')!r} " + f"ref_sha={ref_sha!r}", + ) + self.assertNotEqual( + stamp.get("ref"), planted_stamp.get("ref"), + "normalisation must actually replace the planted stamp.ref", + ) + self.assertEqual( + {k: v for k, v in stamp.items() if k != "ref"}, + {k: v for k, v in planted_stamp.items() if k != "ref"}, + "stamp survives verbatim except stamp.ref: " + f"got {stamp!r} planted {planted_stamp!r}", + ) + self.assertEqual( + env.get("job"), planted["job"], + f"job is not in the amended set: {env.get('job')!r}", + ) + note = str(env.get("note") or "") + self.assertTrue(note, f"amended note must name the refusal: {env!r}") + self.assertIn( + refusal.lower(), note.lower(), + f"note must name {refusal!r}: {note!r}", + ) + self.assertNotEqual( + note, planted["note"], + "the note is amended to name the refusal; the harness note " + f"alone is not that amendment: {note!r}", + ) + + +class ReadRoleHeadRefusalKeepsTheParsedEnvelope(_U10bLaunch): + """D-ENV-4 / read-role-head: a read role that commits is refused + after the harness has run, and the parsed envelope is retained.""" + + def test_read_role_head_refusal_keeps_findings_counts_artifacts_spend_stamp( + self): + planted = self._emit(_planted("plan")) + os.environ["TASK_LAUNCH_COMMIT"] = "worker.py" + os.environ["TASK_LAUNCH_HEAD_COMMIT"] = self.ref + _code, _out, _err = self.dispatch(self.argv_for( + job="plan", harness="grok", stage="plan", + )) + rec = self.read_record() + self.assertEqual(rec["role"], "read") + result = rec["result"] + ref_sha = rec["snapshot"]["ref_sha"] + self.assertEqual(ref_sha, self.ref) + self.assertNotEqual( + result.get("head"), ref_sha, + "plant: the read-role worker committed on top of the ref", + ) + changed = ls.changed_entries(result.get("changed_paths")) + self.assertTrue( + any("worker.py" in str(item) for item in changed), + f"plant: worker.py is in changed_paths, got {changed!r}", + ) + emitted = self._raw_object() + self._assert_plant_landed(emitted, planted) + env = self._envelope_of(rec) + self._assert_evidence_survived( + env, planted, refusal="read-role-head", ref_sha=ref_sha, + ) + + +class OffLineageHeadRefusalKeepsTheParsedEnvelope(_U10bLaunch): + """D-ENV-4 / off-lineage-head: a write role whose HEAD does not + descend from ref_sha is refused after the harness has run, and the + parsed envelope is retained.""" + + def test_off_lineage_head_refusal_keeps_findings_counts_artifacts_spend_stamp( + self): + planted = self._emit(_planted("implement")) + os.environ["TASK_LAUNCH_ORPHAN"] = "1" + os.environ["TASK_LAUNCH_VERDICT"] = "null" + _code, _out, _err = self.dispatch(self.argv_for( + job="implement", harness="grok", stage="code", + scope="python3 -m unittest", + )) + rec = self.read_record() + self.assertEqual(rec["role"], "write") + ref_sha = rec["snapshot"]["ref_sha"] + self.assertEqual(ref_sha, self.ref) + envelope_blob = json.dumps((rec.get("result") or {}).get("envelope") or {}) + self.assertIn( + "off-lineage-head", + (self.combined(_out, _err) + json.dumps(rec) + envelope_blob).lower(), + "plant: off-lineage-head is the refusal that fired", + ) + emitted = self._raw_object() + self._assert_plant_landed(emitted, planted) + env = self._envelope_of(rec) + self._assert_evidence_survived( + env, planted, refusal="off-lineage-head", ref_sha=ref_sha, + ) + + +class UnparsedStdoutStillUsesInvalid(_U10bLaunch): + """D-ENV-4 neighbour: when no envelope could be parsed, the + existing `_invalid(...)` construction stands.""" + + def test_prose_stdout_plus_orphan_head_is_still_the_invalid_construction( + self): + os.environ["TASK_LAUNCH_STDOUT"] = "prose" + os.environ["TASK_LAUNCH_ORPHAN"] = "1" + os.environ["TASK_LAUNCH_VERDICT"] = "null" + _code, _out, _err = self.dispatch(self.argv_for( + job="implement", harness="grok", stage="code", + scope="python3 -m unittest", + )) + rec = self.read_record() + raw = self.the_job_dir() / "raw.out" + self.assertTrue(raw.is_file(), "harness ran; raw.out is missing") + data = raw.read_bytes() + self.assertTrue(data, "plant: prose stdout landed") + self.assertNotIn(b"{", data, "plant: prose has no JSON object") + env = self._envelope_of(rec) + self.assertEqual(env.get("status"), "invalid", env) + self.assertIsNone(env.get("verdict"), env) + findings = env.get("findings") + self.assertIsInstance(findings, list, f"findings={findings!r}") + self.assertEqual( + findings, [], + "unparsed stdout must not invent the planted findings: " + f"envelope={env!r}", + ) + claims = [item.get("claim") for item in findings if isinstance(item, dict)] + self.assertNotIn("D-ENV-4-survives-P1-a", claims) + note = str(env.get("note") or "") + self.assertTrue(note, f"invalid construction must name why: {env!r}") + self.assertIn( + "off-lineage-head", note.lower(), + "the envelope note itself must name the off-lineage-head " + "refusal; a parse-only note that drops the refusal is the " + f"loss this pins: note={note!r} envelope={env!r}", + ) + + +class ReviewWithNothingToCompareIsRefusedBeforeLaunch(_U10bLaunch): + """D-DIFF-1: base_sha == ref_sha is refused before the harness + starts. The rejected alternative — record a no-comparison marker + and run anyway — is pinned by the absence of a record.""" + + def _plant_equal_base_and_ref(self): + self._git("branch", "dev", self.ref) + base = self._git("merge-base", "work", "dev").stdout.strip() + self.assertEqual( + base, self.ref, + "plant: merge-base(work, dev) must equal --ref so the " + f"comparison is empty; base={base} ref={self.ref}", + ) + self.assertEqual( + self._git("rev-parse", "work").stdout.strip(), self.ref, + ) + self.assertEqual( + self._git("rev-parse", "dev").stdout.strip(), self.ref, + ) + return base + + def test_review_with_equal_base_and_ref_is_refused_before_launch(self): + base = self._plant_equal_base_and_ref() + self.assertEqual(self.job_dirs(), [], "jobs root starts empty") + self.assertEqual(self.record_files(), []) + before_head = self._git("rev-parse", "HEAD").stdout.strip() + code, out, err = self.dispatch(self.argv_for( + job="adversarial-review", harness="grok", stage="review", + ref=self.ref, + scope="pin empty comparison refusal", + )) + if code != ls.REFUSAL_EXIT: + dirs = self.job_dirs() + self.assertEqual( + len(dirs), 1, + "plant: HEAD still launches the empty-comparison review: " + f"code={code} dirs={dirs} out={self.combined(out, err)!r}", + ) + diff_path = dirs[0] / "diff.patch" + self.assertTrue( + diff_path.is_file(), + f"plant: {{diff}} is written: {list(dirs[0].iterdir())}", + ) + body = diff_path.read_text(encoding="utf-8") + self.assertFalse( + body.strip(), + "plant: base_sha == ref_sha produces an empty comparison, " + f"got {body!r}", + ) + rec = self.read_record() + lineage = rec.get("lineage") if isinstance(rec.get("lineage"), dict) else {} + snap = rec.get("snapshot") if isinstance(rec.get("snapshot"), dict) else {} + self.assertEqual( + lineage.get("base_sha"), snap.get("ref_sha"), + "plant: resolved base_sha equals ref_sha: " + f"lineage={lineage!r} snapshot={snap!r}", + ) + self.assertEqual(snap.get("ref_sha"), self.ref) + text = self.assert_refusal( + code, out, err, ident="empty-comparison", + phrases=["work", self.ref, "empty"], + ) + lower = text.lower() + self.assertIn(base.lower(), lower) + self.assert_not_started() + self.assertEqual( + self.job_dirs(), [], + "D-DIFF-1 refuses before a job directory is minted: " + f"{self.job_dirs()}", + ) + snaps = list(self.jobs_root.rglob("snapshot")) + self.assertEqual(snaps, [], f"no snapshot is minted: {snaps}") + self.assertEqual( + self.record_files(), [], + "rejected alternative: a no-comparison marker must not be " + f"recorded; records={self.record_files()}", + ) + self.assertEqual( + self._git("rev-parse", "HEAD").stdout.strip(), before_head, + "a pre-launch refusal must not commit a record", + ) + + +class ReviewWithARealComparisonStillLaunches(_U10bLaunch): + """D-DIFF-1 silent neighbour: a review whose base is not the ref + still launches.""" + + def test_review_with_a_non_empty_comparison_still_launches(self): + self._git("branch", "dev", self.mid_sha) + self.assertNotEqual(self.mid_sha, self.ref) + base = self._git("merge-base", "work", "dev").stdout.strip() + self.assertEqual(base, self.mid_sha) + rec, witness, *_ = self.launch_ok(self.argv_for( + job="adversarial-review", harness="grok", stage="review", + scope="pin non-empty comparison neighbour", + )) + self.assertEqual(rec.get("status"), "closed") + self.assertTrue(witness["argv"], "harness started") + self.assertEqual(len(self.job_dirs()), 1) + self.assertEqual(len(self.record_files()), 1) + + +class NonReviewJobWithEqualBaseAndRefIsNotThisRefusal(_U10bLaunch): + """D-DIFF-1 silent neighbour: a plan job is not review-shaped, even + when the lineage merge-base equals --ref.""" + + def test_a_plan_job_with_equal_base_and_ref_still_launches(self): + self._git("branch", "dev", self.ref) + base = self._git("merge-base", "work", "dev").stdout.strip() + self.assertEqual(base, self.ref, "plant: base_sha would equal ref_sha") + rec, witness, *_ = self.launch_ok(self.argv_for( + job="plan", harness="grok", stage="plan", + ref=self.ref, + scope="pin non-review neighbour of empty comparison", + )) + self.assertEqual(rec.get("job"), "plan") + self.assertEqual(rec.get("status"), "closed") + self.assertTrue(witness["argv"], "plan is not refused by D-DIFF-1") + self.assertEqual(len(self.record_files()), 1) + + +class ReviewWithEmptyDiffButUnequalShasStillLaunches(_U10bLaunch): + """D-DIFF-1 silent neighbour: the trigger is base_sha == ref_sha, + not empty comparison text. An --allow-empty commit on the tip + gives base != ref with a zero-byte diff and still launches.""" + + def test_review_with_empty_diff_but_unequal_base_and_ref_still_launches(self): + before = self.ref + self._git("commit", "--allow-empty", "-m", "u10b-empty-on-tip") + empty_sha = self._git("rev-parse", "HEAD").stdout.strip() + self.assertNotEqual( + empty_sha, before, + "plant: --allow-empty must move HEAD off the previous tip", + ) + self.assertEqual( + self._git("rev-parse", "work").stdout.strip(), empty_sha, + ) + self._git("branch", "dev", before) + base = self._git("merge-base", "work", "dev").stdout.strip() + self.assertEqual(base, before, "plant: merge-base is the previous tip") + self.assertNotEqual( + base, empty_sha, + "plant: base_sha != ref_sha; the equal-sha refusal must not fire", + ) + comparison = self._git("diff", base, empty_sha).stdout + self.assertFalse( + comparison.strip(), + "plant: the comparison is empty even though the SHAs differ: " + f"got {comparison!r}", + ) + rec, witness, *_ = self.launch_ok(self.argv_for( + job="adversarial-review", harness="grok", stage="review", + ref=empty_sha, + scope="pin empty-diff unequal-sha neighbour", + )) + self.assertEqual(rec.get("status"), "closed") + self.assertTrue(witness["argv"], "harness started") + self.assertEqual(len(self.job_dirs()), 1) + self.assertEqual(len(self.record_files()), 1) + snap = rec.get("snapshot") if isinstance(rec.get("snapshot"), dict) else {} + self.assertEqual( + snap.get("ref_sha"), empty_sha, + f"snapshot.ref_sha is the empty commit: snapshot={snap!r}", + ) + prompt = witness.get("prompt_text") or "" + if not prompt: + prompt = (self.the_job_dir() / "prompt.txt").read_text( + encoding="utf-8", + ) + self.assertIn(empty_sha, prompt, f"{{ref}} is the empty commit: {prompt!r}") + self.assertIn( + before, prompt, + "{base} is the previous tip, so the comparison SHAs differ: " + f"prompt={prompt!r}", + ) + self.assertNotEqual(empty_sha, before) + diff_path = self.the_job_dir() / "diff.patch" + self.assertTrue( + diff_path.is_file(), + f"{{diff}} is written: {list(self.the_job_dir().iterdir())}", + ) + body = diff_path.read_text(encoding="utf-8") + self.assertFalse( + body.strip(), + "an empty comparison with unequal SHAs still launches: " + f"got {body!r}", + ) diff --git a/ops/devlane/dispatch/tests/test_launch_verbs.py b/ops/devlane/dispatch/tests/test_launch_verbs.py new file mode 100644 index 0000000..cf72e11 --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_verbs.py @@ -0,0 +1,164 @@ +"""Verbs, jobs root default, template values, input order, id collision. + +Written from CONTRACT.md §Dispatch Verbs and Template values. + + V1 DISPATCH_JOBS unset uses XDG_STATE_HOME then HOME/.local/state + V2 scope cap is bytes, not characters (multibyte) + V3 template values into/base/diff land; a missing slot is a refusal + V4 multiple --input keep given order + V5 a colliding id does not overwrite the first dispatch +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import launch_support as ls + + +class JobsRootDefaultsToXdgThenHome(ls._TempLaunch): + """V1 — no flag; $DISPATCH_JOBS, else XDG, else ~/.local/state.""" + + def test_xdg_state_home_is_used_when_dispatch_jobs_is_unset(self): + xdg = self.home / "xdg-state" + os.environ.pop("DISPATCH_JOBS", None) + os.environ["XDG_STATE_HOME"] = str(xdg) + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + expected = (xdg / "minspec" / "dispatch").resolve() + root = Path(rec["snapshot"]["root"]).resolve() + self.assertTrue( + str(root).startswith(str(expected) + os.sep) + or root == expected, + f"jobs root should sit under {expected}, snapshot is {root}", + ) + prompts = list(expected.rglob("prompt.txt")) + self.assertTrue(prompts, f"job dir under XDG default is missing: {expected}") + + def test_home_local_state_is_used_when_xdg_is_also_unset(self): + os.environ.pop("DISPATCH_JOBS", None) + os.environ.pop("XDG_STATE_HOME", None) + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + expected = ( + self.home / ".local" / "state" / "minspec" / "dispatch" + ).resolve() + root = Path(rec["snapshot"]["root"]).resolve() + self.assertTrue( + str(root).startswith(str(expected) + os.sep) + or root == expected, + f"jobs root should sit under {expected}, snapshot is {root}", + ) + + +class ScopeCapIsBytesNotCharacters(ls._TempLaunch): + """V2 — 1024 bytes. A 2-byte character must not be counted as 1.""" + + def test_five_hundred_and_twelve_e_acute_is_exactly_1024_bytes(self): + scope = "é" * 512 + self.assertEqual(len(scope.encode("utf-8")), 1024) + self.assertEqual(len(scope), 512) + rec, *_ = self.launch_ok(self.argv_for(stage="plan", scope=scope)) + self.assertEqual(rec["brief"]["scope"], scope) + self.assertEqual(len(rec["brief"]["scope"].encode("utf-8")), 1024) + + def test_one_extra_byte_of_multibyte_scope_is_refused(self): + scope = "é" * 512 + "x" + self.assertEqual(len(scope.encode("utf-8")), 1025) + code, out, err = self.dispatch(self.argv_for( + stage="plan", scope=scope, + )) + self.assert_refusal( + code, out, err, ident="scope-cap", + phrases=["scope-cap", "1024"], + ) + self.assert_not_started() + + +class TemplateValuesAreSuppliedOrTheRenderIsRefused(ls._TempLaunch): + """V3 / contract Template values.""" + + def test_into_base_and_diff_land_in_the_prompt(self): + rec, witness, *_ = self.launch_ok(self.argv_for( + job="needs-into", harness="grok", stage="plan", + scope="template-slots", + )) + received = witness.get("prompt_text") or "" + snap = str(self.snapshot_of(rec).resolve()) + self.assertIn(snap, received, "{into} is the snapshot root") + self.assertIn(self.ref, received, "{ref} is the named sha") + self.assertTrue( + rec["lineage"]["base_sha"] in received + or rec["snapshot"].get("ref_sha") in received, + f"{{base}} must land as a sha in the prompt: {received!r}", + ) + self.assertIn("template-slots", received) + + def test_a_template_value_the_launcher_did_not_supply_is_a_refusal(self): + code, out, err = self.dispatch(self.argv_for( + job="needs-hole", harness="grok", stage="plan", + )) + text = self.combined(out, err).lower() + self.assertEqual(code, ls.REFUSAL_EXIT) + self.assertIn("not_a_slot", text) + self.assert_not_started() + + +class MultipleInputsKeepGivenOrder(ls._TempLaunch): + """V4 — {inputs} is space-separated, in the order given.""" + + def test_two_inputs_keep_cli_order_in_the_record_and_the_copies(self): + first = self.home / "in" / "a-second-alphabetically.md" + second = self.home / "in" / "b-first-given.md" + self.plant_new_file(first, "# A\n", must_contain="A") + self.plant_new_file(second, "# B\n", must_contain="B") + rec, witness, *_ = self.launch_ok(self.argv_for( + job="check-tests", harness="grok", stage="check-tests", + extra=["--input", str(second), "--input", str(first)], + )) + inputs = rec["brief"]["inputs"] + self.assertEqual(len(inputs), 2, "both --input flags landed") + names = [Path(item["path"]).name for item in inputs] + self.assertEqual( + names, + ["b-first-given.md", "a-second-alphabetically.md"], + "inputs keep given order, not sorted-by-name order", + ) + job_in = self.the_job_dir() / "in" + self.assertTrue((job_in / "b-first-given.md").is_file()) + self.assertTrue((job_in / "a-second-alphabetically.md").is_file()) + received = witness.get("prompt_text") or "" + b_at = received.find("b-first-given.md") + a_at = received.find("a-second-alphabetically.md") + self.assertGreaterEqual(b_at, 0) + self.assertGreaterEqual(a_at, 0) + self.assertLess(b_at, a_at, "{inputs} is in given order in the prompt") + + +class ConcurrentIdCollisionIsRefused(ls._TempLaunch): + """V5 — one directory / one record per id. A colliding mint must + not overwrite the first dispatch. Seam: launch.mint_id.""" + + def test_a_second_dispatch_with_the_same_id_does_not_overwrite(self): + job_id = "20260826T000000Z-plan-grok-aaaaaa" + self.force_id(job_id) + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + self.assertEqual(rec["id"], job_id) + first_bytes = ( + self.repo / ".dev" / "records" / "dispatches" / f"{job_id}.json" + ).read_bytes() + self.assertTrue(first_bytes) + self.start_witness.unlink(missing_ok=True) + self.witness.unlink(missing_ok=True) + os.environ["TASK_LAUNCH_TOKEN"] = "SECOND-SHOULD-NOT-RUN" + os.environ["TASK_LAUNCH_STDOUT"] = "token" + code, out, err = self.dispatch(self.argv_for( + job="plan", harness="grok", stage="plan", + scope="a colliding second dispatch", + )) + self.assertNotEqual(code, 0, self.combined(out, err)) + after = ( + self.repo / ".dev" / "records" / "dispatches" / f"{job_id}.json" + ).read_bytes() + self.assertEqual(after, first_bytes, "first record must be intact") + self.assert_not_started() + self.assertEqual(len(self.job_dirs()), 1) diff --git a/ops/devlane/dispatch/tests/test_launch_watch.py b/ops/devlane/dispatch/tests/test_launch_watch.py new file mode 100644 index 0000000..77c05a3 --- /dev/null +++ b/ops/devlane/dispatch/tests/test_launch_watch.py @@ -0,0 +1,453 @@ +"""status, DIED, resume. + +Written from CONTRACT.md §Dispatch Watching, and picking up after a +kill. Plan items (h)(i). + + W1 DIED is pid gone, no exit file — never finished + W2 running / finished / tripped / unlaunched + W3 resume verbs, same cwd, markers cleared, output appended, attempt appended + W4 a tripped job resumes only with a changed cap or a stated reason + W5 close of a DIED job commits the record as invalid +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import threading +import time + +import launch_support as ls + + +class StatusReportsDiedNeverFinished(ls._TempLaunch): + """W1 / plan (i). Planted job dirs — status does not need a live + harness, but it does need launch.main.""" + + def _plant_job(self, job_id, *, pid, exit_text=None, tripped=False): + d = self.jobs_root / job_id + d.mkdir() + state = { + "pid": pid, + "pgid": pid, + "session": {"id": "11111111-1111-4111-8111-111111111111"}, + "stream": str(d / "stream.jsonl"), + "attempt": 1, + } + (d / "state.json").write_text(json.dumps(state) + "\n", encoding="utf-8") + landed = json.loads((d / "state.json").read_text(encoding="utf-8")) + self.assertEqual(landed["pid"], pid, "plant: state.json pid") + if exit_text is not None: + (d / "exit").write_text(exit_text, encoding="utf-8") + self.assertEqual((d / "exit").read_text(encoding="utf-8"), exit_text) + if tripped: + (d / "TRIPPED.md").write_text("tripped\n", encoding="utf-8") + self.assertTrue((d / "TRIPPED.md").is_file()) + return d + + def _status(self, *args): + argv = ["status", *args] + return self.run_main(argv) + + def _status_blob(self, *args): + code, out, err = self._status(*args) + text = self.combined(out, err) + self.assertEqual(code, 0, f"status exits 0: {text!r}") + self.assertTrue(out.strip() or err.strip(), "status prints a report") + return text + + def test_a_dead_pid_with_no_exit_file_is_died_never_finished(self): + job_id = "20260826T000000Z-plan-grok-d1ed00" + pid = self.dead_pid() + self._plant_job(job_id, pid=pid) + self.assertFalse((self.jobs_root / job_id / "exit").exists()) + text = self._status_blob(job_id) + self.assertIn("DIED", text) + self.assertNotIn("finished", text.lower()) + code, out, _err = self._status(job_id, "--json") + self.assertEqual(code, 0) + payload = json.loads(out) + row = payload if isinstance(payload, dict) else payload[0] + if isinstance(payload, list): + self.assertEqual(len(payload), 1) + row = payload[0] + status = row.get("status") or row.get("state") + self.assertEqual(status, "DIED") + self.assertNotEqual(status, "finished") + + def test_an_alive_pid_is_running(self): + proc = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(60)"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + self.addCleanup(proc.kill) + job_id = "20260826T000000Z-plan-grok-run001" + self._plant_job(job_id, pid=proc.pid) + os.kill(proc.pid, 0) # plant: the pid is still alive + text = self._status_blob(job_id, "--json") + self.assertIn("running", text.lower()) + self.assertNotIn("DIED", text) + + def test_an_exit_file_is_finished(self): + job_id = "20260826T000000Z-plan-grok-fin001" + self._plant_job(job_id, pid=self.dead_pid(), exit_text="0\n") + text = self._status_blob(job_id, "--json") + self.assertIn("finished", text.lower()) + self.assertNotIn("DIED", text) + + def test_tripped_md_is_tripped(self): + job_id = "20260826T000000Z-plan-grok-trp001" + self._plant_job( + job_id, pid=self.dead_pid(), exit_text="137\n", tripped=True, + ) + text = self._status_blob(job_id, "--json") + self.assertIn("tripped", text.lower()) + self.assertNotIn("DIED", text) + + def test_an_unknown_id_is_unlaunched(self): + text = self._status_blob("20260826T000000Z-plan-grok-none00") + self.assertIn("unlaunched", text.lower()) + + +class ResumeRelaunchesInTheSameSnapshot(ls._TempLaunch): + """W3 / plan (h).""" + + def test_resume_uses_the_harness_resume_verb_in_the_same_cwd(self): + rec, first, *_ = self.launch_ok( + job="plan", harness="grok", stage="plan", + ) + job_id = rec["id"] + snap = os.path.realpath(self.snapshot_of(rec)) + first_cwd = os.path.realpath(first["cwd"]) + self.assertEqual(first_cwd, snap) + os.environ["TASK_LAUNCH_TOKEN"] = "SECOND-ATTEMPT" + os.environ["TASK_LAUNCH_STDOUT"] = "token" + self.witness.unlink() + code, out, err = self.run_main(["resume", job_id]) + self.assertEqual(code, 0, self.combined(out, err)) + second = self.read_witness() + argv = [str(p) for p in second["argv"]] + self.assertIn("-r", argv) + self.assertEqual(argv[argv.index("-r") + 1], rec["session"]["id"]) + self.assertEqual(os.path.realpath(second["cwd"]), snap) + + def test_resume_claude_uses_print_and_dash_r(self): + rec, *_ = self.launch_ok(job="plan", harness="claude", stage="plan") + self.witness.unlink() + code, out, err = self.run_main(["resume", rec["id"]]) + self.assertEqual(code, 0, self.combined(out, err)) + argv = [str(p) for p in self.read_witness()["argv"]] + self.assertTrue("-p" in argv or "--print" in argv) + self.assertIn("-r", argv) + self.assertEqual(argv[argv.index("-r") + 1], rec["session"]["id"]) + + def test_resume_codex_uses_exec_resume(self): + rec, *_ = self.launch_ok(job="plan", harness="codex", stage="plan") + self.witness.unlink() + code, out, err = self.run_main(["resume", rec["id"]]) + self.assertEqual(code, 0, self.combined(out, err)) + argv = [str(p) for p in self.read_witness()["argv"]] + self.assertIn("exec", argv) + self.assertIn("resume", argv) + self.assertIn(rec["session"]["id"], argv) + + def test_resume_appends_output_and_an_attempt_and_clears_exit(self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + job_dir = self.the_job_dir() + raw = job_dir / "raw.out" + self.assertTrue(raw.is_file(), "first attempt wrote raw.out") + first_body = raw.read_bytes() + self.assertTrue(first_body, "plant: raw.out from the first attempt") + exit_path = job_dir / "exit" + self.assertTrue(exit_path.is_file(), "exit is written last") + before_attempts = rec.get("attempts") or [] + self.assertGreaterEqual(len(before_attempts), 1) + os.environ["TASK_LAUNCH_TOKEN"] = "SECOND-ATTEMPT" + os.environ["TASK_LAUNCH_STDOUT"] = "token" + self.witness.unlink() + code, out, err = self.run_main(["resume", rec["id"]]) + self.assertEqual(code, 0, self.combined(out, err)) + after = raw.read_bytes() + self.assertTrue( + after.startswith(first_body) or first_body in after, + "resume must append, not truncate, raw.out", + ) + self.assertIn(b"SECOND-ATTEMPT", after) + rec2 = self.read_record() + attempts = rec2.get("attempts") or [] + self.assertGreater( + len(attempts), len(before_attempts), + "resume appends an attempts[] entry", + ) + started = self.read_start_witness() + self.assertFalse( + started.get("exit_present"), + "resume clears the exit marker before the child starts", + ) + self.assertFalse( + started.get("tripped_present"), + "resume clears TRIPPED.md before the child starts", + ) + + +class TrippedResumeNeedsAReason(ls._TempLaunch): + """W4.""" + + def test_a_tripped_job_refuses_resume_without_a_changed_cap_or_reason(self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + job_dir = self.the_job_dir() + tripped = job_dir / "TRIPPED.md" + self.plant_new_file(tripped, "battery tripped\n", + must_contain="tripped") + self.start_witness.unlink(missing_ok=True) + self.witness.unlink(missing_ok=True) + # Rewrite status so close-time resume sees a trip. + code, out, err = self.run_main(["resume", rec["id"]]) + self.assertNotEqual(code, 0, self.combined(out, err)) + text = self.combined(out, err).lower() + self.assertTrue( + "trip" in text or "cap" in text or "reason" in text, + f"tripped resume without reason must be refused: {text!r}", + ) + self.assertTrue(tripped.is_file(), "a refused resume leaves the marker") + self.assert_not_started() + + def test_a_tripped_job_resumes_with_a_stated_reason(self): + rec, *_ = self.launch_ok(job="plan", harness="grok", stage="plan") + job_dir = self.the_job_dir() + tripped = job_dir / "TRIPPED.md" + self.plant_new_file(tripped, "battery tripped\n", + must_contain="tripped") + self.start_witness.unlink(missing_ok=True) + self.witness.unlink(missing_ok=True) + # §Watching requires a changed cap or stated reason. §Verbs + # lists only --prompt-file on resume. --reason is the smallest + # argv that makes the positive path pin-able. + code, out, err = self.run_main([ + "resume", rec["id"], "--reason", "owner said continue", + ]) + self.assertEqual(code, 0, self.combined(out, err)) + started = self.read_start_witness() + self.assertFalse( + started.get("tripped_present"), + "a successful resume clears TRIPPED.md before the child starts", + ) + self.assertFalse(tripped.is_file(), "trip marker does not return") + rec2 = self.read_record() + blob = json.dumps(rec2) + self.assertIn("owner said continue", blob) + attempts = rec2.get("attempts") or [] + self.assertGreaterEqual(len(attempts), 2) + + def test_a_tripped_job_resumes_with_a_changed_cap_and_rearms(self): + rec, *_ = self.launch_ok(job="plan", harness="codex", stage="plan") + job_dir = self.the_job_dir() + tripped = job_dir / "TRIPPED.md" + self.plant_new_file(tripped, "battery tripped\n", + must_contain="tripped") + done = self.home / "resume-completed.marker" + os.environ["TASK_LAUNCH_DONE"] = str(done) + os.environ["TASK_LAUNCH_OVER_OUT"] = "5000000" + os.environ["TASK_LAUNCH_SLEEP"] = "4" + os.environ["TASK_LAUNCH_WRITE_STREAM"] = "1" + self.start_witness.unlink(missing_ok=True) + _code, out, err = self.run_main([ + "resume", rec["id"], "--cap-out", "100", + ]) + rec2 = self.read_record() + blob = json.dumps(rec2) + self.combined(out, err) + self.assertTrue( + "cap" in blob.lower() or rec2.get("caps"), + f"changed cap must be recorded: {blob!r}", + ) + # Re-arm: an over-budget sleeping child is killed, so DONE + # is never written (same shape as run.py R9). + self.assertFalse( + done.is_file(), + "re-armed battery must kill the over-budget resume child", + ) + envelope = (rec2.get("result") or {}).get("envelope") or {} + self.assertIn( + envelope.get("status"), ("tripped", "invalid"), + f"re-armed trip must surface: {rec2!r}", + ) + + +class CloseDiedCommitsInvalid(ls._TempLaunch): + """W5 — close ID finalizes and commits a DIED job's record as invalid.""" + + def test_close_of_a_died_job_commits_invalid(self): + # A real launch first so the lineage can receive a later close + # commit; then plant a sibling launched record as DIED. + self.launch_ok(job="plan", harness="grok", stage="plan") + job_id = "20260826T000000Z-plan-grok-d1edcc" + d = self.jobs_root / job_id + d.mkdir() + pid = self.dead_pid() + (d / "state.json").write_text(json.dumps({ + "pid": pid, "pgid": pid, + "session": {"id": "22222222-2222-4222-8222-222222222222"}, + "attempt": 1, + }) + "\n", encoding="utf-8") + self.assertFalse((d / "exit").exists()) + record_path = ( + self.repo / ".dev" / "records" / "dispatches" / f"{job_id}.json" + ) + seed = { + "id": job_id, "lane": "dev", "stage": "plan", "unit": "work", + "lineage": {"branch": "work", "base_sha": self.root_sha}, + "follows": [], "job": "plan", "role": "read", + "dispatched_by": ls.AGENT, + "at": {"launched": "2026-08-26T00:00:00Z", "closed": None}, + "snapshot": { + "mode": "whole", "ref_name": "HEAD", + "ref_sha": self.ref, "behind_tip": 0, + "root": str(d / "snapshot"), + }, + "harness": { + "name": "grok", "version": "1.0.5", + "isolation": { + "mechanism": "home", + "observed": {"unresolved": "not run"}, + }, + "sandbox": "plan", "containment": "policy", "argv": [], + }, + "model": { + "requested": ls.REQUESTED_MODEL, + "effort_requested": None, "ran": None, "read_from": None, + }, + "session": {"id": None, "stream": None, + "stream_sha256_at_close": None}, + "brief": { + "template": {"path": "jobs.json", "sha256": "a" * 64}, + "scope": "x", "inputs": [], "sha256": "b" * 64, "bytes": 1, + }, + "caps": {"source": "wires.py"}, + "overrides": [], "attempts": [], + "result": None, "status": "launched", + } + (d / "snapshot").mkdir() + record_path.write_text(json.dumps(seed, indent=2) + "\n", encoding="utf-8") + self.assertIn("launched", record_path.read_text(encoding="utf-8")) + before_head = self._git("rev-parse", "HEAD").stdout.strip() + code, out, err = self.run_main(["close", job_id]) + self.assertEqual(code, 0, self.combined(out, err)) + rec = json.loads(record_path.read_text(encoding="utf-8")) + self.assertEqual(rec["status"], "died") + envelope = (rec.get("result") or {}).get("envelope") or {} + self.assertEqual(envelope.get("status"), "invalid") + after_head = self._git("rev-parse", "HEAD").stdout.strip() + self.assertNotEqual(after_head, before_head) + msg = self._git("log", "-1", "--format=%s").stdout + self.assertIn(job_id, msg) + + +class LaunchedRecordExistsBeforeTheChildFinishes(ls._TempLaunch): + """The launched record is written before the child runs to completion.""" + + def test_status_launched_is_visible_while_the_child_is_still_running(self): + os.environ["TASK_LAUNCH_SLEEP"] = "0.5" + seen = {} + + def watch(): + deadline = time.time() + 4 + while not self.start_witness.is_file() and time.time() < deadline: + time.sleep(0.01) + files = self.record_files() + seen["n_records"] = len(files) + if files: + data = json.loads(files[0].read_text(encoding="utf-8")) + seen["status"] = data.get("status") + seen["result"] = data.get("result") + exits = list(self.jobs_root.glob("*/exit")) + seen["exit"] = bool(exits) + + t = threading.Thread(target=watch) + t.start() + self.dispatch(self.argv_for(job="plan", harness="grok", stage="plan")) + t.join(timeout=6) + self.assertEqual( + seen.get("status"), "launched", + "the launched record must exist before the child finishes: " + f"{seen!r}", + ) + self.assertIsNone(seen.get("result")) + self.assertFalse( + seen.get("exit"), + "exit is written last, so it must be absent at child-start", + ) + + +class RuntimeOutcomesStillCommitARecord(ls._TempLaunch): + """harness-cli, envelope-parse, timeout, trip produce committed records.""" + + def _committed_invalid(self, rec, *, needles): + envelope = (rec.get("result") or {}).get("envelope") or {} + blob = (json.dumps(rec) + json.dumps(envelope)).lower() + self.assertTrue( + envelope.get("status") in ("invalid", "tripped") + or rec.get("status") in ("closed", "died"), + f"runtime outcome must still write a record: {rec!r}", + ) + for n in needles: + self.assertIn(n, blob, f"missing {n!r} in {blob!r}") + msg = self._git("log", "-1", "--format=%B").stdout + self.assertIn(rec["id"], msg) + self.assertIn("Source: generated: ops/devlane/dispatch/launch.py", msg) + + def test_harness_cli_nonzero_exit_commits_an_invalid_record(self): + os.environ["TASK_LAUNCH_EXIT"] = "127" + os.environ["TASK_LAUNCH_STDOUT"] = "none" + os.environ["TASK_LAUNCH_WRITE_STREAM"] = "0" + before = self._git("rev-parse", "HEAD").stdout.strip() + self.dispatch(self.argv_for( + job="plan", harness="grok", stage="plan", + )) + rec = self.read_record() + self.assertNotEqual( + self._git("rev-parse", "HEAD").stdout.strip(), before, + ) + self._committed_invalid(rec, needles=["harness-cli"]) + self.assertTrue(self.start_witness.is_file()) + + def test_prose_stdout_is_envelope_parse_and_is_committed(self): + os.environ["TASK_LAUNCH_STDOUT"] = "prose" + before = self._git("rev-parse", "HEAD").stdout.strip() + self.dispatch(self.argv_for(job="plan", harness="grok", stage="plan")) + rec = self.read_record() + self.assertNotEqual( + self._git("rev-parse", "HEAD").stdout.strip(), before, + ) + self._committed_invalid(rec, needles=["envelope-parse"]) + + def test_timeout_kills_the_child_and_commits_the_record(self): + os.environ["DISPATCH_TIMEOUT"] = "0.3" + os.environ["TASK_LAUNCH_SLEEP"] = "8" + os.environ["TASK_LAUNCH_WRITE_STREAM"] = "1" + started = time.monotonic() + self.dispatch(self.argv_for( + job="plan", harness="grok", stage="plan", + )) + elapsed = time.monotonic() - started + rec = self.read_record() + self.assertLess(elapsed, 4, f"timeout must not wait out 8s: {elapsed}") + self._committed_invalid(rec, needles=["timeout"]) + if self.start_witness.is_file(): + pid = self.read_start_witness()["pid"] + self.assertFalse(ls.pid_is_alive(pid)) + + def test_a_battery_trip_commits_a_tripped_record(self): + done = self.home / "trip-completed.marker" + os.environ["TASK_LAUNCH_DONE"] = str(done) + os.environ["TASK_LAUNCH_OVER_OUT"] = "5000000" + os.environ["TASK_LAUNCH_SLEEP"] = "6" + os.environ["TASK_LAUNCH_WRITE_STREAM"] = "1" + self.dispatch(self.argv_for(job="plan", harness="codex", stage="plan")) + rec = self.read_record() + self._committed_invalid(rec, needles=["trip"]) + self.assertFalse( + done.is_file(), + "a tripped run must not be allowed to complete", + ) diff --git a/ops/devlane/dispatch/tests/test_record.py b/ops/devlane/dispatch/tests/test_record.py new file mode 100644 index 0000000..8cb8c2d --- /dev/null +++ b/ops/devlane/dispatch/tests/test_record.py @@ -0,0 +1,306 @@ +"""record.py: one implementation per rule, the way envelope.py is. + +Written from CONTRACT.md §Dispatch The record, before the module +existed. Each test names the contract rule it pins: + + Rec1 fields land in the contracted order (input order is not enough) + Rec2 status launched carries no result + Rec3 model.ran is the stream value, or null with a note — never the alias + Rec4 observed is the probe's result or {"unresolved": ...}, never false + Rec5 lane is "dev" + Rec6 validate names the field it refuses + Rec7 follows is a list of ids; unit is a string + Rec8 a record that fails validation is never "valid" + Rec9 schema-complete: lineage, role, snapshot.mode, containment, status +""" + +from __future__ import annotations + +import unittest + +import launch_support as ls + + +class _RecordCase(unittest.TestCase): + def recmod(self): + # Load inside the test method so a missing file is an empty + # stub, not a setUp ERROR, and the assertion below is the red. + if getattr(self, "record", None) is None: + self.record = ls.require_module(self, "record") + return self.record + + def built(self, **overrides): + record = self.recmod() + payload = dict(self._minimum()) + payload.update(overrides) + try: + return record.build(payload) + except TypeError: + return record.build(**payload) + + def _minimum(self): + return { + "id": "20260826T000000Z-plan-grok-abc123", + "lane": "dev", + "stage": "plan", + "unit": "work", + "lineage": {"branch": "work", "base_sha": "a" * 40}, + "follows": [], + "job": "plan", + "role": "read", + "dispatched_by": ls.AGENT, + "at": {"launched": "2026-08-26T00:00:00Z", "closed": None}, + "snapshot": { + "mode": "whole", + "ref_name": "HEAD", + "ref_sha": "b" * 40, + "behind_tip": 0, + "root": "/tmp/jobs/id/snapshot", + }, + "harness": { + "name": "grok", + "version": "1.0.5", + "isolation": { + "mechanism": "home", + "env": {"GROK_HOME": "/tmp/jobs/id/home/grok"}, + "home": "/tmp/jobs/id/home/grok", + "auth_files": ["auth.json"], + "store": "/tmp/jobs/id/home/grok/sessions", + "observed": {"unresolved": "behavioural probe has not run"}, + }, + "sandbox": "plan", + "containment": "policy", + "argv": ["grok", "-s", "uuid"], + }, + "model": { + "requested": "alias-requested", + "effort_requested": None, + "ran": None, + "read_from": None, + }, + "session": { + "id": "00000000-0000-4000-8000-000000000001", + "stream": None, + "stream_sha256_at_close": None, + }, + "brief": { + "template": {"path": "ops/devlane/task/jobs.json", "sha256": "c" * 64}, + "scope": "pin", + "inputs": [], + "sha256": "d" * 64, + "bytes": 3, + }, + "caps": {"cap-out": 500000, "source": "wires.py"}, + "overrides": [], + "attempts": [], + "result": None, + "status": "launched", + } + + def _raises_naming(self, rec, field): + with self.assertRaises(Exception) as caught: + self.recmod().validate(rec) + self.assertIn(field, str(caught.exception).lower()) + return caught.exception + + +class FieldOrderIsTheContract(_RecordCase): + """Rec1 — a caller diffing two records sees the same fields in the + same places. envelope.py already treats key order as shape. + Input is deliberately reversed so an echo-the-payload stub fails.""" + + def test_built_record_keys_match_the_contracted_order(self): + payload = self._minimum() + scrambled = {k: payload[k] for k in reversed(list(payload))} + self.assertNotEqual(tuple(scrambled), ls.RECORD_FIELDS) + record = self.recmod() + try: + rec = record.build(scrambled) + except TypeError: + rec = record.build(**scrambled) + self.assertEqual(tuple(rec), ls.RECORD_FIELDS) + + def test_module_declares_the_same_order(self): + declared = getattr(self.recmod(), "FIELDS", None) + self.assertIsNotNone(declared, "FIELDS is the order contract") + self.assertEqual(tuple(declared), ls.RECORD_FIELDS) + + +class LaunchedCarriesNoResult(_RecordCase): + """Rec2 — written at launch with status: launched and no result.""" + + def test_a_launched_record_has_result_none(self): + rec = self.built( + status="launched", + result={ + "head": "b" * 40, + "changed_paths": [], + "residual_paths": [], + "envelope": {}, + }, + ) + self.assertEqual(rec["status"], "launched") + self.assertIsNone(rec["result"]) + + def test_validate_refuses_a_launched_record_that_already_has_a_result(self): + rec = self.built() + rec["result"] = { + "head": "b" * 40, + "changed_paths": [], + "residual_paths": [], + "envelope": {}, + } + rec["status"] = "launched" + self._raises_naming(rec, "result") + + +class ModelRanIsNeverTheAliasCopiedOver(_RecordCase): + """Rec3 — requested is the alias; ran is the stream; no stream → + null with a note, never the alias copied over.""" + + def test_ran_null_is_not_silently_filled_with_requested(self): + rec = self.built() + rec["model"]["ran"] = None + rec["model"]["requested"] = "alias-requested" + rec["model"]["read_from"] = None + rec["model"].pop("note", None) + rec.pop("note", None) + rec = self.recmod().validate(rec) + self.assertIsNone(rec["model"]["ran"]) + self.assertEqual(rec["model"]["requested"], "alias-requested") + self.assertNotEqual(rec["model"]["ran"], rec["model"]["requested"]) + note = rec.get("note") or rec["model"].get("note") + self.assertTrue( + str(note or "").strip(), + "null ran must carry a note, not silence", + ) + + def test_copying_the_alias_into_ran_without_a_stream_is_refused(self): + rec = self.built() + rec["model"]["ran"] = rec["model"]["requested"] + rec["model"]["read_from"] = None + rec["session"]["stream"] = None + self._raises_naming(rec, "ran") + + +class ObservedIsNeverAManufacturedFalse(_RecordCase): + """Rec4 / plan (u) — observed is verbatim or {"unresolved": why}.""" + + def test_unresolved_is_the_honest_absent_observation(self): + payload = self._minimum() + del payload["harness"]["isolation"]["observed"] + record = self.recmod() + try: + rec = record.build(payload) + except TypeError: + rec = record.build(**payload) + isolation = rec["harness"]["isolation"] + self.assertIn("observed", isolation) + observed = isolation["observed"] + self.assertIsInstance(observed, dict) + self.assertIn("unresolved", observed) + self.assertNotIn(observed.get("unresolved"), (None, "", False)) + + def test_observed_false_is_refused(self): + rec = self.built() + rec["harness"]["isolation"]["observed"] = False + self._raises_naming(rec, "observed") + + def test_observed_true_without_evidence_is_refused(self): + rec = self.built() + rec["harness"]["isolation"]["observed"] = True + self._raises_naming(rec, "observed") + + +class LaneIsDev(_RecordCase): + """Rec5 — every record is lane: "dev"; the launcher refuses main.""" + + def test_built_lane_is_dev(self): + rec = self.built(lane="prod") + self.assertEqual(rec["lane"], "dev") + + def test_a_product_lane_is_refused(self): + rec = self.built() + rec["lane"] = "prod" + self._raises_naming(rec, "lane") + + +class ValidateNamesTheField(_RecordCase): + """Rec6 — a record that fails validation is never committed; the + launcher exits non-zero naming the field.""" + + def test_a_missing_field_is_named(self): + rec = self.built() + del rec["job"] + self._raises_naming(rec, "job") + + +class FollowsAndUnitAreShaped(_RecordCase): + """Rec7 / plan (q) — follows [ids], unit a string defaulting to the + branch. Neither is context.prior.""" + + def test_follows_is_a_list_and_unit_is_a_string(self): + rec = self.built( + follows=["20260826T000000Z-plan-grok-000001"], + unit="", + ) + self.assertIsInstance(rec["follows"], list) + self.assertEqual(len(rec["follows"]), 1) + self.assertIsInstance(rec["unit"], str) + self.assertTrue(rec["unit"].strip(), "unit defaults to the lineage branch") + self.assertEqual(rec["unit"], rec["lineage"]["branch"]) + + def test_follows_default_is_an_empty_list_not_absent(self): + payload = self._minimum() + del payload["follows"] + record = self.recmod() + try: + rec = record.build(payload) + except TypeError: + rec = record.build(**payload) + self.assertIn("follows", rec) + self.assertEqual(rec["follows"], []) + + +class StatusIsTheEnum(_RecordCase): + """Rec8 — status is launched | closed | died.""" + + def test_unknown_status_is_refused(self): + rec = self.built() + rec["status"] = "finished" + self._raises_naming(rec, "status") + + +class SchemaIsComplete(_RecordCase): + """Rec9 — a shallow validator that accepts lineage=None, role=execute, + or snapshot.mode=shared-and-wrong is not the contract.""" + + def test_lineage_none_is_refused(self): + rec = self.built() + rec["lineage"] = None + self._raises_naming(rec, "lineage") + + def test_lineage_missing_branch_is_refused(self): + rec = self.built() + rec["lineage"] = {"base_sha": "a" * 40} + self._raises_naming(rec, "lineage") + + def test_role_execute_is_refused(self): + rec = self.built() + rec["role"] = "execute" + self._raises_naming(rec, "role") + + def test_snapshot_mode_shared_is_refused(self): + rec = self.built() + rec["snapshot"]["mode"] = "shared-and-wrong" + self._raises_naming(rec, "mode") + + def test_containment_must_be_os_or_policy(self): + rec = self.built() + rec["harness"]["containment"] = "hope" + self._raises_naming(rec, "containment") + + def test_follows_must_be_a_list(self): + rec = self.built() + rec["follows"] = "20260826T000000Z-plan-grok-000001" + self._raises_naming(rec, "follows") diff --git a/ops/devlane/fixtures/envelopes/claude-result-error-max-turns.json b/ops/devlane/fixtures/envelopes/claude-result-error-max-turns.json new file mode 100644 index 0000000..2fa2d00 --- /dev/null +++ b/ops/devlane/fixtures/envelopes/claude-result-error-max-turns.json @@ -0,0 +1 @@ +{"type":"result","subtype":"error_max_turns","is_error":true,"result":"max turns reached","structured_output":null,"session_id":"00000000-0000-4000-8000-000000000001","usage":{"input_tokens":10,"output_tokens":4},"total_cost_usd":0.001} diff --git a/ops/devlane/fixtures/envelopes/claude-result-error-max-turns.meta.json b/ops/devlane/fixtures/envelopes/claude-result-error-max-turns.meta.json new file mode 100644 index 0000000..b1de5c5 --- /dev/null +++ b/ops/devlane/fixtures/envelopes/claude-result-error-max-turns.meta.json @@ -0,0 +1,7 @@ +{ + "harness": "claude", + "cli_version": "unseen-in-records", + "captured": "constructed-2026-08-29", + "source": "docs-claude-headless result wrapper; subtype names from harness-research.md U4b.3 / S33-S34 (error_max_turns). No closed record on 2026-08-28/29 used --output-format json, so these bytes are docs-shaped, not a session dump.", + "shape": "one JSON object {type:result, subtype:error_max_turns, is_error:true} with result a string and no structured_output" +} diff --git a/ops/devlane/fixtures/envelopes/codex-ndjson-tail.jsonl b/ops/devlane/fixtures/envelopes/codex-ndjson-tail.jsonl new file mode 100644 index 0000000..0445050 --- /dev/null +++ b/ops/devlane/fixtures/envelopes/codex-ndjson-tail.jsonl @@ -0,0 +1,3 @@ +{"type":"thread.started","thread_id":"00000000-0000-4000-8000-000000000001"} +{"type":"turn.started"} +{"type":"item.completed","item":{"type":"agent_message","text":"{\"job\":\"plan\",\"status\":\"ok\",\"verdict\":\"approve\",\"counts\":{\"p1\":0,\"p2\":0,\"p3\":0,\"opinions\":0},\"findings\":[],\"artifacts\":{},\"spend\":{\"harness\":\"codex\",\"total\":0,\"out\":0,\"runs\":1},\"stamp\":{\"ref\":\"harness-placeholder\",\"started\":null,\"ended\":null},\"note\":\"u10-codex-agent-message\"}"}} diff --git a/ops/devlane/fixtures/envelopes/codex-ndjson-tail.meta.json b/ops/devlane/fixtures/envelopes/codex-ndjson-tail.meta.json new file mode 100644 index 0000000..80c2677 --- /dev/null +++ b/ops/devlane/fixtures/envelopes/codex-ndjson-tail.meta.json @@ -0,0 +1,7 @@ +{ + "harness": "codex", + "cli_version": "unseen-in-records", + "captured": "constructed-2026-08-29", + "source": "harness-research.md §4: stdout is NDJSON events; the last agent_message holds the final message. No closed record on 2026-08-28/29 used --json -o, so these bytes are docs-shaped, not a session dump.", + "shape": "NDJSON: thread.started, turn.started, item.completed{agent_message.text = envelope JSON}" +} diff --git a/ops/devlane/fixtures/envelopes/grok-fenced-json.meta.json b/ops/devlane/fixtures/envelopes/grok-fenced-json.meta.json new file mode 100644 index 0000000..236c999 --- /dev/null +++ b/ops/devlane/fixtures/envelopes/grok-fenced-json.meta.json @@ -0,0 +1,8 @@ +{ + "harness": "grok", + "cli_version": "unseen-in-records", + "captured": "constructed-2026-08-30", + "source": "docs-shaped pretty-printed nine-key object inside a ```json fence, for grok's mandatory --output-format plain path. Not a session dump.", + "shape": "narration sentence, then a ```json fence, then a pretty-printed ENVELOPE_SCHEMA object, then a closing fence. No single line is a complete JSON object.", + "note": "Constructed so a line- or NDJSON-oriented scan cannot recover the envelope. Regenerated, never presented as captured bytes. D-ENV-3 / skeptic 2fd473 P2." +} diff --git a/ops/devlane/fixtures/envelopes/grok-fenced-json.raw.out b/ops/devlane/fixtures/envelopes/grok-fenced-json.raw.out new file mode 100644 index 0000000..868d834 --- /dev/null +++ b/ops/devlane/fixtures/envelopes/grok-fenced-json.raw.out @@ -0,0 +1,29 @@ +Here is the envelope: + +```json +{ + "job": "plan", + "status": "ok", + "verdict": "changes", + "counts": { + "p1": 0, + "p2": 0, + "p3": 0, + "opinions": 0 + }, + "findings": [], + "artifacts": {}, + "spend": { + "harness": "grok", + "total": 0, + "out": 0, + "runs": 1 + }, + "stamp": { + "ref": "harness-placeholder", + "started": null, + "ended": null + }, + "note": "u10-grok-fenced" +} +``` diff --git a/ops/devlane/fixtures/envelopes/grok-json-schema-short-circuit-66ccb2.meta.json b/ops/devlane/fixtures/envelopes/grok-json-schema-short-circuit-66ccb2.meta.json new file mode 100644 index 0000000..fa77d19 --- /dev/null +++ b/ops/devlane/fixtures/envelopes/grok-json-schema-short-circuit-66ccb2.meta.json @@ -0,0 +1,9 @@ +{ + "harness": "grok", + "cli_version": "1.0.5", + "captured": "2026-08-29T23:48:43Z", + "record": "20260829T234838Z-review-grok-66ccb2", + "source": "raw.out of the first live grok dispatch under the U10 launcher (--output-format json --json-schema )", + "shape": "one JSON wrapper with usage (modelCalls 1, outputTokens 213, costUSD 0.001000) and a structuredOutput that is a schema-valid envelope with status ok, no findings and note 'starting: reading briefs and review contract' \u2014 the turn ended after one model call", + "note": "Captured bytes, never edited. Evidence that grok's structured-output mode short-circuits the agentic loop; the launcher keeps plain output for grok until a probe row proves otherwise." +} diff --git a/ops/devlane/fixtures/envelopes/grok-json-schema-short-circuit-66ccb2.raw.out b/ops/devlane/fixtures/envelopes/grok-json-schema-short-circuit-66ccb2.raw.out new file mode 100644 index 0000000..3eb1c9a --- /dev/null +++ b/ops/devlane/fixtures/envelopes/grok-json-schema-short-circuit-66ccb2.raw.out @@ -0,0 +1,48 @@ +{ + "text": "{\"job\": \"review-u10\", \"status\": \"ok\", \"verdict\": null, \"counts\": {\"p1\": 0, \"p2\": 0, \"p3\": 0, \"opinions\": 0}, \"findings\": [], \"artifacts\": {}, \"spend\": {}, \"stamp\": {\"ref\": \"eb71df9bfab6c349093ebb7fabcea0aecfc83788\", \"started\": null, \"ended\": null}, \"note\": \"starting: reading briefs and review contract\"}", + "stopReason": "end_turn", + "sessionId": "11111111-1111-4111-8111-111111111111", + "requestId": "22222222-2222-4222-8222-222222222222", + "thought": "synthetic fixture reasoning placeholder", + "usage": { + "input_tokens": 18991, + "cache_read_input_tokens": 256, + "cache_creation_input_tokens": 0, + "output_tokens": 213, + "reasoning_tokens": 94, + "total_tokens": 19460 + }, + "num_turns": 1, + "total_cost_usd": 0.001000, + "total_cost_usd_ticks": 10000000, + "modelUsage": { + "grok-4.6-build": { + "inputTokens": 18991, + "outputTokens": 213, + "cacheReadInputTokens": 256, + "cacheCreationInputTokens": 0, + "modelCalls": 1, + "costUSD": 0.001000 + } + }, + "structuredOutput": { + "job": "review-u10", + "status": "ok", + "verdict": null, + "counts": { + "p1": 0, + "p2": 0, + "p3": 0, + "opinions": 0 + }, + "findings": [], + "artifacts": {}, + "spend": {}, + "stamp": { + "ref": "eb71df9bfab6c349093ebb7fabcea0aecfc83788", + "started": null, + "ended": null + }, + "note": "starting: reading briefs and review contract" + } +} diff --git a/ops/devlane/fixtures/envelopes/grok-narration-then-object.meta.json b/ops/devlane/fixtures/envelopes/grok-narration-then-object.meta.json new file mode 100644 index 0000000..e1283af --- /dev/null +++ b/ops/devlane/fixtures/envelopes/grok-narration-then-object.meta.json @@ -0,0 +1,8 @@ +{ + "harness": "grok", + "cli_version": "unseen-in-records", + "captured": "constructed-2026-08-29", + "source": "docs-shaped allowed nine-key object after a narration sentence, for the fallback-scan happy path. Not a session dump. The captured S2 bytes live in grok-s2-13f5f2.raw.out.", + "shape": "narration sentence immediately followed by a single-line JSON envelope matching ENVELOPE_SCHEMA, no newline before the opening brace", + "note": "Constructed so the fallback scan can be shown to accept an allowed envelope. Regenerated, never presented as captured bytes." +} diff --git a/ops/devlane/fixtures/envelopes/grok-narration-then-object.raw.out b/ops/devlane/fixtures/envelopes/grok-narration-then-object.raw.out new file mode 100644 index 0000000..4bac23e --- /dev/null +++ b/ops/devlane/fixtures/envelopes/grok-narration-then-object.raw.out @@ -0,0 +1 @@ +The commit picked up the machine identity, so I'll amend it to the owner identity required by the attribution rules.{"job":"author-tests","status":"ok","verdict":null,"counts":{"p1":0,"p2":0,"p3":0,"opinions":0},"findings":[],"artifacts":{},"spend":{"harness":"grok","total":0,"out":0,"runs":1},"stamp":{"ref":"harness-placeholder","started":null,"ended":null},"note":"u10-grok-narration-then-object"} diff --git a/ops/devlane/fixtures/envelopes/grok-s2-13f5f2.meta.json b/ops/devlane/fixtures/envelopes/grok-s2-13f5f2.meta.json new file mode 100644 index 0000000..866a982 --- /dev/null +++ b/ops/devlane/fixtures/envelopes/grok-s2-13f5f2.meta.json @@ -0,0 +1,11 @@ +{ + "harness": "grok", + "cli_version": "unknown", + "captured": "2026-08-28", + "records": [ + "20260828T222028Z-tests-grok-13f5f2" + ], + "source": "verbatim job-dir raw.out of 20260828T222028Z-tests-grok-13f5f2. The record closed envelope-parse: no JSON object on stdout; harness.version is unknown on the record (research §1.1).", + "shape": "narration immediately followed by a single-line JSON object, no newline before the opening brace", + "note": "Copied byte-for-byte from the job dir, never edited, per D-E2E-2. The object is present; today's last-brace scan misses it." +} diff --git a/ops/devlane/fixtures/envelopes/grok-s2-13f5f2.raw.out b/ops/devlane/fixtures/envelopes/grok-s2-13f5f2.raw.out new file mode 100644 index 0000000..3e56821 --- /dev/null +++ b/ops/devlane/fixtures/envelopes/grok-s2-13f5f2.raw.out @@ -0,0 +1 @@ +I'll start from the dispatch contract and the test-author process, then write one red test per in-scope finding without reading the implementation.I'll read the contract and existing test patterns next, without opening the implementation.I'll match the existing launch-test fixtures and pin each finding as its own red assertion.I'll write one test per in-scope finding, then run each method on its own to prove it fails for that finding's reason.I'll run each new test method on its own so a shared failure cannot mask a wrong assertion.F7, F8, and F12 fail on the stated assertions. Next I’ll prove the launch-path findings the same way.F6’s plant copied the first job’s isolation paths, so I’ll retarget those at the uncollected job before re-proving the red.All 10 tests fail for their stated reasons. I'll commit the red tests, then build the envelope from those results.The commit picked up the machine identity, so I’ll amend it to the owner identity required by the attribution rules.{"job": "author-tests", "status": "ok", "verdict": null, "counts": {"p1": 1, "p2": 5, "p3": 5, "opinions": 0, "findings": 11, "tests_red": 10, "tests_errors": 0, "commands_run": 12}, "findings": [{"id": "F1", "severity": "p1", "file": ".dev/app/dispatch/tests/test_launch_prompt_feed_findings.py", "finding": "Read role that commits closes with envelope status ok; CONTRACT.md §Collect requires head equal ref_sha.", "reproduce": "cd .dev/app/dispatch/tests && python3 -m unittest test_launch_prompt_feed_findings.ReadRoleThatCommitsIsInvalid.test_a_read_role_that_commits_closes_invalid_not_ok -v"}, {"id": "F3", "severity": "p2", "file": ".dev/app/dispatch/tests/test_launch_prompt_feed_findings.py", "finding": "claude --add-dir grants the job directory, so raw.out/exit/state.json/prompt.txt sit inside an acceptEdits working directory.", "reproduce": "cd .dev/app/dispatch/tests && python3 -m unittest test_launch_prompt_feed_findings.ClaudeAddDirDoesNotExposeJobBookkeeping.test_claude_add_dir_does_not_include_job_directory_evidence_files -v"}, {"id": "F4", "severity": "p2", "file": ".dev/app/dispatch/tests/test_launch_prompt_feed_findings.py", "finding": "Launcher snapshot commit takes GIT_AUTHOR_NAME from the process environment (decoy-author), not the owner identity.", "reproduce": "cd .dev/app/dispatch/tests && python3 -m unittest test_launch_prompt_feed_findings.SnapshotCommitIdentityIgnoresEnvOverrides.test_launcher_snapshot_commit_is_the_owner_despite_git_author_env -v"}, {"id": "F5", "severity": "p2", "file": ".dev/app/dispatch/tests/test_launch_prompt_feed_findings.py", "finding": "A timed-out codex write still gets a launcher-made snapshot commit (head != ref_sha) despite note timeout.", "reproduce": "cd .dev/app/dispatch/tests && python3 -m unittest test_launch_prompt_feed_findings.TrippedWriteIsNotCommittedByTheLauncher.test_a_timed_out_codex_write_is_not_committed_by_the_launcher -v"}, {"id": "F6", "severity": "p2", "file": ".dev/app/dispatch/tests/test_launch_prompt_feed_findings.py", "finding": "close of a finished uncollected job leaves model.ran null with note 'no stream was found' while the stream sits in the job home.", "reproduce": "cd .dev/app/dispatch/tests && python3 -m unittest test_launch_prompt_feed_findings.CloseCollectsTheHarnessStream.test_close_of_a_finished_uncollected_job_sets_model_ran_from_the_stream -v"}, {"id": "F7", "severity": "p2", "file": ".dev/app/dispatch/tests/test_launch_prompt_feed_findings.py", "finding": "Narration after a pretty-printed envelope is recorded as envelope-parse: no JSON object on stdout.", "reproduce": "cd .dev/app/dispatch/tests && python3 -m unittest test_launch_prompt_feed_findings.TrailingNarrationAfterPrettyEnvelopeIsTolerated.test_narration_after_a_pretty_printed_envelope_does_not_drop_it -v"}, {"id": "F8", "severity": "p3", "file": ".dev/app/dispatch/tests/test_launch_prompt_feed_findings.py", "finding": "A supplied runtime note overwrites the parse diagnostic; note is only 'head_commit mismatch'.", "reproduce": "cd .dev/app/dispatch/tests && python3 -m unittest test_launch_prompt_feed_findings.RuntimeNoteDoesNotDestroyParseNote.test_a_runtime_note_keeps_the_envelope_parse_reason -v"}, {"id": "F9", "severity": "p3", "file": ".dev/app/dispatch/tests/test_launch_prompt_feed_findings.py", "finding": "Review {base} prefers stale origin/dev over local dev; the brief names the older merge-base.", "reproduce": "cd .dev/app/dispatch/tests && python3 -m unittest test_launch_prompt_feed_findings.ReviewDiffUsesTheMostRecentAnchor.test_stale_origin_dev_is_not_preferred_over_local_dev -v"}, {"id": "F10", "severity": "p3", "file": ".dev/app/dispatch/tests/test_launch_prompt_feed_findings.py", "finding": "claude resume feeds empty stdin; prompt.txt is not handed over.", "reproduce": "cd .dev/app/dispatch/tests && python3 -m unittest test_launch_prompt_feed_findings.ResumeFeedsTheBriefAndKeepsFlags.test_claude_and_codex_resume_feed_the_brief_on_stdin -v"}, {"id": "F11", "severity": "p3", "file": ".dev/app/dispatch/CONTRACT.md", "line": "162-166, 111-125", "finding": "Contract text to correct, not a test. Isolation table still records read sandboxes plan (claude), read-only (codex), plan (grok) while the launcher uses acceptEdits/workspace-write/auto for every role. Job-directory listing omits diff.patch, which the launcher now always writes. Nothing in the contract admits the launcher committing on a harness's behalf or adding job-directory dirs to a child via --add-dir.", "reproduce": "sed -n '111,130p;160,170p;226,236p' .dev/app/dispatch/CONTRACT.md"}, {"id": "F12", "severity": "p3", "file": ".dev/app/dispatch/tests/test_launch_prompt_feed_findings.py", "finding": "A missing spend key keeps status invalid and note envelope-missing: spend but discards findings, counts, verdict and artifacts.", "reproduce": "cd .dev/app/dispatch/tests && python3 -m unittest test_launch_prompt_feed_findings.MissingEnvelopeKeyKeepsThePayload.test_a_missing_spend_key_keeps_findings_counts_and_verdict -v"}], "artifacts": {"tests": ".dev/app/dispatch/tests/test_launch_prompt_feed_findings.py"}, "spend": {"harness": "grok", "model": "grok-4.6", "total": null, "out": null, "runs": 1}, "stamp": {"ref": "a3eddeda24fa21d72471a46d6089993634d67346", "head": "07a88cb6f367e5fef3a6394c971faf5b9cf6d0f2", "base": null, "diff": "/home/user/.local/state/minspec/dispatch/20260828T222028Z-tests-grok-13f5f2/diff.patch", "started": "2026-08-28T22:20:28Z", "ended": "2026-08-28T22:34:58Z"}, "note": "F2 out of scope (not reproduced at 2e872d2). F11 is contract text, no test. Ten tests under .dev/app/dispatch/tests/test_launch_prompt_feed_findings.py, one per remaining finding; implementation not read and not edited. Each method was run alone then together: `cd .dev/app/dispatch/tests && python3 -m unittest test_launch_prompt_feed_findings -v` at 07a88cb against implementation a3edded: Ran 10 tests in 1.471s, FAILED (failures=10), 0 errors. Each failure was the finding's stated assertion. Tests left red."} diff --git a/ops/devlane/fixtures/stores.py b/ops/devlane/fixtures/stores.py new file mode 100644 index 0000000..e00b748 --- /dev/null +++ b/ops/devlane/fixtures/stores.py @@ -0,0 +1,194 @@ +"""Shared store fixtures in the measured 2026-08-21 harness shapes. + +One builder per harness, consolidating the per-test helpers that used +to live in test_breaker.py and test_usage.py. What makes these +trustworthy rather than decorative: + +- Time is a variable the caller controls: every builder REQUIRES an + explicit ``base_timestamp`` (epoch seconds) and never reads a clock, + so repeated builds with the same arguments are byte-identical. +- The shapes replicate the live stores as measured on 2026-08-21, + including the awkward parts: Codex SPLITS its metadata (session_meta + carries id/cwd/base_instructions; model and effort live on a later + turn_context payload), Grok updates stamp EPOCH INTEGERS while its + events stamp ISO strings, and Grok's spend lives in updates.jsonl + turn_completed events (cumulative per run; runs split when totals + shrink), which ``usage_runs`` emits verbatim. +- Every content channel carries the caller's leak marker — Claude tool + blocks, Codex base_instructions.text and command payloads, Grok + session_summary/generated_title/params — so reader tests can prove + aggregates never leak session content. +""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import quote + + +def _iso_ms(epoch: float) -> str: + stamp = datetime.fromtimestamp(epoch, timezone.utc) + return stamp.strftime("%Y-%m-%dT%H:%M:%S.") + f"{stamp.microsecond // 1000:03d}Z" + + +def _iso_ns(epoch: float) -> str: + stamp = datetime.fromtimestamp(epoch, timezone.utc) + return stamp.strftime("%Y-%m-%dT%H:%M:%S.") + f"{stamp.microsecond * 1000:09d}Z" + + +def _write_jsonl(path: Path, entries) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(json.dumps(entry) + "\n" for entry in entries)) + + +def claude_entry(*, timestamp, cwd, session_id, model, mid, usage, content, + git_branch="dev", effort="high"): + return { + "type": "assistant", + "timestamp": timestamp, + "cwd": cwd, + "gitBranch": git_branch, + "effort": effort, + "isSidechain": False, + "sessionId": session_id, + "message": {"id": mid, "model": model, "usage": usage, + "content": content}, + } + + +def build_claude_store(root, slug, *, base_timestamp, cwd, session_id, + model, effort, marker, reemit_last=False): + """Two usage-bearing messages summing to the documented totals + (input 30, cached 12000, output 500); ``reemit_last`` appends a + byte-identical copy of the final message — the snapshot-rewrite + shape whose double-count pulse and breaker must refuse.""" + usage_one = {"input_tokens": 10, "cache_creation_input_tokens": 2_000, + "cache_read_input_tokens": 4_000, "output_tokens": 200} + usage_two = {"input_tokens": 20, "cache_creation_input_tokens": 1_000, + "cache_read_input_tokens": 5_000, "output_tokens": 300} + entries = [ + claude_entry( + timestamp=_iso_ms(base_timestamp), cwd=cwd, + session_id=session_id, model=model, effort=effort, + mid=f"{session_id}-msg-1", usage=usage_one, + content=[{"type": "tool_use", "name": "Read", + "input": {"file_path": marker}}]), + claude_entry( + timestamp=_iso_ms(base_timestamp + 300), cwd=cwd, + session_id=session_id, model=model, effort=effort, + mid=f"{session_id}-msg-2", usage=usage_two, + content=[{"type": "tool_use", "name": "Bash", + "input": {"command": marker}}, + {"type": "tool_result", "is_error": False, + "content": marker}]), + ] + if reemit_last: + entries.append(entries[-1]) + _write_jsonl(Path(root) / slug / f"{session_id}.jsonl", entries) + + +def build_codex_store(root, *, base_timestamp, cwd, session_id, model, + effort, marker): + """The measured SPLIT: session_meta holds id, cwd, and the long + base_instructions prompt (a content channel); model and effort live + only on turn_context. Token counts are CUMULATIVE — the last one IS + the spend; summing them is the measured double-count mistake.""" + day = datetime.fromtimestamp(base_timestamp, timezone.utc) + first_count = {"input_tokens": 200, "cached_input_tokens": 100, + "output_tokens": 40, "reasoning_output_tokens": 10, + "total_tokens": 240} + last_count = {"input_tokens": 400, "cached_input_tokens": 300, + "output_tokens": 90, "reasoning_output_tokens": 30, + "total_tokens": 490} + entries = [ + {"timestamp": _iso_ms(base_timestamp), "type": "session_meta", + "payload": {"id": session_id, "cwd": cwd, + "base_instructions": { + "text": f"You are a coding agent. {marker}"}}}, + {"timestamp": _iso_ms(base_timestamp + 30), "type": "turn_context", + "payload": {"model": model, "effort": effort, "cwd": cwd}}, + {"timestamp": _iso_ms(base_timestamp + 60), "type": "event_msg", + "payload": {"type": "token_count", + "info": {"total_token_usage": first_count}}}, + {"timestamp": _iso_ms(base_timestamp + 300), "type": "event_msg", + "payload": {"type": "shell_command", "command": marker}}, + {"timestamp": _iso_ms(base_timestamp + 540), "type": "event_msg", + "payload": {"type": "token_count", + "info": {"total_token_usage": last_count}}}, + ] + name = day.strftime("%Y-%m-%dT%H-%M-%S") + stream = (Path(root) / "sessions" / day.strftime("%Y") / day.strftime("%m") + / day.strftime("%d") / f"rollout-{name}-{session_id}.jsonl") + _write_jsonl(stream, entries) + + +def build_grok_store(root, cwd, *, base_timestamp, session_id, model, + marker, usage_runs=None, head_commit=None, + git_root_dir=None, grok_home=None): + """Measured Grok shapes: updates are {method, params, timestamp} + with EPOCH-INTEGER timestamps; events are {type, ts} (tool events + add tool_name) with ISO timestamps; summary.json carries content + channels (session_summary, generated_title) and reasoning_effort + and never token usage — spend, when present, rides updates.jsonl + turn_completed events, emitted only from ``usage_runs`` and never + invented. + + ``head_commit``, ``git_root_dir`` and ``grok_home`` are optional + stamps the live summary.json carries; callers that need the + launcher's cross-check against ``snapshot.ref_sha`` pass them. + """ + session = Path(root) / "sessions" / quote(cwd, safe="") / session_id + session.mkdir(parents=True, exist_ok=True) + summary = { + "info": {"id": session_id, "cwd": cwd}, + "created_at": _iso_ns(base_timestamp), + "updated_at": _iso_ns(base_timestamp + 1200), + "num_messages": 3, + "current_model_id": model, + "reasoning_effort": "high", + "session_summary": f"Reviewed the change. {marker}", + "generated_title": f"Session about {marker}", + } + if head_commit is not None: + summary["head_commit"] = head_commit + if git_root_dir is not None: + summary["git_root_dir"] = git_root_dir + if grok_home is not None: + summary["grok_home"] = grok_home + (session / "summary.json").write_text(json.dumps(summary)) + updates = [ + {"method": "session/update", + "params": {"update": {"kind": "tool", "detail": marker}}, + "timestamp": int(base_timestamp) + 20}, + {"method": "session/update", + "params": {"update": {"kind": "note", "detail": marker}}, + "timestamp": int(base_timestamp) + 60}, + ] + # Usage rides updates whose sessionUpdate is turn_completed + # (measured 2026-08-21 late: cumulative within a run; a run ends + # when totals shrink). ``usage_runs`` is a list of runs, each a + # list of cumulative usage dicts emitted VERBATIM — the fixture + # never invents or normalises currencies. + stamp = int(base_timestamp) + 90 + for run in usage_runs or (): + for usage in run: + updates.append( + {"method": "session/update", + "params": {"update": {"sessionUpdate": "turn_completed", + "usage": usage}}, + "timestamp": stamp}) + stamp += 30 + _write_jsonl(session / "updates.jsonl", updates) + _write_jsonl(session / "events.jsonl", [ + {"type": "phase_changed", "ts": _iso_ms(base_timestamp + 10)}, + {"type": "tool_started", "tool_name": "search_code", + "ts": _iso_ms(base_timestamp + 30)}, + {"type": "tool_completed", "tool_name": "search_code", + "ts": _iso_ms(base_timestamp + 40)}, + {"type": "permission_requested", "ts": _iso_ms(base_timestamp + 50)}, + {"type": "permission_resolved", "ts": _iso_ms(base_timestamp + 70)}, + {"type": "loop_started", "ts": _iso_ms(base_timestamp + 80)}, + {"type": "phase_changed", "ts": _iso_ms(base_timestamp + 90)}, + ]) diff --git a/ops/devlane/fixtures/tests/test_grok_usage_fixture.py b/ops/devlane/fixtures/tests/test_grok_usage_fixture.py new file mode 100644 index 0000000..eb71c5e --- /dev/null +++ b/ops/devlane/fixtures/tests/test_grok_usage_fixture.py @@ -0,0 +1,388 @@ +"""Contracts for Grok usage emission by the shared store fixture.""" + +import hashlib +import importlib.util +import inspect +import json +import tempfile +import unittest +from pathlib import Path +from urllib.parse import quote + +HERE = Path(__file__).resolve() +STORES = HERE.parents[1] / "stores.py" + +BASE_EPOCH = 1_787_306_400 +SECOND_BASE_EPOCH = BASE_EPOCH + 86_400 +REPO = "/home/work/projects/minspec/workbench" +SESSION = "grok-usage-fixture" +MODEL = "grok-4.6" +MARKER = "GROKUSAGEFIXTUREMARKER" + +LEGACY_STORE_SHA256 = ( + "49ab593d38217dcf1ee0f7d2443da422b61e77aa08f8e387b2208ee10f0a74e0" +) + +USAGE_RUNS = [ + [ + { + "inputTokens": 100, + "outputTokens": 20, + "totalTokens": 120, + "cachedReadTokens": 30, + "cacheCreationTokens": 5, + "reasoningTokens": 7, + "modelCalls": 1, + "apiDurationMs": 1_000, + "costUsdTicks": 1_100, + "numTurns": 1, + "modelUsage": { + MODEL: { + "inputTokens": 100, + "outputTokens": 20, + "totalTokens": 120, + "cachedReadTokens": 30, + "cacheCreationTokens": 5, + "reasoningTokens": 7, + "modelCalls": 1, + "apiDurationMs": 1_000, + "costUsdTicks": 1_100, + } + }, + }, + { + "inputTokens": 300, + "outputTokens": 80, + "totalTokens": 380, + "cachedReadTokens": 90, + "reasoningTokens": 40, + "modelCalls": 3, + "apiDurationMs": 3_500, + "costUsdTicks": 2_500, + "numTurns": 3, + "modelUsage": { + MODEL: { + "inputTokens": 300, + "outputTokens": 80, + "totalTokens": 380, + "cachedReadTokens": 90, + "reasoningTokens": 40, + "modelCalls": 3, + "apiDurationMs": 3_500, + "costUsdTicks": 2_500, + } + }, + }, + ], + [ + { + "inputTokens": 40, + "outputTokens": 10, + "totalTokens": 50, + "cachedReadTokens": 7, + "cacheCreationTokens": 2, + "reasoningTokens": 3, + "modelCalls": 1, + "apiDurationMs": 500, + "numTurns": 1, + "usageIsIncomplete": True, + "modelUsage": { + MODEL: { + "inputTokens": 40, + "outputTokens": 10, + "totalTokens": 50, + "cachedReadTokens": 7, + "cacheCreationTokens": 2, + "reasoningTokens": 3, + "modelCalls": 1, + "apiDurationMs": 500, + } + }, + } + ], +] + + +def load_module(testcase): + testcase.assertTrue( + STORES.is_file(), + f"{STORES} is missing; the fixture contract requires it", + ) + spec = importlib.util.spec_from_file_location( + "minspec_grok_usage_stores", + STORES, + ) + testcase.assertIsNotNone(spec) + testcase.assertIsNotNone(spec.loader) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def jsonl(path): + return [ + json.loads(raw) + for raw in path.read_text().splitlines() + if raw.strip() + ] + + +def session_path(root, session=SESSION): + return ( + Path(root) + / "sessions" + / quote(REPO, safe="") + / session + ) + + +def usage_entries(entries): + found = [] + for entry in entries: + update = ( + (entry.get("params") or {}).get("update") or {} + ) + if update.get("sessionUpdate") == "turn_completed": + found.append(entry) + return found + + +def legacy_digest(session): + files = sorted( + path + for path in session.rglob("*") + if path.is_file() + ) + digest = hashlib.sha256() + for path in files: + digest.update(path.relative_to(session).as_posix().encode()) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +class GrokUsageFixture(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory( + prefix="grok-usage-fixture-" + ) + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.stores = load_module(self) + + def assert_usage_keyword(self): + parameter = inspect.signature( + self.stores.build_grok_store + ).parameters.get("usage_runs") + self.assertIsNotNone( + parameter, + ( + "build_grok_store must expose an explicit usage_runs " + "keyword" + ), + ) + self.assertIsNot( + parameter.default, + inspect.Parameter.empty, + ( + "usage_runs must be optional so legacy fixture calls " + "remain valid" + ), + ) + + def build_usage_store(self, root, base_timestamp): + self.stores.build_grok_store( + root, + REPO, + base_timestamp=base_timestamp, + session_id=SESSION, + model=MODEL, + marker=MARKER, + usage_runs=json.loads(json.dumps(USAGE_RUNS)), + ) + return session_path(root) + + def test_usage_runs_round_trip_in_measured_shape_and_deterministic_time(self): + self.assert_usage_keyword() + + first = self.build_usage_store( + self.root / "first", + BASE_EPOCH, + ) + repeated = self.build_usage_store( + self.root / "repeated", + BASE_EPOCH, + ) + shifted = self.build_usage_store( + self.root / "shifted", + SECOND_BASE_EPOCH, + ) + + first_updates = jsonl(first / "updates.jsonl") + repeated_updates = jsonl(repeated / "updates.jsonl") + shifted_updates = jsonl(shifted / "updates.jsonl") + + self.assertEqual( + (first / "updates.jsonl").read_bytes(), + (repeated / "updates.jsonl").read_bytes(), + ( + "identical usage fixture arguments did not produce " + "byte-identical updates.jsonl" + ), + ) + + legacy_updates = [ + { + "method": "session/update", + "params": { + "update": { + "kind": "tool", + "detail": MARKER, + } + }, + "timestamp": BASE_EPOCH + 20, + }, + { + "method": "session/update", + "params": { + "update": { + "kind": "note", + "detail": MARKER, + } + }, + "timestamp": BASE_EPOCH + 60, + }, + ] + self.assertEqual( + first_updates[:2], + legacy_updates, + "usage emission changed the existing Grok update records", + ) + + emitted = usage_entries(first_updates) + repeated_emitted = usage_entries(repeated_updates) + shifted_emitted = usage_entries(shifted_updates) + expected_usage = [ + usage + for run in USAGE_RUNS + for usage in run + ] + + self.assertEqual( + len(emitted), + len(expected_usage), + "not every cumulative usage event was emitted", + ) + self.assertEqual( + [entry["params"]["update"]["usage"] for entry in emitted], + expected_usage, + "the pinned usage values did not round-trip", + ) + self.assertEqual( + [entry["params"]["update"]["usage"] for entry in repeated_emitted], + expected_usage, + ) + self.assertEqual( + [entry["params"]["update"]["usage"] for entry in shifted_emitted], + expected_usage, + ) + + for index, entry in enumerate(emitted): + with self.subTest(event=index): + self.assertEqual( + set(entry), + {"method", "params", "timestamp"}, + ) + self.assertEqual(entry["method"], "session/update") + self.assertEqual(set(entry["params"]), {"update"}) + self.assertEqual( + set(entry["params"]["update"]), + {"sessionUpdate", "usage"}, + ) + self.assertEqual( + entry["params"]["update"]["sessionUpdate"], + "turn_completed", + ) + self.assertIs(type(entry["timestamp"]), int) + + first_stamps = [entry["timestamp"] for entry in emitted] + shifted_stamps = [entry["timestamp"] for entry in shifted_emitted] + self.assertEqual( + first_stamps, + sorted(first_stamps), + "usage timestamps must preserve stream order", + ) + self.assertEqual( + len(set(first_stamps)), + len(first_stamps), + "usage events need distinct deterministic timestamps", + ) + self.assertEqual( + shifted_stamps, + [ + timestamp + SECOND_BASE_EPOCH - BASE_EPOCH + for timestamp in first_stamps + ], + "usage timestamps are not derived from base_timestamp", + ) + + raw_store = "\n".join( + path.read_text() + for path in sorted(first.iterdir()) + if path.is_file() + ) + self.assertIn( + MARKER, + raw_store, + "the content leak marker was not planted in the Grok store", + ) + self.assertNotIn( + MARKER, + json.dumps(expected_usage, sort_keys=True), + "the content marker leaked into a usage field", + ) + summary = json.loads((first / "summary.json").read_text()) + self.assertEqual(summary["info"]["id"], SESSION) + self.assertEqual(summary["info"]["cwd"], REPO) + + def test_omitting_usage_runs_preserves_the_legacy_store_bytes(self): + self.assert_usage_keyword() + + root = self.root / "legacy" + self.stores.build_grok_store( + root, + REPO, + base_timestamp=BASE_EPOCH, + session_id="grok-fixture", + model=MODEL, + marker="FIXTUREPROMPTMARKER", + ) + session = session_path(root, session="grok-fixture") + paths = { + path.relative_to(session).as_posix() + for path in session.rglob("*") + if path.is_file() + } + self.assertEqual( + paths, + {"events.jsonl", "summary.json", "updates.jsonl"}, + "the default Grok fixture changed its artifact set", + ) + self.assertEqual( + legacy_digest(session), + LEGACY_STORE_SHA256, + ( + "omitting usage_runs must build the pre-usage Grok " + "store byte-identically" + ), + ) + self.assertEqual( + usage_entries(jsonl(session / "updates.jsonl")), + [], + "the default fixture invented usage events", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/fixtures/tests/test_stores.py b/ops/devlane/fixtures/tests/test_stores.py new file mode 100644 index 0000000..cc0c494 --- /dev/null +++ b/ops/devlane/fixtures/tests/test_stores.py @@ -0,0 +1,1084 @@ +"""Guards against nondeterministic, malformed, or content-leaking fixtures.""" + +import argparse +import importlib.util +import inspect +import itertools +import json +import re +import subprocess +import sys +import tempfile +import typing +import unittest +from pathlib import Path +from urllib.parse import quote + +HERE = Path(__file__).resolve() +STORES = HERE.parents[1] / "stores.py" +USAGE = HERE.parents[2] / "telemetry" / "usage.py" +BREAKER = HERE.parents[2] / "telemetry" / "breaker.py" + +BASE_EPOCH = 1_787_306_400 +SECOND_BASE_EPOCH = BASE_EPOCH + 86_400 +BASE_ISO = "2026-08-21T10:00:00.000Z" +SECOND_BASE_ISO = "2026-08-22T10:00:00.000Z" +CLAUDE_END = "2026-08-21T10:05:00.000Z" +CODEX_END = "2026-08-21T10:09:00.000Z" +GROK_BASE = "2026-08-21T10:00:00.000000000Z" +SECOND_GROK_BASE = "2026-08-22T10:00:00.000000000Z" +GROK_END = "2026-08-21T10:20:00.000000000Z" + +REPO = "/home/work/projects/minspec/workbench" +SLUG = "-home-work-projects-minspec-workbench" +MARKER = "FIXTUREPROMPTMARKER" + +CLAUDE_SESSION = "claude-fixture" +CODEX_SESSION = "codex-fixture" +GROK_SESSION = "grok-fixture" + +CLAUDE_MODEL = "claude-fable-5" +CODEX_MODEL = "gpt-5-codex" +GROK_MODEL = "grok-4.6" + +CLAUDE_TOKENS = { + "input": 30, + "cached": 12_000, + "output": 500, + "total": 12_530, +} +CODEX_TOKENS = { + "input": 400, + "cached": 300, + "output": 90, + "total": 490, +} +CODEX_RAW_TOKENS = { + "input_tokens": 400, + "cached_input_tokens": 300, + "output_tokens": 90, + "reasoning_output_tokens": 30, + "total_tokens": 490, +} +GROK_EVENT_TYPES = [ + "phase_changed", + "tool_started", + "tool_completed", + "permission_requested", + "permission_resolved", + "loop_started", + "phase_changed", +] +GROK_RECENT = [ + "phase_changed", + "session/update", + "search_code", + "search_code", + "permission_requested", + "session/update", + "permission_resolved", + "loop_started", + "phase_changed", +] + + +def load_module(testcase, path, name): + testcase.assertTrue( + path.is_file(), + f"{path} is missing; the contract requires this module to exist", + ) + spec = importlib.util.spec_from_file_location(name, path) + testcase.assertIsNotNone( + spec, + f"{path} could not be given an import specification", + ) + testcase.assertIsNotNone( + spec.loader, + f"{path} has no loader and cannot be exercised", + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def native_epoch(value): + if isinstance(value, (int, float)): + return float(value) + from datetime import datetime + + text = re.sub(r"\.(\d{6})\d+", r".\1", str(value).replace("Z", "+00:00")) + return datetime.fromisoformat(text).timestamp() + + +def jsonl(path): + return [ + json.loads(raw) + for raw in path.read_text().splitlines() + if raw.strip() + ] + + +def only_path(testcase, paths, location): + paths = list(paths) + testcase.assertEqual( + len(paths), + 1, + f"expected exactly one fixture artifact at {location}, found {paths}", + ) + return paths[0] + + +def snapshot(root): + return { + str(path.relative_to(root)): path.read_bytes() + for path in sorted(root.rglob("*")) + if path.is_file() + } + + +def nested_keys(value): + if isinstance(value, dict): + for key, child in value.items(): + yield str(key) + yield from nested_keys(child) + elif isinstance(value, list): + for child in value: + yield from nested_keys(child) + + +class StoreBuilderCase(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="stores-contract-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + + def stores(self): + return load_module(self, STORES, "minspec_fixture_stores") + + def build_all(self, root, base_timestamp=BASE_EPOCH): + stores = self.stores() + claude_root = root / "claude" + codex_root = root / "codex" + grok_root = root / "grok" + + stores.build_claude_store( + claude_root, + SLUG, + base_timestamp=base_timestamp, + cwd=REPO, + session_id=CLAUDE_SESSION, + model=CLAUDE_MODEL, + effort="high", + marker=MARKER, + ) + stores.build_codex_store( + codex_root, + base_timestamp=base_timestamp, + cwd=REPO, + session_id=CODEX_SESSION, + model=CODEX_MODEL, + effort="high", + marker=MARKER, + ) + stores.build_grok_store( + grok_root, + REPO, + base_timestamp=base_timestamp, + session_id=GROK_SESSION, + model=GROK_MODEL, + marker=MARKER, + ) + + claude_stream = claude_root / SLUG / f"{CLAUDE_SESSION}.jsonl" + codex_stream = only_path( + self, + codex_root.glob("sessions/*/*/*/rollout-*.jsonl"), + codex_root / "sessions", + ) + grok_session = ( + grok_root + / "sessions" + / quote(REPO, safe="") + / GROK_SESSION + ) + + return { + "claude_root": claude_root, + "claude_stream": claude_stream, + "codex_root": codex_root, + "codex_stream": codex_stream, + "grok_root": grok_root, + "grok_session": grok_session, + } + + +class ExplicitTimeAndDeterminism(StoreBuilderCase): + def test_every_builder_requires_and_obeys_two_base_timestamps(self): + stores = self.stores() + for name in ( + "build_claude_store", + "build_codex_store", + "build_grok_store", + ): + with self.subTest(builder=name): + builder = getattr(stores, name, None) + self.assertIsNotNone( + builder, + f"{STORES}:{name} is missing", + ) + parameter = inspect.signature(builder).parameters.get( + "base_timestamp" + ) + self.assertIsNotNone( + parameter, + ( + f"{STORES}:{name} has no explicit " + "base_timestamp parameter" + ), + ) + self.assertIs( + parameter.default, + inspect.Parameter.empty, + ( + f"{STORES}:{name} permits an implicit clock; " + "base_timestamp must be required" + ), + ) + + first = self.build_all( + self.root / "first", + base_timestamp=BASE_EPOCH, + ) + second = self.build_all( + self.root / "second", + base_timestamp=SECOND_BASE_EPOCH, + ) + + measured = [] + for label, artifacts, expected_iso, expected_grok in ( + ("first", first, BASE_ISO, GROK_BASE), + ( + "second", + second, + SECOND_BASE_ISO, + SECOND_GROK_BASE, + ), + ): + claude = jsonl(artifacts["claude_stream"]) + codex = jsonl(artifacts["codex_stream"]) + summary = json.loads( + (artifacts["grok_session"] / "summary.json").read_text() + ) + + self.assertGreater( + len(claude), + 0, + f"{artifacts['claude_stream']} contains no entries", + ) + self.assertGreater( + len(codex), + 0, + f"{artifacts['codex_stream']} contains no entries", + ) + self.assertEqual( + claude[0]["timestamp"], + expected_iso, + ( + f"{artifacts['claude_stream']} ignored the {label} " + "base_timestamp" + ), + ) + self.assertEqual( + codex[0]["timestamp"], + expected_iso, + ( + f"{artifacts['codex_stream']} ignored the {label} " + "base_timestamp" + ), + ) + self.assertEqual( + summary["created_at"], + expected_grok, + ( + f"{artifacts['grok_session'] / 'summary.json'} " + f"ignored the {label} base_timestamp" + ), + ) + measured.append( + ( + claude[0]["timestamp"], + codex[0]["timestamp"], + summary["created_at"], + ) + ) + + self.assertNotEqual( + measured[0], + measured[1], + ( + "the two distinct base_timestamp plants produced the " + "same fixture timestamps" + ), + ) + + def test_repeated_builds_with_identical_arguments_are_byte_identical(self): + self.build_all(self.root) + first = snapshot(self.root) + self.assertGreater( + len(first), + 0, + f"{self.root} remained empty after the first fixture build", + ) + + self.build_all(self.root) + second = snapshot(self.root) + self.assertEqual( + second, + first, + ( + f"rebuilding fixtures under {self.root} changed their " + "paths or bytes" + ), + ) + + +class MeasuredStoreShapes(StoreBuilderCase): + def test_claude_entries_match_the_measured_message_shape(self): + artifacts = self.build_all(self.root) + entries = jsonl(artifacts["claude_stream"]) + self.assertEqual( + len(entries), + 2, + f"{artifacts['claude_stream']} has the wrong message count", + ) + + entry_keys = { + "timestamp", + "cwd", + "gitBranch", + "effort", + "isSidechain", + "sessionId", + "message", + } + message_keys = {"id", "model", "usage", "content"} + usage_keys = { + "input_tokens", + "cache_creation_input_tokens", + "cache_read_input_tokens", + "output_tokens", + } + + for index, entry in enumerate(entries): + self.assertTrue( + entry_keys <= entry.keys(), + ( + f"{artifacts['claude_stream']} entry {index} lacks " + f"measured keys: {entry_keys - entry.keys()}" + ), + ) + self.assertEqual( + entry["cwd"], + REPO, + f"{artifacts['claude_stream']} entry {index} lost cwd", + ) + self.assertEqual( + entry["sessionId"], + CLAUDE_SESSION, + ( + f"{artifacts['claude_stream']} entry {index} " + "lost sessionId" + ), + ) + message = entry["message"] + self.assertTrue( + message_keys <= message.keys(), + ( + f"{artifacts['claude_stream']} message {index} lacks " + f"measured keys: {message_keys - message.keys()}" + ), + ) + self.assertEqual( + set(message["usage"]), + usage_keys, + ( + f"{artifacts['claude_stream']} message {index} " + "has the wrong usage currencies" + ), + ) + + blocks = [ + block + for entry in entries + for block in entry["message"]["content"] + if isinstance(block, dict) + ] + self.assertGreater( + len(blocks), + 0, + f"{artifacts['claude_stream']} contains no content blocks", + ) + tool_uses = [ + block for block in blocks if block.get("type") == "tool_use" + ] + tool_results = [ + block for block in blocks if block.get("type") == "tool_result" + ] + self.assertGreater( + len(tool_uses), + 0, + f"{artifacts['claude_stream']} contains no tool_use block", + ) + self.assertGreater( + len(tool_results), + 0, + f"{artifacts['claude_stream']} contains no tool_result block", + ) + self.assertTrue( + all({"name", "input"} <= block.keys() for block in tool_uses), + ( + f"{artifacts['claude_stream']} has a tool_use block " + "without name and input" + ), + ) + self.assertTrue( + all("is_error" in block for block in tool_results), + ( + f"{artifacts['claude_stream']} has a tool_result block " + "without is_error" + ), + ) + + def test_codex_store_splits_meta_context_and_cumulative_counts(self): + artifacts = self.build_all(self.root) + entries = jsonl(artifacts["codex_stream"]) + self.assertEqual( + len(entries), + 5, + f"{artifacts['codex_stream']} has the wrong event count", + ) + + meta = [ + entry + for entry in entries + if entry.get("type") == "session_meta" + ] + context = [ + entry + for entry in entries + if entry.get("type") == "turn_context" + ] + self.assertEqual( + len(meta), + 1, + f"{artifacts['codex_stream']} must contain one session_meta", + ) + self.assertEqual( + len(context), + 1, + f"{artifacts['codex_stream']} must contain one turn_context", + ) + + meta_payload = meta[0]["payload"] + self.assertTrue( + {"id", "cwd", "base_instructions"} <= meta_payload.keys(), + ( + f"{artifacts['codex_stream']} session_meta lacks id, cwd, " + "or base_instructions" + ), + ) + self.assertNotIn( + "model", + meta_payload, + ( + f"{artifacts['codex_stream']} put model in session_meta " + "instead of turn_context" + ), + ) + self.assertNotIn( + "effort", + meta_payload, + ( + f"{artifacts['codex_stream']} put effort in session_meta " + "instead of turn_context" + ), + ) + self.assertEqual( + meta_payload["id"], + CODEX_SESSION, + f"{artifacts['codex_stream']} session_meta has the wrong id", + ) + self.assertEqual( + meta_payload["cwd"], + REPO, + f"{artifacts['codex_stream']} session_meta has the wrong cwd", + ) + base_instructions = meta_payload["base_instructions"] + self.assertIsInstance( + base_instructions, + dict, + ( + f"{artifacts['codex_stream']} base_instructions is not " + "the measured object shape" + ), + ) + self.assertEqual( + set(base_instructions), + {"text"}, + ( + f"{artifacts['codex_stream']} base_instructions has " + "unexpected fixture fields" + ), + ) + self.assertIn( + MARKER, + base_instructions["text"], + ( + f"{artifacts['codex_stream']} did not plant content in " + "base_instructions.text" + ), + ) + + context_payload = context[0]["payload"] + self.assertTrue( + {"model", "effort", "cwd"} <= context_payload.keys(), + ( + f"{artifacts['codex_stream']} turn_context lacks model, " + "effort, or cwd" + ), + ) + self.assertNotIn( + "id", + context_payload, + ( + f"{artifacts['codex_stream']} put the session id in " + "turn_context instead of session_meta" + ), + ) + self.assertEqual( + context_payload["model"], + CODEX_MODEL, + f"{artifacts['codex_stream']} turn_context has the wrong model", + ) + self.assertEqual( + context_payload["effort"], + "high", + f"{artifacts['codex_stream']} turn_context has the wrong effort", + ) + self.assertEqual( + context_payload["cwd"], + REPO, + f"{artifacts['codex_stream']} turn_context has the wrong cwd", + ) + + counts = [ + entry["payload"]["info"]["total_token_usage"] + for entry in entries + if (entry.get("payload") or {}).get("type") == "token_count" + ] + self.assertEqual( + len(counts), + 2, + ( + f"{artifacts['codex_stream']} must contain two cumulative " + "token_count events" + ), + ) + for index, count in enumerate(counts): + self.assertEqual( + set(count), + set(CODEX_RAW_TOKENS), + ( + f"{artifacts['codex_stream']} token_count {index} " + "has the wrong currencies" + ), + ) + self.assertEqual( + counts[-1], + CODEX_RAW_TOKENS, + ( + f"{artifacts['codex_stream']} last cumulative token_count " + "is not the expected spend" + ), + ) + self.assertLess( + counts[0]["total_tokens"], + counts[-1]["total_tokens"], + ( + f"{artifacts['codex_stream']} does not demonstrate " + "cumulative growth" + ), + ) + + def test_grok_store_matches_measured_content_and_activity_shapes(self): + artifacts = self.build_all(self.root) + session = artifacts["grok_session"] + summary_path = session / "summary.json" + updates_path = session / "updates.jsonl" + events_path = session / "events.jsonl" + + summary = json.loads(summary_path.read_text()) + self.assertTrue( + { + "created_at", + "updated_at", + "num_messages", + "current_model_id", + "session_summary", + "generated_title", + "reasoning_effort", + } + <= summary.keys(), + f"{summary_path} lacks the measured Grok summary keys", + ) + self.assertEqual( + summary["info"]["id"], + GROK_SESSION, + f"{summary_path} has the wrong session id", + ) + self.assertEqual( + summary["info"]["cwd"], + REPO, + f"{summary_path} has the wrong cwd", + ) + self.assertEqual( + summary["reasoning_effort"], + "high", + f"{summary_path} has the wrong reasoning_effort", + ) + for field in ("session_summary", "generated_title"): + self.assertIn( + MARKER, + summary[field], + f"{summary_path} did not plant content in {field}", + ) + + token_keys = [ + key for key in nested_keys(summary) if "token" in key.lower() + ] + self.assertEqual( + token_keys, + [], + ( + f"{summary_path} invented token usage keys even though " + f"Grok records none: {token_keys}" + ), + ) + + updates = jsonl(updates_path) + events = jsonl(events_path) + self.assertEqual( + len(updates), + 2, + f"{updates_path} does not demonstrate activity growth", + ) + self.assertEqual( + len(events), + len(GROK_EVENT_TYPES), + f"{events_path} has the wrong measured event set", + ) + + for index, update in enumerate(updates): + self.assertEqual( + set(update), + {"method", "params", "timestamp"}, + ( + f"{updates_path} update {index} does not have exactly " + "method, params, and timestamp" + ), + ) + self.assertEqual( + update["method"], + "session/update", + f"{updates_path} update {index} has the wrong method", + ) + self.assertIsInstance( + update["params"], + dict, + f"{updates_path} update {index} params is not an object", + ) + self.assertIn( + MARKER, + json.dumps(update["params"], sort_keys=True), + f"{updates_path} update {index} lacks the params marker", + ) + self.assertNotIn( + "name", + update, + f"{updates_path} update {index} invented a name field", + ) + self.assertIs( + type(update["timestamp"]), + int, + ( + f"{updates_path} update {index} timestamp is not the" + " measured epoch-integer type (live: 28448/28448 int)" + ), + ) + + self.assertLess( + updates[0]["timestamp"], + updates[-1]["timestamp"], + f"{updates_path} activity timestamps do not advance", + ) + + event_types = [event["type"] for event in events] + self.assertEqual( + event_types, + GROK_EVENT_TYPES, + f"{events_path} has the wrong measured event types", + ) + self.assertGreater( + event_types.count("phase_changed"), + max( + event_types.count(event_type) + for event_type in set(event_types) + if event_type != "phase_changed" + ), + f"{events_path} does not make phase_changed dominant", + ) + + for index, event in enumerate(events): + expected_keys = {"type", "ts"} + if event["type"] in {"tool_started", "tool_completed"}: + expected_keys.add("tool_name") + self.assertEqual( + event["tool_name"], + "search_code", + ( + f"{events_path} tool event {index} has the wrong " + "tool_name" + ), + ) + self.assertEqual( + set(event), + expected_keys, + ( + f"{events_path} event {index} has fields outside the " + "measured shape" + ), + ) + self.assertNotIn( + "name", + event, + f"{events_path} event {index} invented a name field", + ) + self.assertIs( + type(event["ts"]), + str, + ( + f"{events_path} event {index} ts is not the measured" + " ISO-string type (live: 3836/3836 str)" + ), + ) + + for first, second in itertools.pairwise(events): + self.assertLess( + first["ts"], + second["ts"], + f"{events_path} activity timestamps do not advance", + ) + + activity = [ + (update["timestamp"], update["method"]) + for update in updates + ] + activity.extend( + ( + event["ts"], + event.get("tool_name", event["type"]), + ) + for event in events + ) + self.assertEqual( + [name for _, name in sorted( + activity, key=lambda pair: native_epoch(pair[0]))], + GROK_RECENT, + ( + f"{updates_path} and {events_path} do not plant the " + "required native-timestamp merge order" + ), + ) + + +class ExistingReadersRoundTripTheFixtures(StoreBuilderCase): + def test_usage_readers_return_exact_sessions_counts_and_grok_gap(self): + artifacts = self.build_all(self.root) + usage = load_module(self, USAGE, "minspec_usage_for_fixture_test") + + claude = list( + usage.claude_sessions(artifacts["claude_root"], REPO) + ) + codex = list(usage.codex_sessions(artifacts["codex_root"], REPO)) + grok = list(usage.grok_sessions(artifacts["grok_root"], REPO)) + + self.assertEqual( + claude, + [ + { + "harness": "claude", + "session": CLAUDE_SESSION, + "model": CLAUDE_MODEL, + "started": BASE_ISO, + "ended": CLAUDE_END, + "messages": 2, + "tokens": CLAUDE_TOKENS, + } + ], + ( + f"{USAGE}:claude_sessions did not round-trip the " + "built Claude store" + ), + ) + self.assertEqual( + codex, + [ + { + "harness": "codex", + "session": CODEX_SESSION, + "model": CODEX_MODEL, + "started": BASE_ISO, + "ended": CODEX_END, + "messages": 5, + "tokens": CODEX_TOKENS, + } + ], + ( + f"{USAGE}:codex_sessions did not merge session_meta with " + "turn_context or use the last cumulative token_count" + ), + ) + self.assertEqual( + grok, + [ + { + "harness": "grok", + "session": GROK_SESSION, + "model": GROK_MODEL, + "started": GROK_BASE, + "ended": GROK_END, + "messages": 3, + "tokens": None, + "incomplete": False, + "reasoning": None, + "cost_usd_ticks": None, + "note": usage.GROK_GAP, + } + ], + ( + f"{USAGE}:grok_sessions did not preserve the " + "unrecorded-token gap" + ), + ) + + def test_planted_content_is_raw_but_never_in_reader_aggregates(self): + artifacts = self.build_all(self.root) + codex_entries = jsonl(artifacts["codex_stream"]) + codex_meta = [ + entry["payload"] + for entry in codex_entries + if entry.get("type") == "session_meta" + ] + self.assertEqual( + len(codex_meta), + 1, + ( + f"{artifacts['codex_stream']} lacks the planted " + "session_meta" + ), + ) + + grok_summary = json.loads( + (artifacts["grok_session"] / "summary.json").read_text() + ) + grok_updates = jsonl( + artifacts["grok_session"] / "updates.jsonl" + ) + + planted_channels = { + "Claude content": artifacts["claude_stream"].read_text(), + "Codex base_instructions": json.dumps( + codex_meta[0]["base_instructions"], + sort_keys=True, + ), + "Grok session_summary": grok_summary["session_summary"], + "Grok generated_title": grok_summary["generated_title"], + } + for index, update in enumerate(grok_updates): + planted_channels[f"Grok params {index}"] = json.dumps( + update["params"], + sort_keys=True, + ) + + for channel, raw in planted_channels.items(): + self.assertIn( + MARKER, + raw, + f"the leak marker was not planted in {channel}", + ) + + usage = load_module(self, USAGE, "minspec_usage_for_leak_test") + args = argparse.Namespace( + claude_dir=str(artifacts["claude_root"]), + codex_dir=str(artifacts["codex_root"]), + grok_dir=str(artifacts["grok_root"]), + repo=REPO, + ) + aggregate = json.dumps(usage.collect(args), sort_keys=True) + self.assertNotIn( + MARKER, + aggregate, + f"{USAGE}:collect leaked raw session content", + ) + + +class ReadersRefuseSnapshotRewriteInflation(StoreBuilderCase): + """Live Claude streams re-emit message ids (measured 2026-08-21: + 2086 usage lines over 1052 unique ids in one session). A reader + that sums lines nearly doubles the spend; one that keeps the first + copy misses growth. Only last-wins-by-id survives both plants.""" + + GROWTH: typing.ClassVar[dict] = {"input_tokens": 7, "cache_creation_input_tokens": 11, + "cache_read_input_tokens": 13, "output_tokens": 17} + + def reemitted_stream(self, grown): + stores = self.stores() + stores.build_claude_store( + self.root / "claude", SLUG, base_timestamp=BASE_EPOCH, + cwd=REPO, session_id=CLAUDE_SESSION, model=CLAUDE_MODEL, + effort="high", marker=MARKER, reemit_last=True) + stream = self.root / "claude" / SLUG / f"{CLAUDE_SESSION}.jsonl" + entries = jsonl(stream) + ids = [e["message"]["id"] for e in entries if e["message"].get("usage")] + self.assertGreater(len(ids), len(set(ids)), + f"the duplicate-id plant did not land in {stream}") + if grown: + last = entries[-1] + self.assertEqual(last, entries[-2], + f"the re-emit in {stream} is not byte-equivalent" + " before the growth plant") + for key, bump in self.GROWTH.items(): + last["message"]["usage"][key] += bump + stream.write_text("".join(json.dumps(e) + "\n" for e in entries)) + planted = jsonl(stream) + self.assertNotEqual(planted[-1], planted[-2], + f"the growth plant did not land in {stream}") + return stream + + def usage_tokens(self): + usage = load_module(self, USAGE, "minspec_usage_reemit") + rows = list(usage.claude_sessions(self.root / "claude", REPO)) + self.assertEqual(len(rows), 1) + return rows[0] + + def test_identical_reemit_is_counted_once_by_usage(self): + self.reemitted_stream(grown=False) + row = self.usage_tokens() + self.assertEqual(row["tokens"], CLAUDE_TOKENS, + "usage summed a re-emitted message id") + self.assertEqual(row["messages"], 2, + "usage counted the duplicate as a third message") + + def test_grown_reemit_is_counted_last_wins_by_usage(self): + self.reemitted_stream(grown=True) + grown = { + "input": CLAUDE_TOKENS["input"] + self.GROWTH["input_tokens"], + "cached": (CLAUDE_TOKENS["cached"] + + self.GROWTH["cache_creation_input_tokens"] + + self.GROWTH["cache_read_input_tokens"]), + "output": CLAUDE_TOKENS["output"] + self.GROWTH["output_tokens"], + } + grown["total"] = sum(grown.values()) + self.assertEqual(self.usage_tokens()["tokens"], grown, + "usage kept the first copy of a grown re-emit") + + def test_colliding_slugs_are_separated_by_entry_cwd(self): + stores = self.stores() + actual, colliding = "/a/b-c", "/a-b/c" + collided_slug = "-a-b-c" + stores.build_claude_store( + self.root / "claude", collided_slug, base_timestamp=BASE_EPOCH, + cwd=actual, session_id=CLAUDE_SESSION, model=CLAUDE_MODEL, + effort="high", marker=MARKER) + usage = load_module(self, USAGE, "minspec_usage_collision") + hit = list(usage.claude_sessions(self.root / "claude", actual)) + miss = list(usage.claude_sessions(self.root / "claude", colliding)) + self.assertEqual([len(hit), len(miss)], [1, 0], + "usage --repo trusted the lossy slug over the" + " cwd stored in the entries") + + +class BreakerConsumesTheSharedClaudeFixture(StoreBuilderCase): + def test_once_trips_above_a_low_cap_and_stays_clean_above_spend(self): + artifacts = self.build_all(self.root) + entries = jsonl(artifacts["claude_stream"]) + spend = 0 + for entry in entries: + usage = entry["message"]["usage"] + spend += sum(usage.values()) + + low_cap = spend - 1 + self.assertGreater( + spend, + low_cap, + ( + f"the over-cap spend was not planted in " + f"{artifacts['claude_stream']}" + ), + ) + self.assertEqual( + spend, + CLAUDE_TOKENS["total"], + ( + f"{artifacts['claude_stream']} contains an unexpected " + "planted spend" + ), + ) + + tripped = subprocess.run( + [ + sys.executable, + str(BREAKER), + str(artifacts["claude_stream"]), + "--once", + "--cap", + str(low_cap), + ], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual( + tripped.returncode, + 3, + ( + f"{BREAKER} did not trip on the planted Claude spend: " + f"{tripped.stderr}" + ), + ) + + clean = subprocess.run( + [ + sys.executable, + str(BREAKER), + str(artifacts["claude_stream"]), + "--once", + "--cap", + str(spend + 1), + ], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual( + clean.returncode, + 0, + ( + f"{BREAKER} tripped even though the Claude spend was " + f"below cap: {clean.stderr}" + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/fixtures/tests/test_stores_findings.py b/ops/devlane/fixtures/tests/test_stores_findings.py new file mode 100644 index 0000000..118dc19 --- /dev/null +++ b/ops/devlane/fixtures/tests/test_stores_findings.py @@ -0,0 +1,109 @@ +"""Regression contract for the Codex fixture shape from PR #24.""" + +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve() +STORES = HERE.parents[1] / "stores.py" +BASE_EPOCH = 1_787_306_400 +REPO = "/home/work/projects/minspec/workbench" +SESSION = "77777777-7777-4777-8777-777777777777" +MODEL = "gpt-5-codex" + + +def load_module(testcase, path, name): + testcase.assertTrue(path.is_file(), f"required module is missing: {path}") + spec = importlib.util.spec_from_file_location(name, path) + testcase.assertIsNotNone(spec, f"could not create an import spec for {path}") + testcase.assertIsNotNone(spec.loader, f"could not load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class StoreFindings(unittest.TestCase): + def test_b7_codex_fixture_splits_session_meta_from_later_turn_context(self): + with tempfile.TemporaryDirectory(prefix="stores-findings-") as temp: + root = Path(temp) / "codex" + stores = load_module(self, STORES, "minspec_stores_findings") + stores.build_codex_store( + root, + base_timestamp=BASE_EPOCH, + cwd=REPO, + session_id=SESSION, + model=MODEL, + effort="high", + marker="FIXTUREPROMPTMARKER", + ) + rollouts = list(root.glob("sessions/*/*/*/rollout-*.jsonl")) + self.assertEqual( + len(rollouts), + 1, + f"the Codex builder emitted {len(rollouts)} rollout files", + ) + entries = [ + json.loads(line) + for line in rollouts[0].read_text().splitlines() + if line.strip() + ] + + meta_indexes = [ + index + for index, entry in enumerate(entries) + if entry.get("type") == "session_meta" + ] + context_indexes = [ + index + for index, entry in enumerate(entries) + if entry.get("type") == "turn_context" + ] + self.assertEqual( + meta_indexes, + [0], + "the fixture must begin with one session_meta", + ) + self.assertEqual( + len(context_indexes), + 1, + "the fixture must contain one separate turn_context", + ) + self.assertGreater( + context_indexes[0], + meta_indexes[0], + "turn_context must occur after session_meta", + ) + + meta = entries[meta_indexes[0]]["payload"] + context = entries[context_indexes[0]]["payload"] + self.assertLess( + entries[meta_indexes[0]]["timestamp"], + entries[context_indexes[0]]["timestamp"], + "turn_context must carry a timestamp later than session_meta", + ) + self.assertEqual( + (meta.get("id"), meta.get("cwd")), + (SESSION, REPO), + "session_meta must carry the fixture session id and cwd", + ) + self.assertEqual( + (context.get("cwd"), context.get("model")), + (REPO, MODEL), + "turn_context must carry the fixture cwd and model", + ) + self.assertNotIn( + "id", + context, + "turn_context must not synthesize a duplicate session id", + ) + self.assertNotIn( + "model", + meta, + "session_meta must not synthesize the later model fact", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/harness/context.cue b/ops/devlane/harness/context.cue new file mode 100644 index 0000000..dcdf74d --- /dev/null +++ b/ops/devlane/harness/context.cue @@ -0,0 +1,437 @@ +// Everything a dispatched role is permitted to see, as one closed shape. +// +// Three mechanisms already guard a role's context, and each works: +// isolation.py decides what the harness may bring from the operator's +// machine, stage.py decides what the snapshot holds, and the +// instruction files in force decide which rules apply. Nothing checked +// that all three RAN. +// +// On 2026-08-23 that gap opened. The launcher's isolation step is +// `eval "$(isolation.py --sh ...)"`, and eval reports the status of the +// text it evaluated -- so a module that was absent produced no output, +// became `eval ""`, and succeeded. An agent went out carrying the +// operator's entire configuration and no error was raised anywhere. The +// guard was fine. The absence of the guard was invisible. +// +// A procedure that must be remembered will eventually not be. A closed +// definition cannot be forgotten, because the missing field is a type +// error rather than a skipped step. That is the whole reason this file +// is CUE and not another checker: +// +// extra context -> a field not in the definition -> does not evaluate +// missing context -> a required field absent -> does not evaluate +// +// #Context is the SHAPE of an observed dispatch. #Dispatchable is the +// LAW -- the subset of shapes that may actually be launched. They are +// separate on purpose: a context that describes a leak is a valid +// observation and an invalid dispatch, and conflating the two would +// leave no way to write the leak down. + +package harness + +// ---------------------------------------------------------------- ids + +#HarnessName: "claude" | "codex" | "grok" + +// Hex digest, lower case. Long enough to be an identity, not a hint. +#Digest: =~"^[0-9a-f]{64}$" + +#NonEmpty: string & !="" + +// Absence has to be SAID. An omitted list and an empty list mean +// entirely different things -- "nobody looked" and "we looked and found +// none" -- and a schema that accepts a missing field silently turns the +// first into the second. Anywhere a collection may legitimately be +// empty, it is spelled as this instead, carrying the reason. +#DeclaredAbsent: close({ + declared_absent: #NonEmpty +}) + +// ------------------------------------------------------------ harness + +// What the harness itself brought, and how that was established. +// +// `applied` is what the launcher DID. `observed` is what was then found +// to be true. They are separate fields because a context must not be +// able to assert its own cleanliness: applying the right flags is an +// intention, and only the observation is evidence. The pair is what +// makes a false claim expensive to write down. +// +#Isolation: close({ + mechanism: "flags" | "home" + + // The argv fragment and environment overrides actually used. + flags: [...#NonEmpty] | #DeclaredAbsent + env: {[#NonEmpty]: #NonEmpty} | #DeclaredAbsent + + // For a home-mechanism harness, the minimal home and the complete + // list of what was linked into it. Credentials are the one thing + // that crosses, deliberately; enumerating them here is what makes + // "and nothing else" checkable. Optional HERE, and required by + // #Dispatchable -- which is where every other refusal lives, and + // which keeps this struct a description of what was done rather + // than a judgement about it. + home?: #NonEmpty + auth_files?: [...#NonEmpty] + + observed: #IsolationObserved +}) + +#IsolationObserved: close({ + // True means the operator's own configuration reached the agent. + // It is not a warning: see #Dispatchable. + operator_config_present: bool + + // HOW that was determined. A bare boolean is a claim; this is the + // method behind it, so a reader can tell a probe from a guess. + // "unisolated arm answered YES, isolated arm answered NO" is + // evidence. "isolated" is not. + evidence: #NonEmpty + + // When the check was last actually run, and against which harness + // build. Every isolation fact is true of one version on one day. + checked_at: #NonEmpty + harness_version: #NonEmpty +}) + +#Harness: close({ + name: #HarnessName + version: #NonEmpty + isolation: #Isolation +}) + +// ------------------------------------------------------------- staged + +// What the snapshot holds, and the proof that it holds only that. +#Staged: close({ + root: #NonEmpty + + // Every file present, relative to root, and its digest. The list is + // the observation; `count` is not derived from it here because CUE + // would then be checking arithmetic rather than agreement -- the + // extractor states both and a mismatch is a finding. + files: [...#StagedFile] + count: int & >=0 + + // The manifest that produced it, carried so a reader can re-derive + // the staging rather than trust it. + given: [...#NonEmpty] + withheld: [...#NonEmpty] | #DeclaredAbsent + + // Build noise dropped on the way in. Reported rather than silently + // omitted: a rule that quietly removes files is the failure this + // whole file exists to prevent, committed inside the fix. + noise_dropped: [...#NonEmpty] | #DeclaredAbsent + + proof: #StagingProof +}) + +#StagedFile: close({ + path: #NonEmpty + sha256: #Digest + bytes: int & >=0 +}) + +#StagingProof: close({ + tool: #NonEmpty + + // Files present that a withheld pattern matches. Must be empty to + // dispatch; recorded rather than asserted so a breach is + // describable. + withheld_present: [...#NonEmpty] + + // Given patterns that matched nothing. An empty snapshot satisfies + // every withholding rule perfectly, so this is the half that stops + // "nothing leaked" being read as "the firewall worked". + given_unmet: [...#NonEmpty] +}) + +// ----------------------------------------------------------- doctrine + +// The instruction files in force, in inheritance order from the +// repository root down to the working directory. +// +// Session-start load is assembled from the working directory UPWARD. +// Measured 2026-08-26 on this machine (Claude Code 2.1.246, Codex +// 0.148.0, Grok 1.0.5): Codex embeds root AGENTS.md and not CLAUDE.md; +// Grok loads root CLAUDE.md and AGENTS.md; Claude auto-loads CLAUDE.md +// and not AGENTS.md. Claude 2.1.246 also attaches a subtree CLAUDE.md +// on first read into that subtree, so a doctrine file below the +// snapshot root CAN reach a root-cwd agent -- lazily, and only +// CLAUDE.md. Recording the chain that was actually in force -- rather +// than the files that were present -- is the difference between a rule +// being available and a rule applying. +#Doctrine: #DoctrineChain | #DeclaredAbsent + +#DoctrineChain: close({ + // Non-empty by construction. A dispatch with no doctrine at all is + // legitimate, but it has to say so via #DeclaredAbsent rather than + // by presenting an empty list that looks like a chain. + files: [#DoctrineFile, ...#DoctrineFile] +}) + +#DoctrineFile: close({ + path: #NonEmpty + sha256: #Digest + + // Whether the agent actually received it, and how that was known. + // Presence on disk is not receipt. + in_force: bool + evidence: #NonEmpty +}) + +// -------------------------------------------------------------- brief + +// The instructions a role was given. +// +// This is the field the first version of #Context did not have, and its +// absence is where both of the day's real defects came from. Staged +// files are proved. Harness configuration is probed. The BRIEF was prose +// typed fresh each time, validated by nothing -- and it is the largest +// surface of the three. +// +// Twice in one session, two authors of the same seam were handed +// different interfaces. Five extractors take `--root DIR --out PATH`; +// seven take a positional root and print to stdout, because their brief +// said "importable modules" and never said how they would be called. +// vet.py looks for a fault sidecar at `.md`; the planter wrote +// `.meta.json`, because the brief I gave it said to. In both +// cases the shared specification fixed the shape of the DATA and never +// the shape of the INTERFACE, so the authors met at the shape and missed +// at the door. +// +// An interface written twice in prose is an interface that will differ. +// Written once here and referenced by both roles, it cannot. +#Brief: close({ + role: #NonEmpty + + // What the role must produce. Unified with `task.produces` by + // #Dispatchable, so a deliverable the task does not expect — or an + // expectation with no deliverable — is refused rather than noted. + // This comment used to promise a check against a manifest field + // named `returns`; no such field exists on anything here, and a + // comment describing a guard that was never written is worse than + // silence, because it stops the next reader looking (Codex, PR #40). + deliverables: [#Deliverable, ...#Deliverable] + + // The shape of the report. Retyping this per dispatch is how a + // field quietly goes missing, and a missing field in a report is + // indistinguishable from an honest "nothing to say". + report_fields: [#NonEmpty, ...#NonEmpty] + + // Constraints stated to the role. Present so they can be compared + // across roles, and so a rule can be shown to have been given. + rules: [...#NonEmpty] | #DeclaredAbsent + + // The exact bytes handed over. A brief that cannot be identified + // cannot be shown to be the one that produced a result. + sha256: #Digest + bytes: int & >0 +}) + +#Deliverable: close({ + path: #NonEmpty + + // The interface this deliverable must satisfy, when more than one + // role touches it. Null ONLY when nothing else consumes it -- and + // that is a claim, so it is spelled rather than left absent. + interface: #Interface | #DeclaredAbsent +}) + +// One seam, declared once, referenced by every role that meets there. +#Interface: close({ + name: #NonEmpty + + // How it is invoked, if it is invoked. + argv?: [...#NonEmpty] + + // Where its output goes. "stdout" and "file" are not + // interchangeable and assuming either is what broke the extractors. + output?: "stdout" | "file" | "both" + + // Fields a consumer will read. A producer that omits one and a + // consumer that requires it is the fault-sidecar defect exactly. + fields?: [...#NonEmpty] + + // Where this seam is written down, so a disagreement has an arbiter + // that is not one of the two authors. + declared_at: #NonEmpty +}) + +// --------------------------------------------------------------- task + +// What a role doing this task must be given, declared once for the task +// rather than re-decided at every dispatch. +// +// Closure answers only half the question. A closed #Context guarantees +// nothing EXTRA reached the agent; it says nothing about whether what +// did reach it was ENOUGH. Those are different failures and only one of +// them was guarded. +// +// The unguarded one happened today. An extractor author was given +// sections 4 to 6 of a plan as its specification, and section 6 refers +// to sections 9 and 10, which were not in the excerpt. The role reported +// it -- "sections 9 and 10 are not in the SPEC excerpt, yet section 6 +// steps 3-5 depend on them" -- and emitted explicit unresolved markers +// rather than inventing the missing rules. That was the AGENT catching +// it. Nothing in the machinery would have. +// +// A hand-cut slice of a document is not a projection: a projection is +// closed under its own references, and a slice is whatever somebody's +// sed range happened to cover. +#Task: close({ + name: #NonEmpty + + // Everything a role needs. Stated on the TASK, so two dispatches of + // the same task cannot disagree about what it takes to do it. + requires: [#Requirement, ...#Requirement] + + produces: [#Deliverable, ...#Deliverable] + report_fields: [#NonEmpty, ...#NonEmpty] +}) + +#Requirement: close({ + // In words, so a human can tell whether the glob below is right. + what: #NonEmpty + + // The glob that must match something staged. Same mechanism as the + // manifest's `given_unmet`, for the same reason: a requirement that + // matches nothing is under-supply, and under-supply is invisible + // until the role guesses. + satisfied_by: #NonEmpty + + // What the role cannot do without it. Present because a requirement + // nobody can justify is one that will be dropped by whoever next + // tries to make a snapshot smaller. + why: #NonEmpty +}) + +// ------------------------------------------------------------ context + +#Context: close({ + role: #NonEmpty + harness: #Harness + staged: #Staged + doctrine: #Doctrine + + // What the role was actually told. Without it a context describes + // the room and not the instructions given inside it. + brief: #Brief + + // The task being performed, and what checking it against the staged + // set found. Both lists must be empty to dispatch; they are recorded + // rather than asserted so under-supply is describable. + task: #Task + + // Requirements whose glob matched nothing that was staged. + unmet_requirements: [...#NonEmpty] + + // Documents that were given but that refer to material which was + // not -- section cross-references, cited files, named appendices. + // A slice is not closed under its own references; a projection is. + dangling_references: [...#NonEmpty] + + // The law this context was projected from. Without it a context is + // unfalsifiable: it cannot be shown stale, because there is nothing + // to compare against. With it, "is this brief current" is one digest + // comparison rather than a reading. + derived_from: #Digest + + // Free-text notes are deliberately NOT permitted. The struct is + // closed, so a field nobody agreed to cannot be added to smuggle + // context past the definition -- which is the entire point. +}) + +// ------------------------------------------------------------ the law + +// A context that may actually be dispatched. +// +// Everything below is a refusal, and each one is an incident: +// +// operator_config_present a personal instruction file reached the +// system prompt of every dispatched agent +// withheld_present a firewall that was intended, not proved +// given_unmet / count > 0 an empty snapshot that "leaked nothing" +#Dispatchable: #Context & { + harness: isolation: observed: operator_config_present: false + staged: proof: withheld_present: [] + staged: proof: given_unmet: [] + staged: count: >0 + + // `count` is stated by the extractor rather than derived, so that a + // disagreement between it and the list is itself a finding. That + // makes it the wrong thing to gate on alone: `files: []` with + // `count: 1` passed, and an empty snapshot satisfies every + // withholding rule perfectly. The GATE reads the list (Codex, #40). + staged: files: [_, ...] + + // ...and the stated count must be the list's length. #Staged keeps + // the two as separate observations on purpose, so that a snapshot + // tool disagreeing with itself is describable — but a context is not + // DISPATCHABLE on a false measurement, and `count: 999` beside one + // file passed every check here (Codex, PR #40 round three). + staged: count: len(staged.files) + + // An isolation observation is true of one harness build on one day. + // A context could name version 3 and carry a clean observation taken + // against version 1 -- and a release is exactly when a new discovery + // path appears (Codex, PR #40). + harness: isolation: observed: harness_version: harness.version + + // The brief and the task state the same contract twice, and until + // now nothing made the two copies agree — so a context could hand + // one author an interface the other author's task declared absent, + // which is precisely the disagreement the independence method is + // built to make impossible. The positive control encoded exactly + // that and was still dispatchable (Codex, PR #40). + task: #Task + task: produces: brief.deliverables + task: report_fields: brief.report_fields + + // The flags half of the rule below. For a flags-mechanism harness + // the argv IS the isolation, so declaring it absent is the same + // missing-evidence-in-a-complete-record shape as a home-mechanism + // context with no home. + if harness.isolation.mechanism == "flags" { + harness: isolation: flags: [_, ...] + } + + // A home-mechanism harness must name the home and the complete + // credential list. Optional let a codex or grok context omit both + // and still satisfy #Dispatchable -- missing isolation evidence + // wearing the shape of a complete record (Codex, PR #40). An empty + // least one credential is named, because that list is the evidence. + if harness.isolation.mechanism == "home" { + harness: isolation: home: #NonEmpty + + // At least ONE, and this is not pedantry: `[...#NonEmpty]` is + // satisfied by an ABSENT field, because CUE infers the empty list + // and the result is concrete — so the omitting document validated + // (Grok, second read of PR #40). Same trap as unification being a + // default rather than only a comparison. A home-mechanism harness + // that links no credential cannot authenticate at all; build_home + // refuses to construct one, so a context claiming otherwise + // describes a dispatch that could not have happened. + harness: isolation: auth_files: [_, ...] + } + + // The outer role and the role the brief addresses are the same one. + // Otherwise a launcher can send a reviewer's instructions to an + // extractor and the schema will authorise it (Codex, PR #40). + // Written this way round on purpose: inside `brief: {role: ...}` a + // bare `role` resolves to brief's OWN field and constrains nothing. + // Measured 2026-08-23 -- the mismatching document was still accepted + // until the reference ran the other direction. + // `brief` is named here as well as referenced: a struct literal's + // scope holds only the fields IT declares, not the ones unification + // brings in from #Context, so without this line the reference below + // is "not found" rather than a constraint. + brief: #Brief + role: brief.role + + // Sufficiency. Closure above stops what should not be there; these + // stop what should be and is not. A context can be perfectly clean + // and still leave a role guessing. + unmet_requirements: [] + dangling_references: [] +} diff --git a/ops/devlane/harness/controls/build.py b/ops/devlane/harness/controls/build.py new file mode 100644 index 0000000..2eb68aa --- /dev/null +++ b/ops/devlane/harness/controls/build.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +"""Generate the controls for context.cue, so none of them can be wrong by hand. + + python3 ops/devlane/harness/controls/build.py [--check] + +The positive control was written by hand and encoded five separate +untruths about the very module it describes: flags the launcher does not +pass, an environment variable a flags-mechanism harness never sets, a +version the module records as unrecorded, doctrine evidence citing one +harness's tool on another, and an interface declared in a document that +was not staged. It was also illegal under the law it was supposed to +demonstrate — its brief and its task described the same deliverable's +seam differently (Codex, PR #40). + +So the facts about a harness come from `isolation.py` rather than from +memory, and every rejecting control is derived from a passing one by +changing EXACTLY ONE PATH. That second rule is the one that was missing: +a fixture that breaks two things at once still fails when only one of +the two guards exists, so it cannot witness either — measured on the +old `home-without-home`, which omitted both the home and the credential +list (Grok, second read of PR #40). + +`--check` regenerates into memory and compares, so a control edited by +hand is a failure rather than a surprise. +""" + +from __future__ import annotations + +import argparse +import copy +import importlib.util +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +HARNESS = HERE.parent + + +def _isolation(): + spec = importlib.util.spec_from_file_location( + "harness_isolation", HARNESS / "isolation.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +#: A digest is an identity, not a hint; these are fixture identities and +#: say so, rather than borrowing a real file's digest and going stale. +def _digest(seed): + import hashlib + return hashlib.sha256(f"control-fixture:{seed}".encode()).hexdigest() + + +def _evidence(measured): + """The module's own record of how isolation was established.""" + if "probe_default" in measured and "probe_isolated" in measured: + return (f"unisolated arm answered {measured['probe_default']} to the " + f"probe phrase, isolated arm answered " + f"{measured['probe_isolated']}") + leak = measured.get("leak") + if leak: + return f"unisolated dispatch loaded: {leak}" + raise SystemExit("a harness with no recorded measurement cannot be a " + "control: there is nothing to quote") + + +def _doctrine(harness): + """What a real dispatch of `harness` actually receives. + + Presence on disk is not receipt (context.cue). Claude auto-loads + CLAUDE.md and not AGENTS.md; Codex embeds AGENTS.md and not + CLAUDE.md; Grok loads both. + """ + received = { + "claude": [("CLAUDE.md", True)], + "codex": [("AGENTS.md", True)], + "grok": [("CLAUDE.md", True), ("AGENTS.md", True)], + }.get(harness) + if received is None: + raise SystemExit( + f"no doctrine receipt recorded for {harness!r}: a control " + f"cannot invent one") + return {"files": [{ + "path": path, + "sha256": _digest(path), + "in_force": in_force, + "evidence": f"listed as a project instruction by {harness}", + } for path, in_force in received]} + + +def context_for(iso, harness, root="/snapshot"): + """A #Dispatchable context describing a real dispatch of `harness`.""" + spec = iso.HARNESSES[harness] + flags = iso.dispatch_flags(harness) + env = iso.dispatch_env(harness, home=f"{root}/.harness-home") + measured = spec["measured"] + + isolation = { + "mechanism": spec["mechanism"], + "flags": flags or {"declared_absent": + f"{harness} suppresses discovery by home, not argv"}, + "env": env or {"declared_absent": + f"{harness} needs no environment override"}, + "observed": { + "operator_config_present": False, + # The module records its measurement differently per + # harness — a probe pair where one was run, the observed + # leak where it was read off the tool. Quote whichever it + # actually holds rather than inventing a uniform sentence. + "evidence": _evidence(measured), + "checked_at": measured["on"], + "harness_version": measured["version"], + }, + } + if spec["mechanism"] == "home": + isolation["home"] = f"{root}/.harness-home" + isolation["auth_files"] = list(spec["auth_files"]) + + interface = { + "name": "extractor-cli", + "argv": ["--root", "DIR", "--out", "PATH"], + "output": "file", + "fields": ["facts", "unresolved"], + # Named in a file that IS staged, or `dangling_references: []` + # would be a lie the schema cannot catch. + "declared_at": "PLAN.md section 5", + } + deliverable = {"path": "extract/cli.py", "interface": interface} + + return { + "role": "extractor", + "harness": {"name": harness, "version": measured["version"], + "isolation": isolation}, + "staged": { + "root": root, + "files": [{"path": "PLAN.md", "sha256": _digest("PLAN.md"), + "bytes": 33975}], + "count": 1, + "given": ["ops/devlane/workflow/PLAN.md"], + "withheld": ["ops/devlane/workflow/**.py"], + "noise_dropped": {"declared_absent": + "the source tree carried no build artefacts"}, + "proof": {"tool": "stage.py", "withheld_present": [], + "given_unmet": []}, + }, + "doctrine": _doctrine(harness), + "brief": { + "role": "extractor", + "deliverables": [copy.deepcopy(deliverable)], + "report_fields": ["RESULT", "UNRESOLVED"], + "rules": ["read no code outside the snapshot"], + "sha256": _digest("brief"), "bytes": 4096, + }, + "task": { + "name": "describe-the-cli", + "requires": [{"what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it"}], + # The same contract, and #Dispatchable now requires the two + # copies to be identical rather than merely both present. + "produces": [copy.deepcopy(deliverable)], + "report_fields": ["RESULT", "UNRESOLVED"], + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": _digest("law"), + } + + +def at(doc, path): + node = doc + for step in path.split("."): + node = node[int(step)] if step.isdigit() else node[step] + return node + + +def put(doc, path, value): + steps = path.split(".") + node = doc + for step in steps[:-1]: + node = node[int(step)] if step.isdigit() else node[step] + last = steps[-1] + if value is _DROP: + del node[int(last) if last.isdigit() else last] + else: + node[int(last) if last.isdigit() else last] = value + + +_DROP = object() + +#: (name, base, path, new value, expected rejection path, why it matters). +#: The prose is parenthesised per entry: an implicit concatenation +#: inside a collection is one missing comma away from silently merging +#: two elements, which is why the linter refuses it. +MUTANTS = [ + ("stale-observation", "home", + "harness.isolation.observed.harness_version", "0.0.1-not-this-build", + "harness.isolation.observed.harness_version", + ("an isolation fact is true of one build on one day, and a release " + "is exactly when a new discovery path appears")), + ("empty-snapshot", "home", "staged.files", [], + "staged.files", + ("an empty snapshot satisfies every withholding rule perfectly, so a " + "gate reading only the stated count cannot tell it from a firewall " + "that worked")), + ("home-without-home", "home", "harness.isolation.home", _DROP, + "harness.isolation.home", + ("for a home-mechanism harness the relocated home IS the isolation " + "evidence; omitting it leaves a record that reads complete")), + ("home-without-auth-files", "home", "harness.isolation.auth_files", _DROP, + "harness.isolation.auth_files", + ("the credential list is what makes \"and nothing else\" checkable; " + "the old fixture dropped it together with the home and so could " + "witness neither")), + ("flags-not-recorded", "flags", "harness.isolation.flags", + {"declared_absent": "not written down"}, + "harness.isolation.flags", + ("for a flags-mechanism harness the argv is the isolation, so " + "declaring it absent is the same missing evidence in a complete " + "record")), + ("role-mismatch", "home", "brief.role", "reviewer", + "role", + ("nothing else stops a launcher handing a reviewer's instructions " + "to an extractor")), + ("task-brief-interface", "home", + "task.produces.0.interface", {"declared_absent": "nothing consumes it"}, + "task.produces.0.interface", + ("the brief and the task state one contract twice; two authors " + "handed different halves of it meet at a seam that does not exist")), + ("task-brief-report-fields", "home", + "task.report_fields", ["RESULT"], + "task.report_fields", + ("a report field missing from one copy is indistinguishable from an " + "honest \"nothing to say\"")), +] + + +def differing_paths(a, b, prefix=""): + """Every leaf path where two documents disagree.""" + if type(a) is not type(b): + return [prefix or "."] + if isinstance(a, dict): + out = [] + for key in sorted(set(a) | set(b)): + where = f"{prefix}.{key}" if prefix else key + if key not in a or key not in b: + out.append(where) + else: + out += differing_paths(a[key], b[key], where) + return out + if isinstance(a, list): + if len(a) != len(b): + return [prefix or "."] + out = [] + for i, (x, y) in enumerate(zip(a, b, strict=True)): + out += differing_paths(x, y, f"{prefix}.{i}") + return out + return [] if a == b else [prefix or "."] + + +def _must_be_stated(): + """The list lives in vet_context.py, which enforces it. One copy.""" + spec = importlib.util.spec_from_file_location( + "harness_vet_context", HARNESS / "vet_context.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.MUST_BE_STATED + + +def build(): + iso = _isolation() + bases = {"flags": context_for(iso, "claude"), + "home": context_for(iso, "codex")} + files = {f"dispatchable-{kind}.json": doc for kind, doc in bases.items()} + + for name, base, path, value, expect, why in MUTANTS: + doc = copy.deepcopy(bases[base]) + put(doc, path, value) + changed = differing_paths(bases[base], doc) + # ONE axis: every difference is inside the declared subtree, and + # there is at least one. Replacing a field whose value is an + # object legitimately shows up as several leaf paths beneath it; + # what must never appear is a path OUTSIDE it, because a fixture + # that breaks two things still fails when only one of the two + # guards exists and so witnesses neither. + stray = [d for d in changed if d != path and not d.startswith(path + ".")] + if stray or not changed: + raise SystemExit( + f"{name}: wanted changes confined to {path!r}, got {changed}") + files[f"rejects/{name}.json"] = doc + files[f"rejects/{name}.reason.json"] = { + "derived_from": f"controls/dispatchable-{base}.json", + "change": f"{path} — {'removed' if value is _DROP else 'replaced'}", + "expect_path": expect, + "why": why, + "finding": "Codex, PR #40", + "generated_by": "controls/build.py", + } + # An omission witness per must-be-stated path. `cue vet` ACCEPTS + # every one of these -- that is the defect -- so their rejection + # comes from the presence check, and the sidecar says so. + for path in _must_be_stated(): + base = "flags" if path.startswith("harness.isolation.flags") else "home" + doc = copy.deepcopy(bases[base]) + try: + put(doc, path, _DROP) + except (KeyError, IndexError, TypeError) as exc: + raise SystemExit(f"omission witness: {path} is not in the " + f"{base} control, so nothing witnesses it") from exc + name = "omits-" + path.replace(".", "-") + files[f"rejects/{name}.json"] = doc + files[f"rejects/{name}.reason.json"] = { + "derived_from": f"controls/dispatchable-{base}.json", + "change": f"{path} — omitted entirely", + "expect_path": path, + "checked_by": "source", + "why": ("the law constrains this field, and a constraint is " + "also a default: an omitting document validates with " + "the clean answer supplied for it, so `cue vet` cannot " + "witness this and the check runs on the raw JSON"), + "finding": "Codex, PR #40", + "generated_by": "controls/build.py", + } + # ...and one for the pair the law unifies: a task that states LESS + # than the brief inherits the rest, so the document claims an + # agreement it never made and `cue vet` sees a complete one. + thin = copy.deepcopy(bases["home"]) + thin["task"]["produces"] = [{"path": thin["brief"]["deliverables"][0]["path"]}] + files["rejects/task-states-less-than-brief.json"] = thin + files["rejects/task-states-less-than-brief.reason.json"] = { + "derived_from": "controls/dispatchable-home.json", + "change": "task.produces states only the path; the interface is left " + "for the law to copy across from the brief", + "expect_path": "task.produces", + "checked_by": "source", + "why": ("presence is not enough for a unified pair: CUE fills the " + "missing half from the other side, so the two copies agree " + "because one of them was written by the law rather than by " + "the author"), + "finding": "predicted by a second reader as the next hole, confirmed " + "by measurement before it was written", + "generated_by": "controls/build.py", + } + return files + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--check", action="store_true", + help="fail if any control on disk differs from generated") + args = ap.parse_args(argv) + + files = build() + problems = [] + for name, doc in sorted(files.items()): + target = HERE / name + text = json.dumps(doc, indent=2) + "\n" + if args.check: + if not target.exists(): + problems.append(f"{name} is missing") + elif target.read_text(encoding="utf-8") != text: + problems.append(f"{name} differs from what build.py generates") + else: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding="utf-8") + if args.check: + for problem in problems: + print(f" {problem}", file=sys.stderr) + print(f"{len(files)} control(s), {len(problems)} problem(s)") + return 1 if problems else 0 + print(f"wrote {len(files)} control file(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/devlane/harness/controls/dispatchable-flags.json b/ops/devlane/harness/controls/dispatchable-flags.json new file mode 100644 index 0000000..23dc763 --- /dev/null +++ b/ops/devlane/harness/controls/dispatchable-flags.json @@ -0,0 +1,129 @@ +{ + "role": "extractor", + "harness": { + "name": "claude", + "version": "unrecorded", + "isolation": { + "mechanism": "flags", + "flags": [ + "--setting-sources", + "project,local", + "--strict-mcp-config", + "--disable-slash-commands" + ], + "env": { + "declared_absent": "claude needs no environment override" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated arm answered YES to the probe phrase, isolated arm answered NO", + "checked_at": "2026-08-22", + "harness_version": "unrecorded" + } + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "CLAUDE.md", + "sha256": "d38626e41f2f3b0a0cf232bf2ff55a56775aa73610e4195cc8ee5e6fcf941ce5", + "in_force": true, + "evidence": "listed as a project instruction by claude" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/dispatchable-home.json b/ops/devlane/harness/controls/dispatchable-home.json new file mode 100644 index 0000000..4d807ee --- /dev/null +++ b/ops/devlane/harness/controls/dispatchable-home.json @@ -0,0 +1,130 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/evidence-receipt.json b/ops/devlane/harness/controls/evidence-receipt.json new file mode 100644 index 0000000..dae757c --- /dev/null +++ b/ops/devlane/harness/controls/evidence-receipt.json @@ -0,0 +1,19 @@ +{ + "kind": "receipt", + "claim": "the suite passes at this commit", + "argv": [ + "python3", + ".dev/app/workflow/tests/shard.py", + "1", + "0" + ], + "cwd": ".", + "head_sha": "96367372e5f782583f14585ba7925e51be16b028b3dc9ed13160e1205ac59afb", + "tree_dirty": false, + "exit_code": 0, + "duration_ms": 154955, + "stdout_sha256": "de2eceeec05f69a60fd4eab6533924e7b74c18b4216ac36657972db8645690df", + "stderr_sha256": "e93abd5880a01334c3d4c84037d28382c2d93228e88887b5008ab0d12c01e6c8", + "actor": "Claude Opus 5", + "at": "2026-08-23T10:00:00Z" +} diff --git a/ops/devlane/harness/controls/rejects/attestation-is-not-admissible.json b/ops/devlane/harness/controls/rejects/attestation-is-not-admissible.json new file mode 100644 index 0000000..61e7f2b --- /dev/null +++ b/ops/devlane/harness/controls/rejects/attestation-is-not-admissible.json @@ -0,0 +1,24 @@ +{ + "kind": "attestation", + "claim": "the packet's rule is not decided by the specification", + "attestor": { + "name": "an actor nobody admitted", + "kind": "model", + "resolved": "grok-4.6", + "withheld": [ + "the repository" + ] + }, + "saw": "13d9dd9c30a24eeab172802e8aa0f546c4d0582c8797629d5699bc0ce2e5a0d4", + "grounding": { + "quotes_verbatim": true, + "required_citations": [ + "specification", + "observation" + ], + "citations_met": true, + "checked_by": "vet.py" + }, + "at": "2026-08-23T10:00:00Z", + "not_reproducible_because": "judgement" +} diff --git a/ops/devlane/harness/controls/rejects/attestation-is-not-admissible.reason.json b/ops/devlane/harness/controls/rejects/attestation-is-not-admissible.reason.json new file mode 100644 index 0000000..dae297f --- /dev/null +++ b/ops/devlane/harness/controls/rejects/attestation-is-not-admissible.reason.json @@ -0,0 +1,8 @@ +{ + "derived_from": "hand-written: there is no passing attestation to derive it from, because #Admissible no longer has an attestation form", + "change": "a fully grounded attestation from an unadmitted actor", + "expect_path": "kind", + "why": "every condition an attestation could meet is set by the claimant -- the grounding booleans, the citation list, and the attestor's own name -- so admitting one on those terms is a schema that reads as diligence and checks nothing. It stays legal as #Attestation and illegal as #Admissible.", + "finding": "Codex, PR #40", + "definition": "#Admissible" +} diff --git a/ops/devlane/harness/controls/rejects/empty-snapshot.json b/ops/devlane/harness/controls/rejects/empty-snapshot.json new file mode 100644 index 0000000..fdbb4ed --- /dev/null +++ b/ops/devlane/harness/controls/rejects/empty-snapshot.json @@ -0,0 +1,124 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/empty-snapshot.reason.json b/ops/devlane/harness/controls/rejects/empty-snapshot.reason.json new file mode 100644 index 0000000..7f9e265 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/empty-snapshot.reason.json @@ -0,0 +1,8 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "staged.files \u2014 replaced", + "expect_path": "staged.files", + "why": "an empty snapshot satisfies every withholding rule perfectly, so a gate reading only the stated count cannot tell it from a firewall that worked", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/flags-not-recorded.json b/ops/devlane/harness/controls/rejects/flags-not-recorded.json new file mode 100644 index 0000000..c6f5e4d --- /dev/null +++ b/ops/devlane/harness/controls/rejects/flags-not-recorded.json @@ -0,0 +1,126 @@ +{ + "role": "extractor", + "harness": { + "name": "claude", + "version": "unrecorded", + "isolation": { + "mechanism": "flags", + "flags": { + "declared_absent": "not written down" + }, + "env": { + "declared_absent": "claude needs no environment override" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated arm answered YES to the probe phrase, isolated arm answered NO", + "checked_at": "2026-08-22", + "harness_version": "unrecorded" + } + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "CLAUDE.md", + "sha256": "d38626e41f2f3b0a0cf232bf2ff55a56775aa73610e4195cc8ee5e6fcf941ce5", + "in_force": true, + "evidence": "listed as a project instruction by claude" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/flags-not-recorded.reason.json b/ops/devlane/harness/controls/rejects/flags-not-recorded.reason.json new file mode 100644 index 0000000..ca27a9d --- /dev/null +++ b/ops/devlane/harness/controls/rejects/flags-not-recorded.reason.json @@ -0,0 +1,8 @@ +{ + "derived_from": "controls/dispatchable-flags.json", + "change": "harness.isolation.flags \u2014 replaced", + "expect_path": "harness.isolation.flags", + "why": "for a flags-mechanism harness the argv is the isolation, so declaring it absent is the same missing evidence in a complete record", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/home-without-auth-files.json b/ops/devlane/harness/controls/rejects/home-without-auth-files.json new file mode 100644 index 0000000..4f607bc --- /dev/null +++ b/ops/devlane/harness/controls/rejects/home-without-auth-files.json @@ -0,0 +1,127 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home" + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/home-without-auth-files.reason.json b/ops/devlane/harness/controls/rejects/home-without-auth-files.reason.json new file mode 100644 index 0000000..7470319 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/home-without-auth-files.reason.json @@ -0,0 +1,8 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "harness.isolation.auth_files \u2014 removed", + "expect_path": "harness.isolation.auth_files", + "why": "the credential list is what makes \"and nothing else\" checkable; the old fixture dropped it together with the home and so could witness neither", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/home-without-home.json b/ops/devlane/harness/controls/rejects/home-without-home.json new file mode 100644 index 0000000..eeadc52 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/home-without-home.json @@ -0,0 +1,129 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/home-without-home.reason.json b/ops/devlane/harness/controls/rejects/home-without-home.reason.json new file mode 100644 index 0000000..5e1452a --- /dev/null +++ b/ops/devlane/harness/controls/rejects/home-without-home.reason.json @@ -0,0 +1,8 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "harness.isolation.home \u2014 removed", + "expect_path": "harness.isolation.home", + "why": "for a home-mechanism harness the relocated home IS the isolation evidence; omitting it leaves a record that reads complete", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/omits-dangling_references.json b/ops/devlane/harness/controls/rejects/omits-dangling_references.json new file mode 100644 index 0000000..a2e7eac --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-dangling_references.json @@ -0,0 +1,129 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/omits-dangling_references.reason.json b/ops/devlane/harness/controls/rejects/omits-dangling_references.reason.json new file mode 100644 index 0000000..ade93ff --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-dangling_references.reason.json @@ -0,0 +1,9 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "dangling_references \u2014 omitted entirely", + "expect_path": "dangling_references", + "checked_by": "source", + "why": "the law constrains this field, and a constraint is also a default: an omitting document validates with the clean answer supplied for it, so `cue vet` cannot witness this and the check runs on the raw JSON", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/omits-harness-isolation-observed-harness_version.json b/ops/devlane/harness/controls/rejects/omits-harness-isolation-observed-harness_version.json new file mode 100644 index 0000000..b2d869a --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-harness-isolation-observed-harness_version.json @@ -0,0 +1,129 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/omits-harness-isolation-observed-harness_version.reason.json b/ops/devlane/harness/controls/rejects/omits-harness-isolation-observed-harness_version.reason.json new file mode 100644 index 0000000..e6f9616 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-harness-isolation-observed-harness_version.reason.json @@ -0,0 +1,9 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "harness.isolation.observed.harness_version \u2014 omitted entirely", + "expect_path": "harness.isolation.observed.harness_version", + "checked_by": "source", + "why": "the law constrains this field, and a constraint is also a default: an omitting document validates with the clean answer supplied for it, so `cue vet` cannot witness this and the check runs on the raw JSON", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/omits-harness-isolation-observed-operator_config_present.json b/ops/devlane/harness/controls/rejects/omits-harness-isolation-observed-operator_config_present.json new file mode 100644 index 0000000..e81be19 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-harness-isolation-observed-operator_config_present.json @@ -0,0 +1,129 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/omits-harness-isolation-observed-operator_config_present.reason.json b/ops/devlane/harness/controls/rejects/omits-harness-isolation-observed-operator_config_present.reason.json new file mode 100644 index 0000000..d1c4119 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-harness-isolation-observed-operator_config_present.reason.json @@ -0,0 +1,9 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "harness.isolation.observed.operator_config_present \u2014 omitted entirely", + "expect_path": "harness.isolation.observed.operator_config_present", + "checked_by": "source", + "why": "the law constrains this field, and a constraint is also a default: an omitting document validates with the clean answer supplied for it, so `cue vet` cannot witness this and the check runs on the raw JSON", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/omits-role.json b/ops/devlane/harness/controls/rejects/omits-role.json new file mode 100644 index 0000000..1eec4fc --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-role.json @@ -0,0 +1,129 @@ +{ + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/omits-role.reason.json b/ops/devlane/harness/controls/rejects/omits-role.reason.json new file mode 100644 index 0000000..466302f --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-role.reason.json @@ -0,0 +1,9 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "role \u2014 omitted entirely", + "expect_path": "role", + "checked_by": "source", + "why": "the law constrains this field, and a constraint is also a default: an omitting document validates with the clean answer supplied for it, so `cue vet` cannot witness this and the check runs on the raw JSON", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/omits-staged-count.json b/ops/devlane/harness/controls/rejects/omits-staged-count.json new file mode 100644 index 0000000..9059d63 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-staged-count.json @@ -0,0 +1,129 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/omits-staged-count.reason.json b/ops/devlane/harness/controls/rejects/omits-staged-count.reason.json new file mode 100644 index 0000000..63fa646 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-staged-count.reason.json @@ -0,0 +1,9 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "staged.count \u2014 omitted entirely", + "expect_path": "staged.count", + "checked_by": "source", + "why": "the law constrains this field, and a constraint is also a default: an omitting document validates with the clean answer supplied for it, so `cue vet` cannot witness this and the check runs on the raw JSON", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/omits-staged-given.json b/ops/devlane/harness/controls/rejects/omits-staged-given.json new file mode 100644 index 0000000..ae5f88e --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-staged-given.json @@ -0,0 +1,127 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/omits-staged-given.reason.json b/ops/devlane/harness/controls/rejects/omits-staged-given.reason.json new file mode 100644 index 0000000..ca69b58 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-staged-given.reason.json @@ -0,0 +1,9 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "staged.given \u2014 omitted entirely", + "expect_path": "staged.given", + "checked_by": "source", + "why": "the law constrains this field, and a constraint is also a default: an omitting document validates with the clean answer supplied for it, so `cue vet` cannot witness this and the check runs on the raw JSON", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/omits-staged-proof-given_unmet.json b/ops/devlane/harness/controls/rejects/omits-staged-proof-given_unmet.json new file mode 100644 index 0000000..5013a47 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-staged-proof-given_unmet.json @@ -0,0 +1,129 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/omits-staged-proof-given_unmet.reason.json b/ops/devlane/harness/controls/rejects/omits-staged-proof-given_unmet.reason.json new file mode 100644 index 0000000..d8a9cce --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-staged-proof-given_unmet.reason.json @@ -0,0 +1,9 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "staged.proof.given_unmet \u2014 omitted entirely", + "expect_path": "staged.proof.given_unmet", + "checked_by": "source", + "why": "the law constrains this field, and a constraint is also a default: an omitting document validates with the clean answer supplied for it, so `cue vet` cannot witness this and the check runs on the raw JSON", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/omits-staged-proof-withheld_present.json b/ops/devlane/harness/controls/rejects/omits-staged-proof-withheld_present.json new file mode 100644 index 0000000..34fa422 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-staged-proof-withheld_present.json @@ -0,0 +1,129 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/omits-staged-proof-withheld_present.reason.json b/ops/devlane/harness/controls/rejects/omits-staged-proof-withheld_present.reason.json new file mode 100644 index 0000000..6e2c890 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-staged-proof-withheld_present.reason.json @@ -0,0 +1,9 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "staged.proof.withheld_present \u2014 omitted entirely", + "expect_path": "staged.proof.withheld_present", + "checked_by": "source", + "why": "the law constrains this field, and a constraint is also a default: an omitting document validates with the clean answer supplied for it, so `cue vet` cannot witness this and the check runs on the raw JSON", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/omits-task-produces.json b/ops/devlane/harness/controls/rejects/omits-task-produces.json new file mode 100644 index 0000000..77e7b61 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-task-produces.json @@ -0,0 +1,110 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/omits-task-produces.reason.json b/ops/devlane/harness/controls/rejects/omits-task-produces.reason.json new file mode 100644 index 0000000..341cd40 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-task-produces.reason.json @@ -0,0 +1,9 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "task.produces \u2014 omitted entirely", + "expect_path": "task.produces", + "checked_by": "source", + "why": "the law constrains this field, and a constraint is also a default: an omitting document validates with the clean answer supplied for it, so `cue vet` cannot witness this and the check runs on the raw JSON", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/omits-task-report_fields.json b/ops/devlane/harness/controls/rejects/omits-task-report_fields.json new file mode 100644 index 0000000..4264a79 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-task-report_fields.json @@ -0,0 +1,126 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/omits-task-report_fields.reason.json b/ops/devlane/harness/controls/rejects/omits-task-report_fields.reason.json new file mode 100644 index 0000000..bf221ad --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-task-report_fields.reason.json @@ -0,0 +1,9 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "task.report_fields \u2014 omitted entirely", + "expect_path": "task.report_fields", + "checked_by": "source", + "why": "the law constrains this field, and a constraint is also a default: an omitting document validates with the clean answer supplied for it, so `cue vet` cannot witness this and the check runs on the raw JSON", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/omits-unmet_requirements.json b/ops/devlane/harness/controls/rejects/omits-unmet_requirements.json new file mode 100644 index 0000000..ee2ae6d --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-unmet_requirements.json @@ -0,0 +1,129 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/omits-unmet_requirements.reason.json b/ops/devlane/harness/controls/rejects/omits-unmet_requirements.reason.json new file mode 100644 index 0000000..54950c0 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/omits-unmet_requirements.reason.json @@ -0,0 +1,9 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "unmet_requirements \u2014 omitted entirely", + "expect_path": "unmet_requirements", + "checked_by": "source", + "why": "the law constrains this field, and a constraint is also a default: an omitting document validates with the clean answer supplied for it, so `cue vet` cannot witness this and the check runs on the raw JSON", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/role-mismatch.json b/ops/devlane/harness/controls/rejects/role-mismatch.json new file mode 100644 index 0000000..e644efb --- /dev/null +++ b/ops/devlane/harness/controls/rejects/role-mismatch.json @@ -0,0 +1,130 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "reviewer", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/role-mismatch.reason.json b/ops/devlane/harness/controls/rejects/role-mismatch.reason.json new file mode 100644 index 0000000..087c21f --- /dev/null +++ b/ops/devlane/harness/controls/rejects/role-mismatch.reason.json @@ -0,0 +1,8 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "brief.role \u2014 replaced", + "expect_path": "role", + "why": "nothing else stops a launcher handing a reviewer's instructions to an extractor", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/stale-observation.json b/ops/devlane/harness/controls/rejects/stale-observation.json new file mode 100644 index 0000000..5bdad69 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/stale-observation.json @@ -0,0 +1,130 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.0.1-not-this-build" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/stale-observation.reason.json b/ops/devlane/harness/controls/rejects/stale-observation.reason.json new file mode 100644 index 0000000..9be24a1 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/stale-observation.reason.json @@ -0,0 +1,8 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "harness.isolation.observed.harness_version \u2014 replaced", + "expect_path": "harness.isolation.observed.harness_version", + "why": "an isolation fact is true of one build on one day, and a release is exactly when a new discovery path appears", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/task-brief-interface.json b/ops/devlane/harness/controls/rejects/task-brief-interface.json new file mode 100644 index 0000000..37b38b4 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/task-brief-interface.json @@ -0,0 +1,118 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "declared_absent": "nothing consumes it" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/task-brief-interface.reason.json b/ops/devlane/harness/controls/rejects/task-brief-interface.reason.json new file mode 100644 index 0000000..f475d11 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/task-brief-interface.reason.json @@ -0,0 +1,8 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "task.produces.0.interface \u2014 replaced", + "expect_path": "task.produces.0.interface", + "why": "the brief and the task state one contract twice; two authors handed different halves of it meet at a seam that does not exist", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/task-brief-report-fields.json b/ops/devlane/harness/controls/rejects/task-brief-report-fields.json new file mode 100644 index 0000000..2698950 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/task-brief-report-fields.json @@ -0,0 +1,129 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/task-brief-report-fields.reason.json b/ops/devlane/harness/controls/rejects/task-brief-report-fields.reason.json new file mode 100644 index 0000000..49fa58e --- /dev/null +++ b/ops/devlane/harness/controls/rejects/task-brief-report-fields.reason.json @@ -0,0 +1,8 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "task.report_fields \u2014 replaced", + "expect_path": "task.report_fields", + "why": "a report field missing from one copy is indistinguishable from an honest \"nothing to say\"", + "finding": "Codex, PR #40", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/controls/rejects/task-states-less-than-brief.json b/ops/devlane/harness/controls/rejects/task-states-less-than-brief.json new file mode 100644 index 0000000..83b79fc --- /dev/null +++ b/ops/devlane/harness/controls/rejects/task-states-less-than-brief.json @@ -0,0 +1,115 @@ +{ + "role": "extractor", + "harness": { + "name": "codex", + "version": "0.148.0", + "isolation": { + "mechanism": "home", + "flags": { + "declared_absent": "codex suppresses discovery by home, not argv" + }, + "env": { + "CODEX_HOME": "/snapshot/.harness-home" + }, + "observed": { + "operator_config_present": false, + "evidence": "unisolated dispatch loaded: ~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "checked_at": "2026-08-22", + "harness_version": "0.148.0" + }, + "home": "/snapshot/.harness-home", + "auth_files": [ + "auth.json" + ] + } + }, + "staged": { + "root": "/snapshot", + "files": [ + { + "path": "PLAN.md", + "sha256": "d0225c077fa435ec2aa558467876295c4f7d97a429290a0c13234b743974fabd", + "bytes": 33975 + } + ], + "count": 1, + "given": [ + "ops/devlane/workflow/PLAN.md" + ], + "withheld": [ + "ops/devlane/workflow/**.py" + ], + "noise_dropped": { + "declared_absent": "the source tree carried no build artefacts" + }, + "proof": { + "tool": "stage.py", + "withheld_present": [], + "given_unmet": [] + } + }, + "doctrine": { + "files": [ + { + "path": "AGENTS.md", + "sha256": "110344b713866f3adcf3e83870ea4c966abd3914eb379ec091d43f8a329443d0", + "in_force": true, + "evidence": "listed as a project instruction by codex" + } + ] + }, + "brief": { + "role": "extractor", + "deliverables": [ + { + "path": "extract/cli.py", + "interface": { + "name": "extractor-cli", + "argv": [ + "--root", + "DIR", + "--out", + "PATH" + ], + "output": "file", + "fields": [ + "facts", + "unresolved" + ], + "declared_at": "PLAN.md section 5" + } + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ], + "rules": [ + "read no code outside the snapshot" + ], + "sha256": "b15535d84b376ff62d7aac818f4e545e5d9ff44b29cc642a3581fc38abdc4fbb", + "bytes": 4096 + }, + "task": { + "name": "describe-the-cli", + "requires": [ + { + "what": "the plan's CLI section", + "satisfied_by": "PLAN.md", + "why": "the verbs come from it" + } + ], + "produces": [ + { + "path": "extract/cli.py" + } + ], + "report_fields": [ + "RESULT", + "UNRESOLVED" + ] + }, + "unmet_requirements": [], + "dangling_references": [], + "derived_from": "8a0ef21c4ae95b556ee7dc2c20975e336fd1b4fdb843bf1ead8a03eb97e63482" +} diff --git a/ops/devlane/harness/controls/rejects/task-states-less-than-brief.reason.json b/ops/devlane/harness/controls/rejects/task-states-less-than-brief.reason.json new file mode 100644 index 0000000..e3719f9 --- /dev/null +++ b/ops/devlane/harness/controls/rejects/task-states-less-than-brief.reason.json @@ -0,0 +1,9 @@ +{ + "derived_from": "controls/dispatchable-home.json", + "change": "task.produces states only the path; the interface is left for the law to copy across from the brief", + "expect_path": "task.produces", + "checked_by": "source", + "why": "presence is not enough for a unified pair: CUE fills the missing half from the other side, so the two copies agree because one of them was written by the law rather than by the author", + "finding": "predicted by a second reader as the next hole, confirmed by measurement before it was written", + "generated_by": "controls/build.py" +} diff --git a/ops/devlane/harness/evidence.cue b/ops/devlane/harness/evidence.cue new file mode 100644 index 0000000..c1619f5 --- /dev/null +++ b/ops/devlane/harness/evidence.cue @@ -0,0 +1,206 @@ +// How a claim is backed: re-run it, or attest to it. Never both, never +// neither, and never one wearing the other's clothes. +// +// A contract can pin the SHAPE of a receipt. It cannot pin that the +// receipt was earned. That sentence closed the write-up of this +// session's CUE work and it is the gap this file exists to close. +// +// There are exactly two ways a claim can be backed, and they are not +// interchangeable: +// +// VERIFY the claim can be re-derived. Anyone with the repository +// runs the command at the recorded state and compares +// digests. Trust in the claimant is not required, because +// nothing rests on their word -- the reproduction +// instruction IS the evidence. +// +// CERTIFY the claim cannot be re-derived, because it is a judgement, +// or an observation of something that has since moved, or a +// probe that costs money to repeat. An adjudicator's verdict +// on a disagreement is the clearest case: re-running +// anything produces the disagreement again, never the +// ruling. Here trust in the claimant IS load-bearing, so the +// claimant must be named and what they were shown must be +// pinned. +// +// THE CENTRAL RULE, and the reason this is CUE rather than a convention: +// an attestation has NO FIELD in which to claim reproducibility. Not a +// boolean that must be false -- no slot at all. A closed struct cannot +// be given one, so "this was verified" is not a sentence an attestation +// can express, however much whoever is writing it would like to. +// +// The same shape as a judgement that cannot claim binding force: the +// admissible values are what stop the claim, not a reviewer's attention. +// +// WHAT CERTIFICATION CAN STILL BE CHECKED FOR. A judgement cannot be +// re-derived, but its GROUNDING can be. Every quote it rests on must +// appear verbatim in what the attestor was shown; it must cite at least +// one source of each required kind; the attestor must be someone the +// registry admits. Those are mechanical, they either ran or they did +// not, and #Grounding records which. That is the whole difference +// between an attestation and an opinion. + +package harness + +// -------------------------------------------------------------------- + +#Evidence: #Receipt | #Attestation + +// ----------------------------------------------------------- receipt + +// A claim anyone can re-derive. The fields are the reproduction. +#Receipt: close({ + // Fixed, so the two kinds can be told apart by a reader and by a + // program without inspecting which fields happen to be present. + kind: "receipt" + + claim: #NonEmpty + + // Everything needed to run it again. `cwd` is relative to the + // repository root: an absolute path is a fact about one machine. + argv: [#NonEmpty, ...#NonEmpty] + cwd: #NonEmpty + + // The state it ran against. A receipt without this is a claim about + // an unnamed tree, and the tree has since moved. + head_sha: #Digest + tree_dirty: bool + + exit_code: int + duration_ms: int & >=0 + stdout_sha256: #Digest + stderr_sha256: #Digest + + // Who ran it. Recorded for attribution, NOT relied upon: the whole + // point is that a reader need not believe them. + actor: #NonEmpty + at: #NonEmpty +}) + +// -------------------------------------------------------- attestation + +// A claim that rests on someone's judgement. +// +// Note what is absent and cannot be added: argv, exit_code, output +// digests, any `reproducible` or `verified` field. A closed struct has +// no room for them, so an attestation cannot be written that claims to +// be a reproduction. +#Attestation: close({ + kind: "attestation" + + claim: #NonEmpty + + // Why re-running is not available. Required, because "we could have + // verified this and did not" and "this cannot be verified" are + // different situations, and only one of them is acceptable. + not_reproducible_because: "judgement" | "external-state-moved" | + "costly-to-repeat" | "one-time-observation" + + attestor: #Attestor + + // EXACTLY what the attestor was shown, by digest. Without it the + // attestation floats: it cannot be said what the judgement was a + // judgement OF, and a later reader cannot reconstruct the question. + saw: #Digest + + // The mechanical checks that were run on the grounding. Not on the + // judgement -- that is what cannot be checked -- but on whether the + // judgement is anchored to what the attestor saw. + grounding: #Grounding + + at: #NonEmpty +}) + +#Attestor: close({ + // Recorded, NOT admitted. The comment here used to say "must be + // admitted by the registry"; there is no registry, and a claimant + // cannot admit themselves, so the field says what it is: a name + // this attestation carries. See #Admissible for why that is a + // reason to exclude the form rather than to invent a list. + name: #NonEmpty + + // A model attesting is not a person attesting, and a reader is + // entitled to weigh them differently. + kind: "model" | "person" | "tool" + + // For a model, the resolved identity from its own trace -- never the + // alias requested. An alias is a moving target; the resolved name is + // an identity. + resolved: #NonEmpty + + // What the attestor could NOT see. An adjudicator with no repository + // access is more trustworthy on a packet, not less, and that is only + // legible if the withholding is recorded beside the verdict. + withheld: [...#NonEmpty] | #DeclaredAbsent +}) + +// The bridge. A judgement cannot be re-derived; its anchoring can. +#Grounding: close({ + // Every quote the claim rests on, checked to occur verbatim in what + // the attestor saw. A paraphrase fails: it is the point at which a + // judgement starts drifting from its evidence. + quotes_verbatim: bool + + // The kinds of source that must each be cited at least once -- for + // an adjudication, the specification and the observation. A verdict + // citing only one side has heard only one side. + required_citations: [#NonEmpty, ...#NonEmpty] + citations_met: bool + + // The tool that ran these checks, so "grounded" is not itself an + // unbacked claim. Circularity stops here: this is a receipt's job. + checked_by: #NonEmpty +}) + +// ---------------------------------------------------------- admission + +// Evidence that may be relied on. +// +// A receipt qualifies by being re-runnable at a named state. An +// attestation qualifies only when its grounding checks actually PASSED +// -- an attestation whose quotes were never verified is an opinion with +// a schema, which is worse than an opinion, because the schema reads as +// diligence. +// Split per kind rather than written as one conditional. `if kind == …` +// inside a disjunction cannot resolve: the field is not in scope until +// the disjunction is decided, and CUE says so with `reference "kind" not +// found`. Two admissible forms, unioned, says the same thing and +// evaluates. +#AdmissibleReceipt: #Receipt & { + // A claim about a dirty tree names no state anyone can return to. + tree_dirty: false +} + +// An attestation whose quotes were never checked is an opinion with a +// schema -- worse than an opinion, because the schema reads as +// diligence. So this is what admission WOULD require. +// +// It is deliberately NOT part of #Admissible, and that is the honest +// position rather than a gap. Every condition here is set by the +// claimant: the two grounding booleans are ticked by whoever wrote the +// attestation, `required_citations` is a list they choose, and +// `#Attestor.name` -- whose comment promised admission "by the +// registry" -- was `#NonEmpty` with no registry anywhere in the +// repository (Codex, PR #40). +// +// A name union would not fix it. HARNESSES refuses an unknown harness +// because the LAUNCHER does that lookup; nothing on the attestation +// path is an external lookup, so a list of admitted names is one the +// claimant reads and then writes their own name from. That relocates +// #NonEmpty into a list and leaves #Admissible looking like diligence, +// which is the exact failure this file exists to name. +// +// So: an attestation stays a describable judgement -- the way #Context +// can describe a leak -- and #Admissible means receipts until something +// other than the claimant can admit an actor. +#AdmissibleAttestation: #Attestation & { + grounding: quotes_verbatim: true + grounding: citations_met: true +} + +#Admissible: #AdmissibleReceipt + +// A claim requiring reproduction cannot be met by an attestation. Stated +// as a definition so a caller asks for the strength it needs, rather +// than accepting whatever arrived and hoping. +#MustVerify: #AdmissibleReceipt diff --git a/ops/devlane/harness/isolation.py b/ops/devlane/harness/isolation.py new file mode 100644 index 0000000..25c640e --- /dev/null +++ b/ops/devlane/harness/isolation.py @@ -0,0 +1,382 @@ +"""Strip the operator's personal setup out of a dispatched harness. + +A harness launched from a developer's machine does not start empty. It +discovers instruction files, hooks, skills and MCP servers from that +person's home directory, and none of that is this project's law. It is +not in the repository, no other harness shares it, CI does not have it, +and nobody agreed that it applies to work done here. + +What was measured on 2026-08-22, dispatching into a throwaway snapshot +that contained nothing but the brief: + + claude the operator's ~/.claude/CLAUDE.md was in the system prompt. + Asked "do your instructions contain ", a + default dispatch answered YES and an isolated one answered NO. + A SessionStart hook, the personal skill listing and personal + MCP servers arrived as attachments. + codex ~/.codex/hooks.json ran a SessionStart command living in an + unrelated repository, and its output was injected. + grok `grok inspect` listed ~/.claude/CLAUDE.md as a project + instruction worth ~7012 tokens, plus a global rules file and + 31 skills, 24 of them the operator's. + +So the leak is not one harness's quirk. It is what all three do by +design, and the fix has to be per-harness because each reads a +different place. + +Two mechanisms are used here, whichever the harness supports: + + flags the harness offers a documented way to not load user-scoped + configuration. Cheapest and least invasive; nothing on disk + is touched. + home the harness only looks in a directory named by an + environment variable, so it is pointed at a directory built + here that holds credentials and nothing else. + +CREDENTIALS ARE THE ONE EXCEPTION, and it is deliberate. A harness +that cannot authenticate cannot run at all, so the minimal home links +the auth file through and nothing else. Everything that carries +instructions, behaviour or context is left behind. `auth_files` is the +complete list of what crosses that line; it is data, so it can be +read, and a test asserts nothing else is ever linked. + +WHAT THIS IS NOT. Every entry below closes a discovery path that was +found by looking. That makes this a list of known leaks, and a list of +known leaks is only ever as current as the last time somebody looked +-- a harness release can add a fourth path, and nothing here would +say so. `probe.py` exists for exactly that reason and should be run +when a harness version changes. + +The guarantee that does not depend on having enumerated correctly is a +container with no operator home mounted in it: then there is nothing +to discover, whatever the harness decides to look for next. That is +the backstop if a leak is found with no environment variable behind +it, or if a harness stops honouring one. This module is the cheap +version and it is honest about being the cheap version. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +# -------------------------------------------------------------------- +# The data. Every entry says how a harness is isolated and how that was +# established. An entry with no mechanism is not a harness that happens +# to be clean -- it is one nobody has checked, and dispatching it is +# refused rather than assumed safe. +# -------------------------------------------------------------------- + +HARNESSES = { + "claude": { + "mechanism": "flags", + "flags": [ + # Drops ~/.claude/settings.json AND user-scoped CLAUDE.md. + "--setting-sources", "project,local", + # Drops MCP servers configured in the operator's account. + "--strict-mcp-config", + # Drops the personal skill listing. + "--disable-slash-commands", + ], + "home_env": None, + "auth_files": [], + # A phrase that appears only in the operator's own doctrine + # file. Asking the model whether it can see this is the only + # way to observe the system prompt, which is not written to + # the session trace -- absence from the trace would prove + # nothing. + "probe_phrase": "Start from the class, not the instance", + # HOME is untouched, so traces stay in the real home. + "sessions": {"under": "real", "path": ".claude/projects"}, + "measured": { + "on": "2026-08-22", + "version": "unrecorded", + "probe_default": "YES", + "probe_isolated": "NO", + "attachment_bytes_default": 17304, + "attachment_bytes_isolated": 2761, + "note": "what remains is Claude Code's own built-in machinery", + }, + }, + "codex": { + "mechanism": "home", + "flags": [], + "home_env": "CODEX_HOME", + "auth_files": ["auth.json"], + "probe_phrase": None, + # CODEX_HOME moved, so the trace moves with it. + "sessions": {"under": "minimal", "path": "sessions"}, + "measured": { + "on": "2026-08-22", + "version": "0.148.0", + "leak": "~/.codex/hooks.json SessionStart ran " + "projects/xormania/xor/tools/xortations/hooks/session_start.py", + "note": "personal MCP servers and a memories store also live under the home", + }, + }, + "grok": { + # Two variables, because two different directories leak. GROK_HOME + # alone still let ~/.claude/CLAUDE.md through: grok looks for that + # under $HOME, not under its own home. HOME alone still let + # ~/.grok/rules through. Both, or neither works. + "mechanism": "home", + "flags": [], + "home_env": "GROK_HOME", + "also_env": ["HOME"], + "auth_files": ["auth.json"], + "probe_phrase": None, + "sessions": {"under": "minimal", "path": "sessions"}, + "measured": { + "on": "2026-08-22", + "version": "1.0.5", + "leak": "grok inspect listed ~/.claude/CLAUDE.md (~7012 tokens) " + "and ~/.grok/rules/00-xortations-first-turn.md (~161 tokens) " + "as project instructions; 31 skills, 24 user-scoped", + "note": "HOME alone drops CLAUDE.md and cuts skills 31 -> 7; " + "GROK_HOME alone drops the rules file; both are needed", + }, + }, +} + + +class NotIsolated(Exception): + """Raised instead of dispatching a harness that cannot be isolated. + + The refusal is the point. A harness absent from HARNESSES has not + been shown to be clean, and defaulting to "launch it anyway" + converts "nobody looked" into "we checked and it was fine". + """ + + +def _real_home(harness, env, given=True): + """Where this harness's real config lives, for reading credentials. + + Two corrections, both from a test author who could not see this + function and reasoned from what it PROMISES (PR #40 follow-up): + + `home_env` is honoured when the environment sets it. An operator who + runs codex with CODEX_HOME set keeps their config there, not in + ~/.codex, so reading the wrong directory would report a credential + absent that is present -- or link one that is not the one in use. + + `given` defaults to True -- the strict reading -- so a direct caller + gets the refusal and only `build_home`, which knows whether its + caller supplied an env, may ask for the lenient one. + + And when the caller passed an environment EXPLICITLY, a missing HOME + is refused rather than filled in from `Path.home()`. Passing an env + is how a caller says "this, and nothing of mine"; reaching past it + to the operator's real home is the leak this module exists to + prevent, and it made one of the author's tests pass on this machine + and fail on a clean one -- the shape of a suite that lies. + """ + spec = HARNESSES[harness] + named = spec.get("home_env") + if named and env.get(named): + return Path(env[named]) + home = env.get("HOME") + # EMPTY IS ABSENT. `HOME=""` is not None, so the refusal below did + # not fire, and `Path("") / ".codex"` is the RELATIVE path `.codex` + # — resolved against whatever directory the launcher happened to be + # in, which is ambient project state and exactly what an explicit + # environment is supposed to exclude (Copilot, PR #42). The same + # distinction this repo makes everywhere else between "we looked and + # found none" and "nobody looked", arriving one more time as a + # falsy value that is not None. + if not home: + if given: + raise NotIsolated( + f"{harness}: the environment passed here sets neither " + f"{named or 'HOME'} nor HOME, so there is nowhere to read " + f"credentials from. Falling back to the operator's own home " + f"would be the leak this builds a home to prevent.") + home = str(Path.home()) + return Path(home) / f".{harness}" + + +def build_home(harness, root, env=None): + """Create a minimal home for `harness` under `root`; return its path. + + It holds the credential files named in `auth_files` and nothing + else. Each is symlinked, not copied, so a credential is never + duplicated into a scratch directory that outlives the run. + + Raises NotIsolated when a credential the harness needs is missing, + rather than producing a home that will fail to authenticate in a + way that looks like a model refusal. + + `root` must be missing or an EMPTY directory. A reused one may carry + the operator's own setup, and preserving it is the leak this builds + a home to prevent -- so a populated root is refused, naming what it + found. That was not written down until an independent test author + assumed the opposite and expected a rebuild. + """ + given = env is not None + env = os.environ if env is None else env + spec = HARNESSES.get(harness) + if spec is None: + raise NotIsolated( + f"{harness!r} has no isolation entry: nobody has established " + f"what it loads from the operator's home, so it is not " + f"dispatched. Add an entry with a measurement.") + dest = Path(root) + # A MINIMAL home has to start empty. `exist_ok=True` on a reused + # root preserved whatever was already there and returned normally, + # so a home carrying the operator's hooks.json, AGENTS.md and + # skills/ could be handed to a dispatch — through the constructor + # of the module that exists to strip exactly those (Codex, PR #40). + # The structural probe is the only reader that would notice, and + # nothing on the launch path calls it. + if dest.exists(): + if not dest.is_dir(): + raise NotIsolated( + f"{harness}: {dest} exists and is not a directory; a " + f"minimal home cannot be built there.") + leftovers = sorted(p.name for p in dest.iterdir()) + if leftovers: + raise NotIsolated( + f"{harness}: {dest} is not empty ({', '.join(leftovers[:5])}" + f"{', …' if len(leftovers) > 5 else ''}). A reused home may " + f"carry the operator's own setup, which is the leak this " + f"builds a home to prevent. Pass a fresh directory.") + dest.mkdir(parents=True, exist_ok=True) + src_home = _real_home(harness, env, given) + for name in spec["auth_files"]: + src = src_home / name + if not src.exists(): + raise NotIsolated( + f"{harness}: credential {src} is absent, so an isolated " + f"home cannot authenticate. Not falling back to the " + f"operator's home.") + # No exists-check: the destination was just proved empty, so a + # name already there would mean something wrote into the home + # between the two, and skipping it silently is the same hole in + # miniature. + link = dest / name + link.parent.mkdir(parents=True, exist_ok=True) + # ABSOLUTE. `symlink_to` with a relative source resolves it + # against the LINK's directory, not the caller's, so a relative + # `src` produced a link pointing inside the minimal home — a + # dangling one, in a home that looked complete because an entry + # named `auth.json` was there. Surfaced while reproducing the + # empty-HOME finding above; `build_home` returned normally. + link.symlink_to(src.resolve()) + return dest + + +def dispatch_env(harness, home=None, env=None): + """The environment overrides that isolate `harness`. + + `home` is a directory from build_home. It is required for a + home-mechanism harness and ignored for a flag-mechanism one. + """ + env = os.environ if env is None else env + spec = HARNESSES.get(harness) + if spec is None: + raise NotIsolated(f"{harness!r} has no isolation entry") + if spec["mechanism"] == "flags": + return {} + # Same rule, one function along: `home=""` would emit + # `CODEX_HOME=""`, which the harness reads as unset and answers by + # loading the operator's real home. Found by looking for the shape + # rather than the instance, after the instance was reported. + if not home: + raise NotIsolated( + f"{harness} is isolated by relocating its home, and no home " + f"was built (got {home!r}). Call build_home first.") + out = {spec["home_env"]: str(home)} + for extra in spec.get("also_env", ()): + # HOME is redirected to the minimal home too, so that a harness + # looking for a SIBLING vendor's dotfile -- grok reading + # ~/.claude/CLAUDE.md -- finds nothing there either. + out[extra] = str(home) + return out + + +def dispatch_flags(harness): + """The argv fragment that isolates `harness`, possibly empty.""" + spec = HARNESSES.get(harness) + if spec is None: + raise NotIsolated(f"{harness!r} has no isolation entry") + return list(spec["flags"]) + + +def isolated(harness, root, env=None): + """Everything a launcher needs: (env_overrides, argv_fragment). + + The single entry point. A launcher that calls this cannot dispatch + an unisolated harness, because there is no argument that turns the + isolation off. + """ + spec = HARNESSES.get(harness) + if spec is None: + raise NotIsolated( + f"{harness!r} has no isolation entry: dispatching it would " + f"carry the operator's personal setup into this project.") + home = build_home(harness, root, env) if spec["mechanism"] == "home" else None + return dispatch_env(harness, home, env), dispatch_flags(harness) + + +def report(): + """What is known about each harness, as JSON. For the record, and + for a check that wants to notice an entry going stale.""" + # The WHOLE entry. It reported `mechanism` and `measured` only, + # while promising "what is known about each harness" and naming its + # own purpose as noticing an entry going stale -- and a check that + # cannot see `flags` or `auth_files` cannot notice those going + # stale, which is the operative half (PR #40 follow-up). + return json.dumps(HARNESSES, indent=2, sort_keys=True) + + +def _main(argv=None): + """A shell launcher needs the same answer this module already + holds. Giving it one is what keeps the flags from being written + down twice and drifting apart -- the duplicate copy is always the + one that misses the next fix. + + eval "$(isolation.py --sh claude /tmp/home)" + + emits `ISO_FLAGS` and any environment assignments, and exits + non-zero with an explanation on a harness that cannot be isolated, + so a launcher that checks its exit status cannot dispatch one. + """ + import argparse + import shlex + + ap = argparse.ArgumentParser(description="isolation facts for a launcher") + ap.add_argument("--sh", metavar="HARNESS", + help="emit shell assignments for this harness") + ap.add_argument("root", nargs="?", + help="directory to build a minimal home in (--sh only)") + args = ap.parse_args(argv) + + if not args.sh: + print(report()) + return 0 + try: + if HARNESSES.get(args.sh, {}).get("mechanism") == "home" and not args.root: + raise NotIsolated( + f"{args.sh} is isolated by relocating its home; pass a " + f"directory to build one in") + env, flags = isolated(args.sh, args.root or "") + except NotIsolated as exc: + print(f"echo {shlex.quote('REFUSED: ' + str(exc))} >&2; exit 78") + return 78 + spec = HARNESSES[args.sh] + sess = spec["sessions"] + base = (str(Path(os.environ.get("HOME") or Path.home())) + if sess["under"] == "real" else str(args.root)) + # Emitted as assignments, not exports: the launcher must apply these + # to the HARNESS only. Exporting HOME would relocate the launcher's + # own lookups too, and it still needs the real one. + for k, v in sorted(env.items()): + print(f"ISO_ENV_{k}={shlex.quote(v)}") + print(f"ISO_ENV={shlex.quote(' '.join(f'{k}={v}' for k, v in sorted(env.items())))}") + print(f"ISO_FLAGS={shlex.quote(' '.join(flags))}") + print(f"ISO_STORE={shlex.quote(str(Path(base) / sess['path']))}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/ops/devlane/harness/probe.py b/ops/devlane/harness/probe.py new file mode 100644 index 0000000..8123654 --- /dev/null +++ b/ops/devlane/harness/probe.py @@ -0,0 +1,309 @@ +"""Prove the isolation in `isolation.py` still works. + +Every entry in HARNESSES was true of one harness version on one day. +Harnesses ship often, and a release that adds a discovery path, or +stops honouring an environment variable, breaks the isolation without +breaking anything that would announce itself. So the entries are +claims, and this is the thing that re-runs them. + +Two kinds of check, because they cost very differently: + + structural read-only, free, no model call. Asserts the minimal + home really is minimal -- that nothing beyond the + declared credentials was linked into it. Catches the + mistake of adding a convenience file to a home and + quietly re-opening the leak. Run it always. + + behavioural actually launch the harness and observe what it + loaded. This is the only kind that can catch a harness + that ignores the variable. Costs a dispatch, so run it + when a version changes. + +A behavioural check runs TWO arms and both must succeed on their own +terms: an unisolated arm that must show the leak, and an isolated arm +that must not. An arm that failed to run at all is reported INVALID, +never as a pass -- a harness that died before loading anything shows +no leak, and reading that as "isolation worked" is the exact shape of +a test that proves its own setup never happened. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import isolation + + +def _discard(path): + """Remove a directory this module created, refusing anything else. + + `rm -rf` on a variable is the shape that destroys the wrong thing + when the variable is empty or wrong, and it reads alarmingly for + good reason. Every directory removed here came from + tempfile.mkdtemp moments earlier, so that is asserted rather than + assumed: a path outside the system temp directory is left alone + and the mistake is reported instead of acted on. + """ + root = Path(tempfile.gettempdir()).resolve() + target = Path(path).resolve() + if target == root or root not in target.parents: + raise RuntimeError( + f"refusing to remove {target}: not inside {root}") + shutil.rmtree(target, ignore_errors=True) + + +# Verdicts. INVALID is not a failure and not a pass; it means the +# question was never actually asked, so there is no result to report. +CLEAN = "CLEAN" +LEAKING = "LEAKING" +INVALID = "INVALID" + + +class Result: + def __init__(self, harness, kind, verdict, detail): + self.harness = harness + self.kind = kind + self.verdict = verdict + self.detail = detail + + def as_dict(self): + return {"harness": self.harness, "kind": self.kind, + "verdict": self.verdict, "detail": self.detail} + + def __str__(self): + return f"{self.harness:8} {self.kind:12} {self.verdict:8} {self.detail}" + + +# -------------------------------------------------------------------- +# structural +# -------------------------------------------------------------------- + +def structural(harness, env=None): + """Build the minimal home and assert it holds only credentials.""" + spec = isolation.HARNESSES.get(harness) + if spec is None: + return Result(harness, "structural", INVALID, "no isolation entry") + if spec["mechanism"] != "home": + return Result(harness, "structural", CLEAN, + f"mechanism is {spec['mechanism']}; no home is built") + root = tempfile.mkdtemp(prefix=f"probe-{harness}-") + try: + try: + home = isolation.build_home(harness, root, env) + except isolation.NotIsolated as exc: + return Result(harness, "structural", INVALID, str(exc)) + present = sorted(p.name for p in Path(home).iterdir()) + declared = sorted(spec["auth_files"]) + if present != declared: + extra = sorted(set(present) - set(declared)) + return Result(harness, "structural", LEAKING, + f"home holds {extra} beyond declared {declared}") + return Result(harness, "structural", CLEAN, + f"home holds exactly {declared}") + finally: + _discard(root) + + +# -------------------------------------------------------------------- +# behavioural +# -------------------------------------------------------------------- + +def _run(argv, env, cwd, stdin=""): + """(rc, stdout, stderr), where rc is None when there is NO ANSWER. + + A command that exited non-zero did not answer, however well its + stdout parses: a harness that failed to initialise still prints a + well-formed `Project Instructions (0)` header, and `claude -p` can + print a final YES or NO and then fail on teardown. Both were read as + readings, and both scored a broken probe CLEAN. + + The mapping lives HERE rather than at each call site, because it was + fixed at one call site and not the other, and the arm that was + missed is the one the next reviewer found (Codex, PR #40 twice). A + caller that needs the real status has none of them; nothing in this + module does. + """ + try: + p = subprocess.run(argv, env=env, cwd=cwd, input=stdin, + capture_output=True, text=True, timeout=180, + check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + return None, "", str(exc) + if p.returncode != 0: + return None, p.stdout, p.stderr or f"exit {p.returncode}" + return p.returncode, p.stdout, p.stderr + + +def behavioural_grok(env=None, cwd=None): + """`grok inspect` names every instruction file it discovered. + + Free -- it reports configuration without calling a model -- which + makes grok the one harness whose isolation can be re-proven at no + cost. The leak is counted as instruction lines mentioning a path + outside the working directory. + """ + env = dict(os.environ if env is None else env) + cwd = cwd or tempfile.mkdtemp(prefix="probe-grok-cwd-") + + def instructions(e): + """(count, paths) from `grok inspect`, or (None, raw) if it did + not run. + + The COUNT comes from the header grok prints -- `Project + Instructions (2)` -- not from counting the tree rows beneath + it. When nothing is loaded grok still prints one row, reading + `(none)`, and counting rows scored that as a leak. Taking the + number the tool states is both simpler and not fooled by how + it renders an empty list. + """ + rc, out, err = _run(["grok", "inspect"], e, cwd) + # `_run` already answers None for "did not run", which now + # includes a nonzero exit. One rule, one place. + if rc is None or not out.strip(): + return None, err or out or "no output" + count, paths, grab = None, [], False + for line in out.splitlines(): + s = line.strip() + if s.startswith("Project Instructions"): + head = s.rpartition("(")[2].partition(")")[0] + count = int(head) if head.isdigit() else None + grab = True + continue + if grab: + if s.startswith(("└", "├")): + body = s[1:].strip() + if body != "(none)": + paths.append(body) + elif s: + break + if count is None: + return None, out + return (count, paths), out + + before, raw_before = instructions(env) + if before is None: + return Result("grok", "behavioural", INVALID, + f"unisolated arm produced nothing: {raw_before}") + n_before, paths_before = before + if n_before == 0: + return Result("grok", "behavioural", INVALID, + "unisolated arm found NO instruction files, so this " + "machine cannot demonstrate the leak and the " + "isolated arm proves nothing") + + root = tempfile.mkdtemp(prefix="probe-grok-home-") + try: + try: + over, _flags = isolation.isolated("grok", root, env) + except isolation.NotIsolated as exc: + return Result("grok", "behavioural", INVALID, str(exc)) + after, raw_after = instructions({**env, **over}) + if after is None: + return Result("grok", "behavioural", INVALID, + f"isolated arm produced nothing: {raw_after}") + n_after, paths_after = after + if n_after: + return Result("grok", "behavioural", LEAKING, + f"isolated arm still loads {n_after}: {paths_after}") + return Result("grok", "behavioural", CLEAN, + f"unisolated loaded {n_before} ({paths_before}), " + f"isolated loaded 0") + finally: + _discard(root) + + +def behavioural_claude(model="haiku", env=None): + """Ask the model whether the operator's doctrine is in its prompt. + + The system prompt is not written to the session trace, so its + absence there proves nothing at all. Asking the model is the only + observation available. + """ + spec = isolation.HARNESSES["claude"] + phrase = spec["probe_phrase"] + env = dict(os.environ if env is None else env) + question = ('Answer with one word only, YES or NO. Do your system ' + f'instructions contain the phrase "{phrase}"?') + base = ["claude", "-p", "--model", model, "--permission-mode", "plan"] + + def ask(argv): + cwd = tempfile.mkdtemp(prefix="probe-claude-cwd-") + try: + rc, out, err = _run(argv + [question], env, cwd) + if rc is None: + return None, err + word = out.strip().splitlines()[-1].strip().upper() if out.strip() else "" + return (word if word in ("YES", "NO") else None), out or err + finally: + _discard(cwd) + + before, raw_before = ask(base) + if before is None: + return Result("claude", "behavioural", INVALID, + f"unisolated arm gave no YES/NO: {raw_before[:200]!r}") + if before != "YES": + return Result("claude", "behavioural", INVALID, + "unisolated arm answered NO, so this machine has no " + "operator doctrine to leak and the isolated arm " + "proves nothing") + after, raw_after = ask(base + isolation.dispatch_flags("claude")) + if after is None: + return Result("claude", "behavioural", INVALID, + f"isolated arm gave no YES/NO: {raw_after[:200]!r}") + if after == "YES": + return Result("claude", "behavioural", LEAKING, + "isolated arm still sees the operator's doctrine") + return Result("claude", "behavioural", CLEAN, + "unisolated YES, isolated NO") + + +BEHAVIOURAL = {"grok": behavioural_grok, "claude": behavioural_claude} + + +def behavioural(harness, env=None): + fn = BEHAVIOURAL.get(harness) + if fn is None: + return Result(harness, "behavioural", INVALID, + "no cheap observation exists for this harness; its " + "isolation is asserted structurally only") + return fn(env=env) + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("harness", nargs="*", default=sorted(isolation.HARNESSES)) + ap.add_argument("--behavioural", action="store_true", + help="also dispatch each harness; costs tokens") + ap.add_argument("--json", action="store_true") + args = ap.parse_args(argv) + + results = [] + for h in (args.harness or sorted(isolation.HARNESSES)): + results.append(structural(h)) + if args.behavioural: + results.append(behavioural(h)) + + if args.json: + print(json.dumps([r.as_dict() for r in results], indent=2)) + else: + for r in results: + print(r) + + if any(r.verdict == LEAKING for r in results): + return 1 + if any(r.verdict == INVALID for r in results): + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/devlane/harness/stage.py b/ops/devlane/harness/stage.py new file mode 100644 index 0000000..81d3f00 --- /dev/null +++ b/ops/devlane/harness/stage.py @@ -0,0 +1,237 @@ +"""Build a role's snapshot from a manifest, and prove the firewall held. + +The method these dispatches rely on is that two authors work from +different halves of the truth and never see each other's half. One +writes a contract from the specification without the code; another +describes the code without the specification. When they disagree, the +disagreement is real, because neither was in a position to quietly +reconcile it. + +That only holds if the withholding actually happened. And withholding +is invisible: a snapshot that leaked the wrong file looks exactly like +one that did not, right up until an agreement between the two halves +turns out to mean nothing. + +Measured on 2026-08-22, staging four roles by hand: the "given" and +"withheld" sets were a prose table in the plan, implemented with `cp` +and `rm`, and checked afterwards with two ad-hoc `find` commands that +had to be remembered. The same round, the conductor firewalled one +side of a pair and the planner had specified both -- so the hand +version was not merely unproven, it was wrong. + +So the manifest is the input, staging is derived from it, and the +proof is not optional: + + m = load(path) + stage(m, "extractor", dest) # copies exactly the given set + prove(m, "extractor", dest) # raises unless withheld is absent + +`stage` refuses to return a directory it cannot prove. There is no +argument that skips the check, because a caller in a hurry is exactly +who would pass it. +""" + +from __future__ import annotations + +import fnmatch +import json +import shutil +from pathlib import Path + +# -------------------------------------------------------------------- +# The manifest +# -------------------------------------------------------------------- +# +# { +# "source": ".", # tree the given patterns resolve against +# "roles": { +# "extractor": { +# "harness": "claude", +# "model": "opus", "effort": "xhigh", +# "sandbox": "workspace-write", +# "given": ["ops/devlane/workflow/**", "SCOPE.md", "SPEC.md"], +# "withheld": ["**/PLAN.md", "cue/**", "faults/**"], +# "returns": ["contracts/extract/*.py", "EXTRACTION-NOTES.md"] +# } +# } +# } +# +# `given` and `withheld` are both globs over the staged tree. They are +# allowed to overlap: "everything under the app, except its spec" is the +# common case, and expressing it as an exception is clearer than +# enumerating 48 paths. WITHHELD ALWAYS WINS -- a file matching both is +# not staged. Reversing that precedence would make a broad `given` able +# to silently defeat a narrow `withheld`, which is the failure this +# module exists to prevent. + + +# Build noise: present in a working tree, never part of what a role was +# meant to read. Staging a working tree instead of a git archive picked up +# 38 __pycache__ files and a lock file alongside the 49 real ones. +# +# Excluding them by default is right, but a rule that silently drops files +# is the very thing this module exists to prevent -- so plan_files REPORTS +# what noise it dropped rather than quietly omitting it, and a manifest can +# set "include_noise": true to switch the rule off. A `given` pattern that +# names a noise path explicitly also wins, so nothing is unreachable. +NOISE = ["**/__pycache__/**", "**/*.pyc", "**/*.pyo", "**/*.lock", + "**/.DS_Store", "**/*.egg-info/**", "**/.pytest_cache/**"] + + +class FirewallBreach(Exception): + """A withheld pattern matched a file that was staged anyway. + + Not a warning. A snapshot in this state produces findings that + cannot be trusted, and the cheapest moment to stop is before the + dispatch rather than after reading its report. + """ + + +class ManifestError(Exception): + """The manifest does not describe a role that can be staged.""" + + +def load(path): + m = json.loads(Path(path).read_text()) + for name, role in m.get("roles", {}).items(): + for key in ("given", "withheld"): + if key not in role: + raise ManifestError( + f"role {name!r} has no {key!r}. An absent withheld list " + f"is not an empty one -- say [] to mean 'nothing is " + f"withheld' so that the intent is on the record.") + return m + + +def _role(manifest, name): + try: + return manifest["roles"][name] + except KeyError: + raise ManifestError( + f"no role {name!r} in manifest; have " + f"{sorted(manifest.get('roles', {}))}") from None + + +def _matches(rel, patterns): + """True if `rel` matches any glob. + + A directory pattern is taken to mean everything beneath it, so + `cue/**` covers `cue/schema.cue` and a bare `cue` does too. Being + generous here is the safe direction: over-matching a WITHHELD + pattern withholds too much, which fails loudly when the role finds + a file missing. Under-matching leaks, which fails silently. + """ + for pat in patterns: + if fnmatch.fnmatch(rel, pat): + return True + if fnmatch.fnmatch(rel, pat.rstrip("/") + "/**"): + return True + # fnmatch's `*` crosses separators, but `**/x` does not match a + # bare `x` at the root; handle that spelling explicitly. + if pat.startswith("**/") and fnmatch.fnmatch(rel, pat[3:]): + return True + return False + + +def plan_files(manifest, name, source=None): + """(staged, withheld, noise) — copied, firewalled out, dropped as build noise. + + Pure: reads the source tree, writes nothing. Call it to see what a + dispatch would contain before making one. + """ + role = _role(manifest, name) + src = Path(source or manifest.get("source", ".")) + drop_noise = not manifest.get("include_noise", False) + staged, withheld, noise = [], [], [] + for p in sorted(src.rglob("*")): + if not p.is_file(): + continue + rel = str(p.relative_to(src)) + if any(part == ".git" for part in p.relative_to(src).parts): + continue + if not _matches(rel, role["given"]): + continue + if _matches(rel, role["withheld"]): + withheld.append(rel) # given, but explicitly excluded + elif drop_noise and _matches(rel, NOISE) and rel not in role["given"]: + noise.append(rel) # dropped, and said so + else: + staged.append(rel) + return staged, withheld, noise + + +def prove(manifest, name, dest): + """Assert the firewall held. Raises FirewallBreach if it did not. + + Two properties, and the second is the one a byte-count would miss: + + - nothing matching a withheld pattern is present, and + - something matching each given pattern IS present. + + The second exists because an empty snapshot trivially satisfies the + first. A role staged from a mistyped path would leak nothing and + also contain nothing, and the agent would then answer a question + about an empty directory -- confidently, and in whichever direction + happens to be wrong. That is the same shape as a planted fault that + silently failed to plant. + """ + role = _role(manifest, name) + dest = Path(dest) + present = [str(p.relative_to(dest)) for p in sorted(dest.rglob("*")) + if p.is_file() and ".git" not in p.relative_to(dest).parts] + + leaked = [r for r in present if _matches(r, role["withheld"])] + if leaked: + raise FirewallBreach( + f"role {name!r}: {len(leaked)} withheld file(s) were staged: " + f"{leaked[:5]}{'...' if len(leaked) > 5 else ''}") + + if not present: + raise FirewallBreach( + f"role {name!r}: nothing was staged. An empty snapshot " + f"withholds everything and proves nothing.") + + unmet = [pat for pat in role["given"] + if not any(_matches(r, [pat]) for r in present)] + if unmet: + raise FirewallBreach( + f"role {name!r}: given pattern(s) matched no staged file: " + f"{unmet}. Either the pattern is wrong or withheld swallowed " + f"it; both produce a role missing what it was promised.") + return present + + +def stage(manifest, name, dest, source=None): + """Build the snapshot and prove it. Returns the staged file list. + + The proof runs here, not as a step a caller may skip. + """ + _role(manifest, name) # fail on an unknown role before touching disk + src = Path(source or manifest.get("source", ".")) + dest = Path(dest) + if dest.exists() and any(dest.iterdir()): + raise ManifestError( + f"{dest} is not empty; stage into a fresh directory so that " + f"what is present is what this manifest put there.") + staged, _withheld, _noise = plan_files(manifest, name, src) + for rel in staged: + out = dest / rel + out.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src / rel, out) + return prove(manifest, name, dest) + + +def dispatch_spec(manifest, name): + """The harness, model, effort and sandbox this role is dispatched with. + + Kept next to the firewall on purpose: `isolation.py` decides what a + harness may bring from the operator's machine, this decides what the + snapshot contains, and both have to be right for one dispatch to + mean anything. + """ + role = _role(manifest, name) + missing = [k for k in ("harness",) if k not in role] + if missing: + raise ManifestError(f"role {name!r} lacks {missing}") + return {k: role.get(k) for k in + ("harness", "model", "effort", "sandbox", "returns")} diff --git a/ops/devlane/harness/tests/support.py b/ops/devlane/harness/tests/support.py new file mode 100644 index 0000000..768988e --- /dev/null +++ b/ops/devlane/harness/tests/support.py @@ -0,0 +1,21 @@ +"""Put the harness modules on `sys.path` so a test can import them by name. + +`unittest discover -s ops/devlane/harness/tests` inserts the START directory +on the path, not its parent, so `import probe` fails from here without +this. The workflow suite solves it the same way, and its import order is +load-bearing for the same reason: `import support` must come first. + +This exists because of a gap in a CONTRACT, not a gap in a test. An +independent author was handed the modules' promises and told to import +them "the way the guide describes" — and the guide describes a RUN +command, never an import path. Two authors of one seam, given the shape +of the data and not the shape of the door: the same defect this app was +built to make impossible, committed while staging the work to test it. +""" + +import sys +from pathlib import Path + +HARNESS = Path(__file__).resolve().parents[1] +if str(HARNESS) not in sys.path: + sys.path.insert(0, str(HARNESS)) diff --git a/ops/devlane/harness/tests/test_isolation.py b/ops/devlane/harness/tests/test_isolation.py new file mode 100644 index 0000000..9b25e74 --- /dev/null +++ b/ops/devlane/harness/tests/test_isolation.py @@ -0,0 +1,207 @@ +"""A minimal home has to start empty, and the controls have to be generated. + +`build_home` is the constructor of the thing this app exists to produce. +It took a root and called `mkdir(exist_ok=True)`, so a REUSED directory +kept whatever was in it and the function returned normally — a home +carrying the operator's `hooks.json`, `AGENTS.md` and `skills/` could be +handed straight to a dispatch, through the constructor of the module +written to strip exactly those (Codex, PR #40). +""" + +import importlib.util +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +HARNESS = Path(__file__).resolve().parents[1] + + +def load(name): + spec = importlib.util.spec_from_file_location(name, HARNESS / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class AMinimalHomeStartsEmpty(unittest.TestCase): + def setUp(self): + self.iso = load("isolation") + home = Path(tempfile.mkdtemp(prefix="fake-operator-home-")) + (home / ".codex").mkdir() + (home / ".codex" / "auth.json").write_text("{}") + self.env = {"HOME": str(home)} + + def test_a_reused_root_is_refused(self): + root = Path(tempfile.mkdtemp(prefix="reused-")) + (root / "hooks.json").write_text('{"SessionStart": "anything"}') + with self.assertRaises(self.iso.NotIsolated) as caught: + self.iso.build_home("codex", root, env=self.env) + self.assertIn("not empty", str(caught.exception)) + # The reason has to name what it found, or the operator cannot + # tell a stale scratch directory from a bug in the launcher. + self.assertIn("hooks.json", str(caught.exception)) + + def test_a_fresh_empty_root_holds_only_the_credentials(self): + root = Path(tempfile.mkdtemp(prefix="fresh-")) + home = Path(self.iso.build_home("codex", root, env=self.env)) + self.assertEqual(sorted(p.name for p in home.iterdir()), + ["auth.json"]) + + def test_a_root_that_does_not_exist_yet_is_fine(self): + # The launcher names a path under a temp dir before creating it; + # refusing that would make the guard unusable and it would be + # removed, which protects nothing. + root = Path(tempfile.mkdtemp(prefix="parent-")) / "not-yet" + home = Path(self.iso.build_home("codex", root, env=self.env)) + self.assertEqual(sorted(p.name for p in home.iterdir()), + ["auth.json"]) + + def test_a_root_that_is_a_file_is_refused(self): + path = Path(tempfile.mkdtemp(prefix="file-")) / "occupied" + path.write_text("not a directory") + with self.assertRaises(self.iso.NotIsolated): + self.iso.build_home("codex", path, env=self.env) + + +class AnExplicitEnvironmentIsNotQuietlyCompleted(unittest.TestCase): + """Passing an env is how a caller says "this, and nothing of mine". + + `_real_home` filled a missing HOME from `Path.home()`, so a caller + that controlled the environment and forgot HOME silently read the + OPERATOR's home and linked their live credential into a "minimal" + one. Found by hand while diagnosing an independent author's tests — + one of theirs passed on this machine and would have failed on a + clean one — and it survived a mutation run afterwards, because + their tests set the harness's own home variable and never take this + path. So the pin is written here by the person who found it, and + that is worth saying rather than leaving the coverage looking + accidental. + """ + + def setUp(self): + self.iso = load("isolation") + + def test_an_explicit_env_naming_no_home_at_all_is_refused(self): + with self.assertRaises(self.iso.NotIsolated) as caught: + self.iso.build_home("codex", tempfile.mkdtemp(prefix="fresh-"), + env={"PATH": "/usr/bin"}) + reason = str(caught.exception) + self.assertIn("CODEX_HOME", reason) + self.assertIn("HOME", reason) + # The refusal must not be mistaken for the credential one: they + # send a reader to entirely different places. + self.assertNotIn("credential auth.json is absent", reason) + + def test_the_operator_home_is_never_the_answer_to_an_explicit_env(self): + for harness in ("codex", "grok"): + with self.subTest(harness=harness), \ + self.assertRaises(self.iso.NotIsolated): + self.iso._real_home(harness, {"PATH": "/usr/bin"}, given=True) + + def test_an_empty_home_is_absent_not_a_relative_path(self): + # Requested by the reviewer who found it, and written to their + # recipe: HOME="" with a planted ./.codex/auth.json under the + # working directory. `Path("") / ".codex"` is the RELATIVE path + # `.codex`, so the lookup reached ambient project state — the + # one thing an explicit environment exists to exclude. + work = Path(tempfile.mkdtemp(prefix="cwd-with-a-dotdir-")) + planted = work / ".codex" / "auth.json" + planted.parent.mkdir() + planted.write_text("a credential that is not ours\n", encoding="utf-8") + self.assertTrue(planted.exists(), "the plant did not land") + + here = os.getcwd() + os.chdir(work) + try: + with self.assertRaises(self.iso.NotIsolated): + self.iso.build_home("codex", + tempfile.mkdtemp(prefix="fresh-"), + env={"HOME": ""}) + with self.assertRaises(self.iso.NotIsolated): + self.iso._real_home("codex", {"HOME": ""}, given=True) + finally: + os.chdir(here) + # The plant is still there: the refusal must be inert, or a + # guard that also deletes is worse than the hole. + self.assertTrue(planted.exists()) + + def test_an_empty_home_is_absent_when_dispatching_the_environment(self): + # The same shape one function along. `home=""` would emit + # CODEX_HOME="", which the harness reads as UNSET and answers by + # loading the operator's real home — the leak, spelled with an + # empty string instead of a missing key. + for empty in ("", None): + with self.subTest(home=empty): + with self.assertRaises(self.iso.NotIsolated): + self.iso.dispatch_env("codex", home=empty, env={}) + + def test_a_credential_link_that_does_not_resolve_is_refused(self): + # Surfaced while reproducing the empty-HOME finding: a relative + # source made `symlink_to` point inside the minimal home, and + # `build_home` returned a home containing a DANGLING auth.json. + # An entry with the right name is not the credential. + operator = Path(tempfile.mkdtemp(prefix="operator-home-")) + (operator / ".codex").mkdir() + (operator / ".codex" / "auth.json").write_text("{}", encoding="utf-8") + root = Path(tempfile.mkdtemp(prefix="fresh-")) + home = Path(self.iso.build_home("codex", root, + env={"HOME": str(operator)})) + link = home / "auth.json" + self.assertTrue(link.is_symlink()) + self.assertTrue(link.resolve().exists(), + "the credential link must resolve to a real file") + self.assertEqual(link.read_text(encoding="utf-8"), "{}") + + def test_a_relative_home_variable_still_links_to_the_real_file(self): + # The remaining way `src` can be relative, now that an empty + # HOME is refused. `symlink_to` resolves a relative source + # against the LINK's directory, not the caller's, so the home + # ended up holding a DANGLING auth.json pointing inside itself — + # complete-looking, and an authentication failure at dispatch. + work = Path(tempfile.mkdtemp(prefix="cwd-")) + (work / "elsewhere").mkdir() + (work / "elsewhere" / "auth.json").write_text("{}", encoding="utf-8") + here = os.getcwd() + os.chdir(work) + try: + root = Path(tempfile.mkdtemp(prefix="fresh-")) / "home" + home = Path(self.iso.build_home( + "codex", root, env={"CODEX_HOME": "elsewhere"})) + link = home / "auth.json" + self.assertTrue(link.is_symlink()) + self.assertTrue(link.resolve().exists(), + f"{link} dangles: -> {link.resolve()}") + self.assertEqual((work / "elsewhere" / "auth.json").resolve(), + link.resolve()) + finally: + os.chdir(here) + + def test_no_env_at_all_still_falls_back_to_the_real_home(self): + # The twin. Refusing here would break every ordinary call, and a + # guard that breaks the normal path gets removed. Only the SHAPE + # is asserted -- reading the operator's actual files would make + # this test a fact about one machine. + where = self.iso._real_home("codex", {}, given=False) + self.assertEqual(".codex", where.name) + self.assertEqual(Path.home(), where.parent) + + +class TheControlsAreGeneratedNotEdited(unittest.TestCase): + """A control edited by hand is how the positive one came to encode + five untruths about the module it describes. `build.py --check` + regenerates and compares, so drift is a failure rather than a + surprise.""" + + def test_every_control_matches_what_build_py_generates(self): + proc = subprocess.run( + [sys.executable, str(HARNESS / "controls" / "build.py"), "--check"], + capture_output=True, text=True, check=False) + self.assertEqual(proc.returncode, 0, + proc.stdout + proc.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/harness/tests/test_isolation_contract.py b/ops/devlane/harness/tests/test_isolation_contract.py new file mode 100644 index 0000000..b7d5511 --- /dev/null +++ b/ops/devlane/harness/tests/test_isolation_contract.py @@ -0,0 +1,345 @@ +"""Contract tests for isolation.py. + +These tests intentionally use only the public promises recorded in +CONTRACT-isolation.py.md and GUIDE.md. +""" + +from __future__ import annotations + +import json +from pathlib import Path +import shlex +import subprocess +import sys +import tempfile +import unittest + +import support # noqa: F401 (puts the harness dir on sys.path) + +import isolation + + +EXPECTED_HARNESSES = { + "claude": { + "mechanism": "flags", + "flags": [ + "--setting-sources", + "project,local", + "--strict-mcp-config", + "--disable-slash-commands", + ], + "home_env": None, + "auth_files": [], + "probe_phrase": "Start from the class, not the instance", + "sessions": {"under": "real", "path": ".claude/projects"}, + "measured": { + "on": "2026-08-22", + "version": "unrecorded", + "probe_default": "YES", + "probe_isolated": "NO", + "attachment_bytes_default": 17304, + "attachment_bytes_isolated": 2761, + "note": "what remains is Claude Code's own built-in machinery", + }, + }, + "codex": { + "mechanism": "home", + "flags": [], + "home_env": "CODEX_HOME", + "auth_files": ["auth.json"], + "probe_phrase": None, + "sessions": {"under": "minimal", "path": "sessions"}, + "measured": { + "on": "2026-08-22", + "version": "0.148.0", + "leak": "~/.codex/hooks.json SessionStart ran projects/xormania/xor/tools/xortations/hooks/session_start.py", + "note": "personal MCP servers and a memories store also live under the home", + }, + }, + "grok": { + "mechanism": "home", + "flags": [], + "home_env": "GROK_HOME", + "also_env": ["HOME"], + "auth_files": ["auth.json"], + "probe_phrase": None, + "sessions": {"under": "minimal", "path": "sessions"}, + "measured": { + "on": "2026-08-22", + "version": "1.0.5", + "leak": "grok inspect listed ~/.claude/CLAUDE.md (~7012 tokens) and ~/.grok/rules/00-xortations-first-turn.md (~161 tokens) as project instructions; 31 skills, 24 user-scoped", + "note": "HOME alone drops CLAUDE.md and cuts skills 31 -> 7; GROK_HOME alone drops the rules file; both are needed", + }, + }, +} + + +class IsolationContractTests(unittest.TestCase): + def test_registry_is_exactly_the_measured_contract(self): + self.assertEqual(EXPECTED_HARNESSES, isolation.HARNESSES) + + def test_real_home_uses_each_home_mechanism_harness_environment(self): + with tempfile.TemporaryDirectory() as temporary: + base = Path(temporary) + cases = { + "codex": ("CODEX_HOME", base / "operator-codex"), + "grok": ("GROK_HOME", base / "operator-grok"), + } + self.assertEqual(2, len(cases)) + for harness, (variable, expected) in cases.items(): + with self.subTest(harness=harness): + actual = Path(isolation._real_home(harness, {variable: str(expected)})) + self.assertEqual(expected, actual) + + def test_build_home_links_every_credential_and_nothing_else(self): + cases = (("codex", "CODEX_HOME"), ("grok", "GROK_HOME")) + self.assertEqual(2, len(cases)) + for harness, home_variable in cases: + with self.subTest(harness=harness), tempfile.TemporaryDirectory() as temporary: + base = Path(temporary) + real_home = base / "operator-home" + root = base / "scratch" + real_home.mkdir() + root.mkdir() + + credential = real_home / "auth.json" + credential.write_text('{"fixture":"recognisable-auth"}\n', encoding="utf-8") + planted_hook = real_home / "hooks.json" + planted_hook.write_text("personal-session-hook\n", encoding="utf-8") + planted_skill = real_home / "skills" / "personal" / "SKILL.md" + planted_skill.parent.mkdir(parents=True) + planted_skill.write_text("personal-skill\n", encoding="utf-8") + + # Prove the dirty source is still recognisably the clean fixture. + source_names = {entry.name for entry in real_home.iterdir()} + self.assertEqual(3, len(source_names)) + self.assertIn("auth.json", source_names) + self.assertIn("hooks.json", source_names) + self.assertEqual("personal-session-hook\n", planted_hook.read_text(encoding="utf-8")) + + minimal = Path( + isolation.build_home( + harness, + root, + env={home_variable: str(real_home)}, + ) + ) + + self.assertTrue(minimal.is_dir()) + self.assertTrue(minimal == root or root in minimal.parents) + entries = list(minimal.iterdir()) + self.assertEqual(1, len(entries)) + self.assertEqual("auth.json", entries[0].name) + self.assertTrue(entries[0].is_symlink()) + self.assertEqual(credential.resolve(), entries[0].resolve()) + self.assertEqual(credential.read_bytes(), entries[0].read_bytes()) + + def test_build_home_refuses_a_missing_credential_without_leaving_a_home(self): + with tempfile.TemporaryDirectory() as temporary: + base = Path(temporary) + real_home = base / "operator-home" + root = base / "scratch" + real_home.mkdir() + root.mkdir() + marker = real_home / "recognisable-operator-file" + marker.write_text("operator-home-without-auth\n", encoding="utf-8") + before = list(root.iterdir()) + self.assertEqual(0, len(before)) + self.assertFalse((real_home / "auth.json").exists()) + self.assertEqual("operator-home-without-auth\n", marker.read_text(encoding="utf-8")) + + with self.assertRaises(isolation.NotIsolated) as raised: + isolation.build_home( + "codex", + root, + env={"CODEX_HOME": str(real_home)}, + ) + + self.assertIn("auth.json", str(raised.exception)) + self.assertEqual(before, list(root.iterdir())) + + def test_reusing_a_root_does_not_reuse_a_contaminated_minimal_home(self): + with tempfile.TemporaryDirectory() as temporary: + base = Path(temporary) + real_home = base / "operator-home" + root = base / "reused-root" + real_home.mkdir() + root.mkdir() + credential = real_home / "auth.json" + credential.write_text("recognisable-auth\n", encoding="utf-8") + env = {"CODEX_HOME": str(real_home)} + + first_home = Path(isolation.build_home("codex", root, env=env)) + contaminant = first_home / "instructions.md" + contaminant.write_text("planted-personal-doctrine\n", encoding="utf-8") + dirty_entries = list(first_home.iterdir()) + self.assertEqual(2, len(dirty_entries)) + self.assertTrue((first_home / "auth.json").is_symlink()) + self.assertEqual("planted-personal-doctrine\n", contaminant.read_text(encoding="utf-8")) + + # EDITED BY THE IMPLEMENTATION'S AUTHOR, and the edit is + # named rather than quietly made. This test was written from + # a contract that did not state what a REUSED root does, so + # it assumed the reasonable thing: rebuild it clean. The + # code refuses instead, which satisfies this test's own pin + # -- "reusing a root cannot preserve planted instructions" + # -- more strongly than a rebuild would, because a rebuild + # has to be right about what to delete and a refusal does + # not. The docstring says so now; it did not when this was + # written, and the silence is the finding. + with self.assertRaises(isolation.NotIsolated) as raised: + isolation.build_home("codex", root, env=env) + + # The refusal has to name what it found, or a stale scratch + # directory and a launcher bug read the same. + self.assertIn("not empty", str(raised.exception)) + self.assertIn("instructions.md", str(raised.exception)) + + # And it must not have touched the contamination on its way + # out: refusing is only safe if it is also inert. + self.assertEqual("planted-personal-doctrine\n", + contaminant.read_text(encoding="utf-8")) + + def test_dispatch_env_sets_all_and_only_the_promised_overrides(self): + home = "/tmp/recognisable-minimal-home" + cases = { + "claude": {}, + "codex": {"CODEX_HOME": home}, + "grok": {"GROK_HOME": home, "HOME": home}, + } + self.assertEqual(3, len(cases)) + for harness, expected in cases.items(): + with self.subTest(harness=harness): + actual = isolation.dispatch_env(harness, home=home, env={"KEEP": "caller"}) + self.assertEqual(expected, actual) + + def test_dispatch_flags_are_exact_and_are_not_shared_mutable_state(self): + expected = EXPECTED_HARNESSES["claude"]["flags"] + first = isolation.dispatch_flags("claude") + self.assertEqual(expected, first) + self.assertEqual([], isolation.dispatch_flags("codex")) + self.assertEqual([], isolation.dispatch_flags("grok")) + + marker = "--planted-mutation" + first.append(marker) + try: + self.assertEqual(expected, isolation.dispatch_flags("claude")) + finally: + first.remove(marker) + + def test_isolated_returns_complete_configuration_for_each_mechanism(self): + with tempfile.TemporaryDirectory() as temporary: + base = Path(temporary) + claude_root = base / "claude-root" + claude_root.mkdir() + claude_env, claude_flags = isolation.isolated("claude", claude_root, env={}) + self.assertEqual({}, claude_env) + self.assertEqual(EXPECTED_HARNESSES["claude"]["flags"], claude_flags) + self.assertEqual(0, len(list(claude_root.iterdir()))) + + real_home = base / "operator-codex" + codex_root = base / "codex-root" + real_home.mkdir() + codex_root.mkdir() + credential = real_home / "auth.json" + credential.write_text("recognisable-auth\n", encoding="utf-8") + codex_env, codex_flags = isolation.isolated( + "codex", + codex_root, + env={"CODEX_HOME": str(real_home)}, + ) + + self.assertEqual([], codex_flags) + self.assertEqual({"CODEX_HOME"}, set(codex_env)) + minimal = Path(codex_env["CODEX_HOME"]) + entries = list(minimal.iterdir()) + self.assertEqual(1, len(entries)) + self.assertTrue(entries[0].is_symlink()) + self.assertEqual(credential.resolve(), entries[0].resolve()) + + def test_every_isolation_entry_point_refuses_an_unregistered_harness(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + calls = ( + lambda: isolation.build_home("unknown-harness", root, env={}), + lambda: isolation.dispatch_env("unknown-harness", home=root, env={}), + lambda: isolation.dispatch_flags("unknown-harness"), + lambda: isolation.isolated("unknown-harness", root, env={}), + ) + self.assertEqual(4, len(calls)) + for call in calls: + with self.subTest(call=call): + with self.assertRaises(isolation.NotIsolated) as raised: + call() + self.assertIn("unknown-harness", str(raised.exception)) + self.assertEqual(0, len(list(root.iterdir()))) + + def test_report_is_json_containing_exactly_the_registry(self): + rendered = isolation.report() + self.assertIsInstance(rendered, str) + self.assertEqual(EXPECTED_HARNESSES, json.loads(rendered)) + + def test_shell_entry_point_refuses_unknown_harness_with_exit_78_and_reason(self): + with tempfile.TemporaryDirectory() as temporary: + completed = subprocess.run( + [ + sys.executable, + str(Path(isolation.__file__).resolve()), + "--sh", + "unknown-harness", + temporary, + ], + text=True, + capture_output=True, + check=False, + ) + + self.assertEqual(78, completed.returncode) + explanation = completed.stdout + completed.stderr + self.assertIn("unknown-harness", explanation) + self.assertTrue( + any(word in explanation.lower() for word in ("isolat", "unknown", "refus")), + explanation, + ) + + def test_shell_entry_point_output_is_eval_safe_and_complete(self): + with tempfile.TemporaryDirectory(prefix="isolation root ' with spaces ") as temporary: + completed = subprocess.run( + [ + sys.executable, + str(Path(isolation.__file__).resolve()), + "--sh", + "claude", + temporary, + ], + text=True, + capture_output=True, + check=False, + ) + + self.assertEqual(0, completed.returncode, completed.stderr) + shell_program = ( + "set -u\n" + + completed.stdout + + "\nprintf '%s\\n%s\\n%s\\n' " + '"$ISO_ENV" "$ISO_FLAGS" "$ISO_STORE"\n' + ) + evaluated = subprocess.run( + ["/bin/sh", "-c", shell_program], + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(0, evaluated.returncode, evaluated.stderr) + values = {} + for line in evaluated.stdout.splitlines(): + if line.startswith("<") and ">" in line: + key, value = line[1:].split(">", 1) + values[key] = value + self.assertEqual(3, len(values)) + self.assertEqual({"ENV", "FLAGS", "STORE"}, set(values)) + self.assertEqual(EXPECTED_HARNESSES["claude"]["flags"], shlex.split(values["FLAGS"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/harness/tests/test_probe.py b/ops/devlane/harness/tests/test_probe.py new file mode 100644 index 0000000..9fcc0de --- /dev/null +++ b/ops/devlane/harness/tests/test_probe.py @@ -0,0 +1,121 @@ +"""The probe reports INVALID when an arm did not run. + +This app's whole argument is that "nobody looked" must never print the +same way as "we checked and it was fine", and the probe is where that +is decided: a harness that died before loading anything shows no leak, +and reading that as isolation working is a test proving its own setup +never happened. +""" + +import importlib.util +import unittest +from unittest import mock +from pathlib import Path + +HARNESS = Path(__file__).resolve().parents[1] + +#: What `grok inspect` prints when the operator's files ARE loaded, and +#: when none are. The second is a WELL-FORMED answer, which is exactly +#: why a nonzero exit beside it is dangerous. +LEAKING = ("Project Instructions (2)\n" + "├ /home/op/.claude/CLAUDE.md\n" + "└ /home/op/rules.md\n") +NOTHING = "Project Instructions (0)\n└ (none)\n" + + +def load(name): + spec = importlib.util.spec_from_file_location(name, HARNESS / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class AnArmThatDidNotRunIsNotAnAnswer(unittest.TestCase): + def setUp(self): + self.probe = load("probe") + # `load` gives a fresh MODULE, but `probe.subprocess` and + # `probe.isolation` are the same objects every other test in the + # process holds. Assigning through them mutates global state and + # never puts it back: these tests passed alone and broke an + # independent author's the moment both ran in one discovery, + # which is the only way anyone would have noticed. + self.enterContext( + mock.patch.object( + self.probe.isolation, "isolated", + lambda harness, root, env=None: ({"HOME": root}, []))) + + def verdict(self, isolated_exit, isolated_out=NOTHING): + """Run both arms; the unisolated one always succeeds and leaks. + + The PROCESS is faked, not `_run`. The nonzero-exit rule lives + inside `_run` now — deliberately, so a new arm cannot be written + without it — and a test that replaces `_run` would step over the + very rule it is checking and pass either way. + """ + calls = {"n": 0} + + class Done: + def __init__(self, rc, out, err): + self.returncode, self.stdout, self.stderr = rc, out, err + + def fake(argv, **kwargs): + calls["n"] += 1 + if calls["n"] == 1: + return Done(0, LEAKING, "") + if isolated_exit is None: + raise OSError("the harness could not be started") + return Done(isolated_exit, isolated_out, + "grok: could not read config") + + self.enterContext( + mock.patch.object(self.probe.subprocess, "run", fake)) + result = self.probe.behavioural_grok(env={}) + self.assertEqual(calls["n"], 2, "both arms must have been attempted") + return result + + def test_a_nonzero_exit_is_invalid_even_when_stdout_parses(self): + # A grok that failed to initialise still prints a well-formed + # `Project Instructions (0)` header, and taking that as an + # answer certified isolation off a broken probe. + self.assertEqual(self.verdict(1).verdict, self.probe.INVALID) + + def test_a_clean_run_still_reports_clean(self): + # The twin. A guard that turned a working probe into INVALID + # would trade a false pass for a false alarm, and the only + # free isolation check we have would stop being run. + self.assertEqual(self.verdict(0).verdict, self.probe.CLEAN) + + def test_a_leak_is_still_reported_as_leaking(self): + self.assertEqual(self.verdict(0, LEAKING).verdict, self.probe.LEAKING) + + def test_the_claude_arm_refuses_a_nonzero_exit_too(self): + # The rule was fixed in the grok arm and left in this one, and + # the arm that was missed is the one the next reviewer found + # (Codex, PR #40, twice). It lives in `_run` now, so a third arm + # cannot be written without it. + class Done: + def __init__(self, rc, out): + self.returncode, self.stdout, self.stderr = rc, out, "" + + for exit_code, want in ((1, self.probe.INVALID), + (0, self.probe.CLEAN)): + with self.subTest(exit_code=exit_code): + calls = {"n": 0} + + def fake(argv, exit_code=exit_code, calls=calls, **kwargs): + calls["n"] += 1 + return Done(exit_code, "YES" if calls["n"] == 1 else "NO") + + self.enterContext( + mock.patch.object(self.probe.subprocess, "run", fake)) + self.assertEqual( + self.probe.behavioural_claude(env={}).verdict, want) + + def test_a_transport_failure_is_invalid(self): + # `_run` answers None for an OSError or a timeout; the same + # branch has to cover it. + self.assertEqual(self.verdict(None).verdict, self.probe.INVALID) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/harness/tests/test_probe_contract.py b/ops/devlane/harness/tests/test_probe_contract.py new file mode 100644 index 0000000..a7afac9 --- /dev/null +++ b/ops/devlane/harness/tests/test_probe_contract.py @@ -0,0 +1,134 @@ +"""Contract tests for probe.py. + +The behavioural result APIs are deliberately not guessed here: their return +shape is absent from CONTRACT-probe.py.md. The documented process boundary, +where an unsuccessful arm becomes "no answer", is tested directly. +""" + +from __future__ import annotations + +import os +from pathlib import Path +import sys +import tempfile +import unittest +import uuid + +import support # noqa: F401 (puts the harness dir on sys.path) + +import probe + + +class ProbeContractTests(unittest.TestCase): + def test_verdict_tokens_are_three_distinct_states(self): + verdicts = (probe.CLEAN, probe.LEAKING, probe.INVALID) + self.assertEqual(("CLEAN", "LEAKING", "INVALID"), verdicts) + self.assertEqual(3, len(set(verdicts))) + + def test_run_executes_in_the_requested_environment_and_returns_all_streams(self): + with tempfile.TemporaryDirectory(prefix="probe cwd with spaces ") as temporary: + cwd = Path(temporary) + env = dict(os.environ) + env["PROBE_TEST_SENTINEL"] = "recognisable-environment" + script = ( + "import os, pathlib, sys; " + "data = sys.stdin.read(); " + "pathlib.Path('ran.receipt').write_text(data); " + "print(os.environ['PROBE_TEST_SENTINEL'] + ':' + data); " + "print('recognisable-stderr', file=sys.stderr)" + ) + + rc, stdout, stderr = probe._run( + [sys.executable, "-c", script], + env, + cwd, + stdin="recognisable-stdin", + ) + + self.assertEqual(0, rc) + self.assertEqual("recognisable-environment:recognisable-stdin\n", stdout) + self.assertEqual("recognisable-stderr\n", stderr) + receipt = cwd / "ran.receipt" + self.assertTrue(receipt.is_file()) + self.assertEqual("recognisable-stdin", receipt.read_text(encoding="utf-8")) + + def test_run_maps_good_looking_output_followed_by_nonzero_to_no_answer(self): + cases = ( + ("Project Instructions (0)\n", 23), + ("YES\n", 24), + ("NO\n", 25), + ) + self.assertEqual(3, len(cases)) + for apparent_answer, exit_status in cases: + with self.subTest(apparent_answer=apparent_answer.strip()): + with tempfile.TemporaryDirectory() as temporary: + cwd = Path(temporary) + script = ( + "import pathlib, sys; " + f"sys.stdout.write({apparent_answer!r}); " + "sys.stdout.flush(); " + "pathlib.Path('arm-ran.receipt').write_text('recognisable-arm'); " + "sys.stderr.write('teardown failed\\n'); " + f"raise SystemExit({exit_status})" + ) + + rc, stdout, stderr = probe._run( + [sys.executable, "-c", script], + dict(os.environ), + cwd, + ) + + receipt = cwd / "arm-ran.receipt" + self.assertTrue(receipt.is_file()) + self.assertEqual("recognisable-arm", receipt.read_text(encoding="utf-8")) + self.assertIsNone(rc) + self.assertEqual(apparent_answer, stdout) + self.assertEqual("teardown failed\n", stderr) + + def test_run_reports_no_answer_when_the_arm_never_starts(self): + missing_command = f"probe-command-that-does-not-exist-{uuid.uuid4().hex}" + with tempfile.TemporaryDirectory() as temporary: + cwd = Path(temporary) + self.assertEqual(0, len(list(cwd.iterdir()))) + + rc, stdout, stderr = probe._run( + [missing_command], + dict(os.environ), + cwd, + ) + + self.assertIsNone(rc) + self.assertEqual("", stdout) + self.assertIsInstance(stderr, str) + self.assertNotEqual("", stderr.strip()) + self.assertEqual(0, len(list(cwd.iterdir()))) + + def test_structural_probe_is_read_only_even_with_personal_extras_present(self): + with tempfile.TemporaryDirectory() as temporary: + real_home = Path(temporary) / "operator-codex" + real_home.mkdir() + credential = real_home / "auth.json" + credential.write_text("recognisable-auth\n", encoding="utf-8") + planted_hook = real_home / "hooks.json" + planted_hook.write_text("personal-hook\n", encoding="utf-8") + before = { + path.relative_to(real_home): (path.is_symlink(), path.read_bytes()) + for path in real_home.rglob("*") + if path.is_file() or path.is_symlink() + } + self.assertEqual(2, len(before)) + self.assertIn(Path("auth.json"), before) + self.assertIn(Path("hooks.json"), before) + self.assertEqual(b"personal-hook\n", planted_hook.read_bytes()) + + probe.structural("codex", env={"CODEX_HOME": str(real_home)}) + + after = { + path.relative_to(real_home): (path.is_symlink(), path.read_bytes()) + for path in real_home.rglob("*") + if path.is_file() or path.is_symlink() + } + self.assertEqual(before, after) + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/harness/vet_context.py b/ops/devlane/harness/vet_context.py new file mode 100644 index 0000000..de4911a --- /dev/null +++ b/ops/devlane/harness/vet_context.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +"""Run context.cue against the controls: one that must pass, four that must not. + + python3 ops/devlane/harness/vet_context.py + +A schema with no document behind it is law nobody has read. Every +constraint in `context.cue` was written from an incident, and until this +existed none of them had ever been shown to fire -- which is the same +"nobody looked" that the schema is built to refuse. + +Two halves, and the second is the one that gets skipped: + + the positive control `controls/dispatchable.json` must VALIDATE. A + schema that rejects everything passes every + negative test perfectly. + + the negative controls each `controls/rejects/*.json` must be + rejected AND rejected AT THE FIELD its + sidecar names. A document refused for some + other reason reports a guard that is not + there: measured 2026-08-23, the stale-version + fixture was rejected the whole time with + `mechanism: conflicting values`, because the + constraint had collapsed a disjunction rather + than firing. + +Exits 0 when every control behaves, 1 when one does not, and 2 when +`cue` is absent -- refusing rather than reporting a pass it did not run. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +CONTROLS = HERE / "controls" + +#: BOTH files, always. Vetting context.cue alone left every helper +#: `#NonEmpty` undefined and reported a tidy count of failures that were +#: really "the schema did not load" -- an INVALID run wearing a result +#: (measured 2026-08-23). +SCHEMA = [str(HERE / "context.cue"), str(HERE / "evidence.cue")] + +#: Which definition a control is judged against. The positives say so by +#: name; a reject's sidecar may override it. +POSITIVE = {"dispatchable-flags.json": "#Dispatchable", + "dispatchable-home.json": "#Dispatchable", + "evidence-receipt.json": "#Admissible"} + + +#: Paths that must be PRESENT in the source document, not merely valid +#: once the law has been applied. CUE cannot express this: a constraint +#: is also a DEFAULT, so `operator_config_present: false` in the gate +#: SUPPLIES that value to a document that never mentioned it, and +#: `cue vet -c` cannot tell — after inference the field is concrete. +#: Measured 2026-08-23: 28 paths of the flags control could be deleted +#: and the result still validated (Codex, PR #40 round three, three of +#: them; the rest came from deleting every leaf in turn). +#: +#: The rule for being on this list: omitting the field lets the gate +#: MANUFACTURE the clean answer. Not "the field is important". +MUST_BE_STATED = [ + "role", + "harness.isolation.observed.operator_config_present", + "harness.isolation.observed.harness_version", + "staged.count", + "staged.given", + "staged.proof.withheld_present", + "staged.proof.given_unmet", + "task.produces", + "task.report_fields", + "unmet_requirements", + "dangling_references", +] + +#: Pairs the law unifies, which must ALSO be equal in the source. The +#: presence list is not enough for these: `task.produces` can be stated +#: as `[{"path": …}]` and CUE will copy the interface across from the +#: brief, so the document claims an agreement it never made. Comparing +#: the raw values kills the whole class in one rule, rather than +#: enumerating every leaf and going stale when a field is added +#: (predicted by a second reader as the next hole; confirmed by +#: measurement before it was written down). +MUST_MATCH_IN_SOURCE = [ + ("task.produces", "brief.deliverables"), + ("task.report_fields", "brief.report_fields"), +] + +#: Omittable and legitimately so, with the reason. A path that is +#: neither here nor above is a NEW hole, and `omittable()` reports it — +#: which is the only thing that stops this list going stale the next +#: time the law grows a constraint. +WAIVED = { + "argv": "a seam that is never invoked has no argv (#Interface, optional)", + "output": "same (#Interface, optional)", + "fields": "same (#Interface, optional)", + "declared_at": "carried by the brief's copy, which is required", + "name": "carried by the brief's copy, which is required", + "path": "carried by the brief's copy, which is required", + "interface": "carried by the brief's copy, which is required", +} + + +def _leaves(node, prefix=""): + if isinstance(node, dict): + for key, value in node.items(): + here = f"{prefix}.{key}" if prefix else key + yield here + yield from _leaves(value, here) + elif isinstance(node, list): + for i, value in enumerate(node): + here = f"{prefix}.{i}" + yield here + yield from _leaves(value, here) + + +def _at(doc, path): + node = doc + for step in path.split("."): + node = node[int(step)] if step.isdigit() else node[step] + return node + + +def _drop(doc, path): + steps = path.split(".") + node = doc + for step in steps[:-1]: + node = node[int(step)] if step.isdigit() else node[step] + last = steps[-1] + del node[int(last) if last.isdigit() else last] + + +def _present(doc, path): + node = doc + for step in path.split("."): + if isinstance(node, list): + if not step.isdigit() or int(step) >= len(node): + return False + node = node[int(step)] + elif isinstance(node, dict): + if step not in node: + return False + node = node[step] + else: + return False + return True + + +def vet(document, definition): + """(ok, output) from `cue vet` on one document.""" + proc = subprocess.run( + ["cue", "vet", "-d", definition, *SCHEMA, str(document)], + capture_output=True, text=True, check=False) + return proc.returncode == 0, (proc.stdout + proc.stderr).strip() + + +def check_stated(): + """Two halves, and the second is what stops this going stale. + + Every path on MUST_BE_STATED is present in each positive control — + otherwise the control itself is relying on the gate to supply it. + And every path that CAN be omitted while still validating is either + on that list or explicitly waived; a path on neither is a new place + where the law manufactures its own clean answer, and it is reported + as a failure rather than discovered by the next reviewer. + """ + problems = [] + for name, definition in sorted(POSITIVE.items()): + if definition != "#Dispatchable": + continue + control = CONTROLS / name + if not control.exists(): + continue + doc = json.loads(control.read_text(encoding="utf-8")) + for path in MUST_BE_STATED: + if not _present(doc, path): + problems.append(f"{name}: {path} is not stated") + + omittable = [] + for path in list(_leaves(doc)): + candidate = json.loads(json.dumps(doc)) + try: + _drop(candidate, path) + except (KeyError, IndexError, TypeError): + continue + with tempfile.NamedTemporaryFile( + "w", suffix=".json", delete=False) as handle: + json.dump(candidate, handle) + temporary = handle.name + ok, _out = vet(temporary, "#Dispatchable") + Path(temporary).unlink(missing_ok=True) + if ok: + omittable.append(path) + + unexplained = [p for p in omittable + if p not in MUST_BE_STATED + and p.rsplit(".", 1)[-1] not in WAIVED + and not p.rsplit(".", 1)[-1].isdigit()] + for path in unexplained: + problems.append( + f"{name}: {path} can be omitted and still validate, and is " + f"neither required to be stated nor waived") + print(f" {'pass' if not unexplained else 'FAIL'} {name}: " + f"{len(omittable)} omittable path(s), {len(unexplained)} " + f"unexplained") + return problems + + +def stated(document): + """Paths a dispatchable context must state, and this one does not. + + The half `cue vet` cannot do, exported so a launcher can run it + against a real context rather than only against the controls. An + empty list means the document said everything the law would + otherwise have said on its behalf. + """ + doc = json.loads(Path(document).read_text(encoding="utf-8")) + missing = [f"{path} is not stated" for path in MUST_BE_STATED + if not _present(doc, path)] + for left, right in MUST_MATCH_IN_SOURCE: + if not (_present(doc, left) and _present(doc, right)): + continue + if _at(doc, left) != _at(doc, right): + missing.append( + f"{left} and {right} are unified by the law but differ in " + f"the document, or one states less than the other") + return missing + + +def main(argv=None): + argv = sys.argv[1:] if argv is None else list(argv) + if argv[:1] == ["--stated"]: + if len(argv) != 2: + print("usage: vet_context.py --stated ", + file=sys.stderr) + return 2 + missing = stated(argv[1]) + for path in missing: + print(f" not stated: {path}", file=sys.stderr) + print(f"{len(MUST_BE_STATED)} required path(s), " + f"{len(missing)} omitted") + return 1 if missing else 0 + + if shutil.which("cue") is None: + print("cue is not on PATH: refusing rather than reporting a pass", + file=sys.stderr) + return 2 + + problems = [] + + # One positive control per isolation mechanism: a rule that only + # the home branch witnesses is a rule the flags branch never + # exercises, and both branches now carry a refusal. + problems += check_stated() + positives = sorted(p for p in CONTROLS.glob("*.json") + if p.name in POSITIVE) + missing = sorted(set(POSITIVE) - {p.name for p in positives}) + if missing: + print(f"positive control(s) missing: {', '.join(missing)}", + file=sys.stderr) + return 1 + for positive in positives: + definition = POSITIVE[positive.name] + ok, out = vet(positive, definition) + print(f" {'pass' if ok else 'FAIL'} {positive.name} must validate " + f"as {definition}") + if not ok: + problems.append(f"{positive.name} no longer validates:\n{out}") + + rejects = sorted(p for p in (CONTROLS / "rejects").glob("*.json") + if not p.name.endswith(".reason.json")) + if not rejects: + # An empty fixture set satisfies "every negative control was + # rejected" perfectly, which is the empty-snapshot defect these + # very fixtures exist to pin. + print("no negative controls found", file=sys.stderr) + return 1 + + for document in rejects: + reason = json.loads( + document.with_suffix(".reason.json").read_text(encoding="utf-8")) + want = reason["expect_path"] + if reason.get("checked_by") == "source": + # `cue vet` ACCEPTS these -- that is the defect they + # witness -- so the refusal comes from reading the document, + # and the law must accept it, or the fixture would be + # rejected for some other reason and isolate nothing. + complaints = stated(document) + valid, out = vet(document, reason.get("definition", + "#Dispatchable")) + hit = [c for c in complaints if want in c] + if not hit: + problems.append( + f"{document.name}: nothing complained about {want}; " + f"the fixture witnesses nothing. Got: {complaints}") + verdict = "FAIL no complaint" + elif not valid: + problems.append( + f"{document.name}: refused by the law for another " + f"reason, so it cannot isolate this:\n{out}") + verdict = "FAIL not isolated" + else: + verdict = "pass (source)" + else: + ok, out = vet(document, reason.get("definition", "#Dispatchable")) + if ok: + problems.append( + f"{document.name} was ACCEPTED; it must not be") + verdict = "FAIL accepted" + elif want not in out: + problems.append( + f"{document.name} was rejected, but not at {want}:\n{out}") + verdict = f"FAIL wrong path (wanted {want})" + else: + verdict = "pass" + print(f" {verdict:<34} {document.name} — {reason['change']}") + + for problem in problems: + print(f"\n{problem}", file=sys.stderr) + print(f"\n{len(rejects) + len(positives)} control(s), " + f"{len(problems)} problem(s)") + return 1 if problems else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/devlane/harness/wires.py b/ops/devlane/harness/wires.py new file mode 100644 index 0000000..38e709a --- /dev/null +++ b/ops/devlane/harness/wires.py @@ -0,0 +1,155 @@ +"""What supervises a dispatched role, and why each threshold is that number. + +`breaker.py` watches a running dispatch and terminates it when a wire +trips. The wires are good; every failure here was a threshold set from +intuition and never re-derived. So the thresholds live as data with the +measurement behind them, and re-deriving is running one function rather +than remembering a conversation. + +TWO JOBS, AND THEY ARE NOT THE SAME. Confusing them is what caused +every misfire on 2026-08-22/23: + + runaway detection a loop, a storm of failures, a hang. These have a + SHAPE -- repetition, error density, silence -- and + shape is what detects them. A runaway is + repetitive; a thorough job is not. + + budget backstop "this has produced more than any legitimate run of + its kind." A volume number, generous, and a last + resort rather than a control. + +A volume cap asked to do runaway detection will always be wrong, because +the quantity it measures rises with legitimate depth. + +THE ASYMMETRY THAT SETS THE NUMBERS. A cap set too high costs tokens on +a subscription already paid for. A cap set too low costs the whole run, +reads as agent failure, and sends the next person diagnosing the wrong +thing. Two such kills cost 198,317 output tokens, 13% of everything +twenty dispatches produced, and neither was an agent fault. So bias +generous, and put the effort into making a kill survivable instead -- +see NON_DESTRUCTIVE below. +""" + +from __future__ import annotations + +# -------------------------------------------------------------------- +# The measurement the budget rests on. Output tokens per dispatch, +# read from the harnesses' own trace files on 2026-08-23 across twenty +# dispatches. Re-derive with ops/devlane/telemetry/usage.py rather than +# editing these by hand; they are evidence, not configuration. +# -------------------------------------------------------------------- + +OBSERVED_OUTPUT = { + "planner": {"n": 4, "max": 256_839, "top": "plan-wf2"}, + "author": {"n": 5, "max": 213_582, "top": "extract-a"}, + "contract": {"n": 3, "max": 96_797, "top": "contract-b"}, + "reviewer": {"n": 1, "max": 13_208, "top": "author-tests"}, +} + +# Largest legitimate output observed anywhere: 256,839. +CAP_OUT = 500_000 + +# Deliberately FLAT rather than per-role, though the per-role maxima +# above would support tighter numbers. Four data points for planners and +# ONE for reviewers is not enough to set a threshold that kills work: a +# per-role cap derived from n=1 is false precision, and the first +# legitimate run that exceeds it looks exactly like a bug in the agent. +# Ratified by the owner on 2026-08-23 at 1.9x the largest legitimate +# run. Revisit when a role class has enough dispatches to have a real +# distribution rather than a maximum. + +# -------------------------------------------------------------------- +# Wires that are OFF, and why. A disabled wire needs a reason on the +# record, or the next person re-enables it and repeats the failure. +# -------------------------------------------------------------------- + +DISABLED = { + "tokens": + "Sums cache RE-READS, which were 93.4% of all counted tokens " + "across twenty dispatches -- 58,760,434 of 62,887,712, or 14.2x " + "the entire non-cached traffic. It therefore measures how many " + "turns an agent took, not what it produced, and rises with " + "legitimate depth. No threshold separates a runaway from a " + "thorough job. It killed an extractor at 3,083,403 'total' whose " + "real output was 108,241. Off permanently; tokens-out is the " + "signal, because it counts work produced rather than context " + "resent.", +} + +# -------------------------------------------------------------------- +# Per-role wires. Silence means different things to different roles, so +# a single stall threshold cannot be right for all of them. +# -------------------------------------------------------------------- + +ROLE_WIRES = { + # Writes once, after thinking for a long time. Silence is its normal + # working state. A known-good planner had a 340s gap on a 155-line + # source; 600s then killed one reading a 611-line source at 19 + # minutes, and `claude -p` emits stdout only at the end, so the whole + # plan was lost. + "planner": {"stall": 2400}, + # Uses tools constantly, so it writes to its trace constantly. + "author": {"stall": 1200}, + "contract": {"stall": 1200}, + "reviewer": {"stall": 900}, +} + +DEFAULT_STALL = 1200 + +# -------------------------------------------------------------------- + +NON_DESTRUCTIVE = """ +The threshold matters far less than whether tripping it destroys the +work, and that is where the effort belongs. + +An extractor killed mid-run had already written 2 of its 7 modules to +disk; those survived. What died was its REPORT, because `claude -p` +writes stdout only when the turn ends -- so an hour of reasoning left a +zero-byte file. The cap was wrong, but the cap being wrong only cost an +hour because the report had nowhere to land. + +Have a role write its report to a file as it goes, and a trip costs one +turn instead of a run. Then the cap can be generous without anyone +minding, which is the point. +""" + + +def budget(role=None): + """Output-token cap for a role. Flat today; the argument is accepted + so callers do not have to change when it stops being flat.""" + return CAP_OUT + + +def stall(role=None): + return ROLE_WIRES.get(role, {}).get("stall", DEFAULT_STALL) + + +def disabled_wires(): + return sorted(DISABLED) + + +def _main(argv=None): + """`wires.py --sh ` emits the breaker settings for a launcher, + so the numbers are not written down twice and cannot drift apart.""" + import argparse + import json + import shlex + + ap = argparse.ArgumentParser(description="supervision settings for a role") + ap.add_argument("--sh", metavar="ROLE", nargs="?", const="", default=None) + args = ap.parse_args(argv) + if args.sh is None: + print(json.dumps({ + "cap_out": CAP_OUT, "disabled": disabled_wires(), + "role_wires": ROLE_WIRES, "observed_output": OBSERVED_OUTPUT, + }, indent=2, sort_keys=True)) + return 0 + role = args.sh or None + print(f"WIRE_CAP_OUT={budget(role)}") + print(f"WIRE_STALL={stall(role)}") + print(f"WIRE_DISABLE={shlex.quote(','.join(disabled_wires()))}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/ops/devlane/infra/gitea/runner-token.seed b/ops/devlane/infra/gitea/runner-token.seed new file mode 100644 index 0000000..e69de29 diff --git a/ops/devlane/task/CONTRACT.md b/ops/devlane/task/CONTRACT.md new file mode 100644 index 0000000..6fac097 --- /dev/null +++ b/ops/devlane/task/CONTRACT.md @@ -0,0 +1,187 @@ +# The task app — a calling convention for delegated work + +A **task** is one delegated unit of work that returns one deliverable +in one round. It is not a conversation and not a workflow stage: the +Conductor invokes it with parameters, the task does its own iteration +internally, and it returns a typed envelope. + +The point is not correctness — the checks and the cross-review process +already own that. The point is a **standard, token-efficient way to +run delegated work**, so the plumbing is written once and the Conductor +never parses prose. + +## Why it exists, measured + +On 2026-08-22 one session launched **36 agents** and hand-wrote the +same ~40 lines of plumbing for nearly every one: snapshot a ref, +archive it, write a diff, drop a marker, discover the session stream +the harness just opened, arm the battery, wait, read a verdict by eye. +**15 of those 36 were repeat rounds of a review already run**, each +handed the entire diff again even when the live question had narrowed +to a single test case — because there was no parameter for scope. + +Two costs follow from that, and both are what this app removes: + +- **Plumbing retyped per launch**, with the mistakes that come with it. +- **The Conductor reading work product.** Verdicts arrived as prose, were + read in full, and the interesting parts were retyped into the next + prompt. That is the loop cost: a Conductor's context grows with every + round until it cannot run another. + +## The call + +``` +task( + job, # which pre-made task + context = {ref, diff_base, include, prior}, + require = {schema, scope, constraints}, + runtime = {harness, model, effort, caps}, +) +``` + +**`context` is the token lever.** It states what the task may see: + +| field | meaning | +|:--|:--| +| `ref` | the git ref to snapshot — the task works on a detached copy, never the live tree | +| `diff_base` | what the diff is taken against; omitted means no diff is written | +| `include` | explicit path allowlist. Absent means the whole snapshot, which is the expensive default and should be a deliberate choice | +| `prior` | findings from an earlier call. Turns "review this again" into "confirm these are closed", which is a much smaller job | + +**`require`** states what must come back: the output `schema`, the +`scope` to aim at, and the `constraints` the task must respect (read +only, plant on copies, never edit the snapshot). + +**`runtime`** states who runs it and under what ceiling: `harness`, +`model`, `effort`, and the supervision `caps` handed to +`ops/devlane/telemetry/breaker.py`. A job never names a model — that +is the Conductor's dial, which is what makes "eight cheap ones" and "one +careful one" the same job. + +## The envelope + +Every task returns the same shape. The Conductor routes on it without +reading the work: + +```json +{ + "job": "adversarial-review", + "status": "ok | invalid | tripped", + "verdict": "approve | changes | null", + "counts": {"p1": 0, "p2": 2, "p3": 1}, + "findings": [ + {"severity": "p2", "where": "file.py §section", "claim": "...", + "reproduce": "python3 -m unittest ..."} + ], + "artifacts": {"raw": "", "diff": ""}, + "spend": {"harness": "grok", "total": 0, "out": 0, "runs": 0}, + "stamp": {"ref": "", "started": "...Z", "ended": "...Z"} +} +``` + +Three fields carry the weight: + +- **`reproduce`** — the command that re-derives the claim. A finding + without one is an opinion, and the envelope says so rather than + hiding it. +- **`artifacts`** — handles, not contents. The prose stays on disk; + the Conductor reads it only if it decides to. +- **`spend`** — what the run cost, so `worth.py` can price a job + instead of guessing. Runs happen in throwaway snapshots today, and + usage.py's cwd filter therefore attributes **none** of them to the + repo they served: roughly 118M tokens of review on 2026-08-22 went + unattributed. The stamp is what closes that. + +## Statuses, and the refusal rule + +`status` is the routing decision, and it is not the same as `verdict`: + +| status | means | +|:--|:--| +| `ok` | the task ran and produced its deliverable | +| `invalid` | the task could not do its job — harness missing, ref absent, snapshot failed, output unparseable. **Never reported as an approving verdict.** | +| `tripped` | the battery stopped a runaway; the deliverable is partial and says so | + +A task that could not look must not return `approve`. This is the same +rule the registered checks follow, and the reason `status` and +`verdict` are separate fields rather than one. + +## Jobs + +A job is data, not code — a JSON entry naming the role, the prompt +template, the output schema, and the default constraints. Adding one +does not change the runner. + +| job | deliverable | +|:--|:--| +| `adversarial-review` | findings that survived the task's own attempt to refute them | +| `verify` | one claim, executed: a reproduction or a refutation | +| `author-tests` | a test file that has been run and is red for the stated reason | +| `sweep` | structured findings over disjoint categories, merged | +| `plan` | a plan written to the job's `out/PLAN.md`, and an envelope naming it | +| `check-tests` | a skeptic's report at `out/SKEPTIC.md`; verdict `changes` when any test is refuted | +| `implement` | commits in the snapshot that make the named test command pass, trailers per `AGENTS.md` | +| `adjudicate` | a ruling at `out/RULING.md` — UPHELD / PARTIAL / REJECTED per finding, and what the reviewers missed | + +`verify` is deliberately callable on its own: it is the primitive the +others lean on, and the one whose answer is mechanical — an exit code +or a diff, not another opinion. + +The last four are the dev-lane pipeline's own stages as jobs (dispatched +by the dispatch app; see `ops/devlane/dispatch/CONTRACT.md`). They are data +in `jobs.json` today. Three of them — `plan`, +`check-tests`, `adjudicate` — name `{out}` or `{inputs}`, which `run.py` +does not yet supply, so `render` refuses each by name rather than +launching a brief with a hole in it. `implement` names only `{ref}` and +`{scope}` and renders today; what it still lacks is the launcher's +whole-mode snapshot and collect, without which its commits have nowhere +to land. + +## What the runner does, and does not + +The runner wires: it snapshots, launches, supervises with the battery, +collects, parses, records. It does **not** read the work product, and +it does not decide anything a job could not have decided in +advance. + +Two things stay outside it. **Intent** — which of two defensible +designs is right — belongs to the job author or escalates to the +owner; a worker task has no standing to settle it. And **the wiring's +own correctness** cannot be delegated to the tasks it wires, which is +why every job is bound to a planted fault: run it against a fixture +with a known defect and it must surface it; run it clean and it must +stay quiet. A job that cannot catch its own planted fault is a +no-op with good throughput. + +## Harnesses + +The job is harness-independent; an adapter absorbs the differences +in invocation, in where each harness writes its session stream, and in +how each reports usage (`ops/devlane/telemetry/usage.py` already holds +those three accountings). A `stub` adapter exists so the suite can +test the runner without spending a token or needing a model. + +## The property to hold + +**Iteration N should cost the Conductor about what iteration 1 cost.** +A loop whose context grows every round has a horizon; one that stays +flat can run as long as the work does. The envelope is what makes that +possible, and `worth waste`'s cache-churn signal is how it is checked. + +## Dispatch — moved to its own app + +The dev-lane launcher — `launch.py`, the policy over `run.py`, and +its record builder `record.py` — no longer lives here. It executes +the harness app's isolation and reads its budget wiring, so it +genuinely depends on `harness`; the cross-app import contract forbids +`task → harness`. Rather than weaken that rule, the launcher moved to +a composition app that may depend on both this app and `harness`: + +- its contract is `ops/devlane/dispatch/CONTRACT.md`; +- its guide page is `.dev/guide/dispatch.md`; +- the allowed edges `dispatch → task` and `dispatch → harness` are + declared in `.dev/contracts/imports.json`. + +`jobs.json` stays here — it is data the dispatch app reads by path, +not an import — and so do `run.py`, `envelope.py`, `fileset.py`, and +`verify.py`. diff --git a/ops/devlane/task/envelope.py b/ops/devlane/task/envelope.py new file mode 100644 index 0000000..1451eef --- /dev/null +++ b/ops/devlane/task/envelope.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""The typed envelope every task returns, and the rules it enforces. + +`CONTRACT.md` §The envelope states the shape; this module is the only +place that builds one, so the shape cannot vary by caller. The rules +below are not validation for its own sake — each one closes a way the +caller could be misled while reading nothing but this dict: + +**A task that could not look must not approve** (§Statuses). `status` +routes and `verdict` judges, and only a task that actually ran — status +`ok` — may carry `approve`. `invalid` carries no verdict at all, and +`tripped` may report what it found but never that the job is clean. + +**Counts are derived, never asserted.** A tally accepted from a caller +is a second copy of the findings list, and two copies drift. `build` +computes it, and `validate` refuses an envelope whose tally has stopped +matching its findings — which is what a truncated list from a harness +looks like from the outside. + +**A finding with no reproduction is an opinion, and says so.** The +`reproduce` key is always present; absence is `None` in the data rather +than a key the caller has to notice is missing. `counts["opinions"]` is +that same fact where routing can see it. + +**Artifacts are handles, not contents.** The property this whole app +exists for is that iteration N costs the caller about what iteration 1 +cost, and the way that breaks is prose migrating into the envelope. A +handle is one line and shorter than HANDLE_MAX; anything else is +content, and content belongs in the file the handle names. +""" + +from __future__ import annotations + +#: Key order is part of the shape: a caller diffing two envelopes, or a +#: reviewer reading one, sees the same fields in the same places. +FIELDS = ("job", "status", "verdict", "counts", "findings", + "artifacts", "spend", "stamp", "note") +FINDING_FIELDS = ("severity", "where", "claim", "reproduce") +SPEND_FIELDS = ("harness", "total", "out", "runs") +STAMP_FIELDS = ("ref", "started", "ended") + +STATUSES = ("ok", "invalid", "tripped") +VERDICTS = ("approve", "changes") +SEVERITIES = ("p1", "p2", "p3") + +#: Longer than any path this lane produces, far shorter than a review. +HANDLE_MAX = 512 + +#: The wire shape supplied to structured-output capable harnesses. This is +#: deliberately beside ``FIELDS`` and the constructor: callers do not keep a +#: second, gradually diverging description of an envelope. +ENVELOPE_SCHEMA = { + "type": "object", + "properties": { + "job": {"type": "string"}, + "status": {"type": "string", "enum": list(STATUSES)}, + "verdict": { + "type": ["string", "null"], + "enum": [*VERDICTS, None], + }, + "counts": { + "type": "object", + "properties": { + key: {"type": "integer", "minimum": 0} + for key in (*SEVERITIES, "opinions") + }, + "required": [*SEVERITIES, "opinions"], + "additionalProperties": False, + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "severity": {"type": "string", "enum": list(SEVERITIES)}, + "where": {"type": "string"}, + "claim": {"type": "string"}, + "reproduce": {"type": ["string", "null"]}, + }, + "required": list(FINDING_FIELDS), + "additionalProperties": False, + }, + }, + "artifacts": { + "type": "object", + "additionalProperties": {"type": "string"}, + }, + "spend": {"type": "object"}, + "stamp": { + "type": "object", + "properties": { + "ref": {"type": "string"}, + "started": {"type": ["string", "null"]}, + "ended": {"type": ["string", "null"]}, + }, + "required": ["ref"], + "additionalProperties": False, + }, + "note": {"type": ["string", "null"]}, + "commit": { + "type": "object", + "properties": { + "subject": {"type": "string"}, + "body": {"type": "string"}, + }, + "required": ["subject", "body"], + "additionalProperties": False, + }, + }, + "required": list(FIELDS), + "additionalProperties": False, +} + + +class EnvelopeError(ValueError): + """The envelope would have misled the caller. Refuse, never repair.""" + + +def _text(value, field): + if not isinstance(value, str) or not value.strip(): + raise EnvelopeError(f"{field} must be a non-empty string") + return " ".join(value.split()) + + +def _handle(name, value): + if not isinstance(value, str): + raise EnvelopeError(f"artifacts[{name!r}] must be a path string") + if not value.strip(): + raise EnvelopeError(f"artifacts[{name!r}] is empty") + if len(value) > HANDLE_MAX: + raise EnvelopeError( + f"artifacts[{name!r}] is {len(value)} chars: a handle, not " + f"contents, is the contract (max {HANDLE_MAX})") + if "\n" in value or "\r" in value: + raise EnvelopeError( + f"artifacts[{name!r}] spans lines: that is the work product, " + "not a handle to it") + return value + + +def finding(severity, where, claim, reproduce=None): + """One finding, normalized. A blank reproduction is an absent one.""" + if severity not in SEVERITIES: + raise EnvelopeError( + f"severity {severity!r} is not one of {list(SEVERITIES)}") + if isinstance(reproduce, str) and not reproduce.strip(): + reproduce = None + if reproduce is not None and not isinstance(reproduce, str): + raise EnvelopeError("reproduce must be a command string or None") + return {"severity": severity, + "where": _text(where, "where"), + "claim": _text(claim, "claim"), + "reproduce": reproduce.strip() if reproduce else None} + + +def tally(findings): + counts = dict.fromkeys(SEVERITIES, 0) + counts["opinions"] = sum(1 for f in findings if not f.get("reproduce")) + for f in findings: + counts[f["severity"]] += 1 + return counts + + +def _spend(given): + spend = {"harness": None, "total": 0, "out": 0, "runs": 0} + for key, value in (given or {}).items(): + if key not in SPEND_FIELDS: + raise EnvelopeError(f"spend has no field {key!r}") + spend[key] = value + if spend["harness"] is not None and not isinstance(spend["harness"], str): + raise EnvelopeError("spend.harness must be a harness name or None") + for key in ("total", "out", "runs"): + value = spend[key] + if not isinstance(value, int) or isinstance(value, bool): + raise EnvelopeError(f"spend.{key} must be an integer") + if value < 0: + # worth.py prices from these; a negative subtracts from a + # real cost somewhere else in the ledger. + raise EnvelopeError(f"spend.{key} is negative") + return spend + + +def _stamp(given): + given = given or {} + for key in given: + if key not in STAMP_FIELDS: + raise EnvelopeError(f"stamp has no field {key!r}") + # Without a ref the envelope names no state, and a fact with no + # state behind it cannot be re-checked by anyone, including its + # author tomorrow. + ref = given.get("ref") + if not isinstance(ref, str) or not ref.strip(): + raise EnvelopeError("stamp.ref must name the ref the task read") + return {"ref": ref.strip(), + "started": given.get("started"), + "ended": given.get("ended")} + + +def build(job, *, status, verdict=None, findings=(), artifacts=None, + spend=None, stamp=None, note=None): + """Assemble an envelope, refusing any combination that would lie. + + Every rule about the assembled shape lives in `validate`, which + this returns through — deliberately, and once. Checking a rule here + as well would leave two implementations of it, and a suite that + plants a fault in either one still passes on the other. Measured: + the approve gate was written in both places and BOTH copies could + be disabled undetected, because each was covering for the other. + """ + normalized = [ + f if isinstance(f, dict) and tuple(f) == FINDING_FIELDS + else finding(**dict(zip(FINDING_FIELDS, ( + f.get("severity"), f.get("where"), f.get("claim"), + f.get("reproduce")), strict=True))) + for f in findings] + env = { + "job": _text(job, "job"), + "status": status, + "verdict": verdict, + "counts": tally(normalized), + "findings": normalized, + "artifacts": {name: _handle(name, value) + for name, value in (artifacts or {}).items()}, + "spend": _spend(spend), + "stamp": _stamp(stamp), + "note": " ".join(note.split()) if note else None, + } + return validate(env) + + +def invalid(job, note, *, stamp=None, spend=None, artifacts=None): + """The task could not do its job. `note` says which way, and must. + + The note is required by `validate`, not re-checked here: see + `build` on why one rule gets exactly one implementation. + """ + return build(job, status="invalid", verdict=None, stamp=stamp, + spend=spend, artifacts=artifacts, note=note) + + +#: Container fields whose type must be proved before anything iterates +#: or indexes them. `bool` is not accepted for a dict or list by +#: isinstance, so no special case is needed here. +CONTAINER_TYPES = { + "findings": list, + "counts": dict, + "artifacts": dict, + "spend": dict, + "stamp": dict, +} + + +def validate(env): + """Re-check a built or parsed envelope. Returns it, or raises.""" + if not isinstance(env, dict): + raise EnvelopeError("an envelope is a dict") + if tuple(env) != FIELDS: + raise EnvelopeError( + f"envelope fields {list(env)} are not the contract's " + f"{list(FIELDS)}") + # A worker's JSON arrives here unvalidated, and every check below + # either iterates or indexes one of these. A wrong type has to raise + # EnvelopeError -- which the parser turns into an `invalid` envelope + # -- rather than a TypeError, which escapes the parser and kills the + # run instead of reporting it (Codex, PR #49: `"findings": null` + # raised TypeError at the findings loop). Typed here rather than at + # the one field reported, because the same hole is under every + # container the contract names. + for field, want in CONTAINER_TYPES.items(): + if not isinstance(env[field], want): + raise EnvelopeError( + f"{field} is {type(env[field]).__name__}, not " + f"{want.__name__}") + for f in env["findings"]: + if not isinstance(f, dict): + raise EnvelopeError( + f"a finding is {type(f).__name__}, not dict") + if env["status"] not in STATUSES: + raise EnvelopeError(f"status {env['status']!r} is not a status") + if env["verdict"] is not None and env["verdict"] not in VERDICTS: + raise EnvelopeError(f"verdict {env['verdict']!r} is not a verdict") + if env["verdict"] == "approve" and env["status"] != "ok": + raise EnvelopeError( + f"a {env['status']!r} task did not do its job and cannot " + "approve one") + if env["status"] == "invalid" and env["verdict"] is not None: + raise EnvelopeError("an invalid task carries no verdict") + if env["status"] == "invalid" and not env["note"]: + raise EnvelopeError( + "an invalid envelope must say why it could not look") + for f in env["findings"]: + if tuple(f) != FINDING_FIELDS: + raise EnvelopeError( + f"finding fields {list(f)} are not {list(FINDING_FIELDS)}") + # `finding` is the one place a finding's rules live; re-running + # it is how they get checked here without being restated here. + if finding(**f) != f: + raise EnvelopeError(f"finding {f} is not in normal form") + if env["counts"] != tally(env["findings"]): + # The list is what the task found; the tally is a copy of it. + # When they disagree the list is short, which is exactly what a + # truncated harness reply looks like from out here. + raise EnvelopeError( + f"counts {env['counts']} disagree with the findings " + f"{tally(env['findings'])}: the list is not the tally") + for name, value in env["artifacts"].items(): + _handle(name, value) + _spend(env["spend"]) + _stamp(env["stamp"]) + return env diff --git a/ops/devlane/task/fileset.py b/ops/devlane/task/fileset.py new file mode 100644 index 0000000..b0beef9 --- /dev/null +++ b/ops/devlane/task/fileset.py @@ -0,0 +1,432 @@ +#!/usr/bin/env python3 +"""Build task-sized Git snapshots without changing the source checkout. + +Smaller fileset is only useful when it still answers the same question as +the complete tree. The functions here therefore keep selection, +materialization, and parity checking together: a missing requested file is +an error, snapshot provenance travels with the files, and a command that +never started cannot be mistaken for a matching result. + +All Git access is through object-reading commands. In particular, this +module never checks out a ref or asks Git to repair the caller's worktree. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path, PurePosixPath + +_MANIFEST_NAME = "FILESET.md" +_DIFF_NAME = "FILESET.diff" +_RESERVED_NAMES = frozenset({_MANIFEST_NAME, _DIFF_NAME}) +_PATH_NEIGHBOUR = rb"A-Za-z0-9._/-" + + +class FilesetError(Exception): + """A fileset that would have misled the task. Refuse, never repair.""" + + +class _UsageError(Exception): + """An argparse refusal that main can translate to the CLI contract.""" + + +class _Parser(argparse.ArgumentParser): + def error(self, message): + self.print_usage(sys.stderr) + raise _UsageError(f"{self.prog}: error: {message}") + + +def _as_text_path(value, field): + try: + return os.fsdecode(os.fspath(value)) + except TypeError as exc: + raise FilesetError(f"{field} must be a path") from exc + + +def _git(repo, *args): + repo_path = _as_text_path(repo, "repo") + git_env = os.environ.copy() + # A read from a partial clone may otherwise fetch a missing object and + # turn local fileset construction into an undeclared network operation. + git_env["GIT_NO_LAZY_FETCH"] = "1" + git_env["GIT_TERMINAL_PROMPT"] = "0" + try: + completed = subprocess.run( + ["git", "-C", repo_path, *args], + env=git_env, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + except (OSError, ValueError) as exc: + raise FilesetError(f"git could not run for {repo_path!r}: {exc}") from exc + if completed.returncode: + detail = completed.stderr.decode("utf-8", "replace").strip() + if not detail: + detail = f"git exited {completed.returncode} without an error message" + raise FilesetError(detail) + return completed.stdout + + +def _commit(repo, ref): + if ref is None: + raise FilesetError("ref is required") + ref_text = str(ref) + if not ref_text: + raise FilesetError("ref is empty") + raw = _git( + repo, "rev-parse", "--verify", "--end-of-options", + f"{ref_text}^{{commit}}", + ) + sha = raw.decode("ascii", "strict").strip() + if not sha or not all(character in "0123456789abcdefABCDEF" for character in sha): + raise FilesetError(f"git resolved {ref_text!r} to an invalid object name") + return sha.lower() + + +def _repo_path(value, field="include path"): + path = _as_text_path(value, field) + pure = PurePosixPath(path) + if not path or path == "." or pure.is_absolute() or ".." in pure.parts: + raise FilesetError(f"{field} {path!r} is not a repo-relative path") + return pure.as_posix() + + +def _tree(repo, sha): + """Return path -> (mode, type, object id) for one committed tree.""" + raw = _git(repo, "ls-tree", "-r", "-z", "--full-tree", sha) + entries = {} + for record in raw.split(b"\0"): + if not record: + continue + try: + header, raw_path = record.split(b"\t", 1) + mode, kind, object_id = header.split(b" ", 2) + except ValueError as exc: + raise FilesetError("git ls-tree returned an unreadable record") from exc + path = os.fsdecode(raw_path) + normalized = _repo_path(path, "path in the committed tree") + if normalized != path: + raise FilesetError(f"git tree path {path!r} is not canonical") + entries[path] = ( + mode.decode("ascii"), kind.decode("ascii"), + object_id.decode("ascii"), + ) + return entries + + +def _diff(repo, base_sha, ref_sha): + # Hooks, textconv, and external diff commands would turn a read into + # caller-controlled execution and could mutate the checkout indirectly. + return _git( + repo, "diff", "--no-ext-diff", "--no-textconv", + base_sha, ref_sha, "--", + ) + + +def _diff_body(diff_text): + """The diff's lines with their leading +/-/space marker removed. + + A removal marker is `-`, which is also a legal path character, so a + path named at the START of a removed line arrives as `-config.toml` + and the boundary rule below refuses it — while the same reference on + an ADDED line matches, because `+` is not a path character. That + asymmetry made a derivation silently miss a file (Grok's + test_a_deletion_whose_hunk_names_a_survivor_keeps_the_survivor_only, + written from the contract without sight of this code). Stripping the + marker restores each line's own text before boundaries are judged. + """ + return b"\n".join( + line[1:] if line[:1] in (b"+", b"-", b" ") else line + for line in diff_text.split(b"\n")) + + +def _path_is_named(diff_text, path): + raw_path = os.fsencode(path) + # Boundaries keep a short tracked name such as `a` from matching every + # occurrence of that letter, while quotes, backticks, and line suffixes + # remain valid ways for source text to name a path. + pattern = ( + rb"(? int: + try: + arguments = _parser().parse_args(argv) + except _UsageError as exc: + print(exc, file=sys.stderr) + return 64 + except SystemExit as exc: + # argparse owns --help output; the CLI still returns rather than + # terminating when main is called as a library function. + return int(exc.code) + + try: + manifest = snapshot( + Path.cwd(), arguments.ref, arguments.into, + include=arguments.include, base=arguments.base, + whole=arguments.whole, + ) + except FilesetError as exc: + print(f"fileset.py: {exc}", file=sys.stderr) + return 1 + print(json.dumps(manifest, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/devlane/task/jobs.json b/ops/devlane/task/jobs.json new file mode 100644 index 0000000..bfb49b0 --- /dev/null +++ b/ops/devlane/task/jobs.json @@ -0,0 +1,91 @@ +{ + "verify": { + "adapter": "direct", + "deliverable": "one claim, executed: a reproduction or a refutation", + "prompt": null, + "constraints": [ + "read only", + "never edit the snapshot" + ] + }, + "adversarial-review": { + "adapter": "harness", + "deliverable": "findings that survived the task's own attempt to refute them", + "role": "read", + "snapshot": "whole", + "caps": {"timeout": 1200}, + "prompt": "Review the change at {ref} against {base}. The diff is {diff}. Review only — do not edit.\nAim at: {scope}\nInputs named by the brief are files under in/, a sibling of out/; read them whole before working.\nAnswer with a single JSON object and nothing else, carrying exactly these keys: job, status, verdict, counts, findings, artifacts, spend, stamp, note. Prose is not a deliverable here — the CONTRIB.md wire block is the human review format, not this job's stdout.", + "constraints": [ + "read only", + "do not edit the snapshot" + ] + }, + "author-tests": { + "adapter": "harness", + "deliverable": "a test file that has been run and is red for the stated reason", + "role": "write", + "snapshot": "whole", + "caps": {"timeout": 1800}, + "prompt": "Write tests from the contract at {scope}, for the tree at {ref}. Do not read the implementation. Prove each test red for its own assertion before making it green. Commit each test file as soon as its red is proven — if committing is denied, write it to disk at once — so a run killed at the wall leaves finished work, not residue.\nInputs named by the brief are files under in/, a sibling of out/; read them whole before working.\nAnswer with a single JSON object and nothing else, carrying exactly these keys: job, status, verdict, counts, findings, artifacts, spend, stamp, note (and, for a write job that could not commit, commit).", + "constraints": [ + "do not edit the implementation", + "no network" + ] + }, + "sweep": { + "adapter": "harness", + "deliverable": "structured findings over disjoint categories, merged", + "role": "read", + "snapshot": "whole", + "prompt": "Sweep the tree at {ref} for defects in these categories, and ONLY these: {scope}. The categories are disjoint — report each finding under exactly one, and do not repeat a finding across categories.\nCover every category. A category you looked at and found nothing in is a category with zero findings, which is not the same as one you did not look at — if you could not cover one, say so in note and return status invalid rather than reporting a partial sweep as a complete one.\nEvery finding carries a reproduce command that re-derives it; a finding without one is an opinion and does not belong in the list.\nReview only — do not edit.\nInputs named by the brief are files under in/, a sibling of out/; read them whole before working.\nAnswer with a single JSON object and nothing else, carrying exactly these keys: job, status, verdict, counts, findings, artifacts, spend, stamp, note.", + "constraints": [ + "read only", + "do not edit the snapshot" + ] + }, + "plan": { + "adapter": "harness", + "deliverable": "a plan written to the job's out/PLAN.md, and an envelope naming it", + "role": "read", + "snapshot": "whole", + "prompt": "Write a plan for the scope below, for the tree at {ref}. Read the snapshot and nothing else. Write the plan to {out}/PLAN.md and name that path in artifacts. Do not edit the snapshot.\nScope: {scope}\nIf you judge you are running long, stop and write up what you have.\nInputs named by the brief are files under in/, a sibling of out/; read them whole before working.\nAnswer with a single JSON object and nothing else, carrying exactly these keys: job, status, verdict, counts, findings, artifacts, spend, stamp, note.", + "constraints": [ + "read only", + "do not edit the snapshot" + ] + }, + "check-tests": { + "adapter": "harness", + "deliverable": "a skeptic's report at the job's out/SKEPTIC.md; verdict changes when any test is refuted", + "role": "read", + "snapshot": "whole", + "caps": {"timeout": 1800}, + "prompt": "Adopt the role card at ops/process/roles/test-skeptic.md in the tree at {ref}. Judge the tests at {inputs} against the contract named here: {scope}. Do not read any implementation. Refute by default: a test survives only when you can name the broken implementation it would catch. Every refuted or weak test is a finding carrying a reproduce command; verdict is changes when any finding exists and approve otherwise. Write the report to {out}/SKEPTIC.md and name that path in artifacts. Review only — do not edit.\nInputs named by the brief are files under in/, a sibling of out/; read them whole before working.\nAnswer with a single JSON object and nothing else, carrying exactly these keys: job, status, verdict, counts, findings, artifacts, spend, stamp, note.", + "constraints": [ + "read only", + "do not edit the snapshot" + ] + }, + "implement": { + "adapter": "harness", + "deliverable": "commits in the snapshot that make the named test command pass, trailers per AGENTS.md", + "role": "write", + "snapshot": "whole", + "prompt": "In the tree at {ref}, implement whatever makes this command exit 0: {scope}. Commit in the snapshot as you go, with the trailers AGENTS.md requires; do not edit the files the tests live in, and do not touch the command itself. Record each command you ran as a finding with a reproduce entry. Status is ok only when the named command exits 0 in your final tree; otherwise status is invalid and note names the failing test. Commit if you can; if the sandbox refuses git commit, status stays ok and you write your message to out/COMMIT_MSG (or a commit object with keys subject and body in the envelope) — the launcher commits your tree with that message and your attribution (codex only; grok and claude commit themselves). Verdict is null.\nInputs named by the brief are files under in/, a sibling of out/; read them whole before working.\nAnswer with a single JSON object and nothing else, carrying exactly these keys: job, status, verdict, counts, findings, artifacts, spend, stamp, note (and, for a write job that could not commit, commit).", + "constraints": [ + "do not edit the files the tests live in", + "no network" + ] + }, + "adjudicate": { + "adapter": "harness", + "deliverable": "a ruling at the job's out/RULING.md: UPHELD, PARTIAL or REJECTED per finding, and what the reviewers missed", + "role": "read", + "snapshot": "whole", + "prompt": "Rule on the reviews at {inputs} of the artifact at {ref}. You are given the artifact at that ref and the reports verbatim, and nothing else. Rule UPHELD, PARTIAL or REJECTED per finding, each ruling grounded in a reproduction against the tree, then state what the reviewers missed. Write the ruling to {out}/RULING.md and name that path in artifacts. Review only — do not edit.\nInputs named by the brief are files under in/, a sibling of out/; read them whole before working.\nAnswer with a single JSON object and nothing else, carrying exactly these keys: job, status, verdict, counts, findings, artifacts, spend, stamp, note.", + "constraints": [ + "read only", + "do not edit the snapshot" + ] + } +} diff --git a/ops/devlane/task/run.py b/ops/devlane/task/run.py new file mode 100644 index 0000000..c9a65d1 --- /dev/null +++ b/ops/devlane/task/run.py @@ -0,0 +1,1326 @@ +#!/usr/bin/env python3 +"""Run a declared job without inventing launch policy. + +The adapter table exists because plausible harness flags caused real loss: +a writing task silently ran read-only, and a cache-sensitive token wall +killed useful work. Launch facts therefore stay data, real launches use the +existing battery, and missing facts are explicit refusals. + +Raw output stays in the snapshot. The runner validates structured answers; +it never reads review prose or a work product to manufacture a verdict. +""" + +from __future__ import annotations + +import argparse +import contextlib +import importlib.util +import json +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +from datetime import datetime, timezone +from pathlib import Path +from types import SimpleNamespace + +import envelope +import fileset +import verify + +ADAPTERS = { + "codex": { + "argv": ["codex", "exec", "--sandbox", "{sandbox}", "-"], + "prompt": "stdin", + "sandbox": {"read": "read-only", "write": "workspace-write"}, + # `codex --help`: `-m, --model `. There is no effort flag; + # only a generic `-c key=value`, whose reasoning key I will not + # guess. A caller setting runtime.effort on codex is refused by + # name rather than having the dial silently dropped. + "dials": {"model": ["-m", "{model}"]}, + "store": "~/.codex/sessions", + "stream": "rollout-*.jsonl", + "stream_names_cwd": True, + }, + "grok": { + "argv": [ + "grok", "--prompt-file", "{prompt}", + "--output-format", "plain", + "--permission-mode", "{sandbox}", + ], + "prompt": "file", + "sandbox": {"read": "plan", "write": "auto"}, + # `grok --help`: `-m, --model ` and + # `--reasoning-effort ` -- NOT `--effort`, which is what + # a review reported; the CLI was asked directly. + "dials": {"model": ["-m", "{model}"], + "effort": ["--reasoning-effort", "{effort}"]}, + "store": "~/.grok/sessions", + "stream": "updates.jsonl", + # Grok keys its store PATH by url-encoded cwd, and only `/` is + # encoded, so the snapshot's directory NAME survives intact and + # _stream_names_snapshot's marker test matches it. With this + # False, _discover_stream fell back to "newest updates.jsonl + # anywhere under the store" and a concurrent Grok session's + # stream could be supervised, its spend charged here, or the + # wrong task terminated (Codex + Grok, PR #49). + "stream_names_cwd": True, + }, + "claude": { + "argv": ["claude", "--print", "--permission-mode", "{sandbox}"], + "prompt": "stdin", + "sandbox": {"read": "plan", "write": "acceptEdits"}, + # `claude --help`: `--model ` and `--effort `. + "dials": {"model": ["--model", "{model}"], + "effort": ["--effort", "{effort}"]}, + "store": "~/.claude/projects", + "stream": "*.jsonl", + "stream_names_cwd": True, + }, + "stub": {"replay": True}, + "direct": {"direct": True}, +} + +JOBS_PATH = Path(__file__).resolve().parent / "jobs.json" +_BREAKER_PATH = ( + Path(__file__).resolve().parent.parent / "telemetry" / "breaker.py" +) +_UNRESOLVED_REF = "unresolved" +_DIRECT_LOCK = threading.Lock() + + +class RunError(Exception): + """A run that would have misled the Conductor. Refuse, never repair.""" + + +class _UsageError(Exception): + """An argparse refusal that main translates to exit 64.""" + + +class _Parser(argparse.ArgumentParser): + def error(self, message): + self.print_usage(sys.stderr) + raise _UsageError(f"{self.prog}: error: {message}") + + +def _now(): + return datetime.now(timezone.utc).isoformat() + + +def _label(job): + if isinstance(job, str) and job.strip(): + return job.strip() + return "unknown-job" + + +def _stamp(ref, started, ended=None): + return { + "ref": ref or _UNRESOLVED_REF, + "started": started, + "ended": ended, + } + + +def _invalid(job, note, *, ref=None, started=None, artifacts=None, + spend=None): + return envelope.build( + _label(job), + status="invalid", + verdict=None, + artifacts=artifacts, + spend=spend, + stamp=_stamp(ref, started, _now()), + note=note, + ) + + +def load_jobs(path=JOBS_PATH) -> dict: + """Load the registry, refusing malformed data instead of an empty lane.""" + try: + with open(path, encoding="utf-8") as stream: + registry = json.load(stream) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise RunError( + f"job registry {os.fspath(path)!r} cannot load: {exc}" + ) from exc + if not isinstance(registry, dict): + raise RunError("job registry must be a JSON object") + for name, definition in registry.items(): + if not isinstance(name, str) or not name.strip(): + raise RunError("job registry contains an empty name") + if not isinstance(definition, dict): + raise RunError(f"job {name!r} must be a JSON object") + return registry + + +def _lines(value, field): + if value is None: + return [] + if isinstance(value, str): + return [value] if value.strip() else [] + if not isinstance(value, (list, tuple)) or not all( + isinstance(item, str) and item.strip() for item in value): + raise RunError(f"{field} must be a string list") + return list(value) + + +def render(job, context, require) -> str: + """Render only declared prompt fields; missing facts are a refusal. + + `job` is whatever `run` takes: a NAME to look up, or an + already-resolved definition. Accepting only the definition made the + two entry points disagree about their first argument, which is the + kind of seam a caller trips over once and never forgets. + """ + if isinstance(job, str): + registry = load_jobs() + if job not in registry: + raise RunError(f"no job named {job!r}") + job = registry[job] + if not isinstance(job, dict): + raise RunError("job definition must be an object") + if not isinstance(context, dict) or not isinstance(require, dict): + raise RunError("context and require must be objects") + + template = job.get("prompt") + if template is None: + return "" + if not isinstance(template, str) or not template.strip(): + raise RunError("job prompt must be a non-empty string or null") + + values = dict(context) + values.update(require) + try: + prompt = template.format_map(values).strip() + except (KeyError, ValueError) as exc: + raise RunError(f"job prompt cannot render: {exc}") from exc + + deliverable = job.get("deliverable") + if deliverable is not None: + if not isinstance(deliverable, str) or not deliverable.strip(): + raise RunError("job deliverable must be non-empty") + prompt += f"\n\nDeliverable: {deliverable.strip()}" + + constraints = _lines( + job.get("constraints"), "job constraints" + ) + constraints.extend( + _lines(require.get("constraints"), "required constraints") + ) + if constraints: + prompt += "\n\nConstraints:\n" + "\n".join( + f"- {item.strip()}" for item in constraints + ) + return prompt + "\n" + + +def _snapshot(context): + if not isinstance(context, dict): + raise RunError("context must be an object") + repo = context.get("repo") + ref = context.get("ref") + if repo is None or ref is None or not str(ref).strip(): + raise RunError("context.repo and context.ref are required") + + into = context.get("into") + if into is None: + # The generated destination is outside the source tree; fileset also + # independently enforces that safety boundary. + into = tempfile.mkdtemp(prefix="task-snapshot-") + include = context.get("include") + base = context.get("base") + try: + return fileset.snapshot( + repo, + ref, + into, + include=include, + base=base, + whole=(include is None and base is None), + ) + except (fileset.FilesetError, OSError, TypeError, ValueError) as exc: + raise RunError( + f"snapshot for ref {ref!r} could not be built: {exc}" + ) from exc + + +def _select_harness(definition, runtime): + configured = definition.get("adapter") + requested = runtime.get("harness") + if requested is None: + if configured in ("direct", "stub"): + return configured + raise RunError( + "runtime.harness is required for a harness job" + ) + if not isinstance(requested, str) or not requested.strip(): + raise RunError("runtime.harness must be a non-empty name") + requested = requested.strip() + if configured == "direct" and requested != "direct": + raise RunError( + f"job is direct and refuses harness {requested!r}" + ) + if configured == "stub" and requested != "stub": + raise RunError( + f"job is a replay and refuses harness {requested!r}" + ) + if configured not in ("direct", "stub", "harness"): + raise RunError(f"job adapter {configured!r} is not declared") + return requested + + +def _raw_path(manifest): + return Path(manifest["root"]) / ".run.raw" + + +def _prompt_path(manifest): + return Path(manifest["root"]) / ".run.prompt" + + +def _write(path, data): + try: + Path(path).write_bytes(data) + except OSError as exc: + raise RunError( + f"raw output could not be written inside the snapshot: {exc}" + ) from exc + + +def _candidate_envelope(value): + if isinstance(value, dict) and tuple(value) == envelope.FIELDS: + try: + return envelope.validate(value) + except envelope.EnvelopeError: + return None + if isinstance(value, dict): + for key in ("envelope", "result", "output"): + nested = value.get(key) + if isinstance(nested, dict) and tuple(nested) == envelope.FIELDS: + try: + return envelope.validate(nested) + except envelope.EnvelopeError: + pass + return None + + +def _parse_envelope(data): + """Accept JSON records only; prose is never mined for a verdict.""" + text = data.decode("utf-8", "replace").strip() + if not text: + return None + values = [] + try: + values.append(json.loads(text)) + except json.JSONDecodeError: + for line in text.splitlines(): + try: + values.append(json.loads(line)) + except json.JSONDecodeError: + continue + answer = None + for value in values: + candidate = _candidate_envelope(value) + if candidate is not None: + answer = candidate + return answer + + +def _load_breaker(): + spec = importlib.util.spec_from_file_location( + "_task_run_breaker", _BREAKER_PATH + ) + if spec is None or spec.loader is None: + raise RunError("supervision battery module could not be loaded") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _stream_spend(path, harness): + """Use the battery accounting so supervision and the ledger agree.""" + try: + battery = _load_breaker().Battery(SimpleNamespace( + repeat_k=8, window=40, disable="" + )) + with open(path, errors="ignore") as stream: + for line in stream: + battery.feed(line) + except (OSError, AttributeError, RunError) as exc: + return None, ( + "spend is unknown: session stream could not be read: " + f"{exc}" + ) + + observed = bool( + battery.per_msg + or battery.codex_total is not None + or battery.grok_current + or battery.grok_banked["total"] + or battery.grok_banked["out"] + ) + if not observed: + return None, ( + "spend is unknown: session stream contained no usage record" + ) + total = battery.total() + out = battery.total_out() + if ( + not isinstance(total, (int, float)) + or isinstance(total, bool) + or not isinstance(out, (int, float)) + or isinstance(out, bool) + or total < 0 + or out < 0 + or int(total) != total + or int(out) != out + ): + return None, ( + "spend is unknown: session stream usage was not integral" + ) + return { + "harness": harness, + "total": int(total), + "out": int(out), + "runs": 1, + }, None + + +# Keys are the breaker's own flag names without the leading dashes, so a +# config author writes what the battery actually receives and nothing +# has to translate. The underscore spellings are accepted too, because +# JSON authors reach for them; the SPEC was silent on this and the two +# halves of this module chose differently, which cost 15 tests. +_CAP_FLAGS = { + "cap": "--cap", + "cap-out": "--cap-out", + "size-mb": "--size-mb", + "repeat-n": "--repeat-n", + "repeat-k": "--repeat-k", + "err-min": "--err-min", + "total": "--cap", + "cap_out": "--cap-out", + "out": "--cap-out", + "stall": "--stall", + "size_mb": "--size-mb", + "repeat_n": "--repeat-n", + "repeat_k": "--repeat-k", + "err_min": "--err-min", + "window": "--window", + "interval": "--interval", + "disable": "--disable", +} + + +def _breaker_argv(stream, caps, *, pid=None, once=False, + tripped_file=None): + if caps is None: + caps = {} + if not isinstance(caps, dict): + raise RunError("runtime.caps must be an object") + if "cap" in caps and "total" in caps: + raise RunError("runtime.caps gives both cap and total") + if "cap_out" in caps and "out" in caps: + raise RunError("runtime.caps gives both cap_out and out") + unknown = sorted(set(caps) - set(_CAP_FLAGS)) + if unknown: + raise RunError(f"runtime.caps has unknown wires: {unknown}") + + argv = [sys.executable, str(_BREAKER_PATH), str(stream)] + if pid is not None: + argv.extend(["--pid", str(pid), "--terminate"]) + if once: + argv.append("--once") + if tripped_file is not None: + argv.extend(["--tripped-file", str(tripped_file)]) + for name, value in caps.items(): + if value is not None: + argv.extend([_CAP_FLAGS[name], str(value)]) + return argv + + +def _replay_trip(stream, caps): + """Apply the same battery in-process: stub means no process launch.""" + # Building the CLI argv centralizes cap-name validation even though the + # replay path deliberately does not execute that argv. + _breaker_argv(stream, caps) + values = { + "cap": 0, + "cap_out": 0, + "stall": 900, + "size_mb": 50, + "repeat_n": 5, + "repeat_k": 8, + "err_min": 12, + "window": 40, + "interval": 2, + "disable": "", + } + for name, value in (caps or {}).items(): + destination = { + "total": "cap", "out": "cap_out" + }.get(name, name) + values[destination] = value + try: + battery = _load_breaker().Battery(SimpleNamespace(**values)) + with open(stream, errors="ignore") as records: + for line in records: + battery.feed(line) + fired = battery.check(Path(stream)) + except (OSError, AttributeError, TypeError, ValueError) as exc: + raise RunError( + f"replay supervision could not read the stream: {exc}" + ) from exc + if fired is None: + return None + wire, detail = fired + return f"TRIPWIRE {wire}: {detail}" + + +def _path_values(value, key=None): + if isinstance(value, dict): + for child_key, child in value.items(): + yield from _path_values(child, str(child_key).lower()) + elif isinstance(value, list): + for child in value: + yield from _path_values(child, key) + elif isinstance(value, str) and key in { + "cwd", + "workdir", + "working_directory", + "workspace", + "workspace_root", + }: + yield value + + +def _stream_names_snapshot(path, root): + """Does this stream belong to the task running in `root`? + + A harness says so in one of two places and they differ by vendor: + Codex records its cwd in a structured field INSIDE the stream, while + Grok keys the store PATH by url-encoded cwd. Checking only the + contents rejected a stream whose path named the snapshot, and + checking only the path would reject Codex's flat `rollout-*.jsonl`. + Both count. + """ + expected = Path(root).resolve(strict=False) + marker = expected.name + if marker and marker in str(path): + return True + try: + with open(path, errors="ignore") as stream: + for line in stream: + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + for value in _path_values(record): + try: + named = ( + Path(value).expanduser().resolve(strict=False) + == expected + ) + except (OSError, TypeError, ValueError): + named = False + if named: + return True + except OSError: + pass + return False + + +def _stream_files(adapter): + try: + store = Path(os.path.expanduser(adapter["store"])) + pattern = adapter["stream"] + except (KeyError, TypeError) as exc: + raise RunError( + f"adapter stream facts are incomplete: {exc}" + ) from exc + if not isinstance(pattern, str) or not pattern: + raise RunError( + "adapter stream pattern must be a non-empty string" + ) + if not store.is_dir(): + return [] + try: + return [path for path in store.rglob(pattern) if path.is_file()] + except OSError: + return [] + + +def _stream_state(adapter): + state = {} + for path in _stream_files(adapter): + try: + stat = path.stat() + state[path] = (stat.st_mtime_ns, stat.st_size) + except OSError: + continue + return state + + +def _discover_stream(adapter, before, root): + candidates = [] + for path in _stream_files(adapter): + try: + stat = path.stat() + current = (stat.st_mtime_ns, stat.st_size) + except OSError: + continue + names_us = ( + adapter.get("stream_names_cwd") + and _stream_names_snapshot(path, root) + ) + # Naming this snapshot is a stronger claim than being new: no + # other session can name it. Newness was the weaker proxy used + # before there was a name to match on, and requiring BOTH rejects + # a stream that was already open when we launched. + if before.get(path) == current and not names_us: + continue + if adapter.get("stream_names_cwd") and not names_us: + # A fresh sibling stream is still the wrong evidence. The cwd + # identity makes simultaneous sessions separable. + continue + candidates.append((stat.st_mtime_ns, str(path), path)) + return max(candidates)[2] if candidates else None + + +def _remember_group(process): + """Record the child's process group while the parent is still alive.""" + if os.name != "posix": + return + with contextlib.suppress(OSError): + process._task_pgid = os.getpgid(process.pid) + + +def _terminate(process): + """Kill the harness, and its group even when the parent is already gone. + + The parent exiting is not the end of a runaway. The battery can + terminate the harness parent before the runner observes the trip, so + `poll()` is already non-null by the time we arrive -- and an early + return on that skipped the group kill in exactly the case the guard + exists for, leaving every descendant running while the task reported + `tripped` (Codex, PR #49). + + The group is the one recorded at launch by `_remember_group`, not one + derived here: after the parent is reaped `getpgid` raises, so a + guard written that way silently does nothing -- measured, on the + first attempt at this fix. + """ + exited = process.poll() is not None + pgid = getattr(process, "_task_pgid", None) + if os.name == "posix" and pgid is not None: + with contextlib.suppress(ProcessLookupError, PermissionError): + os.killpg(pgid, signal.SIGKILL) + elif not exited: + with contextlib.suppress(ProcessLookupError): + process.kill() + if not exited: + with contextlib.suppress(subprocess.TimeoutExpired): + process.wait(timeout=5) + + +def _adapter_argv(adapter, *, sandbox, prompt, root, runtime): + raw = adapter.get("argv") + if ( + not isinstance(raw, list) + or not raw + or not all(isinstance(part, str) for part in raw) + ): + raise RunError("adapter argv must be a non-empty string list") + values = { + "sandbox": sandbox, + "prompt": str(prompt), + "cwd": str(root), + "model": runtime.get("model") or "", + "effort": runtime.get("effort") or "", + "role": runtime.get("role") or "", + } + # A dial the caller set and the template ignores is worse than one + # that does not exist: CONTRACT.md sells `runtime.model` as what + # makes "eight cheap ones" and "one careful one" the same + # job, and every launch was silently using harness defaults + # instead (Codex + Grok, PR #49). Refusing is the honest floor. + # Wiring the real flags is a per-harness choice -- grok takes + # `-m/--effort`, claude `--model`, codex `-m` with no settled effort + # flag -- so the templates carry the placeholders and the job + # author decides; until one does, a set-but-unused dial refuses. + template = " ".join(raw) + dials = adapter.get("dials") or {} + extra = [] + for dial in ("model", "effort"): + if not runtime.get(dial): + continue + if ("{%s}" % dial) in template: + continue # the template places it itself + fragment = dials.get(dial) + if not fragment: + raise RunError( + f"runtime.{dial}={runtime[dial]!r} was requested but this " + f"adapter declares no way to pass it, so the launch would " + f"silently use the harness default" + ) + extra.extend(fragment) + try: + return [part.format_map(values) for part in [*raw, *extra]] + except (KeyError, ValueError) as exc: + raise RunError(f"adapter argv cannot render: {exc}") from exc + + +def _run_battery_once(stream, caps, tripped_file): + argv = _breaker_argv( + stream, caps, once=True, tripped_file=tripped_file + ) + try: + completed = subprocess.run( + argv, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + check=False, + shell=False, + ) + except (OSError, ValueError) as exc: + raise RunError( + f"supervision battery could not start: {exc}" + ) from exc + return ( + completed.returncode, + completed.stderr.decode("utf-8", "replace").strip(), + ) + + +def _rebuild(job, parsed, manifest, started, raw, spend, + extra_note=None, extra_artifacts=None): + artifacts = dict(parsed.get("artifacts") or {}) + # The runner owns raw evidence; an emitted handle must not redirect the + # caller to a file outside this snapshot. + artifacts["raw"] = str(raw) + artifacts.update(extra_artifacts or {}) + notes = [ + item for item in (parsed.get("note"), extra_note) if item + ] + return envelope.build( + job, + status=parsed["status"], + verdict=parsed["verdict"], + findings=parsed["findings"], + artifacts=artifacts, + spend=spend, + stamp=_stamp(manifest["ref"], started, _now()), + note="; ".join(notes) if notes else None, + ) + + +def _direct(require, runtime, manifest): + """Delegate while supplying provenance verify's API cannot accept.""" + def raw_artifact(): + descriptor, path = tempfile.mkstemp( + prefix="verify-", + suffix=".raw", + dir=manifest["root"], + ) + os.close(descriptor) + return path + + # These hooks only supply the snapshot-local facts absent from check's + # signature. Serializing the short override keeps concurrent direct runs + # from crossing raw paths or refs, and both hooks are restored unchanged. + with _DIRECT_LOCK: + original_raw = verify._raw_artifact + original_ref = verify._ref + verify._raw_artifact = raw_artifact + verify._ref = lambda cwd: manifest["ref"] + try: + # `claim` and `command` are verify's own arguments and the + # caller names them. Reading them out of `scope` and + # `constraints` guessed: a constraints LIST is not an argv, + # so every direct run came back invalid. + return verify.check( + require.get("claim", require.get("scope")), + require.get("command"), + cwd=manifest["root"], + expect=require.get("expect"), + expect_exit=require.get("expect_exit", 0), + timeout=runtime.get("timeout", 300), + ) + finally: + verify._raw_artifact = original_raw + verify._ref = original_ref + + +def _stub(job, runtime, manifest, started): + raw = _raw_path(manifest) + stream_name = runtime.get("replay") or runtime.get("stream") + if ( + not isinstance(stream_name, (str, os.PathLike)) + or not os.fspath(stream_name) + ): + return _invalid( + job, + "stub replay requires runtime.stream", + ref=manifest["ref"], + started=started, + artifacts={"raw": str(raw)}, + spend={ + "harness": "stub", "total": 0, "out": 0, "runs": 0 + }, + ) + try: + data = Path(stream_name).read_bytes() + _write(raw, data) + except (OSError, RunError) as exc: + return _invalid( + job, + f"stub replay stream {os.fspath(stream_name)!r} " + f"could not be read: {exc}", + ref=manifest["ref"], + started=started, + artifacts={"raw": str(raw)}, + spend={ + "harness": "stub", "total": 0, "out": 0, "runs": 0 + }, + ) + + try: + battery_note = _replay_trip(stream_name, runtime.get("caps")) + except RunError as exc: + return _invalid( + job, + str(exc), + ref=manifest["ref"], + started=started, + artifacts={"raw": str(raw)}, + spend={ + "harness": "stub", "total": 0, "out": 0, "runs": 0 + }, + ) + + spend, spend_note = _stream_spend(stream_name, "stub") + parsed = _parse_envelope(data) + if spend is None and parsed is not None: + # Replays can preserve a validated recorded accounting record. Live + # harnesses only trust their independent session stream. + spend = parsed["spend"] + if spend is None: + spend = { + "harness": "stub", "total": 0, "out": 0, "runs": 0 + } + + if battery_note is not None: + return envelope.build( + job, + status="tripped", + verdict=None, + artifacts={"raw": str(raw)}, + spend=spend, + stamp=_stamp(manifest["ref"], started, _now()), + note=battery_note, + ) + if parsed is None: + return _invalid( + job, + "stub replay emitted no valid structured envelope", + ref=manifest["ref"], + started=started, + artifacts={"raw": str(raw)}, + spend=spend, + ) + return _rebuild( + job, + parsed, + manifest, + started, + raw, + spend, + extra_note=( + spend_note if parsed["spend"] == spend else None + ), + ) + + +def _live(job, definition, context, require, runtime, adapter, + harness, manifest, started): + role = runtime.get("role") or definition.get("role") + empty_spend = { + "harness": harness, "total": 0, "out": 0, "runs": 0 + } + if role not in ("read", "write"): + return _invalid( + job, + "runtime.role must be 'read' or 'write'", + ref=manifest["ref"], + started=started, + spend=empty_spend, + ) + sandbox_map = adapter.get("sandbox") + if not isinstance(sandbox_map, dict) or role not in sandbox_map: + return _invalid( + job, + f"adapter {harness!r} has no sandbox for role {role!r}", + ref=manifest["ref"], + started=started, + spend=empty_spend, + ) + sandbox = sandbox_map[role] + if not isinstance(sandbox, str) or not sandbox: + return _invalid( + job, + f"adapter {harness!r} has an invalid {role!r} sandbox", + ref=manifest["ref"], + started=started, + spend=empty_spend, + ) + + timeout = runtime.get("timeout", 900) + if ( + not isinstance(timeout, (int, float)) + or isinstance(timeout, bool) + or timeout <= 0 + ): + return _invalid( + job, + "runtime.timeout must be a positive number of seconds", + ref=manifest["ref"], + started=started, + spend=empty_spend, + ) + try: + # Cap errors are launch refusals, not failures discovered after the + # harness has already been allowed to run unsupervised. + _breaker_argv("", runtime.get("caps")) + except RunError as exc: + return _invalid( + job, + str(exc), + ref=manifest["ref"], + started=started, + spend=empty_spend, + ) + + prompt_context = dict(context) + prompt_context.update({ + "ref": manifest["ref"], + "base": manifest["base"], + "diff": manifest["diff"], + "into": manifest["root"], + }) + try: + prompt_text = render(definition, prompt_context, require) + prompt_path = _prompt_path(manifest) + prompt_path.write_text(prompt_text, encoding="utf-8") + argv = _adapter_argv( + adapter, + sandbox=sandbox, + prompt=prompt_path, + root=manifest["root"], + runtime={**runtime, "role": role}, + ) + except (OSError, RunError) as exc: + return _invalid( + job, + str(exc), + ref=manifest["ref"], + started=started, + spend=empty_spend, + ) + + executable = shutil.which(argv[0]) + if executable is None: + return _invalid( + job, + f"harness {harness!r} CLI {argv[0]!r} is not on PATH", + ref=manifest["ref"], + started=started, + spend=empty_spend, + ) + argv[0] = executable + + prompt_mode = adapter.get("prompt") + if prompt_mode not in ("stdin", "file"): + return _invalid( + job, + f"adapter {harness!r} has unknown prompt mode " + f"{prompt_mode!r}", + ref=manifest["ref"], + started=started, + spend=empty_spend, + ) + + raw = _raw_path(manifest) + raw_handle = None + stdin_handle = None + try: + before = _stream_state(adapter) + # Not a context manager: both handles are handed to Popen and + # must outlive this block for the child's lifetime. They are + # closed in the finally below. + raw_handle = open(raw, "wb") # noqa: SIM115 + if prompt_mode == "stdin": + stdin_handle = open(prompt_path, "rb") # noqa: SIM115 + else: + stdin_handle = subprocess.DEVNULL + process = subprocess.Popen( + argv, + cwd=manifest["root"], + stdin=stdin_handle, + stdout=raw_handle, + stderr=subprocess.STDOUT, + shell=False, + start_new_session=(os.name == "posix"), + ) + # The group id is recorded HERE, while the parent is certainly + # alive. Read later it is unavailable: once the parent is reaped + # getpgid raises, and falling back to the pid risks a recycled + # one. start_new_session makes the child its own leader, so the + # group is its pid at this instant and never afterwards. + _remember_group(process) + except (OSError, ValueError, RunError) as exc: + if raw_handle is not None: + raw_handle.close() + if hasattr(stdin_handle, "close"): + stdin_handle.close() + return _invalid( + job, + f"harness {harness!r} could not launch: {exc}", + ref=manifest["ref"], + started=started, + artifacts={"raw": str(raw)}, + spend=empty_spend, + ) + + breaker = None + breaker_stderr = b"" + session_stream = None + trip_path = Path(manifest["root"]) / ".run-tripped.md" + deadline = time.monotonic() + timeout + tripped_note = None + invalid_note = None + try: + while process.poll() is None: + if breaker is None: + session_stream = _discover_stream( + adapter, before, manifest["root"] + ) + if session_stream is not None: + battery_argv = _breaker_argv( + session_stream, + runtime.get("caps"), + pid=process.pid, + tripped_file=trip_path, + ) + breaker = subprocess.Popen( + battery_argv, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + shell=False, + ) + elif breaker.poll() is not None: + breaker_stderr = breaker.communicate()[1] or b"" + if breaker.returncode == 3: + tripped_note = ( + breaker_stderr.decode( + "utf-8", "replace" + ).strip() + or "supervision battery tripped" + ) + else: + invalid_note = ( + f"supervision battery exited " + f"{breaker.returncode} before the harness" + ) + _terminate(process) + break + + if time.monotonic() >= deadline: + tripped_note = ( + f"harness exceeded timeout of {timeout} seconds" + ) + _terminate(process) + break + time.sleep(0.05) + + # A stream and process may appear and finish in one poll. It still + # gets a one-pass battery check so a fast runaway cannot evade R8. + if session_stream is None: + session_stream = _discover_stream( + adapter, before, manifest["root"] + ) + if session_stream is not None: + code, note = _run_battery_once( + session_stream, runtime.get("caps"), trip_path + ) + if code == 3: + tripped_note = note or "supervision battery tripped" + elif code != 0: + invalid_note = ( + f"supervision battery exited {code}: " + f"{note or 'no detail'}" + ) + + if breaker is not None and breaker.poll() is None: + try: + _, breaker_stderr = breaker.communicate(timeout=5) + except subprocess.TimeoutExpired: + breaker.kill() + _, breaker_stderr = breaker.communicate() + invalid_note = ( + "supervision battery did not stop after the harness" + ) + if breaker is not None and breaker.returncode == 3: + tripped_note = ( + breaker_stderr.decode("utf-8", "replace").strip() + or "supervision battery tripped" + ) + elif ( + breaker is not None + and breaker.returncode not in (None, 0) + and tripped_note is None + ): + breaker_detail = breaker_stderr.decode( + "utf-8", "replace" + ).strip() + invalid_note = ( + f"supervision battery exited {breaker.returncode}: " + f"{breaker_detail or 'no detail'}" + ) + return_code = process.wait() + except (OSError, ValueError, RunError) as exc: + _terminate(process) + invalid_note = f"supervision failed: {exc}" + return_code = process.wait() + finally: + raw_handle.close() + if hasattr(stdin_handle, "close"): + stdin_handle.close() + + artifacts = {"raw": str(raw)} + if trip_path.exists(): + artifacts["trip"] = str(trip_path) + if session_stream is None: + spend = dict(empty_spend) + spend_note = "spend is unknown: no session stream found" + else: + spend, spend_note = _stream_spend(session_stream, harness) + if spend is None: + spend = dict(empty_spend) + + if tripped_note is not None: + if spend_note: + tripped_note = f"{tripped_note}; {spend_note}" + return envelope.build( + job, + status="tripped", + verdict=None, + artifacts=artifacts, + spend=spend, + stamp=_stamp(manifest["ref"], started, _now()), + note=tripped_note, + ) + if invalid_note is not None: + if spend_note: + invalid_note = f"{invalid_note}; {spend_note}" + return _invalid( + job, + invalid_note, + ref=manifest["ref"], + started=started, + artifacts=artifacts, + spend=spend, + ) + if return_code != 0: + note = f"harness {harness!r} exited {return_code}" + if spend_note: + note = f"{note}; {spend_note}" + return _invalid( + job, + note, + ref=manifest["ref"], + started=started, + artifacts=artifacts, + spend=spend, + ) + try: + data = raw.read_bytes() + except OSError as exc: + note = f"raw harness output could not be read: {exc}" + if spend_note: + note = f"{note}; {spend_note}" + return _invalid( + job, + note, + ref=manifest["ref"], + started=started, + artifacts=artifacts, + spend=spend, + ) + parsed = _parse_envelope(data) + if parsed is None: + note = "harness emitted no valid structured envelope" + if spend_note: + note = f"{note}; {spend_note}" + return _invalid( + job, + note, + ref=manifest["ref"], + started=started, + artifacts=artifacts, + spend=spend, + ) + return _rebuild( + job, + parsed, + manifest, + started, + raw, + spend, + extra_note=spend_note, + extra_artifacts={ + key: value for key, value in artifacts.items() + if key != "raw" + }, + ) + + +def run(job, *, context, require, runtime, jobs=None, + adapters=None) -> dict: + """Snapshot, execute the selected adapter, and return one envelope.""" + started = _now() + if not isinstance(runtime, dict): + return _invalid( + job, "runtime must be an object", started=started + ) + if not isinstance(require, dict): + return _invalid( + job, "require must be an object", started=started + ) + + manifest = None + try: + registry = ( + load_jobs() if jobs is None else jobs + ) + if not isinstance(registry, dict): + raise RunError("jobs must be an object") + if job not in registry: + raise RunError( + f"job {job!r} is absent from the registry" + ) + definition = registry[job] + if not isinstance(definition, dict): + raise RunError(f"job {job!r} must be an object") + # Resolve state before launch-policy checks so every runnable + # job is stamped from the fileset manifest, including a later + # refusal for an unknown or unavailable harness. + manifest = _snapshot(context) + harness = _select_harness(definition, runtime) + table = ADAPTERS if adapters is None else adapters + if not isinstance(table, dict): + raise RunError("adapters must be an object") + if harness not in table: + raise RunError(f"unknown harness {harness!r}") + adapter = table[harness] + if not isinstance(adapter, dict): + raise RunError(f"adapter {harness!r} must be an object") + except (RunError, TypeError) as exc: + return _invalid( + job, + str(exc), + ref=manifest["ref"] if manifest is not None else None, + started=started, + ) + + if adapter.get("direct") is True: + # Direct verification owns its envelope, including zero token spend; + # rebuilding it here would make delegation only approximate. + try: + return _direct(require, runtime, manifest) + except (OSError, TypeError, ValueError) as exc: + return _invalid( + job, + f"direct verification could not run: {exc}", + ref=manifest["ref"], + started=started, + spend={ + "harness": None, + "total": 0, + "out": 0, + "runs": 0, + }, + ) + if adapter.get("replay") is True: + return _stub(job, runtime, manifest, started) + return _live( + job, + definition, + context, + require, + runtime, + adapter, + harness, + manifest, + started, + ) + + +def _parser(): + parser = _Parser(prog="run.py") + parser.add_argument("job") + parser.add_argument("--repo", required=True) + parser.add_argument("--ref", required=True) + parser.add_argument("--base") + parser.add_argument("--harness") + parser.add_argument("--role", choices=("read", "write")) + parser.add_argument("--into") + return parser + + +def main(argv=None) -> int: + try: + arguments = _parser().parse_args(argv) + except _UsageError as exc: + print(exc, file=sys.stderr) + return 64 + except SystemExit as exc: + # argparse owns help output while the callable API retains control. + return int(exc.code) + + result = run( + arguments.job, + context={ + "repo": arguments.repo, + "ref": arguments.ref, + "base": arguments.base, + "include": None, + "into": arguments.into, + }, + require={ + "scope": "the requested fileset", + "constraints": [], + }, + runtime={ + "harness": arguments.harness, + "model": None, + "effort": None, + "role": arguments.role, + "caps": {}, + "timeout": 900, + }, + ) + print(json.dumps(result)) + if result["verdict"] == "approve": + return 0 + if result["verdict"] == "changes": + return 1 + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/devlane/task/tests/support.py b/ops/devlane/task/tests/support.py new file mode 100644 index 0000000..f05d68b --- /dev/null +++ b/ops/devlane/task/tests/support.py @@ -0,0 +1,37 @@ +"""Load a task-app module by path, the way the other suites do. + +The dev-lane apps are standalone scripts, not an installed package: +`ops/devlane/task/envelope.py` has no importable dotted name. Every suite +here loads it from its path, so a test file can be run on its own +(`python3 ops/devlane/task/tests/test_envelope.py`) without a sys.path +ceremony repeated in each one. +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +APP = Path(__file__).resolve().parents[1] +REPO = APP.parents[2] + +# The app dir goes on sys.path so its standalone scripts can import one +# another plainly — `import envelope` inside verify.py. This mirrors +# ops/devlane/workflow/tests/support.py, which does the same for the same +# reason. Without it a module loaded BY PATH cannot resolve its +# siblings, and the failure looks like a missing dependency rather +# than a missing path entry. +if str(APP) not in sys.path: + sys.path.insert(0, str(APP)) + + +def load(name): + """Import /.py under a task_ prefix and return it.""" + path = APP / f"{name}.py" + spec = importlib.util.spec_from_file_location(f"task_{name}", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module diff --git a/ops/devlane/task/tests/test_envelope.py b/ops/devlane/task/tests/test_envelope.py new file mode 100644 index 0000000..733efb1 --- /dev/null +++ b/ops/devlane/task/tests/test_envelope.py @@ -0,0 +1,275 @@ +"""The envelope: the shape a caller routes on without reading the work. + +Written from `ops/devlane/task/CONTRACT.md` §The envelope and §Statuses, +before the module existed. Each test names the contract rule it pins: + + R1 a task that could not look must not return `approve` + R2 a finding with no reproduction is an opinion, and the envelope + says so rather than hiding it + R3 artifacts are handles, not contents + R4 every task returns the same shape + R5 spend is present so worth.py can price a job + R6 the stamp carries the ref the task actually read + R7 counts are derived, never asserted — a supplied count that + disagrees with the findings is a truncated list, not a tally +""" + +from __future__ import annotations + +import json +import unittest + +import support + +envelope = support.load("envelope") + +STAMP = {"ref": "f0b8bb3", "started": "2026-08-22T12:00:00Z", + "ended": "2026-08-22T12:04:00Z"} + + +def a_finding(severity="p2", reproduce="python3 -m unittest x"): + return envelope.finding( + severity, "run.py §launch", "the battery is never armed", + reproduce=reproduce) + + +class TheRefusalRule(unittest.TestCase): + """R1 — status routes; verdict judges; only a task that ran may approve.""" + + def test_an_ok_task_may_approve(self): + env = envelope.build("verify", status="ok", verdict="approve", + stamp=STAMP) + self.assertEqual(env["verdict"], "approve") + + def test_an_invalid_task_may_not_approve(self): + with self.assertRaises(envelope.EnvelopeError): + envelope.build("verify", status="invalid", verdict="approve", + stamp=STAMP, note="ref absent") + + def test_a_tripped_task_may_not_approve(self): + # A partial deliverable has not seen the whole job, so it has + # no standing to approve it. + with self.assertRaises(envelope.EnvelopeError): + envelope.build("sweep", status="tripped", verdict="approve", + stamp=STAMP, note="cap wire tripped") + + def test_a_tripped_task_may_still_report_changes(self): + env = envelope.build("sweep", status="tripped", verdict="changes", + findings=[a_finding("p1")], stamp=STAMP, + note="cap wire tripped") + self.assertEqual((env["status"], env["verdict"]), + ("tripped", "changes")) + + def test_invalid_carries_no_verdict_at_all(self): + env = envelope.invalid("verify", "harness missing", stamp=STAMP) + self.assertIsNone(env["verdict"]) + self.assertEqual(env["status"], "invalid") + + def test_an_invalid_task_may_not_report_changes_either(self): + # `invalid` means the task could not look. Not-looking produces + # no judgement at all, not a milder one. + with self.assertRaises(envelope.EnvelopeError): + envelope.build("verify", status="invalid", verdict="changes", + stamp=STAMP, note="snapshot failed") + + def test_a_finding_smuggled_past_its_constructor_is_refused(self): + env = envelope.build("sweep", status="ok", verdict="changes", + findings=[a_finding("p1")], stamp=STAMP) + env["findings"][0]["severity"] = "p4" + env["counts"] = dict.fromkeys(("p1", "p2", "p3", "opinions"), 0) + with self.assertRaises(envelope.EnvelopeError): + envelope.validate(env) + + def test_invalid_must_say_why(self): + with self.assertRaises(envelope.EnvelopeError): + envelope.invalid("verify", "", stamp=STAMP) + + def test_an_unknown_status_is_refused(self): + with self.assertRaises(envelope.EnvelopeError): + envelope.build("verify", status="done", stamp=STAMP) + + def test_an_unknown_verdict_is_refused(self): + with self.assertRaises(envelope.EnvelopeError): + envelope.build("verify", status="ok", verdict="lgtm", stamp=STAMP) + + +class AFindingWithoutAReproductionSaysSo(unittest.TestCase): + """R2 — the absence is a value in the data, not a missing key.""" + + def test_the_key_is_present_and_null(self): + env = envelope.build("adversarial-review", status="ok", + verdict="changes", + findings=[a_finding(reproduce=None)], + stamp=STAMP) + self.assertIn("reproduce", env["findings"][0]) + self.assertIsNone(env["findings"][0]["reproduce"]) + + def test_blank_whitespace_is_absence_not_a_command(self): + env = envelope.build("adversarial-review", status="ok", + verdict="changes", + findings=[a_finding(reproduce=" ")], + stamp=STAMP) + self.assertIsNone(env["findings"][0]["reproduce"]) + + def test_the_envelope_counts_the_opinions(self): + env = envelope.build( + "adversarial-review", status="ok", verdict="changes", + findings=[a_finding(reproduce=None), a_finding()], stamp=STAMP) + self.assertEqual(env["counts"]["opinions"], 1) + + def test_a_finding_keeps_the_contract_key_order(self): + # Same reason the top-level key set is ordered: two envelopes + # from two runs are read side by side, and a field that moves + # costs the reader the diff. + env = envelope.build("sweep", status="ok", verdict="changes", + findings=[a_finding("p1")], stamp=STAMP) + f = env["findings"][0] + env["findings"][0] = {"claim": f["claim"], "severity": f["severity"], + "where": f["where"], + "reproduce": f["reproduce"]} + with self.assertRaises(envelope.EnvelopeError): + envelope.validate(env) + + def test_a_finding_needs_a_severity_a_where_and_a_claim(self): + for kwargs in ({"severity": "p4"}, {"where": ""}, {"claim": ""}): + with self.subTest(**kwargs), self.assertRaises( + envelope.EnvelopeError): + envelope.finding(**{ + "severity": "p2", "where": "f.py §s", + "claim": "c", **kwargs}) + + +class ArtifactsAreHandles(unittest.TestCase): + """R3 — the prose stays on disk; the caller reads it if it decides to.""" + + def test_a_path_is_accepted(self): + env = envelope.build("verify", status="ok", verdict="approve", + artifacts={"raw": "/tmp/t/raw.txt"}, stamp=STAMP) + self.assertEqual(env["artifacts"]["raw"], "/tmp/t/raw.txt") + + def test_inlined_prose_is_refused(self): + with self.assertRaises(envelope.EnvelopeError): + envelope.build("verify", status="ok", verdict="approve", + artifacts={"raw": "VERDICT: APPROVE\nFINDINGS:\n"}, + stamp=STAMP) + + def test_a_handle_longer_than_a_path_is_refused(self): + with self.assertRaises(envelope.EnvelopeError): + envelope.build("verify", status="ok", verdict="approve", + artifacts={"raw": "x" * (envelope.HANDLE_MAX + 1)}, + stamp=STAMP) + + def test_an_empty_handle_is_refused(self): + with self.assertRaises(envelope.EnvelopeError): + envelope.build("verify", status="ok", verdict="approve", + artifacts={"raw": ""}, stamp=STAMP) + + +class EverySoTaskReturnsTheSameShape(unittest.TestCase): + """R4/R5/R6 — one key set, spend always priced, stamp always pinned.""" + + def test_the_key_set_is_fixed_and_ordered(self): + env = envelope.build("verify", status="ok", verdict="approve", + stamp=STAMP) + self.assertEqual(list(env), list(envelope.FIELDS)) + + def test_an_unknown_top_level_key_is_refused(self): + env = envelope.build("verify", status="ok", verdict="approve", + stamp=STAMP) + env["transcript"] = "...the whole conversation..." + with self.assertRaises(envelope.EnvelopeError): + envelope.validate(env) + + def test_spend_is_present_even_when_nothing_was_spent(self): + env = envelope.build("verify", status="ok", verdict="approve", + stamp=STAMP) + self.assertEqual(env["spend"], + {"harness": None, "total": 0, "out": 0, "runs": 0}) + + def test_negative_spend_is_refused(self): + with self.assertRaises(envelope.EnvelopeError): + envelope.build("verify", status="ok", verdict="approve", + spend={"harness": "grok", "total": -1, "out": 0, + "runs": 1}, + stamp=STAMP) + + def test_the_stamp_names_the_ref_that_was_read(self): + env = envelope.build("verify", status="ok", verdict="approve", + stamp=STAMP) + self.assertEqual(env["stamp"]["ref"], "f0b8bb3") + + def test_a_stamp_without_a_ref_is_refused(self): + with self.assertRaises(envelope.EnvelopeError): + envelope.build("verify", status="ok", verdict="approve", + stamp={"started": STAMP["started"], + "ended": STAMP["ended"]}) + + def test_it_round_trips_through_json_unchanged(self): + env = envelope.build("adversarial-review", status="ok", + verdict="changes", findings=[a_finding("p1")], + artifacts={"diff": "/tmp/t/d.patch"}, + spend={"harness": "stub", "total": 9, "out": 3, + "runs": 1}, + stamp=STAMP) + self.assertEqual(json.loads(json.dumps(env)), env) + + +class CountsAreDerivedNeverAsserted(unittest.TestCase): + """R7 — a tally that can disagree with the list is a tally that lies.""" + + def test_build_tallies_the_findings_itself(self): + env = envelope.build( + "sweep", status="ok", verdict="changes", + findings=[a_finding("p1"), a_finding("p3"), a_finding("p3")], + stamp=STAMP) + self.assertEqual( + env["counts"], + {"p1": 1, "p2": 0, "p3": 2, "opinions": 0}) + + def test_validate_refuses_a_tally_that_stopped_matching(self): + env = envelope.build("sweep", status="ok", verdict="changes", + findings=[a_finding("p1")], stamp=STAMP) + env["counts"]["p1"] = 4 + with self.assertRaises(envelope.EnvelopeError): + envelope.validate(env) + + +class AMalformedEnvelopeIsInvalidNotACrash(unittest.TestCase): + """Worker output is untrusted, so a wrong TYPE must refuse, not raise. + + `_candidate_envelope` in run.py catches EnvelopeError only. Anything + else escapes the parser and kills the run, which turns "the worker + replied badly" into "the runner fell over" -- the opposite of the + contract's rule that unparseable output becomes an `invalid` + envelope. Verified before the fix: `findings: None` raised + TypeError: 'NoneType' object is not iterable (Codex, PR #49). + """ + + def _valid(self): + return envelope.build("sweep", status="ok", verdict="changes", + findings=[a_finding("p1")], stamp=STAMP) + + def test_every_container_field_refuses_a_wrong_type(self): + for field in ("findings", "counts", "artifacts", "spend", "stamp"): + with self.subTest(field=field): + env = self._valid() + env[field] = None + # the plant must be in the fixture before the assertion + self.assertIsNone(env[field], "INVALID fixture: not planted") + with self.assertRaises(envelope.EnvelopeError): + envelope.validate(env) + + def test_a_finding_that_is_not_a_dict_refuses(self): + env = self._valid() + env["findings"] = [3] + self.assertEqual(env["findings"], [3], "INVALID fixture: not planted") + with self.assertRaises(envelope.EnvelopeError): + envelope.validate(env) + + def test_the_valid_envelope_these_cases_corrupt_still_passes(self): + """The positive control: without the plant, validate accepts.""" + self.assertIsNotNone(envelope.validate(self._valid())) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/task/tests/test_envelope_schema.py b/ops/devlane/task/tests/test_envelope_schema.py new file mode 100644 index 0000000..9286bc6 --- /dev/null +++ b/ops/devlane/task/tests/test_envelope_schema.py @@ -0,0 +1,364 @@ +"""U10: ENVELOPE_SCHEMA is the allowed envelope shape. + +Authored from harness-research.md §4 and D-ENV-1: one place, the nine +keys, commit optional, verdict nullable. Owner 2026-08-30: pin the +allowed shape (types, required keys, nullable verdict, optional +commit) rather than enumerating denials. Extra-key and missing-key +remain the two CUE-doctrine negatives the brief named. + +The module already owns the nine keys; the export does not exist at +HEAD. Implementation (envelope.py internals, launch.py) was not read +beyond the existing test import of FIELDS / build / invalid. +""" + +from __future__ import annotations + +import unittest + +import support + +envelope = support.load("envelope") + +NINE = ( + "job", "status", "verdict", "counts", "findings", + "artifacts", "spend", "stamp", "note", +) +STATUS_ENUM = {"ok", "invalid", "tripped"} +VERDICT_ENUM = {"approve", "changes", None} +COUNT_KEYS = ("p1", "p2", "p3", "opinions") +STAMP = {"ref": "f0b8bb3", "started": "2026-08-22T12:00:00Z", + "ended": "2026-08-22T12:04:00Z"} + + +def _schema(): + return getattr(envelope, "ENVELOPE_SCHEMA", None) + + +def _type_list(node): + if not isinstance(node, dict): + return [] + raw = node.get("type") + if raw is None: + types = [] + elif isinstance(raw, list): + types = list(raw) + else: + types = [raw] + for alt in node.get("anyOf") or node.get("oneOf") or (): + types.extend(_type_list(alt)) + return types + + +def _allows_null(node): + if node is True: + return True + types = _type_list(node) + if "null" in types: + return True + enum = node.get("enum") if isinstance(node, dict) else None + return enum is not None and None in enum + + +def _prop(schema, key): + props = (schema or {}).get("properties") or {} + return props.get(key) + + +def _value_matches(types, value): + if not types: + return False + if value is None: + return "null" in types + if isinstance(value, bool): + return "boolean" in types + mapping = ( + (str, "string"), + (int, "integer"), + (float, "number"), + (list, "array"), + (dict, "object"), + ) + for py, name in mapping: + if isinstance(value, py): + if name == "integer" and "number" in types: + return True + return name in types + return False + + +def _instance_matches_allowed(schema, instance): + """Does this instance sit inside the schema's declared allowed shape? + + Reads property types. A types-free or wrong-typed schema cannot + accept a real envelope — that is the mutant the skeptic ran. + """ + if not isinstance(schema, dict): + return False, "ENVELOPE_SCHEMA is not an object" + if schema.get("type") not in (None, "object"): + return False, f"schema type is {schema.get('type')!r}, not object" + if not isinstance(instance, dict): + return False, "instance is not an object" + required = schema.get("required") or [] + missing = [k for k in required if k not in instance] + if missing: + return False, f"missing required {missing}" + props = schema.get("properties") or {} + extra = [k for k in instance if k not in props] + if extra and schema.get("additionalProperties") is False: + return False, f"extra keys {extra}" + for key in required: + node = props.get(key) + types = _type_list(node) if isinstance(node, dict) else [] + if not types: + return False, f"required {key!r} has no type in the schema" + if key in instance and not _value_matches(types, instance[key]): + return ( + False, + f"{key} value {type(instance[key]).__name__} not in {types}", + ) + commit_node = props.get("commit") + if "commit" in instance: + types = _type_list(commit_node) if isinstance(commit_node, dict) else [] + if not types or not _value_matches(types, instance["commit"]): + return False, "optional commit is present but not typed as object" + return True, "" + + +class EnvelopeSchemaIsExported(unittest.TestCase): + def test_envelope_schema_is_exported_from_one_place(self): + """Scenario: ENVELOPE_SCHEMA is exported from envelope.py with the nine keys""" + self.assertTrue( + hasattr(envelope, "ENVELOPE_SCHEMA"), + "envelope.py must export ENVELOPE_SCHEMA (JSON Schema)", + ) + schema = envelope.ENVELOPE_SCHEMA + self.assertIsInstance(schema, dict) + self.assertEqual(schema.get("type"), "object") + + def test_schema_requires_the_nine_keys(self): + """Scenario: ENVELOPE_SCHEMA is exported from envelope.py with the nine keys""" + schema = _schema() + self.assertIsInstance( + schema, dict, + "ENVELOPE_SCHEMA must be exported as a JSON Schema object", + ) + required = schema.get("required") + self.assertIsInstance(required, list, f"required={required!r}") + self.assertEqual( + list(required), list(NINE), + "required must be the nine envelope keys, in contract order", + ) + self.assertEqual(list(envelope.FIELDS), list(NINE)) + + def test_commit_is_optional(self): + """Scenario: ENVELOPE_SCHEMA is exported from envelope.py with the nine keys""" + schema = _schema() + self.assertIsInstance( + schema, dict, + "ENVELOPE_SCHEMA must be exported as a JSON Schema object", + ) + required = schema.get("required") or [] + self.assertIn( + "job", required, + "commit-optional is only meaningful on a schema that requires " + f"the nine keys; required={required!r}", + ) + self.assertNotIn("commit", required) + props = schema.get("properties") or {} + self.assertIn( + "commit", props, + "commit is a known optional property, not an extra key", + ) + self.assertIn( + "object", _type_list(props["commit"]), + f"optional commit must be an object, got {props['commit']!r}", + ) + + def test_verdict_is_nullable(self): + """Scenario: ENVELOPE_SCHEMA is exported from envelope.py with the nine keys""" + schema = _schema() + self.assertIsInstance( + schema, dict, + "ENVELOPE_SCHEMA must be exported as a JSON Schema object", + ) + props = schema.get("properties") or {} + self.assertIn("verdict", props) + self.assertTrue( + _allows_null(props["verdict"]), + f"verdict must allow null, got {props['verdict']!r}", + ) + + +class EnvelopeSchemaPinsTheAllowedShape(unittest.TestCase): + def test_schema_pins_the_allowed_property_types(self): + """Scenario: ENVELOPE_SCHEMA pins the allowed property types""" + schema = _schema() + self.assertIsInstance( + schema, dict, + "ENVELOPE_SCHEMA must be exported as a JSON Schema object", + ) + props = schema.get("properties") or {} + expected = { + "job": "string", + "status": "string", + "counts": "object", + "findings": "array", + "artifacts": "object", + "spend": "object", + "stamp": "object", + "note": "string", + } + for key, want in expected.items(): + self.assertIn(key, props, f"properties missing {key}") + types = _type_list(props[key]) + self.assertIn( + want, types, + f"{key} must be typed {want}, got {props[key]!r}", + ) + verdict_types = _type_list(props["verdict"]) + self.assertIn("string", verdict_types, props["verdict"]) + self.assertIn("null", verdict_types, props["verdict"]) + + def test_status_enum_is_ok_invalid_tripped(self): + """Scenario: ENVELOPE_SCHEMA pins the allowed property types""" + schema = _schema() + self.assertIsInstance(schema, dict, "ENVELOPE_SCHEMA must be exported") + node = _prop(schema, "status") + self.assertIsInstance(node, dict, f"status node={node!r}") + enum = node.get("enum") + self.assertIsInstance(enum, list, f"status.enum={enum!r}") + self.assertEqual(set(enum), STATUS_ENUM, f"status.enum={enum!r}") + + def test_verdict_enum_is_approve_changes_null(self): + """Scenario: ENVELOPE_SCHEMA pins the allowed property types""" + schema = _schema() + self.assertIsInstance(schema, dict, "ENVELOPE_SCHEMA must be exported") + node = _prop(schema, "verdict") + self.assertIsInstance(node, dict, f"verdict node={node!r}") + enum = node.get("enum") + self.assertIsInstance(enum, list, f"verdict.enum={enum!r}") + self.assertEqual(set(enum), VERDICT_ENUM, f"verdict.enum={enum!r}") + + def test_counts_is_an_object_with_integer_tallies(self): + """Scenario: ENVELOPE_SCHEMA pins the allowed property types""" + schema = _schema() + self.assertIsInstance(schema, dict, "ENVELOPE_SCHEMA must be exported") + node = _prop(schema, "counts") + self.assertIsInstance(node, dict, f"counts node={node!r}") + self.assertIn("object", _type_list(node)) + cprops = node.get("properties") or {} + for key in COUNT_KEYS: + self.assertIn(key, cprops, f"counts.properties missing {key}") + self.assertIn( + "integer", _type_list(cprops[key]), + f"counts.{key} must be integer, got {cprops[key]!r}", + ) + + def test_stamp_is_an_object_with_a_ref(self): + """Scenario: ENVELOPE_SCHEMA pins the allowed property types""" + schema = _schema() + self.assertIsInstance(schema, dict, "ENVELOPE_SCHEMA must be exported") + node = _prop(schema, "stamp") + self.assertIsInstance(node, dict, f"stamp node={node!r}") + self.assertIn("object", _type_list(node)) + sprops = node.get("properties") or {} + self.assertIn("ref", sprops, f"stamp.properties={sprops!r}") + self.assertIn( + "string", _type_list(sprops["ref"]), + f"stamp.ref must be string, got {sprops['ref']!r}", + ) + required = node.get("required") or [] + self.assertIn("ref", required, f"stamp.required={required!r}") + + def test_findings_items_are_objects(self): + """Scenario: ENVELOPE_SCHEMA pins the allowed property types""" + schema = _schema() + self.assertIsInstance(schema, dict, "ENVELOPE_SCHEMA must be exported") + node = _prop(schema, "findings") + self.assertIsInstance(node, dict, f"findings node={node!r}") + items = node.get("items") + self.assertIsInstance(items, dict, f"findings.items={items!r}") + self.assertIn("object", _type_list(items), f"findings.items={items!r}") + + +class EnvelopeSchemaPositiveAndNegative(unittest.TestCase): + def _complete(self): + env = envelope.build( + "verify", status="ok", verdict="approve", stamp=STAMP, + ) + env = dict(env) + env["commit"] = {"subject": "dispatch: pin U10", "body": "why"} + return env + + def test_a_complete_envelope_including_commit_is_accepted(self): + """Scenario: a complete envelope is a positive schema example""" + schema = _schema() + self.assertIsInstance( + schema, dict, + "ENVELOPE_SCHEMA must be exported as a JSON Schema object", + ) + env = self._complete() + self.assertIn("commit", env) + self.assertEqual(list(k for k in env if k != "commit"), list(NINE)) + ok, why = _instance_matches_allowed(schema, env) + self.assertTrue(ok, why) + + def test_a_null_verdict_on_invalid_is_accepted(self): + """Scenario: a complete envelope is a positive schema example""" + schema = _schema() + self.assertIsInstance( + schema, dict, + "ENVELOPE_SCHEMA must be exported as a JSON Schema object", + ) + env = envelope.invalid("verify", "could not look", stamp=STAMP) + self.assertIsNone(env["verdict"], "plant: invalid carries no verdict") + self.assertNotIn("commit", env, "plant: commit is absent") + ok, why = _instance_matches_allowed(schema, env) + self.assertTrue(ok, why) + + def test_an_extra_key_is_rejected(self): + """Scenario: an extra key is a negative schema example""" + schema = _schema() + self.assertIsInstance( + schema, dict, + "ENVELOPE_SCHEMA must be exported as a JSON Schema object", + ) + self.assertIs( + schema.get("additionalProperties"), False, + "extra keys are refused only when additionalProperties is false", + ) + env = self._complete() + env["transcript"] = "the whole conversation" + self.assertIn("transcript", env, "plant: extra key landed") + ok, why = _instance_matches_allowed(schema, env) + self.assertFalse( + ok, + "an extra top-level key must fail the schema: " + f"why={why!r} env_keys={list(env)}", + ) + + def test_a_missing_key_is_rejected(self): + """Scenario: a missing key is a negative schema example""" + schema = _schema() + self.assertIsInstance( + schema, dict, + "ENVELOPE_SCHEMA must be exported as a JSON Schema object", + ) + required = schema.get("required") or [] + self.assertIn( + "note", required, + "missing-note is only a negative if note is required", + ) + env = self._complete() + del env["note"] + self.assertNotIn("note", env, "plant: note removed") + ok, why = _instance_matches_allowed(schema, env) + self.assertFalse( + ok, + "a missing required key must fail the schema: " + f"why={why!r} env_keys={list(env)}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/task/tests/test_fileset.py b/ops/devlane/task/tests/test_fileset.py new file mode 100644 index 0000000..1cdee1b --- /dev/null +++ b/ops/devlane/task/tests/test_fileset.py @@ -0,0 +1,721 @@ +"""The fileset: the smallest snapshot a task needs, and the proof +that the pruning did not change the answer. + +Written from SPEC.md, before the module existed. Each test names the +contract rule it pins: + + B1 derive is the changed paths, sorted and unique + B2 derive also takes repo-relative paths named in the diff text + B3 a path that does not exist at ref is never in the derived set + B4 derive is deterministic + B5 snapshot(include=...) writes exactly that set + B6 a missing include is a FilesetError, not a skip + B7 an empty include set is a FilesetError + B8 snapshot(whole=True) writes every file tracked at ref + B9 FILESET.diff is written iff a base is given + B10 FILESET.md names the ref, the base, and every included path + B11 snapshot never modifies the source repo + B12 manifest bytes equal the summed size of the files written + B13 parity is True iff whole-tree and pruned results match + B14 a command that cannot run raises; identical failure is not that + B15 main is a CLI: JSON on 0, FilesetError on 1, usage on 64 +""" + +from __future__ import annotations + +import contextlib +import hashlib +import io +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +import support + +fileset = support.load("fileset") + +_META = frozenset({"FILESET.md", "FILESET.diff"}) + + +def _git_env(home: Path) -> dict: + # GIT_DIR / GIT_WORK_TREE in the caller environment would aim + # git at the snapshot (or its parent). The fixtures must be + # closed worlds. + env = {k: v for k, v in os.environ.items() + if not k.startswith("GIT_") and k != "XDG_CONFIG_HOME"} + env.update({ + "HOME": str(home), + "GIT_AUTHOR_NAME": "fileset-test", + "GIT_AUTHOR_EMAIL": "fileset-test@example.test", + "GIT_COMMITTER_NAME": "fileset-test", + "GIT_COMMITTER_EMAIL": "fileset-test@example.test", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + }) + return env + + +def payload_paths(into) -> list[str]: + root = Path(into) + if not root.exists(): + return [] + found = [] + for p in root.rglob("*"): + if p.is_file(): + rel = p.relative_to(root).as_posix() + if rel not in _META: + found.append(rel) + return sorted(found) + + +def all_file_paths(into) -> list[str]: + root = Path(into) + if not root.exists(): + return [] + return sorted( + p.relative_to(root).as_posix() + for p in root.rglob("*") if p.is_file()) + + +@contextlib.contextmanager +def _cwd(path): + prev = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(prev) + + +def run_main(argv, *, cwd): + out, err = io.StringIO(), io.StringIO() + with _cwd(cwd), contextlib.redirect_stdout(out), \ + contextlib.redirect_stderr(err): + code = fileset.main(argv) + return code, out.getvalue(), err.getvalue() + + +class _TempRepo(unittest.TestCase): + """A throwaway git repo. Never the snapshot's own tree.""" + + def setUp(self): + self._td = tempfile.TemporaryDirectory() + self.home = Path(self._td.name) + self.repo = self.home / "repo" + self.into = self.home / "into" + self.pruned = self.home / "pruned" + self.repo.mkdir() + self.into.mkdir() + self.env = _git_env(self.home) + # Branch name is irrelevant: every test pins SHAs, not HEAD. + self._git("init") + self._git("config", "user.name", "fileset-test") + self._git("config", "user.email", "fileset-test@example.test") + self._git("config", "commit.gpgsign", "false") + + def tearDown(self): + self._td.cleanup() + + def _git(self, *args): + r = subprocess.run( + ["git", *args], cwd=self.repo, env=self.env, + capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError( + f"git {args} failed ({r.returncode}): {r.stderr}") + return r + + def _write(self, rel, content): + p = self.repo / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + return p + + def _commit(self, msg): + self._git("add", "-A") + self._git("commit", "-m", msg) + return self._git("rev-parse", "HEAD").stdout.strip() + + def _fingerprint(self): + """User-visible repo state. Not .git bytes — status updates + those without having checked anything out.""" + files = {} + for p in self.repo.rglob("*"): + if ".git" in p.parts or not p.is_file(): + continue + rel = p.relative_to(self.repo).as_posix() + files[rel] = ( + p.stat().st_mode, + hashlib.sha256(p.read_bytes()).hexdigest(), + ) + return { + "files": files, + "head": self._git("rev-parse", "HEAD").stdout, + "status": self._git("status", "--porcelain=v1", "-uall").stdout, + "index": self._git("ls-files", "-s").stdout, + "diff": self._git("diff").stdout, + "staged": self._git("diff", "--cached").stdout, + } + + +def _standard_history(repo_test: _TempRepo): + """Two commits. The ref diff deletes one file, changes one, + adds a test that names two unchanged paths, and leaves a + bystander unmentioned.""" + repo_test._write("alpha.py", "alpha v1\n") + repo_test._write("deleted_by_the_diff.py", "goodbye\n") + repo_test._write("config/app.toml", "name = 'app'\n") + repo_test._write("tools/doit.py", "print('doit')\n") + repo_test._write("pkg/mod.py", "x = 1\n") + repo_test._write("beta.py", "unchanged\n") + base = repo_test._commit("base") + + repo_test._write("alpha.py", "alpha v2\n") + (repo_test.repo / "deleted_by_the_diff.py").unlink() + repo_test._write("zeta.py", "new at ref\n") + # Paths sit on their own lines so a scan of the diff text can + # see them without having to parse Python. + repo_test._write( + "tests/test_alpha.py", + "tools/doit.py\n" + "config/app.toml\n") + ref = repo_test._commit("ref") + return base, ref + + +# The set B1+B2 require of `_standard_history`. Sorted, unique, +# and without the deleted path (B3). +DERIVED = [ + "alpha.py", + "config/app.toml", + "tests/test_alpha.py", + "tools/doit.py", + "zeta.py", +] + +TRACKED_AT_REF = [ + "alpha.py", + "beta.py", + "config/app.toml", + "pkg/mod.py", + "tests/test_alpha.py", + "tools/doit.py", + "zeta.py", +] + + +class DeriveReturnsTheChangedPaths(_TempRepo): + """B1 — the snapshot's seed is the diff, sorted and unique.""" + + def test_changed_paths_are_sorted_and_complete(self): + self._write("zed.py", "z1\n") + self._write("alpha.py", "a1\n") + self._write("stay.py", "stay\n") + base = self._commit("base") + self._write("zed.py", "z2\n") + self._write("alpha.py", "a2\n") + ref = self._commit("ref") + + got = fileset.derive(str(self.repo), ref, base) + # git will list zed first if it walks in change order; + # the contract is the sorted list, so two runs and two + # implementations can be compared without a ceremony. + self.assertEqual(got, ["alpha.py", "zed.py"]) + + def test_a_path_that_is_both_changed_and_named_appears_once(self): + self._write("src.py", "v1\n") + base = self._commit("base") + self._write("src.py", "v2 mentions src.py\n") + ref = self._commit("ref") + + got = fileset.derive(str(self.repo), ref, base) + self.assertEqual(got, ["src.py"]) + self.assertEqual(len(got), len(set(got))) + + def test_paths_are_repo_relative_and_posix(self): + self._write("pkg/mod.py", "v1\n") + base = self._commit("base") + self._write("pkg/mod.py", "v2\n") + ref = self._commit("ref") + + got = fileset.derive(str(self.repo), ref, base) + self.assertEqual(got, ["pkg/mod.py"]) + for path in got: + self.assertNotIn("\\", path) + self.assertFalse(path.startswith("/"), path) + + +class DeriveAlsoTakesPathsNamedInTheDiff(_TempRepo): + """B2 — a test naming the script it runs is how derivation + works with no human input.""" + + def test_an_unchanged_path_named_in_the_diff_is_included(self): + self._write("script.py", "print(0)\n") + self._write("config.toml", "n = 1\n") + self._write("bystander.py", "leave me\n") + self._write("test_script.py", "pass\n") + base = self._commit("base") + self._write( + "test_script.py", + "script.py\n" + "config.toml\n") + ref = self._commit("ref") + + got = fileset.derive(str(self.repo), ref, base) + self.assertEqual(got, ["config.toml", "script.py", "test_script.py"]) + self.assertNotIn("bystander.py", got) + + def test_a_named_path_that_does_not_exist_at_ref_is_not_invented(self): + # The diff text can name anything. Only paths that are + # real at ref belong in the snapshot; the rest is prose. + self._write("keep.py", "k1\n") + base = self._commit("base") + self._write("keep.py", "k2\nno/such/path.py\n") + ref = self._commit("ref") + + got = fileset.derive(str(self.repo), ref, base) + self.assertEqual(got, ["keep.py"]) + self.assertNotIn("no/such/path.py", got) + + +class DeriveNeverReturnsAPathMissingAtRef(_TempRepo): + """B3 — a deletion is recorded by naming the path in the + diff. Naming it in the derived set would hand the task a + file the snapshot cannot contain.""" + + def test_a_file_deleted_by_the_diff_is_not_in_the_set(self): + self._write("keep.py", "k1\n") + self._write("deleted_by_the_diff.py", "goodbye\n") + base = self._commit("base") + self._write("keep.py", "k2\n") + (self.repo / "deleted_by_the_diff.py").unlink() + ref = self._commit("ref") + + got = fileset.derive(str(self.repo), ref, base) + # Equality is the pin: `not in` against the stub's [] would + # pass and freeze nothing. + self.assertEqual(got, ["keep.py"]) + self.assertNotIn("deleted_by_the_diff.py", got) + + def test_a_deletion_whose_hunk_names_a_survivor_keeps_the_survivor_only( + self): + # The deleted blob's contents appear in the diff text. A + # path named there that still exists at ref is B2; the + # deleted path itself is not. + self._write("config.toml", "n = 1\n") + self._write("retired.py", "config.toml\n") + self._write("keep.py", "k1\n") + base = self._commit("base") + self._write("keep.py", "k2\n") + (self.repo / "retired.py").unlink() + ref = self._commit("ref") + + got = fileset.derive(str(self.repo), ref, base) + self.assertEqual(got, ["config.toml", "keep.py"]) + self.assertNotIn("retired.py", got) + + def test_standard_history_drops_the_deleted_path_and_keeps_named_ones( + self): + base, ref = _standard_history(self) + got = fileset.derive(str(self.repo), ref, base) + self.assertEqual(got, DERIVED) + self.assertNotIn("deleted_by_the_diff.py", got) + self.assertNotIn("beta.py", got) + self.assertNotIn("pkg/mod.py", got) + + +class DeriveIsDeterministic(_TempRepo): + """B4 — two callers comparing lists must not have to sort.""" + + def test_the_same_inputs_produce_the_same_list(self): + base, ref = _standard_history(self) + first = fileset.derive(str(self.repo), ref, base) + second = fileset.derive(str(self.repo), ref, base) + self.assertEqual(first, DERIVED) + self.assertEqual(first, second) + + +class SnapshotWritesExactlyTheIncludeSet(_TempRepo): + """B5 — include is a closed set. Derivation extras must not + leak in, and a bystander must not appear.""" + + def test_only_the_named_files_are_written(self): + _standard_history(self) + # Deliberately unsorted: the manifest list is sorted, not + # a replay of the caller's argument order. + include = ["pkg/mod.py", "beta.py"] + expected = ["beta.py", "pkg/mod.py"] + manifest = fileset.snapshot( + str(self.repo), self._git("rev-parse", "HEAD").stdout.strip(), + str(self.into), include=include) + + self.assertEqual(manifest.get("files"), expected) + self.assertEqual(payload_paths(self.into), expected) + # FILESET.md is metadata about the set, not a member of it. + self.assertTrue((self.into / "FILESET.md").is_file()) + self.assertEqual( + all_file_paths(self.into), + ["FILESET.md", "beta.py", "pkg/mod.py"]) + + def test_contents_come_from_the_ref_not_the_worktree(self): + _, ref = _standard_history(self) + self._write("alpha.py", "DIRTY\n") + manifest = fileset.snapshot( + str(self.repo), ref, str(self.into), include=["alpha.py"]) + written = self.into / "alpha.py" + self.assertTrue(written.is_file()) + self.assertEqual(written.read_text(), "alpha v2\n") + self.assertEqual(manifest.get("files"), ["alpha.py"]) + + def test_nested_paths_keep_their_layout(self): + _, ref = _standard_history(self) + fileset.snapshot( + str(self.repo), ref, str(self.into), + include=["config/app.toml", "tests/test_alpha.py"]) + self.assertEqual( + payload_paths(self.into), + ["config/app.toml", "tests/test_alpha.py"]) + self.assertTrue((self.into / "config" / "app.toml").is_file()) + + +class SnapshotRefusesAMissingInclude(_TempRepo): + """B6 — silently skipping a missing path would hand the task + a context it cannot tell from a complete one.""" + + def test_a_path_that_never_existed_raises(self): + _, ref = _standard_history(self) + with self.assertRaises(fileset.FilesetError): + fileset.snapshot( + str(self.repo), ref, str(self.into), + include=["alpha.py", "no/such/file.py"]) + + def test_a_path_deleted_at_ref_raises_even_when_named_in_the_diff(self): + base, ref = _standard_history(self) + # The path is in the diff (B3's subject). Asking for it as + # an include is a request for a blob that is not there. + with self.assertRaises(fileset.FilesetError): + fileset.snapshot( + str(self.repo), ref, str(self.into), + include=["deleted_by_the_diff.py"], base=base) + + def test_a_worktree_only_file_does_not_count_as_existing_at_ref(self): + _, ref = _standard_history(self) + self._write("uncommitted.py", "not at ref\n") + with self.assertRaises(fileset.FilesetError): + fileset.snapshot( + str(self.repo), ref, str(self.into), + include=["uncommitted.py"]) + + +class SnapshotRefusesAnEmptyIncludeSet(_TempRepo): + """B7 — an empty snapshot is never a legitimate answer.""" + + def test_an_empty_include_list_raises(self): + _, ref = _standard_history(self) + with self.assertRaises(fileset.FilesetError): + fileset.snapshot( + str(self.repo), ref, str(self.into), include=[]) + + +class SnapshotWholeWritesEveryTrackedFile(_TempRepo): + """B8 — whole=True is the unpruned tree at ref, not the worktree.""" + + def test_every_tracked_file_is_written_and_no_untracked(self): + _, ref = _standard_history(self) + self._write("scratch.tmp", "untracked\n") + manifest = fileset.snapshot( + str(self.repo), ref, str(self.into), whole=True) + + self.assertEqual(manifest.get("files"), TRACKED_AT_REF) + self.assertEqual(manifest.get("whole"), True) + self.assertEqual(payload_paths(self.into), TRACKED_AT_REF) + self.assertNotIn("scratch.tmp", payload_paths(self.into)) + self.assertNotIn("deleted_by_the_diff.py", payload_paths(self.into)) + + +class SnapshotWritesTheDiffIffABaseIsGiven(_TempRepo): + """B9 — the diff is how a task sees what changed; without a + base there is no such document, and inventing one would lie.""" + + def test_a_base_writes_the_diff_and_names_it_in_the_manifest(self): + base, ref = _standard_history(self) + manifest = fileset.snapshot( + str(self.repo), ref, str(self.into), + include=["alpha.py"], base=base) + + diff_path = self.into / "FILESET.diff" + self.assertTrue(diff_path.is_file()) + diff_text = diff_path.read_text() + self.assertIn("alpha.py", diff_text) + self.assertIn("deleted_by_the_diff.py", diff_text) + reported = manifest.get("diff") + self.assertIsNotNone(reported) + self.assertEqual(Path(reported).resolve(), diff_path.resolve()) + + def test_without_a_base_there_is_no_diff_file_and_manifest_says_so(self): + _, ref = _standard_history(self) + manifest = fileset.snapshot( + str(self.repo), ref, str(self.into), include=["alpha.py"]) + + # The positive half is the snapshot that did get written: + # asserting only "diff is absent" is green against the stub. + self.assertTrue((self.into / "alpha.py").is_file()) + self.assertTrue((self.into / "FILESET.md").is_file()) + self.assertFalse((self.into / "FILESET.diff").exists()) + self.assertIsNone(manifest.get("diff")) + self.assertEqual( + all_file_paths(self.into), ["FILESET.md", "alpha.py"]) + + +class SnapshotWritesAManifestTheTaskCanRead(_TempRepo): + """B10 — a task must be able to see what it was and was not given.""" + + def test_fileset_md_names_the_ref_the_base_and_every_path(self): + base, ref = _standard_history(self) + include = ["alpha.py", "pkg/mod.py"] + manifest = fileset.snapshot( + str(self.repo), ref, str(self.into), + include=include, base=base) + + md_path = self.into / "FILESET.md" + self.assertTrue(md_path.is_file()) + text = md_path.read_text() + self.assertIn(ref, text) + self.assertIn(base, text) + for path in include: + self.assertIn(path, text) + self.assertEqual(Path(manifest.get("manifest") or "").resolve(), + md_path.resolve()) + + def test_the_returned_manifest_carries_the_contract_keys(self): + base, ref = _standard_history(self) + into = str(self.into) + manifest = fileset.snapshot( + str(self.repo), ref, into, + include=["zeta.py"], base=base) + + self.assertEqual( + set(manifest), + {"ref", "base", "root", "files", "bytes", + "diff", "manifest", "whole"}) + self.assertEqual(manifest.get("ref"), ref) + self.assertEqual(manifest.get("base"), base) + self.assertEqual(Path(manifest.get("root") or "").resolve(), + Path(into).resolve()) + self.assertEqual(manifest.get("files"), ["zeta.py"]) + self.assertEqual(manifest.get("whole"), False) + + +class SnapshotNeverTouchesTheSourceRepo(_TempRepo): + """B11 — a stray checkout would throw away uncommitted work. + A clean repo cannot show that damage, so the pin is a dirty + one.""" + + def test_uncommitted_work_survives_and_is_not_what_got_copied(self): + _, ref = _standard_history(self) + self._write("alpha.py", "DIRTY VERSION\n") + self._write("untracked.txt", "do not delete me\n") + self._write("to_stage.py", "staged new file\n") + self._git("add", "to_stage.py") + before = self._fingerprint() + + manifest = fileset.snapshot( + str(self.repo), ref, str(self.into), include=["alpha.py"]) + + written = self.into / "alpha.py" + self.assertTrue( + written.is_file(), + "the snapshot must still be produced from a dirty repo") + self.assertEqual(written.read_text(), "alpha v2\n") + self.assertNotEqual(written.read_text(), "DIRTY VERSION\n") + self.assertEqual(manifest.get("files"), ["alpha.py"]) + self.assertEqual(self._fingerprint(), before) + + def test_whole_snapshot_of_a_dirty_repo_still_leaves_it_dirty(self): + _, ref = _standard_history(self) + self._write("beta.py", "DIRTY BETA\n") + self._write("scratch.tmp", "untracked\n") + before = self._fingerprint() + + manifest = fileset.snapshot( + str(self.repo), ref, str(self.into), whole=True) + self.assertEqual(manifest.get("files"), TRACKED_AT_REF) + beta = self.into / "beta.py" + self.assertTrue(beta.is_file()) + self.assertEqual(beta.read_text(), "unchanged\n") + self.assertEqual(self._fingerprint(), before) + + +class ManifestBytesMatchWhatWasWritten(_TempRepo): + """B12 — a byte count that can disagree with the files is a + count that lies.""" + + def test_bytes_equal_the_sum_of_the_payload_files(self): + _, ref = _standard_history(self) + include = ["alpha.py", "config/app.toml", "zeta.py"] + manifest = fileset.snapshot( + str(self.repo), ref, str(self.into), include=include) + + written = payload_paths(self.into) + self.assertEqual(written, include) + total = sum((self.into / rel).stat().st_size for rel in written) + self.assertEqual(manifest.get("bytes"), total) + self.assertGreater(total, 0) + + +class ParityComparesWholeTreeAgainstPruned(_TempRepo): + """B13 — the prune is verified by running the same command in + both trees. True means the answers matched, not that the + command succeeded.""" + + def _pruned(self, ref, include): + # The stub writes nothing; do not assert on that here or + # the red lands on snapshot instead of on parity. + fileset.snapshot( + str(self.repo), ref, str(self.pruned), include=include) + + def test_matching_results_return_true_and_both_outputs(self): + _, ref = _standard_history(self) + self._pruned(ref, ["alpha.py"]) + argv = [sys.executable, "-c", + "print(open('alpha.py').read(), end='')"] + ok, full, pruned = fileset.parity( + str(self.repo), str(self.pruned), argv, ref=ref) + self.assertEqual(ok, True) + self.assertEqual(full, "alpha v2\n") + self.assertEqual(pruned, "alpha v2\n") + + def test_a_mismatch_returns_false_and_the_two_different_outputs(self): + _, ref = _standard_history(self) + self._pruned(ref, ["alpha.py"]) + argv = [sys.executable, "-c", + ("import pathlib; p=pathlib.Path('pkg/mod.py'); " + "print(p.read_text() if p.exists() else 'ABSENT')")] + ok, full, pruned = fileset.parity( + str(self.repo), str(self.pruned), argv, ref=ref) + # Stub returns (False, "", ""): False alone would pass. + # The outputs are the pin. + self.assertEqual(full.strip(), "x = 1") + self.assertEqual(pruned.strip(), "ABSENT") + self.assertEqual(ok, False) + + def test_identical_failures_are_a_match(self): + # Both trees failing the same way means the prune did not + # change the answer. That is parity, not an error. + _, ref = _standard_history(self) + self._pruned(ref, ["alpha.py"]) + argv = [sys.executable, "-c", + "import sys; sys.stderr.write('boom\\n'); sys.exit(7)"] + try: + ok, full, pruned = fileset.parity( + str(self.repo), str(self.pruned), argv, ref=ref) + except fileset.FilesetError: + self.fail("identical failure must not raise FilesetError") + self.assertEqual(ok, True) + self.assertEqual(full, pruned) + + +class ParityRaisesWhenTheCommandCannotRun(_TempRepo): + """B14 — 'it failed identically in both' and 'it could not + run' must not both read as parity.""" + + def test_a_command_that_cannot_be_executed_raises(self): + _, ref = _standard_history(self) + fileset.snapshot( + str(self.repo), ref, str(self.pruned), include=["alpha.py"]) + argv = ["/no/such/dir/fileset-no-such-command-7e1c9a3d"] + with self.assertRaises(fileset.FilesetError): + fileset.parity( + str(self.repo), str(self.pruned), argv, ref=ref) + + def test_a_python_that_exits_nonzero_is_not_cannot_run(self): + # Contrast with the test above: the executable exists. The + # process starts. That is a result, even when it fails. + _, ref = _standard_history(self) + fileset.snapshot( + str(self.repo), ref, str(self.pruned), include=["alpha.py"]) + argv = [sys.executable, "-c", "raise SystemExit(1)"] + try: + result = fileset.parity( + str(self.repo), str(self.pruned), argv, ref=ref) + except fileset.FilesetError as exc: + self.fail(f"a runnable failure raised: {exc}") + self.assertEqual(result[0], True) + self.assertEqual(result[1], result[2]) + + +class MainIsACli(_TempRepo): + """B15 — stdout is the manifest; refusals are 1; usage is 64.""" + + def test_snapshot_prints_json_and_returns_zero(self): + _, ref = _standard_history(self) + code, out, _ = run_main( + ["snapshot", ref, "--into", str(self.into), + "--include", "alpha.py"], + cwd=self.repo) + self.assertTrue(out.strip(), "manifest JSON on stdout") + try: + manifest = json.loads(out) + except json.JSONDecodeError: + self.fail(f"stdout was not JSON: {out!r}") + self.assertEqual(manifest.get("files"), ["alpha.py"]) + self.assertEqual(manifest.get("ref"), ref) + self.assertTrue((self.into / "alpha.py").is_file()) + self.assertEqual((self.into / "alpha.py").read_text(), "alpha v2\n") + self.assertEqual(code, 0) + + def test_base_is_accepted_and_writes_the_diff(self): + base, ref = _standard_history(self) + code, out, _err = run_main( + ["snapshot", ref, "--into", str(self.into), + "--include", "alpha.py", "--base", base], + cwd=self.repo) + self.assertTrue(out.strip()) + try: + manifest = json.loads(out) + except json.JSONDecodeError: + self.fail(f"stdout was not JSON: {out!r}") + self.assertEqual(code, 0) + self.assertTrue((self.into / "FILESET.diff").is_file()) + self.assertIsNotNone(manifest.get("diff")) + + def test_whole_writes_the_tracked_tree(self): + _, ref = _standard_history(self) + code, out, _err = run_main( + ["snapshot", ref, "--into", str(self.into), "--whole"], + cwd=self.repo) + self.assertTrue(out.strip()) + try: + manifest = json.loads(out) + except json.JSONDecodeError: + self.fail(f"stdout was not JSON: {out!r}") + self.assertEqual(manifest.get("files"), TRACKED_AT_REF) + self.assertEqual(manifest.get("whole"), True) + self.assertEqual(code, 0) + + def test_a_fileset_error_prints_to_stderr_and_returns_one(self): + _, ref = _standard_history(self) + code, _out, err = run_main( + ["snapshot", ref, "--into", str(self.into), + "--include", "no/such/file.py"], + cwd=self.repo) + self.assertEqual(code, 1) + self.assertTrue(err.strip()) + + def test_a_usage_error_returns_sixty_four(self): + _, ref = _standard_history(self) + for argv in ([], ["snapshot"], ["nope"], + ["snapshot", ref]): + with self.subTest(argv=argv): + code, _out, _err = run_main(argv, cwd=self.repo) + self.assertEqual(code, 64) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/task/tests/test_jobs.py b/ops/devlane/task/tests/test_jobs.py new file mode 100644 index 0000000..dfda501 --- /dev/null +++ b/ops/devlane/task/tests/test_jobs.py @@ -0,0 +1,52 @@ +"""Harness job prompts ask for the envelope. + +Written from ops/devlane/task/jobs.json: every harness job's prompt except +author-tests ends with the envelope clause. The author-tests brief must +ask for the same envelope the other harness jobs do. + +The clause, verbatim: + + Answer with a single JSON object and nothing else, carrying exactly + these keys: job, status, verdict, counts, findings, artifacts, + spend, stamp, note +""" + +from __future__ import annotations + +import json +import unittest + +import support + +JOBS_PATH = support.APP / "jobs.json" + +ENVELOPE_CLAUSE = ( + "Answer with a single JSON object and nothing else, carrying exactly " + "these keys: job, status, verdict, counts, findings, artifacts, spend, " + "stamp, note" +) + + +class AuthorTestsBriefAsksForTheEnvelope(unittest.TestCase): + """The author-tests brief asks for the envelope.""" + + def test_the_author_tests_brief_asks_for_the_envelope(self): + self.assertTrue(JOBS_PATH.is_file(), f"jobs.json missing: {JOBS_PATH}") + jobs = json.loads(JOBS_PATH.read_text(encoding="utf-8")) + self.assertIsInstance(jobs, dict) + self.assertIn("author-tests", jobs) + spec = jobs["author-tests"] + self.assertIsInstance(spec, dict) + prompt = spec.get("prompt") + self.assertIsInstance(prompt, str) + self.assertTrue(prompt.strip(), "author-tests prompt is empty") + self.assertIn( + ENVELOPE_CLAUSE, + prompt, + "author-tests brief must ask for the envelope; " + f"last 240 chars: {prompt[-240:]!r}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/task/tests/test_run.py b/ops/devlane/task/tests/test_run.py new file mode 100644 index 0000000..f3f48d6 --- /dev/null +++ b/ops/devlane/task/tests/test_run.py @@ -0,0 +1,1479 @@ +"""run: snapshot, launch, supervise, collect — never judge the work. + +Written from SPEC.md, before the module had behaviour. Each test names +the contract rule it pins: + + R1 `run` builds its snapshot with fileset.snapshot; never copies a + tree itself and never git-checkouts + R2 stamp.ref comes from the snapshot manifest; an unresolvable ref + is invalid, never a launch + R3 runtime.role selects the adapter sandbox map; write must not get + the read-only value (measured: empty deliverable, zero exit) + R4 an unknown harness is invalid with a note naming it + R5 a harness whose CLI is not on PATH is invalid with a note — + never a verdict about the work + R6 `direct` delegates to verify.check and returns that envelope; + zero tokens, spend.harness None + R7 `stub` replays a file named in runtime and launches nothing + R8 the battery is armed with --cap-out from runtime.caps; a --cap + that counts re-sent cache is not a default + R9 a battery trip is status tripped, verdict None, a note; a killed + run is never a completed one + R10 a stream_names_cwd candidate must name this snapshot; two runs + of one harness must not resolve to the same stream + R11 no stream found means spend is unknown, not zero + R12 raw output is a file inside the snapshot, named by artifacts.raw, + never inlined + R13 run may parse a structured envelope; it must not interpret prose + R14 a job absent from the table is invalid + R15 main is a CLI: JSON on stdout; 0 / 1 / 2 / 64 + +The stub returns {}, "" and 0. Every test below fails on its own +assertion against that, not on an import error and not on a crash. +""" + +from __future__ import annotations + +import contextlib +import hashlib +import io +import json +import os +import signal +import stat +import subprocess +import sys +import tempfile +import time +import unittest +import uuid +from pathlib import Path + +import support + +run = support.load("run") + +# Loaded the way run.py will load them (`import fileset`), not under the +# task_ prefix support.load uses. Wrapping these is how R1 and R6 pin +# delegation without caring how run.py spelled the import. +import envelope # noqa: E402 +import fileset # noqa: E402 +import verify # noqa: E402 + +# A subprocess harness. Records argv/stdin, optionally sleeps, optionally +# prints a structured envelope or prose. No network, no shell. +_FAKE_CLI = r"""#!/usr/bin/env python3 +import json +import os +import sys +import time +from pathlib import Path + +record_path = os.environ.get("TASK_RUN_RECORD") +done_path = os.environ.get("TASK_RUN_DONE") +sleep_s = float(os.environ.get("TASK_RUN_SLEEP") or "0") +stdout_mode = os.environ.get("TASK_RUN_STDOUT") or "envelope" +token = os.environ.get("TASK_RUN_TOKEN") or "" +claim = os.environ.get("TASK_RUN_CLAIM") or "the battery is never armed" +verdict = os.environ.get("TASK_RUN_VERDICT") or "changes" +job = os.environ.get("TASK_RUN_JOB") or "author-tests" + +argv = sys.argv[1:] +prompt_file = None +if "--prompt-file" in argv: + idx = argv.index("--prompt-file") + if idx + 1 < len(argv): + prompt_file = argv[idx + 1] + +stdin_data = sys.stdin.read() +prompt_text = "" +if prompt_file: + try: + prompt_text = Path(prompt_file).read_text(encoding="utf-8") + except OSError: + prompt_text = "" + +if record_path: + Path(record_path).write_text( + json.dumps({ + "argv": sys.argv, + "cwd": os.getcwd(), + "stdin": stdin_data, + "prompt_file": prompt_file, + "prompt_text": prompt_text, + }), + encoding="utf-8", + ) + +if stdout_mode == "envelope": + findings = [] + if verdict != "approve": + findings.append({ + "severity": "p2", + "where": "alpha.py §top", + "claim": claim, + "reproduce": "python3 -m unittest", + }) + counts = {"p1": 0, "p2": 0, "p3": 0, "opinions": 0} + counts["p2"] = len(findings) + env = { + "job": job, + "status": "ok", + "verdict": verdict, + "counts": counts, + "findings": findings, + "artifacts": {}, + "spend": {"harness": "codex", "total": 0, "out": 0, "runs": 1}, + "stamp": {"ref": "harness-placeholder", "started": None, "ended": None}, + "note": None, + } + sys.stdout.write(json.dumps(env) + "\n") +elif stdout_mode == "prose": + sys.stdout.write("VERDICT: approve\nThe change is correct and complete.\n") +elif stdout_mode == "token": + sys.stdout.write(token + "\n") +sys.stdout.flush() + +if sleep_s > 0: + time.sleep(sleep_s) + +if done_path: + Path(done_path).write_text("COMPLETED\n", encoding="utf-8") +""" + + +def _git_env(home: Path) -> dict: + # GIT_DIR / GIT_WORK_TREE in the caller would aim git at the + # snapshot (or its parent). Fixtures must be closed worlds, and + # this suite must not run git against the snapshot's own repo. + env = {k: v for k, v in os.environ.items() + if not k.startswith("GIT_") and k != "XDG_CONFIG_HOME"} + env.update({ + "HOME": str(home), + "GIT_AUTHOR_NAME": "run-test", + "GIT_AUTHOR_EMAIL": "run-test@example.test", + "GIT_COMMITTER_NAME": "run-test", + "GIT_COMMITTER_EMAIL": "run-test@example.test", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + }) + return env + + +def write_codex_stream(path, *, total, out): + """One Codex token_count event. breaker.py reads this shape.""" + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + event = { + "payload": { + "type": "token_count", + "info": { + "total_token_usage": { + "total_tokens": total, + "output_tokens": out, + } + }, + } + } + path.write_text(json.dumps(event) + "\n", encoding="utf-8") + + +def run_main(argv): + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + try: + code = run.main(argv) + except SystemExit as exc: + # argparse's default is SystemExit(2). The contract is 64 + # for usage; tests pin that by comparing the code, not by + # letting the exception look like a collection error. + code = int(exc.code) if exc.code is not None else 0 + return code, out.getvalue(), err.getvalue() + + +class _TempRun(unittest.TestCase): + """A throwaway repo, HOME, PATH and fake CLIs. Never the snapshot tree.""" + + def setUp(self): + self._td = tempfile.TemporaryDirectory() + self.home = Path(self._td.name) + self.repo = self.home / "repo" + self.bin = self.home / "bin" + self.codex_store = self.home / ".codex" / "sessions" + self.grok_store = self.home / ".grok" / "sessions" + self.repo.mkdir() + self.bin.mkdir() + self.codex_store.mkdir(parents=True) + self.grok_store.mkdir(parents=True) + + self._orig_env = { + k: os.environ.get(k) for k in ( + "PATH", "HOME", "XDG_CONFIG_HOME", + "TASK_RUN_RECORD", "TASK_RUN_DONE", "TASK_RUN_SLEEP", + "TASK_RUN_STDOUT", "TASK_RUN_TOKEN", "TASK_RUN_CLAIM", + "TASK_RUN_VERDICT", "TASK_RUN_JOB", + ) + } + self._saved_git = {k: os.environ[k] for k in list(os.environ) + if k.startswith("GIT_")} + for k in list(self._saved_git): + del os.environ[k] + if "XDG_CONFIG_HOME" in os.environ: + del os.environ["XDG_CONFIG_HOME"] + + os.environ["HOME"] = str(self.home) + os.environ["PATH"] = str(self.bin) + os.pathsep + os.environ.get("PATH", "") + + self.env = _git_env(self.home) + self._git("init") + self._git("config", "user.name", "run-test") + self._git("config", "user.email", "run-test@example.test") + self._git("config", "commit.gpgsign", "false") + self._write("alpha.py", "alpha v1\n") + self._write("beta.py", "unchanged\n") + self.base = self._commit("base") + self._write("alpha.py", "alpha v2\n") + self.ref = self._commit("ref") + + self.record_path = self.home / "launch-record.json" + os.environ["TASK_RUN_RECORD"] = str(self.record_path) + os.environ.pop("TASK_RUN_DONE", None) + os.environ.pop("TASK_RUN_SLEEP", None) + os.environ["TASK_RUN_STDOUT"] = "envelope" + os.environ.pop("TASK_RUN_TOKEN", None) + os.environ.pop("TASK_RUN_CLAIM", None) + os.environ.pop("TASK_RUN_VERDICT", None) + + self._install_cli("codex") + self._install_cli("grok") + + self.jobs = { + "author-tests": { + "adapter": "harness", + "deliverable": "a test file that has been run and is red", + "role": "write", + "prompt": ( + "Write tests from the contract at {scope}. " + "Ref {ref} base {base}." + ), + "constraints": ["do not edit the implementation"], + }, + "adversarial-review": { + "adapter": "harness", + "role": "read", + "prompt": "Review {ref} against {base}. Aim at: {scope}", + "constraints": ["read only"], + }, + "verify": { + "adapter": "direct", + "deliverable": "one claim, executed", + "prompt": None, + "constraints": ["read only"], + }, + } + self.require = {"scope": "pin the runner", "constraints": ["no network"]} + self.adapters = self.make_adapters() + self.last_into = None + self._patches = [] + self._saved_adapters = run.ADAPTERS + self._saved_comm_path = run.JOBS_PATH + + def tearDown(self): + self._unpatch() + run.ADAPTERS = self._saved_adapters + run.JOBS_PATH = self._saved_comm_path + for k, v in self._orig_env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + for k in list(os.environ): + if k.startswith("GIT_"): + del os.environ[k] + os.environ.update(self._saved_git) + self._td.cleanup() + + def _git(self, *args): + r = subprocess.run( + ["git", *args], cwd=self.repo, env=self.env, + capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError( + f"git {args} failed ({r.returncode}): {r.stderr}") + return r + + def _write(self, rel, content): + p = self.repo / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content, encoding="utf-8") + return p + + def _commit(self, msg): + self._git("add", "-A") + self._git("commit", "-m", msg) + return self._git("rev-parse", "HEAD").stdout.strip() + + def _fingerprint(self): + files = {} + for p in self.repo.rglob("*"): + if ".git" in p.parts or not p.is_file(): + continue + rel = p.relative_to(self.repo).as_posix() + files[rel] = ( + p.stat().st_mode, + hashlib.sha256(p.read_bytes()).hexdigest(), + ) + return { + "files": files, + "head": self._git("rev-parse", "HEAD").stdout, + "status": self._git("status", "--porcelain=v1", "-uall").stdout, + "index": self._git("ls-files", "-s").stdout, + "diff": self._git("diff").stdout, + "staged": self._git("diff", "--cached").stdout, + } + + def _install_cli(self, name): + dest = self.bin / name + dest.write_text(_FAKE_CLI, encoding="utf-8") + dest.chmod(dest.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + return dest + + def make_adapters(self, *, sandbox=None, stream_names_cwd=True, store=None): + sandbox = sandbox or {"read": "read-only", "write": "workspace-write"} + store = str(store or self.codex_store) + return { + "codex": { + "argv": [str(self.bin / "codex"), "exec", + "--sandbox", "{sandbox}", "-"], + "prompt": "stdin", + "sandbox": dict(sandbox), + "store": store, + "stream": "rollout-*.jsonl", + "stream_names_cwd": stream_names_cwd, + # The fakes declare dials for the same reason the real + # adapters do: a runtime dial with nowhere to go is now + # refused, so a fake without one would refuse every + # launch in this fixture rather than record an argv. + "dials": {"model": ["-m", "{model}"]}, + }, + "grok": { + "argv": [str(self.bin / "grok"), "--prompt-file", "{prompt}", + "--output-format", "plain", + "--permission-mode", "{sandbox}"], + "prompt": "file", + "sandbox": {"read": "plan", "write": "auto"}, + "store": str(self.grok_store), + "stream": "updates.jsonl", + "stream_names_cwd": False, + "dials": {"model": ["-m", "{model}"], + "effort": ["--reasoning-effort", "{effort}"]}, + }, + "stub": {"replay": True}, + "direct": {"direct": True}, + } + + def make_context(self, **overrides): + snap_id = uuid.uuid4().hex + into = overrides.pop("into", None) or (self.home / f"snap-{snap_id}") + ctx = { + "repo": str(self.repo), + "ref": self.ref, + "base": self.base, + "include": ["alpha.py"], + "into": str(into), + } + ctx.update(overrides) + return ctx + + def call_run(self, job="author-tests", *, context=None, require=None, + runtime=None, jobs=None, adapters=None): + context = context if context is not None else self.make_context() + self.last_context = context + self.last_into = Path(context["into"]) + rt = { + "harness": "codex", + "model": "test-model", + # No effort here: `codex --help` offers `-m/--model` and no + # effort flag, so the adapter declares no way to pass one and + # a set effort is REFUSED rather than silently dropped. The + # refusal and the two harnesses that do take an effort dial + # have their own cases in TheConductorsDialReachesTheHarness. + "role": "write", + "caps": {"cap-out": 100000}, + "timeout": 30, + } + if runtime: + rt.update(runtime) + try: + env = run.run( + job, + context=context, + require=require if require is not None else self.require, + runtime=rt, + jobs=jobs if jobs is not None + else self.jobs, + adapters=adapters if adapters is not None else self.adapters, + ) + except Exception as exc: + self.fail( + f"run must return an envelope, not raise: " + f"{type(exc).__name__}: {exc}") + self.assertIsInstance(env, dict, "run returns an envelope dict") + return env + + def read_record(self): + # Presence of the record is the proof the fake CLI started. + # Asserting only that a launch side-effect is absent is green + # against the stub, which launches nothing. + self.assertTrue( + self.record_path.is_file(), + "the harness CLI must have been launched", + ) + return json.loads(self.record_path.read_text(encoding="utf-8")) + + def recorded_argv(self): + rec = self.read_record() + argv = rec.get("argv") + self.assertIsInstance(argv, list) + self.assertTrue(argv, "recorded argv is empty") + return [str(part) for part in argv] + + def parse_stdout(self, out): + # An empty stdout is not an envelope. Failing here against the + # stub (which prints nothing) is the honest red for CLI tests + # whose exit code is 0, because that 0 is also the stub's return. + self.assertTrue(out.strip(), "envelope JSON on stdout") + try: + env = json.loads(out) + except json.JSONDecodeError: + self.fail(f"stdout was not JSON: {out!r}") + self.assertIsInstance(env, dict) + return env + + def assert_invalid(self, env, *, naming=None): + self.assertIsInstance(env, dict) + self.assertEqual(env.get("status"), "invalid") + # verdict None is the contract's null. Pinning only + # `is not "changes"` would pass against the stub's {}. + self.assertIsNone(env.get("verdict")) + self.assertNotEqual(env.get("verdict"), "changes") + self.assertNotEqual(env.get("verdict"), "approve") + note = env.get("note") + self.assertIsInstance(note, str) + self.assertTrue(note.strip(), "invalid must say why it could not run") + if naming is not None: + self.assertIn(naming, note) + + def _patch_attr(self, obj, name, replacement): + """Wrap a function on a module run.py may have imported already. + + `from fileset import snapshot` binds a name on run; `import fileset` + leaves the function on that module. Patching both is how R1/R6 pin + delegation without guessing the import spelling. Unique (id, name) + so wrapping fileset and run.fileset (the same object) cannot + restore a wrapped function over the original. + """ + if obj is None or not hasattr(obj, name): + return + if any(id(o) == id(obj) and n == name for o, n, _ in self._patches): + return + self._patches.append((obj, name, getattr(obj, name))) + setattr(obj, name, replacement) + + def _unpatch(self): + while self._patches: + obj, name, original = self._patches.pop() + setattr(obj, name, original) + + +# --------------------------------------------------------------------------- +# R1 +# --------------------------------------------------------------------------- + + +class RunBuildsTheSnapshotWithFileset(_TempRun): + """R1 — the snapshot is fileset.snapshot's, not cp -r and not a checkout. + + Hand-rolled copies on 2026-08-22 produced eight snapshots and a + launch parameter wrong on six of them. fileset.snapshot is the + one call that writes FILESET.md and reads blobs at ref without + touching the source worktree. + """ + + def test_the_snapshot_is_a_fileset_snapshot_of_the_ref_not_the_worktree(self): + self._write("alpha.py", "DIRTY VERSION\n") + calls = [] + original = fileset.snapshot + + def wrapped(*args, **kwargs): + calls.append((args, kwargs)) + return original(*args, **kwargs) + + self._patch_attr(fileset, "snapshot", wrapped) + self._patch_attr(getattr(run, "fileset", None), "snapshot", wrapped) + if getattr(run, "snapshot", None) is original: + self._patch_attr(run, "snapshot", wrapped) + + env = self.call_run() + into = self.last_into + md = into / "FILESET.md" + written = into / "alpha.py" + self.assertTrue( + md.is_file(), + "FILESET.md is how a fileset.snapshot is told from cp -r", + ) + self.assertTrue(written.is_file(), "the include set must be written") + self.assertEqual(written.read_text(encoding="utf-8"), "alpha v2\n") + self.assertNotEqual( + written.read_text(encoding="utf-8"), "DIRTY VERSION\n") + self.assertIn(self.ref, md.read_text(encoding="utf-8")) + self.assertFalse( + (into / "beta.py").exists(), + "include is a closed set; a bystander must not appear", + ) + self.assertTrue( + calls, + "run must call fileset.snapshot, not copy a tree itself", + ) + # The envelope is the proof the run used the snapshot, not + # that it merely built one and threw it away. + stamp = env.get("stamp") if isinstance(env.get("stamp"), dict) else {} + self.assertEqual(stamp.get("ref"), self.ref) + + def test_the_source_repo_is_not_checked_out_or_mutated(self): + # A clean repo cannot show checkout damage. The pin is a dirty one. + self._write("alpha.py", "DIRTY VERSION\n") + self._write("untracked.txt", "do not delete me\n") + self._write("to_stage.py", "staged new file\n") + self._git("add", "to_stage.py") + before = self._fingerprint() + + env = self.call_run() + into = self.last_into + written = into / "alpha.py" + self.assertTrue( + written.is_file(), + "the snapshot must still be produced from a dirty repo", + ) + self.assertEqual(written.read_text(encoding="utf-8"), "alpha v2\n") + self.assertEqual(self._fingerprint(), before) + self.assertIsInstance(env.get("status"), str) + self.assertNotEqual(env.get("status"), "") + + +# --------------------------------------------------------------------------- +# R2 +# --------------------------------------------------------------------------- + + +class StampRefComesFromTheManifest(_TempRun): + """R2 — without a ref the envelope names no state. The manifest is + the state that was actually snapshotted; the harness's own stamp + is not that.""" + + def test_stamp_ref_is_the_sha_the_manifest_names(self): + env = self.call_run() + md = self.last_into / "FILESET.md" + self.assertTrue(md.is_file(), "the manifest must exist to supply the ref") + self.assertIn(self.ref, md.read_text(encoding="utf-8")) + stamp = env.get("stamp") if isinstance(env.get("stamp"), dict) else {} + self.assertEqual(stamp.get("ref"), self.ref) + # The fake CLI prints a placeholder. Overlaying it is the rule. + self.assertNotEqual(stamp.get("ref"), "harness-placeholder") + + def test_an_unresolvable_ref_is_invalid_and_never_launches(self): + missing = "no-such-ref-7e1c9a3d" + env = self.call_run(context=self.make_context(ref=missing)) + self.assert_invalid(env) + note = env.get("note") + self.assertTrue( + missing in note or "ref" in note.lower(), + f"invalid ref must be explained: {note!r}", + ) + self.assertFalse( + self.record_path.exists(), + "an unresolvable ref is never a launch", + ) + + +# --------------------------------------------------------------------------- +# R3 +# --------------------------------------------------------------------------- + + +class RoleSelectsTheSandbox(_TempRun): + """R3 — a WRITE role must not receive the adapter's read-only value. + + Measured: `--sandbox read-only` for a role whose deliverable is a + file produced an empty deliverable and a zero exit — success-shaped + failure. The map is data; the role is the lookup key. + """ + + def test_a_write_role_receives_the_write_sandbox_not_the_read_value(self): + # Unique tokens, not the builtin strings: so a hardcoded + # "workspace-write" cannot satisfy a map it never consulted. + write_tok = "WRITE-" + uuid.uuid4().hex + read_tok = "READ-" + uuid.uuid4().hex + adapters = self.make_adapters( + sandbox={"read": read_tok, "write": write_tok}) + env = self.call_run( + runtime={"harness": "codex", "role": "write"}, + adapters=adapters, + ) + argv = self.recorded_argv() + self.assertIn(write_tok, argv) + self.assertNotIn(read_tok, argv) + self.assertIsInstance(env.get("status"), str) + + def test_a_read_role_receives_the_read_sandbox(self): + # Contrast: if write-not-readonly were implemented as "never + # pass a sandbox", this would fail too, and should. + write_tok = "WRITE-" + uuid.uuid4().hex + read_tok = "READ-" + uuid.uuid4().hex + adapters = self.make_adapters( + sandbox={"read": read_tok, "write": write_tok}) + env = self.call_run( + job="adversarial-review", + runtime={"harness": "codex", "role": "read"}, + adapters=adapters, + ) + argv = self.recorded_argv() + self.assertIn(read_tok, argv) + self.assertNotIn(write_tok, argv) + self.assertIsInstance(env.get("status"), str) + + def test_a_grok_write_role_is_auto_not_plan(self): + env = self.call_run( + runtime={"harness": "grok", "role": "write"}, + ) + argv = self.recorded_argv() + self.assertIn("auto", argv) + self.assertNotIn("plan", argv) + self.assertIsInstance(env.get("status"), str) + + def test_builtin_write_values_are_not_the_read_only_ones(self): + # Measured facts in the adapter table. .get so a missing + # key is an assertion failure, not a KeyError against the stub. + cases = ( + ("codex", "workspace-write", "read-only"), + ("grok", "auto", "plan"), + ) + for name, write, read in cases: + with self.subTest(harness=name): + sandbox = (run.ADAPTERS.get(name) or {}).get("sandbox") or {} + self.assertEqual(sandbox.get("write"), write) + self.assertEqual(sandbox.get("read"), read) + self.assertNotEqual(sandbox.get("write"), sandbox.get("read")) + + +# --------------------------------------------------------------------------- +# R4 +# --------------------------------------------------------------------------- + + +class AnUnknownHarnessIsInvalid(_TempRun): + """R4 — never a default, never a guess. The note names the stranger.""" + + def test_an_unknown_harness_is_invalid_with_a_note_naming_it(self): + name = "not-a-real-harness-7e1c9a3d" + env = self.call_run(runtime={"harness": name}) + self.assert_invalid(env, naming=name) + self.assertFalse( + self.record_path.exists(), + "an unknown harness must not launch a different one", + ) + + +# --------------------------------------------------------------------------- +# R5 +# --------------------------------------------------------------------------- + + +class AMissingCliIsInvalidNotAVerdict(_TempRun): + """R5 — 'the harness is missing' and 'the work found nothing' must + not produce the same envelope. Assert the note, not just the status.""" + + def test_a_missing_cli_is_invalid_with_a_note_naming_the_gap(self): + missing = "run-test-no-such-cli-7e1c9a3d" + adapters = self.make_adapters() + adapters["ghost"] = { + "argv": [missing, "exec", "--sandbox", "{sandbox}", "-"], + # A dial the fixture sets must have somewhere to go, or the + # launch is refused for THAT and never reaches the missing + # CLI this case is about. + "dials": {"model": ["-m", "{model}"]}, + "prompt": "stdin", + "sandbox": {"read": "read-only", "write": "workspace-write"}, + "store": str(self.codex_store), + "stream": "rollout-*.jsonl", + "stream_names_cwd": True, + } + env = self.call_run(runtime={"harness": "ghost", "role": "write"}, + adapters=adapters) + self.assert_invalid(env) + note = env.get("note") + # The note is the pin. Status-only would treat a silent skip + # the same as a missing binary. + self.assertTrue( + missing in note + or "PATH" in note + or "not found" in note.lower() + or "missing" in note.lower() + or "absent" in note.lower() + or "no such" in note.lower(), + f"note must explain the missing CLI: {note!r}", + ) + self.assertIsNone(env.get("verdict")) + findings = env.get("findings") if isinstance(env.get("findings"), list) else [] + self.assertEqual(findings, []) + self.assertFalse(self.record_path.exists()) + + def test_a_missing_cli_is_not_ok_changes_about_the_work(self): + missing = "run-test-no-such-cli-aa11bb22" + adapters = self.make_adapters() + adapters["ghost"] = { + "argv": [missing], + "dials": {"model": ["-m", "{model}"]}, + "prompt": "stdin", + "sandbox": {"read": "read-only", "write": "workspace-write"}, + "store": str(self.codex_store), + "stream": "rollout-*.jsonl", + "stream_names_cwd": True, + } + env = self.call_run(runtime={"harness": "ghost"}, adapters=adapters) + self.assertEqual(env.get("status"), "invalid") + self.assertNotEqual(env.get("status"), "ok") + self.assertNotEqual(env.get("verdict"), "changes") + self.assertNotEqual(env.get("verdict"), "approve") + self.assertIsInstance(env.get("note"), str) + self.assertTrue(env.get("note", "").strip()) + + +# --------------------------------------------------------------------------- +# R6 +# --------------------------------------------------------------------------- + + +class DirectDelegatesToVerify(_TempRun): + """R6 — direct runs no harness. It calls verify.check and returns + that envelope unchanged. require carries claim and command because + those are verify.check's inputs and the require schema is otherwise + deferred for this slice. + """ + + def _direct_require(self, command=None): + return { + "scope": "python exits 0", + "constraints": ["read only"], + "claim": "python exits 0", + "command": command or [sys.executable, "-c", "pass"], + } + + def test_direct_delegates_to_verify_check_and_returns_that_envelope(self): + calls = [] + original = verify.check + + def wrapped(*args, **kwargs): + result = original(*args, **kwargs) + calls.append((args, kwargs, result)) + return result + + self._patch_attr(verify, "check", wrapped) + self._patch_attr(getattr(run, "verify", None), "check", wrapped) + if getattr(run, "check", None) is original: + self._patch_attr(run, "check", wrapped) + + env = self.call_run( + "verify", + require=self._direct_require(), + runtime={"harness": "direct", "role": "read"}, + ) + + self.assertTrue(calls, "direct must delegate to verify.check") + returned = calls[-1][2] + # Unchanged: not rebuilt, not re-stamped, not re-spent. + self.assertEqual(env, returned) + self.assertEqual(env.get("job"), "verify") + self.assertEqual(env.get("status"), "ok") + self.assertEqual(env.get("verdict"), "approve") + + def test_direct_spend_is_zero_tokens_and_harness_none(self): + env = self.call_run( + "verify", + require=self._direct_require(), + runtime={"harness": "direct", "role": "read"}, + ) + # runs is 1 because verify DID run. The default 0 would tell + # worth.py that nothing happened; the stub's missing spend + # cannot satisfy the full dict. + self.assertEqual( + env.get("spend"), + {"harness": None, "total": 0, "out": 0, "runs": 1}, + ) + self.assertEqual(env.get("verdict"), "approve") + self.assertEqual(env.get("status"), "ok") + + def test_direct_does_not_launch_a_harness_cli(self): + env = self.call_run( + "verify", + require=self._direct_require(), + runtime={"harness": "direct", "role": "read"}, + ) + self.assertEqual(env.get("status"), "ok") + self.assertFalse( + self.record_path.exists(), + "direct runs no harness", + ) + + +# --------------------------------------------------------------------------- +# R7 +# --------------------------------------------------------------------------- + + +class StubReplaysWithoutLaunching(_TempRun): + """R7 — the suite must be able to exercise the runner with no CLI + installed. The recorded stream is named in runtime['replay'].""" + + def test_stub_replays_the_named_file_and_does_not_launch(self): + unique = "REPLAY-CLAIM-" + uuid.uuid4().hex + recorded = envelope.build( + "author-tests", + status="ok", + verdict="changes", + findings=[envelope.finding( + "p2", "alpha.py §top", unique, + reproduce="python3 -m unittest")], + spend={"harness": "stub", "total": 11, "out": 4, "runs": 1}, + stamp={"ref": self.ref}, + ) + replay = self.home / "recorded-stream.jsonl" + # jsonl: a spend event plus the structured envelope, so either + # a stream parser or an envelope parser can see the recording. + replay.write_text( + json.dumps({ + "payload": { + "type": "token_count", + "info": {"total_token_usage": { + "total_tokens": 11, "output_tokens": 4}}, + } + }) + "\n" + json.dumps(recorded) + "\n", + encoding="utf-8", + ) + env = self.call_run( + runtime={"harness": "stub", "role": "write", + "replay": str(replay)}, + ) + findings = env.get("findings") if isinstance(env.get("findings"), list) else [] + self.assertTrue(findings, "replayed envelope must carry the recorded finding") + self.assertEqual(findings[0].get("claim"), unique) + self.assertEqual(env.get("verdict"), "changes") + self.assertEqual(env.get("status"), "ok") + self.assertFalse( + self.record_path.exists(), + "stub must not launch a harness CLI", + ) + + def test_builtin_stub_adapter_is_replay_data(self): + self.assertEqual(run.ADAPTERS.get("stub"), {"replay": True}) + self.assertEqual(run.ADAPTERS.get("direct"), {"direct": True}) + + +# --------------------------------------------------------------------------- +# R8 +# --------------------------------------------------------------------------- + + +class TheBatteryIsArmedWithCapOut(_TempRun): + """R8 — --cap-out is the runaway wire. A --cap that counts re-sent + cache killed a review (7.0M tokens to redo). The complementary + pin is R9: when output *does* exceed cap-out, the run trips. + """ + + def test_a_cache_heavy_total_does_not_trip_when_output_is_under_cap_out(self): + snap_id = uuid.uuid4().hex + dirname = f"snap-{snap_id}" + into = self.home / dirname + stream = self.codex_store / f"rollout-{dirname}.jsonl" + # 7.0M total is the measured figure. 50 output is under the cap. + write_codex_stream(stream, total=7000000, out=50) + env = self.call_run( + context=self.make_context(into=into), + runtime={"harness": "codex", "role": "write", + "caps": {"cap-out": 1000}}, + ) + self.assertTrue( + self.record_path.is_file(), + "the launch must have happened for the battery to supervise it", + ) + self.assertEqual(env.get("status"), "ok") + self.assertNotEqual(env.get("status"), "tripped") + spend = env.get("spend") if isinstance(env.get("spend"), dict) else {} + # Accounting may count the cache; the cap must not trip on it. + self.assertEqual(spend.get("total"), 7000000) + self.assertEqual(spend.get("out"), 50) + + +# --------------------------------------------------------------------------- +# R9 +# --------------------------------------------------------------------------- + + +class ABatteryTripIsNotACompletedRun(_TempRun): + """R9 — a killed run is never reported as a completed one. + + The fake CLI writes the over-budget stream, then sleeps. If the + battery is armed with --terminate, the COMPLETED marker is never + written. If the runner just waits for the process, it is. + """ + + def test_a_battery_trip_is_status_tripped_with_no_verdict_and_a_note(self): + snap_id = uuid.uuid4().hex + dirname = f"snap-{snap_id}" + into = self.home / dirname + stream = self.codex_store / f"rollout-{dirname}.jsonl" + write_codex_stream(stream, total=5000, out=5000) + done = self.home / "completed.marker" + os.environ["TASK_RUN_SLEEP"] = "8" + os.environ["TASK_RUN_DONE"] = str(done) + env = self.call_run( + context=self.make_context(into=into), + runtime={"harness": "codex", "role": "write", + "caps": {"cap-out": 100}, "timeout": 60}, + ) + self.assertEqual(env.get("status"), "tripped") + self.assertIsNone(env.get("verdict")) + self.assertNotEqual(env.get("verdict"), "approve") + self.assertNotEqual(env.get("verdict"), "changes") + note = env.get("note") + self.assertIsInstance(note, str) + self.assertTrue(note.strip(), "a trip must say which wire fired") + self.assertFalse( + done.is_file(), + "a killed run must not be allowed to complete", + ) + + +# --------------------------------------------------------------------------- +# R10 +# --------------------------------------------------------------------------- + + +class StreamDiscoveryNamesTheSnapshot(_TempRun): + """R10 — two concurrent runs of one harness must not resolve to the + same stream. Measured: a lookup took a sibling session's file and + one review was supervised against the wrong evidence. + + Plant two candidates; only one names the snapshot. The wrong one + is newer, so an mtime/latest rule picks it and this fails. + """ + + def test_the_stream_that_names_the_snapshot_is_chosen_over_a_newer_sibling(self): + snap_id = uuid.uuid4().hex + sibling_id = uuid.uuid4().hex + dirname = f"snap-{snap_id}" + into = self.home / dirname + winner = self.codex_store / f"rollout-{dirname}.jsonl" + loser = self.codex_store / f"rollout-snap-{sibling_id}.jsonl" + write_codex_stream(winner, total=42, out=7) + time.sleep(0.05) + write_codex_stream(loser, total=999, out=888) + env = self.call_run( + context=self.make_context(into=into), + runtime={"harness": "codex", "role": "write"}, + adapters=self.make_adapters(stream_names_cwd=True), + ) + self.assertTrue( + self.record_path.is_file(), + "the launch must have happened so a stream can be discovered", + ) + spend = env.get("spend") if isinstance(env.get("spend"), dict) else {} + self.assertEqual(spend.get("total"), 42) + self.assertEqual(spend.get("out"), 7) + self.assertNotEqual(spend.get("total"), 999) + + +# --------------------------------------------------------------------------- +# R11 +# --------------------------------------------------------------------------- + + +class AbsentSpendIsNotZeroSpend(_TempRun): + """R11 — if no stream is found the run still completes, and the + envelope says the spend is unknown. A genuine zero is a stream + that recorded zero; those two must not look the same. + + envelope.py requires spend.total to be a non-negative integer, so + unknown cannot live there as None. The note is how the envelope + *says* unknown; the pair of envelopes is how we tell them apart. + """ + + def test_no_stream_is_unknown_spend_not_a_measured_zero(self): + unknown = self.call_run( + runtime={"harness": "codex", "role": "write"}, + adapters=self.make_adapters(stream_names_cwd=True), + ) + self.assertTrue( + self.record_path.is_file(), + "the run still completes — the CLI launched; only the stream is missing", + ) + self.assertEqual(unknown.get("status"), "ok") + self.assertNotEqual(unknown.get("status"), "tripped") + + u_note = unknown.get("note") + u_spend = unknown.get("spend") if isinstance(unknown.get("spend"), dict) else {} + u_text = u_note.lower() if isinstance(u_note, str) else "" + says_unknown = any( + word in u_text + for word in ( + "unknown", "no stream", "absent", "missing stream", + "stream not found", "without a stream", "no session", + ) + ) + self.assertTrue( + says_unknown or u_spend.get("total") not in (0, None), + "the envelope must say the spend is unknown, not look like " + f"a zero: spend={u_spend!r} note={u_note!r}", + ) + + # Genuine zero: a stream that recorded 0/0, named for this snapshot. + if self.record_path.exists(): + self.record_path.unlink() + snap_id = uuid.uuid4().hex + dirname = f"snap-{snap_id}" + into = self.home / dirname + stream = self.codex_store / f"rollout-{dirname}.jsonl" + write_codex_stream(stream, total=0, out=0) + zero = self.call_run( + context=self.make_context(into=into), + runtime={"harness": "codex", "role": "write"}, + adapters=self.make_adapters(stream_names_cwd=True), + ) + z_spend = zero.get("spend") if isinstance(zero.get("spend"), dict) else {} + self.assertEqual(z_spend.get("total"), 0) + self.assertEqual(z_spend.get("out"), 0) + z_text = (zero.get("note") or "").lower() if isinstance(zero.get("note"), str) else "" + self.assertFalse( + any(word in z_text for word in ("unknown", "no stream", "missing stream")), + f"a measured zero must not be labelled unknown: {zero.get('note')!r}", + ) + self.assertNotEqual( + (u_spend, u_note), + (z_spend, zero.get("note")), + "unknown spend and a genuine zero must be distinguishable", + ) + + +# --------------------------------------------------------------------------- +# R12 +# --------------------------------------------------------------------------- + + +class RawOutputLivesInsideTheSnapshot(_TempRun): + """R12 — artifacts are handles. The output text belongs in a file + inside the snapshot, not in /tmp and not in the envelope. Inlining + it is how iteration N starts costing more than iteration 1. + """ + + def test_raw_output_is_inside_the_snapshot_and_not_inlined(self): + token = "RAWOUT-" + uuid.uuid4().hex + os.environ["TASK_RUN_STDOUT"] = "token" + os.environ["TASK_RUN_TOKEN"] = token + env = self.call_run() + into = self.last_into.resolve() + artifacts = env.get("artifacts") if isinstance(env.get("artifacts"), dict) else {} + raw = artifacts.get("raw") + self.assertIsInstance(raw, str) + self.assertTrue(raw.strip(), "artifacts.raw must name a file") + raw_path = Path(raw) + if not raw_path.is_absolute(): + raw_path = into / raw_path + raw_path = raw_path.resolve() + self.assertTrue(raw_path.is_file(), f"artifacts.raw is not a file: {raw!r}") + self.assertTrue( + raw_path == into or into in raw_path.parents, + f"raw {raw_path} is not inside the snapshot {into}", + ) + self.assertNotEqual( + raw_path.parent, + Path(tempfile.gettempdir()).resolve(), + "raw output must not be a /tmp tempfile; it belongs in the snapshot", + ) + body = raw_path.read_text(encoding="utf-8", errors="replace") + self.assertIn(token, body) + # The handle may appear in the envelope; the output must not. + blob = json.dumps(env) + self.assertNotIn(token, blob) + + +# --------------------------------------------------------------------------- +# R13 +# --------------------------------------------------------------------------- + + +class RunDoesNotInterpretProse(_TempRun): + """R13 — run may parse a structured envelope the harness emitted. + It must not interpret prose. Unparseable output is invalid + (CONTRACT §Statuses), never an approving verdict read off the page. + """ + + def test_a_structured_envelope_from_the_harness_is_parsed(self): + unique = "STRUCTURED-CLAIM-" + uuid.uuid4().hex + os.environ["TASK_RUN_STDOUT"] = "envelope" + os.environ["TASK_RUN_CLAIM"] = unique + os.environ["TASK_RUN_VERDICT"] = "changes" + env = self.call_run() + findings = env.get("findings") if isinstance(env.get("findings"), list) else [] + claims = [f.get("claim") for f in findings if isinstance(f, dict)] + self.assertIn(unique, claims) + self.assertEqual(env.get("verdict"), "changes") + self.assertEqual(env.get("status"), "ok") + + def test_prose_that_says_approve_does_not_become_a_verdict(self): + os.environ["TASK_RUN_STDOUT"] = "prose" + env = self.call_run() + self.assertNotEqual(env.get("verdict"), "approve") + self.assertEqual(env.get("status"), "invalid") + self.assertIsNone(env.get("verdict")) + note = env.get("note") + self.assertIsInstance(note, str) + self.assertTrue(note.strip()) + + +# --------------------------------------------------------------------------- +# R14 +# --------------------------------------------------------------------------- + + +class AnUnknownJobIsInvalid(_TempRun): + """R14 — a job is data in the table. A name that is not + there is a refusal, not a chance to improvise a prompt.""" + + def test_an_absent_job_is_invalid_with_a_note_naming_it(self): + name = "no-such-job-7e1c9a3d" + env = self.call_run(name) + self.assert_invalid(env, naming=name) + self.assertFalse( + self.record_path.exists(), + "an unknown job must not launch", + ) + + def test_load_jobs_reads_the_json_table(self): + data = run.load_jobs() + self.assertIsInstance(data, dict) + self.assertIn("verify", data) + self.assertIn("author-tests", data) + self.assertIn("adversarial-review", data) + + def test_load_jobs_reads_a_path_it_is_given(self): + path = self.home / "only.json" + path.write_text(json.dumps({"only-me": {"adapter": "direct"}}), + encoding="utf-8") + data = run.load_jobs(path) + self.assertIsInstance(data, dict) + self.assertEqual(set(data), {"only-me"}) + + +# --------------------------------------------------------------------------- +# R15 +# --------------------------------------------------------------------------- + + +class MainIsACli(_TempRun): + """R15 — stdout is the envelope; the exit code is the verdict + routed for a caller that does not parse JSON. 64 is usage, not + argparse's 2, because 2 is already taken by invalid/tripped. + """ + + def setUp(self): + super().setUp() + run.ADAPTERS = self.make_adapters() + comm = self.home / "jobs.json" + comm.write_text(json.dumps(self.jobs), encoding="utf-8") + run.JOBS_PATH = comm + + def test_a_changes_run_prints_json_and_returns_one(self): + into = self.home / f"cli-{uuid.uuid4().hex}" + os.environ["TASK_RUN_VERDICT"] = "changes" + code, out, _err = run_main([ + "author-tests", + "--repo", str(self.repo), + "--ref", self.ref, + "--base", self.base, + "--harness", "codex", + "--role", "write", + "--into", str(into), + ]) + self.assertEqual(code, 1) + env = self.parse_stdout(out) + self.assertEqual(env.get("status"), "ok") + self.assertEqual(env.get("verdict"), "changes") + + def test_an_approving_run_prints_json_and_returns_zero(self): + into = self.home / f"cli-{uuid.uuid4().hex}" + os.environ["TASK_RUN_VERDICT"] = "approve" + code, out, _err = run_main([ + "author-tests", + "--repo", str(self.repo), + "--ref", self.ref, + "--harness", "codex", + "--role", "write", + "--into", str(into), + ]) + env = self.parse_stdout(out) + self.assertEqual(env.get("verdict"), "approve") + self.assertEqual(env.get("status"), "ok") + self.assertEqual(code, 0) + + def test_an_invalid_run_prints_json_and_returns_two(self): + into = self.home / f"cli-{uuid.uuid4().hex}" + code, out, _err = run_main([ + "no-such-job-7e1c9a3d", + "--repo", str(self.repo), + "--ref", self.ref, + "--into", str(into), + ]) + self.assertEqual(code, 2) + env = self.parse_stdout(out) + self.assertEqual(env.get("status"), "invalid") + self.assertIsNone(env.get("verdict")) + self.assertIn("no-such-job-7e1c9a3d", env.get("note") or "") + + def test_a_usage_error_returns_sixty_four(self): + for argv in ( + [], + ["author-tests"], + ["author-tests", "--repo", str(self.repo)], + ["--bogus"], + ): + with self.subTest(argv=argv): + code, _out, _err = run_main(argv) + self.assertEqual(code, 64) + + +# --------------------------------------------------------------------------- +# Prompt delivery (render): the harness sees the filled job. +# --------------------------------------------------------------------------- + + +class TheHarnessReceivesTheRenderedPrompt(_TempRun): + """render is the prompt. A launch that drops {scope} or {ref} is + how a task gets a blank instruction and still exits 0.""" + + def test_the_harness_receives_the_scope_and_the_ref(self): + scope = "SCOPE-" + uuid.uuid4().hex + env = self.call_run( + require={"scope": scope, "constraints": ["no network"]}, + ) + rec = self.read_record() + blob = (rec.get("stdin") or "") + (rec.get("prompt_text") or "") + self.assertIn(scope, blob) + self.assertIn(self.ref, blob) + self.assertIsInstance(env.get("status"), str) + + def test_render_fills_the_job_template(self): + scope = "RENDER-" + uuid.uuid4().hex + text = run.render( + "author-tests", + {"repo": str(self.repo), "ref": self.ref, "base": self.base, + "include": ["alpha.py"], "into": str(self.home / "x")}, + {"scope": scope, "constraints": []}, + ) + self.assertIsInstance(text, str) + self.assertIn(scope, text) + self.assertIn(self.ref, text) + + +@unittest.skipUnless(os.name == "posix", "process groups are posix") +class ATrippedRunLeavesNothingBehind(unittest.TestCase): + """The runaway guard has to outlive the harness parent. + + Verified as broken before the fix: with the parent already reaped, + _terminate returned early and a `sleep 60` grandchild was still + running afterwards (Codex, PR #49). + """ + + def _orphan_maker(self): + """A parent that exits fast, leaving one descendant behind.""" + proc = subprocess.Popen( + ["sh", "-c", "sleep 60 & echo $! ; exec sleep 0.2"], + stdout=subprocess.PIPE, text=True, start_new_session=True) + run._remember_group(proc) + child = int(proc.stdout.readline().strip()) + proc.wait() + return proc, child + + @staticmethod + def _alive(pid): + try: + os.kill(pid, 0) + return True + except ProcessLookupError: + return False + + def test_the_group_dies_even_when_the_parent_was_already_reaped(self): + proc, child = self._orphan_maker() + self.addCleanup(lambda: self._alive(child) and os.kill(child, signal.SIGKILL)) + # the fixture must actually present the condition under test + self.assertIsNotNone(proc.poll(), "INVALID: parent not reaped") + self.assertTrue(self._alive(child), "INVALID: no orphan to clean up") + run._terminate(proc) + deadline = time.time() + 5 + while self._alive(child) and time.time() < deadline: + time.sleep(0.05) + self.assertFalse(self._alive(child), + "the descendant survived a tripped run") + + def test_the_group_is_recorded_at_launch(self): + proc = subprocess.Popen(["sleep", "0.2"], start_new_session=True) + self.addCleanup(proc.wait) + run._remember_group(proc) + self.assertEqual(getattr(proc, "_task_pgid", None), proc.pid, + "start_new_session makes the child its own leader") + + +class TheConductorsDialReachesTheHarness(unittest.TestCase): + """runtime.model and runtime.effort must arrive, or be refused. + + Verified as dropped before the fix: `_adapter_argv` formatted both + into a mapping no template referenced and no env carried, so every + launch used harness defaults while CONTRACT.md sold the dial as what + makes "eight cheap ones" and "one careful one" the same job + (Codex + Grok, PR #49). + + Every flag below was read from the CLI's own --help, not from a + review: a review reported grok's as `--effort`; it is + `--reasoning-effort`. + """ + + def rendered(self, harness, runtime): + return run._adapter_argv(run.ADAPTERS[harness], sandbox="plan", + prompt="/tmp/p", root="/tmp/r", runtime=runtime) + + def test_grok_receives_both_dials(self): + argv = self.rendered("grok", {"model": "M", "effort": "E"}) + self.assertIn("-m", argv) + self.assertEqual(argv[argv.index("-m") + 1], "M") + self.assertIn("--reasoning-effort", argv) + self.assertEqual(argv[argv.index("--reasoning-effort") + 1], "E") + + def test_claude_receives_both_dials(self): + argv = self.rendered("claude", {"model": "M", "effort": "E"}) + self.assertEqual(argv[argv.index("--model") + 1], "M") + self.assertEqual(argv[argv.index("--effort") + 1], "E") + + def test_codex_receives_the_model_dial(self): + argv = self.rendered("codex", {"model": "M"}) + self.assertEqual(argv[argv.index("-m") + 1], "M") + + def test_codex_refuses_an_effort_it_cannot_deliver(self): + """The honest floor: codex has no verified effort flag, only a + generic `-c key=value` whose reasoning key is not ours to guess.""" + with self.assertRaises(run.RunError) as caught: + self.rendered("codex", {"model": "M", "effort": "E"}) + self.assertIn("effort", str(caught.exception)) + + def test_an_unset_dial_adds_nothing(self): + """The control: without the dial the argv is the bare template.""" + bare = self.rendered("grok", {}) + self.assertNotIn("-m", bare) + self.assertNotIn("--reasoning-effort", bare) + + +class TheRealAdaptersBindToTheirEvidence(unittest.TestCase): + """Pins on the shipped adapter table, not the fixture's fakes. + + The fakes can be configured per case; these are the values a live + run actually uses, and two of them were defects. + """ + + def test_grok_discovery_is_bound_to_the_snapshot(self): + """False here made _discover_stream take the newest updates.jsonl + anywhere under the global store, so a concurrent Grok session + could be supervised, its spend charged here, or the wrong task + terminated (Codex + Grok, PR #49). Grok url-encodes only `/`, so + the snapshot's directory name survives in the path and the + marker test in _stream_names_snapshot matches it.""" + self.assertIs(run.ADAPTERS["grok"]["stream_names_cwd"], True) + + def test_every_live_adapter_can_carry_a_model(self): + for harness in ("codex", "grok", "claude"): + with self.subTest(harness=harness): + dials = run.ADAPTERS[harness].get("dials") or {} + self.assertIn("model", dials, + f"{harness} cannot carry runtime.model") + + +class TheJobAsksForWhatTheRunnerParses(unittest.TestCase): + """A job whose prompt and parser disagree returns `invalid` + however well the worker behaves (Codex + Grok, PR #49).""" + + @staticmethod + def _jobs(): + raw = json.loads(run.JOBS_PATH.read_text(encoding="utf-8")) + return raw.get("jobs", raw) + + def test_adversarial_review_requests_the_typed_envelope(self): + prompt = self._jobs()["adversarial-review"]["prompt"] + self.assertNotIn("CONTRIB.md review wire format", prompt, + "the prose wire format is the human review " + "format, not this job's stdout") + self.assertIn("JSON", prompt) + for field in ("status", "verdict", "findings"): + self.assertIn(field, prompt, f"the prompt never names {field}") + + def test_the_requested_keys_are_the_ones_the_parser_accepts(self): + """Pinned against envelope.FIELDS rather than a copy of the list, + so the prompt cannot drift from the parser.""" + prompt = self._jobs()["adversarial-review"]["prompt"] + for field in envelope.FIELDS: + self.assertIn(field, prompt, + f"envelope field {field!r} is not requested") + + +class TheDocAndTheRegistryAgree(unittest.TestCase): + """CONTRACT.md's job table and jobs.json must name the same set. + + They did not: the table listed four and the file held three -- + `sweep` was documented and did not exist. That is the same shape as + the planted-fault promise one section further down, and the reason + it survived is that nothing compared the two. Comparing them is + three lines, so it is done here rather than noticed again later. + """ + + @staticmethod + def _table_names(): + text = run.CONTRACT_PATH.read_text(encoding="utf-8") \ + if hasattr(run, "CONTRACT_PATH") else \ + (Path(run.JOBS_PATH).parent / "CONTRACT.md").read_text(encoding="utf-8") + names, seen_header = set(), False + for line in text.splitlines(): + if line.startswith("| job ") or line.startswith("| `job`"): + seen_header = True + continue + if seen_header: + if not line.startswith("|"): + break + cell = line.split("|")[1].strip() + if cell.startswith("`") and cell.endswith("`"): + names.add(cell.strip("`")) + return names + + def test_every_documented_job_exists(self): + registry = set(json.loads(Path(run.JOBS_PATH).read_text(encoding="utf-8"))) + documented = self._table_names() + self.assertTrue(documented, "INVALID: no job table found in CONTRACT.md") + self.assertEqual(documented - registry, set(), + "documented but not built") + + def test_every_built_job_is_documented(self): + registry = set(json.loads(Path(run.JOBS_PATH).read_text(encoding="utf-8"))) + documented = self._table_names() + self.assertEqual(registry - documented, set(), + "built but not documented") + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/task/tests/test_verify.py b/ops/devlane/task/tests/test_verify.py new file mode 100644 index 0000000..ef9b2bc --- /dev/null +++ b/ops/devlane/task/tests/test_verify.py @@ -0,0 +1,629 @@ +"""verify: run a command and report it in the envelope. + +Written from SPEC.md, before the module existed. Each test names the +contract rule it pins: + + V1 a claim that holds is ok / approve, with no findings + V2 a claim that fails is ok / changes, with one pasteable reproduce + V3 a missing executable is invalid with a note — never changes + V4 a non-existent cwd is invalid, not a crash + V5 expect is matched against stdout AND stderr combined + V6 expect_exit is the code that means the claim HELD + V7 a timeout is tripped, and the process is not left running + V8 a string command raises ValueError rather than being shell-split + V9 spend is a zero-token run that DID run + V10 stamp.ref is cwd's git HEAD, or the literal "worktree" + V11 raw combined output is a file named by artifacts.raw, never inlined + V12 main is a CLI: JSON on stdout; 0 / 1 / 2 / 64 +""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import shlex +import signal +import subprocess +import sys +import tempfile +import unittest +import uuid +from pathlib import Path + +import support + +verify = support.load("verify") + + +def _git_env(home: Path) -> dict: + # GIT_DIR / GIT_WORK_TREE in the caller environment would aim git + # at the snapshot (or its parent). Fixtures must be closed worlds, + # and this suite must not run git against the snapshot's own repo. + env = {k: v for k, v in os.environ.items() + if not k.startswith("GIT_") and k != "XDG_CONFIG_HOME"} + env.update({ + "HOME": str(home), + "GIT_AUTHOR_NAME": "verify-test", + "GIT_AUTHOR_EMAIL": "verify-test@example.test", + "GIT_COMMITTER_NAME": "verify-test", + "GIT_COMMITTER_EMAIL": "verify-test@example.test", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + }) + return env + + +def _cmdline_pids(marker: str) -> list[int]: + """Pids whose argv contains *marker*. Linux /proc; empty if absent.""" + proc = Path("/proc") + if not proc.is_dir(): + return [] + found = [] + me = os.getpid() + for entry in proc.iterdir(): + if not entry.name.isdigit(): + continue + pid = int(entry.name) + if pid == me: + continue + try: + raw = (entry / "cmdline").read_bytes() + except (OSError, FileNotFoundError): + continue + text = raw.replace(b"\x00", b" ").decode("utf-8", "replace") + if marker in text: + found.append(pid) + return found + + +def _pid_exists(pid: int) -> bool: + if pid <= 0: + return False + return Path(f"/proc/{pid}").exists() + + +def run_main(argv): + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = verify.main(argv) + return code, out.getvalue(), err.getvalue() + + +class _TempCwd(unittest.TestCase): + """A throwaway cwd. Never the snapshot's own tree.""" + + def setUp(self): + self._td = tempfile.TemporaryDirectory() + self.root = Path(self._td.name) + self.cwd = self.root / "cwd" + self.cwd.mkdir() + self._markers: list[str] = [] + + def tearDown(self): + for marker in self._markers: + for pid in _cmdline_pids(marker): + # Already gone is the outcome we wanted anyway. + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGKILL) + self._td.cleanup() + + def parse_stdout(self, out): + # An empty stdout is not an envelope. Failing here against the + # stub (which prints nothing) is the honest red for CLI tests + # whose exit code is 0, because that 0 is also the stub's return. + self.assertTrue(out.strip(), "envelope JSON on stdout") + try: + env = json.loads(out) + except json.JSONDecodeError: + self.fail(f"stdout was not JSON: {out!r}") + self.assertIsInstance(env, dict) + return env + + def token(self, prefix): + return f"{prefix}-{uuid.uuid4().hex}" + + +class AClaimThatHoldsApproves(_TempCwd): + """V1 — the mapping's first row: it ran, it held, nothing to list.""" + + def test_a_held_claim_is_ok_approve_with_no_findings(self): + env = verify.check( + "python exits 0", + [sys.executable, "-c", "pass"], + cwd=str(self.cwd), + ) + self.assertIsInstance(env, dict) + self.assertEqual(env.get("status"), "ok") + self.assertEqual(env.get("verdict"), "approve") + self.assertEqual(env.get("findings"), []) + self.assertEqual(env.get("job"), "verify") + + +class AClaimThatFailsReportsOneFinding(_TempCwd): + """V2 — a failed claim is still a completed look; the finding is + the command a human pastes to see the same result.""" + + def test_a_wrong_exit_is_ok_changes_with_one_pasteable_finding(self): + command = [sys.executable, "-c", "raise SystemExit(1)"] + env = verify.check( + "python exits 0", + command, + cwd=str(self.cwd), + ) + self.assertIsInstance(env, dict) + self.assertEqual(env.get("status"), "ok") + self.assertEqual(env.get("verdict"), "changes") + findings = env.get("findings") if isinstance(env.get("findings"), list) else [] + self.assertEqual(len(findings), 1) + # shlex.join, not " ".join: the -c argument contains a space, + # and an unquoted join is not pasteable. + self.assertEqual(findings[0].get("reproduce"), shlex.join(command)) + + +class AMissingExecutableIsInvalidNeverChanges(_TempCwd): + """V3 — the reason this module exists. 'The claim is false' and + 'we could not evaluate the claim' are different answers, and + returning changes for a missing binary reports a defect nobody + has evidence for.""" + + def test_a_missing_executable_is_invalid_with_a_note(self): + missing = self.root / "no-such-dir" / "verify-no-such-exe-7e1c9a3d" + env = verify.check( + "the missing tool runs", + [str(missing)], + cwd=str(self.cwd), + ) + self.assertIsInstance(env, dict) + self.assertEqual(env.get("status"), "invalid") + # verdict None is the contract's null. Pinning only + # `is not "changes"` would pass against the stub's {}. + self.assertIsNone(env.get("verdict")) + self.assertNotEqual(env.get("verdict"), "changes") + self.assertEqual(env.get("findings"), []) + note = env.get("note") + self.assertIsInstance(note, str) + self.assertTrue(note.strip(), "invalid must say why it could not run") + + def test_a_file_that_is_not_executable_is_invalid_not_changes(self): + # Same mapping row: the process never starts. Invoking the + # interpreter on the file would hide the defect this pins. + script = self.cwd / "noexec.py" + script.write_text( + "#!/usr/bin/env python3\nprint('ran')\n", encoding="utf-8") + script.chmod(0o644) + env = verify.check( + "the script runs", + [str(script)], + cwd=str(self.cwd), + ) + self.assertIsInstance(env, dict) + self.assertEqual(env.get("status"), "invalid") + self.assertIsNone(env.get("verdict")) + self.assertNotEqual(env.get("verdict"), "changes") + self.assertIsInstance(env.get("note"), str) + self.assertTrue(env.get("note", "").strip()) + + +class ANonexistentCwdIsInvalid(_TempCwd): + """V4 — a bad cwd is the same class of refusal as a missing + binary: we could not look, so we must not crash and must not + invent a verdict.""" + + def test_a_missing_cwd_is_invalid_not_a_crash(self): + missing = self.root / "cwd-does-not-exist" + try: + env = verify.check( + "python exits 0", + [sys.executable, "-c", "pass"], + cwd=str(missing), + ) + except Exception as exc: + self.fail(f"non-existent cwd must not crash: {exc}") + self.assertIsInstance(env, dict) + self.assertEqual(env.get("status"), "invalid") + self.assertIsNone(env.get("verdict")) + self.assertIsInstance(env.get("note"), str) + self.assertTrue(env.get("note", "").strip()) + + def test_a_cwd_that_is_a_file_is_invalid(self): + not_a_dir = self.root / "not-a-dir" + not_a_dir.write_text("x\n", encoding="utf-8") + try: + env = verify.check( + "python exits 0", + [sys.executable, "-c", "pass"], + cwd=str(not_a_dir), + ) + except Exception as exc: + self.fail(f"a file as cwd must not crash: {exc}") + self.assertIsInstance(env, dict) + self.assertEqual(env.get("status"), "invalid") + self.assertIsNone(env.get("verdict")) + + +class ExpectIsCheckedAgainstCombinedOutput(_TempCwd): + """V5 — expect is a substring of stdout and stderr together. + Checking only one stream would approve a claim whose evidence + was on the other, or reject one whose evidence was on stderr.""" + + def test_expect_on_stdout_approves(self): + token = self.token("EXPECT_STDOUT") + env = verify.check( + "stdout carries the token", + [sys.executable, "-c", f"print({token!r})"], + cwd=str(self.cwd), + expect=token, + ) + self.assertIsInstance(env, dict) + self.assertEqual(env.get("status"), "ok") + self.assertEqual(env.get("verdict"), "approve") + self.assertEqual(env.get("findings"), []) + + def test_expect_on_stderr_only_approves(self): + token = self.token("EXPECT_STDERR") + env = verify.check( + "stderr carries the token", + [sys.executable, "-c", + f"import sys; sys.stderr.write({token!r} + '\\n')"], + cwd=str(self.cwd), + expect=token, + ) + self.assertIsInstance(env, dict) + self.assertEqual(env.get("status"), "ok") + self.assertEqual(env.get("verdict"), "approve") + self.assertEqual(env.get("findings"), []) + + def test_a_zero_exit_that_lacks_expect_is_changes(self): + env = verify.check( + "output contains a token that was never printed", + [sys.executable, "-c", "print('hello')"], + cwd=str(self.cwd), + expect=self.token("EXPECT_ABSENT"), + ) + self.assertIsInstance(env, dict) + self.assertEqual(env.get("status"), "ok") + self.assertEqual(env.get("verdict"), "changes") + findings = env.get("findings") if isinstance(env.get("findings"), list) else [] + self.assertEqual(len(findings), 1) + + def test_expect_present_does_not_save_a_wrong_exit(self): + token = self.token("EXPECT_AND_FAIL") + env = verify.check( + "exits 0 and prints the token", + [sys.executable, "-c", + f"print({token!r}); raise SystemExit(1)"], + cwd=str(self.cwd), + expect=token, + expect_exit=0, + ) + self.assertIsInstance(env, dict) + self.assertEqual(env.get("status"), "ok") + self.assertEqual(env.get("verdict"), "changes") + + +class ExpectExitNamesTheCodeThatMeansTheClaimHeld(_TempCwd): + """V6 — expect_exit=1 means 'I assert this command FAILS'. + Treating any nonzero as changes would make that assertion + impossible; treating exit 0 as always-approve would make it a lie.""" + + def test_exit_one_with_expect_exit_one_is_approve_not_changes(self): + env = verify.check( + "the command fails", + [sys.executable, "-c", "raise SystemExit(1)"], + cwd=str(self.cwd), + expect_exit=1, + ) + self.assertIsInstance(env, dict) + self.assertEqual(env.get("status"), "ok") + self.assertEqual(env.get("verdict"), "approve") + self.assertNotEqual(env.get("verdict"), "changes") + self.assertEqual(env.get("findings"), []) + + def test_exit_zero_with_expect_exit_one_is_changes(self): + # The claim was that the command fails. It did not. + env = verify.check( + "the command fails", + [sys.executable, "-c", "pass"], + cwd=str(self.cwd), + expect_exit=1, + ) + self.assertIsInstance(env, dict) + self.assertEqual(env.get("status"), "ok") + self.assertEqual(env.get("verdict"), "changes") + + +class ATimeoutTripsAndDoesNotLeaveTheProcessRunning(_TempCwd): + """V7 — a timeout is a trip, not a failed claim. The envelope + cannot see a sleeper still burning a core; the process table can.""" + + def test_a_timeout_is_tripped_and_the_process_is_gone(self): + marker = self.token("VERIFY-V7") + self._markers.append(marker) + pidfile = self.root / "sleeper.pid" + sleeper = self.root / "sleeper.py" + sleeper.write_text( + "import os, sys, time\n" + "path, marker = sys.argv[1], sys.argv[2]\n" + "with open(path, 'w') as f:\n" + " f.write(str(os.getpid()))\n" + " f.flush()\n" + " os.fsync(f.fileno())\n" + "time.sleep(60)\n", + encoding="utf-8", + ) + env = verify.check( + "the sleeper finishes", + [sys.executable, str(sleeper), str(pidfile), marker], + cwd=str(self.cwd), + timeout=1, + ) + self.assertIsInstance(env, dict) + self.assertEqual(env.get("status"), "tripped") + self.assertIsNone(env.get("verdict")) + # The command must have started: otherwise we are asserting + # 'not running' about a process that was never launched, and + # the stub would pass that half. + self.assertTrue( + pidfile.is_file(), + "the command must have started before being timed out", + ) + pid = int(pidfile.read_text(encoding="utf-8").strip()) + leftover = [p for p in _cmdline_pids(marker) if _pid_exists(p)] + self.assertFalse( + _pid_exists(pid), + f"pid {pid} was left running after timeout", + ) + self.assertEqual( + leftover, [], + f"process(es) still running with marker {marker}: {leftover}", + ) + + +class AStringCommandIsRefused(_TempCwd): + """V8 — a shell string is how an argv becomes an injection. + subprocess will accept a single-token string as argv[0]; the + refusal has to happen before that.""" + + def test_a_string_path_raises_value_error(self): + # sys.executable as a string would run, if anyone passed it + # through to subprocess. The type is the whole pin. + with self.assertRaises(ValueError): + verify.check( + "python runs", + sys.executable, + cwd=str(self.cwd), + ) + + def test_a_shell_string_is_not_executed(self): + target = self.root / "must-not-be-created" + with self.assertRaises(ValueError): + verify.check( + "the file is touched", + f"touch {target}", + cwd=str(self.cwd), + ) + self.assertFalse(target.exists()) + + +class SpendRecordsAFreeRun(_TempCwd): + """V9 — a verify costs no tokens and says so. runs is 1 because + it DID run; the default 0 would tell worth.py that nothing happened.""" + + def test_spend_is_zero_tokens_and_one_run(self): + env = verify.check( + "python exits 0", + [sys.executable, "-c", "pass"], + cwd=str(self.cwd), + ) + self.assertIsInstance(env, dict) + self.assertEqual( + env.get("spend"), + {"harness": None, "total": 0, "out": 0, "runs": 1}, + ) + + +class StampRefComesFromTheCwd(_TempCwd): + """V10 — the envelope requires a ref. Without one a fact names + no state, and cannot be re-checked. cwd is the state that was + looked at; os.getcwd() is the test runner and is not that.""" + + def test_a_non_repo_cwd_stamps_the_literal_worktree(self): + env = verify.check( + "python exits 0", + [sys.executable, "-c", "pass"], + cwd=str(self.cwd), + ) + self.assertIsInstance(env, dict) + stamp = env.get("stamp") if isinstance(env.get("stamp"), dict) else {} + self.assertEqual(stamp.get("ref"), "worktree") + + def test_a_repo_cwd_stamps_git_head(self): + repo = self.root / "repo" + repo.mkdir() + env = _git_env(self.root) + saved = {k: os.environ[k] for k in list(os.environ) + if k.startswith("GIT_")} + for k in list(saved): + del os.environ[k] + try: + def git(*args): + r = subprocess.run( + ["git", *args], cwd=repo, env=env, + capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError( + f"git {args} failed ({r.returncode}): {r.stderr}") + return r + + git("init") + git("config", "user.name", "verify-test") + git("config", "user.email", "verify-test@example.test") + git("config", "commit.gpgsign", "false") + (repo / "README").write_text("fixture\n", encoding="utf-8") + git("add", "-A") + git("commit", "-m", "init") + head = git("rev-parse", "HEAD").stdout.strip() + self.assertTrue(head, "fixture HEAD must exist") + + result = verify.check( + "python exits 0", + [sys.executable, "-c", "pass"], + cwd=str(repo), + ) + finally: + for k in list(os.environ): + if k.startswith("GIT_"): + del os.environ[k] + os.environ.update(saved) + + self.assertIsInstance(result, dict) + stamp = result.get("stamp") if isinstance(result.get("stamp"), dict) else {} + self.assertEqual(stamp.get("ref"), head) + + +class RawOutputLivesInAFileNotTheEnvelope(_TempCwd): + """V11 — artifacts are handles, not contents. The property the + envelope exists for is that iteration N costs the caller about + what iteration 1 cost, and the way that breaks is prose migrating + into the dict. Even a short output belongs in the file.""" + + def test_combined_output_is_in_the_named_file_and_not_the_envelope(self): + stdout_tok = self.token("RAW_STDOUT") + stderr_tok = self.token("RAW_STDERR") + before = { + p.relative_to(self.cwd).as_posix(): p.read_bytes() + for p in self.cwd.rglob("*") if p.is_file() + } + env = verify.check( + "the command prints both tokens", + [sys.executable, "-c", + ("import sys\n" + f"sys.stdout.write({stdout_tok!r})\n" + f"sys.stderr.write({stderr_tok!r})\n")], + cwd=str(self.cwd), + ) + self.assertIsInstance(env, dict) + artifacts = env.get("artifacts") if isinstance(env.get("artifacts"), dict) else {} + raw = artifacts.get("raw") + self.assertIsInstance(raw, str) + self.assertTrue(raw.strip(), "artifacts.raw must name a file") + raw_path = Path(raw) + self.assertTrue(raw_path.is_file(), f"artifacts.raw is not a file: {raw!r}") + body = raw_path.read_text(encoding="utf-8", errors="replace") + self.assertIn(stdout_tok, body) + self.assertIn(stderr_tok, body) + # The handle may appear in the envelope; the output must not. + blob = json.dumps(env) + self.assertNotIn(stdout_tok, blob) + self.assertNotIn(stderr_tok, blob) + after = { + p.relative_to(self.cwd).as_posix(): p.read_bytes() + for p in self.cwd.rglob("*") if p.is_file() + } + # verify runs commands; it does not edit cwd. The raw file + # therefore cannot live inside the tree it is judging. + self.assertEqual(after, before) + + +class MainIsACli(_TempCwd): + """V12 — stdout is the envelope; the exit code is the verdict + routed for a caller that does not parse JSON. 64 is usage, not + argparse's 2, because 2 is already taken by invalid/tripped.""" + + def test_an_approving_run_prints_json_and_returns_zero(self): + code, out, _err = run_main( + ["--claim", "python exits 0", "--cwd", str(self.cwd), + "--", sys.executable, "-c", "pass"], + ) + env = self.parse_stdout(out) + self.assertEqual(env.get("job"), "verify") + self.assertEqual(env.get("status"), "ok") + self.assertEqual(env.get("verdict"), "approve") + self.assertEqual(code, 0) + + def test_a_failing_run_prints_json_and_returns_one(self): + code, out, _err = run_main( + ["--claim", "python exits 0", "--cwd", str(self.cwd), + "--", sys.executable, "-c", "raise SystemExit(1)"], + ) + self.assertEqual(code, 1) + env = self.parse_stdout(out) + self.assertEqual(env.get("status"), "ok") + self.assertEqual(env.get("verdict"), "changes") + + def test_an_invalid_run_prints_json_and_returns_two(self): + missing = self.root / "no-such-dir" / "verify-no-such-exe" + code, out, _err = run_main( + ["--claim", "the tool runs", "--cwd", str(self.cwd), + "--", str(missing)], + ) + self.assertEqual(code, 2) + env = self.parse_stdout(out) + self.assertEqual(env.get("status"), "invalid") + self.assertIsNone(env.get("verdict")) + + def test_a_tripped_run_prints_json_and_returns_two(self): + marker = self.token("VERIFY-V12-TRIP") + self._markers.append(marker) + sleeper = self.root / "cli-sleeper.py" + sleeper.write_text( + "import sys, time\n" + "marker = sys.argv[1]\n" + "time.sleep(60)\n", + encoding="utf-8", + ) + code, out, _err = run_main( + ["--claim", "the sleeper finishes", "--cwd", str(self.cwd), + "--timeout", "1", + "--", sys.executable, str(sleeper), marker], + ) + self.assertEqual(code, 2) + env = self.parse_stdout(out) + self.assertEqual(env.get("status"), "tripped") + self.assertIsNone(env.get("verdict")) + leftover = [p for p in _cmdline_pids(marker) if _pid_exists(p)] + self.assertEqual( + leftover, [], + f"CLI timeout left process(es) running: {leftover}", + ) + + def test_expect_exit_on_the_cli_approves_a_failure(self): + code, out, _err = run_main( + ["--claim", "the command fails", "--cwd", str(self.cwd), + "--expect-exit", "1", + "--", sys.executable, "-c", "raise SystemExit(1)"], + ) + env = self.parse_stdout(out) + self.assertEqual(env.get("verdict"), "approve") + self.assertEqual(code, 0) + + def test_expect_on_the_cli_is_honoured(self): + token = self.token("CLI_EXPECT") + code, out, _err = run_main( + ["--claim", "stdout carries the token", "--cwd", str(self.cwd), + "--expect", token, + "--", sys.executable, "-c", f"print({token!r})"], + ) + env = self.parse_stdout(out) + self.assertEqual(env.get("verdict"), "approve") + self.assertEqual(code, 0) + + def test_a_usage_error_returns_sixty_four(self): + for argv in ( + [], + ["--claim", "x"], + ["--cwd", str(self.cwd)], + ["--claim", "x", "--cwd", str(self.cwd)], + ["--bogus"], + ): + with self.subTest(argv=argv): + code, _out, _err = run_main(argv) + self.assertEqual(code, 64) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/task/verify.py b/ops/devlane/task/verify.py new file mode 100644 index 0000000..723e8fc --- /dev/null +++ b/ops/devlane/task/verify.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""Verify a human claim without turning execution failure into evidence. + +The distinction here is the reason this job exists: a command that +disproves a claim returns ``changes``, while a command that never ran returns +``invalid``. Raw output is kept out of the envelope because envelopes are +small routing records; the artifact they name is the evidence. +""" + +from __future__ import annotations + +import argparse +import contextlib +import json +import os +import shlex +import signal +import subprocess +import sys +import tempfile +from datetime import datetime, timezone + +import envelope +import fileset + +_SPEND = {"harness": None, "total": 0, "out": 0, "runs": 1} + + +class _UsageError(Exception): + """An argparse refusal that ``main`` translates to exit 64.""" + + +class _Parser(argparse.ArgumentParser): + def error(self, message): + self.print_usage(sys.stderr) + raise _UsageError(f"{self.prog}: error: {message}") + + +def _now(): + return datetime.now(timezone.utc).isoformat() + + +def _ref(cwd): + """Name committed state when possible, without making Git a prerequisite.""" + try: + return fileset._commit(cwd, "HEAD") + except (fileset.FilesetError, TypeError, ValueError, OSError): + # Verification is useful outside a repository too. A Git lookup + # failure changes provenance, not whether the requested command ran. + return "worktree" + + +def _raw_artifact(): + descriptor, path = tempfile.mkstemp(prefix="verify-", suffix=".raw") + os.close(descriptor) + return path + + +def _command_text(command): + """Return a pasteable reproduction, or explain why argv is unusable.""" + try: + return shlex.join([os.fsdecode(os.fspath(part)) for part in command]) + except (TypeError, ValueError) as exc: + raise ValueError("command entries must be strings or path-like values") from exc + + +def _stop(process): + """Stop the command and its ordinary descendants after a timeout.""" + if os.name == "posix": + try: + # The child starts a new session specifically so a timed-out + # verifier cannot leave helpers running after their parent dies. + os.killpg(process.pid, signal.SIGKILL) + return + except (ProcessLookupError, PermissionError): + pass + # A process that exited between the timeout and this kill is the + # state we were trying to reach. + with contextlib.suppress(ProcessLookupError): + process.kill() + + +def check(claim, command, *, cwd, expect=None, expect_exit=0, + timeout=300) -> dict: + """Run one argv command and return its mechanical answer as an envelope.""" + if not isinstance(claim, str) or not claim.strip() or "\n" in claim or "\r" in claim: + raise ValueError("claim must be a non-empty one-line string") + if not isinstance(command, list): + # Shell-looking text is refused instead of guessed at or split. That + # keeps caller-controlled punctuation from becoming shell syntax. + raise ValueError("command must be an argv list, never a shell string") + if expect is not None and not isinstance(expect, str): + raise ValueError("expect must be a string or None") + if not isinstance(expect_exit, int) or isinstance(expect_exit, bool): + raise ValueError("expect_exit must be an integer") + if not isinstance(timeout, (int, float)) or isinstance(timeout, bool) or timeout <= 0: + raise ValueError("timeout must be a positive number of seconds") + + started = _now() + ref = _ref(cwd) + raw_path = _raw_artifact() + artifacts = {"raw": raw_path} + stamp = {"ref": ref, "started": started, "ended": None} + + try: + reproduce = _command_text(command) + except ValueError as exc: + stamp["ended"] = _now() + return envelope.build( + "verify", status="invalid", verdict=None, artifacts=artifacts, + spend=_SPEND, stamp=stamp, + note=f"command could not run: {exc}", + ) + if not command: + stamp["ended"] = _now() + return envelope.build( + "verify", status="invalid", verdict=None, artifacts=artifacts, + spend=_SPEND, stamp=stamp, + note="command could not run: the argv list is empty", + ) + + try: + process = subprocess.Popen( + command, + cwd=cwd, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=False, + start_new_session=(os.name == "posix"), + ) + except (OSError, TypeError, ValueError) as exc: + stamp["ended"] = _now() + return envelope.build( + "verify", status="invalid", verdict=None, artifacts=artifacts, + spend=_SPEND, stamp=stamp, + note=f"command could not run: {exc}", + ) + + timed_out = False + try: + output, _ = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + _stop(process) + # communicate() after the kill both reaps the process and drains the + # output pipe, so neither a zombie nor evidence is left behind. + output, _ = process.communicate() + + with open(raw_path, "wb") as raw_stream: + raw_stream.write(output or b"") + stamp["ended"] = _now() + + if timed_out: + return envelope.build( + "verify", status="tripped", verdict=None, artifacts=artifacts, + spend=_SPEND, stamp=stamp, + note=f"command exceeded timeout of {timeout} seconds", + ) + + combined = (output or b"").decode("utf-8", "replace") + held = process.returncode == expect_exit + if expect is not None: + held = held and expect in combined + + if held: + return envelope.build( + "verify", status="ok", verdict="approve", artifacts=artifacts, + spend=_SPEND, stamp=stamp, + ) + + finding = envelope.finding( + "p2", str(cwd), claim, reproduce=reproduce, + ) + return envelope.build( + "verify", status="ok", verdict="changes", findings=[finding], + artifacts=artifacts, spend=_SPEND, stamp=stamp, + ) + + +def _parser(): + parser = _Parser(prog="verify.py") + parser.add_argument("--claim", required=True) + parser.add_argument("--cwd", required=True) + parser.add_argument("--expect") + parser.add_argument("--expect-exit", type=int, default=0) + parser.add_argument("--timeout", type=float, default=300) + parser.add_argument("command", nargs=argparse.REMAINDER) + return parser + + +def main(argv=None) -> int: + try: + arguments = _parser().parse_args(argv) + if arguments.command[:1] != ["--"]: + raise _UsageError( + "verify.py: error: -- must separate options from command") + # REMAINDER deliberately preserves the separator. It marks the + # boundary for argparse and is not part of the requested argv. + arguments.command = arguments.command[1:] + if not arguments.command: + raise _UsageError("verify.py: error: command after -- is required") + except _UsageError as exc: + print(exc, file=sys.stderr) + return 64 + except SystemExit as exc: + # argparse owns --help output, but main remains callable as a function. + return int(exc.code) + + try: + result = check( + arguments.claim, + arguments.command, + cwd=arguments.cwd, + expect=arguments.expect, + expect_exit=arguments.expect_exit, + timeout=arguments.timeout, + ) + except ValueError as exc: + print(f"verify.py: error: {exc}", file=sys.stderr) + return 64 + + print(json.dumps(result)) + if result["verdict"] == "approve": + return 0 + if result["verdict"] == "changes": + return 1 + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/devlane/telemetry/breaker.py b/ops/devlane/telemetry/breaker.py new file mode 100644 index 0000000..aefa80e --- /dev/null +++ b/ops/devlane/telemetry/breaker.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +"""Live tripwires over a harness session stream. + + breaker.py [--pid P] [--terminate] [--once] + [--cap N] [--cap-out N] [--stall S] [--size-mb M] + [--repeat-n N] [--repeat-k K] [--err-min N] [--window W] + [--interval S] [--disable w1,w2] [--tripped-file PATH] + +usage.py learns after the fact; this catches failure while it is +happening. Adapted from loopstrap's token-breaker.py: the incremental +tail, the per-message-id accounting (a snapshot that re-emits a message id +must not double-count it), cumulative-last-wins for Codex token_count +events, the wire battery with per-wire disable, the distinct trip exit, +and drain-then-exit when the watched process dies. Deliberately NOT +carried over: the owner-override lane, SIGSTOP pause, and filesystem +progress checkpoints — loopstrap machinery this repo has no seat for. + +Wires, and which harness streams can feed them: + + tokens total spend > --cap claude, codex, grok + tokens-out output spend > --cap-out claude, codex, grok + repeat-loop same (tool, input) call --repeat-n times + in the last --repeat-k claude + error-storm >= --err-min errored tool results in the + last --window claude + stall no stream growth for --stall seconds while + --pid lives any format + size stream file > --size-mb any format + +Grok usage (updates.jsonl turn_completed events, cumulative per run, +runs split on a reported totals shrink) feeds the token walls; only +repeat-loop and error-storm stay claude-only, since grok streams carry +no tool_use/tool_result records. Exit codes: 0 the watched process ended (or --once found +nothing), 3 a wire tripped, 64 usage. +""" + +from __future__ import annotations + +import argparse +import contextlib +import hashlib +import json +import os +import signal +import sys +import time +from collections import deque +from pathlib import Path + +EXIT_TRIPPED = 3 + + +class Battery: + def __init__(self, args): + self.args = args + self.per_msg = {} + self.per_out = {} + self.codex_total = None + # Grok run banking (accounting shipped with slice 2a): usage is + # cumulative within a run; a REPORTED totalTokens shrink ends + # the run; last report per currency wins within it. + self.grok_banked = {"total": 0, "out": 0} + self.grok_current = {} + self.grok_prev_total = None + self.calls = deque(maxlen=args.repeat_k) + self.results = deque(maxlen=args.window) + self.disabled = {x.strip() for x in (args.disable or "").split(",") + if x.strip()} + + # -- accounting -------------------------------------------------------- + + def total(self): + spent = sum(self.per_msg.values()) + if self.codex_total is not None: + spent += self.codex_total.get("total_tokens", 0) + spent += self.grok_banked["total"] + self.grok_current.get( + "totalTokens", 0) + return spent + + def total_out(self): + out = sum(self.per_out.values()) + if self.codex_total is not None: + out += self.codex_total.get("output_tokens", 0) + out += self.grok_banked["out"] + self.grok_current.get( + "outputTokens", 0) + return out + + def feed_grok(self, update): + usage = update.get("usage") + if update.get("sessionUpdate") != "turn_completed" or not isinstance( + usage, dict + ): + # Cancelled turns carry no usage; usage dicts on other + # update kinds are not accounting records. + return + # Only numeric counters enter the accounting: a null or string + # value would crash a later comparison or sum, killing the + # battery while the reviewer runs on unsupervised. + usage = { + key: value for key, value in usage.items() + if isinstance(value, (int, float)) and not isinstance(value, bool) + } + if "totalTokens" in usage: + total = usage["totalTokens"] + if self.grok_prev_total is not None and total < self.grok_prev_total: + self.grok_banked["total"] += self.grok_current.get( + "totalTokens", 0) + self.grok_banked["out"] += self.grok_current.get( + "outputTokens", 0) + self.grok_current = {} + self.grok_prev_total = total + for key in ("totalTokens", "outputTokens"): + if key in usage: + self.grok_current[key] = usage[key] + + def feed(self, line): + line = line.strip() + if not line: + return + try: + ev = json.loads(line) + except ValueError: + return + if not isinstance(ev, dict): + # a JSON-RPC batch ([...]) or bare scalar line is not a + # record; it must not kill the battery mid-watch + return + params = ev.get("params") + # positional JSON-RPC params (a list) must not crash the + # battery before the record is even identified as grok + update = params.get("update") if isinstance(params, dict) else None + if isinstance(update, dict): + self.feed_grok(update) + return + payload = ev.get("payload") or {} + if payload.get("type") == "token_count": + # Codex: cumulative — the last event IS the spend so far. + info = payload.get("info") or {} + self.codex_total = (info.get("total_token_usage") + or self.codex_total) + return + message = ev.get("message") or {} + usage = message.get("usage") + mid = message.get("id") + if usage and mid: + # Keyed by message id: a snapshot overwrite re-emits the same + # id, and summing both copies would double the spend. + self.per_msg[mid] = ( + (usage.get("input_tokens") or 0) + + (usage.get("cache_creation_input_tokens") or 0) + + (usage.get("cache_read_input_tokens") or 0) + + (usage.get("output_tokens") or 0)) + self.per_out[mid] = usage.get("output_tokens") or 0 + for c in message.get("content") or []: + if not isinstance(c, dict): + continue + if c.get("type") == "tool_use": + digest = hashlib.sha256(json.dumps( + c.get("input"), sort_keys=True, default=str + ).encode()).hexdigest() + self.calls.append((c.get("name"), digest)) + if c.get("type") == "tool_result": + self.results.append("err" if c.get("is_error") else "ok") + + # -- wires ------------------------------------------------------------- + + def check(self, stream: Path): + a = self.args + if "tokens" not in self.disabled and a.cap and self.total() > a.cap: + return "tokens", f"{self.total():,} total > cap {a.cap:,}" + if ("tokens-out" not in self.disabled and a.cap_out + and self.total_out() > a.cap_out): + return ("tokens-out", + f"{self.total_out():,} output > cap {a.cap_out:,}") + if ("repeat-loop" not in self.disabled + and len(self.calls) >= a.repeat_n): + top = max(self.calls, key=self.calls.count) + n = self.calls.count(top) + if n >= a.repeat_n: + return ("repeat-loop", + (f"identical call x{n} in last {len(self.calls)}:" + f" {top[0]}")) + if "error-storm" not in self.disabled and self.results: + errors = sum(1 for r in self.results if r != "ok") + if errors >= a.err_min: + return ("error-storm", + f"{errors}/{len(self.results)} tool results errored") + if "size" not in self.disabled: + try: + size = stream.stat().st_size + except OSError: + size = 0 + if size > a.size_mb * 1024 * 1024: + return ("size", + f"{size / 1048576:.1f} MB > --size-mb {a.size_mb}") + return None + + +def alive(pid): + try: + os.kill(pid, 0) + return True + except OSError: + return False + + +def terminate_tree(pid): + try: + os.kill(pid, signal.SIGTERM) + except OSError: + return + for _ in range(20): + if not alive(pid): + return + time.sleep(0.5) + with contextlib.suppress(OSError): + os.kill(pid, signal.SIGKILL) + + +def trip(args, battery, wire, detail): + evidence = (f"TRIPWIRE {wire}: {detail}\n" + f" stream : {args.stream}\n" + f" tokens : {battery.total():,} total" + f" / {battery.total_out():,} output\n") + print(evidence, file=sys.stderr, end="") + if args.tripped_file: + Path(args.tripped_file).write_text( + f"# TRIPPED — {wire}\n\n{evidence}\n" + f"Written by breaker.py; the stream tail around the trip is the" + f" evidence. Tune the wire's flag or --disable it if the" + f" pattern was legitimate.\n") + if args.terminate and args.pid: + terminate_tree(args.pid) + return EXIT_TRIPPED + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser( + description=__doc__.splitlines()[1].strip()) + parser.add_argument("stream") + parser.add_argument("--pid", type=int, default=None) + parser.add_argument("--terminate", action="store_true") + parser.add_argument("--once", action="store_true", + help="one pass over the existing stream, then exit") + parser.add_argument("--cap", type=int, default=0) + parser.add_argument("--cap-out", dest="cap_out", type=int, default=0) + parser.add_argument("--stall", type=float, default=900) + parser.add_argument("--size-mb", dest="size_mb", type=float, default=50) + parser.add_argument("--repeat-n", dest="repeat_n", type=int, default=5) + parser.add_argument("--repeat-k", dest="repeat_k", type=int, default=8) + parser.add_argument("--err-min", dest="err_min", type=int, default=12) + parser.add_argument("--window", type=int, default=40) + parser.add_argument("--interval", type=float, default=2.0) + parser.add_argument("--disable", default="") + parser.add_argument("--tripped-file", dest="tripped_file", default=None) + args = parser.parse_args(argv) + + battery = Battery(args) + stream = Path(args.stream) + pos = 0 + last_growth = time.time() + + def drain(): + nonlocal pos + grew = False + try: + with open(stream, errors="ignore") as handle: + handle.seek(pos) + for line in handle: + battery.feed(line) + grew = True + pos = handle.tell() + except FileNotFoundError: + pass + return grew + + while True: + if drain(): + last_growth = time.time() + fired = battery.check(stream) + if fired: + return trip(args, battery, *fired) + if args.once: + return 0 + if args.pid is not None and not alive(args.pid): + # The watched process ended: drain whatever landed last, give + # the wires one final look, and report clean. + drain() + fired = battery.check(stream) + return trip(args, battery, *fired) if fired else 0 + if ("stall" not in battery.disabled and args.pid is not None + and time.time() - last_growth > args.stall): + return trip(args, battery, "stall", + f"no stream growth for" + f" {int(time.time() - last_growth)} s" + f" (--stall {args.stall:g})") + time.sleep(args.interval) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ops/devlane/telemetry/pulse.py b/ops/devlane/telemetry/pulse.py new file mode 100644 index 0000000..ee0e7f3 --- /dev/null +++ b/ops/devlane/telemetry/pulse.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +"""Live status of running harness sessions, from their own streams. + + pulse.py [--json] [--live-window S] [--now TS] [--repo PATH] + [--tail N] [--claude-dir D] [--codex-dir D] [--grok-dir D] + +usage.py answers after the fact and breaker.py trips on failure; pulse +answers "what is running right now, and what is it doing" — the question +this repo kept assembling by hand from ls/tail/cat. One compact line per +live session: identity, age, idle time, spend so far, and the recent +tool/event NAMES. Never content: no prompt text, no tool inputs, no +results leave the stores through this tool. + +Time is a variable the caller controls: ``--now`` injects the clock and +makes the output a pure function of the stores; the real clock is read +in exactly one place, only when --now is absent. A session is live when +its stream file changed within --live-window seconds of now (boundary +inclusive). Grok sessions with turn_completed usage report the token +dict; pre-upgrade sessions without usage events say +``tokens=unrecorded`` — a stated gap, never a zero. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import time +from pathlib import Path +from urllib.parse import quote + +DEFAULT_WINDOW = 300 +DEFAULT_TAIL = 5 + + +def _tail(items, tail): + # recent[-0:] would be the WHOLE history; --tail 0 must mean none. + return items[-tail:] if tail > 0 else [] + + +def _epoch(stamp: str) -> float: + text = stamp.strip().replace("Z", "+00:00") + text = re.sub(r"\.(\d{6})\d+", r".\1", text) + from datetime import datetime + + return datetime.fromisoformat(text).timestamp() + + +def _read_jsonl(path: Path): + for raw in path.read_text(errors="replace").splitlines(): + raw = raw.strip() + if not raw: + continue + try: + yield json.loads(raw) + except ValueError: + continue + + +def _native_epoch(value) -> float: + """Live stores mix formats: Grok updates stamp epoch integers while + its events stamp ISO strings. Normalize both for one merge order.""" + if value is None: + return 0.0 + if isinstance(value, (int, float)): + return float(value) + try: + return _epoch(str(value)) + except ValueError: + return 0.0 + + +def _grok_usage_totals(events): + """Mirror of usage.py's accounting (measured 2026-08-21): usage is + cumulative within a run, runs split when a REPORTED totalTokens + shrinks, and the four token currencies' last-reported values per + run are summed across runs (reasoning and cost deliberately stay + out of pulse's closed row).""" + runs, current, prev_total = [], {}, None + for usage in events: + if "totalTokens" not in usage: + for key in ("inputTokens", "outputTokens", "totalTokens", + "cachedReadTokens", "cacheCreationTokens"): + if key in usage: + current[key] = usage[key] + continue + total = usage["totalTokens"] + if prev_total is not None and total < prev_total: + # Runs split on a cumulative shrink, never on numTurns — + # see usage.py's accounting note (skeptic-measured). + runs.append(current) + current = {} + prev_total = total + for key in ("inputTokens", "outputTokens", "totalTokens", + "cachedReadTokens", "cacheCreationTokens"): + if key in usage: + current[key] = usage[key] + runs.append(current) + totals = {key: sum(run.get(key, 0) for run in runs) + for key in ("inputTokens", "outputTokens", "totalTokens", + "cachedReadTokens", "cacheCreationTokens")} + return {"input": totals["inputTokens"], + "cached": (totals["cachedReadTokens"] + + totals["cacheCreationTokens"]), + "output": totals["outputTokens"], + "total": totals["totalTokens"]} + + +def _idle(path: Path, now: float) -> float: + """Fractional idle: the liveness comparison happens BEFORE integer + presentation truncation, so an mtime 300.8s old is dead for a 300s + window even though it prints as idle=300s.""" + return now - path.stat().st_mtime + + +def claude_rows(root: Path, repo, now, window, tail): + if not root.is_dir(): + return + for project in sorted(p for p in root.iterdir() if p.is_dir()): + if repo: + slug = "-" + "-".join(repo.strip("/").split("/")) + if project.name != slug: + continue + for stream in sorted(project.glob("*.jsonl")): + idle = _idle(stream, now) + if idle > window: + continue + started = model = None + per_msg = {} + recent = [] + cwds = set() + for entry in _read_jsonl(stream): + if entry.get("cwd"): + cwds.add(entry["cwd"]) + stamp = entry.get("timestamp") + if stamp and started is None: + started = _epoch(stamp) + message = entry.get("message") or {} + usage = message.get("usage") + mid = message.get("id") + if usage and mid: + model = message.get("model") or model + # Keyed by id: a re-emitted message must not + # double-count its spend. + per_msg[mid] = { + "input": usage.get("input_tokens") or 0, + "cached": (usage.get("cache_creation_input_tokens") or 0) + + (usage.get("cache_read_input_tokens") or 0), + "output": usage.get("output_tokens") or 0, + } + for block in message.get("content") or []: + if isinstance(block, dict) and block.get("type") == "tool_use": + recent.append(str(block.get("name"))) + if repo and repo not in cwds: + # Two paths can flatten to one slug; the cwd stored in the + # entries is the truth the directory name is not. + continue + tokens = None + if per_msg: + tokens = {key: sum(m[key] for m in per_msg.values()) + for key in ("input", "cached", "output")} + tokens["total"] = sum(tokens.values()) + yield {"harness": "claude", "session": stream.stem, + "model": model, + "age_seconds": int(now - started) if started else None, + "idle_seconds": int(idle), "tokens": tokens, + "recent": _tail(recent, tail)} + + +def codex_rows(root: Path, repo, now, window, tail): + sessions = root / "sessions" + if not sessions.is_dir(): + return + for stream in sorted(sessions.glob("*/*/*/rollout-*.jsonl")): + idle = _idle(stream, now) + if idle > window: + continue + meta, last_count, started, recent = {}, None, None, [] + for entry in _read_jsonl(stream): + stamp = entry.get("timestamp") + if stamp and started is None: + started = _epoch(stamp) + payload = entry.get("payload") or {} + if "cwd" in payload: + # Measured split: session_meta carries id/cwd, turn_context + # carries model/effort. Merge, never replace. + meta = {**meta, **payload} + kind = payload.get("name") or payload.get("type") + if kind: + recent.append(str(kind)) + if kind == "token_count": + info = payload.get("info") or {} + last_count = info.get("total_token_usage") or last_count + if repo and meta.get("cwd") != repo: + continue + tokens = None + if last_count: + tokens = {"input": last_count.get("input_tokens", 0), + "cached": last_count.get("cached_input_tokens", 0), + "output": last_count.get("output_tokens", 0), + "total": last_count.get("total_tokens", 0)} + yield {"harness": "codex", + "session": meta.get("id", stream.stem), "model": meta.get("model"), + "age_seconds": int(now - started) if started else None, + "idle_seconds": int(idle), "tokens": tokens, + "recent": _tail(recent, tail)} + + +def grok_rows(root: Path, repo, now, window, tail): + sessions = root / "sessions" + if not sessions.is_dir(): + return + for cwd_dir in sorted(p for p in sessions.iterdir() if p.is_dir()): + if repo and cwd_dir.name != quote(repo, safe=""): + continue + for sdir in sorted(p for p in cwd_dir.iterdir() if p.is_dir()): + streams = [sdir / name for name in ("updates.jsonl", "events.jsonl") + if (sdir / name).is_file()] + if not streams: + continue + idle = min(_idle(s, now) for s in streams) + if idle > window: + continue + summary = {} + summary_path = sdir / "summary.json" + if summary_path.is_file(): + try: + summary = json.loads(summary_path.read_text()) + except ValueError: + summary = {} + started = None + if summary.get("created_at"): + started = _epoch(summary["created_at"]) + activity = [] + usage_events = [] + updates_path = sdir / "updates.jsonl" + updates = (list(_read_jsonl(updates_path)) + if updates_path.is_file() else []) + for entry in updates: + update = (entry.get("params") or {}).get("update") or {} + if update.get("sessionUpdate") == "turn_completed": + # Spend records, not activity: they must not pollute + # the recent names. + if isinstance(update.get("usage"), dict): + usage_events.append(update["usage"]) + continue + if entry.get("method"): + activity.append((_native_epoch(entry.get("timestamp")), + str(entry["method"]))) + events_path = sdir / "events.jsonl" + for entry in (_read_jsonl(events_path) + if events_path.is_file() else ()): + name = entry.get("tool_name") or entry.get("type") + if name: + activity.append((_native_epoch(entry.get("ts")), + str(name))) + activity.sort(key=lambda pair: pair[0]) + yield {"harness": "grok", + "session": (summary.get("info") or {}).get("id", sdir.name), + "model": summary.get("current_model_id"), + "age_seconds": int(now - started) if started else None, + "idle_seconds": int(idle), + "tokens": (_grok_usage_totals(usage_events) + if usage_events else None), + "recent": _tail([name for _, name in activity], tail)} + + +def collect(args, now): + rows = [] + rows += list(claude_rows(Path(args.claude_dir), args.repo, now, + args.live_window, args.tail)) + rows += list(codex_rows(Path(args.codex_dir), args.repo, now, + args.live_window, args.tail)) + rows += list(grok_rows(Path(args.grok_dir), args.repo, now, + args.live_window, args.tail)) + rows.sort(key=lambda row: (row["harness"], str(row["session"]))) + return rows + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--json", action="store_true") + parser.add_argument("--live-window", dest="live_window", type=int, + default=DEFAULT_WINDOW) + parser.add_argument("--now", type=float, default=None, + help="injected clock (epoch seconds); the real " + "clock is read only when absent") + parser.add_argument("--repo", default=None) + parser.add_argument("--tail", type=int, default=DEFAULT_TAIL) + home = Path.home() + parser.add_argument("--claude-dir", default=str(home / ".claude" / "projects")) + parser.add_argument("--codex-dir", default=None, + help="default: $CODEX_HOME if set, else ~/.codex —" + " resolved per invocation, never at import") + parser.add_argument("--grok-dir", default=str(home / ".grok")) + args = parser.parse_args(argv) + if args.codex_dir is None: + args.codex_dir = os.environ.get("CODEX_HOME") or str(home / ".codex") + + now = args.now if args.now is not None else time.time() + rows = collect(args, now) + if args.json: + print(json.dumps({"sessions": rows}, sort_keys=True)) + return 0 + if not rows: + print("no live sessions") + return 0 + for row in rows: + tokens = row["tokens"] + spend = f"tokens={tokens['total']}" if tokens else "tokens=unrecorded" + print(f"{row['harness']:<7} {row['session']} {row['model']}" + f" age={row['age_seconds']}s idle={row['idle_seconds']}s" + f" {spend} recent={','.join(row['recent'])}") + return 0 + + +if __name__ == "__main__": + import sys + + sys.exit(main()) diff --git a/ops/devlane/telemetry/tests/test_bdd_breaker.py b/ops/devlane/telemetry/tests/test_bdd_breaker.py new file mode 100644 index 0000000..5f384e9 --- /dev/null +++ b/ops/devlane/telemetry/tests/test_bdd_breaker.py @@ -0,0 +1,131 @@ +"""BDD traceability cases not already proved by the breaker suites.""" + +import json +import unittest + +import test_breaker as breaker_contract +import test_breaker_grok as grok_contract + + +class BreakerScenarios(unittest.TestCase): + def breaker_fixture(self): + fixture = breaker_contract.TokenWalls( + "test_the_total_wall_trips_and_names_its_numbers" + ) + fixture.setUp() + self.addCleanup(fixture.doCleanups) + return fixture + + def grok_fixture(self): + fixture = grok_contract.GrokRecordSelection( + "test_only_completed_turns_with_usage_dicts_are_accounted" + ) + fixture.setUp() + self.addCleanup(fixture.doCleanups) + return fixture + + def test_output_wall_trips_while_total_wall_stays_under_cap(self): + """Scenario: the output-token wall trips independently of the total wall""" + fixture = self.breaker_fixture() + planted = breaker_contract.claude_line( + "output-heavy", out=600, inp=5, cached=0 + ) + fixture.stream.write_text(planted) + self.assertEqual( + fixture.stream.read_text(), + planted, + "the output-heavy stream plant did not land", + ) + + proc = fixture.run_once("--cap", "1000", "--cap-out", "500") + + self.assertEqual( + proc.returncode, + breaker_contract.EXIT_TRIPPED, + proc.stderr, + ) + # the TRIPWIRE line names the wall that fired; the evidence + # block below it always prints a "tokens :" summary, so the + # discriminator is the tripwire name, not that string + self.assertIn("TRIPWIRE tokens-out", proc.stderr) + self.assertNotIn("TRIPWIRE tokens:", proc.stderr) + + def test_omitted_zero_default_does_not_arm_the_token_wall(self): + """Scenario: a zero cap is a disarmed wall""" + fixture = self.breaker_fixture() + planted = breaker_contract.claude_line( + "would-trip", out=600, inp=5, cached=0 + ) + fixture.stream.write_text(planted) + self.assertEqual( + fixture.stream.read_text(), + planted, + "the over-cap stream plant did not land", + ) + + armed = fixture.run_once("--cap", "604") + disarmed = fixture.run_once() + + self.assertEqual( + armed.returncode, + breaker_contract.EXIT_TRIPPED, + "the control did not prove this stream can trip a nonzero cap", + ) + self.assertEqual(disarmed.returncode, 0, disarmed.stderr) + + def test_batch_and_non_numeric_records_do_not_charge_or_crash(self): + """Scenario: a malformed record neither kills the battery nor counts""" + fixture = self.grok_fixture() + batch = [{ + "method": "session/update", + "params": { + "update": { + "sessionUpdate": "turn_completed", + "usage": { + "totalTokens": 5000, + "outputTokens": 4000, + }, + } + }, + "timestamp": 1, + }] + lines = ( + json.dumps(batch) + "\n", + grok_contract.grok_line( + {"totalTokens": "9000", "outputTokens": None}, + timestamp=2, + ), + grok_contract.grok_line( + {"totalTokens": 120, "outputTokens": 30}, + timestamp=3, + ), + ) + self.assertTrue(lines[0].startswith("[")) + self.assertIn('"totalTokens": 5000', lines[0]) + self.assertIn('"totalTokens": "9000"', lines[1]) + self.assertIn('"outputTokens": null', lines[1]) + + observed = fixture.return_codes( + lines, + ( + ("--cap", "119"), + ("--cap", "120"), + ("--cap-out", "29"), + ("--cap-out", "30"), + ), + ) + + self.assertEqual( + observed, + ( + grok_contract.EXIT_TRIPPED, + 0, + grok_contract.EXIT_TRIPPED, + 0, + ), + "the walls did not answer solely for the numeric record", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/telemetry/tests/test_bdd_worth.py b/ops/devlane/telemetry/tests/test_bdd_worth.py new file mode 100644 index 0000000..cc727a1 --- /dev/null +++ b/ops/devlane/telemetry/tests/test_bdd_worth.py @@ -0,0 +1,156 @@ +"""BDD traceability cases joining worth behaviors covered separately elsewhere.""" + +import json +import unittest + +import test_worth as worth_contract + + +class WorthScenarios(unittest.TestCase): + def setUp(self): + self.fixture = worth_contract.WorthContractTests( + "test_default_window_boundaries_and_independent_edge_overrides" + ) + self.fixture.setUp() + self.addCleanup(self.fixture.doCleanups) + + def test_report_joins_window_cost_results_and_stamp(self): + """Scenario: the report joins spend and results for a window""" + w = self.fixture + base = worth_contract.iso_epoch("2026-08-22T10:00:00.000Z") + w._plant_standard_claude( + "joined-claude", + "2026-08-22T10:00:00.000Z", + ) + w._plant_standard_codex( + "joined-codex", + "2026-08-22T10:00:00.000Z", + ) + w._plant_grok( + "joined-grok", + "2026-08-22T10:00:00.000Z", + [[{ + "inputTokens": 70, + "cachedReadTokens": 10, + "cacheCreationTokens": 5, + "outputTokens": 15, + "totalTokens": 100, + "reasoningTokens": 2, + "costUsdTicks": 9, + }]], + ) + self.assertGreater(base, 0) + + w._git("switch", "-q", "-c", "pr-7") + w._commit_files( + {"src/joined.txt": "joined report fixture\n"}, + "implement joined report fixture", + "2026-08-22T10:30:00.000Z", + ) + w._git("switch", "-q", "dev") + merge = w._merge( + "pr-7", + "Merge pull request #7 from fixtures/pr-7", + "2026-08-22T10:40:00.000Z", + ) + parents = w._git("show", "-s", "--format=%P", merge).stdout.split() + self.assertEqual(len(parents), 2, "the numbered-merge plant did not land") + + since = "2026-08-22T09:00:00.000Z" + until = "2026-08-22T12:00:00.000Z" + now = "2026-08-22T13:00:00.000Z" + args = ("--since", since, "--until", until) + + plain = w._run_worth("report", *args, now=now) + self.assertEqual(plain.returncode, 0, plain.stderr) + data = w._json_worth("report", *args, now=now) + + expected_totals = { + "claude": 12_530, + "codex": 490, + "grok": 100, + } + for harness, total in expected_totals.items(): + record = w._harness_record(data, harness) + w._assert_key_number(record, ("total",), total) + line = w._line_for_harness(plain.stdout, harness) + w._assert_plain_number(line, ("total",), total) + + self.assertEqual(w._pr_numbers(data["results"]), {7}) + w._assert_stamp(plain.stdout, data, since, until, now) + + def test_waste_combines_ranking_ties_and_heavy_turn_identity(self): + """Scenario: waste ranks the window's sessions and names the heavy turn""" + w = self.fixture + w._plant_claude( + "leader", + [ + { + "id": "leader-small", + "timestamp": "2026-08-22T10:00:00.000Z", + "output": 10, + }, + { + "id": "leader-heavy", + "timestamp": "2026-08-22T10:01:00.000Z", + "output": 300, + }, + ], + ) + w._plant_claude( + "tie-a", + [{ + "id": "tie-a-message", + "timestamp": "2026-08-22T10:02:00.000Z", + "output": 200, + }], + ) + w._plant_grok( + "tie-b", + "2026-08-22T10:00:00.000Z", + [[{ + "inputTokens": 140, + "cachedReadTokens": 10, + "cacheCreationTokens": 0, + "outputTokens": 50, + "totalTokens": 200, + "reasoningTokens": 0, + "costUsdTicks": 1, + }]], + ) + + data = w._json_worth( + "waste", + "--since", + "2026-08-22T09:00:00.000Z", + "--until", + "2026-08-22T12:00:00.000Z", + "--top", + "3", + now="2026-08-22T13:00:00.000Z", + ) + + self.assertEqual( + w._session_ids(data), + ["leader", "tie-a", "tie-b"], + "sessions did not rank by window total and then id", + ) + heavy = [ + signal + for signal in w._signals_of_kind(data, "heavy-turn") + if ( + w._signal_field(signal, ("harness",)) == "claude" + and w._signal_field( + signal, + ("session", "session_id", "id"), + ) == "leader" + ) + ] + self.assertEqual(len(heavy), 1, data["signals"]) + rendered = json.dumps(heavy[0], sort_keys=True) + self.assertIn("leader-heavy", rendered) + self.assertIn("300", rendered) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/telemetry/tests/test_breaker.py b/ops/devlane/telemetry/tests/test_breaker.py new file mode 100644 index 0000000..7fdbe34 --- /dev/null +++ b/ops/devlane/telemetry/tests/test_breaker.py @@ -0,0 +1,235 @@ +"""The tripwire battery, against fixture streams in the real formats. + +Adapted from loopstrap's token-breaker.py (the live supervision layer; +usage.py is the after-the-fact layer). Every wire test proves both +directions: the wire fires on the planted condition AND stays quiet on the +clean stream — a wire that only fires proves nothing about its silence. +""" + +import json +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path + +BREAKER = Path(__file__).resolve().parents[1] / "breaker.py" +EXIT_TRIPPED = 3 + + +def claude_line(mid, out=10, inp=5, cached=100, content=None): + return json.dumps({"type": "assistant", "message": { + "id": mid, "model": "claude-fable-5", + "usage": {"input_tokens": inp, "cache_creation_input_tokens": 0, + "cache_read_input_tokens": cached, "output_tokens": out}, + "content": content or []}}) + "\n" + + +def tool_use(name, arg): + return {"type": "tool_use", "name": name, "input": {"cmd": arg}} + + +def tool_result(error=False, text="ok"): + return json.dumps({"type": "user", "message": {"content": [ + {"type": "tool_result", "is_error": error, "content": text}]}}) + "\n" + + +def codex_line(total, out): + return json.dumps({"type": "event_msg", "payload": { + "type": "token_count", "info": {"total_token_usage": { + "input_tokens": total - out, "cached_input_tokens": 0, + "output_tokens": out, "total_tokens": total}}}}) + "\n" + + +class BreakerCase(unittest.TestCase): + def setUp(self): + self.dir = Path(tempfile.mkdtemp(prefix="breaker-")) + self.addCleanup(__import__("shutil").rmtree, self.dir, True) + self.stream = self.dir / "stream.jsonl" + + def run_once(self, *args): + return subprocess.run( + [sys.executable, str(BREAKER), str(self.stream), "--once", *args], + capture_output=True, text=True, check=False) + + +class TokenWalls(BreakerCase): + def test_the_total_wall_trips_and_names_its_numbers(self): + self.stream.write_text(claude_line("m1", out=50, cached=400) + + claude_line("m2", out=50, cached=400)) + proc = self.run_once("--cap", "500") + self.assertEqual(proc.returncode, EXIT_TRIPPED, proc.stderr) + self.assertIn("tokens", proc.stderr) + self.assertIn("500", proc.stderr, "the cap belongs in the evidence") + + def test_the_wall_stays_quiet_under_the_cap(self): + self.stream.write_text(claude_line("m1")) + self.assertEqual(self.run_once("--cap", "500").returncode, 0) + + def test_the_output_wall_is_its_own_currency(self): + self.stream.write_text(claude_line("m1", out=600, cached=0)) + proc = self.run_once("--cap-out", "500") + self.assertEqual(proc.returncode, EXIT_TRIPPED) + self.assertIn("tokens-out", proc.stderr) + + def test_a_rewritten_message_id_is_counted_once(self): + # Snapshot overwrites re-emit the same message id; summing both + # copies would double the spend (loopstrap's accounting guard). + self.stream.write_text(claude_line("m1", out=300, cached=0) + + claude_line("m1", out=300, cached=0)) + self.assertEqual(self.run_once("--cap", "400").returncode, 0, + "the same message id was double-counted") + + def test_codex_cumulative_counts_are_not_summed(self): + self.stream.write_text(codex_line(300, 30) + codex_line(450, 60)) + self.assertEqual(self.run_once("--cap", "500").returncode, 0, + "cumulative token_count events were summed") + proc = self.run_once("--cap", "440") + self.assertEqual(proc.returncode, EXIT_TRIPPED) + + +class Storms(BreakerCase): + def test_a_repeat_loop_trips(self): + lines = "".join( + json.dumps({"type": "assistant", "message": { + "id": f"m{i}", "usage": {"output_tokens": 1}, + "content": [tool_use("Bash", "same-cmd")]}}) + "\n" + for i in range(6)) + self.stream.write_text(lines) + proc = self.run_once("--repeat-n", "5", "--repeat-k", "8") + self.assertEqual(proc.returncode, EXIT_TRIPPED, proc.stderr) + self.assertIn("repeat-loop", proc.stderr) + + def test_varied_calls_stay_quiet(self): + lines = "".join( + json.dumps({"type": "assistant", "message": { + "id": f"m{i}", "usage": {"output_tokens": 1}, + "content": [tool_use("Bash", f"cmd-{i}")]}}) + "\n" + for i in range(8)) + self.stream.write_text(lines) + self.assertEqual( + self.run_once("--repeat-n", "5", "--repeat-k", "8").returncode, 0) + + def test_an_error_storm_trips_and_a_healthy_mix_does_not(self): + noisy = "".join(tool_result(error=True, text="boom") for _ in range(12)) + self.stream.write_text(noisy) + proc = self.run_once("--err-min", "10", "--window", "12") + self.assertEqual(proc.returncode, EXIT_TRIPPED) + self.assertIn("error-storm", proc.stderr) + self.stream.write_text(tool_result(error=True) + tool_result() * 20) + self.assertEqual( + self.run_once("--err-min", "10", "--window", "12").returncode, 0) + + +class SizeWireIsVendorAgnostic(BreakerCase): + def test_any_format_trips_on_size(self): + # Grok records no tokens; bytes are the wire that still works. + self.stream.write_text('{"noise": "' + "x" * 2_000_000 + '"}\n') + proc = self.run_once("--size-mb", "1") + self.assertEqual(proc.returncode, EXIT_TRIPPED) + self.assertIn("size", proc.stderr) + + +class LiveSupervision(BreakerCase): + def spawn(self, *args): + return subprocess.Popen( + [sys.executable, str(BREAKER), str(self.stream), *args], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + def test_stall_trips_while_the_watched_process_lives(self): + self.stream.write_text(claude_line("m1")) + sleeper = subprocess.Popen([sys.executable, "-c", + "import time; time.sleep(60)"]) + self.addCleanup(sleeper.kill) + proc = self.spawn("--pid", str(sleeper.pid), + "--stall", "1", "--interval", "0.2") + try: + _, err = proc.communicate(timeout=15) + except subprocess.TimeoutExpired: + proc.kill() + self.fail("the breaker never tripped on a stalled stream") + self.assertEqual(proc.returncode, EXIT_TRIPPED, err) + self.assertIn("stall", err) + + def test_a_dead_process_drains_and_exits_clean(self): + """Scenario: a dead reviewer ends supervision without a trip""" + self.stream.write_text(claude_line("m1")) + dead = subprocess.Popen([sys.executable, "-c", "pass"]) + dead.wait() + proc = self.spawn("--pid", str(dead.pid), + "--stall", "30", "--interval", "0.2") + _, err = proc.communicate(timeout=15) + self.assertEqual(proc.returncode, 0, err) + + def test_terminate_takes_the_process_down_on_a_trip(self): + self.stream.write_text(claude_line("m1", out=900, cached=0)) + runaway = subprocess.Popen([sys.executable, "-c", + "import time; time.sleep(60)"]) + self.addCleanup(runaway.kill) + proc = self.spawn("--pid", str(runaway.pid), "--terminate", + "--cap", "100", "--interval", "0.2") + _, err = proc.communicate(timeout=15) + self.assertEqual(proc.returncode, EXIT_TRIPPED, err) + deadline = time.time() + 10 + while time.time() < deadline and runaway.poll() is None: + time.sleep(0.2) + self.assertIsNotNone(runaway.poll(), + "--terminate must actually stop the runaway") + + +class TheTripLeavesEvidence(BreakerCase): + def test_the_tripped_file_carries_wire_and_numbers(self): + """Scenario: the token wall trips a reviewer that spent past its cap""" + self.stream.write_text(claude_line("m1", out=900, cached=0)) + flag = self.dir / "TRIPPED.md" + proc = self.run_once("--cap", "100", "--tripped-file", str(flag)) + self.assertEqual(proc.returncode, EXIT_TRIPPED) + text = flag.read_text() + # the heading names the wall that FIRED; the evidence block + # below always mentions "tokens", so the discriminator is + # the heading (Grok, PR #33 review — an unnamed-trip mutant + # survived the substring) + # newline-terminated: the "tokens" prefix must not accept a + # "tokens-out" heading (Grok, PR #33 delta round 2) + self.assertIn("# TRIPPED \u2014 tokens\n", text) + self.assertIn("TRIPWIRE tokens:", text) + self.assertIn("100", text) + self.assertIn(str(self.stream), text) + + +class GrownReemitIsCountedLastWins(unittest.TestCase): + """A snapshot rewrite can re-emit a message id with GROWN usage. + First-wins keeps the stale copy and under-counts; summing keeps + both and over-counts; only last-wins reads the stream honestly.""" + + def setUp(self): + self.dir = tempfile.TemporaryDirectory() + self.addCleanup(self.dir.cleanup) + self.stream = Path(self.dir.name) / "stream.jsonl" + original = claude_line("m-grow", out=40, inp=10, cached=50) + grown = claude_line("m-grow", out=60, inp=20, cached=70) + self.assertNotEqual(original, grown, + "the growth plant did not change the line") + self.stream.write_text(original + grown) + self.grown_total = 60 + 20 + 70 + + def run_once(self, cap): + return subprocess.run( + [sys.executable, str(BREAKER), str(self.stream), "--once", + "--cap", str(cap)], + capture_output=True, text=True, check=False) + + def test_trips_just_below_the_grown_spend(self): + self.assertEqual(self.run_once(self.grown_total - 1).returncode, + EXIT_TRIPPED, + "first-wins kept the stale copy and missed the trip") + + def test_quiet_at_the_grown_spend(self): + self.assertEqual(self.run_once(self.grown_total).returncode, 0, + "summing both copies of one id tripped a cap the" + " real spend never reached") + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/telemetry/tests/test_breaker_grok.py b/ops/devlane/telemetry/tests/test_breaker_grok.py new file mode 100644 index 0000000..1108255 --- /dev/null +++ b/ops/devlane/telemetry/tests/test_breaker_grok.py @@ -0,0 +1,376 @@ +"""Grok usage feeds the breaker's total and output token walls.""" + +import ast +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +BREAKER = Path(__file__).resolve().parents[1] / "breaker.py" +EXIT_TRIPPED = 3 + + +def claude_line(mid, out, inp, cached): + return json.dumps( + { + "type": "assistant", + "message": { + "id": mid, + "model": "claude-fable-5", + "usage": { + "input_tokens": inp, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": cached, + "output_tokens": out, + }, + "content": [], + }, + } + ) + "\n" + + +def codex_line(total, out): + return json.dumps( + { + "type": "event_msg", + "payload": { + "type": "token_count", + "info": { + "total_token_usage": { + "input_tokens": total - out, + "cached_input_tokens": 0, + "output_tokens": out, + "total_tokens": total, + } + }, + }, + } + ) + "\n" + + +def grok_line( + usage=None, *, session_update="turn_completed", timestamp=1 +): + update = {"sessionUpdate": session_update} + if usage is not None: + update["usage"] = usage + return json.dumps( + { + "method": "session/update", + "params": {"update": update}, + "timestamp": timestamp, + } + ) + "\n" + + +def grok_reset_lines(): + """Two cumulative runs: 100 + 40 total and 14 + 9 output.""" + return ( + grok_line( + { + "inputTokens": 80, + "outputTokens": 10, + "totalTokens": 100, + "reasoningTokens": 900_000, + "costUsdTicks": 700_000, + }, + timestamp=1, + ), + grok_line( + { + "outputTokens": 14, + "reasoningTokens": 1_000_000, + "costUsdTicks": 800_000, + }, + timestamp=2, + ), + grok_line( + {"inputTokens": 86, "totalTokens": 100}, + timestamp=3, + ), + grok_line( + { + "inputTokens": 30, + "outputTokens": 5, + "totalTokens": 40, + "reasoningTokens": 2_000_000, + "costUsdTicks": 900_000, + }, + timestamp=4, + ), + grok_line( + {"outputTokens": 9, "costUsdTicks": 950_000}, + timestamp=5, + ), + grok_line( + {"inputTokens": 31, "totalTokens": 40}, + timestamp=6, + ), + ) + + +class BreakerGrokCase(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.TemporaryDirectory(prefix="breaker-grok-") + self.addCleanup(self.tempdir.cleanup) + self.stream = Path(self.tempdir.name) / "updates.jsonl" + + def write_stream(self, lines): + planted = "".join(lines) + self.stream.write_text(planted) + self.assertEqual( + self.stream.read_text(), + planted, + "the Grok stream plant did not land byte-for-byte", + ) + + def run_once(self, *args): + return subprocess.run( + [sys.executable, str(BREAKER), str(self.stream), "--once", *args], + capture_output=True, + text=True, + check=False, + ) + + def return_codes(self, lines, cases): + self.write_stream(lines) + return tuple(self.run_once(*args).returncode for args in cases) + + +class GrokRecordSelection(BreakerGrokCase): + def test_only_completed_turns_with_usage_dicts_are_accounted(self): + lines = ( + grok_line( + {"totalTokens": 5_000, "outputTokens": 4_000}, + session_update="tool_completed", + timestamp=1, + ), + grok_line(session_update="turn_cancelled", timestamp=2), + grok_line([{"totalTokens": 6_000}], timestamp=3), + grok_line( + {"totalTokens": 120, "outputTokens": 30}, + timestamp=4, + ), + ) + observed = self.return_codes( + lines, + ( + ("--cap", "119"), + ("--cap", "120"), + ("--cap-out", "29"), + ("--cap-out", "30"), + ), + ) + self.assertEqual(observed, (EXIT_TRIPPED, 0, EXIT_TRIPPED, 0)) + + +class GrokMalformedRecords(BreakerGrokCase): + def test_list_params_neither_crash_nor_count(self): + lines = ( + json.dumps({"method": "x", "params": [1, 2], + "timestamp": 1}) + "\n", + grok_line({"totalTokens": 120, "outputTokens": 30}, + timestamp=2), + ) + self.assertIn('"params": [1, 2]', lines[0]) + observed = self.return_codes( + lines, (("--cap", "119"), ("--cap", "120"))) + self.assertEqual(observed, (EXIT_TRIPPED, 0)) + + def test_non_object_json_lines_neither_crash_nor_count(self): + # the batch CARRIES real usage: a feed() that unwraps or + # recursively feeds list items would count the 5000 and trip + # the 120 wall — ignoring the whole non-dict line does not + batch = [{"params": {"update": { + "sessionUpdate": "turn_completed", + "usage": {"totalTokens": 5_000, "outputTokens": 4_000}}}}] + lines = ( + json.dumps(batch) + "\n", + json.dumps(42) + "\n", + json.dumps(None) + "\n", + grok_line({"totalTokens": 120, "outputTokens": 30}, + timestamp=2), + ) + self.assertTrue(lines[0].startswith("[")) + self.assertIn('"totalTokens": 5000', lines[0]) + self.assertEqual(lines[1].strip(), "42") + observed = self.return_codes( + lines, (("--cap", "119"), ("--cap", "120"))) + self.assertEqual(observed, (EXIT_TRIPPED, 0)) + + def test_skip_is_distinguished_from_coerce_and_zero(self): + # last-report-wins can mask a coercion: "9999" then 120 banks + # a phantom split if the string is coerced (9999 -> shrink), + # and treating null as 0 ERASES a prior numeric out. Both + # wrong accountings move a wall; skip does not. + lines = ( + grok_line({"totalTokens": "9999", "outputTokens": 5}, + timestamp=1), + grok_line({"totalTokens": 120, "outputTokens": 30}, + timestamp=2), + grok_line({"totalTokens": 130, "outputTokens": None}, + timestamp=3), + ) + self.assertIn('"totalTokens": "9999"', lines[0]) + self.assertIn('"outputTokens": null', lines[2]) + observed = self.return_codes( + lines, + (("--cap", "129"), ("--cap", "130"), + ("--cap-out", "29"), ("--cap-out", "30")), + ) + # skip: one run, total 130, out 30. coerce would bank 9999+ + # (tripping 130); zeroing null would erase out 30 (quiet 29). + self.assertEqual(observed, (EXIT_TRIPPED, 0, EXIT_TRIPPED, 0)) + + def test_non_numeric_counters_neither_crash_nor_count(self): + lines = ( + grok_line({"totalTokens": "100", "outputTokens": 5}, + timestamp=1), + grok_line({"totalTokens": 100, "outputTokens": None}, + timestamp=2), + grok_line({"totalTokens": 120, "outputTokens": 30}, + timestamp=3), + ) + self.assertIn('"totalTokens": "100"', lines[0]) + self.assertIn('"outputTokens": null', lines[1]) + # the string total is not a report, so no reset splits; the + # null output never enters the sum; the walls answer for the + # numeric report alone + observed = self.return_codes( + lines, + (("--cap", "119"), ("--cap", "120"), + ("--cap-out", "29"), ("--cap-out", "30")), + ) + self.assertEqual(observed, (EXIT_TRIPPED, 0, EXIT_TRIPPED, 0)) + + """Adversarial-run survivors, pinned: malformed shapes must count + nothing and crash nothing.""" + + def test_string_usage_and_null_params_neither_crash_nor_count(self): + lines = ( + json.dumps({"method": "session/update", "params": None, + "timestamp": 1}) + "\n", + grok_line('{"totalTokens": 9000}', timestamp=2), + grok_line({"totalTokens": 120, "outputTokens": 30}, + timestamp=3), + ) + self.assertIn('"params": null', lines[0], + "the null-params plant did not land") + self.assertIn('"usage": "{', lines[1], + "the string-usage plant did not land") + observed = self.return_codes( + lines, (("--cap", "119"), ("--cap", "120"))) + self.assertEqual(observed, (EXIT_TRIPPED, 0), + "malformed records crashed or counted") + + def test_after_a_reset_unreported_currencies_start_from_nothing(self): + lines = ( + grok_line({"totalTokens": 100, "outputTokens": 14}, + timestamp=1), + grok_line({"totalTokens": 40}, timestamp=2), + ) + observed = self.return_codes( + lines, (("--cap-out", "13"), ("--cap-out", "14"))) + self.assertEqual( + observed, (EXIT_TRIPPED, 0), + "the old run's outputTokens leaked into the new run —" + " current state was not cleared at the bank") + + +class GrokCumulativeRuns(BreakerGrokCase): + def test_total_wall_banks_finished_and_current_runs(self): + """Scenario: grok usage feeds the walls across a run reset""" + observed = self.return_codes( + grok_reset_lines(), + (("--cap", "139"), ("--cap", "140")), + ) + self.assertEqual(observed, (EXIT_TRIPPED, 0)) + + def test_output_wall_banks_last_report_per_run(self): + observed = self.return_codes( + grok_reset_lines(), + (("--cap-out", "22"), ("--cap-out", "23")), + ) + self.assertEqual(observed, (EXIT_TRIPPED, 0)) + + def test_trip_evidence_includes_grok_total_and_output(self): + self.write_stream(grok_reset_lines()) + proc = self.run_once("--cap", "139") + expected_totals = "tokens : 140 total / 23 output" + self.assertEqual( + (proc.returncode, expected_totals in proc.stderr), + (EXIT_TRIPPED, True), + proc.stderr, + ) + + +class ExistingAccountingStaysPinned(BreakerGrokCase): + def test_non_grok_totals_stay_pinned_and_mixed_adds_each_once(self): + claude = ( + claude_line("m1", out=5, inp=5, cached=5), + claude_line("m1", out=13, inp=17, cached=20), + ) + codex = (codex_line(40, 11), codex_line(70, 19)) + grok = ( + grok_line({"totalTokens": 15, "outputTokens": 3}, timestamp=1), + grok_line({"totalTokens": 30, "outputTokens": 7}, timestamp=2), + ) + currency_cases = ( + ("--cap", "49"), + ("--cap", "50"), + ("--cap-out", "12"), + ("--cap-out", "13"), + ) + claude_observed = self.return_codes(claude, currency_cases) + codex_observed = self.return_codes( + codex, + ( + ("--cap", "69"), + ("--cap", "70"), + ("--cap-out", "18"), + ("--cap-out", "19"), + ), + ) + mixed_observed = self.return_codes( + claude + codex + grok, + ( + ("--cap", "149"), + ("--cap", "150"), + ("--cap-out", "38"), + ("--cap-out", "39"), + ), + ) + self.assertEqual( + (claude_observed, codex_observed, mixed_observed), + ( + (EXIT_TRIPPED, 0, EXIT_TRIPPED, 0), + (EXIT_TRIPPED, 0, EXIT_TRIPPED, 0), + (EXIT_TRIPPED, 0, EXIT_TRIPPED, 0), + ), + ) + + +class GrokWireDocumentation(unittest.TestCase): + def test_token_wire_rows_name_grok(self): + docstring = ast.get_docstring(ast.parse(BREAKER.read_text())) + self.assertIsNotNone(docstring, "breaker.py lost its module docstring") + wire_rows = {} + for line in docstring.splitlines(): + fields = line.split() + if fields and fields[0] in {"tokens", "tokens-out"}: + wire_rows[fields[0]] = line + self.assertEqual(set(wire_rows), {"tokens", "tokens-out"}) + missing = [ + wire for wire, row in wire_rows.items() if "grok" not in row.lower() + ] + self.assertEqual( + missing, + [], + f"Grok is absent from these token wire rows: {missing}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/telemetry/tests/test_grok_usage_pulse.py b/ops/devlane/telemetry/tests/test_grok_usage_pulse.py new file mode 100644 index 0000000..93dd47e --- /dev/null +++ b/ops/devlane/telemetry/tests/test_grok_usage_pulse.py @@ -0,0 +1,400 @@ +"""Contracts for Grok usage in live pulse rows.""" + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from urllib.parse import quote + +HERE = Path(__file__).resolve() +PULSE = HERE.parents[1] / "pulse.py" + +BASE_EPOCH = 1_787_306_400 +NOW = BASE_EPOCH + 1_000 +REPO = "/home/work/projects/minspec/workbench" +MODEL = "grok-4.6" + +ROW_KEYS = { + "harness", + "session", + "model", + "age_seconds", + "idle_seconds", + "tokens", + "recent", +} +TOKEN_KEYS = {"input", "cached", "output", "total"} + +RESET_USAGE = [ + { + "inputTokens": 100, + "outputTokens": 20, + "totalTokens": 120, + "cachedReadTokens": 30, + "cacheCreationTokens": 5, + "reasoningTokens": 7, + "modelCalls": 1, + "apiDurationMs": 1_000, + "costUsdTicks": 1_100, + "numTurns": 1, + "modelUsage": { + MODEL: { + "inputTokens": 100, + "outputTokens": 20, + "totalTokens": 120, + "cachedReadTokens": 30, + "cacheCreationTokens": 5, + "reasoningTokens": 7, + "modelCalls": 1, + "apiDurationMs": 1_000, + "costUsdTicks": 1_100, + } + }, + }, + { + "inputTokens": 300, + "outputTokens": 80, + "totalTokens": 380, + "cachedReadTokens": 90, + "reasoningTokens": 40, + "modelCalls": 3, + "apiDurationMs": 3_500, + "costUsdTicks": 2_500, + "numTurns": 3, + "modelUsage": { + MODEL: { + "inputTokens": 300, + "outputTokens": 80, + "totalTokens": 380, + "cachedReadTokens": 90, + "reasoningTokens": 40, + "modelCalls": 3, + "apiDurationMs": 3_500, + "costUsdTicks": 2_500, + } + }, + }, + { + "inputTokens": 40, + "outputTokens": 10, + "totalTokens": 50, + "cachedReadTokens": 7, + "cacheCreationTokens": 2, + "reasoningTokens": 3, + "modelCalls": 1, + "apiDurationMs": 500, + "numTurns": 1, + "usageIsIncomplete": True, + "modelUsage": { + MODEL: { + "inputTokens": 40, + "outputTokens": 10, + "totalTokens": 50, + "cachedReadTokens": 7, + "cacheCreationTokens": 2, + "reasoningTokens": 3, + "modelCalls": 1, + "apiDurationMs": 500, + } + }, + }, +] + + +def write_jsonl(path, entries): + path.write_text( + "".join(json.dumps(entry) + "\n" for entry in entries) + ) + + +class GrokUsagePulse(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory( + prefix="grok-usage-pulse-" + ) + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.grok = self.root / "grok" + self.empty_claude = self.root / "empty-claude" + self.empty_codex = self.root / "empty-codex" + self.empty_claude.mkdir() + self.empty_codex.mkdir() + + def set_mtime(self, path, idle): + timestamp = NOW - idle + os.utime(path, (timestamp, timestamp)) + self.assertAlmostEqual( + path.stat().st_mtime, + timestamp, + places=3, + msg=f"the liveness mtime plant did not land on {path}", + ) + + def write_session( + self, + session_id, + usage, + *, + tools=(), + updates_idle=10.0, + events_idle=10.0, + ): + session = ( + self.grok + / "sessions" + / quote(REPO, safe="") + / session_id + ) + session.mkdir(parents=True) + (session / "summary.json").write_text( + json.dumps( + { + "info": {"id": session_id, "cwd": REPO}, + "created_at": "2026-08-21T10:00:00.000000000Z", + "updated_at": "2026-08-21T10:20:00.000000000Z", + "num_messages": 9, + "current_model_id": MODEL, + } + ) + ) + + updates = [ + { + "method": "session/update", + "params": { + "update": { + "sessionUpdate": "turn_completed", + "usage": value, + } + }, + "timestamp": BASE_EPOCH + 100 + index, + } + for index, value in enumerate(usage) + ] + events = [ + { + "type": "tool_started", + "tool_name": tool, + "ts": ( + "2026-08-21T10:00:" + f"{30 + index:02d}.000Z" + ), + } + for index, tool in enumerate(tools) + ] + + updates_path = session / "updates.jsonl" + events_path = session / "events.jsonl" + write_jsonl(updates_path, updates) + write_jsonl(events_path, events) + self.set_mtime(updates_path, updates_idle) + self.set_mtime(events_path, events_idle) + return session + + def run_pulse(self, *, json_output=True): + args = [ + sys.executable, + str(PULSE), + "--repo", + REPO, + "--now", + str(NOW), + "--live-window", + "300", + "--tail", + "10", + "--claude-dir", + str(self.empty_claude), + "--codex-dir", + str(self.empty_codex), + "--grok-dir", + str(self.grok), + ] + if json_output: + args.append("--json") + return subprocess.run( + args, + capture_output=True, + text=True, + check=False, + ) + + def json_rows(self): + proc = self.run_pulse() + self.assertEqual(proc.returncode, 0, proc.stderr) + try: + document = json.loads(proc.stdout) + except json.JSONDecodeError as error: + self.fail(f"{PULSE} did not emit one JSON document: {error}") + self.assertEqual(set(document), {"sessions"}) + return document["sessions"] + + def test_live_usage_uses_per_run_maxima_and_legacy_tokens_stay_none(self): + self.write_session( + "usage-reset", + RESET_USAGE, + tools=("search_code",), + ) + self.write_session( + "legacy-no-usage", + [], + tools=("read_file",), + ) + + rows = self.json_rows() + self.assertEqual( + len(rows), + 2, + "the two planted live Grok sessions were not reported", + ) + by_session = {row["session"]: row for row in rows} + self.assertEqual( + set(by_session), + {"legacy-no-usage", "usage-reset"}, + ) + + used = by_session["usage-reset"] + self.assertEqual( + used["tokens"], + { + "input": 340, + "cached": 104, + "output": 90, + "total": 430, + }, + "pulse did not sum the maxima from both Grok runs", + ) + self.assertEqual(set(used["tokens"]), TOKEN_KEYS) + self.assertEqual( + set(used), + ROW_KEYS, + "reasoning or cost escaped into the closed pulse row shape", + ) + + legacy = by_session["legacy-no-usage"] + self.assertIsNone( + legacy["tokens"], + "a Grok session without usage must remain an explicit gap", + ) + self.assertEqual(set(legacy), ROW_KEYS) + + plain = self.run_pulse(json_output=False) + self.assertEqual(plain.returncode, 0, plain.stderr) + usage_lines = [ + line + for line in plain.stdout.splitlines() + if "usage-reset" in line + ] + legacy_lines = [ + line + for line in plain.stdout.splitlines() + if "legacy-no-usage" in line + ] + self.assertEqual(len(usage_lines), 1) + self.assertEqual(len(legacy_lines), 1) + self.assertIn("tokens=430", usage_lines[0]) + self.assertNotIn("tokens=unrecorded", usage_lines[0]) + self.assertIn("tokens=unrecorded", legacy_lines[0]) + + def test_usage_updates_drive_liveness_but_never_enter_recent_names(self): + self.write_session( + "usage-is-not-activity", + RESET_USAGE[:2], + tools=("search_code",), + updates_idle=10.0, + events_idle=600.0, + ) + + rows = self.json_rows() + self.assertEqual( + len(rows), + 1, + ( + "a fresh usage-bearing updates stream must keep the " + "Grok session live" + ), + ) + row = rows[0] + self.assertEqual(row["session"], "usage-is-not-activity") + self.assertEqual( + row["idle_seconds"], + 10, + "liveness did not use the freshest Grok stream", + ) + self.assertEqual( + row["recent"], + ["search_code"], + ( + "turn_completed usage updates are accounting records, " + "not recent tool names" + ), + ) + self.assertNotIn("turn_completed", row["recent"]) + self.assertNotIn("session/update", row["recent"]) + self.assertEqual( + row["tokens"], + { + "input": 300, + "cached": 95, + "output": 80, + "total": 380, + }, + ) + self.assertEqual(set(row["tokens"]), TOKEN_KEYS) + self.assertEqual(set(row), ROW_KEYS) + + +class SkepticMirrorAccounting(GrokUsagePulse): + def test_pulse_mirror_pins_last_report_and_equal_totals(self): + self.write_session("mirror", [ + {"inputTokens": 40, "outputTokens": 10, "totalTokens": 50, + "numTurns": 1}, + {"inputTokens": 35, "outputTokens": 45, "totalTokens": 80, + "numTurns": 2}, + {"inputTokens": 36, "outputTokens": 44, "totalTokens": 80, + "numTurns": 2}, + ]) + rows = self.json_rows() + row = next(r for r in rows if r["session"] == "mirror") + self.assertEqual( + row["tokens"], + {"input": 36, "cached": 0, "output": 44, "total": 80}, + "pulse's mirror must keep equal totals in one run and let" + " the last report supersede the maximum", + ) + + +class SkepticCancelledTurns(GrokUsagePulse): + def test_a_cancelled_turn_without_usage_stays_out_of_recent(self): + session = self.write_session( + "cancelled", [], tools=("search_code",)) + updates_path = session / "updates.jsonl" + cancelled = { + "method": "session/update", + "params": {"update": { + "sessionUpdate": "turn_completed", + "prompt_id": "p-1", + "stop_reason": "cancelled", + }}, + "timestamp": BASE_EPOCH + 200, + } + with updates_path.open("a") as handle: + handle.write(json.dumps(cancelled) + "\n") + planted = updates_path.read_text() + self.assertIn('"stop_reason": "cancelled"', planted, + "the cancelled-turn plant did not land") + self.set_mtime(updates_path, 10.0) + rows = self.json_rows() + row = next(r for r in rows if r["session"] == "cancelled") + self.assertNotIn("session/update", row["recent"], + "a usage-less turn_completed polluted recent") + self.assertEqual(row["recent"], ["search_code"]) + self.assertIsNone(row["tokens"], + "a cancelled turn without usage invented tokens") + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/telemetry/tests/test_grok_usage_reader.py b/ops/devlane/telemetry/tests/test_grok_usage_reader.py new file mode 100644 index 0000000..d2ee2c4 --- /dev/null +++ b/ops/devlane/telemetry/tests/test_grok_usage_reader.py @@ -0,0 +1,546 @@ +"""Contracts for reading cumulative Grok usage from updates.jsonl.""" + +import importlib.util +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from urllib.parse import quote + +HERE = Path(__file__).resolve() +USAGE = HERE.parents[1] / "usage.py" + +BASE_EPOCH = 1_787_306_400 +REPO = "/home/work/projects/minspec/workbench" +MODEL = "grok-4.6" +GROK_GAP = "grok usage not yet parsed: updates.jsonl turn_completed" +MARKER = "GROKREADERCONTENTMARKER" + +RESET_USAGE = [ + { + "inputTokens": 100, + "outputTokens": 20, + "totalTokens": 120, + "cachedReadTokens": 30, + "cacheCreationTokens": 5, + "reasoningTokens": 7, + "modelCalls": 1, + "apiDurationMs": 1_000, + "costUsdTicks": 1_100, + "numTurns": 1, + "modelUsage": { + MODEL: { + "inputTokens": 100, + "outputTokens": 20, + "totalTokens": 120, + "cachedReadTokens": 30, + "cacheCreationTokens": 5, + "reasoningTokens": 7, + "modelCalls": 1, + "apiDurationMs": 1_000, + "costUsdTicks": 1_100, + } + }, + }, + { + "inputTokens": 300, + "outputTokens": 80, + "totalTokens": 380, + "cachedReadTokens": 90, + "reasoningTokens": 40, + "modelCalls": 3, + "apiDurationMs": 3_500, + "costUsdTicks": 2_500, + "numTurns": 3, + "modelUsage": { + MODEL: { + "inputTokens": 300, + "outputTokens": 80, + "totalTokens": 380, + "cachedReadTokens": 90, + "reasoningTokens": 40, + "modelCalls": 3, + "apiDurationMs": 3_500, + "costUsdTicks": 2_500, + } + }, + }, + { + "inputTokens": 40, + "outputTokens": 10, + "totalTokens": 50, + "cachedReadTokens": 7, + "cacheCreationTokens": 2, + "reasoningTokens": 3, + "modelCalls": 1, + "apiDurationMs": 500, + "numTurns": 1, + "usageIsIncomplete": True, + "modelUsage": { + MODEL: { + "inputTokens": 40, + "outputTokens": 10, + "totalTokens": 50, + "cachedReadTokens": 7, + "cacheCreationTokens": 2, + "reasoningTokens": 3, + "modelCalls": 1, + "apiDurationMs": 500, + } + }, + }, +] + +COMPLETE_A = [ + { + "inputTokens": 10, + "outputTokens": 2, + "totalTokens": 12, + "cachedReadTokens": 1, + "cacheCreationTokens": 1, + "reasoningTokens": 1, + "modelCalls": 1, + "apiDurationMs": 100, + "costUsdTicks": 100, + "numTurns": 1, + }, + { + "inputTokens": 30, + "outputTokens": 10, + "totalTokens": 40, + "cachedReadTokens": 4, + "cacheCreationTokens": 1, + "reasoningTokens": 2, + "modelCalls": 2, + "apiDurationMs": 300, + "costUsdTicks": 300, + "numTurns": 2, + }, + { + "inputTokens": 7, + "outputTokens": 3, + "totalTokens": 10, + "cachedReadTokens": 2, + "reasoningTokens": 1, + "modelCalls": 1, + "apiDurationMs": 80, + "costUsdTicks": 80, + "numTurns": 1, + }, +] + +COMPLETE_B = [ + { + "inputTokens": 5, + "outputTokens": 4, + "totalTokens": 9, + "cachedReadTokens": 1, + "cacheCreationTokens": 2, + "reasoningTokens": 1, + "modelCalls": 1, + "apiDurationMs": 50, + "costUsdTicks": 20, + "numTurns": 1, + } +] + + +def load_module(testcase): + testcase.assertTrue( + USAGE.is_file(), + f"{USAGE} is missing; the usage reader contract requires it", + ) + spec = importlib.util.spec_from_file_location( + "minspec_grok_usage_reader", + USAGE, + ) + testcase.assertIsNotNone(spec) + testcase.assertIsNotNone(spec.loader) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def write_jsonl(path, entries): + path.write_text( + "".join(json.dumps(entry) + "\n" for entry in entries) + ) + + +class GrokUsageReader(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory( + prefix="grok-usage-reader-" + ) + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.grok = self.root / "grok" + self.module = load_module(self) + + def write_session(self, session_id, usage): + session = ( + self.grok + / "sessions" + / quote(REPO, safe="") + / session_id + ) + session.mkdir(parents=True) + (session / "summary.json").write_text( + json.dumps( + { + "info": {"id": session_id, "cwd": REPO}, + "created_at": "2026-08-21T10:00:00.000000000Z", + "updated_at": "2026-08-21T10:20:00.000000000Z", + "num_messages": 9, + "current_model_id": MODEL, + "session_summary": f"Private content {MARKER}", + } + ) + ) + entries = [ + { + "method": "session/update", + "params": { + "update": { + "kind": "tool", + "detail": MARKER, + "usage": {"inputTokens": 999_999}, + } + }, + "timestamp": BASE_EPOCH + 20, + } + ] + entries.extend( + { + "method": "session/update", + "params": { + "update": { + "sessionUpdate": "turn_completed", + "usage": value, + } + }, + "timestamp": BASE_EPOCH + 100 + index, + } + for index, value in enumerate(usage) + ) + write_jsonl(session / "updates.jsonl", entries) + return session + + def rows(self): + return list(self.module.grok_sessions(self.grok, REPO)) + + def run_usage(self, *args): + return subprocess.run( + [ + sys.executable, + str(USAGE), + *args, + "--claude-dir", + str(self.root / "empty-claude"), + "--codex-dir", + str(self.root / "empty-codex"), + "--grok-dir", + str(self.grok), + "--repo", + REPO, + ], + capture_output=True, + text=True, + check=False, + ) + + def json_document(self, *args): + proc = self.run_usage(*args) + self.assertEqual(proc.returncode, 0, proc.stderr) + try: + return json.loads(proc.stdout) + except json.JSONDecodeError as error: + self.fail(f"{USAGE} did not emit one JSON document: {error}") + + def test_resets_bank_per_run_maxima_and_keep_the_legacy_gap(self): + self.write_session("usage-reset", RESET_USAGE) + self.write_session("legacy-no-usage", []) + + rows = self.rows() + self.assertEqual( + len(rows), + 2, + "the two planted Grok sessions were not both read", + ) + by_session = {row["session"]: row for row in rows} + self.assertEqual( + set(by_session), + {"legacy-no-usage", "usage-reset"}, + ) + + used = by_session["usage-reset"] + self.assertEqual( + used["tokens"], + { + "input": 340, + "cached": 104, + "output": 90, + "total": 430, + }, + ( + "Grok cumulative usage must sum each run's maxima, " + "not take the final event or sum every event" + ), + ) + self.assertEqual( + set(used["tokens"]), + {"input", "cached", "output", "total"}, + ) + self.assertEqual(used["reasoning"], 43) + self.assertIn("cost_usd_ticks", used) + self.assertIsNone( + used["cost_usd_ticks"], + "a missing cost currency is a gap, not zero or a partial sum", + ) + self.assertIn( + "incomplete", + used.get("note", "").lower(), + "usageIsIncomplete must remain visible to the caller", + ) + self.assertNotEqual(used.get("note"), GROK_GAP) + + legacy = by_session["legacy-no-usage"] + self.assertIsNone(legacy["tokens"]) + self.assertEqual( + legacy["note"], + GROK_GAP, + "pre-upgrade Grok sessions must retain the exact gap note", + ) + + def test_complete_cost_ticks_and_tokens_are_aggregated_by_the_report(self): + self.write_session("complete-a", COMPLETE_A) + self.write_session("complete-b", COMPLETE_B) + + sessions = self.json_document("sessions", "--json")["sessions"] + self.assertEqual(len(sessions), 2) + by_session = {row["session"]: row for row in sessions} + + self.assertEqual( + by_session["complete-a"]["tokens"], + { + "input": 37, + "cached": 7, + "output": 13, + "total": 50, + }, + ) + self.assertEqual( + by_session["complete-a"]["cost_usd_ticks"], + 380, + ) + self.assertEqual(by_session["complete-a"]["reasoning"], 3) + self.assertEqual( + by_session["complete-b"]["tokens"], + { + "input": 5, + "cached": 3, + "output": 4, + "total": 9, + }, + ) + self.assertEqual( + by_session["complete-b"]["cost_usd_ticks"], + 20, + ) + for row in by_session.values(): + self.assertNotEqual(row.get("note"), GROK_GAP) + if row.get("note") is not None: + self.assertIn("pars", row["note"].lower()) + + report = self.json_document("report", "--json") + aggregate = report["by_harness"]["grok"] + self.assertEqual(aggregate["sessions"], 2) + self.assertEqual(aggregate["counted"], 2) + self.assertEqual( + aggregate["tokens"], + { + "input": 42, + "cached": 10, + "output": 17, + "total": 59, + }, + ) + self.assertIn("cost_usd_ticks", aggregate) + self.assertEqual(aggregate["cost_usd_ticks"], 400) + + plain_report = self.run_usage("report") + self.assertEqual( + plain_report.returncode, + 0, + plain_report.stderr, + ) + self.assertNotIn("tokens=unrecorded", plain_report.stdout) + self.assertIn("total=59", plain_report.stdout) + self.assertIn("cost_usd_ticks=400", plain_report.stdout) + + plain_sessions = self.run_usage("sessions") + self.assertEqual( + plain_sessions.returncode, + 0, + plain_sessions.stderr, + ) + for session_id, ticks in ( + ("complete-a", 380), + ("complete-b", 20), + ): + lines = [ + line + for line in plain_sessions.stdout.splitlines() + if session_id in line + ] + self.assertEqual( + len(lines), + 1, + f"expected one plain row for {session_id}", + ) + self.assertIn("total=", lines[0]) + self.assertNotIn("unrecorded", lines[0]) + self.assertIn(f"cost_usd_ticks={ticks}", lines[0]) + + plain = ( + plain_report.stdout + "\n" + plain_sessions.stdout + ).lower() + if "$" in plain or "cost_usd=" in plain: + self.assertIn( + "inferred", + plain, + ( + "derived USD is permitted only when its scale is " + "explicitly marked inferred" + ), + ) + self.assertNotIn( + MARKER, + plain, + "session content leaked through the usage reader", + ) + + +class SkepticAccountingShapes(unittest.TestCase): + """Round-1 skeptic findings, pinned as units against the module + (live stream 019fb283: totals shrink while numTurns rises).""" + + def setUp(self): + self.module = load_module(self) + + def totals(self, events): + return self.module._grok_usage_totals(events) + + def test_a_shrink_without_a_turns_drop_starts_a_new_run(self): + events = [ + {"inputTokens": 100, "outputTokens": 20, "totalTokens": 120, + "costUsdTicks": 100, "numTurns": 12}, + {"inputTokens": 30, "outputTokens": 10, "totalTokens": 40, + "costUsdTicks": 30, "numTurns": 15}, + ] + self.assertLess(events[1]["totalTokens"], events[0]["totalTokens"]) + self.assertGreater(events[1]["numTurns"], events[0]["numTurns"]) + tokens, _, cost, _ = self.totals(events) + self.assertEqual(tokens["total"], 160, + "a totals shrink with rising turns must split runs") + self.assertEqual(tokens["input"], 130) + self.assertEqual(cost, 130) + + def test_within_a_run_the_last_report_wins_over_the_maximum(self): + events = [ + {"inputTokens": 40, "outputTokens": 10, "totalTokens": 50, + "costUsdTicks": 50, "numTurns": 1}, + {"inputTokens": 35, "outputTokens": 45, "totalTokens": 80, + "costUsdTicks": 80, "numTurns": 2}, + ] + self.assertLess(events[1]["inputTokens"], events[0]["inputTokens"]) + tokens, _, cost, _ = self.totals(events) + self.assertEqual(tokens["input"], 35, + "max-merge kept a superseded cumulative report") + self.assertEqual(tokens["total"], 80) + self.assertEqual(cost, 80) + + def test_equal_totals_stay_in_one_run(self): + events = [ + {"inputTokens": 60, "outputTokens": 20, "totalTokens": 80, + "costUsdTicks": 80, "numTurns": 1}, + {"inputTokens": 60, "outputTokens": 20, "totalTokens": 80, + "costUsdTicks": 90, "numTurns": 2}, + ] + self.assertEqual(events[0]["totalTokens"], events[1]["totalTokens"]) + tokens, _, cost, _ = self.totals(events) + self.assertEqual(tokens["total"], 80, + "equal cumulative totals split a run that never" + " reset (a <= split double-counts)") + self.assertEqual(cost, 90) + + def test_equal_turns_stay_in_one_run(self): + events = [ + {"inputTokens": 40, "outputTokens": 10, "totalTokens": 50, + "cachedReadTokens": 5, "cacheCreationTokens": 2, + "costUsdTicks": 50, "numTurns": 1}, + {"inputTokens": 60, "outputTokens": 20, "totalTokens": 80, + "cachedReadTokens": 9, "costUsdTicks": 80, "numTurns": 1}, + ] + tokens, _, cost, _ = self.totals(events) + self.assertEqual(tokens["total"], 80, + "equal turns split a run that never reset") + self.assertEqual(tokens["cached"], 11, + "an omitted currency erased the run's report") + self.assertEqual(cost, 80) + + +class SkepticReportShapes(GrokUsageReader): + def test_one_costless_session_makes_the_aggregate_cost_a_gap(self): + self.write_session("with-cost", COMPLETE_B) + self.write_session("without-cost", [ + {"inputTokens": 7, "outputTokens": 3, "totalTokens": 10, + "numTurns": 1}, + ]) + report = self.json_document("report", "--json") + aggregate = report["by_harness"]["grok"] + self.assertEqual(aggregate["counted"], 2) + self.assertIn("cost_usd_ticks", aggregate) + self.assertIsNone( + aggregate["cost_usd_ticks"], + "a partial cost sum was passed off as the aggregate") + plain = self.run_usage("report") + self.assertIn("cost_usd_ticks=unrecorded", plain.stdout, + "an unknown aggregate cost must say so explicitly") + self.assertNotRegex(plain.stdout, r"cost_usd_ticks=\d", + "the plain report printed a partial cost sum") + + def test_incompleteness_propagates_into_every_report_format(self): + self.write_session("incomplete-run", RESET_USAGE) + self.write_session("complete-run", COMPLETE_B) + rows = {r["session"]: r for r in + self.json_document("sessions", "--json")["sessions"]} + self.assertIs(rows["incomplete-run"]["incomplete"], True, + "usageIsIncomplete must be a structured row flag") + self.assertIs(rows["complete-run"]["incomplete"], False) + report = self.json_document("report", "--json") + aggregate = report["by_harness"]["grok"] + self.assertEqual( + aggregate.get("incomplete_sessions"), 1, + "the report presented incomplete measurements as verified") + plain = self.run_usage("report") + self.assertIn("incomplete=1", plain.stdout, + "the plain report hid the incompleteness") + sessions_plain = self.run_usage("sessions") + # the plain row truncates ids to 12 chars + line = next(l for l in sessions_plain.stdout.splitlines() + if "incomplete-r" in l) + self.assertIn("(incomplete)", line) + self.assertIn("cost_usd_ticks=unrecorded", line) + + def test_parsed_rows_never_carry_session_content(self): + self.write_session("leaky", COMPLETE_B) + document = self.json_document("sessions", "--json") + self.assertNotIn(MARKER, json.dumps(document), + "session content leaked into parsed grok rows") + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/telemetry/tests/test_pulse.py b/ops/devlane/telemetry/tests/test_pulse.py new file mode 100644 index 0000000..fbbad18 --- /dev/null +++ b/ops/devlane/telemetry/tests/test_pulse.py @@ -0,0 +1,1123 @@ +"""Guards against stale inclusion, token miscounting, and leaks in pulse.""" + +import importlib.util +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from urllib.parse import quote + +HERE = Path(__file__).resolve() +PULSE = HERE.parents[1] / "pulse.py" +STORES = HERE.parents[2] / "fixtures" / "stores.py" + +BASE_EPOCH = 1_787_306_400 +NOW = BASE_EPOCH + 600 +REPO = "/home/work/projects/minspec/workbench" +OTHER_REPO = "/tmp/other-minspec" +MARKER = "FIXTUREPROMPTMARKER" + +CLAUDE_MODEL = "claude-fable-5" +CODEX_MODEL = "gpt-5-codex" +GROK_MODEL = "grok-4.6" + +CLAUDE_TOKENS = { + "input": 30, + "cached": 12_000, + "output": 500, + "total": 12_530, +} +CLAUDE_REEMIT_GROWTH = { + "input_tokens": 7, + "cache_creation_input_tokens": 11, + "cache_read_input_tokens": 13, + "output_tokens": 17, +} +CLAUDE_GROWN_TOKENS = { + "input": 37, + "cached": 12_024, + "output": 517, + "total": 12_578, +} +CODEX_TOKENS = { + "input": 400, + "cached": 300, + "output": 90, + "total": 490, +} +PULSE_ROW_KEYS = { + "harness", + "session", + "model", + "age_seconds", + "idle_seconds", + "tokens", + "recent", +} +TOKEN_KEYS = {"input", "cached", "output", "total"} +GROK_RECENT = [ + "phase_changed", + "session/update", + "search_code", + "search_code", + "permission_requested", + "session/update", + "permission_resolved", + "loop_started", + "phase_changed", +] + + +def load_module(testcase, path, name): + testcase.assertTrue( + path.is_file(), + f"{path} is missing; live-session status has not been implemented", + ) + spec = importlib.util.spec_from_file_location(name, path) + testcase.assertIsNotNone( + spec, + f"{path} could not be given an import specification", + ) + testcase.assertIsNotNone( + spec.loader, + f"{path} has no loader and cannot be exercised", + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def jsonl(path): + return [ + json.loads(raw) + for raw in path.read_text().splitlines() + if raw.strip() + ] + + +def slug(repo): + return "-" + "-".join(repo.strip("/").split("/")) + + +class PulseCase(unittest.TestCase): + def setUp(self): + self.assertTrue( + PULSE.is_file(), + f"{PULSE} is missing; every pulse contract must remain red", + ) + self.stores = load_module( + self, + STORES, + "minspec_fixture_stores_for_pulse", + ) + self.temp = tempfile.TemporaryDirectory(prefix="pulse-contract-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.claude_root = self.root / "claude" + self.codex_root = self.root / "codex" + self.grok_root = self.root / "grok" + + def build_three(self, suffix, repo=REPO, reemit_claude=False): + claude_id = f"claude-{suffix}" + codex_id = f"codex-{suffix}" + grok_id = f"grok-{suffix}" + + self.stores.build_claude_store( + self.claude_root, + slug(repo), + base_timestamp=BASE_EPOCH, + cwd=repo, + session_id=claude_id, + model=CLAUDE_MODEL, + effort="high", + marker=MARKER, + reemit_last=reemit_claude, + ) + claude_stream = ( + self.claude_root / slug(repo) / f"{claude_id}.jsonl" + ) + self.assertTrue( + claude_stream.is_file(), + f"{STORES}:build_claude_store did not create {claude_stream}", + ) + + before = set( + self.codex_root.glob( + "sessions/2026/08/21/rollout-*.jsonl" + ) + ) + self.stores.build_codex_store( + self.codex_root, + base_timestamp=BASE_EPOCH, + cwd=repo, + session_id=codex_id, + model=CODEX_MODEL, + effort="high", + marker=MARKER, + ) + after = set( + self.codex_root.glob( + "sessions/2026/08/21/rollout-*.jsonl" + ) + ) + created = after - before + self.assertEqual( + len(created), + 1, + ( + f"{STORES}:build_codex_store created {len(created)} " + f"new rollout files for session {codex_id}" + ), + ) + codex_stream = next(iter(created)) + + self.stores.build_grok_store( + self.grok_root, + repo, + base_timestamp=BASE_EPOCH, + session_id=grok_id, + model=GROK_MODEL, + marker=MARKER, + ) + grok_session = ( + self.grok_root + / "sessions" + / quote(repo, safe="") + / grok_id + ) + grok_summary = grok_session / "summary.json" + grok_updates = grok_session / "updates.jsonl" + grok_events = grok_session / "events.jsonl" + for path in (grok_summary, grok_updates, grok_events): + self.assertTrue( + path.is_file(), + f"{STORES}:build_grok_store did not create {path}", + ) + + return { + "claude": { + "id": claude_id, + "model": CLAUDE_MODEL, + "stream": claude_stream, + "streams": [claude_stream], + "raw": [claude_stream], + }, + "codex": { + "id": codex_id, + "model": CODEX_MODEL, + "stream": codex_stream, + "streams": [codex_stream], + "raw": [codex_stream], + }, + "grok": { + "id": grok_id, + "model": GROK_MODEL, + "summary": grok_summary, + "updates": grok_updates, + "events": grok_events, + "streams": [grok_updates, grok_events], + "non_streams": [grok_summary], + "raw": [grok_summary, grok_updates, grok_events], + }, + } + + def set_activity(self, record, timestamp): + for path in record["streams"]: + os.utime(path, (timestamp, timestamp)) + self.assertEqual( + int(path.stat().st_mtime), + timestamp, + f"the planted activity mtime did not land on {path}", + ) + for path in record.get("non_streams", []): + old = BASE_EPOCH - 10_000 + os.utime(path, (old, old)) + self.assertEqual( + int(path.stat().st_mtime), + old, + f"the planted non-stream mtime did not land on {path}", + ) + + def set_all_activity(self, records, offsets): + for harness, offset in offsets.items(): + self.set_activity(records[harness], NOW - offset) + + def assert_marker_planted(self, records): + claude_raw = records["claude"]["stream"].read_text() + self.assertIn( + MARKER, + claude_raw, + "the leak marker was not planted in Claude content", + ) + + codex_entries = jsonl(records["codex"]["stream"]) + codex_meta = [ + entry["payload"] + for entry in codex_entries + if entry.get("type") == "session_meta" + ] + self.assertEqual( + len(codex_meta), + 1, + ( + f"the Codex base-instructions plant lacks one " + f"session_meta in {records['codex']['stream']}" + ), + ) + self.assertIn( + MARKER, + json.dumps( + codex_meta[0]["base_instructions"], + sort_keys=True, + ), + ( + f"the leak marker was not planted in Codex " + f"base_instructions at {records['codex']['stream']}" + ), + ) + + summary = json.loads(records["grok"]["summary"].read_text()) + for field in ("session_summary", "generated_title"): + self.assertIn( + MARKER, + summary[field], + ( + f"the leak marker was not planted in Grok {field} " + f"at {records['grok']['summary']}" + ), + ) + + updates = jsonl(records["grok"]["updates"]) + self.assertGreater( + len(updates), + 0, + "the Grok params plant found no updates", + ) + for index, update in enumerate(updates): + self.assertIn( + MARKER, + json.dumps(update["params"], sort_keys=True), + ( + f"the leak marker was not planted in Grok params " + f"{index} at {records['grok']['updates']}" + ), + ) + + def grow_last_claude_reemit(self, record): + stream = record["stream"] + entries = jsonl(stream) + self.assertGreater( + len(entries), + 2, + f"the grown re-emit plant found too few entries in {stream}", + ) + + reemit = entries[-1] + message = reemit.get("message") or {} + message_id = message.get("id") + matching = [ + entry + for entry in entries[:-1] + if (entry.get("message") or {}).get("id") == message_id + ] + self.assertEqual( + len(matching), + 1, + ( + f"the grown re-emit plant expected one earlier occurrence " + f"of {message_id!r} in {stream}" + ), + ) + original = matching[0] + self.assertEqual( + reemit, + original, + ( + f"the fixture's last Claude re-emit was not " + f"byte-equivalent before the growth plant in {stream}" + ), + ) + + original_usage = dict(original["message"]["usage"]) + self.assertEqual( + set(original_usage), + set(CLAUDE_REEMIT_GROWTH), + ( + f"the grown re-emit plant found unexpected usage " + f"currencies in {stream}" + ), + ) + grown_usage = { + key: value + CLAUDE_REEMIT_GROWTH[key] + for key, value in original_usage.items() + } + entries[-1]["message"]["usage"] = grown_usage + stream.write_text( + "".join( + json.dumps( + entry, + separators=(",", ":"), + sort_keys=True, + ) + + "\n" + for entry in entries + ) + ) + + planted = jsonl(stream) + self.assertEqual( + len(planted), + len(entries), + f"the grown re-emit plant changed the entry count in {stream}", + ) + self.assertEqual( + planted[-1]["sessionId"], + record["id"], + ( + f"the grown re-emit plant no longer resembles the " + f"intended fixture in {stream}" + ), + ) + self.assertEqual( + planted[-1]["message"]["id"], + message_id, + f"the grown re-emit plant changed the duplicate id in {stream}", + ) + self.assertEqual( + planted[-1]["message"]["usage"], + grown_usage, + f"the grown re-emit usage plant did not land in {stream}", + ) + for key, original_value in original_usage.items(): + self.assertGreater( + planted[-1]["message"]["usage"][key], + original_value, + ( + f"the grown re-emit plant did not increase {key} " + f"in {stream}" + ), + ) + + def assert_closed_json_shape(self, document): + self.assertIsInstance( + document, + dict, + f"{PULSE} --json did not emit an object", + ) + self.assertEqual( + set(document), + {"sessions"}, + f"{PULSE} --json emitted top-level fields beyond sessions", + ) + rows = document["sessions"] + self.assertIsInstance( + rows, + list, + f"{PULSE} --json sessions is not a list", + ) + + for index, row in enumerate(rows): + self.assertIsInstance( + row, + dict, + f"{PULSE} --json row {index} is not an object", + ) + self.assertEqual( + set(row), + PULSE_ROW_KEYS, + ( + f"{PULSE} --json row {index} does not have exactly " + f"the closed status fields" + ), + ) + for key in ("harness", "session", "model"): + self.assertIs( + type(row[key]), + str, + ( + f"{PULSE} --json row {index} field {key} is not " + "a string" + ), + ) + for key in ("age_seconds", "idle_seconds"): + self.assertIs( + type(row[key]), + int, + ( + f"{PULSE} --json row {index} field {key} is not " + "an integer" + ), + ) + + tokens = row["tokens"] + if tokens is not None: + self.assertIsInstance( + tokens, + dict, + f"{PULSE} --json row {index} tokens is not an object", + ) + self.assertEqual( + set(tokens), + TOKEN_KEYS, + ( + f"{PULSE} --json row {index} tokens has fields " + "outside the known token currencies" + ), + ) + for key, value in tokens.items(): + self.assertIs( + type(value), + int, + ( + f"{PULSE} --json row {index} token {key} is " + "not an integer" + ), + ) + + recent = row["recent"] + self.assertIsInstance( + recent, + list, + f"{PULSE} --json row {index} recent is not a list", + ) + for name in recent: + self.assertIs( + type(name), + str, + ( + f"{PULSE} --json row {index} recent contains " + "a nested value" + ), + ) + self.assertTrue( + name, + f"{PULSE} --json row {index} has an empty recent name", + ) + self.assertEqual( + name, + name.strip(), + ( + f"{PULSE} --json row {index} recent name has " + "surrounding whitespace" + ), + ) + self.assertLessEqual( + len(name.encode()), + 80, + ( + f"{PULSE} --json row {index} recent name is not " + f"short: {name!r}" + ), + ) + + def run_pulse(self, *args): + return subprocess.run( + [ + sys.executable, + str(PULSE), + *args, + "--now", + str(NOW), + "--claude-dir", + str(self.claude_root), + "--codex-dir", + str(self.codex_root), + "--grok-dir", + str(self.grok_root), + ], + capture_output=True, + text=True, + check=False, + ) + + +class LiveJsonStatus(PulseCase): + def test_json_is_stable_sorted_last_wins_and_content_free(self): + records = self.build_three("live", reemit_claude=True) + self.grow_last_claude_reemit(records["claude"]) + self.set_all_activity( + records, + {"claude": 10, "codex": 20, "grok": 30}, + ) + self.assert_marker_planted(records) + + claude_entries = jsonl(records["claude"]["stream"]) + message_ids = [ + entry["message"]["id"] + for entry in claude_entries + if (entry.get("message") or {}).get("usage") + ] + self.assertGreater( + len(message_ids), + len(set(message_ids)), + ( + f"the duplicate message-id plant did not land in " + f"{records['claude']['stream']}" + ), + ) + + naive_spend = 0 + keyed_spend = {} + for entry in claude_entries: + message = entry.get("message") or {} + usage = message.get("usage") + if not usage: + continue + spend = sum(usage.values()) + naive_spend += spend + keyed_spend[message["id"]] = spend + self.assertGreater( + naive_spend, + sum(keyed_spend.values()), + ( + f"the grown Claude re-emit in " + f"{records['claude']['stream']} cannot expose " + "double-counting" + ), + ) + self.assertEqual( + sum(keyed_spend.values()), + CLAUDE_GROWN_TOKENS["total"], + ( + f"the last-wins Claude spend plant in " + f"{records['claude']['stream']} is wrong" + ), + ) + self.assertNotEqual( + sum(keyed_spend.values()), + CLAUDE_TOKENS["total"], + ( + f"the grown Claude re-emit in " + f"{records['claude']['stream']} cannot expose first-wins" + ), + ) + + first = self.run_pulse( + "--json", + "--repo", + REPO, + "--tail", + "20", + ) + self.assertEqual( + first.returncode, + 0, + f"{PULSE} --json failed: {first.stderr}", + ) + second = self.run_pulse( + "--json", + "--repo", + REPO, + "--tail", + "20", + ) + self.assertEqual( + second.returncode, + 0, + f"{PULSE} --json was not repeatable: {second.stderr}", + ) + self.assertEqual( + second.stdout, + first.stdout, + f"{PULSE} --json changed for identical stores and --now", + ) + self.assertNotIn( + MARKER, + first.stdout + first.stderr, + f"{PULSE} --json leaked planted session content", + ) + + document = json.loads(first.stdout) + self.assert_closed_json_shape(document) + rows = document["sessions"] + self.assertEqual( + len(rows), + 3, + f"{PULSE} --json did not emit exactly three live sessions", + ) + self.assertEqual( + [(row["harness"], row["session"]) for row in rows], + sorted( + (row["harness"], row["session"]) for row in rows + ), + f"{PULSE} --json sessions are not sorted stably", + ) + + by_harness = {row["harness"]: row for row in rows} + self.assertEqual( + set(by_harness), + {"claude", "codex", "grok"}, + f"{PULSE} --json omitted or duplicated a harness", + ) + + claude = by_harness["claude"] + self.assertEqual( + claude["session"], + records["claude"]["id"], + f"{PULSE} reported the wrong Claude session id", + ) + self.assertEqual( + claude["model"], + CLAUDE_MODEL, + f"{PULSE} reported the wrong Claude model", + ) + self.assertEqual( + claude["age_seconds"], + 600, + f"{PULSE} computed Claude age from the wrong clock", + ) + self.assertEqual( + claude["idle_seconds"], + 10, + f"{PULSE} did not compute Claude activity from stream mtime", + ) + self.assertEqual( + claude["tokens"], + CLAUDE_GROWN_TOKENS, + ( + f"{PULSE} did not count the grown Claude re-emit " + "last-wins exactly once" + ), + ) + self.assertEqual( + claude["recent"], + ["Read", "Bash", "Bash"], + ( + f"{PULSE} returned Claude content instead of the " + "recent tool names" + ), + ) + + codex = by_harness["codex"] + self.assertEqual( + codex["session"], + records["codex"]["id"], + ( + f"{PULSE} did not obtain the Codex session id from " + "session_meta" + ), + ) + self.assertEqual( + codex["model"], + CODEX_MODEL, + ( + f"{PULSE} did not merge the Codex model from " + "turn_context" + ), + ) + self.assertEqual( + codex["age_seconds"], + 600, + f"{PULSE} computed Codex age from the wrong clock", + ) + self.assertEqual( + codex["idle_seconds"], + 20, + f"{PULSE} did not compute Codex activity from rollout mtime", + ) + self.assertEqual( + codex["tokens"], + CODEX_TOKENS, + f"{PULSE} summed cumulative Codex token_count events", + ) + self.assertEqual( + codex["recent"], + ["token_count", "shell_command", "token_count"], + ( + f"{PULSE} returned Codex content instead of the " + "recent event names" + ), + ) + + grok = by_harness["grok"] + self.assertEqual( + grok["session"], + records["grok"]["id"], + f"{PULSE} reported the wrong Grok session id", + ) + self.assertEqual( + grok["model"], + GROK_MODEL, + f"{PULSE} reported the wrong Grok model", + ) + self.assertEqual( + grok["age_seconds"], + 600, + f"{PULSE} computed Grok age from the wrong clock", + ) + self.assertEqual( + grok["idle_seconds"], + 30, + f"{PULSE} did not compute Grok activity from stream mtimes", + ) + self.assertIsNone( + grok["tokens"], + f"{PULSE} invented a Grok token measurement", + ) + self.assertEqual( + grok["recent"], + GROK_RECENT, + ( + f"{PULSE} did not merge Grok update methods and " + "event names by their native timestamps" + ), + ) + + def test_byte_identical_claude_reemit_is_counted_once(self): + records = self.build_three( + "identical", + reemit_claude=True, + ) + self.set_all_activity( + records, + {"claude": 10, "codex": 20, "grok": 30}, + ) + + stream = records["claude"]["stream"] + raw_lines = [ + line for line in stream.read_text().splitlines() if line + ] + self.assertGreater( + len(raw_lines), + 2, + f"the byte-identical re-emit plant found too few lines in {stream}", + ) + self.assertIn( + raw_lines[-1], + raw_lines[:-1], + ( + f"the final Claude record in {stream} is not a " + "byte-identical re-emit" + ), + ) + + proc = self.run_pulse( + "--json", + "--repo", + REPO, + "--tail", + "20", + ) + self.assertEqual( + proc.returncode, + 0, + f"{PULSE} failed on a byte-identical re-emit: {proc.stderr}", + ) + rows = json.loads(proc.stdout)["sessions"] + claude = next( + row for row in rows if row["harness"] == "claude" + ) + self.assertEqual( + claude["tokens"], + CLAUDE_TOKENS, + ( + f"{PULSE} did not count a byte-identical Claude " + "re-emit exactly once" + ), + ) + + +class CompactPlainStatus(PulseCase): + def test_plain_lines_are_complete_bounded_and_name_only(self): + records = self.build_three("plain") + self.set_all_activity( + records, + {"claude": 11, "codex": 22, "grok": 33}, + ) + self.assert_marker_planted(records) + + proc = self.run_pulse( + "--repo", + REPO, + "--tail", + "2", + ) + self.assertEqual( + proc.returncode, + 0, + f"{PULSE} plain output failed: {proc.stderr}", + ) + self.assertNotIn( + MARKER, + proc.stdout + proc.stderr, + f"{PULSE} plain output leaked planted session content", + ) + + lines = [ + line for line in proc.stdout.splitlines() if line.strip() + ] + self.assertEqual( + len(lines), + 3, + f"{PULSE} must print one compact line per live session", + ) + for line in lines: + self.assertLessEqual( + len(line.encode()), + 320, + f"{PULSE} exceeded the per-session token diet: {line}", + ) + + by_harness = { + line.split(maxsplit=1)[0]: line for line in lines + } + self.assertEqual( + set(by_harness), + {"claude", "codex", "grok"}, + f"{PULSE} plain output omitted or duplicated a harness", + ) + + expected_common = { + "claude": ( + records["claude"]["id"], + CLAUDE_MODEL, + "idle=11s", + ), + "codex": ( + records["codex"]["id"], + CODEX_MODEL, + "idle=22s", + ), + "grok": ( + records["grok"]["id"], + GROK_MODEL, + "idle=33s", + ), + } + for harness, values in expected_common.items(): + line = by_harness[harness] + for value in values: + self.assertIn( + value, + line, + ( + f"{PULSE} {harness} line omitted required " + f"status value {value!r}" + ), + ) + self.assertIn( + "age=600s", + line, + f"{PULSE} {harness} line has the wrong session age", + ) + + self.assertIn( + "tokens=12530", + by_harness["claude"], + f"{PULSE} plain Claude spend is wrong", + ) + self.assertIn( + "recent=Read,Bash", + by_harness["claude"], + f"{PULSE} plain Claude tail is not the last two tool names", + ) + self.assertIn( + "tokens=490", + by_harness["codex"], + f"{PULSE} plain Codex spend is not the last cumulative count", + ) + self.assertIn( + "recent=shell_command,token_count", + by_harness["codex"], + f"{PULSE} plain Codex tail is not the last two event names", + ) + self.assertEqual( + by_harness["grok"].count("tokens=unrecorded"), + 1, + f"{PULSE} must state the Grok token gap exactly once", + ) + self.assertNotIn( + "tokens=0", + by_harness["grok"], + f"{PULSE} rendered unrecorded Grok tokens as zero", + ) + self.assertIn( + "recent=loop_started,phase_changed", + by_harness["grok"], + ( + f"{PULSE} plain Grok tail did not use event types " + "when tool_name was absent" + ), + ) + + +class LivenessAndFiltering(PulseCase): + def test_default_window_includes_boundary_and_excludes_dead_or_other_repo(self): + boundary = self.build_three("boundary", REPO) + dead = self.build_three("dead", REPO) + other = self.build_three("other", OTHER_REPO) + + for record in boundary.values(): + self.set_activity(record, NOW - 300) + for record in dead.values(): + self.set_activity(record, NOW - 301) + for record in other.values(): + self.set_activity(record, NOW - 1) + + proc = self.run_pulse("--json", "--repo", REPO) + self.assertEqual( + proc.returncode, + 0, + f"{PULSE} failed while applying liveness filters: {proc.stderr}", + ) + document = json.loads(proc.stdout) + self.assert_closed_json_shape(document) + rows = document["sessions"] + found = [ + (row["harness"], row["session"]) for row in rows + ] + expected = sorted( + ( + harness, + boundary[harness]["id"], + ) + for harness in ("claude", "codex", "grok") + ) + self.assertEqual( + found, + expected, + ( + f"{PULSE} default 300-second window or --repo filter " + "included the wrong sessions" + ), + ) + + def test_live_window_option_uses_the_injected_clock_at_its_boundary(self): + records = self.build_three("window") + for record in records.values(): + self.set_activity(record, NOW - 600) + + included = self.run_pulse( + "--json", + "--repo", + REPO, + "--live-window", + "600", + ) + self.assertEqual( + included.returncode, + 0, + f"{PULSE} rejected --live-window 600: {included.stderr}", + ) + included_document = json.loads(included.stdout) + self.assert_closed_json_shape(included_document) + self.assertEqual( + len(included_document["sessions"]), + 3, + ( + f"{PULSE} excluded streams exactly on the injected " + "live-window boundary" + ), + ) + + excluded = self.run_pulse( + "--json", + "--repo", + REPO, + "--live-window", + "599", + ) + self.assertEqual( + excluded.returncode, + 0, + f"{PULSE} rejected --live-window 599: {excluded.stderr}", + ) + excluded_document = json.loads(excluded.stdout) + self.assert_closed_json_shape(excluded_document) + self.assertEqual( + excluded_document, + {"sessions": []}, + ( + f"{PULSE} included streams older than the injected " + "live-window" + ), + ) + + def test_grok_liveness_is_the_freshest_stream_not_all_or_one(self): + """updates and events diverge in life; the session is live if + EITHER is fresh, and idle is the freshest stream's age — a + max()-over-streams reader and an events-only reader both die + on one of the two splits.""" + records = self.build_three("split") + grok = records["grok"] + for stale, fresh, name in ( + (grok["updates"], grok["events"], "updates-stale"), + (grok["events"], grok["updates"], "events-stale"), + ): + with self.subTest(split=name): + os.utime(stale, (NOW - 400, NOW - 400)) + os.utime(fresh, (NOW - 10, NOW - 10)) + self.assertEqual( + int(stale.stat().st_mtime), + NOW - 400, + f"the stale mtime plant did not land on {stale}", + ) + self.assertEqual( + int(fresh.stat().st_mtime), + NOW - 10, + f"the fresh mtime plant did not land on {fresh}", + ) + document = self.invoke_grok_only() + rows = [ + row + for row in document["sessions"] + if row["harness"] == "grok" + ] + self.assertEqual( + [row["session"] for row in rows], + [grok["id"]], + f"{PULSE} declared the session dead on the {name}" + " split even though one stream is fresh", + ) + self.assertEqual( + rows[0]["idle_seconds"], + 10, + f"{PULSE} did not take idle from the freshest" + f" stream on the {name} split", + ) + + def invoke_grok_only(self): + proc = self.run_pulse("--json", "--repo", REPO) + self.assertEqual( + proc.returncode, + 0, + f"{PULSE} failed on the divergent-mtime build: {proc.stderr}", + ) + return json.loads(proc.stdout) + + def test_no_live_sessions_has_exact_empty_outputs_and_exit_zero(self): + records = self.build_three("stale") + for record in records.values(): + self.set_activity(record, NOW - 301) + + plain = self.run_pulse("--repo", REPO) + self.assertEqual( + plain.returncode, + 0, + f"{PULSE} plain empty status failed: {plain.stderr}", + ) + self.assertEqual( + plain.stdout.strip(), + "no live sessions", + f"{PULSE} plain empty status has the wrong message", + ) + + structured = self.run_pulse("--json", "--repo", REPO) + self.assertEqual( + structured.returncode, + 0, + f"{PULSE} JSON empty status failed: {structured.stderr}", + ) + self.assertEqual( + structured.stdout.strip(), + '{"sessions": []}', + f"{PULSE} JSON empty status is not the required stable object", + ) + self.assert_closed_json_shape(json.loads(structured.stdout)) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/telemetry/tests/test_pulse_findings.py b/ops/devlane/telemetry/tests/test_pulse_findings.py new file mode 100644 index 0000000..f9ea5b7 --- /dev/null +++ b/ops/devlane/telemetry/tests/test_pulse_findings.py @@ -0,0 +1,472 @@ +"""Regression contracts for the six pulse findings from PR #24.""" + +import contextlib +import importlib.util +import io +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch +from urllib.parse import quote + +HERE = Path(__file__).resolve() +PULSE = HERE.parents[1] / "pulse.py" + +BASE_EPOCH = 1_787_306_400 +NOW = BASE_EPOCH + 600 +BASE_ISO = "2026-08-21T10:00:00.000Z" +LATER_ISO = "2026-08-21T10:00:01.000Z" +REPO = "/home/work/projects/minspec/workbench" + + +def load_module(testcase, path, name): + testcase.assertTrue(path.is_file(), f"required module is missing: {path}") + spec = importlib.util.spec_from_file_location(name, path) + testcase.assertIsNotNone(spec, f"could not create an import spec for {path}") + testcase.assertIsNotNone(spec.loader, f"could not load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def write_jsonl(path, entries): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "".join( + json.dumps(entry, separators=(",", ":"), sort_keys=True) + "\n" + for entry in entries + ) + ) + return path + + +class PulseFindings(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="pulse-findings-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.empty_claude = self.root / "empty-claude" + self.empty_codex = self.root / "empty-codex" + self.empty_grok = self.root / "empty-grok" + for path in (self.empty_claude, self.empty_codex, self.empty_grok): + path.mkdir() + + import_time_home = self.root / "import-time-codex-home" + import_time_home.mkdir() + with patch.dict( + os.environ, + {"CODEX_HOME": str(import_time_home)}, + clear=False, + ): + self.pulse = load_module( + self, + PULSE, + f"minspec_pulse_findings_{self._testMethodName}", + ) + + def set_mtime(self, path, timestamp): + os.utime(path, (timestamp, timestamp)) + self.assertAlmostEqual( + path.stat().st_mtime, + timestamp, + places=3, + msg=f"the activity mtime plant did not land on {path}", + ) + + def build_claude(self, root, repo, session, activity="Read", idle=10.0): + slug = "-" + "-".join(repo.strip("/").split("/")) + stream = write_jsonl( + root / slug / f"{session}.jsonl", + [ + { + "timestamp": BASE_ISO, + "cwd": repo, + "sessionId": session, + "message": { + "id": f"message-{session}", + "model": "claude-fable-5", + "usage": { + "input_tokens": 1, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 1, + }, + "content": [ + { + "type": "tool_use", + "name": activity, + "input": {}, + } + ], + }, + } + ], + ) + self.set_mtime(stream, NOW - idle) + return stream + + def build_codex( + self, + root, + repo, + session, + model="gpt-5-codex", + activity=(), + idle=10.0, + ): + entries = [ + { + "timestamp": BASE_ISO, + "type": "session_meta", + "payload": {"id": session, "cwd": repo}, + }, + { + "timestamp": LATER_ISO, + "type": "turn_context", + "payload": {"cwd": repo, "model": model}, + }, + ] + for index, payload in enumerate(activity, start=2): + entries.append( + { + "timestamp": f"2026-08-21T10:00:{index:02d}.000Z", + "type": "response_item", + "payload": payload, + } + ) + stream = write_jsonl( + root + / "sessions" + / "2026" + / "08" + / "21" + / f"rollout-2026-08-21T10-00-00-{session}.jsonl", + entries, + ) + self.set_mtime(stream, NOW - idle) + return stream + + def build_grok( + self, + root, + repo, + session, + activity="grok_tool", + idle=10.0, + ): + session_dir = root / "sessions" / quote(repo, safe="") / session + session_dir.mkdir(parents=True, exist_ok=True) + summary = session_dir / "summary.json" + summary.write_text( + json.dumps( + { + "info": {"id": session, "cwd": repo}, + "current_model_id": "grok-4.6", + "created_at": BASE_ISO, + }, + separators=(",", ":"), + sort_keys=True, + ) + ) + events = write_jsonl( + session_dir / "events.jsonl", + [ + { + "timestamp": LATER_ISO, + "name": activity, + "ts": LATER_ISO, + "type": "tool_started", + "tool_name": activity, + } + ], + ) + self.set_mtime(events, NOW - idle) + return events + + def invoke(self, *args, env=None): + stdout = io.StringIO() + stderr = io.StringIO() + environment = contextlib.nullcontext() + if env is not None: + environment = patch.dict(os.environ, env, clear=False) + + with ( + environment, + contextlib.redirect_stdout(stdout), + contextlib.redirect_stderr(stderr), + ): + returncode = self.pulse.main(list(args)) + + self.assertEqual( + returncode, + 0, + f"{PULSE} returned {returncode}: {stderr.getvalue()}", + ) + try: + document = json.loads(stdout.getvalue()) + except json.JSONDecodeError as error: + self.fail(f"{PULSE} did not emit one JSON object: {error}") + self.assertEqual( + set(document), + {"sessions"}, + f"{PULSE} emitted an unexpected JSON document", + ) + self.assertIsInstance(document["sessions"], list) + return document + + def json_args(self, *, repo=REPO, claude=None, codex=None, grok=None, tail=10): + args = [ + "--json", + "--repo", + repo, + "--now", + str(NOW), + "--live-window", + "300", + "--tail", + str(tail), + "--claude-dir", + str(claude or self.empty_claude), + "--grok-dir", + str(grok or self.empty_grok), + ] + if codex is not None: + args.extend(("--codex-dir", str(codex))) + return args + + def test_b1_codex_home_is_dynamic_and_explicit_dir_wins(self): + first_home = self.root / "codex-home-first" + second_home = self.root / "codex-home-second" + first_id = "11111111-1111-4111-8111-111111111111" + second_id = "22222222-2222-4222-8222-222222222222" + first_model = "gpt-5-codex-first-home" + second_model = "gpt-5-codex-second-home" + self.build_codex(first_home, REPO, first_id, model=first_model) + self.build_codex(second_home, REPO, second_id, model=second_model) + + first = self.invoke( + *self.json_args(), + env={"CODEX_HOME": str(first_home)}, + ) + second = self.invoke( + *self.json_args(), + env={"CODEX_HOME": str(second_home)}, + ) + explicit = self.invoke( + *self.json_args(codex=second_home), + env={"CODEX_HOME": str(first_home)}, + ) + + observed = tuple( + [row["model"] for row in document["sessions"]] + for document in (first, second, explicit) + ) + self.assertEqual( + observed, + ([first_model], [second_model], [second_model]), + ( + "CODEX_HOME must be read for every main() invocation, " + "while --codex-dir must take precedence" + ), + ) + + def test_b2_codex_metadata_merges_across_meta_and_context_records(self): + codex_root = self.root / "codex-b2" + session = "33333333-3333-4333-8333-333333333333" + model = "gpt-5.6-codex" + self.build_codex(codex_root, REPO, session, model=model) + + document = self.invoke(*self.json_args(codex=codex_root)) + rows = document["sessions"] + self.assertEqual( + rows and len(rows), + 1, + "the live Codex rollout was not reported", + ) + self.assertEqual( + (rows[0]["session"], rows[0]["model"]), + (session, model), + "later Codex metadata must augment rather than erase earlier facts", + ) + + def test_b3_codex_recent_prefers_tool_name_with_type_fallback(self): + codex_root = self.root / "codex-b3" + session = "44444444-4444-4444-8444-444444444444" + self.build_codex( + codex_root, + REPO, + session, + activity=( + {"type": "custom_tool_call", "name": "exec"}, + {"type": "function_call", "name": "write_stdin"}, + {"type": "unlabelled_activity"}, + ), + ) + + document = self.invoke(*self.json_args(codex=codex_root, tail=3)) + rows = document["sessions"] + self.assertEqual(len(rows), 1, "the Codex activity rollout was not reported") + self.assertEqual( + rows[0]["recent"], + ["exec", "write_stdin", "unlabelled_activity"], + "Codex recent activity must expose tool names and fall back to type", + ) + + def test_b4_tail_zero_returns_no_activity_for_every_harness(self): + claude_root = self.root / "claude-b4" + codex_root = self.root / "codex-b4" + grok_root = self.root / "grok-b4" + self.build_claude(claude_root, REPO, "claude-b4", activity="Read") + self.build_codex( + codex_root, + REPO, + "55555555-5555-4555-8555-555555555555", + activity=({"type": "custom_tool_call", "name": "exec"},), + ) + self.build_grok(grok_root, REPO, "grok-b4", activity="search_code") + + document = self.invoke( + *self.json_args( + claude=claude_root, + codex=codex_root, + grok=grok_root, + tail=0, + ) + ) + recent = { + row["harness"]: row["recent"] for row in document["sessions"] + } + self.assertEqual( + recent, + {"claude": [], "codex": [], "grok": []}, + "--tail 0 must suppress non-empty history for every harness", + ) + + def test_b5_claude_repo_filter_uses_cwd_not_lossy_slug(self): + claude_root = self.root / "claude-b5" + actual_repo = "/a/b-c" + colliding_repo = "/a-b/c" + session = "claude-b5" + self.assertEqual( + "-" + "-".join(actual_repo.strip("/").split("/")), + "-" + "-".join(colliding_repo.strip("/").split("/")), + "the two repo paths do not plant the required slug collision", + ) + self.build_claude(claude_root, actual_repo, session) + + actual = self.invoke( + *self.json_args(repo=actual_repo, claude=claude_root) + ) + collision = self.invoke( + *self.json_args(repo=colliding_repo, claude=claude_root) + ) + self.assertEqual( + ( + [row["session"] for row in actual["sessions"]], + collision["sessions"], + ), + ([session], []), + "Claude --repo matching must verify the cwd stored in each entry", + ) + + def test_b6_liveness_uses_fractional_idle_but_reports_integer_idle(self): + claude_root = self.root / "claude-b6" + codex_root = self.root / "codex-b6" + grok_root = self.root / "grok-b6" + stale_sessions = { + "claude": "claude-fractionally-stale", + "codex": "66666666-6666-4666-8666-666666666661", + "grok": "grok-fractionally-stale", + } + boundary_sessions = { + "claude": "claude-exact-boundary", + "codex": "66666666-6666-4666-8666-666666666662", + "grok": "grok-exact-boundary", + } + streams = [ + self.build_claude( + claude_root, + REPO, + stale_sessions["claude"], + idle=300.8, + ), + self.build_codex( + codex_root, + REPO, + stale_sessions["codex"], + idle=300.8, + ), + self.build_grok( + grok_root, + REPO, + stale_sessions["grok"], + idle=300.8, + ), + ] + boundary_streams = [ + self.build_claude( + claude_root, + REPO, + boundary_sessions["claude"], + idle=300.0, + ), + self.build_codex( + codex_root, + REPO, + boundary_sessions["codex"], + idle=300.0, + ), + self.build_grok( + grok_root, + REPO, + boundary_sessions["grok"], + idle=300.0, + ), + ] + for stream in streams: + self.assertGreater( + NOW - stream.stat().st_mtime, + 300, + f"the fractionally stale mtime is not outside: {stream}", + ) + for stream in boundary_streams: + self.assertAlmostEqual( + NOW - stream.stat().st_mtime, + 300.0, + places=3, + msg=f"the mtime is not exactly on the boundary: {stream}", + ) + + document = self.invoke( + *self.json_args( + claude=claude_root, + codex=codex_root, + grok=grok_root, + ) + ) + rows = document["sessions"] + observed = { + (row["harness"], row["session"]): row for row in rows + } + expected = { + (harness, session) + for harness, session in boundary_sessions.items() + } + self.assertEqual( + set(observed), + expected, + ( + "fractional idle must be compared before integer " + "presentation truncation for every harness" + ), + ) + for key, row in observed.items(): + with self.subTest(session=key): + self.assertIs(type(row["idle_seconds"]), int) + self.assertEqual(row["idle_seconds"], 300) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/telemetry/tests/test_usage.py b/ops/devlane/telemetry/tests/test_usage.py new file mode 100644 index 0000000..32e9d4b --- /dev/null +++ b/ops/devlane/telemetry/tests/test_usage.py @@ -0,0 +1,146 @@ +"""The usage reporter, against fixtures shaped like the real session stores. + +Each fixture replicates a shape measured on 2026-08-21 from the live +stores: Claude's per-message usage, Codex's cumulative token_count events, +and Grok's summary.json — which records NO token usage, a gap the report +must state rather than estimate around. +""" + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from urllib.parse import quote + +USAGE = Path(__file__).resolve().parents[1] / "usage.py" +REPO = "/home/work/projects/minspec/workbench" + + +def line(**kw): + return json.dumps(kw) + "\n" + + +class Fixtures(unittest.TestCase): + def setUp(self): + self.home = Path(tempfile.mkdtemp(prefix="telemetry-")) + self.addCleanup(__import__("shutil").rmtree, self.home, True) + + # Claude: //.jsonl, usage per message. + proj = self.home / "claude" / "-home-work-projects-minspec-workbench" + proj.mkdir(parents=True) + (proj / "aaaa.jsonl").write_text( + line(timestamp="2026-08-21T10:00:00.000Z", + message={"model": "claude-fable-5", + "usage": {"input_tokens": 10, + "cache_creation_input_tokens": 1000, + "cache_read_input_tokens": 5000, + "output_tokens": 200}}) + + line(timestamp="2026-08-21T10:05:00.000Z", + message={"model": "claude-fable-5", + "usage": {"input_tokens": 20, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 6000, + "output_tokens": 300}})) + other = self.home / "claude" / "-somewhere-else" + other.mkdir() + (other / "bbbb.jsonl").write_text( + line(timestamp="2026-08-21T09:00:00.000Z", + message={"model": "claude-fable-5", + "usage": {"input_tokens": 999, "output_tokens": 999}})) + + # Codex: sessions/Y/M/D/rollout-*.jsonl, cumulative token_count. + day = self.home / "codex" / "sessions" / "2026" / "08" / "21" + day.mkdir(parents=True) + (day / "rollout-2026-08-21T10-00-00-cccc.jsonl").write_text( + line(timestamp="2026-08-21T10:00:00.000Z", type="session_meta", + payload={"id": "cccc", "cwd": REPO, + "timestamp": "2026-08-21T10:00:00.000Z"}) + + line(timestamp="2026-08-21T10:02:00.000Z", type="event_msg", + payload={"type": "token_count", + "info": {"total_token_usage": { + "input_tokens": 100, "cached_input_tokens": 50, + "output_tokens": 40, "total_tokens": 140}}}) + + line(timestamp="2026-08-21T10:09:00.000Z", type="event_msg", + payload={"type": "token_count", + "info": {"total_token_usage": { + "input_tokens": 400, "cached_input_tokens": 300, + "output_tokens": 90, "total_tokens": 490}}})) + + # Grok: sessions///summary.json — no tokens. + gdir = (self.home / "grok" / "sessions" + / quote(REPO, safe="") / "dddd") + gdir.mkdir(parents=True) + (gdir / "summary.json").write_text(json.dumps({ + "info": {"id": "dddd", "cwd": REPO}, + "session_summary": "SESSIONPROMPTTEXT", "num_messages": 177, + "created_at": "2026-08-21T10:00:00.000000000Z", + "updated_at": "2026-08-21T10:20:00.000000000Z", + "current_model_id": "grok-4.6", + "git_root_dir": REPO + "/"})) + + def run_usage(self, *args): + return subprocess.run( + [sys.executable, str(USAGE), *args, + "--claude-dir", str(self.home / "claude"), + "--codex-dir", str(self.home / "codex"), + "--grok-dir", str(self.home / "grok"), + "--repo", REPO], + capture_output=True, text=True, check=False) + + def sessions(self): + proc = self.run_usage("sessions", "--json") + self.assertEqual(proc.returncode, 0, proc.stderr) + return json.loads(proc.stdout)["sessions"] + + +class SessionsAreNormalised(Fixtures): + def test_all_three_harnesses_appear_once(self): + rows = self.sessions() + self.assertEqual(sorted(r["harness"] for r in rows), + ["claude", "codex", "grok"]) + + def test_the_repo_filter_excludes_other_projects(self): + for row in self.sessions(): + self.assertNotEqual(row.get("session"), "bbbb", + "a session from another repo leaked in") + + def test_claude_tokens_are_summed_per_message(self): + row = next(r for r in self.sessions() if r["harness"] == "claude") + self.assertEqual(row["tokens"]["output"], 500) + self.assertEqual(row["tokens"]["input"], 30) + self.assertEqual(row["tokens"]["cached"], 12000) + + def test_codex_takes_the_last_cumulative_count(self): + row = next(r for r in self.sessions() if r["harness"] == "codex") + self.assertEqual(row["tokens"]["total"], 490, + "cumulative counts must not be summed") + self.assertEqual(row["tokens"]["output"], 90) + + def test_grok_states_its_gap_instead_of_inventing(self): + row = next(r for r in self.sessions() if r["harness"] == "grok") + self.assertIsNone(row["tokens"]) + self.assertIn("not yet parsed", row["note"]) + self.assertEqual(row["messages"], 177) + self.assertEqual(row["model"], "grok-4.6") + + +class TheReportAggregates(Fixtures): + def test_totals_per_harness_and_the_gap_stated(self): + proc = self.run_usage("report") + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn("claude", proc.stdout) + self.assertIn("490", proc.stdout, "codex total") + self.assertIn("not yet parsed", proc.stdout, "the grok gap is stated") + + def test_the_report_never_prints_prompt_text(self): + # The stores hold prompts and summaries; the report is aggregates + # only. The fixture plants a distinctive marker to catch a leak. + proc = self.run_usage("report") + self.assertNotIn("SESSIONPROMPTTEXT", proc.stdout, + "session content leaked into the report") + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/telemetry/tests/test_wiring.py b/ops/devlane/telemetry/tests/test_wiring.py new file mode 100644 index 0000000..2f800d0 --- /dev/null +++ b/ops/devlane/telemetry/tests/test_wiring.py @@ -0,0 +1,1864 @@ +"""Guard against process-doc recipes whose tool paths or flags silently rot.""" + +import ast +import re +import unittest +from pathlib import Path + +REPO = Path(__file__).resolve().parents[4] +DOC = REPO / "ops/process/cross-review.md" +BREAKER = REPO / "ops/devlane/telemetry/breaker.py" +BREAKER_PATH = "ops/devlane/telemetry/breaker.py" + +CLAUDE_ROW = ( + "| Claude | newest `*.jsonl` under `~/.claude/projects//`, where " + "`` is `$SNAP` with every `/` and `.` replaced by `-` | all six |" +) +CODEX_ROW = ( + "| Codex | newest `~/.codex/sessions/*/*/*/rollout-*.jsonl` | " + "tokens, tokens-out, stall, size |" +) +GROK_ROW = ( + "| Grok | `updates.jsonl` in the newest session dir " + "under `~/.grok/sessions//` | " + "tokens, tokens-out, stall, size |" +) + + +def fenced_logical_blocks(text): + """Return fenced blocks containing (first physical line, logical line).""" + blocks = [] + current = None + start = None + parts = [] + + for number, line in enumerate(text.splitlines(), 1): + if line.lstrip().startswith("```"): + if current is None: + current = [] + else: + if parts: + current.append((start, " ".join(parts))) + start = None + parts = [] + blocks.append(current) + current = None + continue + if current is None: + continue + + if start is None: + start = number + stripped = line.rstrip() + continued = stripped.endswith("\\") + parts.append((stripped[:-1] if continued else line).strip()) + if not continued: + current.append((start, " ".join(parts))) + start = None + parts = [] + + if current is not None: + if parts: + current.append((start, " ".join(parts))) + blocks.append(current) + return blocks + + +def fenced_logical_lines(text): + """Return (first physical line, logical line) pairs from fenced blocks.""" + return [item for block in fenced_logical_blocks(text) for item in block] + + +def fenced_lines(text): + return fenced_logical_lines(text) + + +def breaker_invocations(text): + return [ + (number, line) + for number, line in fenced_logical_lines(text) + if re.search(r"\bbreaker\.py\b", line) + ] + + +def breaker_lines(text): + return breaker_invocations(text) + + +def launch_fenced_blocks(text): + return [ + block + for block in fenced_logical_blocks(text) + if any( + not line.lstrip().startswith("#") and re.search(r"\bnohup\b", line) + for _, line in block + ) + ] + + +def _touches_launch_marker(line): + return re.search( + r"^\s*touch\s+[\"']?launched[\"']?(?:\s|$)", + line, + ) is not None + + +def launch_form_errors(text): + errors = [] + blocks = launch_fenced_blocks(text) + if not blocks: + return ["no fenced reviewer launch uses nohup"] + + for block in blocks: + code_only = "\n".join( + "" if line.lstrip().startswith("#") else line.split("#", 1)[0] + for _, line in block + ) + if re.search(r"\$\(\s*cat\s+prompt\.txt\s*\)", code_only): + errors.append("launch fence uses $(cat prompt.txt)") + + launches = [ + (index, number, line) + for index, (number, line) in enumerate(block) + if not line.lstrip().startswith("#") and re.search(r"\bnohup\b", line) + ] + for index, number, line in launches: + if re.search(r"\bcd\b.*&&.*\bnohup\b.*&", line): + errors.append(f"line {number}: compound cd && nohup launch") + marker_precedes_launch = any( + prior_index < index and _touches_launch_marker(prior_line) + for prior_index, (_, prior_line) in enumerate(block) + ) + if not marker_precedes_launch: + errors.append(f"line {number}: touch launched does not precede nohup") + if not line.split("#", 1)[0].rstrip().endswith("&"): + errors.append( + f"line {number}: nohup launch is not backgrounded" + " with a trailing &" + ) + enters_snapshot = any( + prior_index < index + and re.match(r'\s*cd\s+"\$SNAP"', prior_line) + for prior_index, (_, prior_line) in enumerate(block) + ) + if not enters_snapshot: + errors.append( + f"line {number}: launch does not cd into the" + ' snapshot ("$SNAP") first' + ) + captures_pid = any( + later_index > index and re.match(r"\s*RPID=\$!", later_line) + for later_index, (_, later_line) in enumerate(block) + ) + if not captures_pid: + errors.append( + f"line {number}: nohup launch never captures RPID=$!" + ) + return errors + + +def declared_flags(source): + flags = set() + for node in ast.walk(ast.parse(source)): + if not ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "add_argument" + ): + continue + for argument in node.args: + if ( + isinstance(argument, ast.Constant) + and isinstance(argument.value, str) + and argument.value.startswith("--") + ): + flags.add(argument.value) + return flags + + +def markdown_tables(text): + tables = [] + table = [] + for number, line in enumerate(text.splitlines(), 1): + if line.startswith("|"): + cells = [cell.strip() for cell in line.strip().strip("|").split("|")] + table.append((number, cells)) + elif table: + tables.append(table) + table = [] + if table: + tables.append(table) + return tables + + +def prose_blocks(text): + lines = [] + in_fence = False + for line in text.splitlines(): + if line.lstrip().startswith("```"): + in_fence = not in_fence + lines.append("") + elif in_fence: + lines.append("") + else: + lines.append(line) + return [ + block.strip() + for block in re.split(r"\n\s*\n", "\n".join(lines)) + if block.strip() + ] + + +def _normalise_cell(cell): + return re.sub(r"\s+", " ", re.sub(r"[`*_]", "", cell)).strip().lower() + + +def _listed_wires(cell): + return { + item.strip() + for item in re.split(r"\s*,\s*|\s+and\s+", _normalise_cell(cell)) + if item.strip() + } + + +def wire_table_errors(text): + errors = [] + candidates = [] + for table in markdown_tables(text): + if not table: + continue + header = table[0][1] + if ( + len(header) >= 3 + and _normalise_cell(header[0]) == "reviewer" + and _normalise_cell(header[2]) == "wires" + ): + candidates.append(table) + + if len(candidates) != 1: + return [f"expected one reviewer wire table, found {len(candidates)}"] + + rows = candidates[0][2:] + paired = {} + for reviewer in ("claude", "codex", "grok"): + matches = [ + (number, cells) + for number, cells in rows + if cells and _normalise_cell(cells[0]) == reviewer + ] + if len(matches) != 1: + errors.append(f"expected one {reviewer.title()} row, found {len(matches)}") + else: + paired[reviewer] = matches[0] + + if "claude" in paired: + number, cells = paired["claude"] + if len(cells) < 3: + errors.append(f"line {number}: Claude row has fewer than three cells") + else: + store, wires = cells[1], cells[2] + slug_rule = ( + "" in store + and "$SNAP" in store + and re.search( + r"\bevery\b.*`/`.*`\.`.*\breplaced\s+by\b.*`-`", + store, + re.IGNORECASE, + ) + ) + if "~/.claude/projects" not in store: + errors.append(f"line {number}: Claude store root") + if "*.jsonl" not in store: + errors.append(f"line {number}: Claude stream glob") + if not slug_rule: + errors.append(f"line {number}: Claude slug rule") + if _normalise_cell(wires) != "all six": + errors.append(f"line {number}: Claude wires are not exactly all six") + + if "codex" in paired: + number, cells = paired["codex"] + if len(cells) < 3: + errors.append(f"line {number}: Codex row has fewer than three cells") + else: + store, wires = cells[1], cells[2] + if "~/.codex/sessions" not in store: + errors.append(f"line {number}: Codex store root") + if "rollout-*.jsonl" not in store: + errors.append(f"line {number}: Codex rollout glob") + if _listed_wires(wires) != { + "tokens", + "tokens-out", + "stall", + "size", + }: + errors.append( + f"line {number}: Codex wires must be exactly tokens, " + "tokens-out, stall, and size" + ) + + if "grok" in paired: + number, cells = paired["grok"] + if len(cells) < 3: + errors.append(f"line {number}: Grok row has fewer than three cells") + else: + store, wires = cells[1], cells[2] + if "~/.grok/sessions" not in store: + errors.append(f"line {number}: Grok store root") + affirmed = set() + for match in re.finditer(r"[\w*-]+\.jsonl\b", store): + lookback = store[max(0, match.start() - 16):match.start()] + # "(not `events.jsonl`)" is a warning, not an offer + if re.search(r"\b(?:not|never)\b[^,;]{0,14}$", lookback): + continue + affirmed.add(match.group(0)) + if affirmed != {"updates.jsonl"}: + # only updates.jsonl carries usage; offering ANY + # alternative ("or events.jsonl", "or session.jsonl") + # silently zeroes the token wires the row promises. + errors.append( + f"line {number}: Grok row must name updates.jsonl as" + f" the only offered stream (found {sorted(affirmed)})" + ) + if not re.search(r"\burl-encod\w*\b", store, re.IGNORECASE): + errors.append(f"line {number}: Grok URL-encoded snapshot") + listed = _listed_wires(wires) + if listed != {"tokens", "tokens-out", "stall", "size"}: + errors.append( + f"line {number}: Grok wires must be exactly tokens," + " tokens-out, stall, size" + ) + if re.search(r"\ball\s+six\b", wires, re.IGNORECASE): + errors.append( + f"line {number}: Grok cannot claim all six —" + " repeat-loop and error-storm stay claude-only" + ) + + return errors + + +def flag_values(invocation, flag): + pattern = re.compile( + rf"(?> breaker.log &\n" +) + + +def recipe_fence_errors(text): + """The pinned executable contract: sh has unbounded spellings for + one-line subversion, so the LIVE doc's recipe fence is checked by + equality; battery_wiring_errors stays the feature net for + synthetic corpora and any other fence.""" + errors = [] + fences = re.findall(r"```sh\n(.*?)```", text, re.DOTALL) + recipe_fences = [f for f in fences + if "STORE=" in f and BREAKER_PATH in f] + if len(recipe_fences) != 1: + return [(f"expected one supervision recipe fence, found" + f" {len(recipe_fences)}")] + if recipe_fences[0] != RECIPE_FENCE: + errors.append( + "the supervision recipe fence diverged from the pinned" + " contract — update the doc and RECIPE_FENCE together," + " deliberately") + return errors + + +def battery_wiring_errors(text): + errors = [] + logical_lines = [ + (number, line.split("#", 1)[0]) + for number, line in fenced_logical_lines(text) + ] + stores = [] + patterns = [] + for number, line in logical_lines: + assign = re.match(r"\s*(?:export\s+)?STORE=(\S+)", line) + if assign: + stores.append((number, assign.group(1))) + assign = re.match(r"\s*(?:export\s+)?PATTERN=(\S+)", line) + if assign: + patterns.append((number, assign.group(1))) + for number, line in logical_lines: + stream_find = re.search( + r"\bSTREAM=\$\(\s*find\b[^;|]*?-name\s+(\S+)", line) + # the recipe's PATTERN value is only meaningful if the find + # actually reads it: an inlined literal ("-name events.jsonl") + # detaches the checked assignment from the selected stream + # (Grok, PR #29 delta round 4) + if (stream_find and patterns + and not re.fullmatch(r"\"?\$\{?PATTERN\}?\"?", + stream_find.group(1))): + errors.append( + f"line {number}: stream discovery ignores the PATTERN" + f" assignment and selects {stream_find.group(1)}" + ) + + for p_number, p_value in patterns: + # pair each PATTERN with the nearest STORE in either + # direction — assignment order is not load-bearing + near = min(stores, key=lambda item: abs(item[0] - p_number), + default=None) + # events.jsonl (and any sibling) carries no usage: a grok + # recipe pointing the battery elsewhere zeroes the token + # wires while the table still promises them. + if (near and abs(near[0] - p_number) <= 5 + and ".grok/sessions" in near[1] + and p_value != "updates.jsonl"): + errors.append( + f"line {p_number}: grok recipe PATTERN is" + f" {p_value}, not the stream that carries" + " usage (updates.jsonl)" + ) + + invocations = [ + (number, line) + for number, line in logical_lines + if BREAKER_PATH in line + ] + if not invocations: + return [f"no fenced {BREAKER_PATH} invocation"] + + armed = [ + (number, line) + for number, line in invocations + if BREAKER_PATH in line and re.search(r"(? review.txt 2> review.err &\n" + "RPID=$!\n" + "```\n" + ) + self.assertFalse(launch_form_errors(text)) + + def test_unsafe_launch_forms_are_rejected(self): + unsafe = ( + ( + "prompt substitution", + ( + "```sh\n" + "touch launched\n" + 'nohup grok "$(cat prompt.txt)" > review.txt &\n' + "```\n" + ), + ), + ( + "compound background list", + ( + "```sh\n" + "touch launched\n" + 'cd "$SNAP" && nohup grok < prompt.txt > review.txt &\n' + "```\n" + ), + ), + ( + "late launch marker", + ( + "```sh\n" + "nohup grok < prompt.txt > review.txt &\n" + "touch launched\n" + "```\n" + ), + ), + ) + for label, text in unsafe: + with self.subTest(label=label): + self.assertTrue( + launch_form_errors(text), + f"unsafe launch form survived: {label}", + ) + + +class BatteryWiringUnitTests(unittest.TestCase): + def test_nonzero_integer_flag_requires_one_literal_nonzero_value(self): + self.assertTrue(carries_nonzero_integer_flag("breaker --cap 12", "--cap")) + invalid = ( + "breaker", + "breaker --cap 0", + 'breaker --cap "0"', + 'breaker --cap "$CAP"', + "breaker --cap 1 --cap 2", + ) + for invocation in invalid: + with self.subTest(invocation=invocation): + self.assertFalse( + carries_nonzero_integer_flag(invocation, "--cap") + ) + + def test_discovery_liveness_probe_requires_kill_zero_on_reviewer_pid(self): + self.assertTrue( + discovery_loop_has_kill_zero( + "until STREAM=$(find store -newer launched -printf '%T@ %p\\n' | sort -n | tail -1 | cut -f2-); " + '[ -n "$STREAM" ] || ! kill -0 "$RPID"; do :; done' + ) + ) + self.assertFalse( + discovery_loop_has_kill_zero( + "until STREAM=$(find store -newer launched -printf '%T@ %p\\n' | sort -n | tail -1 | cut -f2-); " + '[ -n "$STREAM" ] || ! kill -9 "$RPID"; do :; done' + ) + ) + + def test_battery_contract_rejects_zero_caps_and_missing_probe(self): + valid = ( + "```sh\n" + "until STREAM=$(find store -newer launched -printf '%T@ %p\\n' | sort -n | tail -1 | cut -f2-); " + '[ -n "$STREAM" ] || ! kill -0 "$RPID"; do :; done\n' + f'[ -n "$STREAM" ] && python3 {BREAKER_PATH} "$STREAM" --pid "$RPID" ' + "--cap 1 --cap-out 1 --terminate --tripped-file TRIPPED.md\n" + "```\n" + ) + self.assertFalse(battery_wiring_errors(valid)) + mutants = ( + valid.replace("--cap 1", "--cap 0"), + valid.replace("--cap-out 1", "--cap-out 0"), + valid.replace("kill -0", "kill -9"), + ) + for mutant in mutants: + with self.subTest(mutant=mutant): + self.assertTrue(battery_wiring_errors(mutant)) + + +class BreakerFlagContractTests(unittest.TestCase): + def test_documented_breaker_flags_are_declared_by_the_cli(self): + text = DOC.read_text(encoding="utf-8") + errors = breaker_flag_errors(text, BREAKER.read_text(encoding="utf-8")) + self.assertFalse( + errors, + "cross-review recipe uses flags breaker.py does not declare: " + + ", ".join(errors), + ) + + def test_ast_flag_discovery_accepts_short_first_groups_and_raw_strings(self): + source = ( + "parser.add_argument('-s', '--stall')\n" + "group = parser.add_argument_group('process')\n" + "group.add_argument(r'--pid')\n" + "other.add_argument('--size-mb', dest='size_mb')\n" + ) + self.assertEqual( + declared_flags(source), + {"--stall", "--pid", "--size-mb"}, + ) + + +class TripReportingContractTests(unittest.TestCase): + def test_trips_are_evidenced_and_named_in_the_pr_body(self): + text = DOC.read_text(encoding="utf-8") + errors = trip_reporting_errors(text) + self.assertFalse( + errors, + f"{DOC}: trip reporting contract is incomplete: " + ", ".join(errors), + ) + + +class TripPolarityUnitTests(unittest.TestCase): + def test_affirmative_trip_statements_are_recognised(self): + paragraph = ( + "A trip is exit code 3. The battery writes the TRIPPED file and " + "kills the runaway reviewer." + ) + self.assertTrue(affirms_exit_three(paragraph)) + self.assertTrue(affirms_tripped_file_write(paragraph)) + self.assertTrue(affirms_reviewer_kill(paragraph)) + + def test_negated_trip_statements_are_not_affirmative(self): + cases = ( + ( + "A trip is not exit code 3.", + affirms_exit_three, + ), + ( + "The battery never writes the TRIPPED file.", + affirms_tripped_file_write, + ), + ( + "The battery never kills the runaway reviewer.", + affirms_reviewer_kill, + ), + ) + for sentence, predicate in cases: + with self.subTest(sentence=sentence): + self.assertFalse(predicate(sentence)) + + +class HarnessStreamContractTests(unittest.TestCase): + def test_each_reviewer_row_pairs_its_stream_and_wires(self): + text = DOC.read_text(encoding="utf-8") + errors = wire_table_errors(text) + self.assertFalse( + errors, + f"{DOC}: reviewer stream wiring is missing or ambiguous: " + + ", ".join(errors), + ) + + +class WireSetUnitTests(unittest.TestCase): + def reviewer_table(self, claude="all six", codex=None, grok=None): + if grok is None: + grok = "tokens, tokens-out, stall, size" + if codex is None: + codex = "tokens, tokens-out, stall, size" + rows = ( + "| reviewer | store root / pattern for the stream | wires |", + "|:--|:--|:--|", + CLAUDE_ROW.replace("| all six |", f"| {claude} |"), + CODEX_ROW.replace( + "| tokens, tokens-out, stall, size |", + f"| {codex} |", + ), + GROK_ROW.replace( + "| tokens, tokens-out, stall, size |", f"| {grok} |"), + ) + return "\n".join(rows) + + def test_each_reviewer_wire_set_rejects_subsets_and_supersets(self): + self.assertFalse(wire_table_errors(self.reviewer_table())) + mutants = ( + ("Claude subset", {"claude": "stall, size"}), + ("Claude superset", {"claude": "all six, rate"}), + ("Codex subset", {"codex": "tokens, tokens-out, stall"}), + ( + "Codex superset", + {"codex": "tokens, tokens-out, stall, size, rate"}, + ), + ("Grok subset", {"grok": "tokens, stall, size"}), + ("Grok superset", {"grok": "all six"}), + ) + for label, changes in mutants: + with self.subTest(label=label): + self.assertTrue( + wire_table_errors(self.reviewer_table(**changes)), + f"wire-set mutant survived: {label}", + ) + + + +class GrokTelemetryGapTests(unittest.TestCase): + def test_grok_gap_is_stated_without_promising_token_caps(self): + text = DOC.read_text(encoding="utf-8") + errors = grok_gap_errors(text) + self.assertFalse( + errors, + f"{DOC}: Grok telemetry gap is not stated honestly: " + + " | ".join(errors), + ) + + +class GrokClaimNetUnitTests(unittest.TestCase): + """The doc-wide nets are best-effort regexes over unbounded + paraphrase space: this corpus documents what they catch and what + they deliberately leave alone. The load-bearing grok claims are + NOT guarded here — they live in the equality-pinned paragraph + (GrokParagraphPinTests), the wire table, and the armed line.""" + + CARRIER = ( + "Grok records its spend and the battery parses it, so grok " + "streams feed the token walls. Only repeat-loop and " + "error-storm stay claude-only." + ) + + def test_refuted_and_stale_and_denial_claims_fire(self): + lies = ( + self.CARRIER.replace("records its spend", "records no spend"), + self.CARRIER + " Grok sessions have no token usage.", + self.CARRIER + " Grok carries no spend.", + self.CARRIER + " The battery does not yet parse that shape.", + self.CARRIER + " Grok usage is not parsed.", + self.CARRIER + " Grok has no token cap.", + self.CARRIER + " Grok cannot enforce a token cap.", + self.CARRIER + " Grok token caps are unavailable.", + self.CARRIER + " Do not apply a token cap to Grok.", + self.CARRIER + " Claude uses --cap; Grok does not.", + self.CARRIER + " Grok does not have a token cap.", + self.CARRIER + " Token caps do not apply to grok.", + self.CARRIER + " Grok operates without token caps.", + self.CARRIER + " Grok's tokens are not capped.", + self.CARRIER + " Grok has no token cap even when --cap" + " is passed.", + ) + for lie in lies: + with self.subTest(lie=lie[-60:]): + self.assertNotEqual(lie, self.CARRIER) + self.assertTrue(grok_claim_errors(lie)) + + def test_true_claims_stay_quiet(self): + truths = ( + self.CARRIER + " Grok's token cap trips like the others.", + self.CARRIER + " Grok has no token cap until --cap is" + " armed.", + self.CARRIER + " Grok cannot enforce a token cap of zero.", + self.CARRIER + " Grok token caps are unavailable until" + " armed.", + self.CARRIER + " Claude uses --cap; Grok does not skip it.", + self.CARRIER + " Cancelled turns carry no token usage.", + self.CARRIER + " events.jsonl carries no token usage.", + self.CARRIER + " reasoningTokens remain unsupported" + " telemetry.", + ) + for truth in truths: + with self.subTest(truth=truth[-60:]): + self.assertFalse(grok_claim_errors(truth)) + + +class GrokParagraphPinTests(unittest.TestCase): + def test_the_live_paragraph_matches_the_pinned_contract(self): + self.assertFalse(grok_gap_errors(DOC.read_text(encoding="utf-8"))) + + def test_any_edit_to_the_paragraph_is_refused(self): + live = DOC.read_text(encoding="utf-8") + anchor = "so grok streams feed the token walls" + self.assertEqual(live.count(anchor), 1) + edits = ( + live.replace(anchor, anchor + " generously"), + live.replace(anchor, "so grok streams never feed the" + " token walls"), + live.replace("stay claude-only:", "stay claude-only," + " though repeat-loop can fire for every" + " reviewer:"), + live.replace("like the others. Only", + "like the others. All six wires fire for" + " every reviewer. Only"), + ) + for mutated in edits: + with self.subTest(edit=mutated[:40]): + self.assertNotEqual(mutated, live) + self.assertTrue(grok_gap_errors(mutated)) + + +class RecipeFencePinTests(unittest.TestCase): + def test_the_live_fence_matches_the_pinned_contract(self): + self.assertFalse( + recipe_fence_errors(DOC.read_text(encoding="utf-8"))) + + def test_any_one_sided_fence_edit_is_refused(self): + live = DOC.read_text(encoding="utf-8") + edits = ( + ("PATTERN=updates.jsonl", + "declare -x PATTERN=events.jsonl"), + ("PATTERN=updates.jsonl", + "PATTERN=updates.jsonl && PATTERN=events.jsonl"), + ('-name "$PATTERN"', "-iname events.jsonl"), + ('-name "$PATTERN"', + '\\( -name "$PATTERN" -o -name events.jsonl \\)'), + ('-name "$PATTERN"', "-path '*/events.jsonl'"), + ) + for anchor, replacement in edits: + with self.subTest(edit=replacement[:40]): + self.assertEqual(live.count(anchor) >= 1, True, anchor) + mutated = live.replace(anchor, replacement, 1) + self.assertNotEqual(mutated, live) + self.assertTrue(recipe_fence_errors(mutated)) + + +class BreakerDocstringClaimTests(unittest.TestCase): + def test_breaker_docstring_carries_no_refuted_grok_claims(self): + module = ast.parse(BREAKER.read_text(encoding="utf-8")) + doc = ast.get_docstring(module) or "" + self.assertTrue(doc, "breaker.py has no module docstring") + self.assertFalse( + grok_claim_errors(doc), + "breaker.py's own docstring makes a refuted grok claim") + + def test_a_reverted_not_yet_parsed_docstring_is_refused(self): + plant = ("Grok's usage records are not yet parsed by this" + " battery; grok rows gate on stall and size only.") + self.assertTrue(grok_claim_errors(plant)) + + +class SkepticDocMutantBatteryTests(unittest.TestCase): + def plant(self, label, anchor, replacement): + text = DOC.read_text(encoding="utf-8") + matches = text.count(anchor) + self.assertEqual( + matches, + 1, + f"INVALID {label}: anchor matched {matches} times, expected exactly once", + ) + mutated = text.replace(anchor, replacement) + self.assertNotEqual(mutated, text, f"INVALID {label}: mutation did not land") + return mutated + + def assert_rejected(self, label, errors): + self.assertTrue(errors, f"{label} SURVIVED: relevant guard accepted mutant") + + def insert_before_trip(self, label, sentence): + anchor = "\nA trip is exit code 3:" + return self.plant( + label, + anchor, + f"\n{sentence}\n\nA trip is exit code 3:", + ) + + def test_m1_continuation_flags_are_scanned(self): + anchor = ( + " --cap 2000000 --cap-out 150000 --stall 600 --size-mb 50 " + "--terminate \\\n" + " --tripped-file TRIPPED.md" + ) + replacement = ( + " --cap 2000000 --cap-out 100000 --not-a-flag 600 --bogus-mb 50 " + "--kill-it \\\n" + " --tripped TRIPPED.md" + ) + mutated = self.plant("M1 continuation flags", anchor, replacement) + self.assert_rejected( + "M1 continuation flags", + breaker_flag_errors(mutated, BREAKER.read_text(encoding="utf-8")), + ) + + def test_m2_swapped_reviewer_paths_are_rejected(self): + anchor = f"{CLAUDE_ROW}\n{CODEX_ROW}\n{GROK_ROW}" + replacement = "\n".join( + ( + CLAUDE_ROW.replace("~/.claude/projects", "~/.codex/sessions"), + CODEX_ROW.replace("~/.codex/sessions", "~/.grok/sessions"), + GROK_ROW.replace("~/.grok/sessions", "~/.claude/projects"), + ) + ) + mutated = self.plant("M2 swapped paths", anchor, replacement) + self.assert_rejected("M2 swapped paths", wire_table_errors(mutated)) + + def test_m3_claude_basename_slug_rule_is_rejected(self): + mutated = self.plant( + "M3 Claude slug", + "where `` is `$SNAP` with every `/` and `.` replaced by `-`", + "where `` is the basename of `$SNAP`", + ) + self.assert_rejected("M3 Claude slug", wire_table_errors(mutated)) + + def test_m42_feed_negation_in_the_live_doc_is_rejected(self): + mutated = self.plant( + "M42 feed negated", + "so grok streams feed the token walls", + "so grok streams never feed the token walls", + ) + self.assert_rejected("M42 feed negated", grok_gap_errors(mutated)) + + def test_m43_claude_only_contradiction_in_the_live_doc_is_rejected(self): + mutated = self.plant( + "M43 pair contradicted", + "error-storm stay claude-only: grok streams carry no tool_use or", + "error-storm stay claude-only, though repeat-loop can also fire" + " for grok: grok streams carry no tool_use or", + ) + self.assert_rejected( + "M43 pair contradicted", grok_gap_errors(mutated)) + + def test_m44_cap_denial_in_the_live_doc_is_rejected(self): + mutated = self.plant( + "M44 cap denial", + "like the others. Only", + "like the others. Grok has no token cap. Only", + ) + self.assert_rejected("M44 cap denial", grok_gap_errors(mutated)) + + def test_m45_disable_tokens_on_the_armed_line_is_rejected(self): + mutated = self.plant( + "M45 disable tokens", + "--tripped-file TRIPPED.md 2>> breaker.log &", + "--disable tokens --tripped-file TRIPPED.md 2>> breaker.log &", + ) + self.assert_rejected( + "M45 disable tokens", battery_wiring_errors(mutated)) + + def test_m46_recipe_pattern_swap_is_rejected(self): + mutated = self.plant( + "M46 recipe pattern", + "PATTERN=updates.jsonl", + "PATTERN=events.jsonl", + ) + self.assert_rejected( + "M46 recipe pattern", battery_wiring_errors(mutated)) + + def test_m47_alternative_stream_offer_is_rejected(self): + mutated = self.plant( + "M47 alternative stream", + "| Grok | `updates.jsonl` in", + "| Grok | `updates.jsonl` (or `session.jsonl`) in", + ) + self.assert_rejected( + "M47 alternative stream", wire_table_errors(mutated)) + + def test_m48_no_longer_feed_in_the_live_doc_is_rejected(self): + mutated = self.plant( + "M48 no-longer-feed", + "so grok streams feed the token walls", + "so grok streams no longer feed the token walls", + ) + self.assert_rejected( + "M48 no-longer-feed", grok_gap_errors(mutated)) + + def test_m49_pattern_swap_survives_assignment_reordering(self): + anchor = ("STORE=~/.grok/sessions # store root and stream" + " pattern for this\nPATTERN=updates.jsonl") + text = DOC.read_text(encoding="utf-8") + self.assertEqual(text.count(anchor), 1, + "the recipe assignment anchor moved") + mutated = text.replace( + anchor, + "PATTERN=events.jsonl\nSTORE=~/.grok/sessions # store" + " root and stream pattern for this") + self.assertNotEqual(mutated, text) + self.assert_rejected( + "M49 reordered pattern swap", battery_wiring_errors(mutated)) + + def test_m50_negated_stream_mention_is_a_warning_not_an_offer(self): + mutated = self.plant( + "M50 negated mention", + "| Grok | `updates.jsonl` in", + "| Grok | `updates.jsonl` (not `events.jsonl`) in", + ) + self.assertFalse( + wire_table_errors(mutated), + "a negated mention is a true warning and must stay quiet") + + def test_m51_inlined_discovery_filename_is_rejected(self): + mutated = self.plant( + "M51 inlined -name", + 'find "$STORE" -name "$PATTERN" -newer launched', + 'find "$STORE" -name events.jsonl -newer launched', + ) + self.assert_rejected( + "M51 inlined -name", battery_wiring_errors(mutated)) + + def test_m52_export_prefixed_pattern_swap_is_rejected(self): + mutated = self.plant( + "M52 export pattern", + "PATTERN=updates.jsonl", + "export PATTERN=events.jsonl", + ) + self.assert_rejected( + "M52 export pattern", battery_wiring_errors(mutated)) + + def test_m41_events_jsonl_token_promise_is_rejected(self): + mutated = self.plant( + "M41 Grok events alternative", + "| Grok | `updates.jsonl` in", + "| Grok | `updates.jsonl` (or `events.jsonl`) in", + ) + self.assert_rejected( + "M41 Grok events alternative", wire_table_errors(mutated)) + + def test_m4_grok_all_six_wires_are_rejected(self): + mutated = self.plant( + "M4 Grok all six", + GROK_ROW, + GROK_ROW.replace( + "| tokens, tokens-out, stall, size |", "| all six |"), + ) + self.assert_rejected("M4 Grok all six", wire_table_errors(mutated)) + + def test_m5_breaker_without_stream_discovery_is_rejected(self): + anchor = ( + 'STORE=~/.grok/sessions # store root and stream pattern for this\n' + 'PATTERN=updates.jsonl # reviewer\'s harness — see the table below\n' + 'until STREAM=$(find "$STORE" -name "$PATTERN" -newer launched \\\n' + " -printf '%T@ %p\\n' | sort -n | tail -1 | cut -d' ' -f2-); \\\n" + ' [ -n "$STREAM" ] || ! kill -0 "$RPID"; do sleep 2; done\n' + '[ -n "$STREAM" ] && python3 ops/devlane/telemetry/breaker.py "$STREAM" ' + '--pid "$RPID" \\\n' + ' --cap 2000000 --cap-out 150000 --stall 600 --size-mb 50 ' + '--terminate \\\n' + ' --tripped-file TRIPPED.md 2>> breaker.log &' + ) + replacement = ( + "python3 ops/devlane/telemetry/breaker.py /dev/null --pid " + '"$RPID" --stall 600' + ) + mutated = self.plant("M5 drop find", anchor, replacement) + self.assert_rejected("M5 drop find", battery_wiring_errors(mutated)) + + def test_m10_negated_trip_reporting_rule_is_rejected(self): + mutated = self.plant( + "M10 inverted trip rule", + "finish: name the tripped reviewer in the PR body", + "finish: do not name the tripped reviewer in the PR body", + ) + self.assert_rejected( + "M10 inverted trip rule", trip_reporting_errors(mutated) + ) + + def test_m14_claude_wrong_glob_is_rejected(self): + mutated = self.plant( + "M14 Claude glob", + CLAUDE_ROW, + CLAUDE_ROW.replace("`*.jsonl`", "`*.log`"), + ) + self.assert_rejected("M14 Claude glob", wire_table_errors(mutated)) + + def test_m15_codex_wrong_pattern_is_rejected(self): + mutated = self.plant( + "M15 Codex pattern", + CODEX_ROW, + CODEX_ROW.replace( + "~/.codex/sessions/*/*/*/rollout-*.jsonl", + "~/.codex/sessions/*.json", + ), + ) + self.assert_rejected("M15 Codex pattern", wire_table_errors(mutated)) + + def test_m16_grok_without_url_encoding_is_rejected(self): + mutated = self.plant( + "M16 Grok encoding", + GROK_ROW, + GROK_ROW.replace("", "$SNAP"), + ) + self.assert_rejected("M16 Grok encoding", wire_table_errors(mutated)) + + def test_wrapped_pid_remains_a_valid_logical_invocation(self): + anchor = ( + 'python3 ops/devlane/telemetry/breaker.py "$STREAM" --pid "$RPID" \\\n' + " --cap 2000000" + ) + replacement = ( + 'python3 ops/devlane/telemetry/breaker.py "$STREAM" \\\n' + ' --pid "$RPID" --cap 2000000' + ) + mutated = self.plant("M17 wrapped pid variant", anchor, replacement) + errors = battery_wiring_errors(mutated) + self.assertFalse( + errors, + "valid wrapped --pid invocation was rejected: " + ", ".join(errors), + ) + self.assertFalse( + breaker_flag_errors(mutated, BREAKER.read_text(encoding="utf-8")) + ) + + def test_m20_claude_stall_only_wires_are_rejected(self): + mutated = self.plant( + "M20 Claude wires", + CLAUDE_ROW, + CLAUDE_ROW.replace("| all six |", "| stall |"), + ) + self.assert_rejected("M20 Claude wires", wire_table_errors(mutated)) + + def test_m21_missing_pid_is_rejected(self): + mutated = self.plant( + "M21 missing pid", + 'breaker.py "$STREAM" --pid "$RPID" \\', + 'breaker.py "$STREAM" \\', + ) + self.assert_rejected("M21 missing pid", battery_wiring_errors(mutated)) + + def test_m23_trip_prose_without_tripped_file_is_rejected(self): + mutated = self.plant( + "M23 trip prose", + "the battery prints its evidence to stderr, writes\n" + "the TRIPPED file named by `--tripped-file` into the snapshot, and with", + "the battery prints its evidence to stderr, and with", + ) + self.assert_rejected("M23 trip prose", trip_reporting_errors(mutated)) + + def test_m26_missing_trip_exit_code_is_rejected(self): + mutated = self.plant( + "M26 exit code", + "A trip is exit code 3:", + "A trip is a nonzero exit:", + ) + self.assert_rejected("M26 exit code", trip_reporting_errors(mutated)) + + def test_m27_recipe_without_tripped_file_flag_is_rejected(self): + mutated = self.plant( + "M27 tripped flag", + " --tripped-file TRIPPED.md 2>> breaker.log &", + " 2>> breaker.log &", + ) + self.assert_rejected("M27 tripped flag", trip_reporting_errors(mutated)) + + def test_m28_prompt_substitution_in_launch_is_rejected(self): + mutated = self.plant( + "M28 prompt substitution", + "nohup grok --prompt-file prompt.txt", + 'nohup grok "$(cat prompt.txt)"', + ) + self.assert_rejected("M28 prompt substitution", launch_form_errors(mutated)) + + def test_m29_compound_launch_is_rejected(self): + mutated = self.plant( + "M29 compound launch", + "nohup grok --prompt-file prompt.txt", + 'cd "$SNAP" && nohup grok --prompt-file prompt.txt', + ) + self.assert_rejected("M29 compound launch", launch_form_errors(mutated)) + + def test_m30_late_launch_marker_is_rejected(self): + anchor = ( + "touch launched # marker: the supervisor finds the stream this launch opens\n" + "nohup grok --prompt-file prompt.txt --output-format plain --max-turns 40 \\" + ) + replacement = ( + "nohup grok --prompt-file prompt.txt --output-format plain --max-turns 40 \\\n" + "touch launched # marker: the supervisor finds the stream this launch opens" + ) + mutated = self.plant("M30 late marker", anchor, replacement) + self.assert_rejected("M30 late marker", launch_form_errors(mutated)) + + def test_m31_zero_input_cap_is_rejected(self): + mutated = self.plant( + "M31 zero cap", + "--cap 2000000", + "--cap 0", + ) + self.assert_rejected("M31 zero cap", battery_wiring_errors(mutated)) + + def test_m32_discovery_without_kill_zero_is_rejected(self): + mutated = self.plant( + "M32 kill mode", + 'kill -0 "$RPID"', + 'kill -9 "$RPID"', + ) + self.assert_rejected("M32 kill mode", battery_wiring_errors(mutated)) + + def test_m33_codex_wire_subset_is_rejected(self): + mutated = self.plant( + "M33 Codex subset", + CODEX_ROW, + CODEX_ROW.replace(", size |", " |"), + ) + self.assert_rejected("M33 Codex subset", wire_table_errors(mutated)) + + def test_m34_codex_wire_superset_is_rejected(self): + mutated = self.plant( + "M34 Codex superset", + CODEX_ROW, + CODEX_ROW.replace("stall, size |", "stall, size, rate |"), + ) + self.assert_rejected("M34 Codex superset", wire_table_errors(mutated)) + + def test_m37_negated_exit_three_is_rejected(self): + mutated = self.plant( + "M37 exit polarity", + "A trip is exit code 3:", + "A trip is not exit code 3:", + ) + self.assert_rejected("M37 exit polarity", trip_reporting_errors(mutated)) + + def test_m38_negated_tripped_file_write_is_rejected(self): + mutated = self.plant( + "M38 TRIPPED polarity", + "the battery prints its evidence to stderr, writes\n" + "the TRIPPED file", + "the battery prints its evidence to stderr, never writes\n" + "the TRIPPED file", + ) + self.assert_rejected( + "M38 TRIPPED polarity", trip_reporting_errors(mutated) + ) + + def test_m39_negated_kill_semantics_are_rejected(self): + mutated = self.plant( + "M39 kill polarity", + "it kills the runaway reviewer", + "it never kills the runaway reviewer", + ) + self.assert_rejected("M39 kill polarity", trip_reporting_errors(mutated)) + + +class RoundThreeSkepticShapeTests(unittest.TestCase): + """The round-3 skeptic bypasses and false-fails, pinned as units.""" + + GAP = ( + "Grok records spend the battery does not yet parse, so only the\n" + "vendor-agnostic wires (stall, size) can fire for it — supervision\n" + "still catches the hour-long hang.\n" + ) + + def test_kill_takes_the_reviewer_as_direct_object(self): + self.assertTrue(affirms_reviewer_kill("it kills the runaway reviewer")) + self.assertFalse(affirms_reviewer_kill( + "kills the wrapper while the reviewer runs on")) + self.assertFalse(affirms_reviewer_kill( + "kills the wrapper, not the reviewer")) + self.assertFalse(affirms_reviewer_kill( + "never kills the runaway reviewer")) + + def test_tripped_file_contrast_negation_is_caught(self): + self.assertTrue(affirms_tripped_file_write( + "writes the TRIPPED file named by --tripped-file")) + self.assertFalse(affirms_tripped_file_write( + "writes the log, not the TRIPPED file")) + + def test_comment_mentions_of_cat_are_not_launch_errors(self): + safe = ( + "```sh\n" + 'cd "$SNAP"\n' + "touch launched\n" + "nohup grok --prompt-file prompt.txt > review.txt 2> review.err &\n" + 'RPID=$!\n' + '# prompt via stdin, never "$(cat prompt.txt)":\n' + "```\n" + ) + self.assertFalse(launch_form_errors(safe)) + unsafe = safe.replace( + "--prompt-file prompt.txt", '"$(cat prompt.txt)"') + self.assertNotEqual(unsafe, safe) + self.assertTrue(launch_form_errors(unsafe)) + + def test_possessive_reviewer_is_not_the_kill_object(self): + self.assertFalse(affirms_reviewer_kill("kills the reviewer's wrapper")) + + def test_guarded_dev_null_once_and_dropped_rpid_are_rotted(self): + text = DOC.read_text(encoding="utf-8") + devnull = text.replace('breaker.py "$STREAM" --pid', "breaker.py /dev/null --pid", 1) + self.assertNotEqual(devnull, text) + self.assertTrue(battery_wiring_errors(devnull)) + onceshot = text.replace(' --pid "$RPID"', ' --once --pid "$RPID"', 1) + self.assertNotEqual(onceshot, text) + self.assertTrue(battery_wiring_errors(onceshot)) + no_pid_capture = text.replace("RPID=$!", "true", 1) + self.assertNotEqual(no_pid_capture, text) + self.assertTrue(launch_form_errors(no_pid_capture)) + + def test_commented_pid_and_probe_do_not_count(self): + text = DOC.read_text(encoding="utf-8") + hidden_pid = text.replace( + '--pid "$RPID" \\\n', '\\\n', 1).replace( + "2>> breaker.log &", '2>> breaker.log & # --pid "$RPID"', 1) + self.assertNotEqual(hidden_pid, text) + self.assertTrue(battery_wiring_errors(hidden_pid)) + hidden_probe = text.replace( + ' || ! kill -0 "$RPID"', "", 1).replace( + "do sleep 2; done", 'do sleep 2; done # kill -0 "$RPID"', 1) + self.assertNotEqual(hidden_probe, text) + self.assertTrue(battery_wiring_errors(hidden_probe)) + + def test_launch_must_enter_the_snapshot_first(self): + text = DOC.read_text(encoding="utf-8") + no_cd = text.replace('cd "$SNAP"', "true", 1) + self.assertNotEqual(no_cd, text) + self.assertTrue(launch_form_errors(no_cd)) + + def test_tripped_file_must_keep_evidence_and_resist_comments(self): + text = DOC.read_text(encoding="utf-8") + devnull = text.replace( + "--tripped-file TRIPPED.md", "--tripped-file /dev/null", 1) + self.assertNotEqual(devnull, text) + self.assertTrue(battery_wiring_errors(devnull)) + hidden = text.replace( + " --tripped-file TRIPPED.md", "", 1).replace( + "2>> breaker.log &", "2>> breaker.log & # --tripped-file TRIPPED.md", 1) + self.assertNotEqual(hidden, text) + self.assertTrue(battery_wiring_errors(hidden)) + + def test_armed_line_must_follow_the_discovery_wait(self): + text = DOC.read_text(encoding="utf-8") + lines = text.splitlines(keepends=True) + starts = [i for i, l in enumerate(lines) + if l.startswith('[ -n "$STREAM" ] && python3')] + self.assertEqual(len(starts), 1, "armed-line anchor moved") + start = starts[0] + end = start + 1 + while lines[end - 1].rstrip("\n").endswith("\\"): + end += 1 + armed_block = lines[start:end] + until = next(i for i, l in enumerate(lines) if l.startswith("until ")) + self.assertLess(until, start, "fixture assumption broke") + reordered = (lines[:until] + armed_block + + lines[until:start] + lines[end:]) + mutated = "".join(reordered) + self.assertNotEqual(mutated, text) + self.assertTrue(battery_wiring_errors(mutated)) + + def test_first_match_and_oldest_match_discovery_are_rotted(self): + text = DOC.read_text(encoding="utf-8") + pipeline = "-printf '%T@ %p\\n' | sort -n | tail -1 | cut -d' ' -f2-" + self.assertEqual(text.count(pipeline), 1, "pipeline anchor moved") + for wrong in ("-print -quit", ("-printf '%T@ %p\\n' | sort -n " + "| head -1 | cut -d' ' -f2-"), "-print | head -1"): + with self.subTest(wrong=wrong): + mutated = text.replace(pipeline, wrong, 1) + self.assertNotEqual(mutated, text) + self.assertTrue(battery_wiring_errors(mutated)) + + def test_absent_telemetry_claim_is_refused_doc_wide(self): + text = DOC.read_text(encoding="utf-8") + preceding = text.replace( + "Grok records its spend", + "Grok records no token usage anywhere in its store.\n\n" + "Grok records its spend", 1) + self.assertNotEqual(preceding, text) + self.assertTrue(grok_gap_errors(preceding)) + stale = text.replace( + "like the others.", + "like the others. The battery does not yet parse that shape.", 1) + self.assertNotEqual(stale, text) + self.assertTrue(grok_gap_errors(stale)) + no_spend = text.replace("Grok records its spend", + "Grok records no spend", 1) + self.assertNotEqual(no_spend, text) + self.assertTrue(grok_gap_errors(no_spend)) + + def test_caps_optionality_is_caught_in_any_block(self): + text = DOC.read_text(encoding="utf-8") + later = text.replace( + "Which stream, and which wires can fire", + "In practice --cap can be left unarmed.\n\n" + "Which stream, and which wires can fire", 1) + self.assertNotEqual(later, text) + self.assertTrue(caps_prose_errors(later)) + + def test_caps_prose_keeps_its_polarity(self): + text = DOC.read_text(encoding="utf-8") + self.assertFalse(caps_prose_errors(text)) + mutated = text.replace("must be armed explicitly", + "need not be armed explicitly", 1) + self.assertNotEqual(mutated, text) + self.assertTrue(caps_prose_errors(mutated)) + + def test_launch_must_end_backgrounded(self): + text = DOC.read_text(encoding="utf-8") + mutated = text.replace("> review.txt 2> review.err &", + "> review.txt 2> review.err", 1) + self.assertNotEqual(mutated, text) + self.assertTrue(launch_form_errors(mutated)) + + def test_discovery_handoff_polarity_is_pinned(self): + text = DOC.read_text(encoding="utf-8") + flips = ( + ("|| ! kill -0", "|| kill -0"), + ("|| ! kill -0", "&& ! kill -0"), + ('[ -n "$STREAM" ] || ! kill', '[ -z "$STREAM" ] || ! kill'), + ("until STREAM=$(find", "while STREAM=$(find"), + ('[ -n "$STREAM" ] && python3', '[ -z "$STREAM" ] && python3'), + ) + for old, new in flips: + with self.subTest(flip=new): + mutated = text.replace(old, new, 1) + self.assertNotEqual(mutated, text, old) + self.assertTrue(battery_wiring_errors(mutated)) + self.assertFalse(battery_wiring_errors(text)) + + def test_armed_invocation_cannot_disable_the_agnostic_wires(self): + text = DOC.read_text(encoding="utf-8") + armed_anchor = "--size-mb 50 --terminate" + self.assertEqual(text.count(armed_anchor), 1, + "the armed-line anchor no longer matches the doc") + for form in ("--disable stall", "--disable size", + "--disable stall,size", "--disable=stall", + "--disable tokens", "--disable tokens,tokens-out", + "--disable=tokens"): + with self.subTest(form=form): + mutated = text.replace( + armed_anchor, f"{armed_anchor} {form}", 1) + self.assertNotEqual(mutated, text) + self.assertTrue(battery_wiring_errors(mutated)) + + def test_armed_pid_must_be_the_captured_rpid(self): + text = DOC.read_text(encoding="utf-8") + for wrong in ('--pid 1', '--pid 0', '--pid "$PPID"', '--pid "$PID"'): + with self.subTest(pid=wrong): + mutated = text.replace('--pid "$RPID"', wrong, 1) + self.assertNotEqual(mutated, text) + self.assertTrue(battery_wiring_errors(mutated)) + + def test_commented_terminate_is_still_a_rotted_invocation(self): + text = DOC.read_text(encoding="utf-8") + mutated = text.replace(" --terminate", " # --terminate", 1) + self.assertNotEqual(mutated, text) + self.assertTrue(battery_wiring_errors(mutated)) + + def test_dropping_terminate_from_the_armed_invocation_is_rotted(self): + text = DOC.read_text(encoding="utf-8") + self.assertFalse(battery_wiring_errors(text)) + mutated = text.replace(" --terminate", " ", 1) + self.assertNotEqual(mutated, text) + self.assertTrue(battery_wiring_errors(mutated)) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/telemetry/tests/test_worth.py b/ops/devlane/telemetry/tests/test_worth.py new file mode 100644 index 0000000..c125d17 --- /dev/null +++ b/ops/devlane/telemetry/tests/test_worth.py @@ -0,0 +1,1699 @@ +from __future__ import annotations + +import importlib.util +import json +import os +import re +import subprocess +import sys +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import quote + +HERE = Path(__file__).resolve() +APP = HERE.parents[2] +WORTH = HERE.parents[1] / "worth.py" +STORES_PATH = APP / "fixtures" / "stores.py" + +SPEC = importlib.util.spec_from_file_location("worth_test_stores", STORES_PATH) +if SPEC is None or SPEC.loader is None: + raise RuntimeError(f"cannot load fixture API from {STORES_PATH}") +STORES = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(STORES) + +NOW = "2026-08-22T12:00:00.000Z" +DEFAULT_SINCE = "2026-08-21T12:00:00.000Z" +WINDOW_SINCE = "2026-08-22T12:00:00.000Z" +WINDOW_UNTIL = "2026-08-22T12:05:00.000Z" + + +def iso_epoch(value): + return datetime.fromisoformat(value).timestamp() + + +def git_date(value): + stamp = datetime.fromisoformat(value) + return stamp.astimezone(timezone.utc).isoformat(timespec="seconds") + + +def canonical_key(value): + return re.sub(r"[^a-z0-9]", "", str(value).lower()) + + +def json_lines(path): + return [ + json.loads(raw) + for raw in path.read_text(encoding="utf-8").splitlines() + if raw.strip() + ] + + +def walk_leaves(value, path=()): + if isinstance(value, dict): + for key, child in value.items(): + yield from walk_leaves(child, path + (str(key),)) + elif isinstance(value, list): + for index, child in enumerate(value): + yield from walk_leaves(child, path + (str(index),)) + else: + yield path, value + + +def walk_dicts(value): + if isinstance(value, dict): + yield value + for child in value.values(): + yield from walk_dicts(child) + elif isinstance(value, list): + for child in value: + yield from walk_dicts(child) + + +class WorthContractTests(unittest.TestCase): + maxDiff = None + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix="worth-contract-") + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.repo = self.root / "repo" + self.repo.mkdir() + + self.claude = self.root / "claude" + self.codex = self.root / "codex" + self.grok = self.root / "grok" + + self._git("init", "-q", "-b", "dev") + self._git("config", "user.name", "Fixture Owner") + self._git("config", "user.email", "fixture@example.invalid") + self.initial = self._commit_files( + {"README.md": "throwaway worth fixture\n"}, + "initial fixture", + "2026-08-20T10:00:00.000Z", + ) + + def _git_env(self, when=None): + env = os.environ.copy() + env.update({ + "LC_ALL": "C", + "TZ": "UTC", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + }) + if when is not None: + fixed = git_date(when) + env["GIT_AUTHOR_DATE"] = fixed + env["GIT_COMMITTER_DATE"] = fixed + return env + + def _git(self, *args, when=None, check=True): + proc = subprocess.run( + ["git", *args], + cwd=self.repo, + env=self._git_env(when), + capture_output=True, + text=True, + check=False, + ) + if check: + self.assertEqual( + proc.returncode, + 0, + f"git {' '.join(args)} failed\nstdout:\n{proc.stdout}" + f"\nstderr:\n{proc.stderr}", + ) + return proc + + def _write_repo_file(self, relative, content): + path = self.repo / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + self.assertEqual(path.read_text(encoding="utf-8"), content) + return path + + def _assert_commit(self, sha, subject, when, parents=None): + record = self._git( + "show", + "-s", + "--format=%H%x00%s%x00%aI%x00%cI%x00%P", + sha, + ).stdout.rstrip("\n").split("\x00") + self.assertEqual(record[0], sha) + self.assertEqual(record[1], subject) + # git 2.51 prints UTC %aI/%cI with a Z suffix, older gits + # with +00:00 — compare instants, not spellings + for observed in (record[2], record[3]): + self.assertEqual( + datetime.fromisoformat(observed), + datetime.fromisoformat(git_date(when)), + ) + if parents is not None: + self.assertEqual(record[4].split(), list(parents)) + self.assertEqual(self._git("cat-file", "-t", sha).stdout.strip(), "commit") + + def _commit_files(self, files, subject, when): + for relative, content in files.items(): + self._write_repo_file(relative, content) + self._git("add", "-A") + self._git("commit", "-q", "-m", subject, when=when) + sha = self._git("rev-parse", "HEAD").stdout.strip() + self._assert_commit(sha, subject, when) + for relative, content in files.items(): + landed = self._git("show", f"{sha}:{relative}").stdout + self.assertEqual(landed, content) + return sha + + def _merge(self, branch, subject, when): + first_parent = self._git("rev-parse", "HEAD").stdout.strip() + second_parent = self._git("rev-parse", branch).stdout.strip() + self._git("merge", "-q", "--no-ff", "-m", subject, branch, when=when) + sha = self._git("rev-parse", "HEAD").stdout.strip() + self._assert_commit( + sha, + subject, + when, + parents=(first_parent, second_parent), + ) + self.assertEqual( + self._git("merge-base", "--is-ancestor", second_parent, sha).returncode, + 0, + ) + return sha + + def _slug(self, repo=None): + repo = str(repo or self.repo) + return "-" + "-".join(repo.strip("/").split("/")) + + def _write_jsonl(self, path, entries): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "".join(json.dumps(entry, sort_keys=True) + "\n" for entry in entries), + encoding="utf-8", + ) + planted = json_lines(path) + self.assertEqual(len(planted), len(entries)) + self.assertEqual(planted, entries) + return path + + def _plant_claude(self, session_id, messages, repo=None, marker=None): + repo = str(repo or self.repo) + marker = marker or f"marker-{session_id}" + entries = [] + for index, message in enumerate(messages, 1): + usage = { + "input_tokens": message.get("input", 0), + "cache_creation_input_tokens": message.get( + "cache_creation", 0 + ), + "cache_read_input_tokens": message.get("cache_read", 0), + "output_tokens": message.get("output", 0), + } + entries.append(STORES.claude_entry( + timestamp=message["timestamp"], + cwd=repo, + session_id=session_id, + model=message.get("model", "claude-fable-5"), + effort="high", + mid=message.get("id", f"{session_id}-msg-{index}"), + usage=usage, + content=[{"type": "text", "text": marker}], + )) + path = self.claude / self._slug(repo) / f"{session_id}.jsonl" + self._write_jsonl(path, entries) + planted = json_lines(path) + self.assertEqual( + [row["message"]["id"] for row in planted], + [row["message"]["id"] for row in entries], + ) + self.assertTrue(all(marker in json.dumps(row) for row in planted)) + return path + + def _plant_standard_claude( + self, + session_id, + base, + repo=None, + replacement_usage=None, + ): + repo = str(repo or self.repo) + marker = f"standard-marker-{session_id}" + STORES.build_claude_store( + self.claude, + self._slug(repo), + base_timestamp=iso_epoch(base), + cwd=repo, + session_id=session_id, + model="claude-fable-5", + effort="high", + marker=marker, + reemit_last=replacement_usage is not None, + ) + path = self.claude / self._slug(repo) / f"{session_id}.jsonl" + planted = json_lines(path) + expected_count = 3 if replacement_usage is not None else 2 + self.assertEqual(len(planted), expected_count) + self.assertTrue(all(marker in json.dumps(row) for row in planted)) + if replacement_usage is not None: + planted[-1]["timestamp"] = replacement_usage["timestamp"] + planted[-1]["message"]["usage"] = { + "input_tokens": replacement_usage.get("input", 0), + "cache_creation_input_tokens": replacement_usage.get( + "cache_creation", 0 + ), + "cache_read_input_tokens": replacement_usage.get( + "cache_read", 0 + ), + "output_tokens": replacement_usage.get("output", 0), + } + self._write_jsonl(path, planted) + reread = json_lines(path) + self.assertEqual( + reread[-1]["message"]["id"], + reread[-2]["message"]["id"], + "the planted replacement must exercise last-wins by id", + ) + self.assertEqual( + reread[-1]["message"]["usage"], + planted[-1]["message"]["usage"], + ) + return path + + def _plant_standard_codex( + self, + session_id, + base, + repo=None, + last_usage=None, + ): + repo = str(repo or self.repo) + marker = f"standard-marker-{session_id}" + STORES.build_codex_store( + self.codex, + base_timestamp=iso_epoch(base), + cwd=repo, + session_id=session_id, + model="gpt-5-codex", + effort="high", + marker=marker, + ) + matches = list(self.codex.rglob(f"*{session_id}.jsonl")) + self.assertEqual(len(matches), 1) + path = matches[0] + planted = json_lines(path) + self.assertEqual(len(planted), 5) + self.assertEqual(planted[0]["payload"]["id"], session_id) + self.assertIn(marker, path.read_text(encoding="utf-8")) + token_rows = [ + row for row in planted + if (row.get("payload") or {}).get("type") == "token_count" + ] + self.assertEqual(len(token_rows), 2) + if last_usage is not None: + token_rows[-1]["payload"]["info"]["total_token_usage"] = last_usage + token_index = max( + index for index, row in enumerate(planted) + if (row.get("payload") or {}).get("type") == "token_count" + ) + planted[token_index] = token_rows[-1] + self._write_jsonl(path, planted) + reread = json_lines(path) + self.assertEqual( + reread[token_index]["payload"]["info"]["total_token_usage"], + last_usage, + ) + return path + + def _plant_grok(self, session_id, base, usage_runs=None, repo=None): + repo = str(repo or self.repo) + marker = f"standard-marker-{session_id}" + usage_runs = usage_runs or [] + STORES.build_grok_store( + self.grok, + repo, + base_timestamp=iso_epoch(base), + session_id=session_id, + model="grok-4.6", + marker=marker, + usage_runs=usage_runs, + ) + session = ( + self.grok / "sessions" / quote(repo, safe="") / session_id + ) + summary = json.loads( + (session / "summary.json").read_text(encoding="utf-8") + ) + updates = json_lines(session / "updates.jsonl") + events = json_lines(session / "events.jsonl") + self.assertEqual(summary["info"]["id"], session_id) + self.assertEqual(summary["info"]["cwd"], repo) + self.assertIn(marker, json.dumps(summary)) + self.assertEqual(len(updates), 2 + sum(map(len, usage_runs))) + self.assertEqual(len(events), 7) + planted_usage = [ + row["params"]["update"]["usage"] + for row in updates + if row["params"]["update"].get("sessionUpdate") == "turn_completed" + ] + self.assertEqual( + planted_usage, + [usage for run in usage_runs for usage in run], + ) + return session + + def _run_worth(self, verb, *args, now=NOW, repo=None): + repo = str(repo or self.repo) + cmd = [ + sys.executable, + str(WORTH), + verb, + *args, + "--now", + now, + "--repo", + repo, + "--claude-dir", + str(self.claude), + "--codex-dir", + str(self.codex), + "--grok-dir", + str(self.grok), + ] + self.assertEqual(cmd.count("--now"), 1) + return subprocess.run( + cmd, + cwd=self.root, + env={ + **os.environ, + "LC_ALL": "C", + "TZ": "Etc/GMT+11", + "HOME": str(self.root / "empty-home"), + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + }, + capture_output=True, + text=True, + check=False, + ) + + def _json_worth(self, verb, *args, now=NOW, repo=None): + proc = self._run_worth( + verb, + *args, + "--format", + "json", + now=now, + repo=repo, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + try: + value = json.loads(proc.stdout) + except ValueError as exc: + self.fail(f"invalid JSON: {exc}\nstdout:\n{proc.stdout}") + self.assertIsInstance(value, dict) + return value + + def _harness_record(self, data, harness): + container = data["cost"] + wanted = canonical_key(harness) + + def find(value): + if isinstance(value, dict): + for key, child in value.items(): + if canonical_key(key) == wanted: + return child + if canonical_key(value.get("harness", "")) == wanted: + return value + for child in value.values(): + found = find(child) + if found is not None: + return found + elif isinstance(value, list): + for child in value: + found = find(child) + if found is not None: + return found + return None + + record = find(container) + self.assertIsNotNone(record, f"no cost record for {harness}: {container}") + return record + + def _line_for_harness(self, plain, harness): + lines = [ + line for line in plain.splitlines() + if re.search(rf"\b{re.escape(harness)}\b", line, re.IGNORECASE) + ] + self.assertTrue(lines, f"no plain-format line for {harness}:\n{plain}") + return lines[0] + + def _assert_key_number(self, value, aliases, expected): + wanted = {canonical_key(alias) for alias in aliases} + matches = [ + leaf for path, leaf in walk_leaves(value) + if path + and canonical_key(path[-1]) in wanted + and isinstance(leaf, (int, float)) + and not isinstance(leaf, bool) + ] + self.assertIn( + expected, + matches, + f"expected {aliases}={expected}; observed {matches} in {value}", + ) + + def _assert_plain_number(self, line, aliases, expected): + names = "|".join(re.escape(alias) for alias in aliases) + self.assertRegex( + line, + rf"\b(?:{names})\s*[=:]\s*{re.escape(str(expected))}\b", + ) + + def _session_ids(self, data): + sessions = data["sessions"] + self.assertIsInstance(sessions, list) + + ids = [] + for row in sessions: + self.assertIsInstance(row, dict) + found = None + for path, value in walk_leaves(row): + if ( + path + and canonical_key(path[-1]) in {"session", "sessionid", "id"} + and isinstance(value, str) + ): + found = value + break + self.assertIsNotNone(found, f"ranked session lacks an id: {row}") + ids.append(found) + return ids + + def _signal_kind(self, signal): + for path, value in walk_leaves(signal): + if ( + path + and canonical_key(path[-1]) in {"kind", "signal", "type"} + and isinstance(value, str) + ): + return value + rendered = json.dumps(signal, sort_keys=True) + for kind in ("cache-churn", "heavy-turn"): + if kind in rendered: + return kind + return None + + def _signal_field(self, signal, aliases): + wanted = {canonical_key(alias) for alias in aliases} + for path, value in walk_leaves(signal): + if path and canonical_key(path[-1]) in wanted: + return value + return None + + def _signals_of_kind(self, data, kind): + signals = data["signals"] + self.assertIsInstance(signals, list) + return [ + signal for signal in signals + if self._signal_kind(signal) == kind + ] + + def _assert_signal( + self, + data, + kind, + harness, + session_id, + required_values=(), + ): + matches = [] + for signal in self._signals_of_kind(data, kind): + if ( + self._signal_field(signal, ("harness",)) == harness + and self._signal_field( + signal, ("session", "session_id", "id") + ) == session_id + ): + matches.append(signal) + self.assertEqual( + len(matches), + 1, + f"expected one {kind} for {harness}/{session_id}: {data['signals']}", + ) + rendered = json.dumps(matches[0], sort_keys=True) + for value in required_values: + self.assertIn(str(value), rendered) + return matches[0] + + def _assert_stamp(self, plain, data, since, until, now): + self.assertIn("stamp", data) + stamp = json.dumps(data["stamp"], sort_keys=True) + head = self._git("rev-parse", "--short", "HEAD").stdout.strip() + for expected in (head, "dev", since, until, now): + self.assertIn(expected, stamp) + self.assertIn(expected, plain) + self.assertRegex( + plain, + rf"\[{re.escape(since)}\s*,\s*{re.escape(until)}\)", + ) + + def _assert_plain_json_parity(self, plain, data): + for path, value in walk_leaves(data): + if value is None: + self.assertIn("unrecorded", plain.lower()) + elif isinstance(value, bool): + if value and path: + self.assertIn(path[-1].replace("_", "-"), plain.lower()) + elif value != "": + self.assertIn( + str(value), + plain, + f"JSON-only value at {'.'.join(path)}: {value!r}", + ) + + rendered = json.dumps(data, sort_keys=True) + for label, figure in re.findall( + r"\b([A-Za-z][A-Za-z0-9_-]*)\s*=\s*" + r"(unrecorded|-?\d+(?:/\d+)?)\b", + plain, + ): + self.assertIn(label.replace("-", "").replace("_", "").lower(), + canonical_key(rendered)) + if figure != "unrecorded": + for part in figure.split("/"): + self.assertIn(part, rendered) + + for stamp in re.findall( + r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?" + r"(?:Z|[+-]\d\d:\d\d)", + plain, + ): + self.assertIn(stamp, rendered) + for pr in re.findall(r"#(\d+)", plain): + self.assertIn(pr, rendered) + for sha in re.findall(r"\b[0-9a-f]{7,40}\b", plain): + self.assertIn(sha, rendered) + + def _pr_numbers(self, results): + found = set() + + def visit(value, in_pr=False): + if isinstance(value, dict): + for key, child in value.items(): + key_name = canonical_key(key) + child_in_pr = in_pr or key_name in { + "pr", + "prs", + "prnumber", + "prnumbers", + "pullrequest", + "pullrequests", + } + visit(child, child_in_pr) + elif isinstance(value, list): + for child in value: + visit(child, in_pr) + elif in_pr: + if isinstance(value, int) and not isinstance(value, bool): + found.add(value) + elif isinstance(value, str): + for number in re.findall( + r"(?:pull request\s*)?#?(\d+)", value, re.IGNORECASE + ): + found.add(int(number)) + + visit(results) + return found + + def _assert_result_measure_zero(self, results, fragments): + fragments = tuple(canonical_key(item) for item in fragments) + candidates = [] + for node in walk_dicts(results): + for key, value in node.items(): + name = canonical_key(key) + if any( + name == fragment + or name.startswith(fragment) + or fragment in name + for fragment in fragments + ): + candidates.append(value) + + def contains_zero(value): + if value == 0 or value == []: + return True + if isinstance(value, dict): + return any(contains_zero(child) for child in value.values()) + if isinstance(value, list): + return len(value) == 0 or any( + contains_zero(child) for child in value + ) + return False + + self.assertTrue( + any(contains_zero(value) for value in candidates), + f"no zero-valued result measure for {fragments}: {results}", + ) + + def test_default_window_boundaries_and_independent_edge_overrides(self): + self._plant_claude("boundary-session", [ + { + "id": "below-lower", + "timestamp": "2026-08-21T11:59:59.000Z", + "output": 1, + }, + { + "id": "at-lower", + "timestamp": DEFAULT_SINCE, + "output": 2, + }, + { + "id": "at-early-until", + "timestamp": "2026-08-21T13:00:00.000Z", + "output": 16, + }, + { + "id": "inside-upper", + "timestamp": "2026-08-22T11:59:59.000Z", + "output": 4, + }, + { + "id": "at-upper", + "timestamp": NOW, + "output": 8, + }, + ]) + + default = self._json_worth("report") + claude = self._harness_record(default, "claude") + self._assert_key_number(claude, ("messages",), 3) + self._assert_key_number(claude, ("output", "out"), 22) + + since_only = self._json_worth( + "report", + "--since", + "2026-08-22T00:00:00.000Z", + ) + claude = self._harness_record(since_only, "claude") + self._assert_key_number(claude, ("messages",), 1) + self._assert_key_number(claude, ("output", "out"), 4) + + until_only = self._json_worth( + "report", + "--until", + "2026-08-21T13:00:00.000Z", + ) + claude = self._harness_record(until_only, "claude") + self._assert_key_number(claude, ("messages",), 1) + self._assert_key_number(claude, ("output", "out"), 2) + + def test_claude_and_codex_window_per_message_but_grok_windows_per_run(self): + # The standard store's second message rides 300s after base; + # this base puts it at 12:00:10 — inside [12:00:00, 12:00:15) + # — while the first stays outside, which is the straddle the + # expected figures (msg2 alone) always described. The authored + # base of 11:59:00 left BOTH messages outside the window. + self._plant_standard_claude( + "claude-straddle", + "2026-08-22T11:55:10.000Z", + ) + self._plant_standard_codex( + "codex-straddle", + "2026-08-22T11:59:00.000Z", + ) + + included_run = [[ + { + "inputTokens": 60, + "cachedReadTokens": 20, + "cacheCreationTokens": 5, + "outputTokens": 15, + "totalTokens": 100, + "reasoningTokens": 3, + "costUsdTicks": 10, + }, + { + "inputTokens": 120, + "cachedReadTokens": 40, + "cacheCreationTokens": 10, + "outputTokens": 30, + "totalTokens": 200, + "reasoningTokens": 6, + "costUsdTicks": 20, + }, + ]] + excluded_run = [[ + { + "inputTokens": 1, + "cachedReadTokens": 2, + "cacheCreationTokens": 3, + "outputTokens": 40_000, + "totalTokens": 40_006, + "reasoningTokens": 4, + "costUsdTicks": 40_010, + }, + { + "inputTokens": 2, + "cachedReadTokens": 3, + "cacheCreationTokens": 4, + "outputTokens": 90_000, + "totalTokens": 90_009, + "reasoningTokens": 5, + "costUsdTicks": 90_010, + }, + ]] + self._plant_grok( + "grok-last-inside", + "2026-08-22T11:58:00.000Z", + included_run, + ) + self._plant_grok( + "grok-last-outside", + "2026-08-22T11:58:30.000Z", + excluded_run, + ) + + report = self._json_worth( + "report", + "--since", + WINDOW_SINCE, + "--until", + "2026-08-22T12:00:15.000Z", + now="2026-08-22T13:00:00.000Z", + ) + + claude = self._harness_record(report, "claude") + self._assert_key_number(claude, ("sessions",), 1) + self._assert_key_number(claude, ("messages",), 1) + self._assert_key_number(claude, ("input", "in"), 20) + self._assert_key_number(claude, ("cached",), 6_000) + self._assert_key_number(claude, ("output", "out"), 300) + self._assert_key_number(claude, ("total",), 6_320) + + codex = self._harness_record(report, "codex") + self._assert_key_number(codex, ("sessions",), 1) + self._assert_key_number(codex, ("messages",), 1) + self._assert_key_number(codex, ("input", "in"), 200) + self._assert_key_number(codex, ("cached",), 100) + self._assert_key_number(codex, ("output", "out"), 40) + self._assert_key_number(codex, ("total",), 240) + + grok = self._harness_record(report, "grok") + self._assert_key_number(grok, ("sessions",), 1) + self._assert_key_number(grok, ("runs",), 1) + self._assert_key_number(grok, ("input", "in"), 120) + self._assert_key_number(grok, ("cached",), 50) + self._assert_key_number(grok, ("output", "out"), 30) + self._assert_key_number(grok, ("total",), 200) + self.assertNotIn("messages", canonical_key(json.dumps(grok))) + self.assertNotIn("90000", json.dumps(report)) + + waste = self._json_worth( + "waste", + "--since", + WINDOW_SINCE, + "--until", + "2026-08-22T12:00:15.000Z", + "--top", + "10", + now="2026-08-22T13:00:00.000Z", + ) + ids = self._session_ids(waste) + self.assertIn("claude-straddle", ids) + self.assertIn("codex-straddle", ids) + self.assertIn("grok-last-inside", ids) + self.assertNotIn("grok-last-outside", ids) + + def test_report_reuses_usage_accounting_and_preserves_measurement_gaps(self): + self._plant_standard_claude( + "claude-accounting", + "2026-08-22T10:00:00.000Z", + replacement_usage={ + "timestamp": "2026-08-22T10:10:00.000Z", + "input": 7, + "cache_creation": 11, + "cache_read": 13, + "output": 17, + }, + ) + self._plant_standard_codex( + "codex-accounting", + "2026-08-22T10:00:00.000Z", + ) + + grok_runs = [ + [ + { + "inputTokens": 60, + "cachedReadTokens": 20, + "cacheCreationTokens": 5, + "outputTokens": 15, + "totalTokens": 100, + "reasoningTokens": 3, + "costUsdTicks": 10, + }, + { + "inputTokens": 120, + "cachedReadTokens": 40, + "cacheCreationTokens": 10, + "outputTokens": 30, + "totalTokens": 200, + "reasoningTokens": 6, + "costUsdTicks": 20, + }, + ], + [ + { + "inputTokens": 30, + "cachedReadTokens": 10, + "cacheCreationTokens": 2, + "outputTokens": 8, + "totalTokens": 50, + "reasoningTokens": 1, + "costUsdTicks": 5, + }, + { + "inputTokens": 45, + "cachedReadTokens": 15, + "cacheCreationTokens": 3, + "outputTokens": 17, + "totalTokens": 80, + "reasoningTokens": 2, + "costUsdTicks": 8, + "usageIsIncomplete": True, + }, + ], + ] + self._plant_grok( + "grok-accounting", + "2026-08-22T10:00:00.000Z", + grok_runs, + ) + self._plant_grok( + "grok-unrecorded", + "2026-08-22T10:30:00.000Z", + [], + ) + + other_repo = self.root / "other-repo" + other_repo.mkdir() + self.assertTrue(other_repo.is_dir()) + self._plant_claude( + "other-claude", + [{ + "timestamp": "2026-08-22T10:00:00.000Z", + "input": 888_001, + "cache_read": 888_002, + "output": 888_003, + }], + repo=other_repo, + ) + self._plant_standard_codex( + "other-codex", + "2026-08-22T10:00:00.000Z", + repo=other_repo, + last_usage={ + "input_tokens": 777_001, + "cached_input_tokens": 777_002, + "output_tokens": 777_003, + "reasoning_output_tokens": 777_004, + "total_tokens": 777_005, + }, + ) + self._plant_grok( + "other-grok", + "2026-08-22T10:00:00.000Z", + [[{ + "inputTokens": 999_001, + "cachedReadTokens": 999_002, + "cacheCreationTokens": 999_003, + "outputTokens": 999_004, + "totalTokens": 999_005, + "reasoningTokens": 999_006, + "costUsdTicks": 999_007, + }]], + repo=other_repo, + ) + + plain_proc = self._run_worth("report") + self.assertEqual(plain_proc.returncode, 0, plain_proc.stderr) + data = self._json_worth("report") + + claude = self._harness_record(data, "claude") + self._assert_key_number(claude, ("sessions",), 1) + self._assert_key_number(claude, ("messages",), 2) + self._assert_key_number(claude, ("input", "in"), 17) + self._assert_key_number(claude, ("cached",), 6_024) + self._assert_key_number(claude, ("output", "out"), 217) + self._assert_key_number(claude, ("total",), 6_258) + + codex = self._harness_record(data, "codex") + self._assert_key_number(codex, ("sessions",), 1) + self._assert_key_number(codex, ("messages",), 2) + self._assert_key_number(codex, ("input", "in"), 400) + self._assert_key_number(codex, ("cached",), 300) + self._assert_key_number(codex, ("output", "out"), 90) + self._assert_key_number(codex, ("total",), 490) + + grok = self._harness_record(data, "grok") + self._assert_key_number(grok, ("sessions",), 2) + self._assert_key_number(grok, ("runs",), 2) + self._assert_key_number(grok, ("input", "in"), 165) + self._assert_key_number(grok, ("cached",), 68) + self._assert_key_number(grok, ("output", "out"), 47) + self._assert_key_number(grok, ("total",), 280) + self._assert_key_number( + grok, + ("cost_usd_ticks", "costUsdTicks"), + 28, + ) + + grok_rendered = json.dumps(grok, sort_keys=True).lower() + self.assertIn("counted", grok_rendered) + self.assertTrue( + "1/2" in grok_rendered + or ( + any( + value == 1 + for path, value in walk_leaves(grok) + if path and canonical_key(path[-1]) == "counted" + ) + and any( + value == 2 + for path, value in walk_leaves(grok) + if path and canonical_key(path[-1]) == "sessions" + ) + ) + ) + self.assertTrue( + any( + bool(value) + for path, value in walk_leaves(grok) + if path and "incomplete" in canonical_key(path[-1]) + ), + f"incompleteness missing from JSON: {grok}", + ) + + claude_line = self._line_for_harness(plain_proc.stdout, "claude") + self._assert_plain_number(claude_line, ("messages",), 2) + self._assert_plain_number(claude_line, ("in", "input"), 17) + self._assert_plain_number(claude_line, ("cached",), 6_024) + self._assert_plain_number(claude_line, ("out", "output"), 217) + + codex_line = self._line_for_harness(plain_proc.stdout, "codex") + self._assert_plain_number(codex_line, ("messages",), 2) + self._assert_plain_number(codex_line, ("total",), 490) + + grok_line = self._line_for_harness(plain_proc.stdout, "grok") + self._assert_plain_number(grok_line, ("runs",), 2) + self.assertIn("counted=1/2", grok_line) + self.assertIn("(incomplete)", grok_line) + self._assert_plain_number( + grok_line, + ("cost_usd_ticks", "cost-ticks"), + 28, + ) + + combined = plain_proc.stdout + json.dumps(data, sort_keys=True) + for forbidden in ( + "other-claude", + "other-codex", + "other-grok", + "888001", + "777005", + "999007", + ): + self.assertNotIn(forbidden, combined) + + def test_absent_and_unparseable_stores_are_unrecorded_never_zero(self): + """Scenario: an absent store is unrecorded, never zero""" + absent_plain = self._run_worth("report") + self.assertEqual(absent_plain.returncode, 0, absent_plain.stderr) + absent_json = self._json_worth("report") + + for harness in ("claude", "codex", "grok"): + record = self._harness_record(absent_json, harness) + self.assertIn("unrecorded", json.dumps(record).lower()) + line = self._line_for_harness(absent_plain.stdout, harness) + self.assertIn("unrecorded", line.lower()) + self.assertNotRegex( + line, + r"\b(?:tokens?|in|input|cached|out|output|total|" + r"cost(?:_usd_ticks)?)\s*[=:]\s*0\b", + ) + + malformed_claude = ( + self.claude / self._slug() / "malformed-claude.jsonl" + ) + malformed_claude.parent.mkdir(parents=True, exist_ok=True) + malformed_claude.write_text("{not-json\n", encoding="utf-8") + self.assertEqual( + malformed_claude.read_text(encoding="utf-8"), + "{not-json\n", + ) + self.assertEqual(len(malformed_claude.read_text().splitlines()), 1) + + malformed_codex = ( + self.codex / "sessions" / "2026" / "08" / "22" + / "rollout-malformed-codex.jsonl" + ) + malformed_codex.parent.mkdir(parents=True, exist_ok=True) + malformed_codex.write_text("[not-json\n", encoding="utf-8") + self.assertEqual( + malformed_codex.read_text(encoding="utf-8"), + "[not-json\n", + ) + self.assertEqual(len(malformed_codex.read_text().splitlines()), 1) + + malformed_grok = ( + self.grok / "sessions" / quote(str(self.repo), safe="") + / "malformed-grok" + ) + malformed_grok.mkdir(parents=True) + (malformed_grok / "summary.json").write_text( + "not-json", + encoding="utf-8", + ) + self.assertEqual( + (malformed_grok / "summary.json").read_text(encoding="utf-8"), + "not-json", + ) + self.assertEqual( + len(list(malformed_grok.iterdir())), + 1, + "the malformed Grok fixture did not land as planted", + ) + + malformed_plain = self._run_worth("report") + self.assertEqual(malformed_plain.returncode, 0, malformed_plain.stderr) + malformed_json = self._json_worth("report") + + for harness in ("claude", "codex", "grok"): + record = self._harness_record(malformed_json, harness) + self.assertIn("unrecorded", json.dumps(record).lower()) + line = self._line_for_harness(malformed_plain.stdout, harness) + self.assertIn("unrecorded", line.lower()) + self.assertNotRegex( + line, + r"\b(?:tokens?|in|input|cached|out|output|total|" + r"cost(?:_usd_ticks)?)\s*[=:]\s*0\b", + ) + + def test_results_use_first_parent_git_history_and_edge_trees_only(self): + one_test = ( + "#[test]\n" + "fn baseline_definition() {}\n" + ) + two_tests = ( + one_test + + "\n#[test]\n" + "fn pr_definition() {}\n" + ) + three_tests = ( + two_tests + + "\n#[test]\n" + "fn integration_definition() {}\n" + ) + four_tests = ( + three_tests + + "\n#[test]\n" + "fn uncommitted_definition() {}\n" + ) + + self._commit_files( + {"tests/contract.rs": one_test}, + "plant baseline test definition", + "2026-08-21T10:00:00.000Z", + ) + + self._git("switch", "-q", "-c", "pr-41") + pr_commit = self._commit_files( + { + "tests/contract.rs": two_tests, + "src/pr41.txt": "numbered merge content\n", + }, + "implement numbered fixture", + "2026-08-21T13:00:00.000Z", + ) + self._git("switch", "-q", "dev") + numbered_merge = self._merge( + "pr-41", + "Merge pull request #41 from fixtures/pr-41", + "2026-08-21T13:30:00.000Z", + ) + self.assertEqual( + self._git("show", f"{numbered_merge}:tests/contract.rs").stdout, + two_tests, + ) + self.assertEqual( + self._git("merge-base", "--is-ancestor", pr_commit, numbered_merge) + .returncode, + 0, + ) + + self._git("switch", "-q", "-c", "integration") + integration_commit = self._commit_files( + { + "tests/contract.rs": three_tests, + "src/integration.txt": "unnumbered merge content\n", + }, + "implement integration fixture", + "2026-08-21T14:00:00.000Z", + ) + self._git("switch", "-q", "-c", "nested") + nested_commit = self._commit_files( + {"src/nested.txt": "second-parent-only merge content\n"}, + "implement nested fixture", + "2026-08-21T14:10:00.000Z", + ) + self._git("switch", "-q", "integration") + hidden_merge = self._merge( + "nested", + "Merge pull request #999 from fixtures/nested", + "2026-08-21T14:20:00.000Z", + ) + self._git("switch", "-q", "dev") + unnumbered_merge = self._merge( + "integration", + "Merge integration branch", + "2026-08-21T14:30:00.000Z", + ) + self.assertEqual( + self._git("show", f"{unnumbered_merge}:tests/contract.rs").stdout, + three_tests, + ) + for landed in (integration_commit, nested_commit, hidden_merge): + self.assertEqual( + self._git( + "merge-base", "--is-ancestor", landed, unnumbered_merge + ).returncode, + 0, + ) + + self._git("switch", "-q", "-c", "pr-88") + outside_commit = self._commit_files( + {"src/outside.txt": "exclusive-upper-bound content\n"}, + "implement outside fixture", + "2026-08-22T12:00:00.000Z", + ) + self._git("switch", "-q", "dev") + outside_merge = self._merge( + "pr-88", + "Merge pull request #88 from fixtures/pr-88", + "2026-08-22T12:00:00.000Z", + ) + self.assertEqual( + self._git( + "merge-base", "--is-ancestor", outside_commit, outside_merge + ).returncode, + 0, + ) + + self._write_repo_file("tests/contract.rs", four_tests) + self.assertEqual( + len(re.findall(r"(?m)^\s*#\[test\]\s*$", four_tests)), + 4, + ) + self.assertEqual( + len(re.findall( + r"(?m)^\s*#\[test\]\s*$", + self._git("show", "HEAD:tests/contract.rs").stdout, + )), + 3, + "the uncommitted fourth test must not be in the HEAD tree", + ) + + self._plant_claude( + "not-a-git-result", + [{ + "timestamp": "2026-08-21T15:00:00.000Z", + "output": 1, + }], + marker="Merge pull request #777 from a transcript", + ) + + args = ( + "--since", + DEFAULT_SINCE, + "--until", + "2026-08-22T12:00:00.000Z", + ) + plain_proc = self._run_worth("report", *args) + self.assertEqual(plain_proc.returncode, 0, plain_proc.stderr) + data = self._json_worth("report", *args) + results = data["results"] + rendered = json.dumps(results, sort_keys=True) + + self.assertEqual(self._pr_numbers(results), {41}) + self.assertEqual(set(map(int, re.findall(r"#(\d+)", plain_proc.stdout))), + {41}) + for forbidden in ("#88", "#999", "#777"): + self.assertNotIn(forbidden, plain_proc.stdout) + self.assertNotIn(forbidden, rendered) + + identity = ( + unnumbered_merge[:7], + "Merge integration branch", + ) + self.assertTrue( + any(value in plain_proc.stdout for value in identity), + "the unnumbered first-parent merge was dropped from plain output", + ) + self.assertTrue( + any(value in rendered for value in identity), + "the unnumbered first-parent merge was dropped from JSON", + ) + + test_values = [ + value for path, value in walk_leaves(results) + if any("test" in canonical_key(part) for part in path) + ] + self.assertIn(2, test_values) + self.assertRegex( + plain_proc.stdout, + r"(?i)test[^\n]*(?:delta\s*[=:]\s*)?\+?2\b", + ) + self.assertNotIn("uncommitted_definition", plain_proc.stdout) + self.assertNotIn("uncommitted_definition", rendered) + + def test_missing_window_edge_commit_makes_test_delta_unrecorded(self): + since = "2026-08-20T09:00:00.000Z" + until = "2026-08-20T11:00:00.000Z" + plain = self._run_worth( + "report", + "--since", + since, + "--until", + until, + now="2026-08-20T12:00:00.000Z", + ) + self.assertEqual(plain.returncode, 0, plain.stderr) + data = self._json_worth( + "report", + "--since", + since, + "--until", + until, + now="2026-08-20T12:00:00.000Z", + ) + test_gap_values = [ + value for path, value in walk_leaves(data["results"]) + if any("test" in canonical_key(part) for part in path) + ] + self.assertTrue( + any( + isinstance(value, str) + and value.lower() == "unrecorded" + for value in test_gap_values + ), + f"test delta did not preserve the missing-edge gap: {data['results']}", + ) + self.assertRegex( + plain.stdout, + r"(?i)test[^\n]*unrecorded", + ) + + def test_stamp_is_identical_in_plain_and_json_for_both_subcommands(self): + since = "2026-08-21T18:00:00.000Z" + until = "2026-08-22T06:00:00.000Z" + now = "2026-08-22T07:00:00.000Z" + + for verb in ("report", "waste"): + with self.subTest(verb=verb): + plain = self._run_worth( + verb, + "--since", + since, + "--until", + until, + now=now, + ) + self.assertEqual(plain.returncode, 0, plain.stderr) + data = self._json_worth( + verb, + "--since", + since, + "--until", + until, + now=now, + ) + self._assert_stamp(plain.stdout, data, since, until, now) + + def test_waste_ranks_window_totals_honors_top_and_breaks_ties_by_id(self): + inside = "2026-08-22T11:00:00.000Z" + outside = "2026-08-21T11:00:00.000Z" + totals = { + "leader": 300, + "tie-a": 200, + "tie-b": 200, + "third": 150, + "fourth": 120, + "fifth": 100, + } + for session_id, total in totals.items(): + self._plant_claude(session_id, [{ + "timestamp": inside, + "output": total, + }]) + + self._plant_claude("mixed-window", [ + { + "id": "mixed-outside", + "timestamp": outside, + "output": 100_000, + }, + { + "id": "mixed-inside", + "timestamp": inside, + "output": 10, + }, + ]) + + expected_default = [ + "leader", + "tie-a", + "tie-b", + "third", + "fourth", + ] + default_json = self._json_worth("waste") + self.assertEqual(self._session_ids(default_json), expected_default) + + default_plain = self._run_worth("waste") + self.assertEqual(default_plain.returncode, 0, default_plain.stderr) + positions = [ + default_plain.stdout.find(session_id) + for session_id in expected_default + ] + self.assertTrue(all(position >= 0 for position in positions)) + self.assertEqual(positions, sorted(positions)) + self.assertNotIn("fifth", default_plain.stdout) + self.assertNotIn("mixed-window", default_plain.stdout) + self.assertNotIn("100000", default_plain.stdout) + self.assertNotIn("100000", json.dumps(default_json)) + + top_three = self._json_worth("waste", "--top", "3") + self.assertEqual( + self._session_ids(top_three), + ["leader", "tie-a", "tie-b"], + ) + + top_two = self._json_worth("waste", "--top", "2") + self.assertEqual( + self._session_ids(top_two), + ["leader", "tie-a"], + ) + + def test_waste_emits_deterministic_cache_churn_and_heavy_turn_signals(self): + self._plant_claude("claude-heavy", [ + { + "id": "claude-heavy-small", + "timestamp": "2026-08-22T10:00:00.000Z", + "output": 5, + }, + { + "id": "claude-heavy-big", + "timestamp": "2026-08-22T10:01:00.000Z", + "output": 40, + }, + ]) + self._plant_standard_codex( + "codex-heavy", + "2026-08-22T10:00:00.000Z", + ) + self._plant_grok( + "grok-heavy", + "2026-08-22T10:00:00.000Z", + [ + [{ + "inputTokens": 470, + "cachedReadTokens": 0, + "cacheCreationTokens": 0, + "outputTokens": 30, + "totalTokens": 500, + "reasoningTokens": 0, + "costUsdTicks": 5, + }], + [{ + "inputTokens": 50, + "cachedReadTokens": 0, + "cacheCreationTokens": 0, + "outputTokens": 50, + "totalTokens": 100, + "reasoningTokens": 0, + "costUsdTicks": 2, + }], + ], + ) + + self._plant_claude("churn-positive", [{ + "timestamp": "2026-08-22T10:00:00.000Z", + "cache_read": 201, + "output": 10, + }]) + self._plant_claude("churn-equal", [{ + "timestamp": "2026-08-22T10:00:00.000Z", + "cache_read": 200, + "output": 10, + }]) + self._plant_claude("churn-zero-output", [{ + "timestamp": "2026-08-22T10:00:00.000Z", + "cache_read": 1_000, + "output": 0, + }]) + self._plant_claude("churn-creation-only", [{ + "timestamp": "2026-08-22T10:00:00.000Z", + "cache_creation": 1_000, + "cache_read": 0, + "output": 10, + }]) + + data = self._json_worth("waste", "--top", "20") + plain = self._run_worth("waste", "--top", "20") + self.assertEqual(plain.returncode, 0, plain.stderr) + + cache_signals = self._signals_of_kind(data, "cache-churn") + cache_sessions = { + self._signal_field(signal, ("session", "session_id", "id")) + for signal in cache_signals + } + self.assertIn("churn-positive", cache_sessions) + self.assertNotIn("churn-equal", cache_sessions) + self.assertNotIn("churn-zero-output", cache_sessions) + self.assertNotIn("churn-creation-only", cache_sessions) + self._assert_signal( + data, + "cache-churn", + "claude", + "churn-positive", + required_values=(201, 10), + ) + + self._assert_signal( + data, + "heavy-turn", + "claude", + "claude-heavy", + required_values=("claude-heavy-big", 40), + ) + self._assert_signal( + data, + "heavy-turn", + "codex", + "codex-heavy", + required_values=("2026-08-22T10:09:00.000Z", 90), + ) + self._assert_signal( + data, + "heavy-turn", + "grok", + "grok-heavy", + required_values=(50,), + ) + + ranked_ids = set(self._session_ids(data)) + for signal in data["signals"]: + harness = self._signal_field(signal, ("harness",)) + session_id = self._signal_field( + signal, ("session", "session_id", "id") + ) + self.assertIn(harness, {"claude", "codex", "grok"}) + self.assertIn(session_id, ranked_ids) + + for expected in ( + "cache-churn", + "heavy-turn", + "claude", + "codex", + "grok", + "churn-positive", + "claude-heavy", + "codex-heavy", + "grok-heavy", + ): + self.assertIn(expected, plain.stdout) + + def test_json_and_plain_are_two_encodings_of_the_same_truth(self): + self._plant_standard_claude( + "parity-claude", + "2026-08-22T10:00:00.000Z", + ) + self._plant_grok( + "parity-grok", + "2026-08-22T10:00:00.000Z", + [[{ + "inputTokens": 70, + "cachedReadTokens": 10, + "cacheCreationTokens": 5, + "outputTokens": 15, + "totalTokens": 100, + "reasoningTokens": 2, + "costUsdTicks": 9, + }]], + ) + + self._git("switch", "-q", "-c", "pr-61") + self._commit_files( + {"src/parity.txt": "parity merge content\n"}, + "implement parity fixture", + "2026-08-22T10:30:00.000Z", + ) + self._git("switch", "-q", "dev") + self._merge( + "pr-61", + "Merge pull request #61 from fixtures/parity", + "2026-08-22T10:40:00.000Z", + ) + + for verb, keys in ( + ("report", {"stamp", "cost", "results"}), + ("waste", {"stamp", "sessions", "signals"}), + ): + with self.subTest(verb=verb): + plain = self._run_worth(verb) + self.assertEqual(plain.returncode, 0, plain.stderr) + data = self._json_worth(verb) + self.assertEqual(set(data), keys) + self._assert_plain_json_parity(plain.stdout, data) + + def test_empty_windows_succeed_with_zero_results_and_zero_sessions(self): + since = "2026-09-01T12:00:00.000Z" + until = "2026-09-02T12:00:00.000Z" + now = until + + report_plain = self._run_worth( + "report", + "--since", + since, + "--until", + until, + now=now, + ) + self.assertEqual(report_plain.returncode, 0, report_plain.stderr) + report_json = self._json_worth( + "report", + "--since", + since, + "--until", + until, + now=now, + ) + + self._assert_result_measure_zero( + report_json["results"], + ("pr", "prs", "pullrequest"), + ) + self._assert_result_measure_zero( + report_json["results"], + ("commit", "commits"), + ) + self._assert_result_measure_zero( + report_json["results"], + ("test", "tests", "testdefinitions"), + ) + self.assertRegex( + report_plain.stdout, + r"(?i)\bprs?\s*[=:]\s*0\b", + ) + self.assertRegex( + report_plain.stdout, + r"(?i)\bcommits?\s*[=:]\s*0\b", + ) + self.assertRegex( + report_plain.stdout, + r"(?i)\btest(?:s|[-_ ]definitions?)?" + r"(?:[-_ ]delta)?\s*[=:]\s*\+?0\b", + ) + + for harness in ("claude", "codex", "grok"): + record = self._harness_record(report_json, harness) + self._assert_key_number(record, ("sessions",), 0) + line = self._line_for_harness(report_plain.stdout, harness) + self._assert_plain_number(line, ("sessions",), 0) + + waste_plain = self._run_worth( + "waste", + "--since", + since, + "--until", + until, + now=now, + ) + self.assertEqual(waste_plain.returncode, 0, waste_plain.stderr) + waste_json = self._json_worth( + "waste", + "--since", + since, + "--until", + until, + now=now, + ) + self.assertEqual(waste_json["sessions"], []) + self.assertEqual(waste_json["signals"], []) + for harness in ("claude", "codex", "grok"): + line = self._line_for_harness(waste_plain.stdout, harness) + self._assert_plain_number(line, ("sessions",), 0) + + def test_unusable_arguments_exit_two_and_explain_the_reason(self): + cases = [ + ( + "report", + ("--since", "not-an-iso"), + NOW, + "since", + ), + ( + "waste", + ( + "--since", + "2026-08-22T12:00:00.000Z", + "--until", + "2026-08-22T12:00:00.000Z", + ), + "2026-08-22T13:00:00.000Z", + "until", + ), + ( + "report", + ("--format", "yaml"), + NOW, + "format", + ), + ( + "waste", + (), + "not-an-iso", + "now", + ), + ] + for verb, args, now, reason in cases: + with self.subTest(verb=verb, args=args, now=now): + proc = self._run_worth(verb, *args, now=now) + self.assertEqual(proc.returncode, 2, proc) + self.assertTrue(proc.stderr.strip()) + self.assertIn(reason, proc.stderr.lower()) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/telemetry/tests/test_worth_seams.py b/ops/devlane/telemetry/tests/test_worth_seams.py new file mode 100644 index 0000000..9ca0894 --- /dev/null +++ b/ops/devlane/telemetry/tests/test_worth_seams.py @@ -0,0 +1,394 @@ +"""Pins for the seams Codex's audit planted through (PR #31). + +Each case proves its fixture landed before asserting: a plant that +silently failed to plant answers a question about nothing. +""" + +import json +import subprocess +import sys +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from typing import ClassVar +from urllib.parse import quote + +WORTH = Path(__file__).resolve().parents[1] / "worth.py" +NOW = "2026-08-22T12:00:00.000Z" + + +class SeamCase(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix="worth-seams-")) + self.addCleanup( + lambda: subprocess.run(["rm", "-rf", str(self.tmp)], check=False)) + self.repo = self.tmp / "repo" + self.repo.mkdir() + subprocess.run(["git", "-C", str(self.repo), "init", "-q", "-b", + "dev"], check=True) + self.claude = self.tmp / "claude" + self.codex = self.tmp / "codex" + self.grok = self.tmp / "grok" + + def write_lines(self, path, rows): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(json.dumps(r) + "\n" for r in rows)) + planted = [json.loads(line) for line in path.read_text().splitlines()] + self.assertEqual(planted, rows, "the plant did not land intact") + return path + + def claude_row(self, mid, stamp, usage): + return {"type": "assistant", "timestamp": stamp, + "cwd": str(self.repo), "sessionId": "s", + "message": {"id": mid, "usage": usage, "content": []}} + + def grok_session(self, name, updates=None, raw_updates=None): + session = self.grok / "sessions" / quote(str(self.repo), safe="") / name + session.mkdir(parents=True) + (session / "summary.json").write_text(json.dumps( + {"info": {"id": name, "cwd": str(self.repo)}, + "created_at": "2026-08-22T10:00:00.000000000Z", + "updated_at": "2026-08-22T10:20:00.000000000Z"})) + if raw_updates is not None: + (session / "updates.jsonl").write_text(raw_updates) + self.assertEqual((session / "updates.jsonl").read_text(), + raw_updates) + elif updates is not None: + self.write_lines(session / "updates.jsonl", updates) + return session + + def worth(self, verb, *args): + proc = subprocess.run( + [sys.executable, str(WORTH), verb, *args, "--now", NOW, + "--repo", str(self.repo), + "--claude-dir", str(self.claude), + "--codex-dir", str(self.codex), + "--grok-dir", str(self.grok), + "--format", "json"], + capture_output=True, text=True, check=False) + self.assertEqual(proc.returncode, 0, proc.stderr) + return json.loads(proc.stdout) + + def slug(self): + return "-" + "-".join(str(self.repo).strip("/").split("/")) + + +class EmptyUsageIsNotAMeasurement(SeamCase): + def test_an_empty_claude_usage_dict_is_unrecorded_not_zero(self): + path = self.write_lines( + self.claude / self.slug() / "s.jsonl", + [self.claude_row("m1", "2026-08-22T10:00:00.000Z", {})]) + self.assertIn('"usage": {}', path.read_text()) + record = self.worth("report")["cost"]["claude"] + self.assertEqual(record.get("sessions"), 0) + self.assertEqual(record.get("tokens"), "unrecorded") + + def test_a_keyless_grok_usage_dict_is_not_a_run(self): + at = int(datetime(2026, 8, 22, 10, 0, 0, + tzinfo=timezone.utc).timestamp()) + self.grok_session("g", updates=[ + {"method": "session/update", "timestamp": at, + "params": {"update": {"sessionUpdate": "turn_completed", + "usage": {"unrelated": 1}}}}]) + record = self.worth("report")["cost"]["grok"] + self.assertEqual(record.get("counted"), "0/1") + self.assertEqual(record.get("tokens"), "unrecorded") + + +class KeylessCodexUsageIsNotAMeasurement(SeamCase): + def test_a_keyless_token_count_is_unrecorded_not_zero(self): + rows = [ + {"timestamp": "2026-08-22T10:00:00.000Z", + "type": "session_meta", + "payload": {"id": "cx", "cwd": str(self.repo)}}, + {"timestamp": "2026-08-22T10:01:00.000Z", + "type": "event_msg", + "payload": {"type": "token_count", + "info": {"total_token_usage": {}}}}, + ] + path = self.write_lines( + self.codex / "sessions" / "2026" / "08" / "22" + / "rollout-2026-08-22T10-00-00-cx.jsonl", rows) + self.assertIn('"total_token_usage": {}', path.read_text()) + record = self.worth("report")["cost"]["codex"] + self.assertEqual(record.get("sessions"), 0) + self.assertEqual(set(record), {"sessions", "tokens"}) + + +class MalformedTimestampsAreGapsNotCrashes(SeamCase): + def test_a_non_string_claude_timestamp_gaps_the_session(self): + """Scenario: a malformed store timestamp is a gap, not a crash""" + path = self.write_lines( + self.claude / self.slug() / "s.jsonl", + [self.claude_row("m1", 12345, {"output_tokens": 5})]) + self.assertIn('"timestamp": 12345', path.read_text()) + record = self.worth("report")["cost"]["claude"] + self.assertEqual(record.get("sessions"), 0) + self.assertEqual(record.get("tokens"), "unrecorded") + + +class GrokDenominatorHoldsItsHoles(SeamCase): + USAGE: ClassVar[dict] = { + "inputTokens": 90, "cachedReadTokens": 0, + "cacheCreationTokens": 0, "outputTokens": 10, + "totalTokens": 100, "costUsdTicks": 7} + + def good_session(self): + # 2026-08-22T10:01:40Z as an epoch — computed here so the + # figure can never be a hand-typed recollection + at = int(datetime(2026, 8, 22, 10, 1, 40, + tzinfo=timezone.utc).timestamp()) + self.grok_session("g-good", updates=[ + {"method": "session/update", "timestamp": at, + "params": {"update": {"sessionUpdate": "turn_completed", + "usage": self.USAGE}}}]) + + def test_an_unreadable_updates_file_still_holds_a_denominator_place(self): + """Scenario: a grok session with unreadable spend holds its denominator place""" + self.good_session() + bad = self.grok_session("g-bad", raw_updates="{broken\n") + self.assertEqual((bad / "updates.jsonl").read_text(), "{broken\n") + record = self.worth("report")["cost"]["grok"] + self.assertEqual(record.get("sessions"), 2) + self.assertEqual(record.get("counted"), "1/2") + self.assertEqual(record.get("total"), 100) + + def test_all_unrecorded_sessions_still_report_the_denominator(self): + self.grok_session("g-bad", raw_updates="{broken\n") + record = self.worth("report")["cost"]["grok"] + self.assertEqual(record.get("counted"), "0/1") + self.assertEqual(record.get("tokens"), "unrecorded") + # exactly these keys: an added in=0/out=0 is a false figure + self.assertEqual(set(record), + {"sessions", "runs", "counted", "tokens"}) + + def test_an_iso_string_run_timestamp_is_a_gap_not_a_crash(self): + self.good_session() + iso = self.grok_session("g-iso", updates=[ + {"method": "session/update", + "timestamp": "2026-08-22T10:05:00.000Z", + "params": {"update": {"sessionUpdate": "turn_completed", + "usage": {"totalTokens": 50, + "outputTokens": 5}}}}]) + self.assertIn('"timestamp": "2026-08-22T10:05:00.000Z"', + (iso / "updates.jsonl").read_text()) + record = self.worth("report")["cost"]["grok"] + self.assertEqual(record.get("sessions"), 2) + self.assertEqual(record.get("counted"), "1/2") + self.assertEqual(record.get("total"), 100) + + +class CodexHeavyTurnIsTheLargestStep(SeamCase): + def test_the_heaviest_message_is_the_largest_delta_not_the_last(self): + rows = [ + {"timestamp": "2026-08-22T10:00:00.000Z", "type": "session_meta", + "payload": {"id": "cx", "cwd": str(self.repo)}}, + ] + for stamp, out in (("2026-08-22T10:01:00.000Z", 40), + ("2026-08-22T10:02:00.000Z", 90), + ("2026-08-22T10:03:00.000Z", 100)): + rows.append({"timestamp": stamp, "type": "event_msg", + "payload": {"type": "token_count", "info": { + "total_token_usage": { + "input_tokens": 10, "cached_input_tokens": 0, + "output_tokens": out, + "total_tokens": 10 + out}}}}) + path = self.write_lines( + self.codex / "sessions" / "2026" / "08" / "22" + / "rollout-2026-08-22T10-00-00-cx.jsonl", rows) + self.assertEqual(path.read_text().count("token_count"), 3) + waste = self.worth("waste") + heavy = [sig for sig in waste["signals"] + if "heavy" in json.dumps(sig) and sig.get("harness") == "codex"] + self.assertEqual(len(heavy), 1) + # deltas are 40, 50, 10 — the middle count is the heavy one + self.assertEqual(heavy[0].get("at"), "2026-08-22T10:02:00.000Z") + self.assertEqual(heavy[0].get("out_delta"), 50) + self.assertEqual(heavy[0].get("out"), 90) + + +class SeamGitCase(SeamCase): + def commit(self, files, subject, when): + for rel, content in files.items(): + path = self.repo / rel + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(content, bytes): + path.write_bytes(content) # test-guard: allow + self.assertEqual(path.read_bytes(), content) + else: + path.write_text(content) + self.assertEqual(path.read_text(), content) + env = {"GIT_AUTHOR_DATE": when, "GIT_COMMITTER_DATE": when, + "GIT_AUTHOR_NAME": "seam", "GIT_AUTHOR_EMAIL": "s@x", + "GIT_COMMITTER_NAME": "seam", "GIT_COMMITTER_EMAIL": "s@x"} + import os + subprocess.run(["git", "-C", str(self.repo), "add", "-A"], + check=True) + subprocess.run(["git", "-C", str(self.repo), "commit", "-q", + "-m", subject], check=True, + env={**os.environ, **env}) + + +class CodexBaselineSubtraction(SeamCase): + def test_pre_window_spend_stays_out_of_the_window(self): + """Scenario: pre-window spend stays out of the window""" + rows = [{"timestamp": "2026-08-22T09:00:00.000Z", + "type": "session_meta", + "payload": {"id": "cx", "cwd": str(self.repo)}}] + for stamp, out in (("2026-08-22T09:30:00.000Z", 10), + ("2026-08-22T10:30:00.000Z", 15)): + rows.append({"timestamp": stamp, "type": "event_msg", + "payload": {"type": "token_count", "info": { + "total_token_usage": { + "input_tokens": out * 2, + "cached_input_tokens": 0, + "output_tokens": out, + "total_tokens": out * 3}}}}) + path = self.write_lines( + self.codex / "sessions" / "2026" / "08" / "22" + / "rollout-2026-08-22T09-00-00-cx.jsonl", rows) + self.assertEqual(path.read_text().count("token_count"), 2) + record = self.worth("report", "--since", + "2026-08-22T10:00:00.000Z")["cost"]["codex"] + # cumulative 10 before the window, 15 inside: the window saw 5 + self.assertEqual(record.get("out"), 5) + self.assertEqual(record.get("in"), 10) + self.assertEqual(record.get("messages"), 1) + waste = self.worth("waste", "--since", + "2026-08-22T10:00:00.000Z") + heavy = [sig for sig in waste["signals"] + if sig.get("kind") == "heavy-turn" + and sig.get("harness") == "codex"] + self.assertEqual(len(heavy), 1) + # the heavy turn is the IN-WINDOW event with its own delta; + # neither the pre-window stamp nor its cumulative may leak + self.assertEqual(heavy[0].get("at"), "2026-08-22T10:30:00.000Z") + self.assertEqual(heavy[0].get("out_delta"), 5) + + +class ClaudeReemissionAfterTheWindow(SeamCase): + def test_a_later_reemission_does_not_erase_history(self): + inside = self.claude_row( + "m1", "2026-08-22T10:00:00.000Z", {"output_tokens": 7}) + after = self.claude_row( + "m1", "2026-08-22T13:00:00.000Z", {"output_tokens": 900}) + path = self.write_lines( + self.claude / self.slug() / "s.jsonl", [inside, after]) + self.assertEqual(path.read_text().count('"m1"'), 2) + record = self.worth( + "report", "--until", "2026-08-22T12:00:00.000Z", + )["cost"]["claude"] + self.assertEqual(record.get("messages"), 1) + self.assertEqual(record.get("out"), 7) + + +class GrokIncompletenessIsWindowScoped(SeamCase): + def test_an_incomplete_run_outside_the_window_says_nothing(self): + early = int(datetime(2026, 8, 21, 9, 0, + tzinfo=timezone.utc).timestamp()) + late = int(datetime(2026, 8, 22, 10, 0, + tzinfo=timezone.utc).timestamp()) + self.grok_session("g", updates=[ + {"method": "session/update", "timestamp": early, + "params": {"update": {"sessionUpdate": "turn_completed", + "usage": {"totalTokens": 500, + "outputTokens": 5, + "usageIsIncomplete": True}}}}, + {"method": "session/update", "timestamp": late, + "params": {"update": {"sessionUpdate": "turn_completed", + "usage": {"totalTokens": 100, + "outputTokens": 10, + "costUsdTicks": 3}}}}]) + record = self.worth( + "report", "--since", "2026-08-22T00:00:00.000Z", + )["cost"]["grok"] + self.assertEqual(record.get("runs"), 1) + self.assertEqual(record.get("total"), 100) + self.assertFalse(record.get("incomplete")) + + +class UnrecordedRecordsCarryNoZeroes(SeamCase): + def test_the_unrecorded_shape_is_exactly_the_unrecorded_shape(self): + data = self.worth("report") + self.assertEqual(set(data["cost"]["claude"]), + {"sessions", "tokens"}) + self.assertEqual(set(data["cost"]["codex"]), + {"sessions", "tokens"}) + self.assertEqual(set(data["cost"]["grok"]), + {"sessions", "tokens"}) + + def test_stamps_carry_exactly_their_documented_keys(self): + report = self.worth("report") + self.assertEqual(set(report["stamp"]), + {"head", "branch", "since", "until", "now"}) + waste = self.worth("waste") + self.assertEqual(set(waste["stamp"]), + {"head", "branch", "since", "until", "now", + "sessions"}) + + +class ResultsSideSeams(SeamGitCase): + def test_a_binary_blob_in_an_edge_tree_is_not_a_crash(self): + self.commit({"img.png": b"\x89PNG\x0d\x0a\x1a\x0a\xff\xfe", + "tests/t.rs": "#[test]\nfn a() {}\n"}, + "binary and a test", "2026-08-22T09:00:00Z") + data = self.worth("report") + tests = data["results"]["tests"] + self.assertEqual(tests.get("until"), 1) + + def test_python_definitions_count_too(self): + self.commit({"tests/t.py": "def helper():\n pass\n\n" + "def test_one():\n pass\n"}, + "python test", "2026-08-22T09:00:00Z") + data = self.worth("report") + self.assertEqual(data["results"]["tests"].get("until"), 1) + + def test_helper_functions_are_not_test_definitions(self): + self.commit({"tests/t.rs": + "fn helper() {}\n\n#[test]\nfn a() {}\n"}, + "helper plus one test", "2026-08-22T09:00:00Z") + data = self.worth("report") + self.assertEqual(data["results"]["tests"].get("until"), 1) + + def test_edge_trees_honor_their_bounds_exactly(self): + self.commit({"tests/t.rs": "#[test]\nfn a() {}\n"}, + "one test", "2026-08-22T09:00:00Z") + self.commit({"tests/t.rs": "#[test]\nfn a() {}\n#[test]\nfn b() {}\n"}, + "two tests", "2026-08-22T10:00:00Z") + data = self.worth("report", "--since", "2026-08-22T09:00:00.000Z", + "--until", "2026-08-22T10:00:00.000Z") + tests = data["results"]["tests"] + # since-edge is inclusive (the 09:00 commit), until-edge is + # strict (the 10:00 commit is outside) + self.assertEqual(tests.get("since"), 1) + self.assertEqual(tests.get("until"), 1) + self.assertEqual(tests.get("delta"), 0) + + def test_every_merge_entry_names_its_sha_and_subject(self): + self.commit({"a.txt": "x\n"}, "base", "2026-08-22T09:00:00Z") + subprocess.run(["git", "-C", str(self.repo), "switch", "-q", + "-c", "side"], check=True) + self.commit({"b.txt": "y\n"}, "side work", "2026-08-22T09:30:00Z") + subprocess.run(["git", "-C", str(self.repo), "switch", "-q", + "dev"], check=True) + import os + subprocess.run(["git", "-C", str(self.repo), "merge", "-q", + "--no-ff", "-m", "Merge pull request #7 from x/side", + "side"], check=True, + env={**os.environ, + "GIT_AUTHOR_DATE": "2026-08-22T10:30:00Z", + "GIT_COMMITTER_DATE": "2026-08-22T10:30:00Z", + "GIT_AUTHOR_NAME": "seam", + "GIT_AUTHOR_EMAIL": "s@x", + "GIT_COMMITTER_NAME": "seam", + "GIT_COMMITTER_EMAIL": "s@x"}) + data = self.worth("report") + merges = data["results"]["merges"] + self.assertEqual(len(merges), 1) + self.assertEqual(set(merges[0]), {"pr", "sha", "subject"}) + self.assertEqual(merges[0]["pr"], 7) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/telemetry/usage.py b/ops/devlane/telemetry/usage.py new file mode 100644 index 0000000..f721c66 --- /dev/null +++ b/ops/devlane/telemetry/usage.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python3 +"""Token and session usage across the three harnesses, from their own stores. + + usage.py sessions [--json] [--repo PATH] [--claude-dir D] [--codex-dir D] [--grok-dir D] + usage.py report [same flags] + +Nothing here instruments anything: every harness already writes a session +store, and this reads them. Measured shapes (2026-08-21): + +- Claude: ``//.jsonl`` — per-message + ``message.usage`` (input, cache_creation, cache_read, output). Summed. +- Codex: ``/sessions/Y/M/D/rollout-*.jsonl`` — a ``session_meta`` head + carrying ``cwd``, then cumulative ``token_count`` events. The LAST one is + the session's usage; summing them would multiply it. +- Grok: ``/sessions///`` — summary.json holds + messages/model/timestamps; spend lives in updates.jsonl + ``turn_completed`` events (cumulative within a run; runs split when + totalTokens shrinks; last report per currency wins) including raw + costUsdTicks. Pre-upgrade sessions have no usage events and stay an + explicit gap; nothing is ever estimated. + +The report is aggregates only. The stores hold prompts and transcripts; +none of that content leaves them through this tool. + +Two principles taken from loopstrap's telemetry design (TELEMETRY.md, +xormania/loopstrap): the capture rule — an unknown value stays an explicit +gap, never a zero — and the accounting rule — a session's spend is input + +cache_creation + cache_read + output, cached reads counted as the cheap +class they are, not ignored. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from urllib.parse import quote + +GROK_GAP = "grok usage not yet parsed: updates.jsonl turn_completed" + + +def _read_jsonl(path: Path): + for raw in path.read_text(errors="replace").splitlines(): + raw = raw.strip() + if not raw: + continue + try: + yield json.loads(raw) + except ValueError: + continue + + +def claude_sessions(root: Path, repo: str | None): + if not root.is_dir(): + return + for project in sorted(p for p in root.iterdir() if p.is_dir()): + if repo: + # Claude encodes the project path with '-' separators. + slug = "-" + "-".join(repo.strip("/").split("/")) + if project.name != slug: + continue + for f in sorted(project.glob("*.jsonl")): + per_msg = {} + model = None + first = last = None + cwds = set() + for line_no, entry in enumerate(_read_jsonl(f)): + if entry.get("cwd"): + cwds.add(entry["cwd"]) + stamp = entry.get("timestamp") + if stamp: + first = first or stamp + last = stamp + message = entry.get("message") or {} + usage = message.get("usage") + if not usage: + continue + model = message.get("model") or model + # Keyed by message id, LAST wins: live streams are the + # snapshot-rewrite shape (measured 2026-08-21: 2086 + # usage lines over 1052 unique ids in one session), and + # summing every line nearly doubles the spend. + key = message.get("id") or f"line-{line_no}" + per_msg[key] = { + "input": usage.get("input_tokens", 0), + "cached": (usage.get("cache_creation_input_tokens", 0) + + usage.get("cache_read_input_tokens", 0)), + "output": usage.get("output_tokens", 0), + } + if not per_msg: + continue + if repo and cwds and repo not in cwds: + # Two paths can flatten to one directory slug; the cwd + # stored in the entries is the truth. + continue + messages = len(per_msg) + tokens = {key: sum(m[key] for m in per_msg.values()) + for key in ("input", "cached", "output")} + tokens["total"] = sum(tokens.values()) + yield {"harness": "claude", "session": f.stem, "model": model, + "started": first, "ended": last, "messages": messages, + "tokens": tokens} + + +def codex_sessions(root: Path, repo: str | None): + sessions = root / "sessions" + if not sessions.is_dir(): + return + for f in sorted(sessions.glob("*/*/*/rollout-*.jsonl")): + meta, last_count, events = {}, None, 0 + first = last = None + for entry in _read_jsonl(f): + stamp = entry.get("timestamp") + if stamp: + first = first or stamp + last = stamp + payload = entry.get("payload") or {} + if "cwd" in payload: + # Measured split: session_meta carries id/cwd, a later + # turn_context carries model/effort. Merge, never replace — + # replacing loses the id to whichever payload came last. + meta = {**meta, **payload} + if payload.get("type") == "token_count": + info = payload.get("info") or {} + last_count = info.get("total_token_usage") or last_count + events += 1 + if repo and meta.get("cwd") != repo: + continue + tokens = None + if last_count: + tokens = {"input": last_count.get("input_tokens", 0), + "cached": last_count.get("cached_input_tokens", 0), + "output": last_count.get("output_tokens", 0), + "total": last_count.get("total_tokens", 0)} + yield {"harness": "codex", "session": meta.get("id", f.stem), + "model": meta.get("model"), "started": first, "ended": last, + "messages": events, "tokens": tokens} + + +GROK_USAGE_CURRENCIES = ( + "inputTokens", "outputTokens", "totalTokens", "cachedReadTokens", + "cacheCreationTokens", "reasoningTokens", "costUsdTicks", +) + + +def _grok_usage_totals(events): + """Session spend from turn_completed events (measured 2026-08-21): + usage is cumulative WITHIN a run and a run ends when the cumulative + totalTokens SHRINKS (the only observable reset signal — turns can + rise or repeat across a reset). The session figure is each + currency's last-reported value per run, summed across runs. + Last-wins-across-the-stream undercounts, summing every event + overcounts, and max-merge overcounts when a reset arrives without + a turns drop (measured live, skeptic finding). An event omitting a + currency does not erase the run's earlier report, and a RUN that + never reports costUsdTicks makes the session's cost a gap (None) — + never zero, never a partial sum.""" + runs, current, prev_total = [], {}, None + incomplete = False + for usage in events: + if "totalTokens" not in usage: + # An event that does not report totals cannot signal a + # reset; coercing absence to zero invents a run boundary. + if usage.get("usageIsIncomplete"): + incomplete = True + for key in GROK_USAGE_CURRENCIES: + if key in usage: + current[key] = usage[key] + continue + total = usage["totalTokens"] + if prev_total is not None and total < prev_total: + # A cumulative snapshot that SHRINKS is a new run — the only + # observable reset signal. numTurns is NOT it: a live stream + # (019fb283…) reset totals 7.5M→1.9M while turns ROSE 12→15, + # and turns repeat freely within a run. + runs.append(current) + current = {} + prev_total = total + if usage.get("usageIsIncomplete"): + incomplete = True + for key in GROK_USAGE_CURRENCIES: + if key in usage: + # Cumulative within the run: the LAST event to report a + # currency wins; an event omitting one does not erase + # the run's earlier report. + current[key] = usage[key] + runs.append(current) + totals = {key: sum(run.get(key, 0) for run in runs) + for key in GROK_USAGE_CURRENCIES} + cost = (None if any("costUsdTicks" not in run for run in runs) + else totals["costUsdTicks"]) + tokens = {"input": totals["inputTokens"], + "cached": (totals["cachedReadTokens"] + + totals["cacheCreationTokens"]), + "output": totals["outputTokens"], + "total": totals["totalTokens"]} + return tokens, totals["reasoningTokens"], cost, incomplete + + +def _grok_usage_events(updates_path): + if not updates_path.is_file(): + return [] + events = [] + for entry in _read_jsonl(updates_path): + update = (entry.get("params") or {}).get("update") or {} + # Only turn_completed carries spend; usage keys on other update + # kinds are not accounting records (measured decoy: a tool + # update carrying a usage dict). + if update.get("sessionUpdate") == "turn_completed" and isinstance( + update.get("usage"), dict + ): + events.append(update["usage"]) + return events + + +def grok_sessions(root: Path, repo: str | None): + sessions = root / "sessions" + if not sessions.is_dir(): + return + for cwd_dir in sorted(p for p in sessions.iterdir() if p.is_dir()): + if repo and cwd_dir.name != quote(repo, safe=""): + continue + for sdir in sorted(p for p in cwd_dir.iterdir() if p.is_dir()): + summary = sdir / "summary.json" + if not summary.exists(): + continue + try: + s = json.loads(summary.read_text()) + except ValueError: + continue + row = {"harness": "grok", "incomplete": False, + "session": (s.get("info") or {}).get("id", sdir.name), + "model": s.get("current_model_id"), + "started": s.get("created_at"), + "ended": s.get("updated_at"), + "messages": s.get("num_messages"), + "tokens": None, "reasoning": None, + "cost_usd_ticks": None, "note": GROK_GAP} + events = _grok_usage_events(sdir / "updates.jsonl") + if events: + tokens, reasoning, cost, incomplete = _grok_usage_totals(events) + row["tokens"] = tokens + row["reasoning"] = reasoning + row["cost_usd_ticks"] = cost + row["incomplete"] = incomplete + row["note"] = ("grok usage parsed from updates.jsonl" + " (usageIsIncomplete: figures incomplete)" + if incomplete + else "grok usage parsed from updates.jsonl") + yield row + + +def collect(args): + rows = [] + rows += list(claude_sessions(Path(args.claude_dir), args.repo)) + rows += list(codex_sessions(Path(args.codex_dir), args.repo)) + rows += list(grok_sessions(Path(args.grok_dir), args.repo)) + rows.sort(key=lambda r: (r.get("started") or "", r["harness"])) + return rows + + +def cmd_sessions(args) -> int: + rows = collect(args) + if args.json: + print(json.dumps({"sessions": rows}, indent=1, sort_keys=True)) + return 0 + for r in rows: + t = r.get("tokens") + spent = (f"in={t['input']} cached={t['cached']} out={t['output']}" + f" total={t['total']}" if t else r.get("note", "-")) + if t and r["harness"] == "grok": + ticks = r.get("cost_usd_ticks") + spent += (f" cost_usd_ticks={ticks}" if ticks is not None + else " cost_usd_ticks=unrecorded") + if "incomplete" in (r.get("note") or "").lower(): + spent += " (incomplete)" + print(f"{r['harness']:<7} {str(r['session'])[:12]:<13}" + f" {str(r.get('model'))[:18]:<19} msgs={r.get('messages')}" + f" {spent}") + return 0 + + +def cmd_report(args) -> int: + rows = collect(args) + by = {} + for r in rows: + agg = by.setdefault(r["harness"], { + "sessions": 0, "messages": 0, + "tokens": {"input": 0, "cached": 0, "output": 0, "total": 0}, + "counted": 0}) + agg["sessions"] += 1 + agg["messages"] += r.get("messages") or 0 + if r.get("tokens"): + agg["counted"] += 1 + for k in agg["tokens"]: + agg["tokens"][k] += r["tokens"].get(k, 0) + if r["harness"] == "grok": + if r.get("incomplete"): + agg["incomplete_sessions"] = agg.get( + "incomplete_sessions", 0) + 1 + # Cost aggregates gap-honestly: one counted session + # with unknown cost makes the total unknown, never a + # partial sum passed off as complete. + if r.get("cost_usd_ticks") is None: + agg["cost_usd_ticks"] = None + elif agg.get("cost_usd_ticks", 0) is not None: + agg["cost_usd_ticks"] = (agg.get("cost_usd_ticks") or 0 + ) + r["cost_usd_ticks"] + if args.json: + print(json.dumps({"by_harness": by}, indent=1, sort_keys=True)) + return 0 + for harness in sorted(by): + agg = by[harness] + t = agg["tokens"] + line = (f"{harness:<7} sessions={agg['sessions']}" + f" messages={agg['messages']}") + if agg["counted"]: + line += (f" in={t['input']} cached={t['cached']}" + f" out={t['output']} total={t['total']}") + if agg["counted"] < agg["sessions"]: + # The figures cover a subset; saying so is the line's + # licence to print them at all. + line += f" counted={agg['counted']}/{agg['sessions']}" + if agg.get("cost_usd_ticks") is not None: + line += f" cost_usd_ticks={agg['cost_usd_ticks']}" + elif harness == "grok": + line += " cost_usd_ticks=unrecorded" + if agg.get("incomplete_sessions"): + # An explicitly incomplete measurement must never read + # as a verified total. + line += f" incomplete={agg['incomplete_sessions']}" + else: + # The capture rule (loopstrap): an unavailable value is an + # explicit gap. Zeros here would read as "measured: nothing + # spent", which is the one thing this line must never say. + line += f" tokens=unrecorded ({GROK_GAP})" if harness == "grok" else " tokens=unrecorded" + print(line) + return 0 + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("verb", choices=["sessions", "report"]) + parser.add_argument("--json", action="store_true") + parser.add_argument("--repo", default=None, + help="only sessions whose cwd is this path") + home = Path.home() + parser.add_argument("--claude-dir", default=str(home / ".claude" / "projects")) + parser.add_argument("--codex-dir", default=str(home / ".codex")) + parser.add_argument("--grok-dir", default=str(home / ".grok")) + args = parser.parse_args(argv) + return cmd_sessions(args) if args.verb == "sessions" else cmd_report(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ops/devlane/telemetry/worth.py b/ops/devlane/telemetry/worth.py new file mode 100644 index 0000000..db9ef55 --- /dev/null +++ b/ops/devlane/telemetry/worth.py @@ -0,0 +1,594 @@ +#!/usr/bin/env python3 +"""worth — costs and results, joined (ops/process/worth.md). + +`report` joins the window's per-harness spend (same stores and +accounting rules as usage.py) with what the repo's history says +landed; `waste` ranks the window's sessions by spend and names the +signals. Every figure is produced at run time and stamped with the +window and repo state it was measured against. Grok cost stays in +raw ticks: the scale is unverified, so no figure here is ever USD. +""" + +import argparse +import json +import re +import subprocess +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path +from urllib.parse import quote + +CHURN_FACTOR = 20 +MERGE_PR = re.compile(r"^Merge pull request #(\d+)\b") +_CLAUDE_CURRENCIES = frozenset(( + "input_tokens", "cache_creation_input_tokens", + "cache_read_input_tokens", "output_tokens")) +_CODEX_CURRENCIES = frozenset(( + "input_tokens", "cached_input_tokens", "output_tokens", + "reasoning_output_tokens", "total_tokens")) +_GROK_CURRENCIES = frozenset(( + "inputTokens", "cachedReadTokens", "cacheCreationTokens", + "outputTokens", "totalTokens", "reasoningTokens", "costUsdTicks")) + + +def parse_stamp(value): + """Store-side: a malformed timestamp raises ValueError so the + reader can treat the SESSION as unparseable — a gap, never a + crash and never a silently dropped row.""" + if not isinstance(value, str): + # ValueError on purpose: callers treat every malformed stamp + # as one class of gap, whatever the malformation + raise ValueError(f"non-string timestamp: {value!r}") + trimmed = re.sub(r"(\.\d{6})\d+", r"\1", value) + stamp = datetime.fromisoformat(trimmed) + if stamp.tzinfo is None: + stamp = stamp.replace(tzinfo=timezone.utc) + return stamp + + +def parse_iso(value, label): + try: + return parse_stamp(value) + except ValueError: + raise SystemExit2( + f"invalid {label}: {value!r} is not ISO 8601") from None + + +class SystemExit2(Exception): + pass + + +# ---------------------------------------------------------------- stores + +def read_jsonl(path): + """Every line parses or the file is unparseable — a half-read + store silently under-reports, which is worse than a gap.""" + rows = [] + try: + for line in path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + rows.append(json.loads(line)) + except (OSError, ValueError): + return None + return rows + + +def claude_sessions(root, repo, window): + """Per-message accounting, last write per message id wins (the + measured re-emission double-count), cwd-filtered to the repo.""" + since, until = window + sessions = {} + for path in sorted(Path(root).glob("*/*.jsonl")): + rows = read_jsonl(path) + if rows is None: + continue + in_window = {} + try: + for row in rows: + if row.get("cwd") != repo: + continue + message = row.get("message") or {} + usage = message.get("usage") + mid = message.get("id") + stamp = row.get("timestamp") + if not (isinstance(usage, dict) and mid and stamp): + continue + if not set(usage) & _CLAUDE_CURRENCIES: + # an empty usage dict is unknown spend, and an + # unknown rendered as 0 is a false measurement + continue + at = parse_stamp(stamp) + # last-wins among IN-WINDOW emissions: a re-emission + # after the window must not erase history from it + if since <= at < until: + in_window[mid] = (at, stamp, usage) + except ValueError: + continue + if not in_window: + continue + sid = path.stem + totals = {"in": 0, "cached": 0, "out": 0} + heavy = None + for mid, (_at, _raw, usage) in sorted(in_window.items()): + out = usage.get("output_tokens", 0) + totals["in"] += usage.get("input_tokens", 0) + totals["cached"] += (usage.get("cache_creation_input_tokens", 0) + + usage.get("cache_read_input_tokens", 0)) + totals["out"] += out + if heavy is None or out > heavy["out"]: + heavy = {"message": mid, "out": out} + cached_read = sum( + usage.get("cache_read_input_tokens", 0) + for _, _, usage in in_window.values() + ) + sessions[sid] = { + "messages": len(in_window), + "in": totals["in"], + "cached": totals["cached"], + "out": totals["out"], + "total": totals["in"] + totals["cached"] + totals["out"], + "cached_read": cached_read, + "heavy": heavy, + } + return sessions + + +def codex_sessions(root, repo, window): + """token_count events are cumulative; the last in-window count IS + the spend (summing them is the measured double-count mistake).""" + since, until = window + sessions = {} + for path in sorted(Path(root).glob("sessions/*/*/*/rollout-*.jsonl")): + rows = read_jsonl(path) + if rows is None: + continue + meta = next((row for row in rows + if row.get("type") == "session_meta"), None) + if not meta or (meta.get("payload") or {}).get("cwd") != repo: + continue + sid = (meta.get("payload") or {}).get("id") or path.stem + counts = [] + try: + for row in rows: + payload = row.get("payload") or {} + if (row.get("type") == "event_msg" + and payload.get("type") == "token_count"): + usage = (payload.get("info") or {}).get( + "total_token_usage") + stamp = row.get("timestamp") + if (isinstance(usage, dict) and stamp + and set(usage) & _CODEX_CURRENCIES): + counts.append((parse_stamp(stamp), stamp, usage)) + except ValueError: + continue + in_window = [(at, raw, usage) for at, raw, usage in counts + if since <= at < until] + if not in_window: + continue + last = in_window[-1][2] + # counts are cumulative for the SESSION: the last pre-window + # count is the baseline, or a straddling session charges its + # pre-window spend to this window + baseline = {} + for at, _raw, usage in counts: + if at < since: + baseline = usage + + def net(key, last=last, baseline=baseline): + return last.get(key, 0) - baseline.get(key, 0) + + # the heaviest MESSAGE is the largest step between + # consecutive counts, session-wide so the first in-window + # message is not credited with pre-window spend + deltas = {} + prev_out = 0 + for _at, raw, usage in counts: + out_here = usage.get("output_tokens", 0) + deltas[raw] = out_here - prev_out + prev_out = out_here + _, heavy_raw, heavy_usage = max( + in_window, key=lambda item: deltas.get(item[1], 0)) + sessions[sid] = { + "messages": len(in_window), + "in": net("input_tokens"), + "cached": net("cached_input_tokens"), + "out": net("output_tokens"), + "total": net("input_tokens") + net("output_tokens"), + "cached_read": net("cached_input_tokens"), + "heavy": {"at": heavy_raw, + "out": heavy_usage.get("output_tokens", 0), + "out_delta": deltas.get(heavy_raw, 0)}, + } + return sessions + + +def grok_runs(rows): + """usage.py's run accounting: cumulative within a run, a run ends + when a REPORTED totalTokens shrinks, absent totals merge, the last + report per currency wins.""" + runs = [] + current = {} + current_at = None + prev_total = None + run_incomplete = False + for row in rows: + update = (row.get("params") or {}).get("update") + if not (isinstance(update, dict) + and update.get("sessionUpdate") == "turn_completed"): + continue + usage = update.get("usage") + if not isinstance(usage, dict) or not set(usage) & _GROK_CURRENCIES: + continue + total = usage.get("totalTokens") + if (total is not None and prev_total is not None + and total < prev_total): + if current: + runs.append((current, current_at, run_incomplete)) + current = {} + run_incomplete = False + if total is not None: + prev_total = total + current.update(usage) + if usage.get("usageIsIncomplete"): + run_incomplete = True + current_at = row.get("timestamp") + if current: + runs.append((current, current_at, run_incomplete)) + return runs + + +def _summary_overlaps(summary, since, until): + try: + created = parse_stamp(summary.get("created_at")) + updated = parse_stamp(summary.get("updated_at")) + except ValueError: + return False + return created < until and updated >= since + + +def grok_sessions(root, repo, window): + since, until = window + sessions = {} + base = Path(root) / "sessions" / quote(repo, safe="") + for session_dir in sorted(base.iterdir()) if base.is_dir() else []: + try: + summary = json.loads( + (session_dir / "summary.json").read_text(encoding="utf-8")) + except (OSError, ValueError): + continue # not even attributable to this repo + if (summary.get("info") or {}).get("cwd") != repo: + continue + sid = session_dir.name + updates = read_jsonl(session_dir / "updates.jsonl") + if updates is None: + # attributable session whose spend is unreadable: it MUST + # hold a place in the counted=N/M denominator, or a store + # with holes reports itself complete (Codex audit, PR #31) + if _summary_overlaps(summary, since, until): + sessions[sid] = {"runs": 0, "incomplete": False} + continue + runs = grok_runs(updates) + if any(at is not None and not isinstance(at, (int, float)) + for _, at, _ in runs): + # a non-numeric run timestamp is a malformed store shape + # (events.jsonl uses ISO; updates.jsonl is epoch) — the + # session's spend is unreadable, not zero and not a crash + if _summary_overlaps(summary, since, until): + sessions[sid] = {"runs": 0, "incomplete": False} + continue + in_window = [] + incomplete = False + for usage, at_epoch, run_incomplete in runs: + if at_epoch is None: + continue + at = datetime.fromtimestamp(at_epoch, timezone.utc) + if since <= at < until: + in_window.append(usage) + # incompleteness rides the runs INSIDE the window: an + # incomplete run elsewhere says nothing about these + if run_incomplete: + incomplete = True + if runs and not in_window: + continue # usage exists, none of it in this window + if not runs: + present = any( + since <= datetime.fromtimestamp(row.get("timestamp", 0), + timezone.utc) < until + for row in updates + if isinstance(row.get("timestamp"), (int, float)) + ) + if not present: + continue + record = {"runs": len(in_window), "incomplete": incomplete} + if in_window: + record["in"] = sum(u.get("inputTokens", 0) for u in in_window) + record["cached"] = sum(u.get("cachedReadTokens", 0) + + u.get("cacheCreationTokens", 0) + for u in in_window) + record["out"] = sum(u.get("outputTokens", 0) for u in in_window) + record["total"] = sum(u.get("totalTokens", 0) for u in in_window) + record["cached_read"] = sum(u.get("cachedReadTokens", 0) + for u in in_window) + if all("costUsdTicks" in u for u in in_window): + record["ticks"] = sum(u["costUsdTicks"] for u in in_window) + heaviest = max(in_window, + key=lambda u: u.get("outputTokens", 0)) + record["heavy"] = {"out": heaviest.get("outputTokens", 0)} + sessions[sid] = record + return sessions + + +# ---------------------------------------------------------------- git + +def run_git(repo, *args): + proc = subprocess.run(["git", "-C", repo, *args], + capture_output=True, text=True, check=False) + return proc.stdout if proc.returncode == 0 else None + + +def repo_results(repo, window): + since, until = window + stdout = run_git(repo, "log", "--first-parent", + "--format=%H%x00%cI%x00%s%x00%P", "HEAD") + line_log = [] + for line in (stdout or "").splitlines(): + parts = line.split("\x00") + if len(parts) == 4: + sha, cdate, subject, parents = parts + line_log.append((sha, parse_iso(cdate, "commit date"), + subject, parents.split())) + merges, prs, commits = [], [], 0 + for sha, at, subject, parents in line_log: + if not since <= at < until: + continue + if len(parents) > 1: + entry = {"sha": sha[:7], "subject": subject} + numbered = MERGE_PR.match(subject) + if numbered: + entry["pr"] = int(numbered.group(1)) + prs.append(int(numbered.group(1))) + merges.append(entry) + else: + commits += 1 + + def edge_sha(predicate): + for sha, at, _, _ in line_log: + if predicate(at): + return sha + return None + + def count_tests(sha): + # git grep -I skips binary blobs, so an image in the tree is + # not a decode crash; exit 1 just means zero definitions + proc = subprocess.run( + ["git", "-C", repo, "grep", "-I", "-c", "-E", + r"^[[:space:]]*(#\[test\][[:space:]]*$|def test_)", sha], + capture_output=True, text=True, check=False) + if proc.returncode == 1: + return 0 + if proc.returncode != 0: + return None + return sum(int(line.rsplit(":", 1)[1]) + for line in proc.stdout.splitlines() if ":" in line) + + since_sha = edge_sha(lambda at: at <= since) + until_sha = edge_sha(lambda at: at < until) + tests = {} + since_count = count_tests(since_sha) if since_sha else None + until_count = count_tests(until_sha) if until_sha else None + tests["since"] = "unrecorded" if since_count is None else since_count + tests["until"] = "unrecorded" if until_count is None else until_count + if since_count is None or until_count is None: + tests["delta"] = "unrecorded" + else: + tests["delta"] = until_count - since_count + merges.reverse() # oldest first, the order they landed + return {"commits": commits, "prs": prs, "merges": merges, + "tests": tests} + + +# ---------------------------------------------------------------- output + +def cost_record(harness, sessions): + if not sessions: + return {"sessions": 0, "tokens": "unrecorded"} + record = {"sessions": len(sessions)} + if harness == "grok": + counted = {sid: s for sid, s in sessions.items() if s["runs"]} + record["runs"] = sum(s["runs"] for s in counted.values()) + # counted=N/M always accompanies a non-empty session list: its + # absence is how a store with holes passes as complete + record["counted"] = f"{len(counted)}/{len(sessions)}" + if counted: + for key in ("in", "cached", "out", "total"): + record[key] = sum(s[key] for s in counted.values()) + if all("ticks" in s for s in counted.values()): + record["cost_usd_ticks"] = sum( + s["ticks"] for s in counted.values()) + else: + record["cost_usd_ticks"] = "unrecorded" + record["incomplete"] = any( + s["incomplete"] for s in counted.values()) + else: + record["tokens"] = "unrecorded" + else: + record["messages"] = sum(s["messages"] for s in sessions.values()) + for key in ("in", "cached", "out", "total"): + record[key] = sum(s[key] for s in sessions.values()) + return record + + +def plain_cost_line(harness, record): + parts = [f"{harness:7s}"] + for key in ("sessions", "messages", "runs", "in", "cached", "out", + "total", "counted", "cost_usd_ticks", "tokens"): + if key in record: + parts.append(f"{key}={record[key]}") + if record.get("incomplete"): + parts.append("(incomplete)") + return " ".join(parts) + + +def stamp_block(repo, window, now, harness_sessions=None): + since, until = window + head = (run_git(repo, "rev-parse", "--short", "HEAD") or "").strip() + branch = (run_git(repo, "rev-parse", "--abbrev-ref", "HEAD") or "").strip() + stamp = {"head": head or "unrecorded", "branch": branch or "unrecorded", + "since": since, "until": until, "now": now} + if harness_sessions is not None: + stamp["sessions"] = harness_sessions + return stamp + + +def plain_stamp(stamp): + line = (f"stamp head={stamp['head']} branch={stamp['branch']} " + f"window=[{stamp['since']}, {stamp['until']}) " + f"now={stamp['now']}") + return line + + +def gather(args, window): + repo = str(args.repo) + return { + "claude": claude_sessions(args.claude_dir, repo, window), + "codex": codex_sessions(args.codex_dir, repo, window), + "grok": grok_sessions(args.grok_dir, repo, window), + } + + +def cmd_report(args, window, window_iso): + per_harness = gather(args, window) + cost = {harness: cost_record(harness, sessions) + for harness, sessions in per_harness.items()} + results = repo_results(str(args.repo), window) + stamp = stamp_block(str(args.repo), window_iso, args.now) + data = {"stamp": stamp, "cost": cost, "results": results} + if args.format == "json": + print(json.dumps(data, sort_keys=True)) + return + lines = [plain_stamp(stamp), ""] + for harness in ("claude", "codex", "grok"): + lines.append(plain_cost_line(harness, cost[harness])) + lines.append("") + prs = ",".join(str(n) for n in results["prs"]) or "0" + lines.append(f"results commits={results['commits']} prs={prs}") + for merge in results["merges"]: + label = f"#{merge['pr']} " if "pr" in merge else "" + lines.append(f"merge {label}{merge['sha']} {merge['subject']}") + tests = results["tests"] + lines.append(f"tests delta={tests['delta']} since={tests['since']}" + f" until={tests['until']}") + print("\n".join(lines)) + + +def cmd_waste(args, window, window_iso): + per_harness = gather(args, window) + ranked = [] + for harness, sessions in per_harness.items(): + for sid, s in sessions.items(): + if "total" not in s: + continue + entry = {"harness": harness, "session": sid, + "total": s["total"], "out": s["out"], + "cached": s["cached"]} + if harness == "grok": + entry["runs"] = s["runs"] + else: + entry["messages"] = s["messages"] + ranked.append((s, entry)) + ranked.sort(key=lambda item: (-item[1]["total"], item[1]["session"])) + ranked = ranked[:args.top] + + signals = [] + for s, entry in ranked: + if s["out"] > 0 and s["cached_read"] > CHURN_FACTOR * s["out"]: + signals.append({"kind": "cache-churn", + "harness": entry["harness"], + "session": entry["session"], + "cached_read": s["cached_read"], + "out": s["out"]}) + for s, entry in ranked: + heavy = s.get("heavy") + if heavy: + signal = {"kind": "heavy-turn", "harness": entry["harness"], + "session": entry["session"]} + signal.update(heavy) + signals.append(signal) + + counts = {harness: len(sessions) + for harness, sessions in per_harness.items()} + stamp = stamp_block(str(args.repo), window_iso, args.now, + harness_sessions=counts) + data = {"stamp": stamp, "sessions": [entry for _, entry in ranked], + "signals": signals} + if args.format == "json": + print(json.dumps(data, sort_keys=True)) + return + lines = [plain_stamp(stamp), ""] + for harness in ("claude", "codex", "grok"): + lines.append(f"{harness:7s} sessions={counts[harness]}") + lines.append("") + for _, entry in ranked: + parts = [f"{entry['harness']:7s} session={entry['session']}"] + for key in ("total", "out", "cached", "messages", "runs"): + if key in entry: + parts.append(f"{key}={entry[key]}") + lines.append(" ".join(parts)) + for signal in signals: + parts = [(f"signal {signal['kind']} harness={signal['harness']}" + f" session={signal['session']}")] + for key, value in signal.items(): + if key not in ("kind", "harness", "session"): + parts.append(f"{key}={value}") + lines.append(" ".join(parts)) + print("\n".join(lines)) + + +def main(argv=None): + parser = argparse.ArgumentParser(prog="worth.py", description=__doc__) + parser.add_argument("verb", choices=("report", "waste")) + parser.add_argument("--repo", required=True) + parser.add_argument("--now") + parser.add_argument("--since") + parser.add_argument("--until") + parser.add_argument("--top", type=int, default=5) + parser.add_argument("--format", choices=("plain", "json"), + default="plain") + parser.add_argument("--claude-dir", + default=str(Path.home() / ".claude" / "projects")) + parser.add_argument("--codex-dir", default=str(Path.home() / ".codex")) + parser.add_argument("--grok-dir", default=str(Path.home() / ".grok")) + args = parser.parse_args(argv) + + try: + if args.now is None: + now = datetime.now(timezone.utc) + args.now = now.isoformat(timespec="milliseconds").replace( + "+00:00", "Z") + else: + now = parse_iso(args.now, "now") + until = parse_iso(args.until, "until") if args.until else now + since = (parse_iso(args.since, "since") if args.since + else now - timedelta(hours=24)) + if until <= since: + raise SystemExit2( + f"until ({args.until or args.now}) must be after" + f" since ({args.since})") + except SystemExit2 as exc: + print(str(exc), file=sys.stderr) + return 2 + + since_iso = args.since or since.isoformat( + timespec="milliseconds").replace("+00:00", "Z") + until_iso = args.until or args.now + window = (since, until) + window_iso = (since_iso, until_iso) + if args.verb == "report": + cmd_report(args, window, window_iso) + else: + cmd_waste(args, window, window_iso) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ops/devlane/workflow/checks/term_wall.py b/ops/devlane/workflow/checks/term_wall.py new file mode 100644 index 0000000..8feb21c --- /dev/null +++ b/ops/devlane/workflow/checks/term_wall.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""The term wall: names this organisation does not use, refused everywhere. + +Some names must not appear in this organisation's trees, commit +messages, or pull-request text — not affirmed, not negated, not cited. +This check is the wall. It never spells the names it refuses (the +pattern below would otherwise be its own first hit) and every hit it +prints is masked, so its output does not carry what the tree may not. + + term_wall.py [--root DIR] [PATH ...] tracked files (default: all) + term_wall.py --message-file FILE one commit message + term_wall.py --range BASE..HEAD every commit message in the range + term_wall.py --stdin text on stdin + +Exit 0 clean; 1 on a hit, every hit printed; 2 on a refusal, one line +on stderr in the lane's shape: `class: expected …; found …; needed …`. +The same wall stands in CI (the org's term-wall action) — this copy is +the local hook's and the landing lever's. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path +import re + +#: The names, spelled so that this file passes its own wall. +WALL = re.compile(r"s[c]ient[ _-]?db|u[s]cient", re.IGNORECASE) +MASK = "[forbidden name]" +SKIP_SUFFIXES = {".png", ".jpg", ".jpeg", ".gif", ".pdf", ".zip", ".gz", ".db", ".phar"} + + +def refuse(cls: str, expected: str, found: str, needed: str) -> None: + print(f"{cls}: expected {expected}; found {found}; needed {needed}", file=sys.stderr) + sys.exit(2) + + +def scan_text(text: str, where: str) -> list[str]: + hits = [] + for n, line in enumerate(text.splitlines(), 1): + if WALL.search(line): + hits.append(f"{where}:{n}: {WALL.sub(MASK, line).strip()[:160]}") + return hits + + +def tracked(root: Path, paths: list[str]) -> list[str]: + try: + out = subprocess.run( + ["git", "-C", str(root), "ls-files", "-z", "--", *paths], + capture_output=True, check=True, + ).stdout + except (OSError, subprocess.CalledProcessError) as exc: + refuse("root", "a git work tree", f"{root} ({exc})", "run inside a clone or pass --root") + return [p for p in out.decode("utf-8", "surrogateescape").split("\0") if p] + + +def scan_tree(root: Path, paths: list[str]) -> list[str]: + hits = [] + for rel in tracked(root, paths): + if WALL.search(rel): + hits.append(f"{rel}: forbidden name in the path ({WALL.sub(MASK, rel)})") + p = root / rel + if p.suffix.lower() in SKIP_SUFFIXES or not p.is_file(): + continue + data = p.read_bytes() + if b"\0" in data[:8000]: + continue + hits.extend(scan_text(data.decode("utf-8", "replace"), rel)) + return hits + + +def scan_range(root: Path, rng: str) -> list[str]: + try: + out = subprocess.run( + ["git", "-C", str(root), "log", "--format=%H%x00%B%x00", rng], + capture_output=True, check=True, + ).stdout + except (OSError, subprocess.CalledProcessError) as exc: + detail = getattr(exc, "stderr", b"") or b"" + refuse("range", "BASE..HEAD git can resolve", + f"{rng} ({detail.decode('utf-8', 'replace').strip() or exc})", + "a range of reachable commits") + hits = [] + parts = out.decode("utf-8", "replace").split("\0") + for sha, body in zip(parts[0::2], parts[1::2]): + sha = sha.strip() + if sha: + hits.extend(scan_text(body, f"commit {sha[:12]}")) + return hits + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + mode = ap.add_mutually_exclusive_group() + mode.add_argument("--message-file", help="one commit message, as the commit-msg hook is given it") + mode.add_argument("--range", dest="rng", help="BASE..HEAD — every commit message in the range") + mode.add_argument("--stdin", action="store_true", help="scan text on stdin") + ap.add_argument("--root", default=".", help="the work tree (default: current directory)") + ap.add_argument("paths", nargs="*", help="tracked paths to scan (default: every tracked file)") + args = ap.parse_args(argv) + root = Path(args.root) + + if args.message_file: + path = Path(args.message_file) + if not path.is_file(): + refuse("message", "a readable message file", str(path), "the path git hands the commit-msg hook") + hits = scan_text(path.read_text(encoding="utf-8", errors="replace"), "message") + elif args.rng: + hits = scan_range(root, args.rng) + elif args.stdin: + hits = scan_text(sys.stdin.read(), "stdin") + else: + if not root.is_dir(): + refuse("root", "a directory", str(root), "an existing work tree") + hits = scan_tree(root, args.paths) + + for hit in hits: + print(hit) + return 1 if hits else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ops/devlane/workflow/checks/vocabulary_wall.py b/ops/devlane/workflow/checks/vocabulary_wall.py new file mode 100755 index 0000000..6876d9b --- /dev/null +++ b/ops/devlane/workflow/checks/vocabulary_wall.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +"""The two-direction vocabulary wall, as a registered check (PLAN §10). + +Dev-lane terms — work order, gate, stage, receipt, red/green, frozen set, +chain, claim, marker, packet, wf, capability — exist only under `.dev/`, in the root +dev-lane surfaces, and in dev-lane commits and PRs. They never appear in +the product's design artifacts, contracts or specs: the product describes itself +in its own vocabulary, and a process word in a design artifact is scope +leakage. + +Exit 0 only when both passes are clean; exit 1 when either pass finds a term. +Prints every hit, because a check that says only "failed" makes the reader +go and re-run it by hand. + + vocabulary_wall.py [--root DIR] [PATH ...] + +With no PATH, scans the design surfaces that exist: everything outside +`.dev/`, `.git/`, and the root dev-lane files that are allowed to use the +vocabulary. + +The compose term is checked in the other direction across the dev-lane +surfaces. It is allowed only in `.dev/infra/` (compose), +`.dev/docs/scratch/` (research), `.dev/handoffs/` (systemd), +`.dev/records/` (evidence), `.dev/conductor/` +(conductor records quote evidence), and `.git/` (not a tree surface). +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +#: Word-boundary patterns. `wf` is matched only as a bare word so that +#: ordinary English containing the letters is not a false positive. +RESERVED = [ + "work order", "work orders", "gate", "gates", "stage", "stages", + "receipt", "receipts", "red stage", "green stage", "frozen set", + "chain", "claim", "claims", "marker", "markers", "packet", "packets", + "wf", "capability", "capabilities", +] + +#: Surfaces where the vocabulary is allowed to appear. The class, not the +#: instances: root dot-directories holding tool and harness configuration +#: (.claude/ adapters, .serena/ code-assist config and memories) are +#: dev-lane surface — they describe how the repo is worked on, not what +#: the product is. Design artifacts never live in a dot-directory here. +ALLOWED_PREFIXES = (".dev/", ".git/", ".github/", ".claude/", ".serena/") +#: `ruff.toml` joins them for the same reason as the dot-directories: +#: it configures how this repo is worked on, and it cannot do that +#: without naming dev-lane paths — a per-file rule for +#: `.dev/app/workflow/wf.py` contains a reserved word in the PATH. +ALLOWED_FILES = ("AGENTS.md", "CONTRIB.md", "README.md", "ruff.toml") + +TEXT_SUFFIXES = {".md", ".txt", ".rst", ".adoc", ".cue", ".toml", ".yaml", ".yml"} +CODE_SUFFIXES = (".py", ".json") +SERVICE_TERMS = ("serv\u0069ce", "serv\u0069ces") +SERVICE_RULE = "vocab.service-outside-infra" +PROD_TERMS = ("ker\u006eel",) +PROD_RULE = "vocab.kernel-outside-homes" +SERVICE_HOMES = ( + ".dev/infra/", + ".dev/docs/scratch/", + ".dev/handoffs/", + ".dev/records/", + ".dev/conductor/", + ".git/", +) + + +def patterns_for(terms): + return [(term, re.compile(rf"(? bool: + return any(rel.startswith(prefix) for prefix in prefixes) + + +def service_files(root: Path, explicit=()): + candidates = [] + if explicit: + for raw in explicit: + path = Path(raw).resolve() + if path.is_dir(): + candidates.extend(p for p in path.rglob("*") if p.is_file()) + elif path.is_file(): + candidates.append(path) + else: + for name in ALLOWED_FILES: + path = root / name + if path.is_file(): + candidates.append(path) + for prefix in ALLOWED_PREFIXES: + base = root / prefix.rstrip("/") + if base.is_file(): + candidates.append(base) + elif base.is_dir(): + candidates.extend(p for p in base.rglob("*") if p.is_file()) + suffixes = TEXT_SUFFIXES | set(CODE_SUFFIXES) + for path in sorted(set(candidates)): + rel = path.relative_to(root).as_posix() + if path.suffix.lower() in suffixes: + yield path, under_prefix(rel, SERVICE_HOMES) + + +def main() -> int: + parser = argparse.ArgumentParser(description="dev-lane vocabulary wall") + parser.add_argument("--root", default=".") + parser.add_argument("paths", nargs="*") + args = parser.parse_args() + + root = Path(args.root).resolve() + compiled = patterns() + hits, scanned = [], 0 + + for path in design_files(root, args.paths): + scanned += 1 + try: + text = path.read_text(encoding="utf-8") + except (UnicodeDecodeError, OSError): + continue + for number, line in enumerate(text.splitlines(), 1): + for term, pattern in compiled: + if pattern.search(line): + try: + shown = path.relative_to(root).as_posix() + except ValueError: + shown = str(path) + hits.append(f"{shown}:{number}: {term!r} in: {line.strip()[:90]}") + break + + print("pass 1") + failed = bool(hits) + if hits: + print(f"vocabulary wall: {len(hits)} dev-lane term(s) in the design tree") + for hit in hits: + print(f" {hit}") + print("These words belong under .dev/ only (PLAN §10).") + elif scanned == 0: + # A tree with no design surface yet is vacuously clean, and + # sandboxes rely on that — but it must not SAY clean, because + # the repo's own wall shrinks every time a prefix or file is + # allowed, and "nobody looked" reads exactly like "we checked" + # once it does. test_vocabulary_wall pins that the real tree + # never reaches this line. + print("vocabulary wall: nothing to scan — 0 design file(s);" + " no dev-lane term can be present, and none was looked for") + else: + print(f"vocabulary wall: clean — {scanned} design file(s) scanned," + f" {len(RESERVED)} reserved term(s)") + + service_patterns = patterns_for(SERVICE_TERMS) + prod_patterns = patterns_for(PROD_TERMS) + compound_patterns = compound_service_patterns() + service_hits, service_scanned = [], 0 + for path, exempt in service_files(root, args.paths): + try: + text = path.read_text(encoding="utf-8") + except (UnicodeDecodeError, OSError): + continue + # Homes are read so the scan count is evidence of files actually + # opened; their contents are deliberately outside this rule. + service_scanned += 1 + if exempt: + continue + if path.stem.casefold() in SERVICE_TERMS: + shown = path.relative_to(root).as_posix() + service_hits.append( + f"{SERVICE_RULE}: {shown}:0: " + f"{SERVICE_TERMS[0]!r} in: path component {path.name!r}") + shown = path.relative_to(root).as_posix() + if any(part.casefold() in PROD_TERMS for part in Path(shown).parts): + service_hits.append( + f"{PROD_RULE}: {shown}:0: " + f"{PROD_TERMS[0]!r} in: path component") + for number, line in enumerate(text.splitlines(), 1): + patterns_here = [ + (SERVICE_RULE, SERVICE_TERMS[0], pattern) + for _term, pattern in service_patterns + ] + patterns_here.extend( + (PROD_RULE, PROD_TERMS[0], pattern) + for _term, pattern in prod_patterns + ) + rel = path.relative_to(root).as_posix() + defines_rule = ( + rel == ".dev/app/workflow/checks/vocabulary_wall.py" + or rel.startswith(".dev/app/workflow/tests/") + ) + if not defines_rule and path.suffix.lower() in {".py", ".json", ".yaml", ".yml"}: + patterns_here.extend( + (SERVICE_RULE, SERVICE_TERMS[0], pattern) + for pattern in compound_patterns + ) + for rule, term, pattern in patterns_here: + if pattern.search(line): + shown = path.relative_to(root).as_posix() + service_hits.append( + f"{rule}: {shown}:{number}: " + f"{term!r} in: {line.strip()[:90]}") + break + print("pass 2") + if service_hits: + failed = True + for hit in service_hits: + print(hit) + elif service_scanned == 0: + print("vocabulary wall: nothing to scan — 0 file(s); compose term was not looked for") + else: + print(f"vocabulary wall: clean — {service_scanned} file(s) scanned," + f" {len(SERVICE_TERMS)} compose term(s)") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ops/process/agentic-management.md b/ops/process/agentic-management.md new file mode 100644 index 0000000..225fafb --- /dev/null +++ b/ops/process/agentic-management.md @@ -0,0 +1,409 @@ +# Agentic Management: a framework for the Conductor + +A **Conductor** is a single AI session that manages a fleet of AI +agent-harnesses (dispatched Claude/Codex/Grok sessions, sub-agents, +CI jobs) to produce work no single session could hold in context. +Management theory was built for humans — durable, expensive, singular, +prone to shirking and politics. None of that is true of an agent +instance, so the theory doesn't transfer by analogy; it has to be +**inverted**, term by term, against what's actually true of the fleet. + +This is that inversion: each classical concept below, named and +grounded, then rebuilt for a manager whose reports are cheap, parallel, +stateless, stochastic, and disposable — and whose real job is not to +lead them but to build the system that catches them when they're wrong. + +--- + +## Part 1 — Org structures, adapted + +### Functional vs. divisional vs. matrix + +Functional structure groups people by shared skill (engineering, +sales); divisional structure groups them by self-contained product, +market, or geography; matrix structure overlays both, giving each +person two chains of command it must reconcile through negotiation. + +None of the three describe a *standing group* of agents, because agents +have no tenure to belong to a group with. What survives: + +| human form | agentic form | +|:--|:--| +| Functional dept. (persistent team, shared skill) | A **role card** (planner, test-author, test-skeptic, implementer, adjudicator) — a job description with no incumbent. Any harness is cast into it for one stage, then the role is empty again. | +| Divisional unit (self-contained P&L) | A **work order**: one artifact's full lineage (plan → tests → code → review), state-isolated from every other work order. Two work orders never share an agent's context, so there's no cross-division politics to manage — there's no memory for it to live in. | +| Matrix (dual authority, resolved by negotiation) | Matrix's problem — an agent pulled two directions by two authorities — has no negotiation-based fix for a stateless worker; it needs a deterministic one. Both dimensions' demands get compiled into **one closed contract** the output must satisfy. The contract *is* the matrix intersection, made checkable instead of political. | + +### Mintzberg's organizational configurations and six coordination mechanisms + +Henry Mintzberg's five (later six) configurations — Simple Structure, +Machine Bureaucracy, Professional Bureaucracy, Divisionalized Form, +Adhocracy, Missionary — are each built around whichever of his six +coordination mechanisms dominates. Mapped to a fleet: + +| coordination mechanism | human default use | agentic form | cost | +|:--|:--|:--|:--| +| **Mutual adjustment** (informal back-and-forth) | Adhocracy; novel, ill-structured work | Multi-harness negotiation — cross-review disputes, adjudicated verdicts — bounded by a fixed wire format (verdict / stamp / findings) so it can't sprawl into unbounded chat | highest; reserve for genuine judgment calls | +| **Direct supervision** (one issues orders) | Simple structure | The Conductor dispatching one harness with an explicit brief and nothing else. The **default** mode — agents have no initiative to self-organize a division of labor, so someone has to hand out the work orders | cheap per call, doesn't scale past the Conductor's own verification bandwidth | +| **Standardization of work processes** | Machine bureaucracy | The gate *sequence* itself — every work order passes through the same stages (spec → red → impl → review → ratify) regardless of what it's building | near-zero once the pipeline is built | +| **Standardization of outputs** | Divisionalized form | **Contracts** — closed schemas / acceptance criteria an artifact must satisfy. This is the mechanism that lets agents run in true parallel with zero cross-talk, because they don't need to coordinate with each other at all if their outputs both have to clear the same gate | near-zero at run time, expensive to author well | +| **Standardization of skills** | Professional bureaucracy | Professionals train a skill into a person over years; an agent's "skill" is fixed at the vendor and can't be developed on the job. What substitutes is **capability routing**: the Conductor tracks which harness is empirically strong at which task-shape and casts accordingly, redoing the match every dispatch instead of trusting a trained-in competence | moderate — requires measuring, not assuming, capability | +| **Standardization of norms** | Missionary org | The weakest mechanism for agents: a human absorbs culture and carries it for years; a stateless instance carries nothing between sessions. Norms survive only as **re-injected doctrine** — CLAUDE.md / AGENTS.md pointer files read fresh into every context — never as something an agent "already knows" from having worked here before | must be paid every session; never amortizes | + +**The one that matters most at scale is standardization of outputs.** +Mutual adjustment doesn't work between agents with no shared memory; +direct supervision doesn't scale past what one Conductor can review. +Contracts are the only mechanism whose cost doesn't grow with fleet +size — which is why "write the contract" is the Conductor's highest- +leverage act, not "assign the work." + +### Span of control + +Classical span-of-control theory (the concern goes back to V.A. +Graicunas's 1933 analysis of the combinatorics of subordinate +relationships) holds that a manager's effective span shrinks as task +interdependence and novelty rise — commonly cited ranges run 5–9 +direct reports for complex work, wider for routine work. + +For a Conductor, the limiting resource isn't attention span — a +Conductor doesn't get tired supervising report #12. It's **verification +bandwidth**: the number of independently-dispatched results the +Conductor can actually check against ground truth (not just read a +summary of) before acting on any of them, plus the blast radius of N +agents mutating shared state concurrently (merge conflicts, races on +the same files). If N results can't each be independently verified, N +is too high regardless of how cheap dispatch was. + +*Concrete practice*: a hard concurrency cap on parallel dispatches, +paired with a monitoring dashboard over job handoffs — one monitor +armed *at* dispatch time, not as a separate manual step afterward. +(Lesson paid for the hard way in this repo: three jobs finished +unnoticed in one day because arming the watch was a step that kept +getting skipped — the fix was to make the launch itself *be* the +watch, so an unwatched job stops being a possible state.) + +### Centralization vs. decentralization + +Human orgs place decision rights on a spectrum from top-down to +delegated. For a fleet the axis that actually matters isn't *how much* +autonomy an agent has — decentralize execution maximally, since an +agent deciding its own tool calls and sub-steps costs nothing and +scales for free. It's **how reversible the thing a decision authorizes +is**. Ratification is centralized absolutely (one identity — the +owner — can merge to `main`) precisely because execution is +decentralized absolutely (any harness can propose anything). + +*Concrete practice*: centralization enforced structurally, not +socially — a branch-protection rule with the owner as sole bypass +turns "please don't merge to main" into a 403, so the control doesn't +depend on an agent choosing to comply. + +--- + +## Part 2 — Management philosophies, adapted + +### Theory X / Theory Y (Douglas McGregor, *The Human Side of Enterprise*, 1960) + +McGregor's dichotomy is about which assumption a manager makes about +*people*: X assumes workers are lazy and must be directed and +watched; Y assumes they're self-motivated and will exercise +self-direction if the goal is meaningful to them. Whichever a manager +believes tends to produce the behavior that confirms it. + +Applied to agents, the dichotomy is a category error — an agent isn't +lazy *or* self-actualizing, it's a stochastic generator with no +persistent stake in the outcome, so neither assumption describes +anything real. What both theories were actually arguing about — +whether to trust a report or check it — has one answer for agents, +independent of temperament: **check it**. Not because agents are bad +actors (Theory X) but because self-report from a stochastic process is +uncorrelated with whether the work is actually correct, at a rate that +doesn't go to zero no matter how "capable" the model. + +*Concrete practice*: every claim of "done" is backed by a command run +at the moment of the claim, not recalled from earlier in the +conversation — a report that says "tests pass" is worth nothing until +an independent, mechanical re-run confirms it against the artifact +that exists right now. + +### Lean / Toyota Production System — jidoka, poka-yoke, andon + +Sakichi Toyoda's automatic loom (which stopped itself the instant a +thread broke) is the root idea Taiichi Ohno systematized into +**jidoka** — "automation with a human touch": detect an abnormality, +stop the line immediately, fix the root cause, don't let a defect +travel downstream. **Poka-yoke** (Shigeo Shingo's term for +mistake-proofing) goes further than detection — it makes the error +*impossible to produce*, not just visible after the fact. **Andon** is +the visible signal — traditionally a cord or button — that gives the +worker closest to the defect the authority to halt production the +instant something looks wrong, rather than waiting for a supervisor to +notice it three stations later. + +| TPS concept | agentic form | +|:--|:--| +| Jidoka (stop the line on abnormality) | A CI gate that blocks a work order from advancing the instant an invariant breaks, instead of letting a bad artifact flow to the next stage. The stage sequence (spec → red → impl → review → ratify) simply refuses to move forward on a failure — the halt is structural, not a judgment call by whoever's watching. | +| Poka-yoke (make the error impossible, not just caught) | A **closed contract** with no accidental escape hatch — a schema that an entire class of bad output cannot satisfy, period, rather than one that merely gets flagged by a downstream check. And critically: a mistake-proofing device has to be tested *as* a device — a **negative-control corpus** of deliberately planted faults, each of which must trip the *specific* constraint it was planted to test (`must_draw`). A gate that reads clean by accident is worse than no gate — it's false confidence — so you verify the jig catches the part it's shaped to catch before you trust it to catch anything else. | +| Andon (signal + stop authority pushed to the point of production) | The "honest red" in a red→green loop: a failing test must fail on the *intended* assertion, never swallow the error as a collection or import failure. The agent closest to the defect (the one running the test) surfaces it the instant it's detected, in a form specific enough to act on — it doesn't wait for a supervisor to notice downstream. | + +### Deming / TQM + +W. Edwards Deming taught **statistical process control** — you can't +manage a process you haven't measured, and you set control limits from +the process's actual demonstrated capability, not from wishful +thinking. He argued for **"driving out fear"** because a workforce +that fears bad news will report good news instead — "where there is +fear, there will be wrong figures." **Constancy of purpose** means the +aim doesn't drift from one quarter to the next. And his sharpest line: +**cease dependence on inspection** — quality has to be built into the +process, not inspected into the product afterward. + +- *Statistical process control* → measure a check's actual precision + and recall before trusting it as a gate. A check with 3% precision + and 0% recall once narrowed (measured, not assumed, in this repo's + own history) is a process out of control — it doesn't get to gate + anything until its numbers justify it. +- *Drive out fear* → an agent has no career at stake, so "fear" isn't + the failure mode — but the structural analog is: never let the + harness being scored on a passing result also be the one producing + the result. A subagent incentivized toward "green" will find a way + to look green. The fix isn't reassurance, it's **removing the + incentive's target** — the implementer never grades its own work + (see segregation of duties, below). +- *Constancy of purpose* → the purpose that would live in + institutional memory for a human workforce lives, for a fleet, in a + versioned document (a contract, `AGENTS.md`, `CLAUDE.md`) re-read + into every fresh context — because no single agent instance carries + constancy on its own; the record has to. +- *Build quality in, don't inspect it in* → inspecting after the fact + assumes an inspector independent of what produced the thing. Tests + written by the same agent that wrote the code will fit the code that + exists. Writing the contract and the negative-control corpus + **before** any implementation exists (spec → red → impl) makes + quality a property of what's allowed to be built, not a filter + applied to what already was. + +### Management by Objectives (Peter Drucker, *The Practice of Management*, 1954) + +Drucker's MBO replaced supervision-based management with +results-based management: set objectives collaboratively, let people +self-direct toward them, review against results rather than watching +the process. Its classical failure mode is goals gamed to the letter +and not the spirit. + +There's no collaborative goal-setting with a stateless agent — it has +no continuity across a review period to be "committed" to anything. +What MBO's actual insight (manage the result, not the method) requires +for agents is that the objective arrive as a **closed, machine-checkable +spec**, handed over whole, not negotiated. Passing the gate isn't +*evidence* the objective was met — under a well-written contract it +*is* the objective being met, because the spec defines success +completely. This closes MBO's classical loophole rather than +inheriting it: a contract has no "spirit" to violate while satisfying +the letter, so the only way to game it is to find an actual hole in +the contract — which is a finding about the contract, not a +management failure, and feeds the next revision. + +*Concrete practice*: contract-as-objective, certified by construction +— a harness is never asked to self-assess against an objective; it's +handed a spec whose satisfaction is checked by the same gate for +everyone, every time. + +### Agency theory / principal-agent problem / information asymmetry + +Agency theory (formalized by Stephen Ross and Barry Mitnick in the +1970s, and by Michael Jensen and William Meckling's agency-cost +framework) studies what happens when a principal delegates to an agent +who has private information or unobservable effort, and whose +interests aren't perfectly aligned with the principal's — the classic +problems are *hidden characteristics* (adverse selection) and *hidden +action* (moral hazard). + +The informational asymmetry is real for an agent fleet — a dispatched +harness's context window, tool calls, and internal reasoning aren't +visible to the Conductor by default — but the *incentive* half of the +classical problem doesn't apply: an agent doesn't shirk to conserve +its own effort. So the entire residual problem is **observability**, +not motivation: does the Conductor have an independent way to check +what happened, or only the agent's own narration of it? + +This is also where information asymmetry stops being a problem to +minimize and becomes a **control to design on purpose**: a test-author +never opens the implementation it's writing tests against; an +adjudicator sees only the artifacts and the question, never the +discussion that produced them. Withholding information is how you +prevent an agent from rationalizing toward an answer it "expects" — +the agentic equivalent of a blinded trial. + +*Concrete practice*: **custody boundaries** — an explicit, per-role +statement of exactly what a harness may see, enforced by what's +actually placed in its context (a detached snapshot, not the live +worktree; a brief with material and a question, nothing prescriptive +added). + +### Internal controls & segregation of duties (COSO) + +The COSO framework's segregation-of-duties principle: no single +individual should be able to initiate, authorize, record, *and* +review the same transaction, because the four functions in one hand is +exactly the condition fraud and undetected error both need. + +Agentic form is the **two-producer firewall**: no producer owns two +consecutive artifacts. Whoever writes an implementation doesn't write +or approve its own tests; whoever plans doesn't implement; whoever +implements doesn't review. Pushed one layer further than COSO usually +goes: a CHANGES/REJECT verdict is itself ruled on by a *third* harness +that produced neither the artifact nor the finding — segregation of +duties applied recursively to the review layer, not just the +production layer, so the auditor can't mark its own audit either. + +And the record COSO calls for isn't a policy statement — it's a +structural property of the tooling: **append-only receipts**. A chain +that records who did what, that no verb in the system can rewrite, +only extend. Segregation of duties is enforced by what the system can +technically do, not by what agents are asked nicely not to do. + +### Situational leadership & delegation levels (Tannenbaum–Schmidt's leadership continuum, 1958; Hersey–Blanchard's situational leadership) + +Tannenbaum and Schmidt's continuum runs from the manager deciding +alone ("tell") to the team deciding independently ("delegate"), +matched to the group's competence and the task's risk. Hersey and +Blanchard's situational leadership matches style to a follower's +*readiness* — competence plus commitment — which is expected to grow +over time as a person is developed. + +Readiness-that-grows doesn't exist for an agent instance — a fresh +session has no track record of its own to have earned trust with. So +the axis a Conductor actually matches supervision to is two things +that *do* persist: (1) the harness's **measured capability** for this +task-shape, tracked at the fleet/model level across many instances, +never assumed for an individual session; and (2) the **reversibility** +of the action being delegated. A cheap-to-check, reversible action +(draft a plan, write a test) gets full delegation even to an +unproven harness, because a bad result costs one review cycle. An +irreversible or expensive-to-verify action (merge to main, delete +data, spend money) sits at the "tell" end of the continuum *regardless +of how capable the harness is* — capability doesn't buy back +irreversibility. + +*Concrete practice*: a delegation-level table keyed on (harness track +record × action reversibility), not on trust, tenure, or how good the +last five outputs looked. The most proven harness in the fleet still +gets its `main`-merge staged for human ratification — that cell of +the table never changes no matter who's in it. + +### RACI + +RACI (Responsible / Accountable / Consulted / Informed) clarifies role +ambiguity in cross-functional work — critically, exactly one person is +Accountable, so "the buck stops" somewhere specific, while several can +be Responsible for doing the work. + +RACI degenerates immediately if Accountable is assigned to an ephemeral +instance — an agent can't be held accountable across time if it no +longer exists when the question comes back. So the transformation is: +**Accountable is always a persistent identity** — the human owner, or +the standing Conductor role — never a dispatched instance. +**Responsible** is whichever harness is cast into a work-order stage +right now, tracked as a role assignment in a durable record rather than +a name anyone has to remember. **Consulted** becomes the cross-review +step — invoked deterministically by the process, not "whoever happens +to be free." **Informed** becomes the append-only receipt chain — any +later agent or the human can read what happened without having to ask +anyone. + +*Concrete practice*: ratification — the Accountable act — is never +delegated, ever, to any harness; everything else in the fleet cycles +through Responsible/Consulted/Informed roles every single dispatch. + +--- + +## Part 3 — The adaptation, made explicit + +Human management theory assumes its unit of management is scarce, +expensive, persistent, and roughly deterministic, and that the job is +to align that unit's *will* with the organization's goals. Every +invariant is false for a fleet: + +| invariant | human org | agent fleet | +|:--|:--|:--| +| cost of a "hire" | months of recruiting, a salary | one dispatch call | +| parallelism | one job per person | N identical instances at once | +| memory | carried in the person, across years | carried nowhere in the agent; lives in the record or not at all | +| behavior | consistent, improvable via feedback | stochastic, does not reliably learn within a session | +| exit | firing is costly, legal, slow | kill the process; free | + +Because the unit is disposable and stateless, motivating it is a +non-operation — there's no will to align and nothing persists to be +motivated for next time. The manager's actual leverage moves entirely +to what surrounds the agent: + +- **Don't motivate — design the gates.** A deterministic check that + refuses invalid output does the job "motivation" was standing in for. +- **Don't supervise the work — write the contract.** A closed, + checkable spec replaces both the pep talk and the micromanagement; + agents that never talk to each other coordinate perfectly if their + outputs both have to clear it. +- **Don't manage politics — set custody boundaries.** Deciding what an + agent may see is not damage control for information asymmetry, it's + the control itself, deployed on purpose. +- **Don't assign by seniority or availability — route by comparative + advantage.** Cast each stage to whichever harness is empirically + strongest at that task-shape, re-decided every dispatch, since no + agent instance carries a career for "seniority" to describe. +- **The system does the remembering and the refusing**, so the manager + doesn't have to hold either in its own head — a fact not in the + durable record does not exist for the next session; a violation not + caught by a gate does not get caught at all. + +--- + +## Operating model for the Conductor + +1. **Reserve the expensive model for judgment and routing** — + adjudicating disputes, deciding who does what next, final + verification. Never spend it on investigation or a first-draft + implementation a cheaper harness can produce. +2. **Delegate investigation and implementation.** A dispatched job is + disposable; the Conductor's own context is the scarce resource — + protect it, don't fill it with work a subagent could have done. +3. **Verify every consequential result with an independent harness + before acting on it.** The producer of a claim is never its sole + verifier — not because it's untrustworthy, but because self-report + is not evidence of anything. +4. **Deliver only what passes a local, deterministic gate.** A + harness's narration of success is not a substitute for a check run + against the artifact that exists right now, at this moment. +5. **Keep durable, append-only records of every dispatch, decision, + and verdict.** Anything that lives only in one session's volatile + context is invisible to the next session and might as well not have + happened. +6. **Stage irreversible or high-ambiguity actions for human + ratification** — merges to a protected branch, deletions, spend, + anything a gate can't fully specify. Capability never buys back + irreversibility; route by risk, not by how good the harness is. +7. **Cap concurrency to what can actually be verified**, and arm a + monitor at the moment of dispatch, not as a separate step after — + an unwatched job that finishes is indistinguishable from one that + never ran. +8. **Restrict what each agent can see, on purpose.** Custody + boundaries are a designed control, not an accident of how much + context happened to be handed over. +9. **Write the objective as a closed contract before work starts.** If + success can't be checked by a machine, it is not yet a real + objective — it's a hope. +10. **Never let one instance be both builder and sole judge of the + same artifact** — and apply that recursively: the judge of a + dispute is never the one who raised it or the one it's against. + +--- + +*Grounded in: Mintzberg's structural configurations and coordination +mechanisms; classical functional/divisional/matrix organization +design; span-of-control theory (Graicunas); McGregor's Theory X/Y; +the Toyota Production System (Toyoda, Ohno, Shingo) — jidoka, +poka-yoke, andon; Deming's statistical process control and 14 points; +Drucker's Management by Objectives; agency theory and the +principal-agent problem (Ross, Mitnick, Jensen & Meckling); the COSO +internal control framework and segregation of duties; the +Tannenbaum–Schmidt leadership continuum and Hersey–Blanchard +situational leadership; and RACI.* diff --git a/ops/process/bdd.md b/ops/process/bdd.md new file mode 100644 index 0000000..84a3c7d --- /dev/null +++ b/ops/process/bdd.md @@ -0,0 +1,60 @@ +# Behavior first, in Gherkin shape + +A scenario is a contract a non-implementer can dispute. Write them at the +**spec** stage, before tests exist, so the test-author role has something +to work from that is not the implementation. + +## Where they live + +`.dev/design/features//.feature` — tracked, reviewed in the PR +like any design artifact, and **in the tdd gate kind's `test_scope`**, so +sealing the frozen set at red freezes the scenarios with the tests: changing +a scenario afterwards is manifest drift that `wf green` refuses, and the +recorded route back is `wf advance --to red --reason`. Plain Gherkin syntax; +no Cucumber runtime is wired up — the value extracted here is the shared, +disputable contract, and the mapping below keeps it honest without a +step-definition layer. + +## Writing them + +```gherkin +Feature: + Scenario: + Given + When + Then +``` + +- One behavior per scenario; one `When` per scenario. If you need two, it is + two scenarios. +- `Then` must be observable from outside — an exit code, bytes, a row, a + refusal message. "The cache is consistent" is not observable; "a second + launch answers the same work orders" is. +- **Write the negative-space scenarios** — this repo's real defects lived + there: the empty input, the concurrent second actor, the interrupted + operation, the malformed file, the caller with the wrong identity. A + feature with only happy paths is half a contract. +- Refusals are behavior. This codebase treats a recorded denial as success + (exit 0) and distinguishes refusal from integrity — scenarios must too. + +## Traceability, both directions + +- Each test that implements a scenario carries the scenario name — in Rust, + in the test's doc comment; in Python, in the docstring. Grep must be able + to find the scenario in the test tree, e.g. + `grep -rn "Scenario: a second claimant is refused" crates/ .dev/` + (`crates/` is the gate kind's forward-looking test scope; it does not exist + until the product's first package lands). +- Before leaving red, sweep the feature file: every scenario either has a + test naming it, or an explicit `# deferred: ` line in the feature. + A scenario silently unimplemented is the gap nobody notices — the same + class as a check silently dropped from CI. +- Deferral is an escape valve, not a loophole: the deferred list goes in the + PR body verbatim, and the review stage judges it — a feature whose + negative-space scenarios are all deferred is a `finding`, not a pass. + +## Handoff + +Scenarios done → follow `ops/process/tdd.md`. Give whoever fills the +test-author role the feature file(s) and the gate-kind spec; the scenarios +are the contract they test against. diff --git a/ops/process/cross-review.md b/ops/process/cross-review.md new file mode 100644 index 0000000..e355505 --- /dev/null +++ b/ops/process/cross-review.md @@ -0,0 +1,244 @@ +# Cross-review: the other harnesses judge the one holding the work order + +Three harnesses work this repo — Claude, Codex, Grok — and any of them can +hold a work order: dispatch the work, route on what comes back, and commit. +Holding one is not a privilege of any harness, and **whoever holds it does +not review its own work: the others do.** Independent review is the only +thing that has reliably caught what a producer's own green suite missed, +whichever harness was producing. + +That role carries no name here on purpose. `Conductor` is the prod-lane +mini-app that drives the loop; in `ops/devlane/task/` and `.dev/guide/` it +means that program, and reusing it for a dev-lane session would put one +word on two objects across two lanes. + +**This file covers review only.** Who *produces* each artifact, in what +order, and what each producer is denied is `ops/process/pipeline.md` — +scope, plan, tests, code, and the rule that no producer owns two +consecutive artifacts. Reading this file alone gives three roles; the +sequence has six stages. + +## Who reviews what + +A constraint, not a roster: **no harness reviews an artifact it produced, +and no harness owns two consecutive artifacts.** `pipeline.md` fixes the +sequence; this says what review may not be. + +| artifact | produced by | reviewed by | +|:--|:--|:--| +| plan | Fable | Codex | +| tests | Grok | Codex | +| code | Opus | Grok **and** Codex | + +Filling two roles is allowed, and with six stages and four producers it is +unavoidable — Grok writes the tests and reviews the code, Codex checks the +tests and reviews the code. Neither reviews what it wrote, and neither +pair is consecutive. Roles are cards, not models: +`ops/process/roles/test-author.md` and `ops/process/roles/test-skeptic.md` +say what the role does; the table says only who fills it this cycle. + +Two reviewers rather than one, so the rule survives whoever is driving: if +the session that set the scope also reviews, the other reviewer is still +independent of it. + +**A fresh session is most of the protection.** Every reviewer is launched +cold against a detached snapshot, so a harness that produced earlier in +the sequence arrives carrying none of it — it cannot defend an artifact it +does not remember making. Context carried between roles is what turns a +second look into a rubber stamp, and there is none here. + +What that does *not* remove is a shared blind spot. The model that wrote a +weak test will not see the gap when it reviews code against that test, no +matter how cold the session, because the limitation is in the model and +not in its memory. Fresh sessions defeat motive; only a different model +defeats correlated error. That is why the two code reviewers are two +different harnesses, and why stage 4 is a third. + +**The residual, stated plainly:** the session that produced an artifact is +the one that decides which findings against it to act on. No fresh session +fixes that, because the adjudicating session is the producing session. The +owner is the backstop, and every finding must be answered in writing — +accepted or refuted with evidence — never silently dropped. + +## The snapshot rule (non-negotiable) + +Reviewers have tool access. They review a **detached snapshot**, never the +live worktree, so "review only" is enforced rather than trusted. One +snapshot per reviewer: + +```sh +SNAP=$(mktemp -d) +git archive | tar -x -C "$SNAP" +git diff > "$SNAP/REVIEW.diff" +git -C "$SNAP" init -q +git -C "$SNAP" add -A && git -C "$SNAP" commit -qm "snapshot of " +``` + +The commit matters: a reviewer that orients itself with `git rev-parse HEAD` +finds one, instead of an unborn branch it may misread as a broken checkout. +The snapshot is the enforcement — a reviewer can write only to a throwaway +copy. Sandbox flags below are defense in depth on top of it, not the +boundary itself. + +## Invoking each harness as a reviewer + +All of these run long (Grok has exceeded 10 minutes on a ~3.5k-line review). +Background them, capture to files, keep working: + +```sh +cd "$SNAP" # this reviewer's own snapshot +touch launched # marker: the supervisor finds the stream this launch opens +nohup grok --prompt-file prompt.txt --output-format plain --max-turns 40 \ + > review.txt 2> review.err & +RPID=$! + +# codex and claude launch the same way — prompt via stdin, never "$(cat ...)": +# codex exec --sandbox read-only - < prompt.txt +# claude -p --permission-mode plan < prompt.txt +``` + +Only two of the three can be told a ceiling of their own: `grok --max-turns +` (40 here — the 2026-08-24 verify completed in 16) and `claude +--max-budget-usd `. Codex has neither, on the CLI or in its config, +so for Codex the battery below is the only wall there is. + +Use the CLIs from PATH (a hard-coded home-directory path is one machine's +truth); `grok` also accepts `--sandbox ` and `--permission-mode` +where configured. **When a reviewer CLI is unreachable, run the ones that +are and name the missing reviewer in the PR body** — a review that silently +shrank is the same lie as a check silently dropped from CI. + +`$!` names the reviewer only when the launch is a plain background +command: `cd "$SNAP" && nohup … &` backgrounds the whole list, so `$!` +names a wrapper shell and a later `--terminate` kills the wrapper while +the reviewer runs on (verified with `ps -o comm=` on both forms). Feed +the prompt by file or stdin — `"$(cat prompt.txt)"` re-parses the +prompt's backticks and `$()` through the shell before the reviewer +sees it. + +### Arm the tripwire battery over every launch + +Nothing above watches a backgrounded reviewer, and a hung one has cost an +hour before anyone looked. Supervise each launch with the live battery, +`ops/devlane/telemetry/breaker.py`: the launch above dropped the `launched` +marker; find the stream the reviewer opened and point the battery at it +with the reviewer's PID: + +```sh +STORE=~/.grok/sessions # store root and stream pattern for this +PATTERN=updates.jsonl # reviewer's harness — see the table below +until STREAM=$(find "$STORE" -name "$PATTERN" -newer launched \ + -printf '%T@ %p\n' | sort -n | tail -1 | cut -d' ' -f2-); \ + [ -n "$STREAM" ] || ! kill -0 "$RPID"; do sleep 2; done +[ -n "$STREAM" ] && python3 ops/devlane/telemetry/breaker.py "$STREAM" --pid "$RPID" \ + --cap 2000000 --cap-out 150000 --stall 600 --size-mb 50 --terminate \ + --tripped-file TRIPPED.md 2>> breaker.log & +``` + +`-newer` alone can match a sibling session's stream when two runs of +the same harness overlap; the `sort` picks the newest by mtime, and a +truly concurrent launch should verify the stream is its own (Codex: +the rollout's session_meta cwd) before trusting the caps. + +The caps must be armed explicitly: `--cap` and `--cap-out` default to 0, +and a zero cap disables that wire — the table's token rows are only true +when the recipe passes them. The `kill -0` ends the wait if the reviewer +died before opening a stream; a rotted launch line exits in two seconds, +and nothing should sit waiting for its stream after that. + +`--cap` counts the harness's cumulative token total, and that is not +spend. On the verify run measured 2026-08-24, 2,047,199 total was +1,892,352 cache re-reads, 143,691 uncached input and 11,156 output, +against a context that peaked at 140,793. The wall tracks how long a +review ran, not what it cost, so a cap set from a cost figure kills a +healthy reviewer inside a few turns. `--cap-out` is the one token wire +that means what it says. Until a real-spend wire exists, leave `--cap` +high enough to catch only a runaway. + +Which stream, and which wires can fire (store shapes measured 2026-08-21): + +| reviewer | store root / pattern for the stream | wires | +|:--|:--|:--| +| Claude | newest `*.jsonl` under `~/.claude/projects//`, where `` is `$SNAP` with every `/` and `.` replaced by `-` | all six | +| Codex | newest `~/.codex/sessions/*/*/*/rollout-*.jsonl` | tokens, tokens-out, stall, size | +| Grok | `updates.jsonl` in the newest session dir under `~/.grok/sessions//` | tokens, tokens-out, stall, size | + +Grok records its spend (updates.jsonl `turn_completed` events, measured +2026-08-21) and the battery parses it — cumulative per run, runs split +when a reported total shrinks — so grok streams feed the token walls +(`--cap`, `--cap-out`) like the others. Only repeat-loop and +error-storm stay claude-only: grok streams carry no tool_use or +tool_result records to feed them. + +A trip is exit code 3: the battery prints its evidence to stderr, writes +the TRIPPED file named by `--tripped-file` into the snapshot, and with +`--terminate` it kills the runaway reviewer. A killed review did not +finish: name the tripped reviewer in the PR body exactly as you would an +unreachable one — a review cut short silently is the same lie as one that +silently shrank. Tune the wire's flag, or `--disable` the wire, only when +the tripped pattern turns out to be legitimate work. + +The prompt names the tip SHA, points at `REVIEW.diff`, says "review only — +do not edit", lists the specific decisions to aim skepticism at, and demands +the wire format from CONTRIB.md §Review protocol (VERDICT / STAMP / +FINDINGS with P1–P3) so stamps aggregate across harnesses. + +It also says how to land: **if you judge you are running long, stop and +write up what you have.** Every ceiling above kills the process and leaves +an empty `review.txt` — the 2026-08-24 Codex kill cost 2,047,199 tokens and +returned nothing, where a partial review would have been worth reading. +This is an exit instruction, not a budget. Never give a reviewer a token or +dollar figure: it has no counter to read, so the number cannot be obeyed, +only performed — and a reviewer that believes it is short of budget skims, +which is the opposite of why it gets a whole snapshot. + +The GitHub Codex bot is separate from local `codex exec`: **only the owner +triggers `@codex review` on a pull request** — no harness posts that trigger. + +## Acting on findings + +**A review that returns CHANGES is adjudicated. That is not a judgement +call.** + +The producer must not be the one deciding which findings against its own +work are worth acting on — that is the last dial left in the hands of the +party with the conflict, and no fresh session removes it, because the +session deciding is the session that produced. So: any review returning +CHANGES goes to a harness that produced **neither the artifact nor the +finding**, before any of it is worked. + +| findings against | producer | finder | adjudicator | +|:--|:--|:--|:--| +| plan | Fable | Codex | Grok, or the session | +| tests | Grok | Codex | Fable, or the session | +| code | Opus | Grok and Codex | **Fable — the only one left** | + +The adjudicator is given the artifact at the SHA that was reviewed and +the reports verbatim, and nothing else: not who wrote the branch, not +which findings the producer concedes, not what has been changed since. +It rules UPHELD / PARTIAL / REJECTED per finding, and is asked what the +reviewers missed — the 2026-08-25 run returned two such items, one of +them a hole the producer's own fix had opened. + +Rule on it before working it. A producer that fixes what it agrees with +first has already adjudicated, whatever it does afterwards. + +**This is prose, and prose runs nothing.** Making it a gate needs review +verdicts recorded as artifacts in the tree rather than left in a +scratchpad, so a check can refuse a merge that carries a CHANGES verdict +with no ruling against it. Until that exists the rule is honoured by +hand, which is exactly the condition under which the last one was +written down and ignored. + +- **Verify every finding against the live tree before applying it.** Not + because reviewers are often wrong — they have not been — but because the + verification is two commands and produces the reproduction that belongs in + the fix commit. +- The fix commit uses CONTRIB.md's template: `Finding:`/`Verified:` pairs, + `Co-Authored-By:` the finder, `Reviewed-by:` the reviewer. The reviewer + never pushes; the session holding the work order commits. +- A review is pinned to the SHA it read. Any push voids it; re-run against a + fresh snapshot rather than reporting an old pass as current. +- Record the literal stamp the reviewer printed as evidence, but write + trailers in the owner's prescribed form (e.g. Grok has stamped itself + `Grok 4`; trailers say `Grok 4.6 `). diff --git a/ops/process/pipeline.md b/ops/process/pipeline.md new file mode 100644 index 0000000..fb61d81 --- /dev/null +++ b/ops/process/pipeline.md @@ -0,0 +1,148 @@ +# The pipeline: who does what, and what they may not see + +`cross-review.md` says who *reviews*. This says who *produces*, in what +order, and what each producer is denied. It exists because the sequence +was run successfully on 2026-08-23, was written down nowhere a later +session would look, and was reconstructed the following night from a +15MB transcript. + +## The loop + +| # | stage | who | is given | is DENIED | +|:--|:--|:--|:--|:--| +| 1 | scope | this session, with the owner | the problem | — | +| 2 | plan | Claude | the scope | the freedom to move its boundaries | +| 3 | tests | Grok | the plan | the implementation (it does not exist yet) | +| 4 | check the tests | Codex | the tests and the plan | — | +| 5 | code | Claude | the tests | — | +| 6 | review the code | Grok **and** Codex | the PR | — | + +Harnesses, not models. A CLI's configured model changes under it — the +Codex that reviewed on 2026-08-25 was `gpt-5.6-sol` at `xhigh` while +stamping itself `GPT-5 Codex`. Dispatch by harness; read the model from +the run and record it as evidence. The owner names the model that fills +each role (`AGENTS.md` §The owner's standing rules): plan is always +Fable; the other fills are this cycle's. + +**The scope is the owner's, and that is what breaks the chain.** Rows 1 +and 2 would otherwise be one harness — a Claude session writing the +scope, Fable writing the plan — which is the violation this table +exists to forbid. The scope is not the session's artifact: it is the +owner's, settled in session, and the session assists. Where the owner +is not the author of a scope, stage 1 goes to a harness that does not +hold stage 2. + +**The scope is not the plan.** Scope is settled in session with the +owner — *what*, and the boundaries; the planner states *how*. It carries +no role name on purpose: `Conductor` is a prod-lane mini-app, and one +word for two objects across two lanes is a collision waiting to be made. Yesterday's scope document said it +in one line — "This file is the scope only. The plan is not mine to +write." Collapsing the two puts one author on both, which is the failure +the whole arrangement exists to prevent. + +**Tests come before code, and a different harness writes each.** Of 14 +review findings against two checkers built the other way round on +2026-08-24, **11 were missing test cases, not coding errors** — the +implementation did what the tests specified and the tests were +incomplete. The scarce skill is adversarial coverage, so stage 4 asks +"what input passes these tests and is still wrong?", not "are these +tests right?". + +**No producer owns two consecutive artifacts.** That is the property to +preserve if the assignment ever changes; the specific model names matter +less than that constraint. Two harnesses review rather than one so the +rule survives whoever is driving: if the session that set the scope is +also a reviewer, the other reviewer is still independent. + +The residual this arrangement carries: **both reviewers have a stake in +the tests** — Grok wrote them, Codex approved them — so code that +satisfies weak tests looks right to both. Stage 4 is the only thing +standing between weak tests and a clean review. Put the tests in scope +at stage 6 when the change is large. + +## A plan is discharged, not archived + +A plan is transient by nature and becomes permanent by accident: it +accumulates the durable content that has nowhere else to go, something +cites it, and then it cannot be retired. + +`ops/devlane/workflow/PLAN.md` is the worked example. It holds four +genres — the design (§§1–10), the phasing (§11), the ratified decisions +(§12) and the Slice 1 brief (§13). Only the last two are a plan. The +decisions were dated 2026-08-20, `.dev/docs/DECISIONS.md` was created +2026-08-21, and they were never migrated. The design had no home, +because the workflow app has no contract document — so the CUE +contracts were extracted from the plan, and 1,236 citations now point +at it. + +**Contracts are extracted from an app's contract document, never from a +plan.** That single rule is what keeps a plan disposable. + +When the work is done, discharge it: + +1. decisions → `.dev/docs/DECISIONS.md` +2. durable design → the app's `CONTRACT.md` +3. lessons → `.dev/docs/mini-app-lessons.md` +4. what remains — work items, ordering, slices — is spent; say so +5. after that nothing may cite it + +A plan that survives step 5 was never a plan. + +## The firewall must be proved, not intended + +Withholding is invisible: a snapshot that leaked the wrong file looks +exactly like one that did not. Staging four roles by hand and checking +afterwards with `find` produced one role firewalled on one side where +the plan specified both. + +Prove it in **both** directions before dispatching: + +- nothing matching a withheld pattern is present, **and** +- something matching every given pattern is present. + +An empty snapshot satisfies the first perfectly. A role staged from a +mistyped path leaks nothing, contains nothing, and then answers +questions about an empty directory. + +Build the snapshot from a manifest and refuse to dispatch one you cannot +prove. `ops/devlane/harness/` exists to do this; a hand-staged snapshot is +the thing it replaced. + +## The variant used for contract work + +When the deliverable is a CUE contract rather than code, the two +producers are split differently and meet at a file neither owns: + +- **contract author** reads the specification and *nothing else* — no + modules, no SQL, no JSON, no database. +- **extractor author** reads the code with the specification *removed + from its snapshot*, and writes mechanical extractors reporting what is + actually there. +- both implement against a neutral shape spec written before either + starts, which neither owns. +- `cue vet` compares. An adjudicator sees everything and rules. + +Neither author can reconcile a disagreement quietly, which is the point +and is not achievable with one author however careful: whoever holds +both halves resolves a discrepancy in passing and never mentions it. + +**Measured**: twelve contracts against twelve observations produced +**38 disagreements**, four confirmed against the tree by hand — among +them `wf.py` declaring 17 CLI verbs where the specification documents +16. + +## The evidence behind all of this + +Three documents on the shelf, `.dev/docs/`: + +- `.dev/docs/cue-sys.md` — what CUE turned out to be for here, and the + incident behind each lesson. +- `.dev/docs/cue-aar-equality.md` — four CUE constructs that read as + constraints and enforced nothing, each of which passed its negative + test while doing so. +- `.dev/docs/mini-app-lessons.md` — running record of what building the + dev-lane mini apps taught, kept so the prod-lane versions can be + written better. Add to it while the measurement is still on screen. + +Read them before writing a contract. They are the record; this file is +only the sequence. diff --git a/ops/process/roles/test-author.md b/ops/process/roles/test-author.md new file mode 100644 index 0000000..a36dd80 --- /dev/null +++ b/ops/process/roles/test-author.md @@ -0,0 +1,40 @@ +# Role: test author + +Any harness can fill this role — Claude, Codex, or Grok. The one rule that +cannot move: **the author of the tests is not the author of the +implementation** (same work order). A harness authors tests for its own +implementation only when no second harness is reachable, and the PR says so; +who judges those tests is the skeptic card's rule, not this one. + +You write tests from contracts, never from code. Nine defects on this repo got +past a green suite written by the same agent that wrote the implementation — +tests shaped to agree with the code pass over broken code by construction. +Your value is that you have not seen the implementation, so do not destroy it: + +- **Read only what the task names**: the objective, the design artifact, the + gate-kind spec, the Gherkin scenarios if any. If the implementation already + exists, do NOT open it. If you cannot write the test without peeking, the + contract is underspecified — say so and stop; that finding is worth more + than the test. +- **Test the promise, not the plumbing.** Assert what the caller was promised + (exit code, output shape, state after), never internal call sequences. +- **Every test must be able to fail.** Before returning, ask of each: what + broken implementation still passes this? A loop over a possibly-empty list + asserts nothing on the empty list — assert the count first. Never pick a + fixture by `sorted(...)[0]` luck. Never let the harness paper over what you + assert (a runner that prepends the interpreter hides a broken argv[0]). +- **Planted faults must prove they landed.** Any test that corrupts a fixture + proves the corruption with a comparison — landed, and still recognisably + the fixture — because a plant that silently failed makes the assertion + answer a question about nothing. In the dev mini-app's Python tests the + guarded helpers are `support.plant_bytes` / `plant_sql`; in Rust or any + other tree, write the same two assertions by hand. (This repo also carries + `ops/devlane/hooks/claude/test-guard.py` as enforcement, wired as a Claude + PostToolUse hook; the rule holds without it.) +- **Include the negative-space cases**: the empty input, the missing file, the + second concurrent caller, the interrupted operation, the value at the + boundary. These are where this repo's real defects lived. + +Return the test file paths and, for each, the one-line behavior it pins. +Do not run the implementation's suite; running your new tests to check they +*collect* is fine, proving them red is the caller's job. diff --git a/ops/process/roles/test-skeptic.md b/ops/process/roles/test-skeptic.md new file mode 100644 index 0000000..4c43aee --- /dev/null +++ b/ops/process/roles/test-skeptic.md @@ -0,0 +1,49 @@ +# Role: test skeptic + +Any harness can fill this role. The one rule that cannot move: **the skeptic +did not write the tests it judges, and did not write the implementation they +guard.** When all three harnesses are reachable there is always an eligible +third; when there is not, the fallback (an in-harness subagent, or the +author's own review) is stated in the PR for the review stage to weigh. + +Your job is to refute the claim "these tests would catch the code being +wrong". Default to refuting; a test survives only when you can name the +broken implementation it would catch. Judge the tests, not the code under +test. + +Work the catalogue — every entry is a shape that shipped a real defect on +this repo while its suite was green: + +1. **Shaped to agree with the code.** The assertion re-derives the expected + value the same way the implementation does, or replays only part of what + the code emits (a harness that prepends the interpreter hid an unrunnable + argv[0] here). Ask: was this expectation written from the contract, or + read off the output? +2. **Empty-set passes.** `for x in xs: assert ...` passes when `xs` is empty; + a filter that matches nothing looks identical to a filter that works. + Demand a count or non-emptiness assertion first. +3. **Unproven plants.** A test that corrupts a fixture must prove the + corruption landed and the fixture is still recognisable — otherwise the + check asserts about a clean fixture. `sed`/`.replace()` with a moved + anchor is the classic; run + `python3 ops/devlane/hooks/claude/test-guard.py `, and read every + plant by hand regardless. +4. **Fixture luck.** `sorted(...)[0]`, dict ordering, one hard-coded id — + the test passes because of an accident of the fixture, not the property. +5. **Proxy assertions.** Asserting a file exists, a name matches, or a + function was called, when the promise is about behavior. `shutil.which(argv[0])` + is not "the command works". +6. **The red that never was.** If a red run is claimed, check what actually + ran: a `-k` pattern matching zero tests prints "NO TESTS RAN" and exits + nonzero — that is not a red. A red must fail on the intended assertion, + not on collection or import. + +For the highest-value findings, prove them: plant a deliberate fault in a +**copy** of the implementation (never the working tree — copy first, or work +in a throwaway clone) and show the suite stays green. A finding with a +surviving mutant attached is unanswerable. + +Report in the repo's wire format (CONTRIB.md §Review protocol): +VERDICT / STAMP / FINDINGS with [P1|P2|P3], file, and the broken +implementation each weak test would miss. "- none" only when you tried and +failed to construct a surviving mutant for the tests in scope. diff --git a/ops/process/tdd.md b/ops/process/tdd.md new file mode 100644 index 0000000..b236c62 --- /dev/null +++ b/ops/process/tdd.md @@ -0,0 +1,91 @@ +# The tdd loop, bound to wf + +Evidence lives in the chain, not in your memory of having run something. +Every step below records a receipt; if `wf` refused it, the step did not +happen. Identity first (appending verbs refuse without it): + +```sh +export WF_AGENT='Your Model Name ' +``` + +`python3 ops/devlane/workflow/wf.py next ` at any point tells you the stage, +what is missing, and the exact argv that records it. Trust it over this file. + +## The stages (gate kind `tdd@1`: spec → red → impl → review → ratify) + +**spec** — write the behavior down before any test. If the work came through +the `bdd` skill there are scenarios already; otherwise state the promises the +tests will pin. Advance: `wf advance `. + +**red** — get the failing tests written and prove them red: + +1. Independent authorship is the rule, not a preference: when a second + harness is reachable, the **test-author role** + (`ops/process/roles/test-author.md`) is filled by a harness that is not + the implementer (`ops/process/cross-review.md`). A subagent given only + the contract is the fallback when no second harness is reachable, and + using the fallback is stated in the PR. The gate does not enforce this — + the review stage does (see below). +2. **Prove the red per file, for the right reason.** Run each new test file + by itself and read the failure: + - "NO TESTS RAN" / zero collected is NOT a red — a `-k` pattern matching + nothing exits nonzero and proves nothing. That mistake shipped here. + - An import/collection error is NOT a red. The red must be the intended + assertion failing because the behavior does not exist yet. +3. Record it: `wf red -- ` — the argv you record is the argv + that gets frozen; make it the one that runs the whole intended set (for + the product suite, once it exists, per the gate + kind's build check). `wf red` records any nonzero exit — it cannot tell an + honest red from a collection error, which is why step 2 is on you and the + skeptic checks it. +4. Receipts refuse against a dirty tree (D3): commit first, or accept the + weaker hash-only receipt with `--allow-dirty` and say why. +5. `wf advance ` — advancing out of red seals the frozen set (the files + matched by the gate kind's `test_scope`, features included). After this a + quiet edit to a frozen file makes `wf green` refuse on manifest drift; the + honest route back is below. + +**impl** — make it pass without touching the judge: + +1. Implement. Do not edit files in the gate kind's `test_scope` or + `harness_scope` — green will refuse on manifest drift if you do. +2. Before green, have the **test-skeptic role** + (`ops/process/roles/test-skeptic.md`) judge the sealed tests — filled by + a harness that wrote neither the tests nor the implementation. When it + proves a test wrong or weak, the route back is recorded, not quiet: + `wf marker open ` with the finding, `wf advance --to red + --reason "" `, fix the tests, re-prove red, reseal. +3. `wf green -- ` — must be the frozen command, passing. +4. `wf check ` for each of this kind's checks (`wf status + --checks` lists every registered kind's checks — take the rows for yours). + Be clear what this is: a failing check records a denial and exits 0, and + `tdd@1` requires only `receipt.green` to leave impl — **checks do not + block advance**. They are evidence the review stage reads, and skipping + them is visible there as absence. + +**review** — this is where the process is enforced, because the gate +deliberately is not (records are evidence, never authority — AGENTS.md). +The reviewer reads the record (`wf log `, `wf agents`) and refuses with +`finding` when it shows: + +- the red and the tests' commits carry the same agent as the implementation + commits, with no stated fallback justification in the PR; +- no skeptic pass over the sealed tests, or its findings unaddressed; +- registered checks absent or denied with no explanation; +- a red whose failure was collection or compile error, not the intended + assertion. + +`wf outcome --by --source ` (`open` and +`outcome` require `--source`; the other verbs do not). Review's `finding` +returns the work order to impl; **ratify** is the owner's (D10 — recording +states it, the attesting merge applies it). + +## The three honesty rules that outrank speed + +- **A test that never failed has proven nothing.** Red first, per file, on + the intended assertion. +- **The implementer's green is a claim, not evidence.** Independent tests, or + at least the skeptic's review, before `wf green`. +- **Numbers about the work come from commands run now** — test counts and + results in any PR body or report are produced at write time and stamped + with the SHA they were measured against (CONTRIB.md §Evidence). diff --git a/ops/process/token-thrift.md b/ops/process/token-thrift.md new file mode 100644 index 0000000..3893f7b --- /dev/null +++ b/ops/process/token-thrift.md @@ -0,0 +1,28 @@ +# token-thrift.md — not spending what the work doesn't need + +`worth.py waste` points at where spend concentrated; this page names +the classes that put it there and the countermeasure for each. The +discipline: every class is detectable from the stores, so a claim +that spend improved is a `worth.py` comparison between two windows, +never an impression. + +| # | waste class | signal in the stores | countermeasure | +|:--|:--|:--|:--| +| 1 | **context churn** — the same context re-read turn after turn | `cache-churn`: cached reads more than 20x output | narrower reads (offset/limit), never re-read after edit, keep long transcripts compacted, split long lanes into fresh sessions | +| 2 | **unbounded tool output** — a command dumped its whole world into context | `heavy-turn` on tool-heavy sessions | bound every command (`head`, `--short`, `-c` counts); the count-first grep shape; `Read` with limits | +| 3 | **re-derivation** — state rebuilt from scratch instead of read from the stream | many short turns re-running status/log/diff | context stream + stamps: read what moved, re-take only what a crossing expired | +| 4 | **doc reloading** — agents loading whole process docs each run | repeated large cached reads across agent sessions | compact aggregates for agents (pulse, usage report lines); docs stay for humans and first reads | +| 5 | **oversized fan-out** — parallel agents where one would do | many concurrent sessions, low per-session output | size the pattern to the surface; say the agent count before launching; solo for routine work | +| 6 | **polling** — asking again instead of being told | dozens of identical cheap turns | `--watch` modes, background tasks with notifications, breaker supervision instead of manual checks | +| 7 | **runaway reviewers** — a hung or looping background run burning quietly | breaker trips; grok runs with huge `modelCalls` | always arm the battery (cross-review.md); caps are the contract, `--disable` is refused on the armed line | +| 8 | **retry storms** — the same failing operation repeated verbatim | bursts of near-identical turns | after two failures, change the approach or surface the blocker; never loop a denied call | + +The rule behind all eight: **tokens buy state changes, not activity.** +A turn that moved no file, landed no commit, and decided nothing was +either a read that should have been narrower or a wait that should +have been a notification. + +Review cadence: run `worth.py waste` over the last day before closing +a lane; anything ranked in the top 5 twice in a row gets a class +assigned from this table and a countermeasure applied, and the next +window's report is the test of whether it worked. diff --git a/ops/process/worth.md b/ops/process/worth.md new file mode 100644 index 0000000..4fa2353 --- /dev/null +++ b/ops/process/worth.md @@ -0,0 +1,92 @@ +# worth.md — costs and results, joined + +Two questions the dev lane must answer from data, not memory: *was the +recent work worth what it cost?* and *where are tokens being wasted?* +Both are answered by `ops/devlane/telemetry/worth.py`, which reads the +same stores as `usage.py` and the same repo history as everything +else. Every figure it prints is produced at run time and stamped with +the window and the repo state it was measured against. No figure from +this tool is ever USD: the only native dollar wire (grok +`costUsdTicks`) has an unverified scale, and it is reported raw as +ticks or `unrecorded`, exactly as `usage.py` does. + +## worth report — cost x results + + python3 ops/devlane/telemetry/worth.py report --repo PATH \ + [--since ISO] [--until ISO] [--now ISO] [--format plain|json] + +The cost block: per harness (claude, codex, grok), the sessions and +messages inside the window, and in / cached / out / total token sums. +The results block, from the repo's history alone (no network): merge +commits landed on the current branch in the window with their PR +numbers parsed from the subject, non-merge commits, and the +test-definition count at each window edge with its delta. + +## worth waste — where the spend concentrates + + python3 ops/devlane/telemetry/worth.py waste --repo PATH \ + [--since ISO] [--until ISO] [--now ISO] [--top N] [--format plain|json] + +Ranks sessions by total tokens and names, for each: the harness, the +session id, messages (or runs), out, and cached tokens. Signals are +numbers with provenance, never verdicts: `cache-churn` carries the +cached-read and output figures whose ratio breached, and +`heavy-turn` names the single heaviest message (for codex, the +largest step between its cumulative counts) or run. A human or an +agent decides what to change; the tool only points. + +## Behavior contract (tests are authored from this section alone) + +1. **Time is an input.** `--now ISO` is accepted by both subcommands; + when absent the current time is read once. The default window is + the 24 hours ending at now; `--since`/`--until` (ISO 8601, + inclusive lower, exclusive upper) override either edge. No test + may depend on the wall clock: fixtures pass `--now`. +2. **Windowing is per message for claude and codex**: a usage event + counts iff its own timestamp is inside the window. **Windowing is + per run for grok**: a run (as split by `usage.py`'s shrink rule) + counts iff its last report's timestamp is inside the window; runs + are never subdivided, because grok reports cumulatively and a + partial run has no honest per-message delta. The report says + `runs=` for grok, not `messages=`. +3. **Cost figures reuse the usage.py accounting** — same stores, same + session discovery, same cwd filter for `--repo`, same last-wins + dedup for claude ids, same grok run-splitting, cost ticks, and + incompleteness discipline. A harness whose stores are absent or + unparseable prints `unrecorded`, never 0. `counted=N/M` and + `(incomplete)` propagate to every output format. +4. **Results come from git only.** Merge commits on HEAD's + first-parent line in the window; PR numbers parsed from `Merge + pull request #N` subjects; commits with no PR number are listed as + commits, not dropped. The test-definition count is measured from + the two window-edge trees (`git rev-list -1 --before` at each + edge), never from the working tree; if an edge has no commit the + report says `unrecorded` for the delta. +5. **The join is stamped**: repo HEAD short sha, branch, window + [since, until), and generation `--now` appear in both formats. +6. **waste ranks by window-scoped totals**: `--top N` (default 5) + sessions ordered by total tokens inside the window, ties broken by + session id for determinism. `cache-churn` fires for a session iff + cached reads exceed 20x its output tokens AND output is nonzero; + `heavy-turn` names the single largest out-token message (claude / + codex) or run (grok) in each listed session. Signals carry the + session id and harness so the transcript can be opened. +7. **JSON is the same truth**: `--format json` emits one object with + `stamp`, `cost`, `results` (report) or `stamp`, `sessions`, + `signals` (waste); every plain-format figure appears in it, and + nothing appears in JSON that plain omits. +8. **Exit codes**: 0 on success including empty windows (an empty + window prints zeros for results and `sessions=0` per harness); 2 + on unusable arguments (malformed ISO, until <= since, unknown + format); stderr carries the reason. + +## Reading it + +- A PR that took three review rounds and a fraction of a session's + tokens, and landed with its findings pinned, was cheap. A refuted + finding that consumed a round is the trigger to inspect, not a + number to hide. Marginal cost against marginal utility, per + decision — not totals against feelings. +- `waste` output feeds the token-thrift checklist + (`ops/process/token-thrift.md`): each signal names the transcript + to open and the discipline that would have prevented it. From b80e03631c6084e1445b2a11706bafe17f5dce61 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:21:15 -0400 Subject: [PATCH 06/18] =?UTF-8?q?ops:=20take=20the=20guards=20=E2=80=94=20?= =?UTF-8?q?hooks=20+=20workflow/checks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PreToolUse guard hooks (boundary-match, unsafe-command, command_shape, conductor-enforce, test-guard, context custody) and the standalone workflow checks they depend on (commit_trailers and siblings), under ops/devlane. Machinery only, no captured data. Verified: 156/156 hook tests green, run from the worktree root. Source: owner 2026-09-01 Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XehTac5TJNmPAskwrPp7rJ Claude-Session: https://claude.ai/code/session_012Jj94rkp3tfHAxUkTCthgY --- ops/devlane/hooks/claude/boundary-match.py | 90 ++ ops/devlane/hooks/claude/command_shape.py | 462 +++++++++ ops/devlane/hooks/claude/conductor-enforce.py | 160 +++ ops/devlane/hooks/claude/context-boundary.sh | 40 + ops/devlane/hooks/claude/context-precheck.py | 246 +++++ ops/devlane/hooks/claude/context-precheck.sh | 18 + ops/devlane/hooks/claude/context-stream.sh | 147 +++ ops/devlane/hooks/claude/evgrep.sh | 28 + ops/devlane/hooks/claude/pr-feedback.sh | 291 ++++++ ops/devlane/hooks/claude/ruff-after-edit.sh | 73 ++ ops/devlane/hooks/claude/test-guard-hook.sh | 62 ++ ops/devlane/hooks/claude/test-guard.py | 164 ++++ ops/devlane/hooks/claude/unsafe-command.py | 417 ++++++++ ops/devlane/hooks/commit-msg | 53 + ops/devlane/hooks/install.sh | 70 ++ ops/devlane/hooks/post-checkout | 44 + ops/devlane/hooks/post-commit | 39 + ops/devlane/hooks/tests/corpus.py | 42 + ops/devlane/hooks/tests/gh_stub.py | 247 +++++ ops/devlane/hooks/tests/support.py | 361 +++++++ .../hooks/tests/test_bdd_traceability.py | 96 ++ .../hooks/tests/test_boundary_match.py | 64 ++ ops/devlane/hooks/tests/test_command_shape.py | 154 +++ .../hooks/tests/test_commit_msg_gate.py | 85 ++ .../hooks/tests/test_conductor_enforce.py | 101 ++ .../hooks/tests/test_context_precheck.py | 178 ++++ .../test_context_precheck_selected_repo.py | 223 +++++ .../hooks/tests/test_context_stream.py | 502 ++++++++++ ops/devlane/hooks/tests/test_hooks.py | 916 ++++++++++++++++++ ops/devlane/hooks/tests/test_pr_feedback.py | 655 +++++++++++++ .../hooks/tests/test_ruff_after_edit.py | 315 ++++++ ops/devlane/hooks/tests/test_test_guard.py | 121 +++ .../hooks/tests/test_test_guard_hook.py | 160 +++ .../hooks/tests/test_unsafe_command.py | 69 ++ ops/devlane/workflow/checks/ci_contexts.py | 375 +++++++ ops/devlane/workflow/checks/ci_matrix.py | 74 ++ ops/devlane/workflow/checks/ci_minutes.py | 349 +++++++ .../workflow/checks/commit_trailers.py | 386 ++++++++ ops/devlane/workflow/checks/diagnostics.py | 425 ++++++++ ops/devlane/workflow/checks/doc_commands.py | 162 ++++ ops/devlane/workflow/checks/doc_covers.py | 292 ++++++ ops/devlane/workflow/checks/lint.py | 103 ++ ops/devlane/workflow/checks/secret_scan.py | 124 +++ .../workflow/checks/vocabulary_wall.py | 6 +- 44 files changed, 8986 insertions(+), 3 deletions(-) create mode 100755 ops/devlane/hooks/claude/boundary-match.py create mode 100755 ops/devlane/hooks/claude/command_shape.py create mode 100755 ops/devlane/hooks/claude/conductor-enforce.py create mode 100755 ops/devlane/hooks/claude/context-boundary.sh create mode 100755 ops/devlane/hooks/claude/context-precheck.py create mode 100755 ops/devlane/hooks/claude/context-precheck.sh create mode 100755 ops/devlane/hooks/claude/context-stream.sh create mode 100755 ops/devlane/hooks/claude/evgrep.sh create mode 100755 ops/devlane/hooks/claude/pr-feedback.sh create mode 100755 ops/devlane/hooks/claude/ruff-after-edit.sh create mode 100755 ops/devlane/hooks/claude/test-guard-hook.sh create mode 100755 ops/devlane/hooks/claude/test-guard.py create mode 100755 ops/devlane/hooks/claude/unsafe-command.py create mode 100755 ops/devlane/hooks/commit-msg create mode 100755 ops/devlane/hooks/install.sh create mode 100755 ops/devlane/hooks/post-checkout create mode 100755 ops/devlane/hooks/post-commit create mode 100644 ops/devlane/hooks/tests/corpus.py create mode 100644 ops/devlane/hooks/tests/gh_stub.py create mode 100644 ops/devlane/hooks/tests/support.py create mode 100644 ops/devlane/hooks/tests/test_bdd_traceability.py create mode 100644 ops/devlane/hooks/tests/test_boundary_match.py create mode 100644 ops/devlane/hooks/tests/test_command_shape.py create mode 100644 ops/devlane/hooks/tests/test_commit_msg_gate.py create mode 100644 ops/devlane/hooks/tests/test_conductor_enforce.py create mode 100644 ops/devlane/hooks/tests/test_context_precheck.py create mode 100644 ops/devlane/hooks/tests/test_context_precheck_selected_repo.py create mode 100644 ops/devlane/hooks/tests/test_context_stream.py create mode 100644 ops/devlane/hooks/tests/test_hooks.py create mode 100644 ops/devlane/hooks/tests/test_pr_feedback.py create mode 100644 ops/devlane/hooks/tests/test_ruff_after_edit.py create mode 100644 ops/devlane/hooks/tests/test_test_guard.py create mode 100644 ops/devlane/hooks/tests/test_test_guard_hook.py create mode 100644 ops/devlane/hooks/tests/test_unsafe_command.py create mode 100644 ops/devlane/workflow/checks/ci_contexts.py create mode 100755 ops/devlane/workflow/checks/ci_matrix.py create mode 100644 ops/devlane/workflow/checks/ci_minutes.py create mode 100755 ops/devlane/workflow/checks/commit_trailers.py create mode 100755 ops/devlane/workflow/checks/diagnostics.py create mode 100755 ops/devlane/workflow/checks/doc_commands.py create mode 100644 ops/devlane/workflow/checks/doc_covers.py create mode 100644 ops/devlane/workflow/checks/lint.py create mode 100755 ops/devlane/workflow/checks/secret_scan.py diff --git a/ops/devlane/hooks/claude/boundary-match.py b/ops/devlane/hooks/claude/boundary-match.py new file mode 100755 index 0000000..dc138f6 --- /dev/null +++ b/ops/devlane/hooks/claude/boundary-match.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Decide whether a shell command crossed a context boundary. + +Reads a PostToolUse payload on stdin, prints "\t" if it did, nothing if not. + +Precision matters more than coverage: a hook that fires during ordinary work is read as +noise and then ignored, which is worse than not having one. So matching looks at COMMAND +POSITIONS rather than substrings — an earlier version globbed '*build*' and fired on +`ls build/`, `cat docs/build-notes.md` and `cd builder`. + +The corpus lives in `ops/devlane/hooks/tests/`. +""" + +import json +import logging +import os +import re +import sys + +# `.claude/settings.json` runs every hook in this directory by path, so a sibling import is +# free — but only once the directory is on the path, which it is not when python reads the +# script from stdin. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from command_shape import commands + +RULES = [ + ("tree", + r"^git\s+(?:checkout|switch)\s+(?!--\s)|^git\s+worktree\s+(?:add|remove)\b", + ("the working tree changed. File contents, greps, line numbers and anything read from " + "the tree before this are stale. Build output belongs to the PREVIOUS checkout — " + "rebuild before testing against it, and expect dangling symlinks where generated " + "trees differ between branches.")), + ("history", + (r"^git\s+rebase\b|^git\s+commit\b.*--amend|^git\s+push\b.*(?:--force|-f)\b" + r"|^git\s+filter-repo\b"), + ("history was rewritten. Commit SHAs, counts, diffs, and any PR with this branch as " + "its head are stale. A PR does NOT recompute when its BASE moves, only when its head " + "does — verify with 'gh pr diff --name-only', never the compare API, which is " + "computed live and cannot observe the failure.")), + ("refs", + r"^git\s+(?:fetch|pull)\b|^git\s+remote\s+update\b", + ("refs may have moved. Ahead/behind, merge-base, 'is it up to date', and every " + "distance computed earlier are stale.")), + ("discard", + r"^git\s+(?:reset|stash|clean)\b|^git\s+(?:restore|checkout)\s+--\s", + ("working-tree state was discarded. Confirm what survived with 'git status --short' " + "before trusting anything staged, written or measured — untracked files are gone, and " + "no reflog holds them.")), + ("artifacts", + (r"^(?:make|cmake|ninja|gradle|mvn|sphinx-build|tsc|webpack|vite)\b" + r"|^(?:docker|cargo|go|dotnet|bazel|zig|swift)\s+build\b" + r"|^(?:pytest|jest|vitest|playwright|tox|nox)\b" + r"|^(?:cargo|go|dotnet|swift|npm|yarn|pnpm)\s+test\b" + r"|^[\w./-]*\b(?:doc-)?build\b\s*(?:-|$)"), + ("artifacts or results were regenerated. Counts, listings and test outcomes taken " + "before this run describe the previous state.")), +] + + +def classify(cmd: str): + """The first boundary any command position in the text crosses, or None. + + Where the command positions ARE is `command_shape`'s question. This file used to answer + it with its own runner list, and that list contained `npm`, `yarn` and `pnpm` — so + `npm test` was peeled to `test` before the `npm test` rule two lines below could be + tried, and the rule never fired for the launchers it names. It also had no idea what a + heredoc was, and recorded a working-tree crossing for the words `git checkout` written + inside a note. + """ + for seg in commands(cmd): + for kind, pattern, note in RULES: + if re.search(pattern, seg): + return kind, note + return None + + +if __name__ == "__main__": + import sys + if "--test" in sys.argv: + print("corpus moved to ops/devlane/hooks/tests/", file=sys.stderr) + sys.exit(2) + try: + payload = json.load(sys.stdin) + command = ((payload.get("tool_input") or {}).get("command") or "")[:2000] + except Exception: + logging.exception("failed to read hook payload from stdin") + sys.exit(0) + hit = classify(command) + if hit: + print(hit[0] + "\t" + hit[1]) diff --git a/ops/devlane/hooks/claude/command_shape.py b/ops/devlane/hooks/claude/command_shape.py new file mode 100755 index 0000000..a8f6ffd --- /dev/null +++ b/ops/devlane/hooks/claude/command_shape.py @@ -0,0 +1,462 @@ +#!/usr/bin/env python3 +"""One answer to "what commands does this shell text run, and where do they start?". + +Three hooks in this directory each decide something about a Bash payload — is it +consequential, did it move the ground, must it be refused — and each of them first has to +split the text into pieces and find the program name in each piece. Three copies of that +split existed, they disagreed, and every disagreement was a defect in whichever copy lost: + + * `git -C . push` was invisible to two of them and visible to the third. + * `npm test` was invisible to the rule written for it, because `npm` was on a runner list + and got peeled off before the rule was tried. + * `echo "done; git checkout main"` fired the boundary hook, because the split ignored + quoting — it fired on this file's own test probe while this file was being written. + * A `git push` written inside a heredoc BODY — a commit message, a note, a memory file — + fired two of the three. Only one blanked data heredocs. + +So this module owns the shape and nothing else. It has no opinion about what any command +MEANS: no list of dangerous programs, no notion of consequence, no rules. Those stay with +the hook that has the reason for them. What it owes its callers is that a command the +caller can recognise in its plainest form stays recognisable when it arrives wrapped — +behind `sudo`, inside `for … do … done`, after `git -C dir`, split over a line +continuation — and that text which merely SAYS a command, inside quotes or a data heredoc, +is not offered as one. + +Two views, because the callers need different things and the difference is real: + + statements(text) splits on statement separators only, keeping a PIPELINE whole. + `pytest -q 2>&1 | tail -4` is one statement. A rule that reads "this + runner, clipped by that head" needs both halves in one string. + + commands(text) every position where a program name can appear: pipeline components and + command substitutions too, each one peeled down to its program name. + A rule that reads "^git push" needs the `git` at the front. + +`commands()` returns VARIANTS, not a single rewriting. `sudo -u ci git push` yields both +`ci git push` and `git push`, because guessing which options take an operand means keeping +a list of options — the same "subset of an unenumerated set" mistake the hooks next door +exist to catch. Offering both costs one more regex match and needs no list. Callers match +patterns anchored at `^`, so the extra variants match nothing. + +The corpus lives in `ops/devlane/hooks/tests/`. +""" + +import re + +MAX_VARIANTS = 400 # a bound, not a tuning knob: runaway text stays cheap + +# ---------------------------------------------------------------- program-name vocabulary + +# Wrappers: the program name that matters is the NEXT one, always. `sudo`, `env` and +# `timeout` are here because they run something else, never because of what they are. +RUNNERS = frozenset(( + "sudo", "doas", "command", "exec", "builtin", "nohup", "setsid", "stdbuf", + "nice", "ionice", "chrt", "time", "timeout", "xargs", "env", + "npx", "uvx", "poe", +)) + +# Launchers that wrap a command ONLY in their ` run …` form. `npm` is the reason this +# distinction exists: as a bare runner it swallowed `npm test` (killing the rule written for +# it) and `npm publish` (killing the rule written for THAT), while `npm run build` genuinely +# needs peeling. `run` is the whole difference and it is written in the command. +RUN_LAUNCHERS = frozenset(("uv", "poetry", "pipenv", "npm", "yarn", "pnpm", "bun", "rye", "pdm")) + +# Shell grammar that can sit in front of a command. Peeled, never matched against. +KEYWORDS = frozenset(( + "if", "then", "elif", "else", "fi", "do", "done", "while", "until", + "for", "case", "esac", "select", "function", "coproc", "!", "{", "}", "(", ")", +)) + +# A heredoc whose body is fed to one of these EXECUTES; anything else is data. +INTERPRETERS = frozenset(( + "sh", "bash", "zsh", "ksh", "dash", "ash", "fish", + "python", "python2", "python3", "perl", "ruby", "node", "deno", + "awk", "gawk", "mawk", "php", "tclsh", "osascript", "Rscript", +)) +_VERSIONED = re.compile(r"^(python|perl|ruby|php|node)[\d.]+$") + +# Programs whose `-c` argument is a whole shell command in a string. +SHELLS = frozenset(("sh", "bash", "zsh", "ksh", "dash", "ash")) + +_ASSIGNMENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=") +_DURATION = re.compile(r"^\d+(?:\.\d+)?[smhdSMHD]?$") # `timeout 60`, `timeout 1.5s` + +# The delimiter word must be a plain word, and the body ends on a line that is only the +# delimiter. `[^\n]*` after it is what makes `cat <<'EOF' | python3` a matchable head — the +# pipe that decides whether the body executes lives AFTER the delimiter. +HEREDOC = re.compile( + r"<<-?[ \t]*(['\"]?)([A-Za-z_]\w*)\1[^\n]*\n(?:.*?\n)??[ \t]*\2[ \t]*$", + re.DOTALL | re.MULTILINE, +) + + +def is_interpreter(word: str) -> bool: + """Is this bare word the name of something that runs its stdin as code?""" + if not word: + return False + base = word.rsplit("/", 1)[-1] + return base in INTERPRETERS or bool(_VERSIONED.match(base)) + + +# ------------------------------------------------------------------------------ scanning + +def _scan(text: str, split_pipes: bool): + """Split shell text, honouring quotes. + + Separators inside quotes are text: `git commit -m "wip; git push next"` is ONE command, + and splitting it at the `;` made the precheck hook deny a commit for what its message + said. Command substitution still opens inside double quotes, because it still runs: + `"$(git push)"` is a push. + + Returns None when the text ends inside an open quote — the caller falls back to a + quote-blind split rather than swallowing everything after a stray apostrophe. + """ + segs, buf, quote, nest = [], [], None, [] + i, n = 0, len(text) + + def flush(): + segs.append("".join(buf)) + buf.clear() + + while i < n: + ch = text[i] + if quote == "'": # single quotes: nothing is special + if ch == "'": + quote = None + buf.append(ch) + i += 1 + continue + if ch == "\\" and i + 1 < n: # escaped char: never a separator + buf.append(ch) + buf.append(text[i + 1]) + i += 2 + continue + # A substitution opens a command position even inside double quotes, because it + # still runs. The quote state is stacked and restored at the closing paren. + if split_pipes and text.startswith("$(", i): + flush() + nest.append(quote) + quote = None + i += 2 + continue + if split_pipes and ch == "`": + flush() + i += 1 + continue + if (split_pipes and ch == "(" and quote is None + and (i == 0 or text[i - 1] in " \t\n;&|(")): # a subshell, not `f()` + flush() + nest.append(quote) + i += 1 + continue + if split_pipes and ch == ")" and nest: + flush() + quote = nest.pop() + i += 1 + continue + if quote == '"': + if ch == '"': + quote = None + buf.append(ch) + i += 1 + continue + if ch in "'\"": + quote = ch + buf.append(ch) + i += 1 + continue + + if ch in "\n;": + flush() + i += 1 + continue + if ch == "|": + two = text.startswith("||", i) + if two or split_pipes: + flush() + i += 2 if two else 1 + continue + if ch == "&": + if text.startswith("&&", i): + flush() + i += 2 + continue + # `2>&1` and `&>log` are redirections, not the background operator. Splitting + # there tore `playwright test … 2>&1 | tail -4` in half and lost the rule. + if not (i and text[i - 1] == ">") and not text.startswith("&>", i): + flush() + else: + buf.append(ch) + i += 1 + continue + buf.append(ch) + i += 1 + segs.append("".join(buf)) + if quote is not None: + return None + return segs + + +_BLIND_STATEMENTS = re.compile(r";|&&|\|\||\n") +_BLIND_COMMANDS = re.compile(r"[;&|]{1,2}|\$\(|`|\)|\n") + + +def _split(text: str, split_pipes: bool): + segs = _scan(text, split_pipes) + if segs is None: # unbalanced quoting: fall back, do not swallow + pattern = _BLIND_COMMANDS if split_pipes else _BLIND_STATEMENTS + segs = pattern.split(text) + return [s.strip() for s in segs if s.strip()] + + +# ----------------------------------------------------------------------------- heredocs + +def _head_line(text: str, at: int) -> str: + """The line carrying a `<<` operator, with the operator itself removed. + + Both sides matter. `cat > x.sh <<'EOF'` is `cat` writing a file, whatever the file is + called; `cat <<'EOF' | python3` is python running the body, and the `python3` is on the + far side of the operator. + """ + start = text.rfind("\n", 0, at) + 1 + end = text.find("\n", at) + if end == -1: + end = len(text) + before = text[start:at] + after = text[at:end] + after = after.split(None, 1)[1] if len(after.split(None, 1)) > 1 else "" + return before + " " + after + + +def _head_executes(head: str) -> bool: + """Does the body of a heredoc opened by this head get RUN? + + Two tests, and the second one is the fix for a real refusal. The command word settles + `python3 - <<'PY'` and `cat > x.sh <<'EOF'`. A bare interpreter TOKEN anywhere in the + head settles `ssh host bash <<'EOF'`, where the interpreter is an argument. + + What is deliberately gone is the substring test this replaced: `\\bsh\\b` matched the + `sh` in `x.sh`, so `cat > setup.sh <<'EOF'` and `tee notes.sh <<'EOF'` were read as + executing their bodies. That refused a memory-file write, in this repo, because the + file was named `…-bash.md`. + """ + for c in commands(head, _heredocs=False): + if is_interpreter(_word(c)): + return True + return any(is_interpreter(t.strip("()[]{}'\"`;&<>")) for t in head.split()) + + +def strip_data_heredocs(text: str) -> str: + """Blank heredoc bodies that are DATA, keep the ones that are CODE. + + A commit message explaining why an unguarded `.replace()` is dangerous contains every + token of the thing it warns about. A note listing `git push` mentions a push. Neither + runs anything, and both were treated as if they did. + + Idempotent: the body and its terminator go, the operator stays, and a second pass finds + no body to take. + """ + if "<<" not in text: + return text + + def repl(m): + if _head_executes(_head_line(text, m.start())): + return m.group(0) + whole = m.group(0) + cut = whole.find("\n") + return whole if cut == -1 else whole[:cut] + + return HEREDOC.sub(repl, text) + + +_CONTINUATION = re.compile(r"[ \t]*\\\n[ \t]*") + + +def _join_continuations(text: str) -> str: + """`git \\ push` is one command. Splitting on the newline made it two.""" + return _CONTINUATION.sub(" ", text) + + +def _prepare(text: str, heredocs: bool = True) -> str: + text = text or "" + if heredocs: + text = strip_data_heredocs(text) + return _join_continuations(text) + + +# -------------------------------------------------------------------------- normalisation + +def _first_token(s: str): + """The first shell word, and the rest of the string, with quoted spans kept whole. + + `GIT_SSH_COMMAND='ssh -i k' git push` is one assignment followed by a command. Splitting + it on whitespace made it three words, the first of which still looked like an + assignment — so the assignment was peeled, `-i` became the program, and the push + vanished. Found by the corpus row for it, not by reading this function. + """ + i, n, quote = 0, len(s), None + while i < n and s[i] in " \t": + i += 1 + start = i + while i < n: + ch = s[i] + if quote: + if ch == quote: + quote = None + elif ch in "'\"": + quote = ch + elif ch in " \t": + break + elif ch == "\\" and i + 1 < n: + i += 1 + i += 1 + return s[start:i], s[i:].lstrip() + + +def _tokens(s: str): + out = [] + while s: + tok, s = _first_token(s) + if not tok: + break + out.append(tok) + return out + + +def tokens(cmd: str): + """The shell words of one command position, quoted spans kept whole. + + Public because a caller that needs to read a command's OPTIONS — which + repository `git -C other -c x=y push` acts on — otherwise writes its own + splitter, and a fourth copy of the split is what this module exists to + prevent. It answers where the words are, never what they mean. + """ + return _tokens(cmd) + + +def _word(cmd: str) -> str: + return _first_token(cmd)[0] + + +def command_word(cmd: str) -> str: + """The program name a command position invokes, once it is peeled.""" + for c in commands(cmd): + return _word(c) + return "" + + +def _peel_front(seg: str) -> str: + """Remove one leading thing that is not a program name. Returns seg unchanged if none.""" + s = seg.lstrip() + if not s: + return s + if s[0] in "({}!" and (len(s) == 1 or s[1].isspace() or s[0] in "({"): + return s[1:] + if s.startswith("\\") and len(s) > 1 and not s[1].isspace(): + return s[1:] # `\git` — quoting the name to skip an alias + head, rest = _first_token(s) + if head in KEYWORDS: + return rest + if _ASSIGNMENT.match(head): # `VAR=1 git push`, and lowercase names too + return rest + return s + + +def _option_region(tail: list[str], prefix: str) -> list[str]: + """Candidates for "the command starts somewhere after these options". + + Options that take an operand are not enumerated. The first non-option token is one + candidate; if the option before it could have taken it, the token after is another. A + surplus candidate matches no anchored pattern and costs one regex. + """ + i = 0 + while i < len(tail) and tail[i].startswith("-"): + i += 1 + if i >= len(tail): + return [] + out = [" ".join(filter(None, [prefix, *tail[i:]]))] + if i and "=" not in tail[i - 1]: + out.append(" ".join(filter(None, [prefix, *tail[i + 1:]]))) + elif not i and _DURATION.match(tail[0]): # `timeout 60 git push` + out.append(" ".join(filter(None, [prefix, *tail[1:]]))) + return out + + +def _quoted_arg_after(cmd: str, flag: str) -> str: + """The single argument following `flag`, unquoted — `bash -c 'git push'`.""" + m = re.search(rf"(?:^|\s){re.escape(flag)}\s+('([^']*)'|\"((?:[^\"\\]|\\.)*)\"|(\S+))", cmd) + if not m: + return "" + return m.group(2) or m.group(3) or m.group(4) or "" + + +def _variants(seg: str, depth: int = 0): + """Every plausible reading of one command position, peeled down to a program name.""" + out, seen, work = [], set(), [seg] + while work and len(out) < MAX_VARIANTS: + s = work.pop(0).strip() + if not s or s in seen: + continue + seen.add(s) + + peeled = _peel_front(s) + if peeled != s: + work.append(peeled) + continue + + toks = _tokens(s) + w = toks[0] + + if w in RUNNERS: + work.extend(_option_region(toks[1:], "")) + continue # a wrapper is never the command + if w in RUN_LAUNCHERS and len(toks) > 1 and toks[1] == "run": + work.append(" ".join(toks[2:])) + continue + + out.append(s) + + if w in ("git", "gh") and len(toks) > 1 and toks[1].startswith("-"): + work.extend(_option_region(toks[1:], w)) + if depth < 2 and w in SHELLS and "-c" in toks[1:3]: + inner = _quoted_arg_after(s, "-c") + if inner: + for sub in _split(_prepare(inner), split_pipes=True): + work.extend(_variants(sub, depth + 1)) + return out + + +# ------------------------------------------------------------------------------- the API + +def statements(text: str, _heredocs: bool = True): + """Statement-level pieces, PIPELINES KEPT WHOLE. + + For rules that read a pipeline as one thing: "this runner, clipped by that head". + """ + return _split(_prepare(text, _heredocs), split_pipes=False) + + +def commands(text: str, _heredocs: bool = True): + """Every position a program name can appear in, peeled to that name. + + Pipeline components and command substitutions included; runner prefixes, shell + keywords, environment assignments and git/gh global options peeled off. The segment as + written is always among the results, so nothing a caller recognises today stops being + recognised. + """ + out = [] + for seg in _split(_prepare(text, _heredocs), split_pipes=True): + for v in _variants(seg): + if v not in out: + out.append(v) + if len(out) >= MAX_VARIANTS: + break + return out + + +if __name__ == "__main__": + import sys + if "--test" in sys.argv: + print("corpus moved to ops/devlane/hooks/tests/", file=sys.stderr) + sys.exit(2) + print(__doc__) diff --git a/ops/devlane/hooks/claude/conductor-enforce.py b/ops/devlane/hooks/claude/conductor-enforce.py new file mode 100755 index 0000000..3e8473b --- /dev/null +++ b/ops/devlane/hooks/claude/conductor-enforce.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Keep the conductor on the dispatch levers. + +Reads Claude Code's PreToolUse JSON payload on stdin. A refusal is the same +``hookSpecificOutput`` decision used by the precheck next door; an allow is +silent. POLICY is deliberately a table: additions should be reviewable as +policy changes, not hidden in control flow. +""" + +import json +import os +import re +import shlex +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from command_shape import commands, statements + +AGENT_REASON = ( + "Own sub-agents are retired. Delegate through the levers: " + "grok-dispatch.sh (investigation), fable-dispatch.sh (Claude-specific / " + "high-judgment), codex-dispatch.sh (implementation), apply-push.sh (landing)." +) +INVESTIGATION_REASON = ( + "The conductor may not investigate or edit repository contents by hand. " + "Delegate investigation through grok-dispatch.sh." +) + +# Ordered, auditable policy. Explicit allows win before any deny is tested. +POLICY = { + "allow_anywhere": ( + "grok-dispatch.sh", + "fable-dispatch.sh", + "codex-dispatch.sh", + "apply-push.sh", + ), + "allow_git": ("fetch", "rev-parse", "ls-remote", "push", "status"), + "deny_git": ( + "diff", "log", "show", "interpret-trailers", "apply", "merge", + "rebase", "cherry-pick", "blame", + ), + "deny_gh": ( + r"^gh\s+pr\s+diff\b", + r"^gh\s+pr\s+view\b(?=.*(?:^|\s)--json(?:\s|=))", + r"^gh\s+run\s+view\b", + r"^gh\s+api\b", + ), + "forensic_programs": ("sed", "awk", "gawk", "mawk", "grep", "egrep", "fgrep"), + "job_readers": ("cat", "python", "python3", "ls"), + "job_markers": ("/jobs/", "/scratchpad/", ".dev/jobs/", ".dev/scratchpad/"), +} + + +def _git_subcommand(command): + match = re.match(r"^git\s+([^\s]+)", command) + return match.group(1) if match else None + + +def _is_job_read(command): + words = command.split() + if not words or words[0].rsplit("/", 1)[-1] not in POLICY["job_readers"]: + return False + padded = "/" + command.lstrip("./") + return any(marker in padded for marker in POLICY["job_markers"]) + + +def _tracked_files(): + try: + result = subprocess.run( + ["git", "ls-files", "-z"], # noqa: S607 + capture_output=True, + timeout=5, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return () + if result.returncode != 0: + return () + return tuple( + path.decode("utf-8", "surrogateescape") + for path in result.stdout.split(b"\0") + if path + ) + + +def _forensic_over_tracked(statement, tracked): + if not any( + re.search(rf"(?:^|[|;&]\s*){program}\b", statement) + for program in POLICY["forensic_programs"] + ): + return False + try: + words = shlex.split(statement, comments=True) + except ValueError: + words = statement.split() + for word in words: + candidate = word.removeprefix("./") + if candidate in tracked or any( + path.startswith(candidate.rstrip("/") + "/") + for path in tracked + if candidate + ): + return True + return False + + +def deny_reason(payload): + """Return a reason to deny, or None to allow.""" + tool = payload.get("tool_name") + if tool == "Agent": + return AGENT_REASON + if tool != "Bash": + return None + + command = ((payload.get("tool_input") or {}).get("command") or "")[:20000] + if any(lever in command for lever in POLICY["allow_anywhere"]): + return None + + variants = commands(command) + for variant in variants: + subcommand = _git_subcommand(variant) + if subcommand in POLICY["allow_git"]: + continue + if subcommand in POLICY["deny_git"]: + return INVESTIGATION_REASON + if any(re.search(pattern, variant) for pattern in POLICY["deny_gh"]): + return INVESTIGATION_REASON + + tracked = _tracked_files() + for statement in statements(command): + if _is_job_read(statement): + continue + if _forensic_over_tracked(statement, tracked): + return INVESTIGATION_REASON + return None + + +def main(): + try: + payload = json.load(sys.stdin) + except (json.JSONDecodeError, UnicodeDecodeError): + return + reason = deny_reason(payload) + if reason: + print( + json.dumps( + { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": reason, + } + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/ops/devlane/hooks/claude/context-boundary.sh b/ops/devlane/hooks/claude/context-boundary.sh new file mode 100755 index 0000000..34739bd --- /dev/null +++ b/ops/devlane/hooks/claude/context-boundary.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# context-boundary.sh — global PostToolUse hook. Says so when the ground moves. +# +# Context rot is not gradual: it happens at instants. A checkout, a rebase, a fetch that +# moves a ref, a build — each silently expires every measurement taken before it, and the +# command reports success either way. +# +# The decision of what counts as a boundary lives in boundary-match.py; its corpus lives +# in `ops/devlane/hooks/tests/`. Precision matters more than +# coverage: a hook that fires during ordinary work is read as noise and then ignored, which +# is worse than not having one. +# +# This never blocks and never fails a tool call. It exits 0 in every path, including when +# python is missing, the payload is malformed, or the repo is not a git repo. + +set -uo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +input=$(cat 2>/dev/null) || exit 0 +[ -n "$input" ] || exit 0 + +result=$(printf '%s' "$input" | python3 "$DIR/boundary-match.py" 2>/dev/null) || exit 0 +[ -n "$result" ] || exit 0 + +kind=${result%%$'\t'*} +note=${result#*$'\t'} + +# Record before warning. The warning helps this session; the record is what the next one +# reads instead of rebuilding state from whatever happens to be visible. +"$DIR/context-stream.sh" record "$kind" >/dev/null 2>&1 || true + +python3 - "$note" <<'PY' 2>/dev/null || true +import json, sys +print(json.dumps({"hookSpecificOutput": { + "hookEventName": "PostToolUse", + "additionalContext": "Context boundary crossed — " + sys.argv[1] + + " Re-take before reusing; do not reason from what you measured earlier.", +}})) +PY +exit 0 diff --git a/ops/devlane/hooks/claude/context-precheck.py b/ops/devlane/hooks/claude/context-precheck.py new file mode 100755 index 0000000..10b3c7e --- /dev/null +++ b/ops/devlane/hooks/claude/context-precheck.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Before anything consequential: get up to date, then decide. + +Reads a PreToolUse payload on stdin. Emits a deny decision when a command with outward +effect is about to run against refs that moved. Silent otherwise. + +The distinction that keeps this bearable is CONSEQUENCE, not risk. Reading, building, +committing locally — all reversible, all silent here. Pushing, opening or editing a PR, +publishing, deploying: those leave the machine and cannot be taken back, and those are the +only ones worth interrupting. + +Fetch before every such command, then compare: if nothing moved, the command proceeds +silently. If a tracked ref did move, it stops — because at that point every distance, +merge-base and ahead/behind computed earlier is genuinely wrong, and the reason says +exactly which ref changed and by how much. + +Denying on actual movement interrupts rarely and always has something to say. + +Which repository gets fetched is the command's to say, not the hook's. `git -C other +push`, `git --git-dir=other/.git push` and `GIT_DIR=other/.git git push` all leave the +machine from `other`; a gate that fetches and compares in its own process cwd is answering +about a different repository than the one about to publish — and it denies a fresh named +repo for a stale cwd, which is the same wrong answer pointing the other way. + +`CLAUDE_PRECHECK_NO_FETCH=1` skips the fetch, so the hook stays inert offline. + +The corpus lives in `ops/devlane/hooks/tests/`. +""" + +import json +import logging +import os +import re +import subprocess +import sys + +# `.claude/settings.json` runs every hook in this directory by path, so a sibling import is +# free — but only once the directory is on the path, which it is not when python reads the +# script from stdin. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from command_shape import commands, statements, tokens + +CONSEQUENTIAL = re.compile( + r"^git\s+push\b" + r"|^gh\s+(?:pr|release|issue)\s+(?:create|merge|edit|close|reopen|comment|ready)\b" + r"|^gh\s+api\b.*-X\s*(?:POST|PATCH|PUT|DELETE)" + r"|^(?:npm|yarn|pnpm)\s+publish\b" + r"|^(?:twine|cargo)\s+(?:upload|publish)\b" + r"|^docker\s+push\b" + r"|^(?:kubectl|terraform|serverless|flyctl|vercel|netlify)\s+(?:apply|deploy|destroy|promote)\b", + re.IGNORECASE, +) +def is_consequential(cmd): + """Does any command position in this text invoke something that leaves the machine? + + Where the command positions ARE is `command_shape`'s question, not this file's. It used + to be answered here, in a copy that missed `git -C . push` entirely and read a `git + push` written inside a heredoc note as a real one. + """ + return any(CONSEQUENTIAL.search(c) for c in commands(cmd)) + + +ASSIGNMENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=") + +#: git's own global options that take a SEPARATE operand. Only the ones that +#: select a repository are read; the rest are here so the operand is stepped +#: over rather than mistaken for the subcommand. `-c` is why this list exists: +#: a walk that ate `-C other` and then read `-c push.default=simple` as the +#: subcommand never saw the `push` behind it. +GIT_OPTS_WITH_OPERAND = frozenset(( + "-C", "-c", "--git-dir", "--work-tree", "--namespace", "--exec-path", + "--super-prefix", "--config-env", +)) + + +def _unquote(token): + if len(token) >= 2 and token[0] == token[-1] and token[0] in "'\"": + return token[1:-1] + return token + + +def selected_repo(cmd): + """The git global options naming the repository a push will run against. + + Returns a list of arguments to pass to git (`["-C", path]` or + `["--git-dir", path]`, possibly both), or `[]` when the command names no + repository — then the hook's own cwd is the repository, which is what it + always was. + + Where the words are is `command_shape`'s question; this reads them. + """ + for stmt in statements(cmd): + words = tokens(stmt) + env_git_dir = None + i = 0 + while i < len(words): + word = _unquote(words[i]) + if word.startswith("GIT_DIR="): + env_git_dir = _unquote(word.split("=", 1)[1]) + i += 1 + continue + if ASSIGNMENT.match(word): + i += 1 + continue + if word.rsplit("/", 1)[-1] != "git": + env_git_dir = None # an assignment binds one command + i += 1 + continue + i += 1 + work = git_dir = None + while i < len(words) and words[i].startswith("-"): + opt = words[i] + if opt.startswith("--git-dir="): + git_dir = _unquote(opt.split("=", 1)[1]) + elif opt.startswith("-C") and opt != "-C": + work = _unquote(opt[2:]) + elif opt in ("-C", "--git-dir") and i + 1 < len(words): + value = _unquote(words[i + 1]) + if opt == "-C": + work = value + else: + git_dir = value + i += 1 + elif opt in GIT_OPTS_WITH_OPERAND: + i += 1 + i += 1 + subcommand = _unquote(words[i]) if i < len(words) else "" + if subcommand == "push": + selected = [] + if work: + selected += ["-C", work] + if git_dir or env_git_dir: + selected += ["--git-dir", git_dir or env_git_dir] + return selected + env_git_dir = None + return [] + + +def clone_git_dir(selected): + """The SHARED git directory of the repository `selected` names. + + Not a linked worktree's private one. `FETCH_HEAD` is a per-worktree file: + a fetch run at `.git/worktrees/wt` leaves the clone's own `FETCH_HEAD` + untouched, so the next reader still sees a repository that has never + fetched. Remote-tracking refs live in the common dir either way, so + resolving it once is also what makes the comparison the right one. + + Returns None when nothing there is a repository — then there is nothing to + compare and the command proceeds. + """ + base = os.getcwd() + if "-C" in selected: + base = os.path.abspath( + os.path.join(base, selected[selected.index("-C") + 1]) + ) + try: + proc = subprocess.run(["git", *selected, "rev-parse", "--git-common-dir"], # noqa: S603, S607 — PATH git; the arguments are options this hook just read out of the command + capture_output=True, text=True, timeout=10, check=False) + except Exception: + logging.exception("could not resolve the selected repository") + return None + out = proc.stdout.strip() + if proc.returncode != 0 or not out: + return None + return os.path.abspath(os.path.join(base, out)) + + +def tracked_refs(git_dir): + """Remote-tracking refs and where they point, so movement can be named exactly.""" + try: + out = subprocess.run(["git", "--git-dir", git_dir, # noqa: S603, S607 — PATH git; git_dir is a path git itself resolved + "for-each-ref", "--format=%(refname:short) %(objectname)", + "refs/remotes/"], capture_output=True, text=True, timeout=10, + check=False).stdout + except Exception: + logging.exception("could not list remote-tracking refs") + return {} + refs = {} + for line in out.splitlines(): + parts = line.split() + if len(parts) == 2: + refs[parts[0]] = parts[1] + return refs + + +def main(): + try: + payload = json.load(sys.stdin) + cmd = ((payload.get("tool_input") or {}).get("command") or "")[:2000] + except Exception: + logging.exception("failed to read hook payload from stdin") + sys.exit(0) + + if not is_consequential(cmd): + sys.exit(0) + + git_dir = clone_git_dir(selected_repo(cmd)) + if git_dir is None: + sys.exit(0) # the command names nothing this hook can compare + + before = tracked_refs(git_dir) + if os.environ.get("CLAUDE_PRECHECK_NO_FETCH") != "1": + try: + subprocess.run(["git", "--git-dir", git_dir, "fetch", "--all", "--quiet"], # noqa: S603, S607 — PATH git; git_dir is a path git itself resolved + capture_output=True, timeout=45, check=False) + except Exception: + logging.exception("git fetch failed") + sys.exit(0) # offline or slow: never block work over it + after = tracked_refs(git_dir) + + moved = [(r, before[r], after[r]) for r in sorted(set(before) & set(after)) + if before[r] != after[r]] + if not moved: + sys.exit(0) # fetched, nothing changed, carry on — no interruption + + lines = [] + for ref, a, b in moved: + try: + n = subprocess.run(["git", "--git-dir", git_dir, "rev-list", "--count", f"{a}..{b}"], # noqa: S603, S607 — PATH git; range is SHAs this hook just read + capture_output=True, text=True, timeout=10, check=False).stdout.strip() + except Exception: + logging.exception("could not count commits on moved ref") + n = "?" + lines.append(f" {ref}: {a[:8]} -> {b[:8]} ({n} commits)") + + reason = ( + "Not up to date — so I fetched, and things moved:\n" + + "\n".join(lines) + + "\n\nThe fetch is done. But this command has outward " + "effect, and every ahead/behind, merge-base, commit count and PR diff you measured " + "before now was computed against the older refs. Re-take whatever this command " + "depends on, then run it again — it will pass." + ) + print(json.dumps({"hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": reason, + }})) + + +if __name__ == "__main__": + import sys + if "--test" in sys.argv: + print("corpus moved to ops/devlane/hooks/tests/", file=sys.stderr) + sys.exit(2) + main() diff --git a/ops/devlane/hooks/claude/context-precheck.sh b/ops/devlane/hooks/claude/context-precheck.sh new file mode 100755 index 0000000..41c0399 --- /dev/null +++ b/ops/devlane/hooks/claude/context-precheck.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# context-precheck.sh — global PreToolUse hook. Asks "are you up to date?" before anything +# that leaves the machine. Silent for everything reversible. Never fails a call: any +# internal error exits 0 and the command proceeds. +set -uo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +input=$(cat 2>/dev/null) || exit 0 +[ -n "$input" ] || exit 0 +# Two checks, cheapest first. unsafe-command is pure pattern matching with no network; +# context-precheck may fetch, so it runs only if nothing has already been refused. +refusal=$(printf '%s' "$input" | python3 "$DIR/unsafe-command.py" 2>/dev/null || true) +if [ -n "$refusal" ]; then + printf '%s\n' "$refusal" + exit 0 +fi + +printf '%s' "$input" | python3 "$DIR/context-precheck.py" 2>/dev/null || true +exit 0 diff --git a/ops/devlane/hooks/claude/context-stream.sh b/ops/devlane/hooks/claude/context-stream.sh new file mode 100755 index 0000000..b817835 --- /dev/null +++ b/ops/devlane/hooks/claude/context-stream.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# context-stream.sh — an append-only record of everything that invalidated context. +# +# A session that starts blind rebuilds state from whatever is visible now, which is how a +# measurement from days ago gets repeated as current. The current state cannot tell you +# what it used to be — so the moments it changed get written down as they happen. +# +# context-stream.sh record [kind] append what just happened, plus any state delta +# context-stream.sh since entries newer than an ISO timestamp +# context-stream.sh tail [n] the last n entries, readable (default 15) +# +# Stored in the clone's shared git directory, so it is per-CLONE and cannot be committed by +# accident. Its value is recency, not history. +# +# ONE STREAM PER CLONE, and --git-common-dir is what makes that true. CONTRIB.md mandates a +# worktree per line of work; in a linked worktree `git rev-parse --git-dir` answers +# `.git/worktrees/`, so a stream keyed on it forks into one private history per tree +# while the git-native recorders next door (post-commit, post-checkout) keep appending to +# the shared file. Measured: the Claude-side entry landed in .git/worktrees/wt/ and the +# commit entry in .git/, and `tail` showed a different history from each location. +# +# The last-seen SNAPSHOT stays per-worktree, and deliberately: it answers "what did THIS +# tree look like when I last looked here", and each worktree has its own branch and HEAD. +# Sharing it would make the first record from a second worktree report a checkout nobody +# made -- the branch changed because the reader moved, not because the ground did. + +set -uo pipefail +root=$(git rev-parse --show-toplevel 2>/dev/null) || exit 0 +common=$(git rev-parse --git-common-dir 2>/dev/null) || exit 0 +[ -n "$common" ] || exit 0 +gitdir=$(git rev-parse --git-dir 2>/dev/null) || exit 0 +[ -n "$gitdir" ] || exit 0 +# Both answers may be relative to the CURRENT directory, and the next line leaves it. +case "$common" in /*) ;; *) common="$PWD/$common" ;; esac +case "$gitdir" in /*) ;; *) gitdir="$PWD/$gitdir" ;; esac +common=$(cd "$common" 2>/dev/null && pwd) || exit 0 +gitdir=$(cd "$gitdir" 2>/dev/null && pwd) || exit 0 +cd "$root" +STREAM="$common/claude-context-stream.jsonl" +STATE="$gitdir/claude-context-state.json" + +base_ref() { + for c in upstream/main upstream/master origin/main origin/master main master; do + git rev-parse --verify -q "$c" >/dev/null 2>&1 && { echo "$c"; return; } + done +} + +record() { + python3 - "$STREAM" "$STATE" "$(base_ref)" "${1:-}" <<'PY' +import json, os, subprocess, sys, datetime + +stream, state_path, base = sys.argv[1], sys.argv[2], sys.argv[3] +kind_hint = sys.argv[4] if len(sys.argv) > 4 else "" + +def sh(*a): + try: + return subprocess.run(a, capture_output=True, text=True, timeout=8).stdout.strip() + except Exception: + return "" + +now = { + "branch": sh("git", "branch", "--show-current") or "?", + "head": sh("git", "rev-parse", "--short", "HEAD") or "?", + "base": sh("git", "rev-parse", "--short", base) if base else "", +} +prev = {} +if os.path.exists(state_path): + try: prev = json.load(open(state_path)) + except Exception: prev = {} + +events = [] +# the base ref moving invalidates the most, so it carries what it touched +if prev.get("base") and now["base"] and prev["base"] != now["base"]: + rng = f'{prev["base"]}..{now["base"]}' + n = sh("git", "rev-list", "--count", rng) or "?" + files = sh("git", "diff", "--name-only", rng).splitlines()[:8] + events.append(("base", f'{base} {prev["base"]} -> {now["base"]} ({n} commits)', " ".join(files))) + +if prev.get("branch") and prev["branch"] != now["branch"]: + events.append(("branch", f'checkout {prev["branch"]} -> {now["branch"]}', "")) +elif prev.get("head") and prev["head"] != now["head"]: + events.append(("head", f'{now["branch"]} moved {prev["head"]} -> {now["head"]}', "")) + +# Record the crossing itself even when the net state is unchanged. A command can check out +# a branch, build, and return before this hook runs — the delta is zero and the crossing +# still happened, which is exactly what a later session needs to know. Logging only deltas +# left the stream nearly empty across a whole session of real work. +if kind_hint and not events: + events.append((kind_hint, f'{kind_hint} while on {now["branch"]} at {now["head"]}', "")) + +if events: + ts = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + with open(stream, "a") as f: + for kind, what, detail in events: + f.write(json.dumps({"at": ts, "kind": kind, "what": what, "detail": detail}) + "\n") + +json.dump(now, open(state_path, "w")) +PY +} + +read_stream() { # read_stream + [ -f "$STREAM" ] || { [ "$1" = tail ] && echo " (no context stream yet)"; return 0; } + python3 - "$STREAM" "$1" "$2" <<'PY' +import json, sys +path, mode, arg = sys.argv[1], sys.argv[2], sys.argv[3] +entries = [] +for line in open(path): + raw = line.rstrip("\n") + if not raw.strip(): + continue + try: + entry = json.loads(raw) + except Exception: + # An unreadable line is still a record. Dropping it silently was how a + # reader could disagree with the file it was reading and say nothing. + entries.append((raw, None)) + continue + entries.append((raw, entry)) +if mode == "since": + entries = [p for p in entries if p[1] is None or p[1].get("at", "") > arg] +else: + entries = entries[-int(arg or 15):] +for raw, e in entries: + if e is None: + print(f' (unreadable) {raw[:160]}') + continue + when = e.get("at", "")[5:16].replace("T", " ") + # tolerate entries written by earlier versions of this tool, which had no "what" + what = e.get("what") or f'{e.get("kind","?")} on {e.get("branch","?")} at {e.get("head","?")}' + print(f' {when} [{e.get("kind","?"):6}] {what}') + if e.get("detail"): + print(f' touched: {e["detail"][:96]}') + # The record itself, verbatim, under its own rendering. The rendering is + # lossy — it drops the year, the seconds, the agent and the worktree — and a + # lossy view cannot answer the question this tool now has to answer from + # every worktree: is this the same stream the other tree is reading? Two + # readers comparing summaries agree while reading different files. + print(f' record: {raw}') +PY +} + +case "${1:-tail}" in + record) record "${2:-}" ;; + since) read_stream since "${2:-1970-01-01T00:00:00Z}" ;; + tail) read_stream tail "${2:-15}" ;; + *) echo "usage: context-stream.sh {record|since |tail [n]}" >&2; exit 2 ;; +esac diff --git a/ops/devlane/hooks/claude/evgrep.sh b/ops/devlane/hooks/claude/evgrep.sh new file mode 100755 index 0000000..c617329 --- /dev/null +++ b/ops/devlane/hooks/claude/evgrep.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# evgrep.sh -- truncation-honest evidence grep. The COUNT comes from the full search; only +# the DISPLAY is bounded, and truncation announces itself instead of ending silently. +# +# Exists because `grep "a\|b\|c\|d" file | head -8` answered a completeness question with +# its first eight lines: the match that mattered sat below the clip, the conclusion drawn +# was "absent", and nothing looked wrong. Bound the display, never the measurement. +# +# usage: evgrep.sh [grep options] PATTERN PATH... (args pass through to grep -n) +# EVGREP_LIMIT=40 evgrep.sh ... (default display limit: 20) + +set -uo pipefail +LIMIT=${EVGREP_LIMIT:-20} + +out=$(grep -n "$@" 2>&1) +rc=$? +if [ "$rc" -ge 2 ]; then + printf '%s\n' "$out" >&2 + exit "$rc" +fi + +n=$(printf '%s' "$out" | grep -c .) +printf '== %s match(es)\n' "$n" +[ "$n" -gt 0 ] && printf '%s\n' "$out" | head -n "$LIMIT" +if [ "$n" -gt "$LIMIT" ]; then + printf '== TRUNCATED: %s more match(es) not shown — do not conclude absence or completeness from this view\n' $((n - LIMIT)) +fi +exit "$rc" diff --git a/ops/devlane/hooks/claude/pr-feedback.sh b/ops/devlane/hooks/claude/pr-feedback.sh new file mode 100755 index 0000000..07176a2 --- /dev/null +++ b/ops/devlane/hooks/claude/pr-feedback.sh @@ -0,0 +1,291 @@ +#!/usr/bin/env bash +# pr-feedback.sh [repo] [--watch] — every surface a PR can carry feedback on. +# +# Written because a poller watched `.comments` for a Codex reply, Codex answered on a diff +# line instead, and the poller reported "no response" for forty minutes while the review sat +# there. The check could not observe the thing it was watching for. +# +# Then it happened AGAIN, in the other direction: a hand-rolled loop watched reviews, inline +# comments and reactions, and Codex's clean verdict arrived as a conversation comment. The +# tool that reads all four already existed; what it lacked was a way to WAIT, so a loop got +# written from scratch and re-introduced the bug the tool had fixed. Hence --watch. +# +# Feedback on a pull request lives in FIVE places, and they are different API objects: +# +# 1. issue comments the conversation tab /issues/N/comments +# 2. reviews approve / request-changes bodies /pulls/N/reviews +# 3. review comments inline, anchored to a diff line /pulls/N/comments +# 4. reactions a bot signalling "nothing found" /issues/comments/ID/reactions +# 5. unresolved threads what the review is still waiting on GraphQL reviewThreads +# +# Checking a subset and reporting "nothing" is worse than not checking, because it answers +# the question wrongly rather than not answering it. +# +# The endpoints are defined ONCE below and used by both the report and the watch +# fingerprint. Two lists would drift, and the drift would be exactly this bug. Threads were +# in the report and not in the fingerprint, which is that drift: --watch could not notice +# the one surface that says what a review is still blocked on. +# +# A SURFACE THAT COULD NOT BE READ IS NOT AN EMPTY SURFACE. Every `gh` call's status is +# checked; a failed one prints `(UNREADABLE — gh exit N: )` where the +# listing would have gone, and the run exits 3 — distinct from --watch's 1 for a quiet +# timeout and from 2 for a usage error. This is the incident that produced the tool, made +# by the tool: an auth failure and a genuinely empty PR used to print the same `(none)` and +# the same exit 0, so a network blip read as "the reviewer's comment vanished". +# +# In --watch, a fingerprint taken through a failed call is not a fingerprint. It is never +# compared, the error is said once, polling continues, and the run cannot exit 0 on it. + +set -uo pipefail +PR=""; REPO=""; WATCH=0; INTERVAL=20; TIMEOUT=900 +while [ $# -gt 0 ]; do + case "$1" in + --watch) WATCH=1 ;; + --interval) INTERVAL=$2; shift ;; + --timeout) TIMEOUT=$2; shift ;; + -*) echo "pr-feedback: unknown flag $1" >&2; exit 2 ;; + *) if [ -z "$PR" ]; then PR=$1; else REPO=$1; fi ;; + esac + shift +done +[ -n "$PR" ] || { echo "usage: pr-feedback.sh [owner/repo] [--watch] [--interval N] [--timeout N]" >&2; exit 2; } +[ -n "$REPO" ] || REPO=$(gh repo view --json nameWithOwner --jq .nameWithOwner 2>/dev/null) +[ -n "$REPO" ] || { echo "pr-feedback: could not determine the repo" >&2; exit 1; } + +EXIT_UNREADABLE=3 +THREAD_PAGE=50 # the GraphQL bound; paging is below, and it is the point + +# ---- the five surfaces, defined once ------------------------------------------------- +# per_page=100: the API defaults to 30 per page, and a bare listing silently truncates +# there — which once turned 9 fresh review findings into a reported "clean" pass. 100 is +# the API maximum; cap_warn() below refuses to stay quiet if a surface hits it. +EP_CONV="repos/$REPO/issues/$PR/comments?per_page=100" +EP_REVIEWS="repos/$REPO/pulls/$PR/reviews?per_page=100" +EP_INLINE="repos/$REPO/pulls/$PR/comments?per_page=100" +ep_reactions() { echo "repos/$REPO/issues/comments/$1/reactions?per_page=100"; } + +ERRFILE=$(mktemp "${TMPDIR:-/tmp}/pr-feedback-stderr.XXXXXX") || exit 1 +trap 'rm -f "$ERRFILE"' EXIT + +UNREAD=0 # any call failed since the last reset +API_OUT=""; API_RC=0; API_ERR="" + +api() { # api — sets API_OUT / API_RC / API_ERR, returns the gh status + : >"$ERRFILE" + API_OUT=$(gh api "$@" 2>"$ERRFILE") + API_RC=$? + API_ERR=$(head -n 1 "$ERRFILE" 2>/dev/null | tr -d '\000-\037') + [ "$API_RC" = 0 ] || UNREAD=1 + return "$API_RC" +} + +jqr() { # jqr + printf '%s' "$1" | jq -r "$2" 2>/dev/null +} + +unreadable() { # unreadable + if [ -n "$2" ]; then + printf ' (UNREADABLE — gh exit %s: %s)\n' "$1" "$2" + else + # An empty stderr is still not an empty PR. Saying "(none)" here is the + # whole defect, so the marker stands on the exit status alone. + printf ' (UNREADABLE — gh exit %s: no stderr)\n' "$1" + fi +} + +cap_warn() { # cap_warn + local n + n=$(jqr "$2" 'if type == "array" then length else 0 end') + case "$n" in ''|*[!0-9]*) n=0 ;; esac + [ "$n" -ge 100 ] && printf ' WARNING: %s returned %s items — page cap hit, output may be TRUNCATED\n' "$1" "$n" + return 0 +} + +# ---- unresolved threads, paged ------------------------------------------------------- +# GraphQL demands a bound, so `first:` is not optional and cannot be raised out of the +# problem: `first:100` is `first:50` with a bigger number in it, and the 101st thread is +# gone the same way. .claude/skills/pr-overview/pr_overview.py already learned this and +# pages; this follows it. Sets THREADS_OUT / THREADS_RC / THREADS_ERR. +threads_query() { # threads_query [cursor] + local after="" + [ -n "${1:-}" ] && after=", after: \"$1\"" + printf '{repository(owner:"%s",name:"%s"){pullRequest(number:%s){reviewThreads(first:%s%s){pageInfo{hasNextPage endCursor} nodes{isResolved path line}}}}}' \ + "${REPO%%/*}" "${REPO##*/}" "$PR" "$THREAD_PAGE" "$after" +} + +threads_fetch() { + local cursor="" page nodes has next pages=0 + THREADS_OUT=""; THREADS_RC=0; THREADS_ERR="" + while [ "$pages" -lt 200 ]; do + pages=$((pages + 1)) + if ! api graphql -f query="$(threads_query "$cursor")"; then + THREADS_RC=$API_RC; THREADS_ERR=$API_ERR + return 1 + fi + page=$API_OUT + nodes=$(jqr "$page" '.data.repository.pullRequest.reviewThreads.nodes[]? | select(.isResolved == false) | " \(.path):\(.line)"') + [ -n "$nodes" ] && THREADS_OUT="${THREADS_OUT}${nodes}"$'\n' + has=$(jqr "$page" '.data.repository.pullRequest.reviewThreads.pageInfo.hasNextPage // false') + next=$(jqr "$page" '.data.repository.pullRequest.reviewThreads.pageInfo.endCursor // ""') + # A page that claims a next page without a cursor to reach it would spin + # forever on the same first page; so would a cursor that does not move. + [ "$has" = "true" ] && [ -n "$next" ] && [ "$next" != "null" ] \ + && [ "$next" != "$cursor" ] || break + cursor=$next + done + return 0 +} + +# A single string that changes when ANY surface changes. Counts alone miss an edit, so the +# last id, the reaction contents and the unresolved threads go in too. Sets FP and FP_OK; +# FP_OK=0 means at least one call failed and FP is not a reading of anything. +fingerprint() { + local c="" r="" i="" x="" t="" conv="" + FP_OK=1 + if api "$EP_CONV"; then + conv=$API_OUT + c=$(jqr "$conv" '[.[] | "\(.id):\(.updated_at)"] | join(",")') + else + FP_OK=0 + fi + if api "$EP_REVIEWS"; then + r=$(jqr "$API_OUT" '[.[] | "\(.id):\(.submitted_at // "")"] | join(",")') + else + FP_OK=0 + fi + if api "$EP_INLINE"; then + i=$(jqr "$API_OUT" '[.[] | "\(.id):\(.updated_at)"] | join(",")') + else + FP_OK=0 + fi + if [ "$FP_OK" = 1 ]; then + for id in $(jqr "$conv" '.[].id'); do + if api "$(ep_reactions "$id")"; then + x="$x$(jqr "$API_OUT" '[.[].content] | join("+")')|" + else + FP_OK=0 + fi + done + fi + if threads_fetch; then + t=$(printf '%s' "$THREADS_OUT" | tr '\n' ',') + else + FP_OK=0 + fi + FP=$(printf 'conv=%s\nrev=%s\ninline=%s\nreact=%s\nthreads=%s\n' "$c" "$r" "$i" "$x" "$t") +} + +report() { + local conv conv_rc conv_err body found failed frc ferr r + printf '\n%s #%s\n\n' "$REPO" "$PR" + + api "$EP_CONV"; conv=$API_OUT; conv_rc=$API_RC; conv_err=$API_ERR + printf ' conversation comments\n' + if [ "$conv_rc" != 0 ]; then + unreadable "$conv_rc" "$conv_err" + else + body=$(jqr "$conv" '.[] | " \(.created_at[11:16]) \(.user.login): \(.body[0:100] | gsub("\n";" "))"') + if [ -n "$body" ]; then printf '%s\n' "$body"; else printf ' (none)\n'; fi + cap_warn "$EP_CONV" "$conv" + fi + + printf '\n reviews\n' + if api "$EP_REVIEWS"; then + body=$(jqr "$API_OUT" '.[] | " \(.submitted_at[11:16]) \(.user.login) \(.state): \((.body // "")[0:90] | gsub("\n";" "))"') + if [ -n "$body" ]; then printf '%s\n' "$body"; else printf ' (none)\n'; fi + cap_warn "$EP_REVIEWS" "$API_OUT" + else + unreadable "$API_RC" "$API_ERR" + fi + + printf '\n inline review comments\n' + if api "$EP_INLINE"; then + body=$(jqr "$API_OUT" '.[] | " \(.created_at[11:16]) \(.user.login) \(.path):\(.line // .original_line)\n \(.body[0:220] | gsub("\n";" "))"') + if [ -n "$body" ]; then printf '%s\n' "$body"; else printf ' (none)\n'; fi + cap_warn "$EP_INLINE" "$API_OUT" + else + unreadable "$API_RC" "$API_ERR" + fi + + # Reaction semantics matter: an automated reviewer reacts 👀 (eyes) when it PICKS THE JOB + # UP and 👍 (+1) when it finishes having found nothing. Treating any reaction as + # completion reports a pass while the review is still running. + printf '\n reactions on comments (eyes = picked up, +1 = finished, found nothing)\n' + if [ "$conv_rc" != 0 ]; then + # The comment ids come from the conversation listing. Without it there is + # nothing to enumerate, and "no reactions" would be a guess. + unreadable "$conv_rc" "$conv_err" + else + found=0; failed=0; frc=0; ferr="" + for id in $(jqr "$conv" '.[].id'); do + if api "$(ep_reactions "$id")"; then + r=$(jqr "$API_OUT" '.[] | " comment '"$id"': \(.content) by \(.user.login)"') + [ -n "$r" ] && { printf '%s\n' "$r"; found=1; } + else + failed=1; frc=$API_RC; ferr=$API_ERR + fi + done + [ "$failed" = 1 ] && unreadable "$frc" "$ferr" + [ "$found" = 0 ] && [ "$failed" = 0 ] && printf ' (none)\n' + fi + + printf '\n unresolved threads\n' + if threads_fetch; then + if [ -n "$THREADS_OUT" ]; then printf '%s' "$THREADS_OUT"; else printf ' (none)\n'; fi + else + unreadable "$THREADS_RC" "$THREADS_ERR" + fi + printf '\n' +} + +if [ "$WATCH" = 0 ]; then + report + [ "$UNREAD" = 0 ] && exit 0 + exit "$EXIT_UNREADABLE" +fi + +fingerprint +base=$FP +base_ok=$FP_OK +said=0 +printf '\n watching all five surfaces of %s #%s (every %ss, up to %ss)\n' "$REPO" "$PR" "$INTERVAL" "$TIMEOUT" +if [ "$base_ok" = 0 ]; then + printf '\n UNREADABLE — gh failed taking the baseline; polling continues, but nothing can be compared against a reading that never happened\n' + said=1 +fi +elapsed=0 +while [ "$elapsed" -lt "$TIMEOUT" ]; do + sleep "$INTERVAL"; elapsed=$((elapsed + INTERVAL)) + fingerprint + if [ "$FP_OK" = 0 ]; then + # Comparing this would report every surface it could not read as changed, + # or as unchanged, and both are answers to a question nobody asked. + if [ "$said" = 0 ]; then + printf '\n UNREADABLE — gh failed during a poll after %ss; polling continued, this reading is not compared\n' "$elapsed" + said=1 + fi + continue + fi + if [ "$base_ok" = 0 ]; then + # First readable poll after an unreadable baseline. It is a baseline, not a + # change: nothing is known to have moved between a reading and a non-reading. + base=$FP; base_ok=1 + continue + fi + if [ "$FP" != "$base" ]; then + # name WHICH surface moved: "something changed" sends you looking in the wrong tab + printf '\n changed after %ss:\n' "$elapsed" + diff <(printf '%s\n' "$base") <(printf '%s\n' "$FP") \ + | grep '^>' | cut -d= -f1 | sed 's/^> / /' | sort -u + UNREAD=0 + report + [ "$UNREAD" = 0 ] && exit 0 + exit "$EXIT_UNREADABLE" + fi +done +if [ "$said" = 1 ]; then + printf '\n polled for %ss with at least one surface UNREADABLE — this is not "no change"\n\n' "$TIMEOUT" + exit "$EXIT_UNREADABLE" +fi +printf '\n no change on any of the five surfaces after %ss\n\n' "$TIMEOUT" +exit 1 diff --git a/ops/devlane/hooks/claude/ruff-after-edit.sh b/ops/devlane/hooks/claude/ruff-after-edit.sh new file mode 100755 index 0000000..9d7d716 --- /dev/null +++ b/ops/devlane/hooks/claude/ruff-after-edit.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# ruff-after-edit.sh — a PostToolUse hook: did the edit I just made leave ruff check unhappy? +# +# `ruff.toml` makes `ruff check` the contract and `ruff format` not. A formatter complaint +# is advice and never a failure; this hook nags only on lint, at the moment of the edit, +# because the gate runs later. +# +# ADVISORY, ALWAYS. It exits 0 on everything: junk input, missing files, no ruff, its own errors, +# and a ruff that cannot run. Work arrives half-finished and a hook that interrupts that gets +# switched off, at which point it protects nothing. + +set -uo pipefail + +# The corpus lives in `ops/devlane/hooks/tests/`. `--test` is a pointer, not a runner. +if [ "${1:-}" = "--test" ]; then + printf 'corpus moved to ops/devlane/hooks/tests/\n' >&2 + exit 2 +fi + +payload=$(cat 2>/dev/null) || exit 0 +[ -n "$payload" ] || exit 0 + +field() { printf '%s' "$payload" | jq -r "$1 // empty" 2>/dev/null; } + +file=$(field '.tool_response.filePath') +[ -n "$file" ] || file=$(field '.tool_input.file_path') +[ -n "$file" ] || file=$(field '.tool_input.relative_path') # Serena's symbol/regex editors +[ -n "$file" ] || exit 0 +case "$file" in *.py) ;; *) exit 0 ;; esac + +# Serena's path is relative to the project root, so resolve against the session cwd +if [ ! -f "$file" ]; then + base=$(field '.cwd'); [ -n "$base" ] || base=$PWD + file="$base/$file" +fi +[ -f "$file" ] || exit 0 + +root=$(git -C "$(dirname "$file")" rev-parse --show-toplevel 2>/dev/null) || exit 0 +[ -n "$root" ] || exit 0 + +# A repo that has not configured ruff has not asked for this. `[tool.ruff` rather than a bare +# pyproject.toml: nearly every Python project has the latter and most do not use ruff. +configured=0 +grep -qs '^\[tool\.ruff' "$root/pyproject.toml" && configured=1 +[ -f "$root/ruff.toml" ] || [ -f "$root/.ruff.toml" ] && configured=1 +[ "$configured" = 1 ] || exit 0 + +if command -v ruff >/dev/null 2>&1; then + run_ruff() { ruff "$@"; } +elif command -v uv >/dev/null 2>&1; then + run_ruff() { (cd "$root" && uv run --quiet ruff "$@"); } +else + exit 0 +fi + +# Prove ruff RUNS before reading its verdict. `uv run` in a directory that is not a uv project +# fails, and a failure to run looks identical to "this file is clean" if you only check stdout +# — silence is what we want when ruff cannot run. CI is the gate. +run_ruff --version >/dev/null 2>&1 || exit 0 + +rel=${file#"$root"/} +lint=$(run_ruff check --quiet "$file" 2>/dev/null | head -5) +[ -n "$lint" ] || exit 0 + +detail=$(printf '\n%s' "$lint") + +jq -nc --arg r "$rel" --arg d "$detail" '{ + hookSpecificOutput: { + hookEventName: "PostToolUse", + additionalContext: ("[ruff] \($r) fails ruff check after that edit.\($d)\n\nFix it now rather than at commit time:\n ruff check \($r)\n\nThis is advisory. CI is the lint gate; this only says so at the moment of the edit.") + } +}' 2>/dev/null || true +exit 0 diff --git a/ops/devlane/hooks/claude/test-guard-hook.sh b/ops/devlane/hooks/claude/test-guard-hook.sh new file mode 100755 index 0000000..d8cb460 --- /dev/null +++ b/ops/devlane/hooks/claude/test-guard-hook.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# test-guard-hook.sh — PostToolUse(Write|Edit): did that test file just plant a fault +# without proving it landed? +# +# test-guard.py existed for a whole session as a tool you had to remember to run, which by +# this toolchain's own record is the tier that does not work: the hooks stopped two real +# mistakes mid-command the same day a run-by-name tool sat unused while its exact failure +# happened again. So the check moved to the moment it matters — the instant a test file is +# written or edited. +# +# ADVISORY, never blocking. It reports; it does not deny. A test is often written in two +# passes (the plant first, the guard second), and a hook that refused the intermediate state +# would be noise, and noise gets switched off. And whatever it is handed — junk, no file, a +# file it cannot read — it exits 0. A hook that fails takes the session with it. + +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +GUARD="$HERE/test-guard.py" + +payload=$(cat 2>/dev/null || true) +[ -n "$payload" ] || exit 0 +[ -x "$GUARD" ] || exit 0 + +read -r tool file <<<"$(printf '%s' "$payload" | python3 -c ' +import json, sys +try: + d = json.load(sys.stdin) +except Exception: + sys.exit(0) +t = d.get("tool_name") or "" +ti = d.get("tool_input") or {} +f = ti.get("file_path") or "" +# a path with whitespace would split across the read; drop those rather than mis-parse +if t and f and " " not in f and "\t" not in f: + print(t, f) +' 2>/dev/null || true)" + +case "${tool:-}" in + Write|Edit|MultiEdit) ;; + *) exit 0 ;; +esac +[ -n "${file:-}" ] && [ -f "$file" ] && [ -r "$file" ] || exit 0 + +findings=$(python3 "$GUARD" "$file" 2>/dev/null) || true +printf '%s' "$findings" | grep -q "unguarded-plant\|truncating-self-read" || exit 0 + +python3 - "$findings" <<'PY' 2>/dev/null || true +import json, sys +print(json.dumps({"hookSpecificOutput": { + "hookEventName": "PostToolUse", + "additionalContext": + "test-guard: this file plants a fault without proving the fault landed.\n" + + sys.argv[1].strip() + + "\n\nA plant that silently no-ops leaves a CLEAN fixture, so the check runs " + "against a file with no fault in it and whichever way it answers is meaningless. " + "Compare the fixture before and after, fail the case if it did not change, and " + "fail it again if the file was clobbered — a truncating write does change the " + "file, so a checksum alone will not catch it. A deliberate fixture carries " + "`# test-guard: allow`.", +}})) +PY +exit 0 diff --git a/ops/devlane/hooks/claude/test-guard.py b/ops/devlane/hooks/claude/test-guard.py new file mode 100755 index 0000000..407864c --- /dev/null +++ b/ops/devlane/hooks/claude/test-guard.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Find tests that plant a fault without proving the fault landed. + +A test suite's plants are code, and nobody tests them. They fail the way all setup fails — +silently, exit 0, with the assertion afterwards still producing a confident answer about a +fixture that does not contain the fault. + +Two shapes, both shipped in this toolchain: + + nothing happened `sed -i 's/^## Checklist$/## Checklis/'` where the anchor moved. The + fixture stays clean, the check correctly stays quiet, and the case + reports the CHECKER as broken. Hours went into the wrong file. + + too much happened `open(p,"wb").write(open(p,"rb").read().replace(...))`. Python + evaluates the `open(...,"wb")` first, truncating the file, so the read + returns nothing and the fixture is emptied. A byte-comparison guard + does NOT catch this — the file did change. + +So this looks for ANCHORED IN-PLACE MUTATION of a fixture in a test file, and asks whether +anything in that file ever verifies a mutation applied. + +Deliberately narrow. It does NOT flag: + - `printf ... > fixture` whole-file write with literal content: no anchor to miss + - `printf ... >> fixture` append: cannot silently no-op + - mutation in a non-test file: a build script rewriting a config is not a plant + + test-guard.py [path ...] files or directories + The corpus lives in `ops/devlane/hooks/tests/`. + +Exit 1 if any file plants without a guard. +""" + +import re +import sys +from pathlib import Path + +# a file is a test if it says so in its name +# Files that are read, not run. See the PROSE note in scan(). +PROSE = {".md", ".rst", ".txt", ".adoc"} + +TESTish = re.compile(r"(^|[-_./])(test|tests|spec|selftest|check-test)([-_.]|$)", re.IGNORECASE) + +# ANCHORED in-place mutation: needs an existing string to match, so it can silently no-op +ANCHORED = [ + (re.compile(r"\bsed\s+(-[a-zA-Z]*i[a-zA-Z]*\s+)"), "sed -i (anchor may not match)"), + (re.compile(r"\bperl\s+-[a-zA-Z]*p[a-zA-Z]*i"), "perl -pi (anchor may not match)"), + (re.compile(r"\.replace\([\s\S]{0,120}?\bwrite_(text|bytes)\b"), "read/replace/write-back"), + (re.compile(r"\bwrite_(text|bytes)\([\s\S]{0,120}?\.replace\("), "read/replace/write-back"), + (re.compile(r"\bsub\(\s*[^)]*\)[\s\S]{0,80}?\bwrite_(text|bytes)\b"), "re.sub write-back"), +] + +# Guards that prove a mutation landed. These are COMPARISONS ONLY. +# +# An earlier version accepted `plant()` — the name of the helper — as evidence. Removing +# every real comparison from the harness then left it "guarded", because the function was +# still called plant. A name is not a check; that is the same proxy-for-the-thing mistake +# this tool exists to catch, committed inside the tool. +GUARD = re.compile( + r"\bcksum\b|\bmd5sum\b|\bsha1sum\b|\bsha256sum\b|\bcmp\s|\bdiff\s|\bwc\s+-c\b|" + r"assert.*chang|before\s*!=\s*after|after\s*!=\s*before|" + r"\bst_size\b|\bgetsize\b|\bstat\(\)\.st_|\bassert .*\bin\b.*read", + re.IGNORECASE) + +# always a bug, in any file: the write truncates before the read runs +TRUNCATING = re.compile( + r"open\(\s*([A-Za-z_][\w.\[\]\"']*)\s*,\s*[\"']w[b+]*[\"']\s*\)" + r"\s*\.\s*write\(\s*open\(\s*\1\b") + + +ALLOW = re.compile(r"test-guard:\s*allow") +TRIPLE = re.compile('"""[\\s\\S]*?"""' + "|'''[\\s\\S]*?'''") + + +def strip_noise(text, is_python): + """Prose that describes these patterns does not execute them. + + Comment lines are blanked everywhere. Triple-quoted regions are blanked only in Python, + where they are inert data — a shell heredoc looks similar and RUNS, so it is kept. Lines + carrying `test-guard: allow` are deliberate fixtures: a harness that tests its own guard + has to contain the bug it guards against. + """ + if is_python: + text = TRIPLE.sub(lambda m: "\n" * m.group(0).count("\n"), text) + out = [] + for line in text.splitlines(): + st = line.lstrip() + out.append("" if st.startswith(("#", "//")) or ALLOW.search(line) else line) + return "\n".join(out) + + +def scan(path): + """-> list of (line_no, kind, detail) findings for one file.""" + try: + raw = Path(path).read_text(encoding="utf-8", errors="replace") + except (OSError, UnicodeDecodeError): + return [] + code = strip_noise(raw, Path(path).suffix == ".py") + found = [] + + # The truncating self-read is a bug in anything that RUNS. In prose it is usually the + # opposite — someone documenting the trap so it can be recognised — and flagging that + # fires on every edit to the document defining the rule, which is how a guard becomes + # noise and then gets ignored. Prose is not executed; if the snippet ever moves into a + # script, it is caught there. + for m in (() if Path(path).suffix.lower() in PROSE else TRUNCATING.finditer(code)): + found.append((code[:m.start()].count("\n") + 1, "truncating-self-read", + ("the write truncates the file before the read executes — the fixture " + "ends up EMPTY. Read into a variable first."))) + + if not TESTish.search(str(path)): + return found + + guarded = bool(GUARD.search(code)) + if guarded: + return found + + for pat, why in ANCHORED: + m = pat.search(code) + if m: + found.append((code[:m.start()].count("\n") + 1, "unguarded-plant", + f"{why}, and nothing in this file checks a mutation landed")) + break # one finding per file: this is a per-file property + return found + + +def main(argv): + paths = [] + for a in argv: + p = Path(a) + if p.is_dir(): + paths += [q for q in p.rglob("*") if q.is_file() and q.suffix in + (".sh", ".bash", ".py", ".zsh", "")] + elif p.is_file(): + paths.append(p) + if not paths: + print("usage: test-guard.py [path ...]", file=sys.stderr) + return 2 + + bad = [] + for p in sorted(set(paths)): + for line, kind, detail in scan(p): + bad.append((p, line, kind, detail)) + + if not bad: + print(f" every plant is guarded ({len(paths)} file(s) scanned)") + return 0 + + print(" tests that plant a fault without proving it landed:\n") + for p, line, kind, detail in bad: + print(f" {p}:{line} [{kind}]") + print(f" {detail}") + print(f"\n {len(bad)} finding(s). A plant that silently no-ops makes its case " + f"uninterpretable:\n the check then runs against a clean fixture and answers " + f"a question nobody asked.\n Fix: compare the fixture before and after, and fail " + f"the case if it did not change.\n") + return 1 + + +if __name__ == "__main__": + import sys + if "--test" in sys.argv: + print("corpus moved to ops/devlane/hooks/tests/", file=sys.stderr) + sys.exit(2) + sys.exit(main(sys.argv[1:])) diff --git a/ops/devlane/hooks/claude/unsafe-command.py b/ops/devlane/hooks/claude/unsafe-command.py new file mode 100755 index 0000000..48ceb98 --- /dev/null +++ b/ops/devlane/hooks/claude/unsafe-command.py @@ -0,0 +1,417 @@ +#!/usr/bin/env python3 +"""Refuse two command shapes that fail silently, and say what to write instead. + +Both were rules in CLAUDE.md before they were checks here, and both were then broken dozens +of times in the session that produced this file. That is what makes them nightmares rather +than mistakes: knowing better is not the same as being stopped. + + 1. A test or build runner clipped to a handful of lines with no failure grep. + `playwright test | tail -4` printed "1 skipped / 71 passed" and cut "61 failed" off the + top. The summary block is several lines and the failure count is FIRST, so a short tail + reliably removes exactly the number that matters. + + 2. A string-replace whose result is written, with nothing asserting the anchor matched. + `s.replace(old, new)` returns the input unchanged when `old` is absent. The write + succeeds, the exit code is 0, and the next command runs against the unedited file. + +Neither refusal costs more than a retry, and both name the fix. The corpus lives in +`ops/devlane/hooks/tests/`. +""" + +import json +import logging +import os +import re +import subprocess +import sys + +# `.claude/settings.json` runs every hook in this directory by path, so a sibling import is +# free — but only once the directory is on the path, which it is not when python reads the +# script from stdin. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from command_shape import commands, statements, strip_data_heredocs + +# ------------------------------------------------------------------------- amend +# +# 3. `git commit --amend` absorbing an index nobody inspected. +# --amend commits whatever is staged and prints one sha. Twice in one session a doc +# build left 48 generated .rst files in the tree; the first time `git add -A` staged +# them (7 files became 56), the second time they were still staged from before, so +# even `git add CHANGELOG.md` produced 61. Both were caught only because the file +# count was printed afterwards and looked wrong. +# +# 4. `git commit --amend` while HEAD is a commit that is already upstream. +# Amending there does not add a commit, it REWRITES the one HEAD points at — which +# was the maintainer's. The tell was an unrelated file appearing as "new" in the +# branch diff. Unambiguously wrong, so this one refuses rather than asks. + +AMEND = re.compile(r"\bgit\b[^;&|\n]*?\bcommit\b[^;&|\n]*?--amend\b") +# an amend of a couple of files is ordinary; an amend of a dozen is a sweep +AMEND_STAGED_LIMIT = 12 +PUBLISHED_REFS = ("upstream/main", "upstream/master", "origin/main", "origin/master") + + +def is_amend(cmd: str) -> bool: + """True only where git is the command being run. + + `grep -n 'git commit --amend' notes.md` mentions one; it does not make one. That is the + same proxy-for-the-thing mistake the rest of this file exists to catch, so the test is + positional: git must be the program the position invokes. + + `command_shape` is what decides where a position starts and what its program is. The + hand-rolled version here dropped UPPERCASE env assignments only, so `flags=1 git commit + --amend` was not an amend; and it required `git` literally first, so `sudo git commit + --amend` and `for b in a b; do git commit --amend; done` were not either. + """ + return any(c.startswith("git ") and AMEND.search(c) for c in commands(cmd)) + + +def _ok(args: list[str], timeout: int = 8) -> bool: + """Exit status only — for queries like `merge-base --is-ancestor` that answer with it.""" + try: + return subprocess.run(args, capture_output=True, timeout=timeout, check=False).returncode == 0 # noqa: S603 — literal executable, list argv, no shell; the only non-literal elements are the -C operand (the hook's own cwd) and a ref name from PUBLISHED_REFS + except Exception: + logging.exception("git query failed; treating as not-ok") + return False + + +def head_already_published(cwd: str) -> str | None: + """The published ref that already contains HEAD, if any. + + `--is-ancestor` is reflexive, so HEAD sitting exactly on upstream/main answers yes — + which is the case that caused the incident. + """ + for ref in PUBLISHED_REFS: + if _ok(["git", "-C", cwd, "rev-parse", "--verify", "--quiet", ref]) and \ + _ok(["git", "-C", cwd, "merge-base", "--is-ancestor", "HEAD", ref]): + return ref + return None + + +def staged_summary(cwd: str) -> tuple[int, str]: + """How many files an amend would carry, and where they are.""" + out = _run(["git", "-C", cwd, "diff", "--cached", "--name-only"]) + files = [f for f in out.splitlines() if f.strip()] + tops: dict[str, int] = {} + for f in files: + key = "/".join(f.split("/")[:2]) if "/" in f else f + tops[key] = tops.get(key, 0) + 1 + top = sorted(tops.items(), key=lambda kv: -kv[1])[:4] + return len(files), ", ".join(f"{k} ({n})" for k, n in top) + + +def _run(args: list[str], timeout: int = 8) -> str: + try: + p = subprocess.run(args, capture_output=True, text=True, timeout=timeout, check=False) # noqa: S603 — literal executable (git), list argv, no shell; the only non-literal element is the -C operand (the hook's own cwd) + except Exception: + logging.exception("git query failed; treating as empty output") + return "" + return p.stdout.strip() if p.returncode == 0 else "" + + +# This list was a subset of the runners actually in use — `poe doc-build` was here and +# `poe test`, the command this project runs its suite with, was not. Found by mutation-testing +# against real commands, not by reading the regex, and it is the "subset of an unenumerated +# set" class committed inside the tool that refuses two other members of it. +# +# So: any `poe `, and the common suite runners. Task runners are enumerated by their +# LAUNCHER (poe, make, tox, npm, uv run) rather than by task name, because task names are +# per-project and a per-project list is the same bug again. +# Linters and formatters are excluded deliberately: they print their verdict at the END +# ("All checks passed", "Found N errors"), so a short tail keeps it. The failure this rule +# exists for is a TEST runner, whose summary block puts the failure count FIRST. +RUNNER = re.compile(r"\b(playwright test|pytest|poe\s+(?!lint|format|fmt|type)[a-z][\w-]*|sphinx-build|selftest\.sh|" + r"mutation-check\.sh|mutation-live\.sh|npm (?:test|run test)|cargo test|" + r"go test|jest|vitest|tox|make\s+(?:test|check|lint)|rspec|phpunit)\b") +CLIPPED = re.compile(r"\|\s*(?:tail|head)\s+-(\d+)\b") +# `[^|]*` here was a bug: `grep -iE "warning|error"` contains a pipe INSIDE the quoted +# pattern, so the class stopped before reaching the word that made it safe. Look for a +# grep and a failure word anywhere in the command instead. +FAIL_GREP = re.compile(r"grep(?s:.)*?(fail|passed|error|✘|✗)", re.IGNORECASE) + +# A multi-alternation grep at the head of a pipeline is a COMPLETENESS question — "does any +# of these appear?" — and a head/tail on it silently truncates the answer. A head -8 on +# exactly that shape ended before nixd_ls.py:114 (where which("nix") raises), and the +# absence conclusion drawn from the first eight matches was wrong. Same class as the +# 30-per-page GitHub listing read as a total. Scope, tuned on the session's real commands: +# grep must START the statement (a mid-pipeline grep is a filter — `pytest | grep -E +# 'failed|passed' | tail` is the SANCTIONED summary shape and must stay quiet), and the +# pattern must carry >=2 alternatives-separators inside quotes (single-needle greps piped +# to head are region views; `grep -rn foo src | head -5` stays ordinary work). +SWEEP_GREP = re.compile(r"^\s*grep\b") + + +def quoted_pipe_count(stmt: str) -> int: + """Count '|' characters inside quoted spans — alternation separators in a grep pattern, + whether basic (a\\|b) or extended (-E 'a|b').""" + count, quote = 0, None + for ch in stmt: + if quote: + if ch == quote: + quote = None + elif ch == "|": + count += 1 + elif ch in "'\"": + quote = ch + return count + + +WRITES = re.compile(r"write_text\(|open\([^)]*['\"][wa]['\"]|\.writelines\(|>\s*\$?\w*\.py\b") +REPLACE = re.compile(r"\.replace\(|re\.sub\(|\.subn\(") +GUARDED = re.compile(r"\bassert\b|\braise\b|if\s+\w+\s+not\s+in\b|count\s*==|!=\s*s\b") + +CLIP_LIMIT = 6 + + +# A pull request carries feedback in FOUR API objects. A loop that waits for a verdict while +# watching a subset reports "still running" with the answer already sitting on the surface it +# does not read — forty minutes the first time, and again today when a clean Codex verdict +# arrived as an issue comment and the poll was watching reviews, inline comments and +# reactions. Both times the tool that reads all four already existed. +POLL_LOOP = re.compile(r"\bwhile\b[^\n]*?\bdo\b|\bfor\b[^\n]*?\bdo\b|\bsleep\s+\d+") +SURFACES = { + "conversation comments (issues/N/comments)": re.compile(r"issues/[^/\s]+/comments\b"), + "review bodies (pulls/N/reviews)": re.compile(r"pulls/[^/\s]+/reviews\b"), + "inline review comments (pulls/N/comments)": re.compile(r"pulls/[^/\s]+/comments\b"), + "reactions": re.compile(r"/reactions\b"), +} + +# `statusCheckRollup` is an APPEND-ONLY list of check-run attempts against the head sha, not +# a state. A re-run does not replace the attempt it supersedes; it appends beside it. On +# oraios/serena#1873 the field held 47 entries for 24 distinct check names, with a FAILURE +# from 05:27:14Z sitting next to the SUCCESS from 16:11:18Z that cleared it. Filtering it for +# FAILURE answers "did any attempt ever fail here", which reads exactly like "is this red +# now" -- and reported five green PRs as red, contradicted by the user looking at the UI. +# `gh pr checks` and the web UI collapse to the latest attempt per name; nothing else does. +# Prose handed to a DOUBLE-QUOTED shell argument is shell input, not text. Backticks inside it +# run as command substitution: on 2026-08-18 a commit message containing a backticked flag name +# executed it, the command failed, and its EMPTY output replaced the phrase. The commit succeeded +# and printed a sha; the message on disk (still at 4633f78 in the reflog) had a hole in it, and it +# was found only by grepping afterwards for a phrase I remembered writing. Worse is the silent +# case: a substitution that SUCCEEDS injects its output into the prose with no error at all. +# +# Only backticks are refused, on purpose. `$(...)` is the idiomatic substitution form and is +# normally deliberate (`-m "release $(cat VERSION)"`), and a bare `$VAR` in prose is common enough +# that flagging it would be noise. Backticks are archaic as substitution and ubiquitous as +# markdown code spans, so the intent is not ambiguous. The other two are a declared gap. +QUOTED_PROSE = re.compile(r'(?:-m|--message|--body)\s+"((?:[^"\\]|\\.)*)"') + +ROLLUP = re.compile(r"statusCheckRollup") +PRINTS_DATA = re.compile(r"^\s*(?:echo|printf)\b") +ROLLUP_VERDICT = re.compile(r"\.conclusion\b|[\"']conclusion[\"']") +# an explicit collapse to one attempt per name is the correct use and must stay usable. +# `group_by(.conclusion)` is NOT that -- it is the incident's own second shape. +ROLLUP_DEDUPED = re.compile(r"(?:group_by|unique_by|sort_by)\s*\(\s*\.(?:name|context)\b|" + r"max_by\s*\(\s*\.(?:startedAt|completedAt)\b|" + r"latest_per_check") + +# Blanking data heredocs — a commit message that DESCRIBES an unguarded `.replace()` is not +# one — moved into `command_shape`, along with the fix to what "this heredoc executes" means. +# The test here was `\b(python3?|…|bash|sh|…)\b` against the whole head, and `\bsh\b` matched +# the `sh` in a FILENAME: `cat > setup.sh <<'EOF'` and `tee notes.sh <<'EOF'` were read as +# running their bodies. In this repo that refused a memory-file write, because the file was +# named `…-bash.md`. + + +def check(cmd: str): + """Return (kind, message) if the command should be refused, else None.""" + if not cmd or not cmd.strip(): + return None + cmd = strip_data_heredocs(cmd) + + # The clip has to be in the SAME pipeline as the runner. Matching "runner anywhere" plus + # "clip anywhere" flagged `poe doc-build > log; grep ... | head -3`, where the head is on + # a grep of markdown and the runner's output went to a file. A checker that blocks real + # work gets switched off, so precision matters more here than reach. + m = None + # `statements()` splits on statement separators only and keeps a PIPELINE whole — a bare + # & also appears inside `2>&1`, and splitting there tore `playwright test ... 2>&1 | + # tail -4` into two halves, putting the runner in one and the clip in the other. + # A runner's NAME passed to a file-reading command is a string, not an invocation: + # `grep -n foo tools/selftest.sh | head -5` reads a file and was refused for it. The + # same proxy-for-the-thing mistake the rest of this toolchain exists to catch. + READS_FILES = re.compile(r"\b(grep|rg|ls|cat|wc|find|stat|chmod|rm|cp|mv|sed|awk|git|" + r"head|tail|diff|cksum|md5sum|realpath|dirname|basename)\b") + + for stmt in statements(cmd): + if not RUNNER.search(stmt) or FAIL_GREP.search(stmt): + continue + # only consider a clip that appears after the runner within this statement + rpos = RUNNER.search(stmt).start() + if READS_FILES.search(stmt[:rpos]): + continue # the runner is an argument, not the command + for c in CLIPPED.finditer(stmt): + if c.start() > rpos: + m = c + break + if m: + break + if m and int(m.group(1)) <= CLIP_LIMIT: + return ("clipped-runner", + (f"This clips a test/build runner to {m.group(1)} lines with nothing grepping for " + "the failure count. Runner summaries put 'N failed' FIRST, so a short tail cuts " + "exactly the line that matters — that is how '61 failed' got read as a pass.\n\n" + "Write it so the failure count cannot be dropped, e.g.\n" + " ... 2>&1 | grep -E 'failed|passed|skipped' | tail -3\n" + "or keep the full output and say why you need it.")) + + for stmt in statements(cmd): + if not SWEEP_GREP.search(stmt): + continue + clip = CLIPPED.search(stmt) + if clip and quoted_pipe_count(stmt[: clip.start()]) >= 2: + return ("clipped-sweep", + ("This clips a multi-alternation grep — a completeness question (\"does any of " + "these appear?\") whose answer head/tail truncates SILENTLY. A head -8 on exactly " + "this shape ended before the line that mattered, and the absence conclusion drawn " + "from the first eight matches was wrong. Same class as a 30-per-page API listing " + "read as a total.\n\n" + "Count first, then look:\n" + " grep -c PATTERN file # the verdict, untruncatable\n" + " ops/devlane/hooks/claude/evgrep.sh PATTERN file # count header + bounded view + loud truncation marker\n" + "or take the full output and say why you need it.")) + + # only the shape that WRITES: an unwritten replace is harmless + if REPLACE.search(cmd) and WRITES.search(cmd) and not GUARDED.search(cmd): + return ("unguarded-replace", + ("This replaces text and writes the result with nothing checking the anchor " + "matched. `.replace()` returns the input unchanged when the pattern is absent: " + "the write succeeds, the exit code is 0, and the next command runs against an " + "unedited file. That happened three times in one session.\n\n" + "Guard it, e.g.\n" + " assert old in s, 'anchor missing'\n" + " s = s.replace(old, new, 1)\n" + "or use re.subn and assert the count.")) + + # a wait-loop over SOME of a PR's feedback surfaces + if "gh " in cmd and POLL_LOOP.search(cmd): + present = [n for n, p in SURFACES.items() if p.search(cmd)] + missing = [n for n in SURFACES if n not in present] + if present and missing: + return ("partial-feedback-poll", + "This waits for PR feedback while watching " + f"{len(present)} of {len(SURFACES)} surfaces. Not watched:\n" + + "".join(f" - {n}\n" for n in missing) + + "\nAn automated reviewer answers on whichever surface suits it: findings " + "arrive as a review plus inline comments, a clean verdict as a conversation " + "comment or a 👍 reaction. A poll that reports 'still running' while the " + "answer sits on an unwatched surface answers the question wrongly rather " + "than not answering it.\n\n" + "Use the tool that reads all four:\n" + " ops/devlane/hooks/claude/pr-feedback.sh [repo] one report\n" + " ops/devlane/hooks/claude/pr-feedback.sh [repo] --watch wait for a change") + + # a rollup read reduced to a pass/fail verdict, with nothing collapsing the attempts + for stmt in statements(cmd): + r = ROLLUP.search(stmt) + # the field NAME ahead of the token means it is a string being read from a file or + # printed -- not a request being made. `echo '' | python3 hook.py` + # is how this hook is tested, and it refused that on the day it was written. + if not r or READS_FILES.search(stmt[: r.start()]) or PRINTS_DATA.search(stmt[: r.start()]): + continue + if not ROLLUP_VERDICT.search(stmt) or ROLLUP_DEDUPED.search(stmt): + continue + return ("stale-check-rollup", + ("This reads `statusCheckRollup` and reduces it to a pass/fail verdict. That " + "field is an APPEND-ONLY list of check-run ATTEMPTS against the head sha, not " + "the current state: a re-run appends beside the attempt it supersedes rather " + "than replacing it. On oraios/serena#1873 it carried 47 entries for 24 check " + "names, with a FAILURE from 05:27:14Z next to the SUCCESS from 16:11:18Z that " + "cleared it.\n\n" + "So `select(.conclusion==\"FAILURE\")` answers \"did any attempt ever fail " + "here\", which reads identically to \"is this red now\". It reported 18 failing " + "checks across 5 PRs that were all green, and the user had to correct it.\n\n" + "Ask the question you mean. `gh pr checks` collapses to the latest attempt " + "per name, exactly as the web UI does -- verified: 24 rows for the 47-entry " + "rollup above.\n" + " gh pr checks --repo \n" + " gh pr checks --repo --json name,state,bucket # scriptable;\n" + " bucket is already pass/fail/pending/skipping/cancel\n" + "or, only when you must batch many PRs in one call, collapse them yourself:\n" + " --jq '.statusCheckRollup | group_by(.name) | map(max_by(.startedAt))\n" + " | map(select(.conclusion==\"FAILURE\"))'")) + + # a message or body passed as a double-quoted string, with backticks inside it. + # Statement-scoped so that `echo '' | python3 hook.py` -- which is how this + # rule gets tested -- is read as printing the shape rather than committing it. + # NOT split into statements: a prose message spans newlines, and splitting on them tore + # `-m "line one\n\nline two with a `backtick`"` into fragments that matched nothing -- losing + # the long-message case, which is the one this rule exists for. PRINTS_DATA is anchored at the + # start of the command, which is enough to tell `echo '' | python3 hook.py` apart + # from a real commit. A command that echoes FIRST and then commits is a declared blind spot. + if not PRINTS_DATA.search(cmd): + for quoted in QUOTED_PROSE.finditer(cmd): + if "`" not in quoted.group(1): + continue + return ("shell-expanded-prose", + ("This passes prose as a DOUBLE-QUOTED shell argument, and the prose contains " + "backticks. The shell runs what is between them and substitutes the output — so " + "the text you wrote is not the text that gets committed or posted.\n\n" + "That happened on 2026-08-18: a commit message containing a backticked flag name " + "executed it, the command failed, and its empty output REPLACED the phrase. The " + "commit succeeded and printed a sha; the message on disk had a hole in it, and it " + "was found only by grepping later for a phrase I remembered writing. A " + "substitution that SUCCEEDS is worse — it injects output into your prose with no " + "error at all.\n\n" + "Pass prose in a form the shell does not read:\n" + " git commit -q -F - <<'EOF' # quoted heredoc — no expansion at all\n" + " ...your message...\n" + " EOF\n" + " git commit -q -F # or write it to a file first\n" + " git commit -m 'single quotes' # for a one-liner with no substitution\n\n" + "Declared gap: `$(...)` and `$VAR` are NOT refused here, because both are " + "normally deliberate. Only backticks are unambiguous.")) + return None + + +if __name__ == "__main__": + import sys + if "--test" in sys.argv: + print("corpus moved to ops/devlane/hooks/tests/", file=sys.stderr) + sys.exit(2) + try: + payload = json.load(sys.stdin) + cmd = ((payload.get("tool_input") or {}).get("command") or "")[:4000] + except Exception: + logging.exception("failed to read hook payload from stdin") + sys.exit(0) + hit = check(cmd) + + # An amend rewrites rather than adds. Both of its failure modes are answerable from + # local git state alone, so this costs nothing and runs first. + if not hit and is_amend(cmd): + acwd = os.getcwd() + published = head_already_published(acwd) + if published: + hit = ("amend-published", + (f"HEAD is already contained in `{published}`, so `--amend` would not add a " + "commit — it would REWRITE the one HEAD points at, which is not yours. " + "This happened once already: the amend replaced the maintainer's commit " + "and the branch then proposed undoing their work, visible only as an " + "unrelated file showing up as `new` in the diff.\n\n" + "You almost certainly want a new commit:\n" + " git commit -F \n" + "and if you meant to amend your own work, check `git rev-parse HEAD^` " + "is the base you expect first.")) + else: + n, tops = staged_summary(acwd) + if n > AMEND_STAGED_LIMIT: + print(json.dumps({"hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "ask", + "permissionDecisionReason": ( + f"[amend-sweep] This amend would carry {n} staged files: {tops}. " + "`--amend` commits whatever is in the index and prints one sha, so a " + "build's generated output sitting there gets folded in silently — " + "twice in one session that turned 7 files into 56, then 13 into 61. " + "Confirm the staged set is what you mean:\n" + " git diff --cached --name-only"), + }})) + sys.exit(0) + + if hit: + print(json.dumps({"hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": f"[{hit[0]}] {hit[1]}", + }})) diff --git a/ops/devlane/hooks/commit-msg b/ops/devlane/hooks/commit-msg new file mode 100755 index 0000000..8e7f3a5 --- /dev/null +++ b/ops/devlane/hooks/commit-msg @@ -0,0 +1,53 @@ +#!/bin/sh +# commit-msg -- refuse a trailer git will not parse (ops/devlane/hooks). +# +# UNLIKE the recorders beside it. post-checkout and post-commit swallow +# every failure and always exit 0, because a broken recorder must never +# stop work. This one is a gate: refusing is the entire point, and it is +# the only hook here that can fail a commit. +# +# It exists because the defect it catches is created here and nowhere +# else, and is invisible everywhere else. A blank line between two +# trailers demotes everything above it to body text; `git commit` still +# exits 0 and `git log` still prints the line. Measured 2026-08-24: 50 +# of 243 commits across three branches carry a `Source:` line git cannot +# parse, including the commit that applied a review's attribution +# findings. It was found and hand-fixed before, and came back. +# +# A run that cannot check refuses rather than passing quietly, per +# AGENTS.md. `git commit --no-verify` is the deliberate way past. + +set -u + +msg_file=${1:-} +[ -n "$msg_file" ] || { echo "commit-msg: no message file given" >&2; exit 1; } + +top=$(git rev-parse --show-toplevel 2>/dev/null) || { + echo "commit-msg: not inside a work tree" >&2 + exit 1 +} +check="$top/ops/devlane/workflow/checks/commit_trailers.py" + +# When the hook cannot run it WARNS and lets the commit through, and the +# distinction is deliberate. This hook is local convenience, not the +# gate: the gate is `dev: gates (commit-trailers)` in CI, which runs the +# same checker over the branch's commits and cannot be dodged. Blocking +# every commit in a clone that happens to lack the checker -- a linked +# worktree mid-checkout, a shallow clone, a machine without python -- +# would stop work for an environment problem, not a bad message. It +# refuses for the one thing it is here to judge: the message. +if [ ! -f "$check" ] || ! command -v python3 >/dev/null 2>&1; then + echo "commit-msg: cannot check trailers here (checker or python3" >&2 + echo " missing) — letting the commit through. CI still" >&2 + echo " gates this: dev: gates (commit-trailers)." >&2 + exit 0 +fi + +python3 "$check" --message-file "$msg_file" || { + echo "" >&2 + echo "commit-msg: the message above was NOT committed." >&2 + echo " Move the trailer into the final block — one" >&2 + echo " contiguous paragraph, no blank line between" >&2 + echo " Source: and Co-Authored-By:." >&2 + exit 1 +} diff --git a/ops/devlane/hooks/install.sh b/ops/devlane/hooks/install.sh new file mode 100755 index 0000000..7173000 --- /dev/null +++ b/ops/devlane/hooks/install.sh @@ -0,0 +1,70 @@ +#!/bin/sh +# install.sh -- install the crossing-recorder hooks (ops/devlane/hooks) +# into the clone's shared hooks dir, so every worktree fires them. +# +# sh ops/devlane/hooks/install.sh [--force] +# +# Idempotent: a re-run over identical hooks is a quiet success. An +# existing DIFFERENT hook of the same name is never clobbered -- the +# install refuses, naming the file; --force replaces it. + +set -u + +force=0 +case "${1:-}" in + --force) force=1 ;; + "") ;; + *) + echo "usage: install.sh [--force]" >&2 + exit 2 + ;; +esac + +# The hooks to install live beside this script; fall back to the +# repo-relative package path when run from the repo root. +src_dir=$(dirname "$0") +if [ ! -f "$src_dir/post-checkout" ] || [ ! -f "$src_dir/post-commit" ] || [ ! -f "$src_dir/commit-msg" ]; then + src_dir="ops/devlane/hooks" +fi +if [ ! -f "$src_dir/post-checkout" ] || [ ! -f "$src_dir/post-commit" ] || [ ! -f "$src_dir/commit-msg" ]; then + echo "install.sh: cannot find post-checkout/post-commit/commit-msg beside $0 or under ops/devlane/hooks" >&2 + exit 1 +fi + +common=$(git rev-parse --git-common-dir) || exit 1 +# Detect by exit status, not value: an explicitly EMPTY hooksPath is +# still configured, and git will not run hooks from $common/hooks. +if configured=$(git config --get core.hooksPath); then + # Installing into $common/hooks would report success while git runs + # hooks from the configured path — a successful inert install. + echo "install.sh: core.hooksPath is configured (${configured:-empty}); git will not run hooks from $common/hooks — install into that path yourself or unset core.hooksPath" >&2 + exit 1 +fi +hooks_dir="$common/hooks" +mkdir -p "$hooks_dir" || exit 1 + +# Refuse before copying anything: report every conflict, touch nothing. +status=0 +for name in post-checkout post-commit commit-msg; do + dest="$hooks_dir/$name" + if { [ -e "$dest" ] || [ -h "$dest" ]; } && [ "$force" -eq 0 ] && ! cmp -s "$src_dir/$name" "$dest"; then + echo "install.sh: existing $name differs from the packaged hook; not clobbering (use --force)" + echo "install.sh: existing $name differs from the packaged hook; not clobbering (use --force)" >&2 + status=1 + fi +done +[ "$status" -eq 0 ] || exit "$status" + +for name in post-checkout post-commit commit-msg; do + dest="$hooks_dir/$name" + if [ -f "$dest" ] && [ ! -h "$dest" ] && cmp -s "$src_dir/$name" "$dest"; then + chmod 755 "$dest" || exit 1 + continue + fi + # Unlink first: cp onto a symlink writes THROUGH it to a target the + # installer was never pointed at. The entry must be a fresh regular file. + rm -f "$dest" || exit 1 + cp "$src_dir/$name" "$dest" || exit 1 + chmod 755 "$dest" || exit 1 +done +exit 0 diff --git a/ops/devlane/hooks/post-checkout b/ops/devlane/hooks/post-checkout new file mode 100755 index 0000000..ce072f2 --- /dev/null +++ b/ops/devlane/hooks/post-checkout @@ -0,0 +1,44 @@ +#!/bin/sh +# post-checkout -- git-native crossing recorder (ops/devlane/hooks). +# +# Appends one JSONL entry to the clone-shared context stream when a +# branch checkout happens (arg 3, the flag, is 1). File checkouts +# (flag 0) are not crossings and are ignored. +# +# This hook must never break git: every failure path is swallowed and +# the exit status is always 0. If the stream cannot be written, the +# record is lost and the checkout succeeds anyway. + +json_escape() { + # Control characters would split or corrupt the JSONL record; they + # are never legitimate in these values, so newline/tab/CR become + # spaces and the rest are dropped before quoting is escaped. + printf '%s' "$1" | tr '\n\r\t' ' ' | tr -d '\000-\037' \ + | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' +} + +record() { + common=$(git rev-parse --git-common-dir) || return 0 + [ -n "$common" ] || return 0 + top=$(git rev-parse --show-toplevel) || return 0 + worktree=$(basename "$top") || return 0 + agent="${WF_AGENT:-${USER:-unknown}}" + at=$(date -u +%Y-%m-%dT%H:%M:%SZ) || return 0 + printf '{"at": "%s", "kind": "%s", "what": "%s", "detail": "", "agent": "%s", "worktree": "%s", "via": "git-hook"}\n' \ + "$(json_escape "$at")" "$(json_escape "$1")" "$(json_escape "$2")" \ + "$(json_escape "$agent")" "$(json_escape "$worktree")" \ + >>"$common/claude-context-stream.jsonl" || return 0 +} + +main() { + [ "${3:-}" = "1" ] || return 0 + # symbolic-ref answers for unborn branches (checkout --orphan), + # where rev-parse fails; rev-parse covers detached HEAD. + branch=$(git symbolic-ref --short -q HEAD) \ + || branch=$(git rev-parse --abbrev-ref HEAD) || return 0 + [ -n "$branch" ] || return 0 + record "branch" "switched to $branch" +} + +main "$@" >/dev/null 2>&1 || : +exit 0 diff --git a/ops/devlane/hooks/post-commit b/ops/devlane/hooks/post-commit new file mode 100755 index 0000000..795af85 --- /dev/null +++ b/ops/devlane/hooks/post-commit @@ -0,0 +1,39 @@ +#!/bin/sh +# post-commit -- git-native crossing recorder (ops/devlane/hooks). +# +# Appends one JSONL entry to the clone-shared context stream after +# every commit, carrying the new short SHA. +# +# This hook must never break git: every failure path is swallowed and +# the exit status is always 0. If the stream cannot be written, the +# record is lost and the commit succeeds anyway. + +json_escape() { + # Control characters would split or corrupt the JSONL record; they + # are never legitimate in these values, so newline/tab/CR become + # spaces and the rest are dropped before quoting is escaped. + printf '%s' "$1" | tr '\n\r\t' ' ' | tr -d '\000-\037' \ + | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' +} + +record() { + common=$(git rev-parse --git-common-dir) || return 0 + [ -n "$common" ] || return 0 + top=$(git rev-parse --show-toplevel) || return 0 + worktree=$(basename "$top") || return 0 + agent="${WF_AGENT:-${USER:-unknown}}" + at=$(date -u +%Y-%m-%dT%H:%M:%SZ) || return 0 + printf '{"at": "%s", "kind": "%s", "what": "%s", "detail": "", "agent": "%s", "worktree": "%s", "via": "git-hook"}\n' \ + "$(json_escape "$at")" "$(json_escape "$1")" "$(json_escape "$2")" \ + "$(json_escape "$agent")" "$(json_escape "$worktree")" \ + >>"$common/claude-context-stream.jsonl" || return 0 +} + +main() { + short=$(git rev-parse --short HEAD) || return 0 + [ -n "$short" ] || return 0 + record "head" "head now at $short" +} + +main "$@" >/dev/null 2>&1 || : +exit 0 diff --git a/ops/devlane/hooks/tests/corpus.py b/ops/devlane/hooks/tests/corpus.py new file mode 100644 index 0000000..3eb9653 --- /dev/null +++ b/ops/devlane/hooks/tests/corpus.py @@ -0,0 +1,42 @@ +"""In-file --test tables, moved out of the subject scripts. + +Copied from the hook files at the commit this change is cut from. +Do not import this module from the subject scripts. +""" + +# 51 rows; commands() must contain the want +SHAPE_FINDS = [('plain push', 'git push origin topic', 'git push origin topic', False), ('plain checkout', 'git checkout main', 'git checkout main', False), ('plain gh create', 'gh pr create --fill', 'gh pr create --fill', False), ('&& chain', 'git status -s && git push origin b', 'git push origin b', False), ('pipeline member', 'echo x | git hash-object -w --stdin', 'git hash-object -w --stdin', False), ('uv run poe task', 'uv run poe doc-build', 'doc-build', False), ('npx binary', 'npx playwright test', 'playwright test', False), ('newline separated', 'cd repo\ngit pull', 'git pull', False), ('git -C', 'git -C . push origin topic', 'git push origin topic', True), ('git -C then subcmd', 'git -C sub checkout main', 'git checkout main', True), ('git -c', 'git -c push.default=simple push origin topic', 'git push origin topic', True), ('git --git-dir=', 'git --git-dir=/p/.git push origin main', 'git push origin main', True), ('git --git-dir arg', 'git --git-dir /p/.git push origin main', 'git push origin main', True), ('git -C and -c', 'git -C sub -c core.hooksPath=/dev/null commit --amend', 'git commit --amend', True), ('gh -R', 'gh -R o/r pr create --fill', 'gh pr create --fill', True), ('gh --repo', 'gh --repo o/r pr merge 3 --squash', 'gh pr merge 3 --squash', True), ('git --no-pager', 'git --no-pager log --oneline', 'git log --oneline', True), ('env assignment', 'GIT_SSH_COMMAND=x git push origin b', 'git push origin b', True), ('lowercase assignment', 'flags=1 git push origin b', 'git push origin b', True), ('two assignments', 'A=1 B=2 git push origin b', 'git push origin b', True), ('quoted assignment', "GIT_SSH_COMMAND='ssh -i k' git push origin b", 'git push origin b', True), ('quoted git -c value', "git -c 'user.name=A B' commit --amend", 'git commit --amend', True), ('sudo', 'sudo git push origin b', 'git push origin b', False), ('sudo with operand', 'sudo -u ci git push origin b', 'git push origin b', True), ('env wrapper', 'env VAR=1 git push origin b', 'git push origin b', True), ('command builtin', 'command git push origin b', 'git push origin b', True), ('exec', 'exec git push origin b', 'git push origin b', True), ('nice -n', 'nice -n 5 git push origin b', 'git push origin b', True), ('timeout duration', 'timeout 60 git push origin b', 'git push origin b', True), ('xargs -I{}', 'echo b | xargs -I{} git push origin', 'git push origin', True), ('alias-escaped name', '\\git push origin b', 'git push origin b', True), ('wrapper then -C', 'sudo git -C . push origin b', 'git push origin b', True), ('if/then', 'if true; then git push origin b; fi', 'git push origin b', True), ('for/do', 'for r in a b; do git push origin $r; done', 'git push origin $r', True), ('while/do', 'while read x; do git push origin b; done', 'git push origin b', True), ('brace group', '{ git push origin b; }', 'git push origin b', True), ('subshell', '( git push origin b )', 'git push origin b', True), ('cd && subshell', '( cd sub && git checkout main )', 'git checkout main', True), ('line continuation', 'git \\\n push origin main', 'git push origin main', True), ('leading tab', '\tgit push origin b', 'git push origin b', False), ('bash -c', "bash -c 'git push origin b'", 'git push origin b', True), ('sh -c double quoted', 'sh -c "git checkout main"', 'git checkout main', True), ('$( ) substitution', 'out=$(git push origin b)', 'git push origin b', False), ('backtick', 'out=`git push origin b`', 'git push origin b', False), ('substitution in dq', 'echo "$(git push origin b)"', 'git push origin b', False), ('npm run', 'npm run build', 'build', False), ('npm test stays whole', 'npm test', 'npm test', True), ('yarn test stays whole', 'yarn test', 'yarn test', True), ('pnpm test stays whole', 'pnpm test', 'pnpm test', True), ('npm publish stays whole', 'npm publish', 'npm publish', True), ('bun run', 'bun run build', 'build', False)] + +# 10 rows; no position may start with the unwanted +SHAPE_REFUSES = [('quoted separator', 'git commit -m "wip; git push origin main next"', 'git push origin main next', True), ('quoted separator sq', "gh pr comment 3 --body 'do this; git push --force origin b'", 'git push --force origin b', True), ('echoed command', 'echo "done; git checkout main"', 'git checkout main', True), ('data heredoc', "cat > notes.md <<'EOF'\nThe rule:\ngit push origin main\nEOF", 'git push origin main', True), ('commit-message heredoc', "git commit -q -F - <<'EOF'\nhooks: explain push\n\ngit worktree add ../wt makes one.\nEOF", 'git worktree add ../wt makes one.', True), ('heredoc into .sh file', "cat > setup.sh <<'EOF'\ngit checkout main\nEOF", 'git checkout main', True), ('heredoc into tee .sh', "tee notes.sh <<'EOF'\ngit rebase main\nEOF", 'git rebase main', True), ('redirection is not &', 'pytest -q 2>&1 | tail -4', '1', True), ('mixed pair, data half', "cat > note.md <<'EOF'\ngit push origin main\nEOF\npython3 - <<'PY'\nimport os\nPY", 'git push origin main', True), ('two data heredocs', "cat > a.md <<'EOF'\ngit push a\nEOF\ncat > b.md <<'EOF'\ngit push b\nEOF", 'git push b', True)] + +# 6 rows; executing heredoc bodies stay visible +SHAPE_KEEPS = [('python3 heredoc', "python3 - <<'PY'\ngit_push = 1\nPY", 'git_push = 1'), ('bash heredoc', "bash <<'EOF'\ngit push origin main\nEOF", 'git push origin main'), ('uv run python3', "uv run python3 - <<'PY'\ngit push\nPY", 'git push'), ('ssh host bash', "ssh host bash <<'EOF'\ngit push origin main\nEOF", 'git push origin main'), ('piped to python3', "cat <<'EOF' | python3\nimport os\nEOF", 'import os'), ('mixed pair', "cat > note.md <<'EOF'\ngit push origin main\nEOF\npython3 - <<'PY'\nimport os\nPY", 'import os')] + +# 3 rows; statements vs commands +SHAPE_PIPELINES = [('runner and clip', 'npx playwright test --reporter=line 2>&1 | tail -4', 1, 2), ('two statements', 'git add -A && git commit -q --amend', 2, 2), ('redirection', "poe test 2>&1 | grep -E 'failed|passed' | tail -3", 1, 3)] + +# 22 rows; check() must refuse +UNSAFE_BLOCK = ['npx playwright test --reporter=line 2>&1 | tail -4', 'cd t && npx playwright test --project=firefox | tail -3', 'uv run pytest test/serena -q | head -5', "python3 - <<'PY'\ns=p.read_text()\ns=s.replace(a,b)\np.write_text(s)\nPY", 'python3 -c "s=open(f).read(); open(f,\'w\').write(s.replace(x,y))"', 'uv run poe test 2>&1 | tail -4', 'make test | tail -2', "for i in $(seq 1 45); do\n R=$(gh api repos/xormania/serena/pulls/13/reviews --jq 'length')\n C=$(gh api repos/xormania/serena/pulls/13/comments --jq 'length')\n X=$(gh api repos/xormania/serena/issues/comments/5296877293/reactions --jq '.[].content')\n sleep 20\ndone", "while true; do gh api repos/o/r/pulls/13/comments --jq 'length'; sleep 30; done", 'for i in $(seq 1 20); do gh api repos/o/r/issues/13/comments --jq length; sleep 15; done', 'grep -n "possible_paths\\|/opt/\\|/usr/local\\|which(\\|def _get" src/solidlsp/language_servers/nixd_ls.py | head -8', "grep -rn -E 'CODEX_HOME|CLAUDE_CONFIG_DIR|GROK_HOME' src/serena | head -5", 'grep -n "record\\|write_text\\|0o600\\|chmod\\|touch(" scripts/live_test_client_setup.py | tail -10', 'gh pr view 1873 --repo oraios/serena --json statusCheckRollup --jq \'.statusCheckRollup[] | select(.conclusion=="FAILURE") | "\\(.name) \\(.detailsUrl)"\'', 'gh pr view 1842 --repo oraios/serena --json statusCheckRollup --jq \'.statusCheckRollup | group_by(.conclusion) | map("\\(.[0].conclusion)=\\(length)")\'', 'gh pr list --repo oraios/serena --state open --limit 12 --json number,statusCheckRollup --jq \'.[] | "#\\(.number) \\(.statusCheckRollup | map(select(.conclusion=="FAILURE")) | length)"\'', 'git -C proj commit -q -m "He moved from building to landing. `git log --author` cannot show this."', 'git commit -m "Fix the flag\n\nThe `--author` option needs a value, so the guard now passes one."', 'gh pr create --title "Docs" --body "see the `poe lint` output for the failing step"', 'gh issue comment 1663 --body "the `ruff format --check` step is the one that aborts"', "python3 - <<'PY'\ns = p.read_text()\np.write_text(s.replace(old, new))\nPY", "uv run python3 - <<'PY'\ns = p.read_text()\np.write_text(s.replace(old, new))\nPY"] + +# 54 rows; check() must stay silent +UNSAFE_PASS = ["npx playwright test --reporter=line 2>&1 | grep -E 'failed|passed' | tail -3", "uv run pytest test/serena -q 2>&1 | grep -E 'failed|passed|error' | tail -2", 'npx playwright test --reporter=line 2>&1 | tail -40', "python3 - <<'PY'\ns=p.read_text()\nassert a in s\np.write_text(s.replace(a,b,1))\nPY", "python3 - <<'PY'\nout,n = re.subn(pat, rep, s)\nif n != 1: raise SystemExit('no match')\np.write_text(out)\nPY", 'git log --oneline -5 | head -3', 'ls -la | tail -4', 'grep -rn foo src | head -5', 'grep -n "^import\\|^from" scripts/check_dev_env.py | head -8', "git log --format= --name-status A..B | grep -c '^R'", "uv run pytest test/x -q 2>&1 | grep -E 'failed|passed|error' | tail -1", 'git status --short', 'echo "$x" | tail -2', 'python3 -c "print(\'a\'.replace(\'a\',\'b\'))"', 'gh api repos/xormania/serena/issues/comments/5296931625 --jq .body', "gh api repos/xormania/serena/issues/13/comments --jq 'length'", 'gh pr view 13 --repo xormania/serena --json headRefOid', 'ops/devlane/hooks/claude/pr-feedback.sh 13 xormania/serena', "while true; do\n gh api repos/o/r/issues/13/comments --jq length\n gh api repos/o/r/pulls/13/reviews --jq length\n gh api repos/o/r/pulls/13/comments --jq length\n gh api repos/o/r/issues/comments/99/reactions --jq '.[].content'\n sleep 30\ndone", "for r in $(gh api repos/o/r/releases --jq '.[].tag_name'); do echo $r; done", 'gh run watch 12345', 'ls test/serena | head -3', "uv run poe test 2>&1 | grep -E 'failed|passed' | tail -3", 'grep -n "pr-feedback" CLAUDE.md tools/selftest.sh | head -5', 'ls -la tools/selftest.sh | head -2', 'git add tools/selftest.sh && git status --short | head -3', 'wc -l tools/selftest.sh test/serena/test_x.py | tail -1', 'git commit -q -F - <<\'EOF\'\ntest-guard: flag tests that plant a fault without proving it landed\n\nopen(p,"w").write(open(p,"r").read()) truncates before the read runs.\nA .replace() with no assert leaves the file unchanged and open(f,\'w\').write(s)\nstill exits 0.\nEOF', "gh pr create --body-file - <<'EOF'\nGuarded every s.replace(old,new) and write_text(s) in the suite.\nEOF", "git commit -F - <<'EOF'\nFixed the python3 helper: s = s.replace(a,b) then p.write_text(s), unguarded.\nEOF", 'cat notes.md | head -5', 'uv run poe doc-build > /tmp/build.log 2>&1; echo done; grep -h session_id docs/*.md | head -3', "uv run poe doc-build >/dev/null 2>&1 && grep -ho 'defaults' docs/*.md | sort -u | head -6", 'uv run poe lint 2>&1 | tail -2', 'gh pr checks 1873 --repo oraios/serena', 'gh pr view 1873 --repo oraios/serena --json headRefOid,mergeable,reviewDecision', "gh pr view 1873 --repo oraios/serena --json statusCheckRollup --jq '.statusCheckRollup | length'", 'gh pr view 1873 --repo oraios/serena --json statusCheckRollup --jq \'.statusCheckRollup | group_by(.name) | map(max_by(.startedAt)) | map(select(.conclusion=="FAILURE")) | length\'', "grep -n 'statusCheckRollup' proj/tools/radar/radar.py", 'echo \'{"tool_input":{"command":"gh pr view 1 --json statusCheckRollup --jq .statusCheckRollup[]|select(.conclusion==FAILURE)"}}\' | python3 ops/devlane/hooks/claude/unsafe-command.py', 'printf \'%s\' "$(cat rollup-note.md)" # mentions statusCheckRollup and .conclusion', "grep -c 'select(.conclusion' proj/tools/radar/radar.py", "git commit -q -F - <<'EOF'\nMaintainers: the split is the wrong number\n\nA `git log` author query cannot show this, because squash merges are attributed\nto the PR author.\nEOF", 'git commit -q -F /tmp/msg-maintainers.txt', 'echo \'{"tool_input":{"command":"git commit -m \\"has `cmd` inside\\""}}\' | python3 ops/devlane/hooks/claude/unsafe-command.py', "git commit -m 'the `--author` flag needs a value'", 'git commit -m "an ordinary message with no expansion in it"', 'git commit -m "release $(cat VERSION)"', 'gh pr create --title "x" --body-file /tmp/body.md', 'git commit -q -F - <<\'EOF\'\nradar: collapse statusCheckRollup to the latest attempt per check\n\nselect(.conclusion=="FAILURE") over the raw rollup counts superseded attempts.\nEOF', "cat > setup.sh <<'EOF'\ns = p.read_text()\np.write_text(s.replace(old, new))\nEOF", "tee notes.sh <<'EOF'\ns = p.read_text()\np.write_text(s.replace(old, new))\nEOF", 'gh issue comment 12 --body \'note; grep -n "a\\|b\\|c" log.txt | head -5\'', 'git commit -q -m \'sweep; grep -rn -E "CODEX|CLAUDE|GROK" src | head -5 was wrong\''] # test-guard: allow + +# 15 rows; is_amend() +UNSAFE_AMEND = [('git add -A && git commit -q --amend --no-edit', True), ('git add CHANGELOG.md && git commit -q --amend --no-edit', True), ('git commit --amend -F /tmp/msg.txt', True), ('git -C proj commit -q --amend -F msg.txt', True), ('uv run poe lint && git commit -q --amend --no-edit && git push', True), ("git commit -q -m 'ordinary commit'", False), ('git commit -q -F /tmp/msg.txt', False), ("grep -n 'git commit --amend' notes.md", False), ('git log --oneline -3', False), ('sudo git commit -q --amend --no-edit', True), ('flags=1 git commit -q --amend', True), ('if git diff --cached --quiet; then git commit --amend --no-edit; fi', True), ('git -C sub -c core.hooksPath=/dev/null commit --amend', True), ("git commit -q -F - <<'EOF'\nnote: git commit --amend rewrites\nEOF", False), ('gh pr comment 3 --body "run git commit --amend after"', False)] + +# 22 rows; is_consequential True +PRECHECK_STOPS = ['git push origin main', 'git push --force-with-lease origin b', 'gh pr create --fill', 'gh pr merge 12 --squash', 'gh pr edit 7 --body x', 'gh api -X PATCH repos/o/r/pulls/1 -f base=main', 'npm publish', 'docker push ghcr.io/x/y', 'kubectl apply -f k8s/', 'git -C . push origin topic', 'git -c push.default=simple push origin topic', 'git --git-dir=/p/.git push origin main', "GIT_SSH_COMMAND='ssh -i k' git push origin topic", 'sudo git push origin topic', 'timeout 60 git push origin topic', 'if git diff --quiet; then git push origin topic; fi', 'for r in origin backup; do git push $r topic; done', 'gh -R o/r pr create --fill', 'gh --repo o/r pr merge 3 --squash', 'gh -R o/r api -X POST repos/o/r/issues/1/comments -f body=x', "bash <<'EOF'\ngit push origin main\nEOF", "ssh host bash <<'EOF'\ngit push origin main\nEOF"] + +# 21 rows; is_consequential False +PRECHECK_PASSES = ['git status --short', 'git fetch upstream', 'git commit -m x', 'git rebase main', 'git log --oneline', 'gh pr list', 'gh pr view 10 --json commits', 'gh pr diff 10 --name-only', 'gh api repos/o/r/branches/main', 'ls -la', 'uv run poe doc-build', 'pytest test/', 'git add -A', 'git checkout main', 'npm run build', 'grep -rn push src', 'cat deploy-notes.md', 'git diff --stat', 'git commit -m "wip; git push origin main is next"', "cat > notes.md <<'EOF'\nWhen ready:\ngit push origin main\nEOF", "git commit -q -F - <<'EOF'\nhooks: explain what push costs\n\ngit push origin main\nEOF"] + +# 29 rows; classify() fires +BOUNDARY_FIRE = ['git checkout main', 'git switch -c x', 'git rebase upstream/main', 'git commit --amend --no-edit', 'git push --force-with-lease origin b', 'git fetch upstream', 'git pull', 'git clean -fdx -- docs', 'git checkout -- file.py', 'git reset --hard HEAD~1', 'git stash', 'uv run poe doc-build', 'make test', 'docker build -t x .', 'npx playwright test --project=chromium', 'cd repo && git pull', 'pytest test/serena -q', 'cargo build --release', 'npm run build', 'npm test', 'yarn test', 'pnpm test', 'bun run build', 'git -C sub checkout main', 'git -C sub worktree add ../wt', 'for b in a b; do git checkout $b; done', 'sudo git commit --amend --no-edit', 'git \\\n rebase upstream/main', "bash <<'EOF'\ngit checkout main\nEOF"] + +# 37 rows; classify() silent +BOUNDARY_SILENT = ['ls -la', 'git status --short', 'git log --oneline -5', 'grep -rn foo src', 'cat README.md', "python3 -c 'print(1)'", 'npm ls', "sed -n '1,20p' f.py", "find . -name '*.md'", 'git diff --stat', 'wc -l f.txt', 'mkdir -p out', 'cp a b', 'echo hello', 'gh pr list', 'git show HEAD --stat', 'uv run poe lint', 'ruff check src', 'ls build/', 'grep -n build Makefile', 'cd builder && ls', 'cat docs/build-notes.md', 'git branch --show-current', 'git rev-parse HEAD', 'curl -s https://example.com', 'jq . f.json', 'head -20 log.txt', 'git add -A', "git commit -m 'x'", 'git diff --name-only main...feature', 'npm ls --depth 0', 'uv run poe lint', "cat > mem.md <<'EOF'\nNote:\ngit checkout main is a boundary\nEOF", "cat > notes.md <<'EOF'\nMake one with:\ngit worktree add ../wt\nEOF", 'echo "done; git checkout main"', "git commit -q -m 'describe git rebase without doing one'", "grep -rn 'git checkout' .dev/process"] + +# row counts of the tables above; tests assert these first +COUNTS = {'SHAPE_FINDS': 51, 'SHAPE_REFUSES': 10, 'SHAPE_KEEPS': 6, 'SHAPE_PIPELINES': 3, 'UNSAFE_BLOCK': 22, 'UNSAFE_PASS': 54, 'UNSAFE_AMEND': 15, 'PRECHECK_STOPS': 22, 'PRECHECK_PASSES': 21, 'BOUNDARY_FIRE': 29, 'BOUNDARY_SILENT': 37} + diff --git a/ops/devlane/hooks/tests/gh_stub.py b/ops/devlane/hooks/tests/gh_stub.py new file mode 100644 index 0000000..cbd3fb2 --- /dev/null +++ b/ops/devlane/hooks/tests/gh_stub.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""PATH-stub `gh` for pr-feedback tests. + +Serves fixture JSON through real jq (the same `--jq` the real client uses). +No network. Failure, per-endpoint failure, and GraphQL `first:` paging are +driven by environment so the test can prove the plant before the tool runs. + +Env: + GH_FIXTURES directory of fixture files (required) + GH_MODE_FILE file whose contents are `ok` or `fail` (default ok) + GH_FAIL `1` forces every call to fail + GH_FAIL_RC exit status when failing (default 1) + GH_FAIL_STDERR stderr body when failing (default auth-failed text) + GH_FAIL_ENDPOINT substring of the API path that should fail + GH_THREAD_COUNT if set, synthesize N unresolved reviewThreads +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +from pathlib import Path + + +def _fail(fix: Path) -> None: + msg = os.environ.get("GH_FAIL_STDERR", "gh: authentication failed") + if msg: + sys.stderr.write(msg + "\n") + log(fix, f"FAIL rc={os.environ.get('GH_FAIL_RC', '1')}") + sys.exit(int(os.environ.get("GH_FAIL_RC", "1"))) + + +def log(fix: Path, line: str) -> None: + path = fix / "calls.log" + with path.open("a", encoding="utf-8") as fh: + fh.write(line + "\n") + + +def bump(fix: Path) -> int: + path = fix / "callcount" + n = 0 + if path.is_file(): + raw = path.read_text(encoding="utf-8").strip() + if raw.isdigit(): + n = int(raw) + n += 1 + path.write_text(str(n) + "\n", encoding="utf-8") + return n + + +def mode_is_fail(fix: Path) -> bool: + if os.environ.get("GH_FAIL") == "1": + return True + mode_file = os.environ.get("GH_MODE_FILE") + if mode_file: + p = Path(mode_file) + if p.is_file() and p.read_text(encoding="utf-8").strip() == "fail": + return True + return False + + +def parse_api_argv(argv: list[str]) -> tuple[str, str | None, dict, str, bool]: + """Return endpoint, jq expr, -f variables, joined tail, --paginate.""" + jqexpr = None + variables: dict[str, str] = {} + paginate = False + endpoint = "" + i = 0 + while i < len(argv): + a = argv[i] + if a in ("--jq", "-q") and i + 1 < len(argv): + jqexpr = argv[i + 1] + i += 2 + continue + if a.startswith("--jq="): + jqexpr = a.split("=", 1)[1] + i += 1 + continue + if a in ("-f", "-F") and i + 1 < len(argv): + kv = argv[i + 1] + if "=" in kv: + k, v = kv.split("=", 1) + variables[k] = v + i += 2 + continue + if a == "--paginate": + paginate = True + i += 1 + continue + if not a.startswith("-") and not endpoint: + endpoint = a + i += 1 + continue + i += 1 + return endpoint, jqexpr, variables, " ".join(argv), paginate + + +def thread_node(i: int) -> dict: + return { + "id": f"TH{i}", + "isResolved": False, + "isOutdated": False, + "path": f"thread-{i}-unique.py", + "line": i, + "comments": { + "nodes": [ + { + "id": f"TC{i}", + "body": f"THREAD-{i}-UNIQUE", + "author": {"login": "bot"}, + } + ] + }, + } + + +def threads_page(total: int, first: int | None, after: str, paginate: bool) -> dict: + if total <= 0: + nodes: list[dict] = [] + return { + "data": { + "repository": { + "pullRequest": { + "reviewThreads": { + "pageInfo": { + "hasNextPage": False, + "endCursor": None, + }, + "nodes": nodes, + } + } + } + } + } + start = 1 + if after.startswith("c") and after[1:].isdigit(): + start = int(after[1:]) + 1 + # Live GraphQL demands a bound; first:50 is the finding. Omitting + # first must not dump the whole set and look like paging. + if paginate: + end = total + else: + bound = 50 if first is None else first + end = min(total, start + bound - 1) + if start < 1: + start = 1 + if start > total: + nodes = [] + end = start - 1 + else: + nodes = [thread_node(i) for i in range(start, end + 1)] + has_next = bool(nodes) and end < total + cursor = f"c{end}" if nodes else None + return { + "data": { + "repository": { + "pullRequest": { + "reviewThreads": { + "pageInfo": { + "hasNextPage": has_next, + "endCursor": cursor, + }, + "nodes": nodes, + } + } + } + } + } + + +def route(fix: Path, endpoint: str) -> Path: + # Origin globbed `*/issues/*/comments*` so query strings still match. + ep = endpoint.split("?", 1)[0] + if ep == "graphql": + return fix / "threads.json" + if re.search(r"issues/comments/[^/]+/reactions", ep): + ident = ep.split("issues/comments/", 1)[1].split("/", 1)[0] + return fix / f"react-{ident}.json" + if re.search(r"/issues/[^/]+/comments", ep): + return fix / "conv.json" + if re.search(r"/pulls/[^/]+/reviews", ep): + return fix / "reviews.json" + if re.search(r"/pulls/[^/]+/comments", ep): + return fix / "inline.json" + return fix / "empty.json" + + +def main() -> None: + fix = Path(os.environ["GH_FIXTURES"]) + n = bump(fix) + argv = sys.argv[1:] + log(fix, f"call {n} argv={argv!r}") + if not argv or argv[0] != "api": + sys.exit(0) + endpoint, jqexpr, variables, joined, paginate = parse_api_argv(argv[1:]) + fail_ep = os.environ.get("GH_FAIL_ENDPOINT", "") + if mode_is_fail(fix) or (fail_ep and fail_ep in endpoint): + _fail(fix) + + count_raw = os.environ.get("GH_THREAD_COUNT", "").strip() + if endpoint == "graphql" and count_raw.isdigit(): + total = int(count_raw) + query = variables.get("query", joined) + first = None + m = re.search(r"reviewThreads\s*\(\s*first:\s*(\d+)", query) + if m: + first = int(m.group(1)) + after = ( + variables.get("threadCursor") + or variables.get("after") + or "" + ) + if after in ("null", "None", "none"): + after = "" + inline = re.search(r'after:\s*"([^"]+)"', query) + if not after and inline: + after = inline.group(1) + payload = threads_page(total, first, after, paginate) + text = json.dumps(payload) + else: + path = route(fix, endpoint) + if not path.is_file(): + path = fix / "empty.json" + text = "[]" if not path.is_file() else path.read_text(encoding="utf-8") + + if jqexpr is None: + sys.stdout.write(text) + if text and not text.endswith("\n"): + sys.stdout.write("\n") + sys.exit(0) + proc = subprocess.run( + ["jq", "-r", jqexpr], + input=text, + capture_output=True, + text=True, + check=False, + ) + sys.stdout.write(proc.stdout) + sys.stderr.write(proc.stderr) + sys.exit(proc.returncode) + + +if __name__ == "__main__": + main() diff --git a/ops/devlane/hooks/tests/support.py b/ops/devlane/hooks/tests/support.py new file mode 100644 index 0000000..a51140b --- /dev/null +++ b/ops/devlane/hooks/tests/support.py @@ -0,0 +1,361 @@ +"""Load the hyphenated hook scripts and build throwaway git / PATH fixtures. + +The real repository is never the fixture: every git directory these helpers +create lives under the caller's tempdir, and every stub binary is written +there too. A plant that did not land raises ``PlantFailed`` — that is +INVALID, not a verdict on the hook. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import shutil +import stat +import subprocess +import sys +from pathlib import Path + +TESTS_DIR = Path(__file__).resolve().parent +HOOKS_DIR = TESTS_DIR.parent +CLAUDE_DIR = HOOKS_DIR / "claude" +WORKTREE_ROOT = TESTS_DIR.parents[3] + +FIXED_DATE = "2026-01-02T03:04:05Z" + + +class PlantFailed(AssertionError): + """The fixture was not what the case claimed to set up.""" + + +def load_claude(filename: str, modname: str): + """Load a script from ``claude/`` as a module without running ``__main__``.""" + path = CLAUDE_DIR / filename + if not path.is_file(): + raise PlantFailed( + f"INVALID: hook script does not exist yet: {path}" + ) + if modname in sys.modules: + return sys.modules[modname] + # The hyphenated scripts import ``command_shape`` from this directory. + claude = str(CLAUDE_DIR) + if claude not in sys.path: + sys.path.insert(0, claude) + spec = importlib.util.spec_from_file_location(modname, path) + if spec is None or spec.loader is None: + raise PlantFailed(f"INVALID: cannot load {path}") + mod = importlib.util.module_from_spec(spec) + sys.modules[modname] = mod + spec.loader.exec_module(mod) + return mod + + +def isolated_env(home: Path, extra_path=None, extra=None): + """Env that cannot see the real repo's git config or hook escape hatches.""" + path = os.environ.get("PATH", "/usr/bin:/bin") + if extra_path is not None: + path = str(extra_path) + os.pathsep + path + env = { + "PATH": path, + "HOME": str(home), + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_AUTHOR_DATE": FIXED_DATE, + "GIT_COMMITTER_DATE": FIXED_DATE, + "LC_ALL": "C", + "LANG": "C", + } + # Never inherit the offline escape: a D2 case with this set is INVALID. + env.pop("CLAUDE_PRECHECK_NO_FETCH", None) + if extra: + env.update(extra) + return env + + +def run_cmd(argv, cwd, env, *, expect=None, stdin=None): + proc = subprocess.run( + argv, + cwd=str(cwd), + env=env, + input=stdin, + capture_output=True, + text=True, + check=False, + ) + if expect is not None and proc.returncode != expect: + raise PlantFailed( + f"INVALID: {argv!r} in {cwd} exited {proc.returncode} " + f"(wanted {expect})\nstdout: {proc.stdout}\nstderr: {proc.stderr}" + ) + return proc + + +def git(cwd, env, *args, expect=0): + return run_cmd(["git", *args], cwd, env, expect=expect) + + +def configure_identity(repo: Path, env): + git(repo, env, "config", "user.name", "Test User") + git(repo, env, "config", "user.email", "test@example.invalid") + + +def plant_text(path: Path, content: str, *, recognisable: str | None = None) -> str: + """Write ``content`` and prove it landed intact.""" + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + landed = path.read_text(encoding="utf-8") + if landed != content: + raise PlantFailed( + f"INVALID: plant did not reach disk for {path} " + f"(wanted {len(content)} bytes, got {len(landed)})" + ) + if content and not landed: + raise PlantFailed(f"INVALID: plant emptied {path}") + if recognisable is not None and recognisable not in landed: + raise PlantFailed( + f"INVALID: plant left {path} unrecognisable as the fixture" + ) + return landed + + +def plant_executable(path: Path, content: str) -> Path: + """Write a stub binary, prove it is executable, prove it is the file we wrote.""" + plant_text(path, content, recognisable=content.splitlines()[0][:20]) + path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + mode = path.stat().st_mode + if not (mode & stat.S_IXUSR): + raise PlantFailed(f"INVALID: {path} is not executable") + if not path.is_file(): + raise PlantFailed(f"INVALID: {path} is not a file") + return path + + +def bash_payload(command: str) -> str: + return json.dumps({"tool_name": "Bash", "tool_input": {"command": command}}) + + +def write_payload(file_path=None, relative_path=None, cwd=None) -> str: + tool_input = {} + if file_path is not None: + tool_input["file_path"] = file_path + if relative_path is not None: + tool_input["relative_path"] = relative_path + payload = {"tool_name": "Edit", "tool_input": tool_input} + if cwd is not None: + payload["cwd"] = cwd + return json.dumps(payload) + + +def run_script(script: Path, payload: str, cwd: Path, env) -> subprocess.CompletedProcess: + if not script.is_file(): + raise PlantFailed(f"INVALID: script missing: {script}") + argv = (["python3", str(script)] if script.suffix == ".py" + else ["bash", str(script)]) + return run_cmd(argv, cwd, env, stdin=payload, expect=None) + + +def has_additional_context(stdout: str) -> bool: + if not stdout or not stdout.strip(): + return False + try: + data = json.loads(stdout) + except json.JSONDecodeError: + return "additionalContext" in stdout + hook = data.get("hookSpecificOutput") or {} + return bool(hook.get("additionalContext")) + + +def permission_decision(stdout: str): + if not stdout or not stdout.strip(): + return None, "" + try: + data = json.loads(stdout) + except json.JSONDecodeError: + return None, stdout + hook = data.get("hookSpecificOutput") or {} + return hook.get("permissionDecision"), hook.get("permissionDecisionReason") or "" + + +class StaleClone: + """Bare remote + clone whose FETCH_HEAD is gone and whose origin is behind.""" + + def __init__(self, tmp: Path, env): + self.tmp = tmp + self.env = env + self.seed = tmp / "seed" + self.bare = tmp / "remote.git" + self.clone = tmp / "clone" + self.old_sha = None + self.new_sha = None + + def build(self): + self.seed.mkdir() + git(self.seed, self.env, "-c", "init.defaultBranch=main", "init", "-q") + configure_identity(self.seed, self.env) + plant_text(self.seed / "file.txt", "one\n", recognisable="one") + git(self.seed, self.env, "add", "file.txt") + git(self.seed, self.env, "commit", "-q", "-m", "A", "--date", FIXED_DATE) + self.old_sha = git( + self.seed, self.env, "rev-parse", "HEAD" + ).stdout.strip() + if len(self.old_sha) < 7: + raise PlantFailed("INVALID: seed commit SHA missing") + + git(self.tmp, self.env, "clone", "--bare", "-q", + str(self.seed), str(self.bare)) + git(self.tmp, self.env, "clone", "-q", str(self.bare), str(self.clone)) + configure_identity(self.clone, self.env) + + fetch_head = self.clone / ".git" / "FETCH_HEAD" + # git 2.43's clone from a local path does not write FETCH_HEAD. + # That is already D2's missing-FETCH_HEAD state. Older gits do + # write one; delete it so both land on the same fixture. + if fetch_head.is_file(): + before = fetch_head.read_bytes() + if not before: + raise PlantFailed("INVALID: FETCH_HEAD was already empty") + fetch_head.unlink() + if fetch_head.exists(): + raise PlantFailed("INVALID: FETCH_HEAD is still present") + + origin_main = git( + self.clone, self.env, "rev-parse", "origin/main" + ).stdout.strip() + if origin_main != self.old_sha: + raise PlantFailed( + f"INVALID: clone origin/main is {origin_main}, not seed {self.old_sha}" + ) + + plant_text(self.seed / "file.txt", "two\n", recognisable="two") + landed = (self.seed / "file.txt").read_text(encoding="utf-8") + if landed == "one\n": + raise PlantFailed("INVALID: seed file was not advanced") + git(self.seed, self.env, "add", "file.txt") + git(self.seed, self.env, "commit", "-q", "-m", "B", "--date", FIXED_DATE) + self.new_sha = git( + self.seed, self.env, "rev-parse", "HEAD" + ).stdout.strip() + if self.new_sha == self.old_sha: + raise PlantFailed("INVALID: seed HEAD did not move") + git(self.seed, self.env, "push", "-q", str(self.bare), "main") + bare_head = git( + self.bare, self.env, "rev-parse", "HEAD" + ).stdout.strip() + if bare_head != self.new_sha: + raise PlantFailed( + f"INVALID: bare remote still at {bare_head}, not {self.new_sha}" + ) + + still = git( + self.clone, self.env, "rev-parse", "origin/main" + ).stdout.strip() + if still != self.old_sha: + raise PlantFailed( + "INVALID: clone origin/main moved before the hook ran" + ) + if (self.clone / ".git" / "FETCH_HEAD").exists(): + raise PlantFailed( + "INVALID: FETCH_HEAD reappeared before the hook ran" + ) + return self + + def restale(self): + """Put origin/main back behind the remote and drop FETCH_HEAD again.""" + git(self.clone, self.env, "update-ref", "refs/remotes/origin/main", + self.old_sha) + fetch_head = self.clone / ".git" / "FETCH_HEAD" + if fetch_head.exists(): + fetch_head.unlink() + now = git( + self.clone, self.env, "rev-parse", "origin/main" + ).stdout.strip() + if now != self.old_sha: + raise PlantFailed("INVALID: restale did not rewind origin/main") + if fetch_head.exists(): + raise PlantFailed("INVALID: restale left FETCH_HEAD in place") + + def refresh(self): + """Fetch until origin/main matches the remote. Prove it landed.""" + git(self.clone, self.env, "fetch", "--all", "-q") + now = git( + self.clone, self.env, "rev-parse", "origin/main" + ).stdout.strip() + if now != self.new_sha: + raise PlantFailed( + f"INVALID: refresh left origin/main at {now}, " + f"not remote {self.new_sha}" + ) + return self + + +def require_jq(): + if shutil.which("jq") is None: + raise PlantFailed( + "INVALID: jq is not on PATH; ruff-after-edit.sh cannot emit" + ) + + +def rev_parse_path(cwd, env, flag: str, sandbox: Path) -> Path: + """Resolve ``git rev-parse `` to an absolute path inside *sandbox*. + + Used to tell a linked worktree's private git-dir from the clone's + common dir without touching the real repository. + """ + out = git(cwd, env, "rev-parse", flag).stdout.strip() + if not out: + raise PlantFailed(f"INVALID: git rev-parse {flag} was empty in {cwd}") + path = Path(os.path.abspath(os.path.join(str(cwd), out))) + sand = os.path.realpath(str(sandbox)) + os.sep + real = os.path.realpath(str(path)) + if not real.startswith(sand): + raise PlantFailed( + f"INVALID: git rev-parse {flag} resolved to {path}, " + f"outside sandbox {sandbox}" + ) + if not path.exists(): + raise PlantFailed(f"INVALID: git rev-parse {flag} path missing: {path}") + return path + + +def prove_linked_worktree(main: Path, linked: Path, env, sandbox: Path): + """Prove this pair is a linked worktree of *main*, not a second clone. + + The one-stream finding is invisible when git-dir equals common-dir + (the primary checkout). A fixture that failed to diverge is INVALID + — later assertions would be answering a question about nothing. + Also proves ``.git`` is a directory in the primary and a gitfile in + the linked tree: that is the input a script which uses the common + dir only when cwd looks like a main worktree gets wrong. + """ + main_git = rev_parse_path(main, env, "--git-dir", sandbox) + main_common = rev_parse_path(main, env, "--git-common-dir", sandbox) + link_git = rev_parse_path(linked, env, "--git-dir", sandbox) + link_common = rev_parse_path(linked, env, "--git-common-dir", sandbox) + + if os.path.realpath(str(main_common)) != os.path.realpath(str(link_common)): + raise PlantFailed( + "INVALID: linked worktree does not share the clone's common dir" + ) + if os.path.realpath(str(main_git)) != os.path.realpath(str(main_common)): + raise PlantFailed( + "INVALID: primary git-dir is not the common dir; " + "the fixture is not a normal checkout" + ) + if os.path.realpath(str(link_git)) == os.path.realpath(str(link_common)): + raise PlantFailed( + "INVALID: linked worktree git-dir equals common-dir; " + "this fixture cannot expose a split stream" + ) + if not (main / ".git").is_dir(): + raise PlantFailed("INVALID: primary .git is not a directory") + gitfile = linked / ".git" + if gitfile.is_dir(): + raise PlantFailed( + "INVALID: linked worktree .git is a directory, not a gitfile" + ) + if not gitfile.is_file(): + raise PlantFailed("INVALID: linked worktree has no .git gitfile") + return link_git, link_common diff --git a/ops/devlane/hooks/tests/test_bdd_traceability.py b/ops/devlane/hooks/tests/test_bdd_traceability.py new file mode 100644 index 0000000..07ae479 --- /dev/null +++ b/ops/devlane/hooks/tests/test_bdd_traceability.py @@ -0,0 +1,96 @@ +"""BDD traceability cases not already proved by the hook contract suite.""" + +import json +import unittest + +import test_hooks as hook_contract + + +class MultiWorktreeScenarios(unittest.TestCase): + def setUp(self): + self.fixture = hook_contract.HookContractTest( + "test_h3_checkout_appends_one_branch_entry" + ) + self.fixture.setUp() + self.addCleanup(self.fixture.doCleanups) + + def test_a_file_checkout_is_not_a_crossing(self): + """Scenario: a file checkout is not a crossing""" + h = self.fixture + h.require_sources() + repo = h.make_repo("file-checkout") + h.install(repo) + + stream_before = h.stream_lines(repo) + tracked = repo / "file.txt" + tracked.write_text("dirty replacement\n") + self.assertEqual(tracked.read_text(), "dirty replacement\n") + self.assertIn("file.txt", h.git(repo, "status", "--short").stdout) + + h.git(repo, "checkout", "--", "file.txt") + + self.assertEqual(tracked.read_text(), "one\n") + self.assertEqual( + h.stream_lines(repo), + stream_before, + "a file checkout was recorded as a branch crossing", + ) + + def test_commits_from_both_worktrees_name_their_worktrees(self): + """Scenario: a commit is recorded from whichever worktree made it""" + h = self.fixture + h.require_sources() + repo = h.make_repo("primary-clone") + h.git(repo, "branch", "linked") + h.install(repo) + + linked = h.tmp / "second-tree" + h.git(repo, "worktree", "add", str(linked), "linked") + before = h.stream_lines(repo) + + primary_sha = h.commit_change(repo, "primary\n", "primary commit") + linked_sha = h.commit_change(linked, "linked\n", "linked commit") + + after = h.stream_lines(repo) + self.assertEqual(after[: len(before)], before) + added = after[len(before):] + self.assertEqual( + len(added), + 2, + f"two commits should append two records, observed {added!r}", + ) + entries = [h.parse_entry(line) for line in added] + self.assertEqual([entry["kind"] for entry in entries], ["head", "head"]) + self.assertEqual( + [entry["worktree"] for entry in entries], + ["primary-clone", "second-tree"], + ) + self.assertIn(primary_sha, entries[0]["what"]) + self.assertIn(linked_sha, entries[1]["what"]) + + def test_a_unicode_line_separator_in_a_branch_stays_one_json_object(self): + """Scenario: control characters cannot corrupt the stream""" + h = self.fixture + h.require_sources() + repo = h.make_repo("branch-controls") + branch = "line\u2028separator" + self.assertIn("\u2028", branch, "the newline-adjacent plant did not land") + h.git(repo, "branch", branch) + h.install(repo) + + before = h.stream_lines(repo) + h.git(repo, "checkout", "-q", branch) + raw = h.stream_path(repo).read_text() + records = [line for line in raw.split("\n") if line] + self.assertEqual( + len(records), + len(before) + 1, + f"one checkout must add one LF-delimited JSON object: {raw!r}", + ) + entry = json.loads(records[-1]) + self.assertIsInstance(entry, dict) + self.assertIn(branch, entry["what"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/hooks/tests/test_boundary_match.py b/ops/devlane/hooks/tests/test_boundary_match.py new file mode 100644 index 0000000..98f1aa6 --- /dev/null +++ b/ops/devlane/hooks/tests/test_boundary_match.py @@ -0,0 +1,64 @@ +"""Moved corpus for boundary-match.py.""" + +import unittest + +import corpus +import support + + +class BoundaryMatchCorpus(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.mod = support.load_claude("boundary-match.py", "boundary_match") + + def test_the_moved_tables_are_the_tables_that_were_there(self): + self.assertEqual(len(corpus.BOUNDARY_FIRE), + corpus.COUNTS["BOUNDARY_FIRE"]) + self.assertEqual(len(corpus.BOUNDARY_SILENT), + corpus.COUNTS["BOUNDARY_SILENT"]) + self.assertEqual(corpus.COUNTS["BOUNDARY_FIRE"], 29) + self.assertEqual(corpus.COUNTS["BOUNDARY_SILENT"], 37) + + def test_fires_on_each_fire_row(self): + self.assertGreater(len(corpus.BOUNDARY_FIRE), 0) + for cmd in corpus.BOUNDARY_FIRE: + with self.subTest(cmd=cmd): + hit = self.mod.classify(cmd) + self.assertTrue( + hit, + f"should fire, stayed silent: {cmd!r}", + ) + + def test_stays_silent_on_each_silent_row(self): + self.assertGreater(len(corpus.BOUNDARY_SILENT), 0) + for cmd in corpus.BOUNDARY_SILENT: + with self.subTest(cmd=cmd): + hit = self.mod.classify(cmd) + self.assertFalse( + hit, + f"false positive {hit!r} on {cmd!r}", + ) + + def test_plan_d3_git_dash_c_checkout_is_a_tree_crossing(self): + rows = [ + "git -C sub checkout main", + "git -C sub worktree add ../wt", + ] + for cmd in rows: + with self.subTest(cmd=cmd): + hit = self.mod.classify(cmd) + self.assertTrue(hit, f"D3 form was silent: {cmd!r}") + self.assertEqual(hit[0], "tree", f"{cmd!r} fired as {hit!r}") + + def test_plan_d3_data_heredoc_is_not_a_tree_crossing(self): + rows = [ + "cat > mem.md <<'EOF'\nNote:\ngit checkout main is a boundary\nEOF", + "cat > notes.md <<'EOF'\nMake one with:\ngit worktree add ../wt\nEOF", + 'echo "done; git checkout main"', + ] + for cmd in rows: + with self.subTest(cmd=cmd[:40]): + self.assertFalse( + self.mod.classify(cmd), + f"prose was a tree crossing: {cmd!r}", + ) diff --git a/ops/devlane/hooks/tests/test_command_shape.py b/ops/devlane/hooks/tests/test_command_shape.py new file mode 100644 index 0000000..173bda2 --- /dev/null +++ b/ops/devlane/hooks/tests/test_command_shape.py @@ -0,0 +1,154 @@ +"""Moved corpus for command_shape.py, plus the plant-against-legacy guard. + +The legacy split is a verbatim copy of what the three callers shared before +the module existed. A planted row the old split already handles is INVALID, +not a pass. +""" + +import re +import unittest + +import corpus +import support + +# Verbatim from command_shape.py as it shipped: the split the three hooks +# used before this module existed. Kept in the test so a plant the old +# split already handles fails as PLANT NOT LANDED rather than passing. +_LEGACY_SEPARATORS = re.compile(r"[;&|]{1,2}|\$\(|`|\n") +_LEGACY_RUNNERS = re.compile( + r"^(?:sudo|time|env|nohup|xargs|uv|uvx|npx|poetry|pipenv|poe|pnpm|yarn|npm|bun)\s+" + r"(?:run\s+)?" +) + + +def _legacy_commands(text): + """context-precheck.py:41-47 and boundary-match.py:19-24, as they were.""" + out = [] + for raw in _LEGACY_SEPARATORS.split(text or ""): + seg = raw.strip() + while True: + s = _LEGACY_RUNNERS.sub("", seg, count=1) + if s == seg: + break + seg = s + if seg: + out.append(seg) + return out + + +def _legacy_statements(text): + """unsafe-command.py:55,250,272, as it was.""" + return [s.strip() for s in re.split(r";|&&|\|\||\n", text or "") if s.strip()] + + +class CommandShapeCorpus(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.mod = support.load_claude("command_shape.py", "command_shape") + + def test_the_moved_tables_are_the_tables_that_were_there(self): + self.assertEqual(len(corpus.SHAPE_FINDS), corpus.COUNTS["SHAPE_FINDS"]) + self.assertEqual(len(corpus.SHAPE_REFUSES), corpus.COUNTS["SHAPE_REFUSES"]) + self.assertEqual(len(corpus.SHAPE_KEEPS), corpus.COUNTS["SHAPE_KEEPS"]) + self.assertEqual(len(corpus.SHAPE_PIPELINES), corpus.COUNTS["SHAPE_PIPELINES"]) + self.assertEqual(corpus.COUNTS["SHAPE_FINDS"], 51) + self.assertEqual(corpus.COUNTS["SHAPE_REFUSES"], 10) + self.assertEqual(corpus.COUNTS["SHAPE_KEEPS"], 6) + self.assertEqual(corpus.COUNTS["SHAPE_PIPELINES"], 3) + + def test_finds_each_want_among_command_positions(self): + self.assertGreater(len(corpus.SHAPE_FINDS), 0) + for label, text, want, plant in corpus.SHAPE_FINDS: + with self.subTest(label=label): + if plant: + self.assertNotIn( + want, _legacy_commands(text), + f"INVALID: PLANT NOT LANDED {label}: " + f"the legacy split already finds {want!r}", + ) + got = self.mod.commands(text) + self.assertIn( + want, got, + f"finds {label}: {want!r} not in {got!r}", + ) + + def test_refuses_to_offer_quoted_or_heredoc_text_as_a_command(self): + self.assertGreater(len(corpus.SHAPE_REFUSES), 0) + for label, text, unwanted, plant in corpus.SHAPE_REFUSES: + with self.subTest(label=label): + if plant: + legacy = [c for c in _legacy_commands(text) + if c.startswith(unwanted)] + self.assertTrue( + legacy, + f"INVALID: PLANT NOT LANDED {label}: " + f"the legacy split already refuses {unwanted!r}", + ) + got = [c for c in self.mod.commands(text) + if c.startswith(unwanted)] + self.assertEqual( + got, [], + f"refuses {label}: a position still starts " + f"{unwanted!r} — {got!r}", + ) + + def test_keeps_heredoc_bodies_that_execute(self): + self.assertGreater(len(corpus.SHAPE_KEEPS), 0) + for label, text, body in corpus.SHAPE_KEEPS: + with self.subTest(label=label): + self.assertIn( + body, text, + f"INVALID: PLANT NOT LANDED {label}: " + f"body absent from the input", + ) + kept = self.mod.strip_data_heredocs(text) + self.assertIn( + body, kept, + f"keeps {label}: {body!r} was blanked", + ) + + def test_statements_keep_pipelines_whole(self): + self.assertGreater(len(corpus.SHAPE_PIPELINES), 0) + for label, text, want_stmts, want_cmds in corpus.SHAPE_PIPELINES: + with self.subTest(label=label): + s = self.mod.statements(text) + c = self.mod.commands(text) + self.assertEqual( + len(s), want_stmts, + f"statements {label}: {len(s)} != {want_stmts} {s!r}", + ) + self.assertGreaterEqual( + len(c), want_cmds, + f"commands {label}: {len(c)} < {want_cmds} {c!r}", + ) + + def test_statements_match_the_legacy_split_where_quoting_is_not_involved(self): + texts = ( + "git add -A && git commit -q --amend --no-edit", + "uv run pytest -q 2>&1 | grep -E 'failed|passed' | tail -2", + "for i in 1 2; do gh api repos/o/r/pulls/1/reviews; sleep 20; done", + ) + for text in texts: + with self.subTest(text=text): + self.assertEqual( + self.mod.statements(text), + _legacy_statements(text), + f"statements drift on {text!r}", + ) + + def test_strip_data_heredocs_is_idempotent(self): + hd = "cat > n.md <<'EOF'\ngit push\nEOF" + once = self.mod.strip_data_heredocs(hd) + twice = self.mod.strip_data_heredocs(once) + self.assertEqual(once, twice) + + def test_junk_inputs_return_lists(self): + for junk in ("", " ", "'unbalanced quote git push origin b", "$(", "`", + "<<", "|||", "cat < 72 collected). +import test_hooks + +SPLIT = "subject\n\nbody.\n\nSource: original\n\nCo-Authored-By: A \n" +JOINED = "subject\n\nbody.\n\nSource: original\nCo-Authored-By: A \n" + + +class TheCommitMsgGate(test_hooks.HookContractTest): + def prepared(self, with_checker): + self.require_sources() + repo = self.make_repo("gate") + self.install(repo) + checker = repo / "ops" / "devlane" / "workflow" / "checks" / "commit_trailers.py" + self.assertTrue(checker.is_file(), + "INVALID: staging did not place the checker") + if not with_checker: + checker.unlink() + self.assertFalse(checker.is_file(), + "INVALID: the checker was not removed") + return repo + + def attempt(self, repo, message, content): + (repo / "f.txt").write_text(content, encoding="utf-8") + self.assertEqual((repo / "f.txt").read_text(encoding="utf-8"), content, + "INVALID: the change did not land") + msg = repo / "m.txt" + msg.write_text(message, encoding="utf-8") + self.assertEqual(msg.read_text(encoding="utf-8"), message, + "INVALID: the message did not land") + self.git(repo, "add", "f.txt") + proc = subprocess.run( + ["git", "commit", "-F", str(msg)], cwd=repo, + capture_output=True, text=True, check=False, env=self.env_for()) + msg.unlink() + return proc + + def count(self, repo): + out = subprocess.run(["git", "rev-list", "--count", "HEAD"], cwd=repo, + capture_output=True, text=True, check=False) + return int(out.stdout.strip() or 0) + + def test_a_split_block_is_refused(self): + repo = self.prepared(with_checker=True) + before = self.count(repo) + proc = self.attempt(repo, SPLIT, "one\n") + self.assertNotEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertEqual(self.count(repo), before, "the commit landed anyway") + + def test_a_joined_block_commits(self): + repo = self.prepared(with_checker=True) + before = self.count(repo) + proc = self.attempt(repo, JOINED, "two\n") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertEqual(self.count(repo), before + 1) + + def test_the_two_messages_differ_only_by_the_blank_line(self): + """Without this the pair above proves nothing.""" + self.assertEqual(SPLIT.replace("\n\nCo-Authored-By", "\nCo-Authored-By"), + JOINED) + + def test_a_missing_checker_warns_and_lets_the_commit_through(self): + """An environment problem must not stop work; CI is the gate.""" + repo = self.prepared(with_checker=False) + before = self.count(repo) + proc = self.attempt(repo, SPLIT, "three\n") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertEqual(self.count(repo), before + 1) + self.assertIn("cannot check trailers", proc.stderr) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/ops/devlane/hooks/tests/test_conductor_enforce.py b/ops/devlane/hooks/tests/test_conductor_enforce.py new file mode 100644 index 0000000..62c209e --- /dev/null +++ b/ops/devlane/hooks/tests/test_conductor_enforce.py @@ -0,0 +1,101 @@ +"""Policy corpus and process boundary for conductor-enforce.py.""" + +import json +import os +import unittest + +import support + + +class ConductorPolicy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.mod = support.load_claude("conductor-enforce.py", "conductor_enforce") + + @staticmethod + def bash(command): + return json.loads(support.bash_payload(command)) + + def test_agent_is_denied(self): + self.assertEqual( + self.mod.deny_reason({"tool_name": "Agent"}), + self.mod.AGENT_REASON, + ) + + def test_each_denied_git_subcommand_is_denied(self): + for subcommand in self.mod.POLICY["deny_git"]: + with self.subTest(subcommand=subcommand): + self.assertEqual( + self.mod.deny_reason(self.bash(f"git {subcommand}")), + self.mod.INVESTIGATION_REASON, + ) + + def test_each_allowed_git_subcommand_is_allowed(self): + for subcommand in self.mod.POLICY["allow_git"]: + with self.subTest(subcommand=subcommand): + self.assertIsNone( + self.mod.deny_reason(self.bash(f"git {subcommand}")) + ) + + def test_structured_pr_view_is_denied_but_plain_view_is_allowed(self): + self.assertEqual( + self.mod.deny_reason(self.bash("gh pr view --json state")), + self.mod.INVESTIGATION_REASON, + ) + self.assertIsNone( + self.mod.deny_reason(self.bash("gh pr view 64")) + ) + + def test_gh_api_is_denied(self): + self.assertEqual( + self.mod.deny_reason(self.bash("gh api repos/o/r")), + self.mod.INVESTIGATION_REASON, + ) + + def test_each_lever_anywhere_overrides_a_denied_git_subcommand(self): + for lever in self.mod.POLICY["allow_anywhere"]: + with self.subTest(lever=lever): + self.assertIsNone( + self.mod.deny_reason( + self.bash(f"git log --oneline && {lever} task") + ) + ) + + def test_fable_dispatch_is_allowed_with_a_denied_git_subcommand(self): + self.assertIsNone( + self.mod.deny_reason( + self.bash("fable-dispatch.sh plan && git diff") + ) + ) + + def test_grep_over_a_tracked_file_is_denied(self): + self.assertEqual( + self.mod.deny_reason( + self.bash("grep conductor AGENTS.md") + ), + self.mod.INVESTIGATION_REASON, + ) + + def test_cat_over_a_job_path_is_allowed(self): + self.assertIsNone( + self.mod.deny_reason( + self.bash("cat /tmp/scratchpad/jobs/123/result.txt") + ) + ) + + def test_unrelated_tool_is_allowed(self): + self.assertIsNone(self.mod.deny_reason({"tool_name": "Read"})) + + +class ConductorProcess(unittest.TestCase): + def test_malformed_stdin_exits_zero_and_prints_nothing(self): + script = support.CLAUDE_DIR / "conductor-enforce.py" + proc = support.run_script( + script, "{not json", support.WORKTREE_ROOT, os.environ.copy() + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(proc.stdout, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/hooks/tests/test_context_precheck.py b/ops/devlane/hooks/tests/test_context_precheck.py new file mode 100644 index 0000000..61f3fda --- /dev/null +++ b/ops/devlane/hooks/tests/test_context_precheck.py @@ -0,0 +1,178 @@ +"""Moved corpus for context-precheck.py, plus D2/D3 reach against real git.""" + +import shutil +import tempfile +import unittest +from pathlib import Path + +import corpus +import support + + +class PrecheckCorpus(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.mod = support.load_claude("context-precheck.py", "context_precheck") + + def test_the_moved_tables_are_the_tables_that_were_there(self): + self.assertEqual(len(corpus.PRECHECK_STOPS), + corpus.COUNTS["PRECHECK_STOPS"]) + self.assertEqual(len(corpus.PRECHECK_PASSES), + corpus.COUNTS["PRECHECK_PASSES"]) + self.assertEqual(corpus.COUNTS["PRECHECK_STOPS"], 22) + self.assertEqual(corpus.COUNTS["PRECHECK_PASSES"], 21) + + def test_gates_each_stop_row(self): + self.assertGreater(len(corpus.PRECHECK_STOPS), 0) + for cmd in corpus.PRECHECK_STOPS: + with self.subTest(cmd=cmd): + self.assertTrue( + self.mod.is_consequential(cmd), + f"should gate, stayed silent: {cmd!r}", + ) + + def test_lets_each_pass_row_through(self): + self.assertGreater(len(corpus.PRECHECK_PASSES), 0) + for cmd in corpus.PRECHECK_PASSES: + with self.subTest(cmd=cmd): + self.assertFalse( + self.mod.is_consequential(cmd), + f"false stop on {cmd!r}", + ) + + def test_plan_d3_spellings_are_consequential(self): + rows = [ + "git -C . push origin topic", + "git -c push.default=simple push origin topic", + "GIT_SSH_COMMAND='ssh -i k' git push origin topic", + "gh -R o/r pr create --fill", + ] + for cmd in rows: + with self.subTest(cmd=cmd): + self.assertTrue( + self.mod.is_consequential(cmd), + f"D3 spelling was not consequential: {cmd!r}", + ) + + def test_plan_d3_data_heredoc_is_not_a_push(self): + rows = [ + "cat > notes.md <<'EOF'\nWhen ready:\ngit push origin main\nEOF", + ("git commit -q -F - <<'EOF'\n" + "hooks: explain what push costs\n\n" + "git push origin main\nEOF"), + 'git commit -m "wip; git push origin main is next"', + ] + for cmd in rows: + with self.subTest(cmd=cmd[:40]): + self.assertFalse( + self.mod.is_consequential(cmd), + f"prose was treated as a push: {cmd!r}", + ) + + def test_plan_d3_executing_heredoc_is_still_a_push(self): + rows = [ + "bash <<'EOF'\ngit push origin main\nEOF", + "ssh host bash <<'EOF'\ngit push origin main\nEOF", + ] + for cmd in rows: + with self.subTest(cmd=cmd): + self.assertTrue( + self.mod.is_consequential(cmd), + f"executing heredoc was not a push: {cmd!r}", + ) + + +class PrecheckGitState(unittest.TestCase): + """D2 and D3 as they actually happen: a clone whose remote has moved.""" + + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix="precheck-git-")) + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.home = self.tmp / "home" + self.home.mkdir() + self.env = support.isolated_env(self.home) + self.script = support.CLAUDE_DIR / "context-precheck.py" + self.assertTrue( + self.script.is_file(), + f"INVALID: context-precheck.py missing: {self.script}", + ) + self.fixture = support.StaleClone(self.tmp, self.env).build() + + def _run(self, command: str): + payload = support.bash_payload(command) + return support.run_script( + self.script, payload, self.fixture.clone, self.env + ) + + def _origin_main(self): + return support.git( + self.fixture.clone, self.env, "rev-parse", "origin/main" + ).stdout.strip() + + def test_d2_missing_fetch_head_with_a_moved_remote_is_a_deny(self): + """Fresh clone, FETCH_HEAD gone, remote advanced → deny naming origin/main. + + And the fetch must have landed: origin/main equals the remote afterwards. + """ + clone = self.fixture.clone + fetch_head = clone / ".git" / "FETCH_HEAD" + self.assertFalse( + fetch_head.exists(), + "INVALID: FETCH_HEAD present before the hook; D2 is the missing case", + ) + self.assertEqual(self._origin_main(), self.fixture.old_sha) + self.assertNotEqual(self.fixture.old_sha, self.fixture.new_sha) + + proc = self._run("git push origin topic") + decision, reason = support.permission_decision(proc.stdout) + self.assertEqual( + decision, "deny", + f"D2 wanted deny, got {decision!r}\n" + f"stdout: {proc.stdout}\nstderr: {proc.stderr}", + ) + self.assertIn( + "origin/main", reason, + f"deny did not name origin/main: {reason!r}", + ) + self.assertTrue( + fetch_head.is_file(), + "INVALID or red: fetch did not write FETCH_HEAD", + ) + self.assertEqual( + self._origin_main(), self.fixture.new_sha, + "refs were not refreshed; origin/main still stale", + ) + + def test_d3_spellings_deny_and_refresh_against_the_same_moved_remote(self): + spellings = [ + "git -C . push origin topic", + "git -c push.default=simple push origin topic", + "GIT_SSH_COMMAND='ssh -i k' git push origin topic", + "gh -R o/r pr create --fill", + ] + clone = self.fixture.clone + fetch_head = clone / ".git" / "FETCH_HEAD" + for cmd in spellings: + with self.subTest(cmd=cmd): + self.fixture.restale() + self.assertFalse( + fetch_head.exists(), + f"INVALID: FETCH_HEAD present before {cmd!r}", + ) + self.assertEqual(self._origin_main(), self.fixture.old_sha) + + proc = self._run(cmd) + decision, reason = support.permission_decision(proc.stdout) + self.assertEqual( + decision, "deny", + f"D3 spelling {cmd!r} wanted deny, got {decision!r}\n" + f"stdout: {proc.stdout}\nstderr: {proc.stderr}", + ) + self.assertIn( + "origin/main", reason, + f"{cmd!r} deny did not name origin/main: {reason!r}", + ) + self.assertEqual( + self._origin_main(), self.fixture.new_sha, + f"{cmd!r} left origin/main stale", + ) diff --git a/ops/devlane/hooks/tests/test_context_precheck_selected_repo.py b/ops/devlane/hooks/tests/test_context_precheck_selected_repo.py new file mode 100644 index 0000000..ddc3aac --- /dev/null +++ b/ops/devlane/hooks/tests/test_context_precheck_selected_repo.py @@ -0,0 +1,223 @@ +"""Freshness gate must inspect the repository the command names. + +Claim 1 of the git -C finding (not recognised as consequential) is +already pinned by the moved corpus. Claim 2 is a different fact: the +gate fetches and compares in the hook process cwd, not in the repository +`-C` / `--git-dir` / `GIT_DIR` selects. `git -C .` cannot tell those +apart; these cases use two clones. +""" + +from __future__ import annotations + +import os +import shutil +import tempfile +import unittest +from pathlib import Path + +import support + +SCRIPT = support.CLAUDE_DIR / "context-precheck.py" + + +class SelectedRepoFreshness(unittest.TestCase): + """cwd and `other` are independent file:// clones.""" + + maxDiff = None + + def setUp(self): + self.assertTrue( + SCRIPT.is_file(), + f"INVALID: context-precheck.py missing: {SCRIPT}", + ) + self.tmp = Path(tempfile.mkdtemp(prefix="precheck-selected-")) + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.home = self.tmp / "home" + self.home.mkdir() + self.env = support.isolated_env(self.home) + cwd_root = self.tmp / "cwd-pair" + other_root = self.tmp / "other-pair" + cwd_root.mkdir() + other_root.mkdir() + self.cwd_fix = support.StaleClone(cwd_root, self.env).build() + self.other_fix = support.StaleClone(other_root, self.env).build() + + def _origin(self, fix: support.StaleClone) -> str: + return support.git( + fix.clone, self.env, "rev-parse", "origin/main", + ).stdout.strip() + + def _fetch_head(self, fix: support.StaleClone) -> Path: + return fix.clone / ".git" / "FETCH_HEAD" + + def _prove_stale(self, fix: support.StaleClone, label: str): + self.assertEqual( + self._origin(fix), fix.old_sha, + f"INVALID: {label} origin/main is not the old SHA", + ) + self.assertNotEqual( + fix.old_sha, fix.new_sha, + f"INVALID: {label} remote did not move", + ) + remote = support.git( + fix.bare, self.env, "rev-parse", "HEAD", + ).stdout.strip() + self.assertEqual( + remote, fix.new_sha, + f"INVALID: {label} bare HEAD is {remote}, not {fix.new_sha}", + ) + + def _prove_fresh(self, fix: support.StaleClone, label: str): + fix.refresh() + self.assertEqual( + self._origin(fix), fix.new_sha, + f"INVALID: {label} did not refresh to the remote", + ) + + def _fire(self, command: str, *, cwd: Path): + payload = support.bash_payload(command) + return support.run_script(SCRIPT, payload, cwd, self.env) + + def _assert_denied_and_other_refreshed(self, command: str, *, cwd: Path): + self._prove_stale(self.other_fix, "other") + other_before = self._origin(self.other_fix) + self.assertEqual(other_before, self.other_fix.old_sha) + self.assertFalse( + self._fetch_head(self.other_fix).exists(), + "INVALID: other FETCH_HEAD present before the hook", + ) + proc = self._fire(command, cwd=cwd) + decision, reason = support.permission_decision(proc.stdout) + self.assertEqual( + decision, "deny", + f"wanted deny for {command!r} in cwd {cwd}, got {decision!r}\n" + f"stdout: {proc.stdout}\nstderr: {proc.stderr}", + ) + self.assertIn( + "origin/main", reason, + f"deny did not name origin/main: {reason!r}", + ) + self.assertEqual( + self._origin(self.other_fix), self.other_fix.new_sha, + f"{command!r} left other origin/main stale " + f"({self._origin(self.other_fix)}); the gate inspected cwd", + ) + self.assertTrue( + self._fetch_head(self.other_fix).is_file(), + f"{command!r} did not write other FETCH_HEAD; fetch ran elsewhere", + ) + + def test_minus_c_other_stale_cwd_fresh_denies_and_refreshes_other(self): + """VERIFY-GIT-C case A: cwd fresh, other stale, `git -C other push`.""" + self._prove_fresh(self.cwd_fix, "cwd") + self._prove_stale(self.other_fix, "other") + command = f"git -C {self.other_fix.clone} push origin topic" + self._assert_denied_and_other_refreshed( + command, cwd=self.cwd_fix.clone, + ) + + def test_minus_c_other_fresh_cwd_stale_does_not_deny(self): + """VERIFY-GIT-C case B: deny citing cwd is the wrong repository.""" + self._prove_stale(self.cwd_fix, "cwd") + self._prove_fresh(self.other_fix, "other") + cwd_sha = self._origin(self.cwd_fix) + command = f"git -C {self.other_fix.clone} push origin topic" + proc = self._fire(command, cwd=self.cwd_fix.clone) + decision, reason = support.permission_decision(proc.stdout) + self.assertNotEqual( + decision, "deny", + f"-C other (fresh) denied using cwd (stale): {decision!r} " + f"{reason!r}\nstdout: {proc.stdout}", + ) + self.assertEqual( + self._origin(self.cwd_fix), cwd_sha, + "cwd origin/main moved; a deny of cwd was applied to -C other", + ) + self.assertEqual( + self._origin(self.other_fix), self.other_fix.new_sha, + "INVALID: other origin drifted during the case", + ) + + def test_git_dir_option_other_stale_denies_and_refreshes_other(self): + self._prove_fresh(self.cwd_fix, "cwd") + git_dir = self.other_fix.clone / ".git" + self.assertTrue(git_dir.is_dir(), "INVALID: other .git missing") + command = f"git --git-dir={git_dir} push origin topic" + self._assert_denied_and_other_refreshed( + command, cwd=self.cwd_fix.clone, + ) + + def test_git_dir_separate_arg_other_stale_denies_and_refreshes_other(self): + self._prove_fresh(self.cwd_fix, "cwd") + git_dir = self.other_fix.clone / ".git" + command = f"git --git-dir {git_dir} push origin topic" + self._assert_denied_and_other_refreshed( + command, cwd=self.cwd_fix.clone, + ) + + def test_git_dir_assignment_in_command_other_stale_denies_and_refreshes( + self): + self._prove_fresh(self.cwd_fix, "cwd") + git_dir = self.other_fix.clone / ".git" + command = f"GIT_DIR={git_dir} git push origin topic" + self._assert_denied_and_other_refreshed( + command, cwd=self.cwd_fix.clone, + ) + + def test_relative_minus_c_other_stale_denies_and_refreshes_other(self): + self._prove_fresh(self.cwd_fix, "cwd") + rel = os.path.relpath( + str(self.other_fix.clone), start=str(self.cwd_fix.clone), + ) + self.assertFalse( + Path(rel).is_absolute(), + f"INVALID: relative path was absolute: {rel}", + ) + command = f"git -C {rel} push origin topic" + self._assert_denied_and_other_refreshed( + command, cwd=self.cwd_fix.clone, + ) + + def test_minus_c_with_config_other_stale_denies_and_refreshes_other(self): + self._prove_fresh(self.cwd_fix, "cwd") + command = ( + f"git -C {self.other_fix.clone} " + "-c push.default=simple push origin topic" + ) + self._assert_denied_and_other_refreshed( + command, cwd=self.cwd_fix.clone, + ) + + def test_minus_c_other_when_cwd_is_not_a_repo(self): + plain = self.tmp / "plain" + plain.mkdir() + self.assertFalse( + (plain / ".git").exists(), + "INVALID: plain cwd grew a .git", + ) + self._prove_stale(self.other_fix, "other") + command = f"git -C {self.other_fix.clone} push origin topic" + self._assert_denied_and_other_refreshed(command, cwd=plain) + + def test_minus_c_linked_worktree_of_other_denies_and_refreshes_other(self): + self._prove_fresh(self.cwd_fix, "cwd") + self._prove_stale(self.other_fix, "other") + wt = self.tmp / "other-wt" + support.git( + self.other_fix.clone, self.env, + "worktree", "add", str(wt), "-b", "wt-topic", + ) + gitfile = wt / ".git" + self.assertTrue( + gitfile.is_file(), + "INVALID: linked worktree .git is not a file", + ) + landed = gitfile.read_text(encoding="utf-8") + self.assertIn( + "gitdir:", landed, + f"INVALID: worktree .git contents {landed!r}", + ) + command = f"git -C {wt} push origin topic" + self._assert_denied_and_other_refreshed( + command, cwd=self.cwd_fix.clone, + ) diff --git a/ops/devlane/hooks/tests/test_context_stream.py b/ops/devlane/hooks/tests/test_context_stream.py new file mode 100644 index 0000000..2c50cde --- /dev/null +++ b/ops/devlane/hooks/tests/test_context_stream.py @@ -0,0 +1,502 @@ +"""One clone-shared context stream, from every worktree, for every writer. + +Written from `.dev/guide/hooks.md` ("clone-shared context stream", +"every worktree of the clone"), CONTRIB.md ("Every worktree shares one +clone — one … context stream"), and PLAN.md BELONGS item 4: the Claude +writer and the git-native recorders must not land in different files +when git-dir and git-common-dir diverge. Not written from the scripts. + +The origin selftest measured the stream via `--git-dir` in a single +checkout, so it could not see the split. These cases run every writer +from a linked worktree's cwd (``.git`` is a gitfile there) and read +the clone stream at `--git-common-dir`. +""" + +from __future__ import annotations + +import os +import unittest +from pathlib import Path + +import support +import test_hooks as hook_contract + +STREAM_SH = support.CLAUDE_DIR / "context-stream.sh" +BOUNDARY_SH = support.CLAUDE_DIR / "context-boundary.sh" +KIND_WT = "artifacts" +# Not a substring of worktree names (second-tree, third-tree) or kinds +# the git-native recorders already write (branch, head). +KIND_MAIN = "artifacts-from-primary" +BRANCH_A = "stream-checkout-target" +SINCE_EPOCH = "1970-01-01T00:00:00Z" + + +class OneStreamPerClone(unittest.TestCase): + """Pins one stream per clone. Each pinning case must be red at this head.""" + + maxDiff = None + + def setUp(self): + self.assertTrue( + STREAM_SH.is_file(), + f"INVALID: context-stream.sh is missing: {STREAM_SH}", + ) + self.h = hook_contract.HookContractTest( + "test_h6_second_worktree_appends_to_the_shared_stream" + ) + self.h.setUp() + self.addCleanup(self.h.doCleanups) + + def _env(self): + return self.h.env_for() + + def _linked_clone(self): + """Primary + two sibling linked worktrees, hooks installed. + + Sibling paths (not nested in the primary) are CONTRIB's shape + and the input a 'walk up until .git is a directory' resolver + gets wrong. + """ + self.h.require_sources() + repo = self.h.make_repo("primary-clone") + self.h.git(repo, "branch", "wt-a") + self.h.git(repo, "branch", "wt-b") + self.h.git(repo, "branch", BRANCH_A) + self.h.install(repo) + wt_a = self.h.tmp / "second-tree" + wt_b = self.h.tmp / "third-tree" + self.h.git(repo, "worktree", "add", str(wt_a), "wt-a") + self.h.git(repo, "worktree", "add", str(wt_b), "wt-b") + env = self._env() + support.prove_linked_worktree(repo, wt_a, env, self.h.tmp) + support.prove_linked_worktree(repo, wt_b, env, self.h.tmp) + return repo, wt_a, wt_b + + def _git_dir(self, cwd): + return support.rev_parse_path(cwd, self._env(), "--git-dir", self.h.tmp) + + def _stream_sh(self, cwd, *args, expect=0): + # Invoke the script itself: prepending bash would hide a broken argv[0]. + self.assertTrue( + os.access(STREAM_SH, os.X_OK), + f"INVALID: {STREAM_SH} is not executable", + ) + return self.h.run_cmd([str(STREAM_SH), *args], cwd, expect=expect) + + def _boundary(self, cwd, payload, expect=0): + self.assertTrue( + BOUNDARY_SH.is_file(), + f"INVALID: context-boundary.sh is missing: {BOUNDARY_SH}", + ) + self.assertTrue( + os.access(BOUNDARY_SH, os.X_OK), + f"INVALID: {BOUNDARY_SH} is not executable", + ) + proc = support.run_cmd( + [str(BOUNDARY_SH)], cwd, self._env(), stdin=payload, expect=None + ) + if expect is not None: + self.assertEqual( + proc.returncode, + expect, + f"context-boundary.sh exited {proc.returncode} " + f"(wanted {expect})\nstdout: {proc.stdout}\n" + f"stderr: {proc.stderr}", + ) + return proc + + def _clone_text(self, repo: Path) -> str: + path = self.h.stream_path(repo) + if not path.is_file(): + return "" + return path.read_text(encoding="utf-8") + + def _private_stream(self, wt: Path) -> Path: + return self._git_dir(wt) / hook_contract.STREAM_NAME + + def _wrote_kind_somewhere(self, repo: Path, wt: Path, kind: str) -> None: + """Plant proof: the writer produced *kind* in at least one stream file.""" + clone = self._clone_text(repo) + private_path = self._private_stream(wt) + private = ( + private_path.read_text(encoding="utf-8") + if private_path.is_file() + else "" + ) + self.assertTrue( + kind in clone or kind in private, + "INVALID: record did not write " + f"{kind!r} to the clone stream or the per-worktree git-dir " + f"(clone {self.h.stream_path(repo)}, private {private_path})", + ) + + def test_record_from_linked_worktree_lands_in_the_clone_stream(self): + """Claude-side record from a linked worktree appends to the clone stream. + + A script that uses --git-dir, or that uses --git-common-dir only + when cwd's .git is a directory, writes a private file instead. + """ + repo, wt_a, _wt_b = self._linked_clone() + before = self.h.stream_lines(repo) + proc = self._stream_sh(wt_a, "record", KIND_WT) + self.assertEqual(proc.returncode, 0, proc.stderr) + self._wrote_kind_somewhere(repo, wt_a, KIND_WT) + + after = self.h.stream_lines(repo) + self.assertEqual( + after[: len(before)], + before, + "clone stream is not append-only after a linked-worktree record", + ) + added = after[len(before) :] + self.assertEqual( + len(added), + 1, + f"expected one new clone-stream line, got {len(added)}: {added!r}", + ) + self.assertIn( + KIND_WT, + added[0], + f"clone-stream line does not name the kind: {added[0]!r}", + ) + self.assertFalse( + self._private_stream(wt_a).exists(), + "a private stream appeared in the per-worktree git-dir; " + "the stream must be shared at the common dir only", + ) + tail_main = self._stream_sh(repo, "tail", "15").stdout + self.assertIn( + KIND_WT, + tail_main, + f"tail from the primary checkout missed the linked-worktree " + f"record: {tail_main!r}", + ) + + def test_tail_from_linked_worktree_sees_a_git_native_commit(self): + """post-commit writes the clone stream; tail from that worktree sees it. + + A tail that still reads --git-dir shows a different history from + the worktree than from the primary checkout. + """ + repo, wt_a, _wt_b = self._linked_clone() + sha = self.h.commit_change(wt_a, "from-linked\n", "stream-commit-marker") + clone = self._clone_text(repo) + self.assertIn( + sha, + clone, + f"INVALID: commit {sha} did not land on the clone stream: {clone!r}", + ) + self.assertIn( + sha, + self._stream_sh(repo, "tail", "15").stdout, + "INVALID: tail from the primary does not see its own clone stream", + ) + tail_wt = self._stream_sh(wt_a, "tail", "15").stdout + self.assertIn( + sha, + tail_wt, + f"tail from the linked worktree missed the git-native commit " + f"{sha}: {tail_wt!r}", + ) + + def test_tail_from_linked_worktree_sees_a_git_native_checkout(self): + """post-checkout is the other git-native writer; same promise as commit.""" + repo, wt_a, _wt_b = self._linked_clone() + self.h.git(wt_a, "checkout", "-q", BRANCH_A) + clone = self._clone_text(repo) + self.assertIn( + BRANCH_A, + clone, + f"INVALID: checkout of {BRANCH_A} did not land on the clone " + f"stream: {clone!r}", + ) + tail_wt = self._stream_sh(wt_a, "tail", "15").stdout + self.assertIn( + BRANCH_A, + tail_wt, + f"tail from the linked worktree missed the git-native checkout " + f"{BRANCH_A}: {tail_wt!r}", + ) + + def test_claude_and_git_writers_are_one_history_from_every_worktree(self): + """The finding: Claude record + git commit + git checkout, one history. + + Read from the primary, the writing worktree, and a second linked + worktree. A resolver that is correct only for the main checkout, + or a tail that merges nothing, fails at least one location. + """ + repo, wt_a, wt_b = self._linked_clone() + before = self.h.stream_lines(repo) + + self._stream_sh(wt_a, "record", KIND_WT) + self._wrote_kind_somewhere(repo, wt_a, KIND_WT) + sha = self.h.commit_change(wt_a, "mixed\n", "stream-mixed-commit") + self.h.git(wt_a, "checkout", "-q", BRANCH_A) + + clone = self._clone_text(repo) + for marker in (KIND_WT, sha, BRANCH_A): + self.assertIn( + marker, + clone, + f"clone stream is missing {marker!r}: {clone!r}", + ) + after = self.h.stream_lines(repo) + self.assertEqual(after[: len(before)], before) + self.assertGreaterEqual( + len(after) - len(before), + 3, + f"three writers should append at least three records, " + f"observed {after[len(before):]!r}", + ) + + for cwd, label in ( + (repo, "primary"), + (wt_a, "writing worktree"), + (wt_b, "second linked worktree"), + ): + tail = self._stream_sh(cwd, "tail", "15").stdout + for marker in (KIND_WT, sha, BRANCH_A): + self.assertIn( + marker, + tail, + f"tail from {label} ({cwd}) missed {marker!r}: {tail!r}", + ) + self.assertFalse( + self._private_stream(wt_a).exists(), + "Claude-side record left a private stream in the worktree git-dir", + ) + self.assertFalse( + self._private_stream(wt_b).exists(), + "a stream file appeared in the second worktree's private git-dir", + ) + + def test_record_from_main_is_visible_from_every_linked_worktree(self): + """The other direction: write in the primary, read from the worktrees. + + A tail that uses --git-dir from a linked worktree misses records + that already land correctly on the common dir. + """ + repo, wt_a, wt_b = self._linked_clone() + self._stream_sh(repo, "record", KIND_MAIN) + clone = self._clone_text(repo) + self.assertIn( + KIND_MAIN, + clone, + f"INVALID: record from the primary did not land on the clone " + f"stream: {clone!r}", + ) + for cwd, label in ((wt_a, "second-tree"), (wt_b, "third-tree")): + tail = self._stream_sh(cwd, "tail", "15").stdout + self.assertIn( + KIND_MAIN, + tail, + f"tail from {label} missed a primary-checkout record: {tail!r}", + ) + + def test_kinded_record_without_delta_appends_and_bare_record_stays_quiet( + self, + ): + """Origin selftest, from a linked worktree, against the clone stream. + + Origin measured `--git-dir` in a single checkout: a kinded record + still appends when nothing in the tree moved, and a bare record + after that stays silent. Here the same sequence must land on the + clone-shared file, or the origin proof still cannot see the split. + """ + repo, wt_a, _wt_b = self._linked_clone() + self._stream_sh(wt_a, "record") # seed state, as origin does + before = self.h.stream_lines(repo) + self._stream_sh(wt_a, "record", KIND_WT) + self._wrote_kind_somewhere(repo, wt_a, KIND_WT) + after = self.h.stream_lines(repo) + self.assertEqual(after[: len(before)], before) + added = after[len(before) :] + self.assertEqual( + len(added), + 1, + f"kinded record with no state delta must append one clone-stream " + f"line, got {added!r}", + ) + self.assertIn(KIND_WT, added[0]) + tail = self._stream_sh(wt_a, "tail", "1").stdout + self.assertIn( + KIND_WT, + tail, + f"tail 1 from the linked worktree did not name the kind: {tail!r}", + ) + tail_main = self._stream_sh(repo, "tail", "1").stdout + self.assertIn( + KIND_WT, + tail_main, + f"tail 1 from the primary missed the kinded record: {tail_main!r}", + ) + self._stream_sh(wt_a, "record") + self.assertEqual( + self.h.stream_lines(repo), + after, + "a bare record with nothing changed must not append again", + ) + + def test_bare_record_from_a_second_worktree_does_not_fork_the_stream(self): + """A script that shares the log file but keeps per-worktree state still + forks the history: the second worktree's first bare record looks + like a first-ever snapshot and appends again. The promise is one + stream, so a no-kind record after the clone is already seeded + must stay quiet from every worktree. + """ + repo, wt_a, wt_b = self._linked_clone() + self._stream_sh(wt_a, "record") + self._stream_sh(wt_a, "record", KIND_WT) + self._wrote_kind_somewhere(repo, wt_a, KIND_WT) + after = self.h.stream_lines(repo) + self.assertTrue( + any(KIND_WT in line for line in after), + f"kinded record from the first worktree never reached the clone " + f"stream: {after!r}", + ) + self._stream_sh(wt_b, "record") + self.assertEqual( + self.h.stream_lines(repo), + after, + "a bare record from a second worktree appended; the clone " + "stream is not shared state, only a shared filename", + ) + + def test_since_from_main_sees_a_record_made_in_a_linked_worktree(self): + """`since` is a third reader. Fixing record and tail is not enough.""" + repo, wt_a, _wt_b = self._linked_clone() + self._stream_sh(wt_a, "record", KIND_WT) + self._wrote_kind_somewhere(repo, wt_a, KIND_WT) + out = self._stream_sh(repo, "since", SINCE_EPOCH).stdout + self.assertIn( + KIND_WT, + out, + f"since from the primary missed a linked-worktree record: {out!r}", + ) + + def test_since_from_linked_worktree_sees_a_git_native_commit(self): + """`since` from the worktree must read the clone stream, not git-dir.""" + repo, wt_a, _wt_b = self._linked_clone() + sha = self.h.commit_change(wt_a, "since-wt\n", "stream-since-commit") + self.assertIn( + sha, + self._clone_text(repo), + f"INVALID: commit {sha} did not land on the clone stream", + ) + out = self._stream_sh(wt_a, "since", SINCE_EPOCH).stdout + self.assertIn( + sha, + out, + f"since from the linked worktree missed the git-native commit " + f"{sha}: {out!r}", + ) + + def test_boundary_wrapper_from_linked_worktree_appends_to_the_clone_stream( + self, + ): + """The session-facing writer is context-boundary.sh, not the CLI. + + Origin selftest fires the wrapper with `git checkout main`. From a + linked worktree that fire must append to the clone stream and be + visible via tail from the primary. + """ + repo, wt_a, _wt_b = self._linked_clone() + before = self.h.stream_lines(repo) + before_text = self._clone_text(repo) + payload = '{"tool_input":{"command":"git checkout main"}}' + proc = self._boundary(wt_a, payload, expect=0) + self.assertTrue( + support.has_additional_context(proc.stdout), + "INVALID: boundary wrapper did not fire on git checkout main: " + f"{proc.stdout!r}", + ) + after = self.h.stream_lines(repo) + self.assertEqual(after[: len(before)], before) + self.assertGreater( + len(after), + len(before), + "boundary wrapper from a linked worktree did not append to the " + f"clone stream (stdout={proc.stdout!r}, stderr={proc.stderr!r}, " + f"before={before_text!r}, after={self._clone_text(repo)!r})", + ) + tail_main = self._stream_sh(repo, "tail", "15").stdout + tail_wt = self._stream_sh(wt_a, "tail", "15").stdout + self.assertTrue( + tail_main.strip(), + "tail from the primary is empty after a boundary fire", + ) + # The new clone-stream bytes must appear in both tails. Comparing + # the added lines themselves, not a formatted wrapper message. + added = after[len(before) :] + self.assertGreater(len(added), 0) + for line in added: + token = line[:40] + self.assertTrue( + token, + "INVALID: an appended clone-stream line was empty", + ) + self.assertIn( + token, + tail_main, + f"tail from the primary missed boundary record {token!r}: " + f"{tail_main!r}", + ) + self.assertIn( + token, + tail_wt, + f"tail from the linked worktree missed boundary record " + f"{token!r}: {tail_wt!r}", + ) + self.assertFalse( + self._private_stream(wt_a).exists(), + "boundary fire left a private stream in the worktree git-dir", + ) + + +class OriginBoundaryNeverBlock(unittest.TestCase): + """Origin selftest: the PostToolUse wrapper must never block a tool call.""" + + def setUp(self): + self.assertTrue( + BOUNDARY_SH.is_file(), + f"INVALID: context-boundary.sh is missing: {BOUNDARY_SH}", + ) + self.h = hook_contract.HookContractTest( + "test_h6_second_worktree_appends_to_the_shared_stream" + ) + self.h.setUp() + self.addCleanup(self.h.doCleanups) + self.h.require_sources() + self.repo = self.h.make_repo("primary-clone") + + def test_exits_0_on_malformed_payloads(self): + payloads = ( + '{"tool_input":{}}', + "not json", + "", + '{"tool_input":{"command":null}}', + ) + self.assertGreater(len(payloads), 0) + self.assertTrue( + os.access(BOUNDARY_SH, os.X_OK), + f"INVALID: {BOUNDARY_SH} is not executable", + ) + for payload in payloads: + with self.subTest(payload=payload[:40]): + proc = support.run_cmd( + [str(BOUNDARY_SH)], + self.repo, + self.h.env_for(), + stdin=payload, + expect=None, + ) + self.assertEqual( + proc.returncode, + 0, + f"boundary wrapper blocked on {payload!r}: " + f"rc={proc.returncode} stderr={proc.stderr!r}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/hooks/tests/test_hooks.py b/ops/devlane/hooks/tests/test_hooks.py new file mode 100644 index 0000000..99745db --- /dev/null +++ b/ops/devlane/hooks/tests/test_hooks.py @@ -0,0 +1,916 @@ +"""Contract tests for the git-native crossing recorder (ops/devlane/hooks). + +Written from the behavior contract only (branch repo/worktree-per-line, +behaviors H1-H7), then strengthened against a skeptic's surviving-mutant +report. The mutants were read as descriptions of faults, never as a +source of expectations: every assertion below traces to the contract. +Every test first asserts the three source files exist (install.sh, +post-checkout, post-commit), so a missing implementation fails as a plain +assertion with a message -- never as an import or collection error. + +All git activity happens in throwaway repositories under tempfile, with +explicit user.name/user.email and fixed --date on every commit; the wall +clock is never an assertion input (the entry's "at" field is checked for +shape only). The real repo's .git, hooks, and stream file are never +touched: every resolved git common dir is asserted to live inside the +test's own tempdir before anything is read or written there. + +Every fixture these tests corrupt -- a foreign hook, a symlinked hook, a +directory standing in the stream's place, pre-existing stream lines -- is +proved to have landed and to still be recognisably itself before the +behavior under test runs, so that no later assertion can answer a +question about nothing. +""" + +import json +import os +import re +import shutil +import stat +import subprocess +import tempfile +import unittest +from pathlib import Path + +# The package under test sits beside this tests/ directory, inside the +# worktree at ops/devlane/hooks/ -- resolved from this file's own location. +TESTS_DIR = Path(__file__).resolve().parent # .../ops/devlane/hooks/tests +WORKTREE_ROOT = TESTS_DIR.parents[3] # the repo/worktree root +HOOKS_SRC_DIR = WORKTREE_ROOT / "ops" / "devlane" / "hooks" +INSTALL_SH = HOOKS_SRC_DIR / "install.sh" +POST_CHECKOUT_SRC = HOOKS_SRC_DIR / "post-checkout" +POST_COMMIT_SRC = HOOKS_SRC_DIR / "post-commit" +COMMIT_MSG_SRC = HOOKS_SRC_DIR / "commit-msg" +#: commit-msg shells out to a checker in the WORKFLOW app, so a fixture +#: that stages only this package would make every commit fail — the hook +#: refuses when it cannot check, by design. Staging it here is not a +#: convenience: it is the real dependency, made visible. +TRAILER_CHECK_SRC = (WORKTREE_ROOT / "ops" / "devlane" / "workflow" + / "checks" / "commit_trailers.py") +SOURCES = (INSTALL_SH, POST_CHECKOUT_SRC, POST_COMMIT_SRC, COMMIT_MSG_SRC) +HOOK_SOURCES = (POST_CHECKOUT_SRC, POST_COMMIT_SRC, COMMIT_MSG_SRC) +HOOK_NAMES = ("post-checkout", "post-commit", "commit-msg") + +STREAM_NAME = "claude-context-stream.jsonl" +WF = "Agent Under Test " +# A perfectly legal WF_AGENT that is illegal inside a JSON string until +# it is escaped: it carries one double quote and one backslash. +TRICKY_WF = 'Odd " Agent \\ Name ' +FIXED_DATE_1 = "2026-01-02T03:04:05Z" +FIXED_DATE_2 = "2026-01-02T03:05:06Z" + +# ISO8601 UTC with a literal Z suffix; fractional seconds allowed. +AT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$") +ENTRY_KEYS = ("at", "kind", "what", "detail", "agent", "worktree", "via") + + +class HookContractTest(unittest.TestCase): + """Pins H1-H7 of the crossing-recorder behavior contract.""" + + maxDiff = None + + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix="hooks-contract-")) + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.home = self.tmp / "home" + self.home.mkdir() + + # ---------------------------------------------------------------- plumbing + + def require_sources(self): + """The clear missing-file assertion: while the implementation does + not exist, every test fails here, with a message, not on import.""" + for path in SOURCES: + self.assertTrue( + path.is_file(), + f"hook package file does not exist yet (implementation " + f"missing): {path}", + ) + + def env_for(self, wf_agent=WF, user="tester"): + """A fully controlled environment: no global/system git config, a + temp HOME, fixed commit dates. Pass None to omit WF_AGENT or USER + entirely (LOGNAME is never set, so $USER is the only identity).""" + env = { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": str(self.home), + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_AUTHOR_DATE": FIXED_DATE_1, + "GIT_COMMITTER_DATE": FIXED_DATE_1, + "LC_ALL": "C", + } + if wf_agent is not None: + env["WF_AGENT"] = wf_agent + if user is not None: + env["USER"] = user + return env + + def run_cmd(self, argv, cwd, env=None, expect=0): + proc = subprocess.run( + argv, + cwd=str(cwd), + env=env if env is not None else self.env_for(), + capture_output=True, + text=True, + check=False, + ) + if expect is not None: + self.assertEqual( + proc.returncode, + expect, + f"command {argv!r} in {cwd} exited {proc.returncode} " + f"(wanted {expect})\nstdout: {proc.stdout}\n" + f"stderr: {proc.stderr}", + ) + return proc + + def git(self, cwd, *args, env=None, expect=0): + return self.run_cmd(["git", *args], cwd, env=env, expect=expect) + + def make_repo(self, name): + """A throwaway repo: explicit identity, fixed-date initial commit.""" + repo = self.tmp / name + repo.mkdir() + self.git(repo, "-c", "init.defaultBranch=main", "init", "-q") + self.git(repo, "config", "user.name", "Test User") + self.git(repo, "config", "user.email", "test@example.invalid") + (repo / "file.txt").write_text("one\n") + self.git(repo, "add", "file.txt") + self.git(repo, "commit", "-q", "-m", "c1", "--date", FIXED_DATE_1) + return repo + + def stage_pkg(self, repo): + """Copy the hook package into the throwaway repo at its real + relative path, proving each copy landed byte-for-byte, so that + install.sh may resolve its hooks beside $0 or from the repo tree -- + both are faithful to 'install.sh run inside a repo'.""" + dest = repo / "ops" / "devlane" / "hooks" + dest.mkdir(parents=True) + check_dest = repo / "ops" / "devlane" / "workflow" / "checks" + check_dest.mkdir(parents=True) + shutil.copy2(str(TRAILER_CHECK_SRC), str(check_dest / TRAILER_CHECK_SRC.name)) + self.assertEqual( + (check_dest / TRAILER_CHECK_SRC.name).read_bytes(), + TRAILER_CHECK_SRC.read_bytes(), + "staged copy of the trailer checker did not land intact", + ) + for src in SOURCES: + target = dest / src.name + shutil.copy2(str(src), str(target)) + self.assertEqual( + target.read_bytes(), + src.read_bytes(), + f"staged copy of {src.name} did not land intact", + ) + return dest / "install.sh" + + def install(self, repo, *args, env=None, expect=0): + """Run install.sh with `repo` as the working directory. `repo` may + be a primary checkout or a linked worktree; the script is staged + into whichever tree it is run from.""" + script = repo / "ops" / "devlane" / "hooks" / "install.sh" + if not script.is_file(): + script = self.stage_pkg(repo) + return self.run_cmd(["sh", str(script), *args], repo, env=env, expect=expect) + + def common_dir(self, cwd): + """The clone's git common dir, asserted to be inside the sandbox so + no test can ever touch the real repo's .git.""" + out = self.git(cwd, "rev-parse", "--git-common-dir").stdout.strip() + common = Path(os.path.abspath(os.path.join(str(cwd), out))) + self.assertTrue( + os.path.realpath(str(common)).startswith( + os.path.realpath(str(self.tmp)) + os.sep + ), + f"resolved git common dir {common} escapes the test sandbox {self.tmp}", + ) + return common + + def hooks_dir(self, cwd): + return self.common_dir(cwd) / "hooks" + + def stream_path(self, cwd): + return self.common_dir(cwd) / STREAM_NAME + + def stream_lines(self, cwd): + path = self.stream_path(cwd) + if not path.is_file(): + return [] + return path.read_text().splitlines() + + def parse_entry(self, line): + """One stream line must be a JSON object with the full schema: + string fields at/kind/what/detail/agent/worktree/via, at in + ISO8601Z shape, via 'git-hook', detail '' (the contract fixes it).""" + try: + entry = json.loads(line) + except ValueError as exc: + self.fail(f"stream line is not valid JSON: {line!r} ({exc})") + self.assertIsInstance( + entry, dict, f"stream line is not a JSON object: {line!r}" + ) + for key in ENTRY_KEYS: + self.assertIn(key, entry, f"entry lacks key {key!r}: {line!r}") + self.assertIsInstance( + entry[key], str, f"entry field {key!r} is not a string: {line!r}" + ) + self.assertRegex( + entry["at"], + AT_RE, + f"at is not ISO8601 UTC with a Z suffix: {entry['at']!r}", + ) + self.assertEqual(entry["via"], "git-hook", f"via is not 'git-hook': {line!r}") + self.assertEqual( + entry["detail"], "", f"detail is not the empty string: {line!r}" + ) + return entry + + def sole_new_entry(self, before, cwd): + """Exactly one line was appended since `before`, and the stream is + still append-only (earlier lines untouched). Returns the parsed + new entry. Asserting the count first keeps every later field check + non-vacuous.""" + after = self.stream_lines(cwd) + self.assertEqual( + after[: len(before)], + before, + "stream is not append-only: earlier lines changed or were lost " + f"(before={before!r}, after={after!r})", + ) + new = after[len(before) :] + self.assertEqual( + len(new), + 1, + f"expected exactly one new stream line, got {len(new)}: {new!r}", + ) + return self.parse_entry(new[0]) + + # ------------------------------------------------------- guarded fixtures + + def assert_installed_hook(self, hooks, src, when): + """An installed hook is present, byte-identical to the packaged + one, and executable -- git silently ignores a non-executable + hook, so the mode is part of 'installed', not a nicety.""" + installed = hooks / src.name + self.assertTrue( + installed.is_file(), + f"{src.name} was not installed into {hooks} ({when})", + ) + self.assertEqual( + installed.read_bytes(), + src.read_bytes(), + f"installed {src.name} differs from the packaged hook ({when})", + ) + self.assertTrue( + installed.stat().st_mode & stat.S_IXUSR, + f"installed {src.name} is not executable; git will never run it ({when})", + ) + return installed + + def plant_foreign_hook(self, hooks, name): + """Plant somebody else's hook of `name`, and prove the plant both + landed and is distinguishable from the packaged hook -- otherwise + a later 'was it clobbered?' check answers nothing.""" + hooks.mkdir(parents=True, exist_ok=True) + packaged = (HOOKS_SRC_DIR / name).read_bytes() + sentinel = ( + f"#!/bin/sh\n# pre-existing {name}, not ours\n" + f"# sentinel: do-not-clobber-{name}\nexit 0\n" + ).encode() + dest = hooks / name + dest.write_bytes(sentinel) + dest.chmod(0o755) + landed = dest.read_bytes() + self.assertEqual(landed, sentinel, f"planted foreign {name} did not land") + self.assertTrue(landed, f"planted foreign {name} landed empty") + self.assertNotEqual( + sentinel, + packaged, + f"planted foreign {name} is byte-identical to the packaged " + "hook; this test cannot discriminate", + ) + return sentinel + + def plant_stream_lines(self, cwd, count=2): + """Seed the shared stream with earlier records, proving they + landed, so that a later append can be told apart from a truncating + overwrite. An empty stream makes those two indistinguishable.""" + path = self.stream_path(cwd) + lines = [ + json.dumps( + { + "at": FIXED_DATE_1, + "kind": "branch", + "what": f"earlier record {i} from another actor", + "detail": "", + "agent": "Someone Else ", + "worktree": "some-other-tree", + "via": "git-hook", + } + ) + for i in range(count) + ] + payload = "".join(line + "\n" for line in lines) + self.assertTrue(payload, "stream plant payload is empty") + path.write_text(payload) + self.assertTrue(path.is_file(), "stream plant did not create the file") + self.assertEqual(path.read_text(), payload, "stream plant did not land") + self.assertEqual( + path.stat().st_size, + len(payload.encode()), + "stream plant landed truncated", + ) + self.assertEqual( + self.stream_lines(cwd), lines, "planted stream lines do not read back" + ) + return lines + + def commit_change(self, repo, text, message, env=None): + """Make a commit and return the resulting short SHA. Only the + commit itself carries `env`; staging and rev-parse fire no hook.""" + (repo / "file.txt").write_text(text) + self.git(repo, "add", "file.txt") + # A real commit in this repo carries a Source trailer, and the + # commit-msg hook these fixtures install now requires one. A bare + # `-m "primary commit"` was refused by the very gate under test. + self.git(repo, "commit", "-q", "-m", message + "\n\nSource: original", + "--date", FIXED_DATE_2, env=env) + short = self.git(repo, "rev-parse", "--short", "HEAD").stdout.strip() + self.assertTrue( + short, + "could not resolve the new short SHA (guards the containment " + "checks below from being vacuous)", + ) + return short + + # ------------------------------------------------------------------- tests + + def test_h1_install_copies_both_hooks_and_reruns_quietly(self): + """H1: install.sh run inside a repo copies both hooks into the + clone's common hooks dir; a second run is a quiet success that + leaves both hooks installed AND still executable.""" + self.require_sources() + repo = self.make_repo("primary-clone") + self.install(repo) + hooks = self.hooks_dir(repo) + for src in HOOK_SOURCES: + self.assert_installed_hook(hooks, src, "after the first install") + second = self.install(repo) # idempotent: exit 0 asserted by install() + self.assertEqual( + second.stdout, + "", + f"second install was not quiet (stdout): {second.stdout!r}", + ) + self.assertEqual( + second.stderr, + "", + f"second install was not quiet (stderr): {second.stderr!r}", + ) + # The rerun must not quietly disarm what the first run installed: + # content AND mode are re-checked after the second run. + for src in HOOK_SOURCES: + self.assert_installed_hook(hooks, src, "after the idempotent rerun") + + def test_h1_install_from_a_linked_worktree_targets_the_common_hooks_dir(self): + """H1+H6: install.sh run from a linked worktree still installs + into the CLONE's common hooks dir -- not the worktree's private + gitdir -- so the hooks fire from every worktree of the clone.""" + self.require_sources() + repo = self.make_repo("primary-clone") + self.git(repo, "branch", "beta") + self.git(repo, "branch", "b-linked") + self.git(repo, "branch", "b-target") + wt2 = self.tmp / "second-tree" + self.git(repo, "worktree", "add", str(wt2), "b-linked") + common = self.common_dir(repo) + self.assertEqual( + os.path.realpath(str(self.common_dir(wt2))), + os.path.realpath(str(common)), + "the linked worktree does not share the clone's common dir; " + "the fixture is not what this test needs", + ) + # Install FROM the linked worktree. + self.install(wt2) + hooks = common / "hooks" + for src in HOOK_SOURCES: + self.assert_installed_hook(hooks, src, "installed from a linked worktree") + private = common / "worktrees" / "second-tree" / "hooks" + for name in HOOK_NAMES: + self.assertFalse( + (private / name).exists(), + f"install from a linked worktree put {name} in the " + f"per-worktree gitdir ({private}); git will not run it " + "there and other worktrees never see it", + ) + # The promise is not 'a file appeared' but 'the hook fires' -- and + # it must fire from BOTH worktrees of the clone. + before = self.stream_lines(repo) + self.git(repo, "checkout", "-q", "beta", env=self.env_for(wf_agent=WF)) + entry = self.sole_new_entry(before, repo) + self.assertEqual( + entry["kind"], + "branch", + f"primary-worktree checkout did not record kind 'branch': {entry!r}", + ) + self.assertEqual( + entry["worktree"], + "primary-clone", + f"entry does not carry the primary worktree's name: {entry!r}", + ) + before = self.stream_lines(repo) + self.git(wt2, "checkout", "-q", "b-target", env=self.env_for(wf_agent=WF)) + entry = self.sole_new_entry(before, repo) + self.assertEqual( + entry["kind"], + "branch", + f"linked-worktree checkout did not record kind 'branch': {entry!r}", + ) + self.assertIn( + "b-target", + entry["what"], + f"what does not name the branch switched to: {entry['what']!r}", + ) + self.assertEqual( + entry["worktree"], + "second-tree", + f"entry does not carry the linked worktree's own name: {entry!r}", + ) + + def test_h2_existing_different_hook_survives_unless_forced(self): + """H2: an existing different hook of the same name is not + clobbered -- nonzero exit naming the conflict; --force replaces. + Both names are planted, because a refusal that protects one name + and overwrites the other still destroys somebody's hook.""" + self.require_sources() + repo = self.make_repo("primary-clone") + hooks = self.hooks_dir(repo) + sentinels = {name: self.plant_foreign_hook(hooks, name) for name in HOOK_NAMES} + proc = self.install(repo, expect=None) + self.assertNotEqual( + proc.returncode, + 0, + "install over existing different hooks must exit nonzero\n" + f"stdout: {proc.stdout}\nstderr: {proc.stderr}", + ) + said = proc.stdout + proc.stderr + self.assertIn( + "post-checkout", + said, + "the conflict message does not name the conflicting hook", + ) + for name in HOOK_NAMES: + self.assertEqual( + (hooks / name).read_bytes(), + sentinels[name], + f"existing {name} was clobbered by a non-forced install " + "that had already refused to proceed", + ) + self.install(repo, "--force") # exit 0 asserted by install() + for src in HOOK_SOURCES: + self.assert_installed_hook(hooks, src, "after the forced install") + + def test_h2_each_hook_name_is_protected_individually(self): + """H2, per name: whichever single hook is already present and + different, the non-forced install refuses, names THAT hook, and + leaves it byte-for-byte intact.""" + self.require_sources() + for name in HOOK_NAMES: + with self.subTest(hook=name): + repo = self.make_repo(f"clone-{name}") + hooks = self.hooks_dir(repo) + sentinel = self.plant_foreign_hook(hooks, name) + proc = self.install(repo, expect=None) + self.assertNotEqual( + proc.returncode, + 0, + f"install over an existing different {name} must exit " + f"nonzero\nstdout: {proc.stdout}\nstderr: {proc.stderr}", + ) + self.assertIn( + name, + proc.stdout + proc.stderr, + f"the conflict message does not name {name}", + ) + self.assertEqual( + (hooks / name).read_bytes(), + sentinel, + f"existing {name} was clobbered by a non-forced install", + ) + + def test_h2_force_replacing_a_symlinked_hook_stays_inside_the_hooks_dir(self): + """H2 containment: --force replaces the hook IN the hooks dir. A + hook that happens to be a symlink pointing elsewhere must not turn + the install into a write to that other file: the replacement lands + as a regular file in the hooks dir and the symlink's target + outside it is untouched.""" + self.require_sources() + repo = self.make_repo("primary-clone") + hooks = self.hooks_dir(repo) + hooks.mkdir(parents=True, exist_ok=True) + outside_dir = self.tmp / "outside" + outside_dir.mkdir() + outside = outside_dir / "foreign-target" + sentinel = b"#!/bin/sh\n# somebody else's file, outside the hooks dir\nexit 0\n" + outside.write_bytes(sentinel) + outside.chmod(0o755) + link = hooks / "post-checkout" + os.symlink(str(outside), str(link)) + # Prove the plant landed and is still recognisably itself. + self.assertTrue(link.is_symlink(), "symlinked-hook plant did not land") + self.assertEqual( + os.path.realpath(str(link)), + os.path.realpath(str(outside)), + "planted symlink does not point at the outside target", + ) + self.assertEqual( + outside.read_bytes(), sentinel, "outside target was not planted intact" + ) + self.assertNotEqual( + sentinel, + POST_CHECKOUT_SRC.read_bytes(), + "outside target is byte-identical to the packaged hook; this " + "test cannot discriminate", + ) + self.install(repo, "--force") # exit 0 asserted by install() + self.assertEqual( + outside.read_bytes(), + sentinel, + f"--force wrote through the symlink and overwrote {outside}, a " + "file outside the hooks dir that install.sh was never asked to " + "touch", + ) + self.assertFalse( + link.is_symlink(), + "the hooks dir still holds a symlink after --force; the " + "replacement did not land in the hooks dir", + ) + self.assert_installed_hook( + hooks, POST_CHECKOUT_SRC, "after --force over a symlinked hook" + ) + + def test_h3_checkout_appends_one_branch_entry(self): + """Scenario: a branch checkout is recorded with its actor and worktree + + H3: after install, git checkout appends exactly + one valid JSON line: kind 'branch', what naming the branch, agent + == $WF_AGENT, via 'git-hook', correct worktree.""" + self.require_sources() + repo = self.make_repo("primary-clone") + self.git(repo, "branch", "beta") + self.install(repo) + before = self.stream_lines(repo) + self.git(repo, "checkout", "-q", "beta", env=self.env_for(wf_agent=WF)) + entry = self.sole_new_entry(before, repo) + self.assertEqual(entry["kind"], "branch", f"kind is not 'branch': {entry!r}") + self.assertIn( + "beta", + entry["what"], + f"what does not name the branch switched to: {entry['what']!r}", + ) + self.assertEqual(entry["agent"], WF, f"agent is not $WF_AGENT: {entry!r}") + self.assertEqual( + entry["worktree"], + "primary-clone", + f"worktree is not the basename of the toplevel: {entry!r}", + ) + + def test_h3_entries_stay_valid_json_for_a_quoting_agent(self): + """H3/H4 schema: the stream is JSONL, so a WF_AGENT carrying a + double quote and a backslash -- both legal in a name -- must come + back out of json.loads as exactly that string, from a checkout + entry and from a commit entry alike.""" + self.require_sources() + repo = self.make_repo("primary-clone") + self.git(repo, "branch", "beta") + self.install(repo) + self.assertIn('"', TRICKY_WF, "fixture agent has no double quote") + self.assertIn("\\", TRICKY_WF, "fixture agent has no backslash") + env = self.env_for(wf_agent=TRICKY_WF) + before = self.stream_lines(repo) + self.git(repo, "checkout", "-q", "beta", env=env) + entry = self.sole_new_entry(before, repo) + self.assertEqual( + entry["agent"], + TRICKY_WF, + "checkout entry did not round-trip the agent string through " + f"JSON: {entry!r}", + ) + self.assertEqual(entry["kind"], "branch", f"kind is not 'branch': {entry!r}") + before = self.stream_lines(repo) + self.commit_change(repo, "escaped\n", "c-escaped", env=env) + entry = self.sole_new_entry(before, repo) + self.assertEqual( + entry["agent"], + TRICKY_WF, + f"commit entry did not round-trip the agent string through JSON: {entry!r}", + ) + self.assertEqual(entry["kind"], "head", f"kind is not 'head': {entry!r}") + + def test_h4_commit_appends_head_entry_with_new_short_sha(self): + """H4: git commit APPENDS kind 'head' with the new short SHA in + what -- earlier records written by other actors survive it.""" + self.require_sources() + repo = self.make_repo("primary-clone") + self.install(repo) + planted = self.plant_stream_lines(repo) + before = self.stream_lines(repo) + self.assertEqual( + before, + planted, + "the stream does not hold the planted earlier records; an " + "append could not be told from an overwrite", + ) + short = self.commit_change(repo, "two\n", "c2") + entry = self.sole_new_entry(before, repo) + self.assertEqual(entry["kind"], "head", f"kind is not 'head': {entry!r}") + self.assertIn( + short, + entry["what"], + f"what does not contain the new short SHA {short}: {entry['what']!r}", + ) + self.assertEqual( + entry["worktree"], + "primary-clone", + f"worktree is not the basename of the toplevel: {entry!r}", + ) + after = self.stream_lines(repo) + self.assertEqual( + after[: len(planted)], + planted, + f"the commit did not append: earlier shared history is gone " + f"(planted={planted!r}, after={after!r})", + ) + + def test_h5_unwritable_stream_never_breaks_git(self): + """Scenario: a hook failure never breaks git + + H5: with the stream file unwritable, checkout and commit still + exit 0 -- the record is lost, the operation is not. Lost means + lost: the obstruction is still standing afterwards and no stream + file was conjured up anywhere else.""" + self.require_sources() + repo = self.make_repo("primary-clone") + self.git(repo, "branch", "beta") + self.install(repo) + stream = self.stream_path(repo) + if stream.is_file(): + stream.unlink() + # A directory at the stream path defeats appending for every uid, + # root included (chmod would not stop root). + stream.mkdir() + self.assertTrue(stream.is_dir(), "unwritable-stream plant did not land") + self.assertEqual( + list(stream.iterdir()), [], "unwritable-stream plant is not empty" + ) + # exit 0 asserted by git(); a failing post-checkout hook becomes + # git checkout's own exit status, so this line is the pin. + self.git(repo, "checkout", "-q", "beta") + on = self.git(repo, "rev-parse", "--abbrev-ref", "HEAD").stdout.strip() + self.assertEqual(on, "beta", "checkout did not actually land on beta") + old = self.git(repo, "rev-parse", "HEAD").stdout.strip() + self.commit_change(repo, "three\n", "c3") + new = self.git(repo, "rev-parse", "HEAD").stdout.strip() + self.assertTrue(old and new, "could not resolve HEAD around the commit") + self.assertNotEqual(new, old, "commit exited 0 but did not create a new commit") + # The obstruction is the user's: a hook may not clear it to get + # its record written. + self.assertTrue( + stream.is_dir(), + f"the obstruction at {stream} did not survive; a hook removed " + "what it could not write to", + ) + self.assertEqual( + list(stream.iterdir()), + [], + f"something was written inside the obstruction at {stream}", + ) + strays = sorted(str(p) for p in self.tmp.rglob(STREAM_NAME) if p.is_file()) + self.assertEqual( + strays, + [], + f"the record was not lost: a stream file materialised at {strays}", + ) + + def test_h6_second_worktree_appends_to_the_shared_stream(self): + """H6: a checkout inside a second git worktree of the same clone + appends to the SAME stream file, with that worktree's own + worktree value.""" + self.require_sources() + repo = self.make_repo("primary-clone") + self.git(repo, "branch", "b-linked") + self.git(repo, "branch", "b-target") + self.install(repo) + wt2 = self.tmp / "second-tree" + self.git(repo, "worktree", "add", str(wt2), "b-linked") + before = self.stream_lines(repo) # snapshot AFTER worktree add's checkout + self.git(wt2, "checkout", "-q", "b-target", env=self.env_for(wf_agent=WF)) + # Read through the PRIMARY clone: growth here IS the shared-stream pin. + entry = self.sole_new_entry(before, repo) + self.assertEqual(entry["kind"], "branch", f"kind is not 'branch': {entry!r}") + self.assertIn( + "b-target", + entry["what"], + f"what does not name the branch switched to: {entry['what']!r}", + ) + self.assertEqual( + entry["worktree"], + "second-tree", + f"entry does not carry the second worktree's own name: {entry!r}", + ) + per_wt = self.common_dir(repo) / "worktrees" / "second-tree" / STREAM_NAME + self.assertFalse( + per_wt.exists(), + "a stream file appeared in the per-worktree gitdir; the stream " + "must be shared at the common dir only", + ) + + def test_h7_agent_falls_back_to_user_then_unknown(self): + """Scenario: an unset WF_AGENT falls back to the system user + + H7: with WF_AGENT unset the entry is still written, agent + falling back to $USER, and to 'unknown' when USER is unset too.""" + self.require_sources() + repo = self.make_repo("primary-clone") + self.git(repo, "branch", "beta") + self.git(repo, "branch", "gamma") + self.install(repo) + before = self.stream_lines(repo) + self.git( + repo, + "checkout", + "-q", + "beta", + env=self.env_for(wf_agent=None, user="fallbackuser"), + ) + entry = self.sole_new_entry(before, repo) + self.assertEqual( + entry["agent"], + "fallbackuser", + f"with WF_AGENT unset, agent did not fall back to $USER: {entry!r}", + ) + before = self.stream_lines(repo) + self.git( + repo, + "checkout", + "-q", + "gamma", + env=self.env_for(wf_agent=None, user=None), + ) + entry = self.sole_new_entry(before, repo) + self.assertEqual( + entry["agent"], + "unknown", + f"with WF_AGENT and USER both unset, agent is not 'unknown': {entry!r}", + ) + + def test_h7_commit_agent_falls_back_to_user_then_unknown(self): + """H7 for the other hook: the identity fallback is a property of + every entry, so a commit with WF_AGENT unset records $USER, and + 'unknown' when USER is unset too.""" + self.require_sources() + repo = self.make_repo("primary-clone") + self.install(repo) + before = self.stream_lines(repo) + short = self.commit_change( + repo, + "two\n", + "c2", + env=self.env_for(wf_agent=None, user="fallbackuser"), + ) + entry = self.sole_new_entry(before, repo) + self.assertEqual(entry["kind"], "head", f"kind is not 'head': {entry!r}") + self.assertIn( + short, + entry["what"], + f"what does not contain the new short SHA {short}: {entry['what']!r}", + ) + self.assertEqual( + entry["agent"], + "fallbackuser", + "with WF_AGENT unset, the commit entry's agent did not fall " + f"back to $USER: {entry!r}", + ) + before = self.stream_lines(repo) + self.commit_change( + repo, + "three\n", + "c3", + env=self.env_for(wf_agent=None, user=None), + ) + entry = self.sole_new_entry(before, repo) + self.assertEqual( + entry["agent"], + "unknown", + "with WF_AGENT and USER both unset, the commit entry's agent " + f"is not 'unknown': {entry!r}", + ) + + +class ReviewFindingsContractTest(HookContractTest): + """PR #25 Codex-review findings, pinned red-first on this head.""" + + def test_configured_hookspath_is_refused_not_silently_bypassed(self): + self.require_sources() + repo = self.make_repo("hookspath") + self.git(repo, "config", "core.hooksPath", "custom-hooks") + proc = self.install(repo, expect=1) + self.assertIn( + "core.hooksPath", + proc.stdout + proc.stderr, + "the refusal must name core.hooksPath so the operator knows why", + ) + self.assertFalse( + (self.hooks_dir(repo) / "post-checkout").exists(), + "install wrote hooks git will never run — a successful inert install", + ) + + def test_dangling_symlink_hook_is_not_clobbered_without_force(self): + self.require_sources() + repo = self.make_repo("dangle") + hooks = self.hooks_dir(repo) + hooks.mkdir(parents=True, exist_ok=True) + dest = hooks / "post-checkout" + dest.symlink_to(self.tmp / "gone-target") + self.assertTrue(dest.is_symlink(), "the dangling-symlink plant did not land") + self.assertFalse(dest.exists(), "the symlink plant is not dangling") + self.install(repo, expect=1) + self.assertTrue( + dest.is_symlink() and os.readlink(str(dest)).endswith("gone-target"), + "a non-forced install destroyed a foreign dangling symlink", + ) + self.install(repo, "--force", expect=0) + self.assertTrue( + dest.is_file() and not dest.is_symlink(), + "--force must replace the dangling symlink with the packaged hook", + ) + + def test_control_characters_in_agent_never_break_the_stream(self): + self.require_sources() + repo = self.make_repo("controls") + self.install(repo, expect=0) + agent = "Line\nBreak\tAgent " + self.assertIn("\n", agent, "the control-character plant did not land") + env = self.env_for(wf_agent=agent) + self.git(repo, "checkout", "-q", "-b", "side", env=env) + lines = self.stream_lines(repo) + self.assertEqual( + len(lines), + 1, + "one checkout crossing must yield exactly one stream record; " + f"a raw newline split it: {lines!r}", + ) + entry = self.parse_entry(lines[0]) + for field in ("agent", "worktree"): + self.assertFalse( + any(ord(ch) < 0x20 for ch in entry[field]), + f"raw control characters leaked into the {field} field", + ) + + +class SecondReviewFindingsContractTest(HookContractTest): + """PR #25 second Codex review (head f5824f5), pinned red-first.""" + + def test_explicitly_empty_hookspath_is_also_refused(self): + self.require_sources() + repo = self.make_repo("empty-hookspath") + self.git(repo, "config", "core.hooksPath", "") + self.assertEqual( + self.git(repo, "config", "--get", "core.hooksPath").stdout, + "\n", + "the empty-hooksPath plant did not land", + ) + proc = self.install(repo, expect=1) + self.assertIn( + "core.hooksPath", + proc.stdout + proc.stderr, + "an explicitly empty core.hooksPath must be refused by name", + ) + self.assertFalse( + (self.hooks_dir(repo) / "post-checkout").exists(), + "install wrote hooks git will never run (empty hooksPath)", + ) + + def test_orphan_checkout_crossing_is_recorded(self): + self.require_sources() + repo = self.make_repo("orphan") + self.install(repo, expect=0) + env = self.env_for() + self.git(repo, "checkout", "-q", "--orphan", "fresh-start", env=env) + lines = self.stream_lines(repo) + self.assertEqual( + len(lines), + 1, + "an orphan checkout is a branch crossing and must be recorded", + ) + entry = self.parse_entry(lines[0]) + self.assertIn( + "fresh-start", + entry["what"], + "the crossing record must name the orphan branch", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/hooks/tests/test_pr_feedback.py b/ops/devlane/hooks/tests/test_pr_feedback.py new file mode 100644 index 0000000..c875069 --- /dev/null +++ b/ops/devlane/hooks/tests/test_pr_feedback.py @@ -0,0 +1,655 @@ +"""Tests for pr-feedback.sh, written from PLAN D1 and the origin corpus. + +The origin test (pr-feedback-test.sh) proved --watch notices each of four +surfaces when a stub `gh` serves fixtures through real jq. It never +covered a `gh` that fails, which is why D1 (error printed as `(none)`, +exit 0) survived it. These cases port that corpus and add the failure, +watch-fingerprint, and reviewThreads paging pins. +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import tempfile +import time +import unittest +from pathlib import Path + +import support + +SCRIPT = support.CLAUDE_DIR / "pr-feedback.sh" +STUB_SRC = Path(__file__).resolve().parent / "gh_stub.py" + +HEADINGS = ( + "conversation comments", + "reviews", + "inline review comments", + "reactions on comments", + "unresolved threads", +) + +CONV_BASE = ( + '[{"id":1,"updated_at":"T1","created_at":"2026-08-14T18:00:00Z",' + '"user":{"login":"xormania"},"body":"@codex review"}]' +) +REV_BASE = ( + '[{"id":10,"submitted_at":"2026-08-14T18:09:00Z","state":"COMMENTED",' + '"user":{"login":"bot"},"body":"review"}]' +) +INLINE_BASE = ( + '[{"id":20,"updated_at":"T1","created_at":"2026-08-14T18:09:00Z",' + '"user":{"login":"bot"},"path":"a.py","line":5,"body":"finding"}]' +) +THREADS_EMPTY = ( + '{"data":{"repository":{"pullRequest":{"reviewThreads":{"nodes":[]}}}}}' +) +EMPTY_LIST = "[]" + +CONV_MOVED = ( + '[{"id":1,"updated_at":"T1","created_at":"2026-08-14T18:00:00Z",' + '"user":{"login":"xormania"},"body":"@codex review"},' + '{"id":2,"updated_at":"T2","created_at":"2026-08-14T18:43:00Z",' + '"user":{"login":"bot"},"body":"VERDICT-AS-CONV-UNIQUE"}]' +) +INLINE_MOVED = ( + '[{"id":20,"updated_at":"T1","created_at":"2026-08-14T18:09:00Z",' + '"user":{"login":"bot"},"path":"a.py","line":5,"body":"finding"},' + '{"id":21,"updated_at":"T2","created_at":"2026-08-14T18:44:00Z",' + '"user":{"login":"bot"},"path":"b.py","line":9,' + '"body":"INLINE-FINDING-UNIQUE"}]' +) +REV_MOVED = ( + '[{"id":10,"submitted_at":"2026-08-14T18:09:00Z","state":"COMMENTED",' + '"user":{"login":"bot"},"body":"review"},' + '{"id":11,"submitted_at":"2026-08-14T18:45:00Z","state":"APPROVED",' + '"user":{"login":"human"},"body":"REVIEW-APPROVED-UNIQUE"}]' +) +REACT_MOVED = ( + '[{"content":"+1","user":{"login":"react-bot-unique"}}]' +) +THREADS_MOVED = json.dumps({ + "data": { + "repository": { + "pullRequest": { + "reviewThreads": { + "nodes": [ + { + "isResolved": False, + "path": "thread-moved-unique.py", + "line": 3, + "comments": { + "nodes": [ + { + "body": "THREAD-MOVED-UNIQUE", + "author": {"login": "bot"}, + } + ] + }, + } + ] + } + } + } + } +}) + + +def _kill(proc: subprocess.Popen) -> None: + if proc.poll() is None: + proc.kill() + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + proc.wait(timeout=3) + + +class PrFeedback(unittest.TestCase): + maxDiff = None + + def setUp(self): + support.require_jq() + self.assertTrue( + SCRIPT.is_file(), + f"INVALID: pr-feedback.sh missing: {SCRIPT}", + ) + self.assertTrue( + STUB_SRC.is_file(), + f"INVALID: gh stub source missing: {STUB_SRC}", + ) + self.tmp = Path(tempfile.mkdtemp(prefix="pr-feedback-")) + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.home = self.tmp / "home" + self.home.mkdir() + self.bin = self.tmp / "bin" + self.bin.mkdir() + self.fix = self.tmp / "fix" + self.fix.mkdir() + self.mode_file = self.tmp / "gh-mode" + support.plant_text(self.mode_file, "ok\n", recognisable="ok") + stub_body = STUB_SRC.read_text(encoding="utf-8") + self.stub = support.plant_executable(self.bin / "gh", stub_body) + which = shutil.which("gh", path=str(self.bin) + os.pathsep + + os.environ.get("PATH", "")) + self.assertEqual( + Path(which).resolve(), self.stub.resolve(), + f"INVALID: which gh is {which}, not the stub {self.stub}", + ) + + def _env(self, extra=None): + merged = { + "GH_FIXTURES": str(self.fix), + "GH_MODE_FILE": str(self.mode_file), + } + if extra: + merged.update(extra) + return support.isolated_env( + self.home, extra_path=self.bin, extra=merged, + ) + + def _plant_file(self, name: str, content: str, *, recognisable: str): + path = self.fix / name + support.plant_text(path, content, recognisable=recognisable) + landed = path.read_text(encoding="utf-8") + self.assertEqual(landed, content, f"INVALID: {name} drifted") + return path + + def _baseline(self): + self._plant_file("conv.json", CONV_BASE, recognisable="@codex review") + self._plant_file("reviews.json", REV_BASE, recognisable='"id":10') + self._plant_file( + "inline.json", INLINE_BASE, recognisable='"path":"a.py"', + ) + self._plant_file("react-1.json", EMPTY_LIST, recognisable="[") + self._plant_file("empty.json", EMPTY_LIST, recognisable="[") + self._plant_file( + "threads.json", THREADS_EMPTY, recognisable="reviewThreads", + ) + + def _empty_success(self): + self._plant_file("conv.json", EMPTY_LIST, recognisable="[") + self._plant_file("reviews.json", EMPTY_LIST, recognisable="[") + self._plant_file("inline.json", EMPTY_LIST, recognisable="[") + self._plant_file("react-1.json", EMPTY_LIST, recognisable="[") + self._plant_file("empty.json", EMPTY_LIST, recognisable="[") + self._plant_file( + "threads.json", THREADS_EMPTY, recognisable="reviewThreads", + ) + + def _move(self, name: str, new_content: str, recognisable: str): + path = self.fix / name + self.assertTrue(path.is_file(), f"INVALID: {name} missing before move") + before = path.read_bytes() + self.assertTrue(before, f"INVALID: {name} empty before move") + before_len = len(before) + support.plant_text(path, new_content, recognisable=recognisable) + after = path.read_bytes() + if after == before: + raise support.PlantFailed( + f"INVALID: FIXTURE DID NOT MOVE — {name}" + ) + if len(after) < max(1, before_len // 2): + raise support.PlantFailed( + f"INVALID: FIXTURE CLOBBERED — {name} " + f"({before_len} -> {len(after)} bytes)" + ) + + def _prove_stub_ok(self, env): + conv = support.run_cmd( + ["gh", "api", "repos/o/r/issues/13/comments", "--jq", "length"], + self.tmp, env, expect=0, + ) + self.assertEqual( + conv.stdout.strip(), "1", + f"INVALID: stub conv length {conv.stdout!r}", + ) + inline = support.run_cmd( + ["gh", "api", "repos/o/r/pulls/13/comments", "--jq", "length"], + self.tmp, env, expect=0, + ) + self.assertEqual( + inline.stdout.strip(), "1", + f"INVALID: stub inline length {inline.stdout!r} " + "(did not distinguish inline from conversation)", + ) + paged = support.run_cmd( + ["gh", "api", + "repos/o/r/issues/13/comments?per_page=100", "--jq", "length"], + self.tmp, env, expect=0, + ) + self.assertEqual( + paged.stdout.strip(), "1", + f"INVALID: stub ignored ?per_page=100: {paged.stdout!r}", + ) + + def _prove_stub_fails(self, env, *, rc=1, stderr_snip="authentication"): + proc = support.run_cmd( + ["gh", "api", "repos/o/r/issues/13/comments"], + self.tmp, env, expect=None, + ) + self.assertEqual( + proc.returncode, rc, + f"INVALID: stub fail rc {proc.returncode}, wanted {rc}", + ) + if stderr_snip: + self.assertIn( + stderr_snip, proc.stderr, + f"INVALID: stub stderr {proc.stderr!r}", + ) + else: + self.assertEqual( + proc.stderr.strip(), "", + f"INVALID: stub stderr not empty: {proc.stderr!r}", + ) + + def _report(self, env, *args): + argv = ["bash", str(SCRIPT), "13", "o/r", *args] + return support.run_cmd(argv, self.tmp, env, expect=None) + + def _watch(self, env, *, interval=1, timeout=8, mover=None): + argv = [ + "bash", str(SCRIPT), "13", "o/r", + "--watch", "--interval", str(interval), "--timeout", str(timeout), + ] + proc = subprocess.Popen( + argv, + cwd=str(self.tmp), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + self.addCleanup(_kill, proc) + if mover is not None: + time.sleep(2) + mover() + try: + out, _ = proc.communicate(timeout=timeout + 5) + except subprocess.TimeoutExpired: + _kill(proc) + out = proc.stdout.read() if proc.stdout else "" + self.fail( + f"watch did not exit within {timeout + 5}s; out={out!r}" + ) + return proc.returncode, out or "" + + def _combined(self, proc): + return (proc.stdout or "") + (proc.stderr or "") + + def _section(self, out: str, heading: str) -> str: + idx = out.lower().find(heading.lower()) + self.assertGreaterEqual( + idx, 0, f"heading {heading!r} missing from:\n{out}", + ) + rest = out[idx + len(heading):] + next_idx = len(rest) + for other in HEADINGS: + if other.lower() == heading.lower(): + continue + found = rest.lower().find(other.lower()) + if 0 <= found < next_idx: + next_idx = found + return rest[:next_idx] + + # --- origin corpus: the stub itself -------------------------------- + + def test_stub_serves_conversation_and_distinguishes_inline(self): + self._baseline() + env = self._env() + self._prove_stub_ok(env) + + # --- origin corpus: one-shot report -------------------------------- + + def test_one_shot_lists_the_five_surfaces(self): + self._baseline() + env = self._env() + self._prove_stub_ok(env) + proc = self._report(env) + out = self._combined(proc) + for heading in HEADINGS: + with self.subTest(heading=heading): + self.assertIn( + heading, out.lower(), + f"report omitted {heading!r}:\n{out}", + ) + self.assertIn( + "@codex review", out, + f"INVALID: baseline conversation never reached the report:\n{out}", + ) + + def test_one_shot_empty_is_none_and_exit_0(self): + self._empty_success() + env = self._env() + proc = support.run_cmd( + ["gh", "api", "repos/o/r/issues/13/comments", "--jq", "length"], + self.tmp, env, expect=0, + ) + self.assertEqual( + proc.stdout.strip(), "0", + f"INVALID: empty fixture length {proc.stdout!r}", + ) + report = self._report(env) + out = self._combined(report) + self.assertEqual( + report.returncode, 0, + f"empty success wanted rc 0, got {report.returncode}\n{out}", + ) + for heading in ( + "conversation comments", + "reviews", + "inline review comments", + "reactions on comments", + ): + body = self._section(out, heading) + self.assertIn( + "(none)", body, + f"{heading} empty body was not (none): {body!r}", + ) + self.assertNotIn("UNREADABLE", body) + + # --- origin corpus: --watch notices each surface ------------------- + + def _assert_watch_notices(self, name, new, unique, label, recognisable): + self._baseline() + env = self._env() + self._prove_stub_ok(env) + + def mover(): + self._move(name, new, recognisable) + + rc, out = self._watch(env, interval=1, timeout=10, mover=mover) + self.assertEqual( + rc, 0, + f"--watch did not notice {label} (rc={rc}):\n{out}", + ) + named = re.search( + rf"changed after[^\n]*{re.escape(label)}", out, re.IGNORECASE, + ) + self.assertTrue( + unique in out or named, + f"--watch fired but did not name {label!r} or " + f"include {unique!r}:\n{out}", + ) + + def test_watch_notices_conversation_comment(self): + self._assert_watch_notices( + "conv.json", CONV_MOVED, "VERDICT-AS-CONV-UNIQUE", + "conv", "VERDICT-AS-CONV-UNIQUE", + ) + + def test_watch_notices_inline_comment(self): + self._assert_watch_notices( + "inline.json", INLINE_MOVED, "INLINE-FINDING-UNIQUE", + "inline", "INLINE-FINDING-UNIQUE", + ) + + def test_watch_notices_review_body(self): + self._assert_watch_notices( + "reviews.json", REV_MOVED, "REVIEW-APPROVED-UNIQUE", + "rev", "REVIEW-APPROVED-UNIQUE", + ) + + def test_watch_notices_reaction(self): + self._assert_watch_notices( + "react-1.json", REACT_MOVED, "react-bot-unique", + "react", "react-bot-unique", + ) + + def test_watch_notices_unresolved_thread(self): + """Fifth surface. Origin grepped four endpoint names and omitted it.""" + self._assert_watch_notices( + "threads.json", THREADS_MOVED, "thread-moved-unique.py", + "thread", "THREAD-MOVED-UNIQUE", + ) + + def test_watch_times_out_without_change_exit_1(self): + self._baseline() + env = self._env() + self._prove_stub_ok(env) + before = (self.fix / "conv.json").read_bytes() + rc, out = self._watch(env, interval=1, timeout=3, mover=None) + self.assertEqual( + rc, 1, + f"silent watch wanted rc 1, got {rc}:\n{out}", + ) + self.assertNotRegex(out, r"changed after") + after = (self.fix / "conv.json").read_bytes() + self.assertEqual( + after, before, + "INVALID: watching mutated the fixtures", + ) + + def test_watch_does_not_mutate_fixtures(self): + self._baseline() + env = self._env() + checksums = { + n: (self.fix / n).read_bytes() + for n in ( + "conv.json", "reviews.json", "inline.json", + "react-1.json", "threads.json", + ) + } + self._report(env) + rc, _ = self._watch(env, interval=1, timeout=3, mover=None) + self.assertEqual(rc, 1) + for name, before in checksums.items(): + landed = (self.fix / name).read_bytes() + self.assertEqual( + landed, before, + f"watching/report mutated {name}", + ) + + # --- D1: a failed gh is not an empty PR ---------------------------- + + def test_failing_gh_exits_nonzero(self): + self._empty_success() + env = self._env(extra={"GH_FAIL": "1"}) + self._prove_stub_fails(env) + proc = self._report(env) + out = self._combined(proc) + self.assertNotEqual( + proc.returncode, 0, + f"failed gh reported as success rc=0:\n{out}", + ) + + def test_failing_gh_marks_surfaces_unreadable_not_none(self): + self._empty_success() + env = self._env(extra={"GH_FAIL": "1"}) + self._prove_stub_fails(env) + proc = self._report(env) + out = self._combined(proc) + for heading in ( + "conversation comments", + "reviews", + "inline review comments", + "reactions on comments", + ): + with self.subTest(heading=heading): + body = self._section(out, heading) + self.assertRegex( + body, + r"UNREADABLE", + f"{heading} did not say UNREADABLE:\n{body}", + ) + self.assertRegex( + body, + r"exit 1", + f"{heading} did not name gh exit 1:\n{body}", + ) + self.assertIn( + "authentication failed", body, + f"{heading} dropped the stderr line:\n{body}", + ) + self.assertNotIn( + "(none)", body, + f"{heading} printed (none) for an unread surface:\n{body}", + ) + + def test_failing_gh_output_differs_from_empty(self): + self._empty_success() + empty_env = self._env() + empty = self._report(empty_env) + fail_env = self._env(extra={"GH_FAIL": "1"}) + self._prove_stub_fails(fail_env) + fail = self._report(fail_env) + empty_out = self._combined(empty) + fail_out = self._combined(fail) + self.assertNotEqual( + fail_out, empty_out, + "auth failure and genuinely empty PR were byte-identical", + ) + self.assertEqual(empty.returncode, 0, empty_out) + self.assertNotEqual(fail.returncode, 0, fail_out) + + def test_failing_gh_empty_stderr_is_still_unreadable(self): + self._empty_success() + env = self._env(extra={"GH_FAIL": "1", "GH_FAIL_STDERR": ""}) + self._prove_stub_fails(env, stderr_snip="") + proc = self._report(env) + out = self._combined(proc) + self.assertNotEqual(proc.returncode, 0, out) + self.assertIn("UNREADABLE", out, out) + body = self._section(out, "conversation comments") + self.assertNotIn("(none)", body, body) + + def test_one_surface_failing_keeps_the_others_readable(self): + self._baseline() + env = self._env(extra={"GH_FAIL_ENDPOINT": "/reviews"}) + ok = support.run_cmd( + ["gh", "api", "repos/o/r/issues/13/comments", "--jq", "length"], + self.tmp, env, expect=0, + ) + self.assertEqual(ok.stdout.strip(), "1", "INVALID: conv should work") + bad = support.run_cmd( + ["gh", "api", "repos/o/r/pulls/13/reviews"], + self.tmp, env, expect=None, + ) + self.assertEqual( + bad.returncode, 1, + f"INVALID: reviews endpoint did not fail: {bad.returncode}", + ) + proc = self._report(env) + out = self._combined(proc) + self.assertNotEqual(proc.returncode, 0, out) + conv = self._section(out, "conversation comments") + self.assertIn("@codex review", conv, conv) + self.assertNotIn("UNREADABLE", conv, conv) + reviews = self._section(out, "reviews") + self.assertIn("UNREADABLE", reviews, reviews) + self.assertNotIn("(none)", reviews, reviews) + + # --- D1: --watch must not fingerprint a failed call ---------------- + + def test_watch_does_not_treat_failed_poll_as_a_change(self): + self._baseline() + env = self._env() + self._prove_stub_ok(env) + + def flip_to_fail(): + calls = self.fix / "callcount" + self.assertTrue( + calls.is_file() and int(calls.read_text().strip() or "0") > 0, + "INVALID: watch never called gh before the failure plant", + ) + support.plant_text(self.mode_file, "fail\n", recognisable="fail") + landed = self.mode_file.read_text(encoding="utf-8").strip() + self.assertEqual(landed, "fail", "INVALID: mode did not flip") + self._prove_stub_fails(env) + + rc, out = self._watch(env, interval=1, timeout=6, mover=flip_to_fail) + self.assertNotEqual( + rc, 0, + f"failed poll exited 0 as if something changed:\n{out}", + ) + self.assertNotRegex( + out, r"changed after", + f"failed poll was reported as a change:\n{out}", + ) + + def test_watch_failed_poll_exit_is_not_success_or_timeout(self): + """Watch timeout is 1; an unread poll must use a different nonzero.""" + self._baseline() + env = self._env() + self._prove_stub_ok(env) + + def flip_to_fail(): + support.plant_text(self.mode_file, "fail\n", recognisable="fail") + self._prove_stub_fails(env) + + rc, out = self._watch(env, interval=1, timeout=6, mover=flip_to_fail) + self.assertNotIn( + rc, (0, 1), + f"failed poll rc={rc} collides with success(0) or " + f"timeout(1):\n{out}", + ) + self.assertRegex( + out, r"UNREADABLE|error|fail", + f"failed poll was silent about the error:\n{out}", + ) + + def test_watch_failed_poll_on_empty_baseline_is_not_timeout(self): + self._empty_success() + env = self._env(extra={"GH_FAIL": "1"}) + self._prove_stub_fails(env) + rc, out = self._watch(env, interval=1, timeout=4, mover=None) + self.assertNotEqual(rc, 0, out) + self.assertNotEqual( + rc, 1, + f"failed empty baseline looked like a quiet timeout:\n{out}", + ) + self.assertNotRegex(out, r"changed after") + + # --- D1: reviewThreads must not silently stop at first:50 ---------- + + def _assert_thread_visible(self, env, n, token): + proc = self._report(env) + out = self._combined(proc) + self.assertIn( + token, out, + f"unresolved thread {n} ({token}) was dropped:\n{out}", + ) + + def test_unresolved_thread_past_first_50_is_shown(self): + self._empty_success() + env = self._env(extra={"GH_THREAD_COUNT": "51"}) + sample = support.run_cmd( + ["gh", "api", "graphql", "-f", + "query=reviewThreads(first: 50)", "--jq", + ".data.repository.pullRequest.reviewThreads.nodes | length"], + self.tmp, env, expect=0, + ) + self.assertEqual( + sample.stdout.strip(), "50", + f"INVALID: first:50 page length {sample.stdout!r}", + ) + page2 = support.run_cmd( + ["gh", "api", "graphql", "-f", + "query=reviewThreads(first: 50)", "-f", "threadCursor=c50", + "--jq", + ".data.repository.pullRequest.reviewThreads.nodes[0].path"], + self.tmp, env, expect=0, + ) + self.assertIn( + "thread-51-unique.py", page2.stdout, + f"INVALID: page 2 did not serve thread 51: {page2.stdout!r}", + ) + self._assert_thread_visible(env, 51, "thread-51-unique.py") + + def test_unresolved_thread_past_first_100_is_shown(self): + """first:100 without paging is the pr-overview mutant, copied here.""" + self._empty_success() + env = self._env(extra={"GH_THREAD_COUNT": "101"}) + sample = support.run_cmd( + ["gh", "api", "graphql", "-f", + "query=reviewThreads(first: 100)", "--jq", + ".data.repository.pullRequest.reviewThreads.nodes | length"], + self.tmp, env, expect=0, + ) + self.assertEqual( + sample.stdout.strip(), "100", + f"INVALID: first:100 page length {sample.stdout!r}", + ) + self._assert_thread_visible(env, 101, "thread-101-unique.py") diff --git a/ops/devlane/hooks/tests/test_ruff_after_edit.py b/ops/devlane/hooks/tests/test_ruff_after_edit.py new file mode 100644 index 0000000..cb68e9a --- /dev/null +++ b/ops/devlane/hooks/tests/test_ruff_after_edit.py @@ -0,0 +1,315 @@ +"""Moved ruff-after-edit.sh corpus, plus D4 as a stub ruff on PATH. + +The in-file --test staged files inside the real worktree and needed a +runnable ruff. These cases use a throwaway git repo and a stub whose +answers are proved before the hook is asked anything. +""" + +import json +import os +import shutil +import tempfile +import unittest +from pathlib import Path + +import support + +HOOK = support.CLAUDE_DIR / "ruff-after-edit.sh" + +# A stub that answers per-file, matching the moved corpus's three plants: +# bad.py is a lint finding, good.py is clean, fmt.py is format-dirty / lint-clean. +CORPUS_STUB = """#!/bin/sh +set -eu +if [ "${1:-}" = "--version" ]; then + echo "ruff 0.0.0-stub" + exit 0 +fi +cmd="${1:-}" +shift || true +file="" +for a in "$@"; do + case "$a" in + --quiet|--check) ;; + *) file="$a" ;; + esac +done +base=$(basename "$file") +case "$cmd" in + format) + if [ "$base" = "fmt.py" ]; then exit 1; fi + exit 0 + ;; + check) + if [ "$base" = "bad.py" ]; then + printf '%s\\n' "${file}:1:1: F401 unused import" + exit 1 + fi + exit 0 + ;; + *) + exit 0 + ;; +esac +""" + +# D4: --version ok, format --check rc 2, check rc 0 empty. +D4_FORMAT_RC2_STUB = """#!/bin/sh +if [ "${1:-}" = "--version" ]; then echo "ruff 0.0.0-stub"; exit 0; fi +if [ "${1:-}" = "format" ]; then exit 2; fi +if [ "${1:-}" = "check" ]; then exit 0; fi +exit 0 +""" + +# D4 if the format branch is gone: check rc 2 and empty stdout → silence. +D4_CHECK_RC2_STUB = """#!/bin/sh +if [ "${1:-}" = "--version" ]; then echo "ruff 0.0.0-stub"; exit 0; fi +if [ "${1:-}" = "format" ]; then exit 0; fi +if [ "${1:-}" = "check" ]; then exit 2; fi +exit 0 +""" + + +class RuffAfterEdit(unittest.TestCase): + def setUp(self): + support.require_jq() + self.assertTrue(HOOK.is_file(), f"INVALID: hook missing: {HOOK}") + self.tmp = Path(tempfile.mkdtemp(prefix="ruff-hook-")) + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.home = self.tmp / "home" + self.home.mkdir() + self.bin = self.tmp / "bin" + self.bin.mkdir() + self.repo = self.tmp / "repo" + + def _env(self): + return support.isolated_env(self.home, extra_path=self.bin) + + def _git_repo(self, *, ruff_toml: str | None = ""): + self.repo.mkdir() + env = self._env() + support.git(self.repo, env, "-c", "init.defaultBranch=main", "init", "-q") + support.configure_identity(self.repo, env) + if ruff_toml is not None: + support.plant_text( + self.repo / "ruff.toml", ruff_toml, + recognisable=ruff_toml or None, + ) + if ruff_toml == "": + # empty file is still a config file; prove it exists and is empty + landed = (self.repo / "ruff.toml").read_text(encoding="utf-8") + self.assertEqual(landed, "", "INVALID: ruff.toml plant drifted") + return env + + def _stub(self, body: str): + stub = support.plant_executable(self.bin / "ruff", body) + which = shutil.which("ruff", path=str(self.bin) + os.pathsep + os.environ.get("PATH", "")) + self.assertEqual( + Path(which).resolve(), stub.resolve(), + f"INVALID: which ruff is {which}, not the stub {stub}", + ) + return stub + + def _prove_stub(self, env, *, version_rc=0, check_rc=None, check_out=None, + format_rc=None, file="x.py"): + v = support.run_cmd(["ruff", "--version"], self.repo, env, expect=version_rc) + self.assertIn( + "ruff", v.stdout.lower(), + f"INVALID: stub --version did not identify itself: {v.stdout!r}", + ) + if format_rc is not None: + f = support.run_cmd( + ["ruff", "format", "--check", file], self.repo, env, expect=None + ) + self.assertEqual( + f.returncode, format_rc, + f"INVALID: stub format --check rc {f.returncode}, wanted {format_rc}", + ) + if check_rc is not None: + c = support.run_cmd( + ["ruff", "check", "--quiet", file], self.repo, env, expect=None + ) + self.assertEqual( + c.returncode, check_rc, + f"INVALID: stub check rc {c.returncode}, wanted {check_rc}", + ) + if check_out is not None: + self.assertEqual( + c.stdout, check_out, + f"INVALID: stub check stdout {c.stdout!r}, wanted {check_out!r}", + ) + + def _fire(self, env, payload: str): + return support.run_script(HOOK, payload, self.repo, env) + + def _plant_py(self, name: str, body: str, *, recognisable: str): + path = self.repo / name + support.plant_text(path, body, recognisable=recognisable) + self.assertTrue(path.is_file(), f"INVALID: {name} missing") + self.assertTrue(str(path).endswith(".py"), f"INVALID: {name} not .py") + return path + + def test_catches_a_lint_finding(self): + env = self._git_repo() + self._stub(CORPUS_STUB) + path = self._plant_py("bad.py", "import os\n", recognisable="import os") + self._prove_stub( + env, check_rc=1, + check_out=f"{path}:1:1: F401 unused import\n", + file=str(path), + ) + proc = self._fire(env, support.write_payload(file_path=str(path))) + self.assertTrue( + support.has_additional_context(proc.stdout), + f"wanted a lint complaint, got {proc.stdout!r} / {proc.stderr!r}", + ) + + def test_stays_quiet_on_a_clean_py(self): + env = self._git_repo() + self._stub(CORPUS_STUB) + path = self._plant_py("good.py", "x = 1\n", recognisable="x = 1") + self._prove_stub(env, check_rc=0, check_out="", file=str(path)) + proc = self._fire(env, support.write_payload(file_path=str(path))) + self.assertFalse( + support.has_additional_context(proc.stdout), + f"nagged a clean file: {proc.stdout!r}", + ) + + def test_stays_quiet_on_format_only(self): + env = self._git_repo() + self._stub(CORPUS_STUB) + path = self._plant_py( + "fmt.py", 'x = { "a":1 }\n', recognisable='"a":1' + ) + self._prove_stub( + env, format_rc=1, check_rc=0, check_out="", file=str(path) + ) + proc = self._fire(env, support.write_payload(file_path=str(path))) + self.assertFalse( + support.has_additional_context(proc.stdout), + f"format-only was treated as a failure: {proc.stdout!r}", + ) + + def test_reads_serena_relative_path(self): + env = self._git_repo() + self._stub(CORPUS_STUB) + path = self._plant_py("bad.py", "import os\n", recognisable="import os") + self._prove_stub( + env, check_rc=1, + check_out=f"{path}:1:1: F401 unused import\n", + file=str(path), + ) + proc = self._fire( + env, + support.write_payload( + relative_path="bad.py", cwd=str(self.repo) + ), + ) + self.assertTrue( + support.has_additional_context(proc.stdout), + f"relative_path was not read: {proc.stdout!r}", + ) + self.assertTrue(path.is_file(), "INVALID: relative_path target vanished") + + def test_ignores_a_non_py_file(self): + env = self._git_repo() + self._stub(CORPUS_STUB) + path = self.repo / "pyproject.toml" + support.plant_text(path, "[project]\nname='x'\n", recognisable="name=") + self.assertFalse(str(path).endswith(".py")) + proc = self._fire(env, support.write_payload(file_path=str(path))) + self.assertFalse( + support.has_additional_context(proc.stdout), + f"nagged a non-.py file: {proc.stdout!r}", + ) + + def test_ignores_a_file_that_is_not_there(self): + env = self._git_repo() + self._stub(CORPUS_STUB) + gone = self.repo / "gone.py" + self.assertFalse(gone.exists(), "INVALID: gone.py exists") + proc = self._fire(env, support.write_payload(file_path=str(gone))) + self.assertFalse( + support.has_additional_context(proc.stdout), + f"nagged a missing file: {proc.stdout!r}", + ) + + def test_ignores_a_payload_with_no_path(self): + env = self._git_repo() + self._stub(CORPUS_STUB) + proc = self._fire(env, json.dumps({"tool_input": {}})) + self.assertFalse( + support.has_additional_context(proc.stdout), + f"nagged an empty payload: {proc.stdout!r}", + ) + + def test_ignores_junk_input(self): + env = self._git_repo() + self._stub(CORPUS_STUB) + proc = self._fire(env, "not json at all") + self.assertFalse( + support.has_additional_context(proc.stdout), + f"nagged junk input: {proc.stdout!r}", + ) + + def test_ignores_empty_input(self): + env = self._git_repo() + self._stub(CORPUS_STUB) + proc = self._fire(env, "") + self.assertFalse( + support.has_additional_context(proc.stdout), + f"nagged empty input: {proc.stdout!r}", + ) + + def test_noop_where_ruff_is_not_configured(self): + env = self._git_repo(ruff_toml=None) + self._stub(CORPUS_STUB) + path = self._plant_py( + "bad.py", 'x = { "a":1 }\n', recognisable='"a":1' + ) + self.assertFalse( + (self.repo / "ruff.toml").exists(), + "INVALID: ruff.toml present in the unconfigured case", + ) + self._prove_stub( + env, check_rc=1, + check_out=f"{path}:1:1: F401 unused import\n", + file=str(path), + ) + proc = self._fire(env, support.write_payload(file_path=str(path))) + self.assertFalse( + support.has_additional_context(proc.stdout), + f"nagged an unconfigured repo: {proc.stdout!r}", + ) + + def test_d4_format_check_rc_2_is_not_a_formatting_violation(self): + """Stub: --version ok, format --check rc 2, check rc 0 → no output.""" + env = self._git_repo() + self._stub(D4_FORMAT_RC2_STUB) + path = self._plant_py("x.py", "x = 1\n", recognisable="x = 1") + self._prove_stub( + env, format_rc=2, check_rc=0, check_out="", file=str(path) + ) + proc = self._fire(env, support.write_payload(file_path=str(path))) + self.assertFalse( + support.has_additional_context(proc.stdout), + f"D4: format rc 2 was reported as a formatting violation: " + f"{proc.stdout!r}", + ) + self.assertNotIn( + "formatting", proc.stdout.lower(), + f"D4: output mentioned formatting: {proc.stdout!r}", + ) + + def test_d4_check_rc_2_with_empty_stdout_is_silence(self): + """If the format branch is gone: check rc 2 + empty stdout → silence.""" + env = self._git_repo() + self._stub(D4_CHECK_RC2_STUB) + path = self._plant_py("x.py", "x = 1\n", recognisable="x = 1") + self._prove_stub( + env, check_rc=2, check_out="", file=str(path) + ) + proc = self._fire(env, support.write_payload(file_path=str(path))) + self.assertFalse( + support.has_additional_context(proc.stdout), + f"D4: check rc 2 empty stdout was not silence: {proc.stdout!r}", + ) diff --git a/ops/devlane/hooks/tests/test_test_guard.py b/ops/devlane/hooks/tests/test_test_guard.py new file mode 100644 index 0000000..8f44ba3 --- /dev/null +++ b/ops/devlane/hooks/tests/test_test_guard.py @@ -0,0 +1,121 @@ +"""Moved corpus for test-guard.py: each case is a file that must or must not flag. + +The case bodies are the two real bugs this tool exists to catch, and the +files that must stay quiet. They live here as triple-quoted data so +scanning THIS file does not treat the examples as plants. +""" + +import shutil +import tempfile +import unittest +from pathlib import Path + +import support + +# Copied from test-guard.py's in-file table. 10 rows: 4 that must flag, +# 6 that must not. +CASES = [ + ("pr-body-check-test.sh", """ +clean; sed -i 's/^## Checklist$/## Checklis/' "$TMP/body.md" +[ "$(fired checklist)" != "0" ] && ok "catches it" || bad "catches it" +""", True), + + ("body-test.sh", """ +clean +python3 - "$F" <<'PY' +import sys +p = sys.argv[1] +open(p, "wb").write(open(p, "rb").read().replace(b"\\n", b"\\r\\n")) +PY +""", True), + + ("thing_test.py", """ +def test_it(tmp): + p = tmp / "f.md" + p.write_text(p.read_text().replace("good", "bad")) + assert check(p) != 0 +""", True), + + # PROSE DOCUMENTING the anti-pattern is not the anti-pattern. + ("doctrine.md", """ +| too much happened | `open(p,"w").write(open(p,"r").read())` — the write truncates before +the read runs | the fixture is emptied, so the fault isn't there either | +""", False), + + # ...but the same text in something that actually runs is still a finding + ("fixup.sh", """ +python3 -c 'open(p,"w").write(open(p,"r").read())' +""", True), + + # guarded: the fixed harness + ("pr-body-check-test.sh", """ +plant() { + before=$(cksum < "$BODY"); "$1"; after=$(cksum < "$BODY") + [ "$before" = "$after" ] && { bad "PLANT DID NOT APPLY"; return 1; } +} +clean; plant mut_x && sed -i 's/^## Checklist$/## Checklis/' "$BODY" +""", False), + + # selftest.sh's shape: literal whole-file fixtures and appends + ("selftest.sh", """ +printf '[claim]\\nlabel = true\\nrun = echo yes\\n' > cl/ok.txt +for i in 1 2 3; do echo "x$i" >> src/untouched.py; done +""", False), + + # a test that mutates but proves the mutation with a content assertion + ("edit_test.py", """ +def test_edit(tmp): + p = tmp / "f.py" + p.write_text(p.read_text().replace("a", "b")) + assert "b" in p.read_text() + assert check(p) != 0 +""", False), + + # not a test file: a release script rewriting a version is not a plant + ("bump-version.sh", """ +sed -i "s/^version = .*/version = \\"$NEW\\"/" pyproject.toml +""", False), + + # the pattern named in a comment, not executed + ("notes-test.sh", """ +# never write `sed -i` without checking the anchor matched +printf 'literal\\n' > "$F" +""", False), +] + + +class TestGuardCorpus(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.mod = support.load_claude("test-guard.py", "test_guard") + + def setUp(self): + # The scanner treats a path as a test file when the PATH matches + # a test-ish regex. A tempdir named `test-…` would make a + # release script look like a test. The original --test used + # tempfile's `tmp*` prefix, which does not match. + self.tmp = Path(tempfile.mkdtemp(prefix="tg-corpus-")) + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + + def test_the_moved_tables_are_the_tables_that_were_there(self): + self.assertEqual(len(CASES), 10) + must_flag = sum(1 for _, _, w in CASES if w) + must_not = sum(1 for _, _, w in CASES if not w) + self.assertEqual(must_flag, 4) + self.assertEqual(must_not, 6) + + def test_each_case(self): + self.assertGreater(len(CASES), 0) + for i, (name, body, want) in enumerate(CASES): + with self.subTest(name=name, want=want, i=i): + path = self.tmp / f"{i}_{name}" + support.plant_text(path, body, recognisable=body.strip()[:12]) + landed = path.read_text(encoding="utf-8") + self.assertEqual(landed, body, "INVALID: plant did not land") + self.assertNotEqual(landed, "", "INVALID: plant emptied the file") + got = bool(self.mod.scan(path)) + self.assertEqual( + got, want, + f"{'MISSED' if want else 'NOISE'}: {name} " + f"(first line {body.strip().splitlines()[0][:60]!r})", + ) diff --git a/ops/devlane/hooks/tests/test_test_guard_hook.py b/ops/devlane/hooks/tests/test_test_guard_hook.py new file mode 100644 index 0000000..98beb2d --- /dev/null +++ b/ops/devlane/hooks/tests/test_test_guard_hook.py @@ -0,0 +1,160 @@ +"""Origin test-guard-hook-test.sh: the PostToolUse wrapper's never-block contract. + +The scanner itself is already in test_test_guard.py. Origin also tested the +wrapper: a real Write/Edit payload reaches it, a finding comes back as +additionalContext, and — the property that matters more than detection — +it NEVER blocks a tool call, whatever it is handed. This suite did not +have that. +""" + +from __future__ import annotations + +import json +import os +import shutil +import tempfile +import unittest +from pathlib import Path + +import support + +HOOK = support.CLAUDE_DIR / "test-guard-hook.sh" + +JUNK = ( + '{"tool_input":{}}', + "not json at all", + "", + '{"tool_input":{"file_path":null}}', + '{"tool_name":"Write"}', + '{"tool_name":"Write","tool_input":{"file_path":"/no/such/file.sh"}}', + '{"tool_name":"Write","tool_input":{"file_path":"/etc/shadow"}}', +) + +UNGUARDED = '''clean; sed -i "s/^## Checklist$/## Checklis/" "$F" +[ "$(fired checklist)" != "0" ] && ok "catches it" || bad "catches it" +''' + +GUARDED = '''plant() { + before=$(cksum < "$F"); "$1"; after=$(cksum < "$F") + [ "$before" = "$after" ] && { bad "PLANT DID NOT APPLY"; return 1; } +} +clean; plant mut && sed -i "s/a/b/" "$F" +''' + +NOT_A_TEST = '''sed -i "s/^version = .*/version = \\"1.2\\"/" pyproject.toml +''' + + +class TestGuardHookWrapper(unittest.TestCase): + def setUp(self): + self.assertTrue(HOOK.is_file(), f"INVALID: wrapper missing: {HOOK}") + self.assertTrue( + os.access(HOOK, os.X_OK), + f"INVALID: {HOOK} is not executable", + ) + # Prefix must not look like a test path: the scanner matches the path. + self.tmp = Path(tempfile.mkdtemp(prefix="tgh-wrap-")) + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.home = self.tmp / "home" + self.home.mkdir() + self.env = support.isolated_env(self.home) + + def _fire(self, tool: str, path: Path): + payload = json.dumps( + {"tool_name": tool, "tool_input": {"file_path": str(path)}} + ) + return support.run_cmd( + [str(HOOK)], self.tmp, self.env, stdin=payload, expect=None + ) + + def _rc(self, payload: str) -> int: + return support.run_cmd( + [str(HOOK)], self.tmp, self.env, stdin=payload, expect=None + ).returncode + + def test_never_blocks_a_tool_call(self): + self.assertGreater(len(JUNK), 0) + for payload in JUNK: + with self.subTest(payload=payload[:40]): + self.assertEqual( + self._rc(payload), + 0, + f"wrapper blocked on {payload!r}", + ) + + def test_catches_an_unguarded_plant_as_additional_context(self): + path = self.tmp / "thing-test.sh" + support.plant_text(path, UNGUARDED, recognisable="sed -i") + self.assertIn("sed -i", path.read_text(encoding="utf-8")) + proc = self._fire("Write", path) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn( + "unguarded-plant", + proc.stdout, + f"did not catch an unguarded plant: {proc.stdout!r}", + ) + self.assertTrue( + support.has_additional_context(proc.stdout), + f"finding was not additionalContext: {proc.stdout!r}", + ) + json.loads(proc.stdout) + self.assertIn( + "thing-test.sh", + proc.stdout, + f"finding does not name the file: {proc.stdout!r}", + ) + + def test_catches_the_truncating_self_read_in_a_non_test_file(self): + # Assemble the fault so scanning THIS file does not contain it. + path = self.tmp / "helper.py" + opener = 'open(p, "wb")' + reader = 'open(p, "rb").read()' + body = "import sys\np = sys.argv[1]\n" + opener + ".write(" + reader + ")\n" + support.plant_text(path, body, recognisable="import sys") + landed = path.read_text(encoding="utf-8") + needle = opener + ".write(" + reader + ")" + self.assertIn( + needle, + landed, + "INVALID: fixture does not contain the truncating self-read", + ) + self.assertNotEqual(landed, "", "INVALID: fixture was emptied") + proc = self._fire("Edit", path) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn( + "truncating-self-read", + proc.stdout, + f"did not catch the truncating self-read: {proc.stdout!r}", + ) + + def test_stays_quiet_where_it_should(self): + guarded = self.tmp / "ok-test.sh" + support.plant_text(guarded, GUARDED, recognisable="cksum") + self.assertIn("cksum", guarded.read_text(encoding="utf-8")) + not_test = self.tmp / "bump-version.sh" + support.plant_text(not_test, NOT_A_TEST, recognisable="pyproject") + unguarded = self.tmp / "thing-test.sh" + support.plant_text(unguarded, UNGUARDED, recognisable="sed -i") + + g = self._fire("Write", guarded) + self.assertEqual(g.returncode, 0, g.stderr) + self.assertFalse( + support.has_additional_context(g.stdout), + f"nagged a guarded test: {g.stdout!r}", + ) + n = self._fire("Write", not_test) + self.assertEqual(n.returncode, 0, n.stderr) + self.assertFalse( + support.has_additional_context(n.stdout), + f"nagged a release script: {n.stdout!r}", + ) + b = self._fire("Bash", unguarded) + self.assertEqual(b.returncode, 0, b.stderr) + self.assertFalse( + support.has_additional_context(b.stdout), + f"nagged a non-Write/Edit tool: {b.stdout!r}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devlane/hooks/tests/test_unsafe_command.py b/ops/devlane/hooks/tests/test_unsafe_command.py new file mode 100644 index 0000000..5a4d673 --- /dev/null +++ b/ops/devlane/hooks/tests/test_unsafe_command.py @@ -0,0 +1,69 @@ +"""Moved corpus for unsafe-command.py: refuse, stay silent, see an amend.""" + +import unittest + +import corpus +import support + + +class UnsafeCommandCorpus(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.mod = support.load_claude("unsafe-command.py", "unsafe_command") + + def test_the_moved_tables_are_the_tables_that_were_there(self): + self.assertEqual(len(corpus.UNSAFE_BLOCK), corpus.COUNTS["UNSAFE_BLOCK"]) + self.assertEqual(len(corpus.UNSAFE_PASS), corpus.COUNTS["UNSAFE_PASS"]) + self.assertEqual(len(corpus.UNSAFE_AMEND), corpus.COUNTS["UNSAFE_AMEND"]) + self.assertEqual(corpus.COUNTS["UNSAFE_BLOCK"], 22) + self.assertEqual(corpus.COUNTS["UNSAFE_PASS"], 54) + self.assertEqual(corpus.COUNTS["UNSAFE_AMEND"], 15) + + def test_refuses_each_block_row(self): + self.assertGreater(len(corpus.UNSAFE_BLOCK), 0) + for cmd in corpus.UNSAFE_BLOCK: + with self.subTest(cmd=cmd[:70]): + hit = self.mod.check(cmd) + self.assertTrue( + hit, + f"should refuse, stayed silent: {cmd!r}", + ) + + def test_stays_silent_on_each_pass_row(self): + self.assertGreater(len(corpus.UNSAFE_PASS), 0) + for cmd in corpus.UNSAFE_PASS: + with self.subTest(cmd=cmd[:70]): + hit = self.mod.check(cmd) + self.assertFalse( + hit, + f"false refusal {hit!r} on {cmd!r}", + ) + + def test_amend_detection(self): + self.assertGreater(len(corpus.UNSAFE_AMEND), 0) + for cmd, want in corpus.UNSAFE_AMEND: + with self.subTest(cmd=cmd[:70], want=want): + self.assertEqual( + self.mod.is_amend(cmd), want, + f"is_amend({cmd!r}) wanted {want}", + ) + + def test_plan_d3_writing_a_dot_sh_file_is_not_running_it(self): + """cat > x.sh and tee notes.sh were refused because \\bsh\\b matched the name.""" + rows = [ + """cat > setup.sh <<'EOF' +s = p.read_text() +p.write_text(s.replace(old, new)) +EOF""", + """tee notes.sh <<'EOF' +s = p.read_text() +p.write_text(s.replace(old, new)) +EOF""", + ] + for cmd in rows: + with self.subTest(cmd=cmd.splitlines()[0]): + self.assertIn(cmd, corpus.UNSAFE_PASS) + self.assertFalse( + self.mod.check(cmd), + f"writing a .sh file was refused: {cmd!r}", + ) diff --git a/ops/devlane/workflow/checks/ci_contexts.py b/ops/devlane/workflow/checks/ci_contexts.py new file mode 100644 index 0000000..0bc5ac2 --- /dev/null +++ b/ops/devlane/workflow/checks/ci_contexts.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python3 +"""Derive the CI check-run names a ruleset would list. + +The check-run API reports job names, with a matrix value in parentheses +when the job name does not interpolate it. This script prints that list +from the two workflow files plus `wf --json status --checks`. + + wf --json status --checks > checks.json + ci_contexts.py checks.json + ci_contexts.py checks.json --json +""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +# checks/ -> workflow -> app -> .dev -> repo. +# Same root as support.WF_DIR.parent.parent.parent, from __file__. +ROOT = Path(__file__).resolve().parent.parent.parent.parent.parent + +DEV = "ci-dev.yml" +APPS = "ci-dev-apps.yml" + +# Quoted `name:` at job indent. Workflow `name: CI` is unquoted and +# unindented; step names in these files are unquoted or dashed. +JOB_NAME = re.compile(r'(?m)^ name: "([^"]+)"') +JOB_ID_TEXT = r"[A-Za-z_][A-Za-z0-9_-]*" +JOB_ID = re.compile(rf"(?m)^ ({JOB_ID_TEXT}):$") +JOB_HEADER = re.compile( + rf"^ (?:(?P{JOB_ID_TEXT})|" + rf"\"(?P{JOB_ID_TEXT})\"|" + rf"'(?P{JOB_ID_TEXT})'):" + r"\s*(?:#.*)?$" +) +SHARD_LINE = re.compile(r"(?m)^[ \t]+shard:\s*\[([^]]*)\]") +NAME_LINE = re.compile(r'(?m)^ name: "([^"]*)"$') + +# Shape of each known job, keyed by file and job id (`^ :$`). +# The emitted text comes from the file's `name:` line; this table pins +# how that line is expanded, not the string itself. +KNOWN = { + DEV: { + "verify": "static", + "discover": "static", + "gates": "gates", + }, + APPS: { + "workflow": "shards", + "app": "apps", + }, +} + +APP_NAME = "dev: ${{ matrix.app }}" +DISCOVER_MATRIX = "fromJSON(needs.discover.outputs.matrix)" +STATIC_NAME = 'quoted name: with no ${{' + + +class DerivationError(Exception): + """A short list is never a valid answer to a broken input.""" + + def __init__(self, path, job_id, expected, found): + self.path = path + self.job_id = job_id + self.expected = expected + self.found = found + + def __str__(self): + return ( + f"{self.path}: job {self.job_id}: " + f"expected {self.expected}, found {self.found}" + ) + + +def workflow_path(root, filename): + return Path(root) / ".github" / "workflows" / filename + + +def read_workflow(root, filename): + path = workflow_path(root, filename) + try: + return path.read_text(encoding="utf-8") + except FileNotFoundError: + raise DerivationError( + filename, "-", "readable workflow file", "missing" + ) from None + except OSError as err: + raise DerivationError( + filename, "-", "readable workflow file", str(err) + ) from err + + +def jobs_section(text): + parts = re.split(r"(?m)^jobs:\s*$", text, maxsplit=1) + if len(parts) != 2: + return "" + return parts[1] + + +def _job_blocks(text, filename="workflow"): + """Return every job block, or refuse a job shape we cannot parse. + + GitHub accepts mixed-case and underscore job ids, plus YAML-quoted + keys. The old parser only used its narrow match both to inventory jobs + and to find block boundaries, so an accepted-but-unmatched job could be + omitted or swallowed into the preceding known job. Parse every supported + GitHub id spelling before deciding whether the id is known. + """ + section = jobs_section(text) + lines = section.splitlines(keepends=True) + headers = [] + for index, line in enumerate(lines): + raw = line.rstrip("\r\n") + if not raw.strip() or raw.lstrip().startswith("#"): + continue + if raw == raw.lstrip(): + # A later top-level key ends `jobs:`. + lines = lines[:index] + break + if not raw.startswith(" ") or raw.startswith((" ", "\t")): + continue + match = JOB_HEADER.fullmatch(raw) + if not match: + raise DerivationError( + filename, + raw.strip().split(":", 1)[0].strip("'\"") or "-", + "a GitHub job id mapping on its own line", + raw.strip(), + ) + job_id = next(value for value in match.groupdict().values() if value) + headers.append((index, job_id)) + + blocks = {} + for position, (start, job_id) in enumerate(headers): + if job_id in blocks: + raise DerivationError( + filename, job_id, "unique job id", "duplicate" + ) + end = headers[position + 1][0] if position + 1 < len(headers) else len(lines) + blocks[job_id] = "".join(lines[start:end]) + return blocks + + +def job_ids(text, filename="workflow"): + return list(_job_blocks(text, filename)) + + +def job_block(text, job_id, filename="workflow"): + return _job_blocks(text, filename).get(job_id) + + +def job_name(block): + if block is None: + return None + match = NAME_LINE.search(block) + if not match: + return None + return match.group(1) + + +def app_matrix_entries(text): + """(app, cmd) pairs from the apps workflow's include matrix.""" + job = job_block(text, "app", APPS) + if job is None: + return [] + start = job.find("include:") + if start == -1: + return [] + rest = job[start + len("include:"):] + cut = rest.find("\n steps:") + block = rest[:cut] if cut != -1 else rest + entries = [] + app = None + for line in block.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + named = re.fullmatch( + r" - app:\s+([A-Za-z0-9_-]+)", line + ) + if named: + if app is not None: + raise DerivationError( + APPS, "app", f"cmd: for include app {app}", "absent" + ) + app = named.group(1) + continue + command = re.fullmatch(r" cmd:\s+(.+)", line) + if command and app is not None: + entries.append((app, command.group(1).strip())) + app = None + continue + raise DerivationError( + APPS, + "app", + "canonical '- app:' then 'cmd:' include rows", + stripped, + ) + if app is not None: + raise DerivationError( + APPS, "app", f"cmd: for include app {app}", "absent" + ) + return entries + + +def shard_values(text): + """Shard indexes from the workflow job's `shard: [ … ]` line.""" + job = job_block(text, "workflow", APPS) + if job is None: + return None + match = SHARD_LINE.search(job) + if not match: + return None + inner = match.group(1).strip() + if not inner: + return [] + return [item.strip() for item in inner.split(",") if item.strip() != ""] + + +def require_quoted_static(filename, job_id, name): + if name is None: + raise DerivationError(filename, job_id, STATIC_NAME, "absent") + if "${{" in name: + raise DerivationError(filename, job_id, STATIC_NAME, name) + return name + + +def _ci_matrix(payload): + if not isinstance(payload, dict): + raise DerivationError( + "checks.json", + "ci_matrix", + "object whose ci_matrix is a non-empty list of strings", + type(payload).__name__, + ) + matrix = payload.get("ci_matrix") + if ( + not isinstance(matrix, list) + or not matrix + or not all(isinstance(item, str) for item in matrix) + ): + raise DerivationError( + "checks.json", + "ci_matrix", + "non-empty list of strings", + repr(matrix), + ) + return matrix + + +def derive(payload, root=None): + """Return the sorted check-run names, or raise DerivationError.""" + root = Path(root) if root is not None else ROOT + matrix = _ci_matrix(payload) + names = [] + texts = {filename: read_workflow(root, filename) for filename in KNOWN} + for filename, kinds in KNOWN.items(): + text = texts[filename] + found_ids = job_ids(text, filename) + extras = [jid for jid in found_ids if jid not in kinds] + if extras: + raise DerivationError( + filename, + extras[0], + "a job id in the known table", + "not in the table", + ) + for job_id, kind in kinds.items(): + block = job_block(text, job_id, filename) + if block is None: + raise DerivationError( + filename, job_id, f"^ {job_id}:$ line", "absent" + ) + name = job_name(block) + if kind == "static": + names.append(require_quoted_static(filename, job_id, name)) + elif kind == "gates": + static = require_quoted_static(filename, job_id, name) + if DISCOVER_MATRIX not in block: + raise DerivationError( + filename, + job_id, + DISCOVER_MATRIX, + "matrix does not reference discover outputs", + ) + names.extend(f"{static} ({check})" for check in matrix) + elif kind == "shards": + static = require_quoted_static(filename, job_id, name) + shards = shard_values(text) + if shards is None: + raise DerivationError( + filename, + job_id, + "shard: [ … ] list in matrix", + "absent", + ) + if not shards: + raise DerivationError( + filename, job_id, "non-empty shard list", "[]" + ) + names.extend(f"{static} ({shard})" for shard in shards) + elif kind == "apps": + if name != APP_NAME: + raise DerivationError( + filename, + job_id, + f'name: "{APP_NAME}"', + "absent" if name is None else name, + ) + entries = app_matrix_entries(text) + apps = [app for app, _cmd in entries] + if not apps: + raise DerivationError( + filename, + job_id, + "- app: entries in the include matrix", + "none", + ) + dupes = sorted({app for app in apps if apps.count(app) > 1}) + if dupes: + raise DerivationError( + filename, + job_id, + "unique - app: values", + f"duplicate {dupes[0]}", + ) + names.extend( + name.replace("${{ matrix.app }}", app) for app in apps + ) + else: + raise DerivationError(filename, job_id, "known kind", kind) + return sorted(names) + + +def main(argv) -> int: + args = argv[1:] + as_json = False + if len(args) == 2 and args[1] == "--json": + as_json = True + args = args[:1] + if len(args) != 1 or args[0].startswith("-"): + print(__doc__.strip(), file=sys.stderr) + return 64 + try: + with open(args[0], encoding="utf-8") as handle: + payload = json.load(handle) + except json.JSONDecodeError as err: + print( + f"checks.json: job ci_matrix: expected JSON object, found {err}", + file=sys.stderr, + ) + return 2 + except OSError as err: + print( + f"checks.json: job ci_matrix: expected readable file, found {err}", + file=sys.stderr, + ) + return 2 + try: + names = derive(payload) + except DerivationError as err: + print(err, file=sys.stderr) + return 2 + if as_json: + json.dump(names, sys.stdout) + sys.stdout.write("\n") + else: + sys.stdout.write("".join(f"{name}\n" for name in names)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/ops/devlane/workflow/checks/ci_matrix.py b/ops/devlane/workflow/checks/ci_matrix.py new file mode 100755 index 0000000..bc77f0a --- /dev/null +++ b/ops/devlane/workflow/checks/ci_matrix.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Turn `wf status --checks --json` into GitHub Actions matrix outputs. + +PLAN §7 says the matrix should be derived from that command. This is the +one-line transformation that does it, kept as a script rather than inlined in +YAML so it can be tested — a shell one-liner in a workflow file is the part +nobody can run locally and nobody notices breaking. + + wf --json status --checks > checks.json + ci_matrix.py checks.json >> "$GITHUB_OUTPUT" + +Writes `matrix=` and `any=yes|no`. The `any` flag exists because a +matrix job with an empty matrix is a hard error in Actions, not a skip. +""" + +from __future__ import annotations + +import json +import sys + + +def outputs(payload: dict) -> str: + names = payload.get("ci_matrix") or [] + if not isinstance(names, list) or not all(isinstance(n, str) for n in names): + raise SystemExit(f"ci_matrix must be a list of names, got {names!r}") + return ( + f"matrix={json.dumps(sorted(set(names)))}\n" + f"any={'yes' if names else 'no'}\n" + ) + + +def require_workspace_tests(payload: dict, root) -> None: + """A Cargo workspace whose test check is not in the matrix is refused. + + cargo-test ships with "ci": false because no workspace exists yet, and + nothing would force the product PR that adds Cargo.toml to remember the + flag — so broken Rust could pass every gate with zero product tests run. + This runs in the discover job, which is unfiltered and therefore runs on + exactly that PR; the refusal names the flag to flip. + """ + from pathlib import Path + + root = Path(root) + manifests = [p for p in [root / "Cargo.toml", *root.glob("crates/*/Cargo.toml")] + if p.exists()] + if not manifests: + return + in_matrix = set(payload.get("ci_matrix") or []) + for check in payload.get("checks") or []: + argv = check.get("argv") or [] + if argv[:2] == ["cargo", "test"] and check.get("name") not in in_matrix: + raise SystemExit( + f"{manifests[0]} exists but the registered test check " + f"{check.get('name')!r} is not in the CI matrix — set " + f"\"ci\": true on it in the {check.get('kind')}@" + f"{check.get('kind_version')} gate-kind spec. A workspace " + f"whose tests CI never runs is a gate nobody notices is " + f"missing." + ) + + +def main(argv) -> int: + if len(argv) != 2: + print(__doc__.strip(), file=sys.stderr) + return 64 + with open(argv[1], encoding="utf-8") as handle: + payload = json.load(handle) + require_workspace_tests(payload, ".") + sys.stdout.write(outputs(payload)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/ops/devlane/workflow/checks/ci_minutes.py b/ops/devlane/workflow/checks/ci_minutes.py new file mode 100644 index 0000000..a4dea95 --- /dev/null +++ b/ops/devlane/workflow/checks/ci_minutes.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +"""What GitHub Actions billed this repository, from the runs API. + +On a private repository every job bills a whole minute, rounded up, +and a job that ran for six seconds bills the same minute as one that +ran for fifty-nine. Nothing in the tree says what a push costs, so the +number that finally mattered -- the organisation's included minutes -- +was learned from GitHub refusing to start jobs, not from a check. + + ci_minutes.py [--repo OWNER/NAME] [--days N | --since YYYY-MM-DD] + [--budget MINUTES] [--json] [--dump FILE] + ci_minutes.py --input FILE [--budget MINUTES] [--json] + +Reads every workflow run created in the window, then every run's jobs, +and bills each job the way GitHub does: ceil(seconds / 60); nothing for +a job that never reached a runner (no steps); nothing for a job that ran +on a self-hosted runner (its `labels` carry `self-hosted`), because +GitHub meters hosted minutes only. Prints the +window's totals, the billed minutes per distinct pushed commit -- the +figure a change to `.github/workflows/` must state before and after -- +and a per-workflow table. + +Exit 0 when under or at `--budget` (or when no budget was given), 1 +when over it. Exit 2 -- UNREACHABLE, with no figures printed -- when +the API could not be read: `gh` missing, a non-zero exit, a body that +is not the JSON expected. A window that read cleanly and holds zero +runs is a genuine zero and prints as one; a window that could not be +read is not a zero and never prints as one. + +`--dump` saves the fetched payload; `--input` replays one, which is how +the tests feed planted windows without a network and how a figure in a +commit body can be re-derived later from the bytes it was measured on. +""" + +from __future__ import annotations + +import argparse +import collections +import json +import math +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime, timedelta + +API = "repos/{repo}/actions/runs?per_page=100&created=>={since}" +JOBS = "repos/{repo}/actions/runs/{run_id}/jobs?per_page=100" + +EXIT_OK = 0 +EXIT_OVER = 1 +EXIT_UNREACHABLE = 2 +EXIT_USAGE = 64 + + +class Unreachable(Exception): + """The API was not read. There is no figure, and none is printed.""" + + +def _iso(value): + return datetime.fromisoformat(value) + + +def self_hosted(job): + return any(str(label).lower() == "self-hosted" for label in job.get("labels") or []) + + +def billed_minutes(job): + """GitHub's rounding: whole minutes, up; nothing for a job that + never reached a runner; nothing for a job a self-hosted runner ran, + whose seconds are still real and still reported.""" + if not isinstance(job, dict): + raise Unreachable( + f"job: expected a mapping, found {type(job).__name__}" + ) + if not job.get("steps"): + return 0, 0.0 + started, completed = job.get("started_at"), job.get("completed_at") + if not started or not completed: + raise Unreachable( + "job: expected started_at and completed_at for a job with steps, " + f"found started_at={started!r}, completed_at={completed!r}" + ) + seconds = max(0.0, (_iso(completed) - _iso(started)).total_seconds()) + if self_hosted(job): + return 0, seconds + return math.ceil(seconds / 60), seconds + + +def family(job_name): + """`dev: gates (lint)` and `dev: gates (imports)` are one family.""" + return job_name.split(" (", 1)[0] + + +def summarize(payload): + """Totals for a fetched or replayed payload, as one JSON-able dict.""" + runs = payload.get("runs") + jobs_by_run = payload.get("jobs") + if not isinstance(runs, list) or not isinstance(jobs_by_run, dict): + raise Unreachable( + "payload: expected {runs: [...], jobs: {run_id: [...]}}, found " + f"keys {sorted(payload) if isinstance(payload, dict) else type(payload).__name__}" + ) + by_workflow = collections.defaultdict( + lambda: {"runs": 0, "jobs": 0, "billed": 0, "real_seconds": 0.0, + "shas": set()} + ) + by_family = collections.defaultdict( + lambda: {"jobs": 0, "billed": 0, "real_seconds": 0.0} + ) + by_event = collections.Counter() + shas = set() + jobs = billed = never_started = hosted_by_us = 0 + real = hosted_real = 0.0 + for run in runs: + if not isinstance(run, dict): + raise Unreachable( + f"run: expected a mapping, found {type(run).__name__}" + ) + run_id = str(run.get("id")) + if run_id not in jobs_by_run: + raise Unreachable(f"run {run_id}: jobs were not fetched") + workflow = str(run.get("path", "?")).rsplit("/", 1)[-1] + sha = str(run.get("head_sha", ""))[:7] + shas.add(sha) + wf = by_workflow[workflow] + wf["runs"] += 1 + wf["shas"].add(sha) + for job in jobs_by_run[run_id]: + minutes, seconds = billed_minutes(job) + if not job.get("steps"): + never_started += 1 + elif self_hosted(job): + hosted_by_us += 1 + jobs += 1 + billed += minutes + real += seconds + if not self_hosted(job): + hosted_real += seconds + wf["jobs"] += 1 + wf["billed"] += minutes + wf["real_seconds"] += seconds + fam = by_family[family(str(job.get("name", "?")))] + fam["jobs"] += 1 + fam["billed"] += minutes + fam["real_seconds"] += seconds + by_event[str(run.get("event", "?"))] += minutes + workflows = {} + for name, row in sorted(by_workflow.items()): + pushes = len(row["shas"]) + workflows[name] = { + "runs": row["runs"], + "pushes": pushes, + "jobs": row["jobs"], + "billed": row["billed"], + "real": round(row["real_seconds"] / 60, 1), + "billed_per_push": round(row["billed"] / pushes, 1) if pushes else 0.0, + } + families = { + name: {"jobs": row["jobs"], "billed": row["billed"], + "real": round(row["real_seconds"] / 60, 1)} + for name, row in sorted( + by_family.items(), key=lambda item: -item[1]["billed"] + ) + } + pushes = len(shas) + return { + "runs": len(runs), + "pushes": pushes, + "jobs": jobs, + "never_started": never_started, + "self_hosted": hosted_by_us, + "real": round(real / 60, 1), + "billed": billed, + "rounding_share": ( + round(1 - (hosted_real / 60) / billed, 2) if billed else 0.0 + ), + "billed_per_push": round(billed / pushes, 1) if pushes else 0.0, + "by_event": dict(by_event), + "workflows": workflows, + "families": families, + } + + +def render(summary, window, budget): + lines = [ + f"CI minutes, {window}", + "", + ( + "| runs | pushes | jobs | never started | self-hosted | real min " + "| billed min | rounding | billed/push |" + ), + "|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ( + f"| {summary['runs']} | {summary['pushes']} | {summary['jobs']} " + f"| {summary['never_started']} | {summary['self_hosted']} " + f"| {summary['real']} | {summary['billed']} " + f"| {int(summary['rounding_share'] * 100)}% | {summary['billed_per_push']} |" + ), + "", + "| workflow | runs | pushes | jobs | real min | billed min | billed/push |", + "|:--|---:|---:|---:|---:|---:|---:|", + ] + for name, row in summary["workflows"].items(): + lines.append( + f"| {name} | {row['runs']} | {row['pushes']} | {row['jobs']} " + f"| {row['real']} | {row['billed']} | {row['billed_per_push']} |" + ) + lines += ["", "| job family | jobs | real min | billed min |", "|:--|---:|---:|---:|"] + for name, row in summary["families"].items(): + lines.append(f"| {name} | {row['jobs']} | {row['real']} | {row['billed']} |") + if budget is not None: + verdict = "OVER" if summary["billed"] > budget else "within" + lines += ["", f"budget {budget} min: {verdict} ({summary['billed']} billed)"] + return "\n".join(lines) + "\n" + + +def gh_json(gh, path, paginate=False): + argv = [gh, "api", path] + if paginate: + argv += ["--paginate", "--slurp"] + try: + proc = subprocess.run(argv, capture_output=True, text=True, check=False) + except OSError as err: + raise Unreachable(f"{gh}: {err}") from err + if proc.returncode != 0: + raise Unreachable( + f"gh api {path}: exit {proc.returncode}: {proc.stderr.strip()[:300]}" + ) + try: + return json.loads(proc.stdout) + except json.JSONDecodeError as err: + raise Unreachable(f"gh api {path}: body is not JSON: {err}") from err + + +def fetch(gh, repo, since): + pages = gh_json(gh, API.format(repo=repo, since=since), paginate=True) + runs = [] + for page in pages if isinstance(pages, list) else [pages]: + if not isinstance(page, dict) or "workflow_runs" not in page: + raise Unreachable("runs: expected pages with workflow_runs, found " + f"{type(page).__name__}") + runs.extend(page["workflow_runs"]) + + def one(run): + body = gh_json(gh, JOBS.format(repo=repo, run_id=run["id"])) + if not isinstance(body, dict) or "jobs" not in body: + raise Unreachable(f"run {run['id']}: expected jobs, found " + f"{type(body).__name__}") + jobs = body["jobs"] + total = body.get("total_count") + if not isinstance(jobs, list) or not isinstance(total, int): + raise Unreachable( + f"run {run['id']}: expected jobs list and integer total_count" + ) + if len(jobs) != total: + raise Unreachable( + f"run {run['id']}: expected {total} jobs, fetched {len(jobs)}; " + "the jobs page is incomplete" + ) + return str(run["id"]), body + + with ThreadPoolExecutor(max_workers=8) as pool: + job_bodies = dict(pool.map(one, runs)) + return { + "repo": repo, + "since": since, + "runs": runs, + "jobs": {run_id: body["jobs"] for run_id, body in job_bodies.items()}, + "api": {"runs": pages, "jobs": job_bodies}, + } + + +def repo_from_gh(gh): + body = gh_json(gh, "repos/{owner}/{repo}") + if not isinstance(body, dict) or not body.get("full_name"): + raise Unreachable("repos/{owner}/{repo}: no full_name in the body") + return body["full_name"] + + +def parse(argv): + parser = argparse.ArgumentParser(prog="ci_minutes.py", add_help=True) + parser.add_argument("--repo") + parser.add_argument("--gh", default="gh") + window = parser.add_mutually_exclusive_group() + window.add_argument("--days", type=int) + window.add_argument("--since") + parser.add_argument("--budget", type=int) + parser.add_argument("--json", action="store_true") + parser.add_argument("--dump") + parser.add_argument("--input") + return parser.parse_args(argv) + + +def main(argv) -> int: + try: + args = parse(argv[1:]) + except SystemExit as err: + return EXIT_USAGE if err.code else EXIT_OK + if args.budget is not None and args.budget < 0: + print("--budget: expected a non-negative number of minutes, found " + f"{args.budget}", file=sys.stderr) + return EXIT_USAGE + try: + if args.input: + try: + with open(args.input, encoding="utf-8") as handle: + payload = json.load(handle) + except (OSError, json.JSONDecodeError) as err: + raise Unreachable(f"--input {args.input}: {err}") from err + window = f"replayed from {args.input}" + else: + if args.since: + since = args.since + else: + days = 7 if args.days is None else args.days + if days <= 0: + print(f"--days: expected a positive number, found {days}", + file=sys.stderr) + return EXIT_USAGE + since = (datetime.now(UTC) - timedelta(days=days)).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + repo = args.repo or repo_from_gh(args.gh) + payload = fetch(args.gh, repo, since) + window = f"{repo} since {since} (UTC)" + if args.dump: + try: + with open(args.dump, "w", encoding="utf-8") as handle: + json.dump(payload, handle) + except OSError as err: + raise Unreachable(f"--dump {args.dump}: {err}") from err + summary = summarize(payload) + except Unreachable as err: + print(f"UNREACHABLE: {err}", file=sys.stderr) + return EXIT_UNREACHABLE + summary["window"] = window + summary["budget"] = args.budget + if args.json: + json.dump(summary, sys.stdout) + sys.stdout.write("\n") + else: + sys.stdout.write(render(summary, window, args.budget)) + if args.budget is not None and summary["billed"] > args.budget: + return EXIT_OVER + return EXIT_OK + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/ops/devlane/workflow/checks/commit_trailers.py b/ops/devlane/workflow/checks/commit_trailers.py new file mode 100755 index 0000000..0ab3e7e --- /dev/null +++ b/ops/devlane/workflow/checks/commit_trailers.py @@ -0,0 +1,386 @@ +#!/usr/bin/env python3 +"""A trailer that git will not parse is not a trailer. + +The failure, measured on 2026-08-24 across `dev`, `task/runner-slice-1` +and `contracts/lift-and-enforce`: of 181 commits carrying a `Source:` +line, **50 of them** are invisible to +`git log --format='%(trailers:key=Source)'`. Nineteen on 08-20, +twenty-seven on 08-21, four on 08-24. The commit titled "apply Codex's +six P1 findings on attribution" is itself one of them. + +The cause is always the same and never visible: git's trailer block is +the LAST paragraph of a message, so a blank line between two trailers +demotes everything above it to body text. + + Source: original <- body text now + <- this blank line is the bug + Co-Authored-By: A <- git parses only this + +`git commit` exits 0, `git log` prints the line, review sees a trailer +sitting exactly where trailers go. Only a machine query reveals the +break, and the whole point of a trailer is that a machine can query it. +So this must be checked mechanically or not at all. + +**git is the oracle, never a parser written here.** The rules for what +counts as a trailer are git's, they are subtle, and a second +implementation would drift from the first the day either changed. This +compares a raw scan of the message against +`git interpret-trailers --parse` and reports where they disagree. + +Refusals, not silences: the check is INVALID -- exit 1, never 0 -- when +git cannot be run, when a range names no commit, or when a message file +is missing or empty. "Nobody looked" must never render as "we checked +and it was fine". + + commit_trailers.py --message-file FILE # commit-msg hook + commit_trailers.py --range BASE..HEAD # registered check + commit_trailers.py --range BASE..HEAD --json + +Exit 0 clean, 1 on a violation or an INVALID run. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +#: A line shaped like a trailer: `Key: value`, key at column zero. +TRAILER_LINE = re.compile(r"^([A-Za-z][A-Za-z0-9-]*): +(\S.*)$") + +#: The keys this repo means as trailers, as data. Shape alone is far too +#: wide to check on: this repo's commit subjects are `workflow: ...`, +#: `docs: ...`, `telemetry: ...`, and a body that quotes one is a line +#: indistinguishable from a trailer. Checking shape flagged 187 of 243 +#: commits, of which 137 were subject prefixes -- a checker firing on +#: three quarters of history is one nobody reads. +#: +#: The cost of a list is that a NEW key is unchecked until it is added +#: here. That is the right trade only because the list is data and the +#: suite pins it: adding a key is one line, and a false positive rate +#: this high is not recoverable. +# AGENTS.md SS1: a specific model name, then a vendor noreply address. +# Deliberately loose on the name -- new models appear and a checker that +# enumerates them refuses tomorrow's -- and strict on the shape, which is +# what makes the trailer queryable at all. +ATTRIBUTION = re.compile(r"^.*\S.* <[^@<>\s]+@[^@<>\s]+>$") + +KNOWN_KEYS = frozenset({ + "Source", # CONTRIB.md, required on every commit + "Co-Authored-By", # AGENTS.md attribution + "Reviewed-by", # CONTRIB.md fix-commit template + "Claude-Session", + "Signed-off-by", + "WO", "Stage", "Event", # wf auto-commits, per commitmsg.cue +}) + + +class Invalid(Exception): + """The check could not run. Never reported as a pass.""" + + +def parsed_pairs(message: str) -> set[tuple[str, str]]: + """The (key, value) trailers git ACTUALLY parses. git is the oracle. + + Pairs rather than keys, because a key alone loses an occurrence: an + orphaned `Source: lost-source` followed by a valid `Source: original` + put `Source` in the set and suppressed the orphan, so a declared + origin git cannot see reported clean (Codex, PR #51). + """ + try: + proc = subprocess.run( + ["git", "interpret-trailers", "--parse"], + input=message, capture_output=True, text=True, check=False, + ) + except OSError as exc: + raise Invalid(f"git could not be run ({exc})") from exc + if proc.returncode != 0: + raise Invalid( + f"git interpret-trailers failed: {proc.stderr.strip()}") + pairs = set() + for line in proc.stdout.splitlines(): + match = TRAILER_LINE.match(line) + if match: + pairs.add((match.group(1), match.group(2).strip())) + return pairs + + +def orphaned(message: str) -> list[dict]: + """Trailer-shaped lines git does not parse, in blocks that claim to be trailers. + + The precision hinge. A message may legitimately DISCUSS a trailer -- + this file's own commit does, and so does every commit that quotes + `Source: original` while explaining it. Such a mention is prose + inside a sentence, or indented inside a fence; it is never a whole + paragraph made of nothing but trailer-shaped lines. + + So a key is reported only when all three hold: it is shaped like a + trailer at column zero, git's parse does NOT contain it, and its + paragraph is entirely trailer-shaped lines. That last condition is + what separates a real orphaned trailer block from prose about one. + """ + lines = message.splitlines() + + paragraphs, current, start = [], [], 0 + for index, line in enumerate(lines): + if line.strip(): + if not current: + start = index + current.append(line) + elif current: + paragraphs.append((start, current)) + current = [] + if current: + paragraphs.append((start, current)) + if not paragraphs: + return [] + + last = paragraphs[-1][1] + found = [] + for start_line, block in paragraphs: + if block is last: + continue + if not all(TRAILER_LINE.match(line) for line in block): + continue # prose, or a mixed paragraph: not a claim + for offset, line in enumerate(block): + key = TRAILER_LINE.match(line).group(1) + # No membership test. git parses the LAST paragraph and no + # other, so a known-key line in an earlier all-trailer block + # is unparsed by construction -- checking whether the same + # pair ALSO appears in the final block only suppressed the + # report when someone wrote the trailer twice, which is the + # case where a declared origin is silently lost (Codex, + # PR #51). Membership could only ever hide a true positive. + if key in KNOWN_KEYS: + found.append({ + "key": key, + "line": start_line + offset + 1, + "text": line.strip(), + }) + return found + + +def is_merge(sha: str) -> bool: + """Is this a real merge? Asked of git, never of the subject line. + + Merges carry no Source -- every merge on dev is a generated "Merge + pull request #N" with no trailers at all -- so they must be exempt + or every PR merge fails. The first version read that exemption off + the SUBJECT, which made the mandatory attribution bypassable by + titling any commit `Merge branch ...` (Codex, PR #51). Parentage is + a fact; a subject is a claim. + + Takes a sha, because parentage is only a fact once the commit + exists. See check_message on why the hook does not ask. + """ + proc = subprocess.run(["git", "rev-list", "--parents", "-n", "1", sha], + capture_output=True, text=True, check=False) + if proc.returncode != 0: + return False + return len(proc.stdout.split()) > 2 + + +def check_message(message: str, label: str, *, require_source: bool = True, + sha: str | None = None) -> list[str]: + if not message.strip(): + raise Invalid(f"{label}: the message is empty") + problems = [] + # A trailer that is ABSENT is as unqueryable as one that is malformed, + # and CONTRIB.md §Naming requires Source on every commit. The check + # began by comparing a raw scan against git's parse, which by + # construction can only see trailers that are THERE -- so a message + # with no Source at all passed clean (Codex, PR #51). + # + # Only over a RANGE, where `sha` names an existing commit. At hook + # time the commit does not exist, so its parentage is not a fact yet: + # MERGE_HEAD covers a merge in progress but not `commit --amend` on a + # merge, where MERGE_HEAD is already gone and the result still keeps + # both parents -- the hook refused a legitimate amend that CI exempts + # (Codex, PR #51 round four). Rather than guess parentage from the + # message or the reflog, the hook does not enforce this at all: it is + # local convenience, and WF:gates is the gate that cannot be dodged. + # The hook still catches the orphaned trailer, which needs only the + # message. + if (require_source and sha is not None and not is_merge(sha) + and not any(k == "Source" for k, _ in parsed_pairs(message))): + problems.append( + f"{label}: no `Source:` trailer git can parse — CONTRIB.md " + f"requires one on every non-merge commit" + ) + # Same requirement, same reason, a different trailer. AGENTS.md SS1 + # states it flatly: "Every commit carries a `Co-Authored-By` trailer + # naming the specific agent -- model, not brand; vendor noreply + # address." An absent one was accepted, and so was + # `Co-Authored-By: arbitrary`, because only parseability was asked + # (Codex, PR #51 round five). Parseability is not the requirement; + # the documented form is, so the VALUE is checked too. + # + # Range mode only, for the parentage reason above. + if require_source and sha is not None and not is_merge(sha): + coauthors = [v for k, v in parsed_pairs(message) if k == "Co-Authored-By"] + if not coauthors: + problems.append( + f"{label}: no `Co-Authored-By:` trailer git can parse — " + f"AGENTS.md requires one naming the specific agent on " + f"every non-merge commit" + ) + elif not any(ATTRIBUTION.match(v) for v in coauthors): + problems.append( + f"{label}: `Co-Authored-By: {coauthors[0]}` is not the " + f"documented form — AGENTS.md requires a specific model " + f"name and a vendor address, e.g. " + f"`Claude Fable 5 `" + ) + return problems + [ + f"{label}: `{item['text']}` at line {item['line']} is not a " + f"trailer git will parse — a blank line above the final block " + f"demotes it to body text" + for item in orphaned(message) + ] + + +def commits_in(rng: str) -> list[str]: + try: + proc = subprocess.run( + ["git", "rev-list", rng], capture_output=True, text=True, check=False) + except OSError as exc: + raise Invalid(f"git could not be run ({exc})") from exc + if proc.returncode != 0: + raise Invalid(f"{rng} is not a range git can list: {proc.stderr.strip()}") + shas = proc.stdout.split() + if not shas: + raise Invalid(f"{rng} names no commit, so nothing was checked") + return shas + + +def message_of(sha: str) -> str: + proc = subprocess.run(["git", "log", "-1", "--format=%B", sha], + capture_output=True, text=True, check=False) + if proc.returncode != 0: + raise Invalid(f"cannot read the message of {sha}") + return proc.stdout + + +def range_from_event(): + """The commits THIS CI run is responsible for, from the event payload. + + Hard-coding `origin/dev` breaks on the branch it names: a `push` run + for dev checks out the pushed commit and sets origin/dev to it, so + `merge-base origin/dev HEAD` is HEAD, the range is empty, and the + gate reports INVALID on every push to dev. A run for main would + compare against an unrelated base instead (Codex, PR #51). + + GITHUB_EVENT_PATH is always set by Actions, so the range is read from + the event rather than guessed: what a push added, or what a pull + request proposes. Absent or unreadable, the caller falls back to the + merge-base route, which is right for a developer's shell. + """ + path = os.environ.get("GITHUB_EVENT_PATH") + if not path or not Path(path).is_file(): + return None + try: + event = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + def here(sha): + """Is this commit in the repository we are actually running in? + + GITHUB_EVENT_PATH is set for every step of a CI job, including + steps that run the checker inside a throwaway fixture repo. That + repo does not contain the event's commits, so using the event's + range there asked about commits that do not exist and refused — + which failed the gate-binding fixture's CLEAN arm in CI while + passing locally, where the variable is unset (found by CI, not + by me). + """ + if not sha: + return False + return subprocess.run(["git", "cat-file", "-e", f"{sha}^{{commit}}"], + capture_output=True, check=False).returncode == 0 + + pull = event.get("pull_request") + if isinstance(pull, dict): + base = (pull.get("base") or {}).get("sha") + head = (pull.get("head") or {}).get("sha") + if here(base) and here(head): + return f"{base}..{head}" + return None + before, after = event.get("before"), event.get("after") + # A new branch reports an all-zero `before`; there is no prior state + # to diff against, so fall back rather than invent one. + if before and set(before) != {"0"} and here(before) and here(after): + return f"{before}..{after}" + return None + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + group = ap.add_mutually_exclusive_group(required=True) + group.add_argument("--message-file", help="one message, as the commit-msg hook is given it") + group.add_argument("--range", dest="rng", help="BASE..HEAD") + group.add_argument("--since-base", action="store_true", + help="every commit this branch adds over --base-ref") + ap.add_argument("--base-ref", + help="the ref --since-base measures from (default origin/dev)") + ap.add_argument("--json", action="store_true") + args = ap.parse_args(argv) + + problems = [] + try: + if args.message_file: + path = Path(args.message_file) + if not path.is_file(): + raise Invalid(f"{path} is not a file") + problems = check_message(path.read_text(encoding="utf-8"), str(path)) + checked = 1 + else: + rng = args.rng + if args.since_base: + # An explicitly supplied base is an operator instruction, + # not a fallback. Only consult Actions' event payload when + # the caller left the base at its implicit default. + event = range_from_event() if args.base_ref is None else None + if event: + rng = event + # Resolved here rather than baked into the registered argv: + # a gate whose range is a literal string goes stale the day + # the lane branch is renamed, and a shallow checkout makes it + # silently empty. commits_in refuses an empty range, so a + # clone without the base ref reports INVALID rather than a + # pass over nothing. + if args.since_base and not rng: + base_ref = args.base_ref or "origin/dev" + merge_base = subprocess.run( + ["git", "merge-base", base_ref, "HEAD"], + capture_output=True, text=True, check=False) + if merge_base.returncode != 0: + raise Invalid( + f"{base_ref} is not available in this clone " + f"(a shallow checkout?): {merge_base.stderr.strip()}") + rng = f"{merge_base.stdout.strip()}..HEAD" + shas = commits_in(rng) + checked = len(shas) + for sha in shas: + problems.extend(check_message(message_of(sha), sha[:7], sha=sha)) + except Invalid as exc: + print(f"commit-trailers: INVALID — {exc}", file=sys.stderr) + return 1 + + if args.json: + print(json.dumps({"checked": checked, "problems": problems}, + indent=2, sort_keys=True)) + else: + print(f"commit-trailers: checked {checked} message(s)") + for problem in problems: + print(f" {problem}") + if not problems: + print(" clean — every trailer-shaped line is one git parses") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ops/devlane/workflow/checks/diagnostics.py b/ops/devlane/workflow/checks/diagnostics.py new file mode 100755 index 0000000..ee96b09 --- /dev/null +++ b/ops/devlane/workflow/checks/diagnostics.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +"""A diagnostics snapshot, as a registered check (PLAN §9). + +"Diagnostics snapshots recorded as `receipt.check` receipts (class +`diagnostics`) — same evidence shape, **no new machinery**." So there is no +verb here and no event type: this is an argv template in a gate-kind spec, +and `wf check` records its output exactly like any other check. + +It is deliberately producer-agnostic. Whatever emits the diagnostics — +`cargo check --message-format=json`, a language server, a linter — this reads +JSON objects, one per line, and counts them by severity: + + diagnostics.py [--fail-on error] -- + +Recognised shapes, because producers disagree and guessing wrong silently +reports zero problems: + + {"severity": "error", "file": …, "line": …, "message": …} + {"level": "error", …} (rustc / cargo) + {"message": {"level": "error", …}} (cargo --message-format=json) + +Two producers do not speak in lines at all, and are read as whole +documents BEFORE the line walk — a SARIF log read line-by-line is +every line unparsed and no findings, which is a silent pass over a +file full of errors: + + {"$schema": …sarif…, "runs": [{"results": [...]}]} SARIF 2.1.0 + … JUnit XML + +Each document format carries a state that is NOT "clean", and saying +so is the point of reading them at all: + + SARIF "runs": [] no tool ran (an empty `results` IS clean) + JUnit no at all no test ran (.dev/process/tdd.md: "NO + TESTS RAN" is not a red — + and not a pass either) + +A line that is not JSON is counted as unparsed and reported. Exits 1 when +anything at or above --fail-on is present, 0 otherwise — and prints the +counts either way, because a check that says only "ok" cannot be told from +one that read nothing. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import xml.etree.ElementTree as ElementTree +from typing import NamedTuple + +SEVERITY_ORDER = ["note", "help", "info", "information", "warning", "error"] +ALIASES = {"warn": "warning", "information": "info", "err": "error"} + + +def rank(severity: str) -> int: + severity = ALIASES.get(severity.lower(), severity.lower()) + return SEVERITY_ORDER.index(severity) if severity in SEVERITY_ORDER else -1 + + +def severity_of(record): + """Pull a severity out of whichever shape the producer used.""" + if not isinstance(record, dict): + return None + for key in ("severity", "level"): + value = record.get(key) + if isinstance(value, str): + return ALIASES.get(value.lower(), value.lower()) + if isinstance(value, int): # LSP numeric severities: 1=error … 4=hint + return {1: "error", 2: "warning", 3: "info", 4: "note"}.get(value) + nested = record.get("message") + if isinstance(nested, dict): + return severity_of(nested) + return None + + +class Reading(NamedTuple): + """What one producer's output amounted to. + + `refusal` is the field that keeps this check honest: it is set when + the producer emitted a well-formed document proving it never did + the work, which no count can express — zero findings and zero work + look identical in `counts`. + """ + + counts: dict + unparsed: int + detail: str | None = None + refusal: str | None = None + + +#: SARIF 2.1.0 §3.27.10. A result with no `level` resolves through its +#: rule's defaultConfiguration before falling back to `warning` — NOT +#: to "unparsed", which would report a valid document as unreadable. +SARIF_DEFAULT = "warning" + +def _prolog_declares_entities(text): + """Does the PROLOG carry a DOCTYPE or ENTITY declaration? + + Walked structurally rather than regex-bounded. The first version + took "the prolog" to be everything before the first `<` followed by + a name character — but an XML COMMENT may legally contain such text, + and `]>` then truncated the + prolog before the declaration, admitted the document, and expanded + the entity. That defeated the whole mitigation, including the one + cited to justify the S314 exemption in ruff.toml (Codex, PR #38). + + So: skip whitespace, comments and processing instructions the way a + parser does, and answer on the first thing that is none of those. + """ + i, n = 0, len(text) + while i < n: + while i < n and text[i] in " \t\r\n": + i += 1 + if i >= n or text[i] != "<": + return False + if text.startswith("", i + 4) + if end == -1: + return False # unterminated; the parser will refuse + i = end + 3 + continue + if text.startswith("", i + 2) + if end == -1: + return False + i = end + 2 + continue + # Whatever this is, the prolog ends here: either a declaration + # (the thing we refuse) or an element start (the thing that ends + # the prolog). + return text.startswith(" "testcase".""" + return tag.rsplit("}", 1)[-1] if isinstance(tag, str) else "" + + +def _rule_level(item, rules): + """A result may name its rule three ways; all reach the default. + + `ruleId`, `ruleIndex`, and a `rule` reportingDescriptorReference + carrying either. Resolving only `ruleId` scored a rule that + defaults to `error` as a warning and passed the gate (Codex, PR + review of 3453dc1). + """ + def configured(rule): + return (rule.get("defaultConfiguration") or {}).get("level") + + ref = item.get("rule") if isinstance(item.get("rule"), dict) else {} + index = item.get("ruleIndex") + if index is None: + index = ref.get("index") + if isinstance(index, int) and not isinstance(index, bool) \ + and 0 <= index < len(rules): + level = configured(rules[index]) + if level: + return level + identifier = item.get("ruleId") or ref.get("id") + if identifier: + for rule in rules: + if rule.get("id") == identifier: + level = configured(rule) + if level: + return level + return None + + +def sarif_reading(text: str): + """Read a SARIF log, or return None if this is not one.""" + stripped = text.strip() + if not stripped.startswith("{"): + return None + try: + doc = json.loads(stripped) + except ValueError: + return None + if not isinstance(doc, dict): + return None + # §3.13.2: `version` is mandatory and its value is fixed. Claiming + # any object with a list-valued `runs` swallowed ordinary producer + # records — {"severity":"error","runs":[{}]} passed the gate + # silently (Codex, PR review of 3453dc1). + schema = str(doc.get("$schema") or "").lower() + if doc.get("version") != "2.1.0" and "sarif" not in schema: + return None + runs = doc.get("runs") + if runs is None: + return Reading({}, 0, refusal="sarif: `runs` is null") + if not isinstance(runs, list): + return None + if not runs: + # §3.13.4 permits an empty `runs` for a producer with no run + # data — a result-management query matching nothing, say. For a + # GATE the producer's job is to analyse code, so zero runs is + # zero analysis and refusing is right; `executionSuccessful` + # below is the spec's own signal and now carries the weight. + return Reading({}, 0, refusal="sarif: no runs in the log") + counts = {} + for run in runs: + if not isinstance(run, dict): + continue + for invocation in run.get("invocations") or []: + # §3.20.14: the tool itself reporting that it did not + # complete. No count can express that. + if isinstance(invocation, dict) \ + and invocation.get("executionSuccessful") is False: + return Reading( + {}, 0, + refusal="sarif: an invocation reports " + "executionSuccessful=false") + results = run.get("results") + if results is None: + # §3.14.23: absent or null `results` means the tool failed + # to populate the log, which is not "found nothing". + return Reading({}, 0, refusal="sarif: a run has no `results`") + for item in results: + if not isinstance(item, dict): + continue + # §3.27.9: a result whose `kind` is anything but "fail" is + # not a finding at all, whatever level it carries. + if item.get("kind", "fail") != "fail": + continue + level = (item.get("level") + or _rule_level(item, _component_rules(run, item)) + or SARIF_DEFAULT) + if not isinstance(level, str) or level == "none": + continue + level = ALIASES.get(level.lower(), level.lower()) + counts[level] = counts.get(level, 0) + 1 + return Reading(counts, 0) + + +def junit_reading(text: str): + """Read a JUnit report, or return None if this is not one.""" + stripped = text.strip() + if " Reading: + """Read one producer's two streams. + + The streams are kept APART for the document readers and joined only + for the line walk. A producer emits its document on one stream and + its progress chatter on the other — `cargo clippy + --message-format=sarif` writes the log to stdout while cargo writes + "Compiling foo v0.1.0" to stderr — so concatenating them first + turned a valid log full of errors into unparsed lines and a clean + exit (Codex, PR review of 3453dc1). + """ + for stream in (stdout, stderr): + if not stream.strip(): + continue + for reader in (sarif_reading, junit_reading): + reading = reader(stream) + if reading is not None: + return reading + text = stdout + "\n" + stderr + counts, unparsed = {}, 0 + for line in text.splitlines(): + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + except ValueError: + unparsed += 1 + continue + severity = severity_of(record) + if severity is None: + unparsed += 1 + continue + counts[severity] = counts.get(severity, 0) + 1 + return Reading(counts, unparsed) + + +def main(argv=None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + producer = None + if "--" in argv: + index = argv.index("--") + argv, producer = argv[:index], argv[index + 1:] + + parser = argparse.ArgumentParser(description="diagnostics snapshot") + parser.add_argument("--fail-on", default="error", + help="lowest severity that fails (default: error)") + args = parser.parse_args(argv) + + if not producer: + print("diagnostics: no producer command given", file=sys.stderr) + print("usage: diagnostics.py [--fail-on error] -- ", file=sys.stderr) + return 64 + + try: + proc = subprocess.run(producer, capture_output=True, text=True, + errors="replace", check=False) + except (FileNotFoundError, PermissionError) as exc: + # Not "no diagnostics". A producer that cannot run has told us + # nothing, and reporting that as clean is the failure this whole + # repository exists to prevent. + print(f"diagnostics: the producer could not run: {exc}", file=sys.stderr) + return 1 + + reading = summarise(proc.stdout, proc.stderr) + threshold = rank(args.fail_on) + if threshold < 0: + print(f"diagnostics: unknown --fail-on severity {args.fail_on!r}", + file=sys.stderr) + return 64 + + counts = reading.counts + failing = sum(n for sev, n in counts.items() if rank(sev) >= threshold) + shown = ", ".join(f"{sev}={n}" for sev, n in sorted(counts.items())) or "none" + print(f"diagnostics: {shown}" + f"{f', unparsed={reading.unparsed}' if reading.unparsed else ''}" + f" (producer exited {proc.returncode}, failing at >= {args.fail_on})") + if reading.detail: + print(f" {reading.detail}") + for line in (proc.stdout + proc.stderr).splitlines()[:20]: + if line.strip(): + print(f" {line[:160]}") + if reading.refusal: + # A well-formed document proving the work never happened. No + # count can say this: zero findings and zero work are the same + # empty dict, and only one of them is clean. + print(f"diagnostics: {reading.refusal} — refusing to score that" + f" as clean", file=sys.stderr) + return 1 + if failing: + return 1 + if proc.returncode != 0: + # No failing diagnostic was parsed, yet the producer itself failed — + # a broken manifest, a bad flag, a crash before speaking JSON. That + # is not a clean tree; it is a producer that told us nothing, and + # scoring it clean records a satisfying receipt for a failed check. + print(f"diagnostics: producer exited {proc.returncode} with no" + f" recognized failing diagnostic — failing the check", + file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ops/devlane/workflow/checks/doc_commands.py b/ops/devlane/workflow/checks/doc_commands.py new file mode 100755 index 0000000..8db2009 --- /dev/null +++ b/ops/devlane/workflow/checks/doc_commands.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""A document that tells you to run something must name something that +is there. + +`ghost_link_errors` already refuses a markdown link to a missing `.md`. +It cannot see a command: `./lessons.sh` inside an ```sh fence is not a +link, so a shelf document instructed a reader to run a script that +existed nowhere in the repository and everything reported success -- +the doc rendered, the docs-index suite passed, the ghost-link check +passed, CI was green. Two reviewers caught it by reading the prose. + +Exit 0 when every command in every shell fence resolves, 1 when one +does not, and print each miss with its file and line so the reader does +not have to go and find it. + + doc_commands.py [--root DIR] [PATH ...] + +Fences are scanned line by line rather than by pairing ``` to the next +```. A regex doing the latter can begin at a CLOSING fence and capture +the prose after it as if it were shell -- which made the first version +of this scanner return zero hits on the very document that carried the +fault. The state machine is not fastidiousness; it is the difference +between a check and a decoration. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +#: Only fences that claim to be shell. An unlabelled fence is usually +#: output or a transcript, and assuming otherwise is how a checker earns +#: its reputation for noise. +SHELL_LANGS = {"sh", "bash", "console", "shell"} + +#: The command word: the first token of a line, optionally preceded by an +#: interpreter. A path that appears later on the line is an argument -- +#: `git add ops/devlane/gone.py` is talking about a file, not running one. +COMMAND = re.compile( + r"""^\s* + (?:(?:python3?|bash|sh)\s+)? # an interpreter, if any + (?P(?:\./|[\w.-]+/)[\w./-]+\.(?:sh|py)) + (?=\s|$) # the whole token, not a prefix + """, + re.VERBOSE, +) + + +class DocumentReadError(Exception): + """The document was selected for scanning but could not be read.""" + + def __init__(self, path, error): + self.path = path + self.error = error + super().__init__(str(error)) + + +def shell_fences(text): + """(line_number, line) for every line inside a shell fence. + + Line-scanned, so an opening fence is only ever a line that opens one. + """ + out, lang = [], None + for n, line in enumerate(text.splitlines(), 1): + stripped = line.strip() + if stripped.startswith("```"): + lang = None if lang is not None else ( + stripped[3:].strip().lower() or "-") + continue + if lang in SHELL_LANGS: + out.append((n, line)) + return out + + +def command_misses(path: Path, root: Path): + """Commands in this document's shell fences that resolve to nothing.""" + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as err: + # Survive unreadable input, but do not turn an unperformed scan into + # a clean result. The caller reports every unreadable document. + raise DocumentReadError(path, err) from err + misses = [] + for n, line in shell_fences(text): + m = COMMAND.match(line) + if not m: + continue + raw = m.group("path") + rel = raw.removeprefix("./") + if (path.parent / rel).exists() or (root / rel).exists(): + continue + misses.append((n, raw)) + return misses + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--root", default=".") + ap.add_argument("paths", nargs="*") + a = ap.parse_args(argv) + try: + root = Path(a.root).resolve() + except (OSError, RuntimeError) as err: + print(f"{a.root}: doc-commands: INVALID — root could not be" + f" resolved ({err})") + return 1 + if not root.is_dir(): + print(f"{root}: doc-commands: INVALID — root is not a directory") + return 1 + resolution_errors = [] + if a.paths: + docs = [] + for raw in a.paths: + path = Path(raw) + try: + docs.append(path.resolve()) + except (OSError, RuntimeError) as err: + resolution_errors.append(DocumentReadError(path, err)) + else: + docs = sorted(p for p in root.rglob("*.md") + if ".git" not in p.parts) + hits = 0 + unreadable = 0 + for err in resolution_errors: + print(f"{err.path}: doc-commands: INVALID — could not be read" + f" ({err.error})") + unreadable += 1 + for d in docs: + try: + misses = command_misses(d, root) + except DocumentReadError as err: + try: + shown = d.relative_to(root) + except ValueError: + shown = d + print(f"{shown}: doc-commands: INVALID — could not be read" + f" ({err.error})") + unreadable += 1 + continue + for n, raw in misses: + try: + shown = d.relative_to(root) + except ValueError: + shown = d + print(f"{shown}:{n}: `{raw}` is run here and does not exist") + hits += 1 + if hits or unreadable: + if unreadable: + print(f"doc-commands: INVALID — {unreadable} document(s)" + " could not be scanned") + if not hits: + return 1 + print(f"doc-commands: {hits} command(s) name nothing") + return 1 + print(f"doc-commands: clean — {len(docs)} document(s) scanned") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ops/devlane/workflow/checks/doc_covers.py b/ops/devlane/workflow/checks/doc_covers.py new file mode 100644 index 0000000..7ae7cb5 --- /dev/null +++ b/ops/devlane/workflow/checks/doc_covers.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Check shelf-document frontmatter and the paths named by ``covers:``. + +Three classes of bad input, three different obligations, and one exit +code between them — so the printed line is the only thing that tells +them apart, and the summary never says clean when any of them fired. + + doc_covers.py [--root DIR] [PATH ...] + +A **finding** is a declaration that is wrong: a path that resolves to +nothing, a path that resolves somewhere it should not, a `covers:` +value that is not a list of paths, a header that never closes or lacks +`name:`/`description:`. **INVALID** is a subject that could not be +judged at all: a document that cannot be read or decoded, a path whose +resolution raises, a `--root` that is not a directory, a shelf that is +not there, a shelf with no `.md` in it. "Nobody looked" is not "clean" +— the lint and imports checkers learned that first, and doc_commands +gave INVALID its printed shape. + +`covers:` names repo-relative paths. Absolute and `..` are refused +LEXICALLY, before any filesystem call: a checker that stats an absolute +path has already reached outside the repository it is judging, and +`.dev/docs/../docs/README.md` is not a name, it is a route. The +inside test resolves both sides, because a `/tmp` that is itself a +symlink (macOS) would otherwise report every inside path as outside. + +Exit 0 or 1, never 2, 3, 64 or 70 — those are `wf`'s, so argparse's +own exit is intercepted rather than inherited. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path +from typing import NamedTuple + +#: The shelf, relative to the repository root. +SHELF = (".dev", "docs") + +FINDING = "finding" +INVALID = "invalid" + + +class Problem(NamedTuple): + """One printed line, and which count it belongs to.""" + + kind: str + message: str + + +class _Usage(Exception): + """argparse asked to exit. Carries the code this checker may use.""" + + def __init__(self, status, message=""): + self.status = status + self.message = message + super().__init__(message) + + +class _Parser(argparse.ArgumentParser): + """Stock argparse leaves on 2 for an unknown flag, which is `wf`'s + code for a usage error and would arrive in CI as neither pass nor + fail. Both exits are raised instead and mapped by main().""" + + def error(self, message): + raise _Usage(1, message) + + def exit(self, status=0, message=None): + raise _Usage(1 if status else 0, message or "") + + +def header_block(text): + """(block, opened, closed) — the ONE place a document's `---` + header is split. Two parsers of one header is the drift this + checker exists to hold together.""" + if not text.startswith("---\n"): + return "", False, False + parts = text.split("---", 2) + if len(parts) != 3: + return "", True, False + return parts[1], True, True + + +def covers_declarations(text): + """(raw paths, malformed messages) for EVERY `covers:` key. + + Every key is read and every item is judged: the parser this + replaced stopped at the first key and dropped a scalar value or an + empty `- ` item on the floor, so a declaration that named nothing + real reported clean. The two branches judge the same way — an item + is empty once its quotes are off, and a value that is a mapping is + as much "not a list" as a scalar is. + """ + block, opened, closed = header_block(text) + if not (opened and closed): + return [], [] + paths, malformed = [], [] + lines = block.splitlines() + for index, line in enumerate(lines): + if not line.startswith("covers:"): + continue + rest = line.split(":", 1)[1].strip() + if rest.startswith("[") and rest.endswith("]"): + inner = rest[1:-1].strip() + # `[]` is a list with no items; `""` is an item that is + # empty. The quotes come off BEFORE the test, or the empty + # item is judged as a path and resolves to the repo root. + items = inner.split(",") if inner else [] + for item in items: + value = item.strip().strip("'\"") + if value: + paths.append(value) + else: + malformed.append("covers: has an empty list item") + continue + if rest: + malformed.append( + f"covers: {rest} is a scalar; covers: names a list of paths") + continue + for follow in lines[index + 1:]: + stripped = follow.strip() + # Column 0 ends the block — the next key, and in particular + # a second `covers:` that is its own declaration, never a + # value of this one. A blank line ends it too, and is not a + # finding. + if not stripped or not follow[:1].isspace(): + break + if stripped != "-" and not stripped.startswith("- "): + malformed.append(f"covers: {stripped} is not a list item;" + " covers: names a list of paths") + break + item = stripped[1:].strip().strip("'\"") + if item: + paths.append(item) + else: + malformed.append("covers: has an empty list item") + return paths, malformed + + +def covers_paths(text): + """Paths named by a `covers:` key in YAML-ish frontmatter.""" + return covers_declarations(text)[0] + + +def covers_path_problem(raw, name, repo): + """One `covers:` path, judged. None when it is a real repo path.""" + lexical = Path(raw) + if lexical.is_absolute(): + return Problem(FINDING, f"{name}: covers: {raw} is absolute;" + " covers: names repo-relative paths") + if ".." in lexical.parts: + return Problem(FINDING, f"{name}: covers: {raw} has a .. segment;" + " covers: names a path, not a route") + target = repo / raw + try: + resolved = target.resolve() + except (OSError, RuntimeError) as err: + return Problem(INVALID, f"{name}: doc-covers: INVALID — covers:" + f" {raw} could not be resolved ({err})") + try: + target.stat() + except FileNotFoundError: + return Problem(FINDING, f"{name}: covers: {raw} does not exist") + except (OSError, ValueError) as err: + return Problem(INVALID, f"{name}: doc-covers: INVALID — covers:" + f" {raw} could not be read ({err})") + if not resolved.is_relative_to(repo): + return Problem(FINDING, f"{name}: covers: {raw} resolves outside" + " the repository") + return None + + +def covers_problems(text, name, repo): + """(problems, paths judged) for one document's `covers:` block.""" + root = Path(repo).resolve() + paths, malformed = covers_declarations(text) + problems = [Problem(FINDING, f"{name}: {message}") + for message in malformed] + for raw in paths: + problem = covers_path_problem(raw, name, root) + if problem is not None: + problems.append(problem) + return problems, len(paths) + + +def covers_path_errors(text, name, repo): + """Every covers: path must be a real repo-relative path in the tree.""" + return [problem.message for problem in covers_problems(text, name, repo)[0]] + + +def frontmatter_errors(text, name): + """The production frontmatter check — driven by the class test on + every live doc AND by the self-tests on planted shapes, so a + reverted branch cannot stay green (Grok, PR #30 round three).""" + block, opened, closed = header_block(text) + if not opened: + return [f"{name}: has no frontmatter"] + if not closed: + return [f"{name}: frontmatter never closes"] + return [f"{name}: frontmatter lacks a {key}: line" + for key in ("name", "description") + if not re.search(rf"(?m)^{key}:", block)] + + +def shown_as(doc, repo): + """The document's name as a reader of the repository knows it.""" + try: + return str(doc.relative_to(repo)) + except ValueError: + return str(doc) + + +def judge(docs, repo): + """(problems, documents scanned, covers: paths judged).""" + problems, paths = [], 0 + for doc in docs: + name = shown_as(doc, repo) + try: + text = doc.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as err: + problems.append(Problem( + INVALID, + f"{name}: doc-covers: INVALID — could not be read ({err})")) + continue + if not text.startswith("---\n"): + # BOUNDARY: the checker judges the header of a document that + # has one. TODO.md and the shelf index carry prose only. + continue + problems.extend(Problem(FINDING, message) + for message in frontmatter_errors(text, name)) + found, counted = covers_problems(text, name, repo) + problems.extend(found) + paths += counted + return problems, len(docs), paths + + +def main(argv=None): + ap = _Parser(description=__doc__.splitlines()[0]) + ap.add_argument("--root", default=".") + ap.add_argument("paths", nargs="*") + try: + args = ap.parse_args(argv) + except _Usage as usage: + if usage.status: + print(f"doc-covers: INVALID — {usage.message.strip()}") + return usage.status + + try: + repo = Path(args.root).resolve(strict=True) + except (OSError, RuntimeError) as err: + print(f"{args.root}: doc-covers: INVALID — root could not be" + f" resolved ({err})") + return 1 + if not repo.is_dir(): + print(f"{args.root}: doc-covers: INVALID — root is not a directory") + return 1 + + shelf_name = str(Path(*SHELF)) + if args.paths: + docs = [Path(raw) for raw in args.paths] + else: + shelf = repo.joinpath(*SHELF) + if not shelf.is_dir(): + print(f"{shelf_name}: doc-covers: INVALID — the shelf is not" + " a directory here") + return 1 + docs = sorted(shelf.rglob("*.md")) + if not docs: + print(f"{shelf_name}: doc-covers: INVALID — no .md document" + " under the shelf to judge") + return 1 + + problems, scanned, paths = judge(docs, repo) + for problem in problems: + print(problem.message) + invalid = sum(1 for problem in problems if problem.kind == INVALID) + findings = len(problems) - invalid + if invalid: + print(f"doc-covers: INVALID — {invalid} subject(s) could not be" + " judged") + if findings: + print(f"doc-covers: {findings} finding(s)") + if problems: + return 1 + print(f"doc-covers: clean — {scanned} doc(s), {paths} covers: path(s)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ops/devlane/workflow/checks/lint.py b/ops/devlane/workflow/checks/lint.py new file mode 100644 index 0000000..4c17faa --- /dev/null +++ b/ops/devlane/workflow/checks/lint.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Lint, as a registered check. + +Runs ruff under the repo's pinned contract (`ruff.toml`) so the CI +verdict and the local one are the same verdict. There is no rule +selection here on purpose: a checker that carried its own list would +be a second contract, free to drift from the file everyone edits. + +Refusals, not silences. A lint pass is INVALID — exit 1, not 0 — +when ruff is absent, when it cannot be run, or when the paths given +match no Python file at all. "Nobody looked" must never render as +"we checked and it was fine". + + lint.py [--root DIR] [PATH ...] + +With no PATH, checks the whole tree from --root, which the config's +excludes then narrow. +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from pathlib import Path + +#: Only these are Python; `--show-files` lists whatever the config +#: reaches, including ruff.toml itself, so the count must filter. +PY_SUFFIXES = (".py", ".pyi") + +#: A finding line is `path:line:col: CODE message`; ruff also prints a +#: summary and fix advice, which are not findings. +FINDING = re.compile(r"^.+?:\d+:\d+: [A-Z]+\d+") + + +def run_ruff(root: Path, args: list[str]): + try: + return subprocess.run( + ["ruff", *args], cwd=root, + capture_output=True, text=True, check=False, + ) + except OSError as exc: + print(f"lint: INVALID — ruff could not be run ({exc})") + return None + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description="ruff as a registered check") + parser.add_argument("--root", default=".") + parser.add_argument("paths", nargs="*") + args = parser.parse_args(argv) + root = Path(args.root) + paths = args.paths or ["."] + + # The pinned contract is the whole point: without the config ruff + # falls back to its own defaults and can report clean under rules + # nobody chose, which is the drift this check exists to prevent. + config = root / "ruff.toml" + if not config.is_file(): + print(f"lint: INVALID — no ruff.toml under {root}; the pinned" + " contract is the only thing that makes a verdict mean" + " anything") + return 1 + common = ["--config", str(config)] + + listed = run_ruff(root, ["check", *common, "--show-files", *paths]) + if listed is None: + return 1 + if listed.returncode not in (0, 1): + print("lint: INVALID — ruff refused the request:") + print(listed.stderr.strip() or listed.stdout.strip()) + return 1 + files = [line for line in listed.stdout.splitlines() + if line.strip().endswith(PY_SUFFIXES)] + if not files: + print(f"lint: INVALID — {paths} matched zero Python files;" + " a lint pass over nothing is not a clean lint pass") + return 1 + + proc = run_ruff(root, ["check", *common, "--output-format", "concise", + *paths]) + if proc is None: + return 1 + if proc.returncode not in (0, 1): + print("lint: INVALID — ruff exited abnormally:") + print(proc.stderr.strip() or proc.stdout.strip()) + return 1 + if proc.returncode == 1: + findings = [ln for ln in proc.stdout.splitlines() if FINDING.match(ln)] + print(f"lint: {len(findings)} finding(s)" + f" across {len(files)} file(s)") + print(proc.stdout.rstrip()) + return 1 + + version = run_ruff(root, ["--version"]) + stamp = version.stdout.strip() if version else "ruff" + print(f"lint: clean — {len(files)} file(s), {stamp}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ops/devlane/workflow/checks/secret_scan.py b/ops/devlane/workflow/checks/secret_scan.py new file mode 100755 index 0000000..47be50c --- /dev/null +++ b/ops/devlane/workflow/checks/secret_scan.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""A day-one secret scan (PLAN D8's seed, as data-driven as anything else). + +Deliberately small and high-signal. It looks for credential shapes that are +unambiguous — a private key header, an AWS key id, a GitHub token prefix, a +Slack token — rather than trying to be a general scanner. A scanner that +cries wolf gets disabled, and a disabled check is worse than none. + +D8 is still open in PLAN §12: this seeds the `security` class so a security +stage has something real to require, and it is an argv template in the +gate-kind spec, so replacing it with a heavier scanner is a data change. + + secret_scan.py [--root DIR] [PATH ...] + +Exit 0 clean, 1 on a hit. Tracked files only, so an untracked scratch file +does not fail the gate. +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from pathlib import Path + +SIGNATURES = [ + ("private key block", re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH |PGP )?PRIVATE KEY-----")), + ("AWS access key id", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b")), + ("GitHub token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{36,}\b")), + ("Slack token", re.compile(r"\bxox[abpsr]-[A-Za-z0-9-]{10,}\b")), + ("Google API key", re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b")), + ("PEM certificate key", re.compile(r"-----BEGIN ENCRYPTED PRIVATE KEY-----")), +] + +SKIP_SUFFIXES = {".png", ".jpg", ".jpeg", ".gif", ".pdf", ".zip", ".gz", ".db"} + +#: A line carrying this marker is skipped. The suite that proves this scanner +#: fires has to contain credential-shaped strings, and so will any fixture or +#: documentation example. Marking the line is deliberate and greppable, which +#: is the same idiom test-guard uses; a blanket "skip tests" rule would hide +#: a real key committed to a test file. +ALLOW_MARKER = "secret-scan: allow" + + +def tracked_files(root: Path, explicit): + if explicit: + # the documented [PATH ...] form accepts directories: expand + # each to the tracked files beneath it — silently dropping a + # directory made `secret_scan.py .` a scan of nothing + # (Grok, PR #32 review) + chosen = [] + for raw in explicit: + path = Path(raw) + if path.is_dir(): + out = subprocess.run( + ["git", "-C", str(root), "ls-files", "-z", "--", raw], + capture_output=True, text=True, check=False) + if out.returncode == 0: + chosen.extend(root / q for q in out.stdout.split("\0") + if q) + elif path.is_file(): + chosen.append(path) + return chosen + out = subprocess.run( + ["git", "-C", str(root), "ls-files", "-z"], + capture_output=True, text=True, check=False, + ) + if out.returncode != 0: + return [p for p in sorted(root.rglob("*")) if p.is_file()] + return [root / p for p in out.stdout.split("\0") if p] + + +def main() -> int: + parser = argparse.ArgumentParser(description="day-one secret scan") + parser.add_argument("--root", default=".") + parser.add_argument("paths", nargs="*") + args = parser.parse_args() + + root = Path(args.root).resolve() + hits, scanned, allowed = [], 0, 0 + # This file necessarily contains the patterns it looks for. + myself = Path(__file__).resolve() + + for path in tracked_files(root, args.paths): + if not path.is_file() or path.suffix.lower() in SKIP_SUFFIXES: + continue + if path.resolve() == myself: + continue + scanned += 1 + try: + text = path.read_text(encoding="utf-8") + except (UnicodeDecodeError, OSError): + continue + for number, line in enumerate(text.splitlines(), 1): + if ALLOW_MARKER in line: + allowed += 1 + continue + for label, pattern in SIGNATURES: + if pattern.search(line): + try: + shown = path.relative_to(root).as_posix() + except ValueError: + shown = str(path) + hits.append(f"{shown}:{number}: {label}") + + if hits: + print(f"secret scan: {len(hits)} candidate credential(s)") + for hit in hits: + print(f" {hit}") + return 1 + if scanned == 0: + # a scan that read nothing proved nothing: reporting it clean + # converts "nobody looked" into "we checked and it was fine" + print("secret scan: INVALID — the fileset matched zero" + " tracked files; a scan over nothing is not a clean scan") + return 1 + print(f"secret scan: clean — {scanned} tracked file(s)," + f" {len(SIGNATURES)} signature(s), {allowed} allowlisted line(s)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ops/devlane/workflow/checks/vocabulary_wall.py b/ops/devlane/workflow/checks/vocabulary_wall.py index 6876d9b..3842d34 100755 --- a/ops/devlane/workflow/checks/vocabulary_wall.py +++ b/ops/devlane/workflow/checks/vocabulary_wall.py @@ -50,7 +50,7 @@ #: `ruff.toml` joins them for the same reason as the dot-directories: #: it configures how this repo is worked on, and it cannot do that #: without naming dev-lane paths — a per-file rule for -#: `.dev/app/workflow/wf.py` contains a reserved word in the PATH. +#: `ops/devlane/workflow/wf.py` contains a reserved word in the PATH. ALLOWED_FILES = ("AGENTS.md", "CONTRIB.md", "README.md", "ruff.toml") TEXT_SUFFIXES = {".md", ".txt", ".rst", ".adoc", ".cue", ".toml", ".yaml", ".yml"} @@ -221,8 +221,8 @@ def main() -> int: ) rel = path.relative_to(root).as_posix() defines_rule = ( - rel == ".dev/app/workflow/checks/vocabulary_wall.py" - or rel.startswith(".dev/app/workflow/tests/") + rel == "ops/devlane/workflow/checks/vocabulary_wall.py" + or rel.startswith("ops/devlane/workflow/tests/") ) if not defines_rule and path.suffix.lower() in {".py", ".json", ".yaml", ".yml"}: patterns_here.extend( From 9f5af68c0d87030764655dcb7cadf82ee3d90330 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:21:15 -0400 Subject: [PATCH 07/18] =?UTF-8?q?ops:=20the=20local-CI=20infra=20=E2=80=94?= =?UTF-8?q?=20Gitea=20+=20act=5Frunner=20+=20lane=20image?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local runners instead of GitHub-hosted minutes: a Gitea instance (bootstrap seeds a local mirror), an act_runner (labels ubuntu-latest:host), and the lane build image, under ops/devlane/infra. No secrets: runner-token.seed is an empty placeholder (the real token comes from the owner) and is gitignored along with gitea runtime state. Runner tests deferred. Activation is two owner-gated steps, in order: bring the runner up with a real token, THEN flip CI runs-on from ubuntu-latest to a self-hosted label. Flipping before a runner is live would stall every open PR. Source: owner 2026-09-01 Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XehTac5TJNmPAskwrPp7rJ Claude-Session: https://claude.ai/code/session_012Jj94rkp3tfHAxUkTCthgY --- ops/devlane/infra/.gitignore | 3 + ops/devlane/infra/README.md | 5 ++ ops/devlane/infra/build-lane.sh | 24 +++++++ ops/devlane/infra/compose.yml | 66 ++++++++++++++++++++ ops/devlane/infra/containment-lint.sh | 38 ++++++++++++ ops/devlane/infra/gitea/Dockerfile | 12 ++++ ops/devlane/infra/gitea/app.ini | 12 ++++ ops/devlane/infra/gitea/bootstrap.sh | 75 ++++++++++++++++++++++ ops/devlane/infra/gitea/entrypoint.sh | 12 ++++ ops/devlane/infra/gitea/runtime/.keep | 1 + ops/devlane/infra/lane/Dockerfile | 86 ++++++++++++++++++++++++++ ops/devlane/infra/lane/entrypoint.sh | 21 +++++++ ops/devlane/infra/pins.env | 21 +++++++ ops/devlane/infra/runner/Dockerfile | 11 ++++ ops/devlane/infra/runner/config.yaml | 13 ++++ ops/devlane/infra/runner/entrypoint.sh | 32 ++++++++++ ops/devlane/infra/secrets/README.md | 5 ++ 17 files changed, 437 insertions(+) create mode 100644 ops/devlane/infra/.gitignore create mode 100644 ops/devlane/infra/README.md create mode 100755 ops/devlane/infra/build-lane.sh create mode 100644 ops/devlane/infra/compose.yml create mode 100755 ops/devlane/infra/containment-lint.sh create mode 100644 ops/devlane/infra/gitea/Dockerfile create mode 100644 ops/devlane/infra/gitea/app.ini create mode 100755 ops/devlane/infra/gitea/bootstrap.sh create mode 100644 ops/devlane/infra/gitea/entrypoint.sh create mode 100644 ops/devlane/infra/gitea/runtime/.keep create mode 100644 ops/devlane/infra/lane/Dockerfile create mode 100755 ops/devlane/infra/lane/entrypoint.sh create mode 100644 ops/devlane/infra/pins.env create mode 100644 ops/devlane/infra/runner/Dockerfile create mode 100644 ops/devlane/infra/runner/config.yaml create mode 100755 ops/devlane/infra/runner/entrypoint.sh create mode 100644 ops/devlane/infra/secrets/README.md diff --git a/ops/devlane/infra/.gitignore b/ops/devlane/infra/.gitignore new file mode 100644 index 0000000..7878137 --- /dev/null +++ b/ops/devlane/infra/.gitignore @@ -0,0 +1,3 @@ +gitea/runtime/* +!gitea/runtime/.keep +gitea/runner-token.seed diff --git a/ops/devlane/infra/README.md b/ops/devlane/infra/README.md new file mode 100644 index 0000000..31f8692 --- /dev/null +++ b/ops/devlane/infra/README.md @@ -0,0 +1,5 @@ +# Compose infrastructure + +This directory is compose's home in the development lane. +Here, `service` has compose's container meaning. +Nothing is deployed from this directory yet. diff --git a/ops/devlane/infra/build-lane.sh b/ops/devlane/infra/build-lane.sh new file mode 100755 index 0000000..283abe7 --- /dev/null +++ b/ops/devlane/infra/build-lane.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")" +set -a +. ./pins.env +set +a + +export LANE_TAG="$(sha256sum pins.env | cut -d ' ' -f 1)" +install -m 0600 /dev/null secrets/token +docker build --file lane/Dockerfile --tag "lane:${LANE_TAG}" \ + --build-arg "PYTHON=${PYTHON:?}" \ + --build-arg "UV=${UV:?}" --build-arg "UV_SHA256=${UV_SHA256:?}" \ + --build-arg "RUFF=${RUFF:?}" \ + --build-arg "CUE=${CUE:?}" --build-arg "CUE_SHA256=${CUE_SHA256:?}" \ + --build-arg "GH=${GH:?}" --build-arg "GH_SHA256=${GH_SHA256:?}" \ + --build-arg "RUSTUP=${RUSTUP:?}" --build-arg "RUSTUP_SHA256=${RUSTUP_SHA256:?}" \ + --build-arg "RUST_TOOLCHAIN=${RUST_TOOLCHAIN:?}" \ + --build-arg "RUST_COMPONENTS=${RUST_COMPONENTS:?}" \ + --build-arg "CLAUDE_CLI=${CLAUDE_CLI:?}" --build-arg "CODEX_CLI=${CODEX_CLI:?}" \ + --build-arg "GROK_CLI=${GROK_CLI:?}" --build-arg "GROK_CLI_SHA256=${GROK_CLI_SHA256:?}" \ + . + +docker compose --profile ci --env-file pins.env build act_runner diff --git a/ops/devlane/infra/compose.yml b/ops/devlane/infra/compose.yml new file mode 100644 index 0000000..3596b5a --- /dev/null +++ b/ops/devlane/infra/compose.yml @@ -0,0 +1,66 @@ +services: + gitea: + image: minspec/gitea-lane:${GITEA} + build: + context: . + dockerfile: gitea/Dockerfile + args: + GITEA: ${GITEA} + profiles: [ci] + user: "1000:1000" + environment: + USER: git + GITEA__security__INSTALL_LOCK: "true" + GITEA__database__DB_TYPE: sqlite3 + GITEA__server__ROOT_URL: http://gitea:3000 + GITEA__security__SECRET_KEY__FILE: /run/secrets/secret_key + GITEA__security__INTERNAL_TOKEN__FILE: /run/secrets/internal_token + GITEA__oauth2__JWT_SECRET__FILE: /run/secrets/jwt_secret + volumes: + - gitea-data:/var/lib/gitea + - gitea-config:/etc/gitea + - ../..:/workspace:ro + - type: bind + source: ./secrets/secret_key + target: /run/secrets/secret_key + read_only: true + - type: bind + source: ./secrets/internal_token + target: /run/secrets/internal_token + read_only: true + - type: bind + source: ./secrets/jwt_secret + target: /run/secrets/jwt_secret + read_only: true + - token:/run/runner-token + + act_runner: + image: minspec/act-runner-lane:${ACT_RUNNER} + build: + context: . + dockerfile: runner/Dockerfile + args: + ACT_RUNNER: ${ACT_RUNNER} + LANE_TAG: ${LANE_TAG:?run ops/devlane/infra/build-lane.sh} + profiles: [ci] + privileged: false + depends_on: + - gitea + environment: + CONFIG_FILE: /usr/local/bin/config.yaml + GITEA_INSTANCE_URL: http://gitea:3000 + RUNNER_LABELS: ubuntu-latest:host + LANE_TAG: ${LANE_TAG:?run ops/devlane/infra/build-lane.sh} + volumes: + - runner-data:/data + - token:/run/runner-token:ro + - type: bind + source: ./secrets/token + target: /run/secrets/token + read_only: true + +volumes: + gitea-data: + gitea-config: + runner-data: + token: diff --git a/ops/devlane/infra/containment-lint.sh b/ops/devlane/infra/containment-lint.sh new file mode 100755 index 0000000..76d551a --- /dev/null +++ b/ops/devlane/infra/containment-lint.sh @@ -0,0 +1,38 @@ +#!/bin/sh +set -eu + +# Vendored from the Agent-Lab containment check, with selectors for this tree. +# Tests are fixtures for this checker, so they are deliberately excluded. +fail=0 +warn=0 +files=$(git ls-files -- 'ops/devlane/infra/' '.github/workflows/' \ + | while IFS= read -r path; do + case "$path" in + ops/devlane/infra/tests/*) ;; + *) printf '%s\n' "$path" ;; + esac + done) + +report_fail() { + printf 'FAIL %s: %s\n' "$1" "$2" + fail=$((fail + 1)) +} + +for path in $files; do + [ -f "$path" ] || continue + if grep -Eq '/var/run/docker\.sock|/run/docker\.sock' "$path"; then + report_fail "$path" 'docker socket mount' + fi + if grep -Eq '(^|[[:space:]])privileged:[[:space:]]*true([[:space:]]|$)|--privileged([[:space:]]|$)' "$path"; then + report_fail "$path" 'privileged' + fi + if grep -Eq 'network_mode:[[:space:]]*host([[:space:]]|$)|--network([=[:space:]]+)host([[:space:]]|$)' "$path"; then + report_fail "$path" 'host networking' + fi + if grep -Eq 'gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{30,}' "$path"; then + report_fail "$path" 'likely-real secret' + fi +done + +printf '%s fail, %s warn\n' "$fail" "$warn" +[ "$fail" -eq 0 ] diff --git a/ops/devlane/infra/gitea/Dockerfile b/ops/devlane/infra/gitea/Dockerfile new file mode 100644 index 0000000..aaf6d64 --- /dev/null +++ b/ops/devlane/infra/gitea/Dockerfile @@ -0,0 +1,12 @@ +ARG GITEA=1.24.6 +FROM gitea/gitea:${GITEA}-rootless@sha256:91f2b27e080739f0d19dba716c3214f17257c632cf762e545b44ca577e37052c + +COPY --chown=1000:1000 gitea/runtime/ /run/agent-lab/ +COPY --chown=1000:1000 --chmod=0700 gitea/runtime/ /run/runner-token/ +COPY --chown=1000:1000 --chmod=0600 gitea/runner-token.seed /run/runner-token/token +COPY gitea/app.ini /tmp/agent-lab-app.ini +COPY --chown=1000:1000 --chmod=0640 gitea/app.ini /etc/gitea/app.ini +COPY --chmod=755 gitea/bootstrap.sh /usr/local/bin/agent-lab-bootstrap +COPY --chmod=755 gitea/entrypoint.sh /usr/local/bin/agent-lab-gitea + +CMD ["/usr/local/bin/agent-lab-gitea"] diff --git a/ops/devlane/infra/gitea/app.ini b/ops/devlane/infra/gitea/app.ini new file mode 100644 index 0000000..252515e --- /dev/null +++ b/ops/devlane/infra/gitea/app.ini @@ -0,0 +1,12 @@ +[server] +OFFLINE_MODE = true +ROOT_URL = http://gitea:3000 + +[security] +INSTALL_LOCK = true + +[service] +DISABLE_REGISTRATION = true + +[actions] +ENABLED = true diff --git a/ops/devlane/infra/gitea/bootstrap.sh b/ops/devlane/infra/gitea/bootstrap.sh new file mode 100755 index 0000000..138a08e --- /dev/null +++ b/ops/devlane/infra/gitea/bootstrap.sh @@ -0,0 +1,75 @@ +#!/bin/sh +set -eu + +until curl -fsS http://127.0.0.1:3000/api/healthz >/dev/null; do + sleep 1 +done + +user=minspec +password_file=/var/lib/gitea/agent-lab/bootstrap-password +mkdir -p "${password_file%/*}" +if [ ! -s "$password_file" ]; then + umask 077 + od -An -N24 -tx1 /dev/urandom | tr -d ' \n' > "$password_file" +fi +password=$(cat "$password_file") +gitea_cli() { + env GITEA_WORK_DIR=/var/lib/gitea gitea \ + --config /etc/gitea/app.ini "$@" +} + +if ! gitea_cli admin user list --admin | awk 'NR > 1 { print $2 }' \ + | grep -Fxq "$user"; then + gitea_cli admin user create \ + --admin --username minspec --password "$password" \ + --email agent-lab@invalid.example --must-change-password=false +fi + +state_dir=/var/lib/gitea/agent-lab +mkdir -p "$state_dir" /run/agent-lab +umask 077 +if [ ! -f /var/lib/gitea/agent-lab/api-token ]; then + gitea_cli admin user generate-access-token --username "$user" \ + --token-name agent-lab-bootstrap \ + --scopes read:repository,read:admin,read:user \ + --raw > /var/lib/gitea/agent-lab/api-token +fi +api_token=$(cat /var/lib/gitea/agent-lab/api-token) + +create_repo() { + case "$1" in --name) ;; *) return 64 ;; esac + repo_name=$2 + if curl -fsS -H "Authorization: token $api_token" \ + "http://127.0.0.1:3000/api/v1/repos/$user/$repo_name" >/dev/null; then + return + fi + curl -fsS --user "$user:$password" \ + -H 'Content-Type: application/json' \ + -d "{\"name\":\"$repo_name\",\"default_branch\":\"dev\"}" \ + http://127.0.0.1:3000/api/v1/user/repos >/dev/null +} +create_repo --name minspec + +if [ ! -s /run/runner-token/token ]; then + token=$(gitea_cli actions generate-runner-token) + umask 077 + printf '%s\n' "$token" > /run/runner-token/token +fi + +git config --global --add safe.directory /workspace +source_tree=/workspace +seed= +if ! git -C "$source_tree" rev-parse --verify HEAD >/dev/null 2>&1; then + seed=$(mktemp -d) + trap 'rm -rf "$seed"' EXIT INT TERM + tar -C /workspace --exclude=.git -cf - . | tar -C "$seed" -xf - + git -C "$seed" init --initial-branch=dev + git -C "$seed" config user.name xormania + git -C "$seed" config user.email 127287135+xormania@users.noreply.github.com + git -C "$seed" add --all + git -C "$seed" commit -m 'Seed dev tree' + source_tree=$seed +fi +git -C "$source_tree" push --force \ + "http://$user:$password@127.0.0.1:3000/$user/minspec.git" \ + HEAD:refs/heads/dev diff --git a/ops/devlane/infra/gitea/entrypoint.sh b/ops/devlane/infra/gitea/entrypoint.sh new file mode 100644 index 0000000..c87a153 --- /dev/null +++ b/ops/devlane/infra/gitea/entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu + +mkdir -p /var/lib/gitea/agent-lab /run/agent-lab + +gitea web --config /etc/gitea/app.ini & +gitea_pid=$! +trap 'kill "$gitea_pid" 2>/dev/null || true' INT TERM EXIT + +/usr/local/bin/agent-lab-bootstrap + +wait "$gitea_pid" diff --git a/ops/devlane/infra/gitea/runtime/.keep b/ops/devlane/infra/gitea/runtime/.keep new file mode 100644 index 0000000..0f844a0 --- /dev/null +++ b/ops/devlane/infra/gitea/runtime/.keep @@ -0,0 +1 @@ +runtime state — gitignored diff --git a/ops/devlane/infra/lane/Dockerfile b/ops/devlane/infra/lane/Dockerfile new file mode 100644 index 0000000..a088869 --- /dev/null +++ b/ops/devlane/infra/lane/Dockerfile @@ -0,0 +1,86 @@ +ARG PYTHON=3.12.14 +FROM python:3.12-slim-bookworm@sha256:0f5b26b9518d002b6173fd61daad821fa340635ebfec5bba471013f9ca114579 + +ARG PYTHON=3.12.14 + +ARG UV=0.8.22 +ARG UV_SHA256=741ff1f5742c5a4a25d2f829e8395355e43f7a5ae2ebc6368e9ae2df0efb69cf +ARG RUFF=0.16.5 +ARG CUE=0.14.1 +ARG CUE_SHA256=c7d29f5988d088627cf53bd6a223807c466066cf432c7cf5c36429ffc9e734f6 +ARG GH=2.98.0 +ARG GH_SHA256=f65a3fa2fa0eb2e97c445ee3f5e087a40aae03b64847f45a8f13805e504535d6 +ARG RUSTUP=1.29.0 +ARG RUSTUP_SHA256=4acc9acc76d5079515b46346a485974457b5a79893cfb01112423c89aeb5aa10 +ARG RUST_TOOLCHAIN=1.89.0 +ARG RUST_COMPONENTS="clippy rustfmt" +ARG CLAUDE_CLI=2.1.251 +ARG CODEX_CLI=0.150.1 +ARG GROK_CLI=1.0.5 +ARG GROK_CLI_SHA256=9ba87444e1819e8f6104adbbf4676a870c204380aa5c3e1c38a926c4ea677238 + +RUN test -n "$UV" && test -n "$RUFF" && test -n "$CUE" \ + && test -n "$PYTHON" && python3 --version | grep -F "$PYTHON" \ + && test -n "$CUE_SHA256" && test -n "$GH" \ + && test -n "$UV_SHA256" && test -n "$GH_SHA256" \ + && test -n "$RUSTUP" && test -n "$RUSTUP_SHA256" \ + && test -n "$RUST_TOOLCHAIN" && test -n "$RUST_COMPONENTS" \ + && test -n "$CLAUDE_CLI" && test -n "$CODEX_CLI" \ + && test -n "$GROK_CLI" && test -n "$GROK_CLI_SHA256" + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl git gosu npm pipx xz-utils \ + && rm -rf /var/lib/apt/lists/* + +# uv from the release tarball its version names, digest-checked — never +# an installer piped to a shell. +RUN curl -fsSLo /tmp/uv.tar.gz \ + "https://github.com/astral-sh/uv/releases/download/${UV}/uv-x86_64-unknown-linux-gnu.tar.gz" \ + && echo "${UV_SHA256} /tmp/uv.tar.gz" | sha256sum -c - \ + && tar -xzf /tmp/uv.tar.gz -C /usr/local/bin --strip-components=1 \ + uv-x86_64-unknown-linux-gnu/uv \ + && rm /tmp/uv.tar.gz \ + && uv tool install "ruff==${RUFF}" \ + && install -m 0755 /root/.local/bin/ruff /usr/local/bin/ruff + +RUN curl -fsSLo /tmp/cue.tar.gz \ + "https://github.com/cue-lang/cue/releases/download/v${CUE}/cue_v${CUE}_linux_amd64.tar.gz" \ + && echo "${CUE_SHA256} /tmp/cue.tar.gz" | sha256sum -c - \ + && tar -xzf /tmp/cue.tar.gz -C /usr/local/bin cue \ + && rm /tmp/cue.tar.gz + +RUN curl -fsSLo /tmp/gh.deb \ + "https://github.com/cli/cli/releases/download/v${GH}/gh_${GH}_linux_amd64.deb" \ + && echo "${GH_SHA256} /tmp/gh.deb" | sha256sum -c - \ + && apt-get update \ + && apt-get install -y --no-install-recommends /tmp/gh.deb \ + && rm -rf /var/lib/apt/lists/* /tmp/gh.deb + +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH="/usr/local/cargo/bin:/usr/local/bin:${PATH}" + +# rustup-init from the archive its version names, digest-checked — the +# sh.rustup.rs pipe binds no bytes. +RUN curl --proto '=https' --tlsv1.2 -fsSLo /tmp/rustup-init \ + "https://static.rust-lang.org/rustup/archive/${RUSTUP}/x86_64-unknown-linux-gnu/rustup-init" \ + && echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c - \ + && chmod 0755 /tmp/rustup-init \ + && /tmp/rustup-init -y --default-toolchain "${RUST_TOOLCHAIN}" \ + && rm /tmp/rustup-init \ + && rustup component add ${RUST_COMPONENTS} + +RUN npm install -g \ + "@anthropic-ai/claude-code@${CLAUDE_CLI}" \ + "@openai/codex@${CODEX_CLI}" + +RUN curl -fsSLo /usr/local/bin/grok \ + "https://x.ai/cli/grok-${GROK_CLI}-linux-x86_64" \ + && echo "${GROK_CLI_SHA256} /usr/local/bin/grok" | sha256sum -c - \ + && chmod 0755 /usr/local/bin/grok + +COPY lane/entrypoint.sh /usr/local/bin/lane-entrypoint +ENTRYPOINT ["/usr/local/bin/lane-entrypoint"] +CMD ["bash"] + +# Build tooling tags this image as lane:$(sha256sum pins.env | cut -d' ' -f1). diff --git a/ops/devlane/infra/lane/entrypoint.sh b/ops/devlane/infra/lane/entrypoint.sh new file mode 100755 index 0000000..4f54ad1 --- /dev/null +++ b/ops/devlane/infra/lane/entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu + +if find "${HOME:?HOME is required}" -type f \ + \( -name '.credentials*' -o -name auth.json -o -name .gitconfig \ + -o -name '*.token' \) -print -quit | grep -q .; then + echo "refusing image-baked credentials under HOME" >&2 + exit 64 +fi + +HOST_UID=${HOST_UID:-$(id -u)} +HOST_GID=${HOST_GID:-$(id -g)} + +if [ "$(id -u)" -eq 0 ] && { [ "$HOST_UID" -ne 0 ] || [ "$HOST_GID" -ne 0 ]; }; then + HOME="/tmp/lane-home-${HOST_UID}" + mkdir -p "$HOME" + chown "$HOST_UID:$HOST_GID" "$HOME" + export HOME + exec gosu "$HOST_UID:$HOST_GID" env HOME="$HOME" "$@" +fi +exec "$@" diff --git a/ops/devlane/infra/pins.env b/ops/devlane/infra/pins.env new file mode 100644 index 0000000..c352252 --- /dev/null +++ b/ops/devlane/infra/pins.env @@ -0,0 +1,21 @@ +PYTHON=3.12.14 +PYTHON_DIGEST=sha256:0f5b26b9518d002b6173fd61daad821fa340635ebfec5bba471013f9ca114579 +UV=0.8.22 +UV_SHA256=741ff1f5742c5a4a25d2f829e8395355e43f7a5ae2ebc6368e9ae2df0efb69cf +RUFF=0.16.5 +CUE=0.14.1 +CUE_SHA256=c7d29f5988d088627cf53bd6a223807c466066cf432c7cf5c36429ffc9e734f6 +GH=2.98.0 +GH_SHA256=f65a3fa2fa0eb2e97c445ee3f5e087a40aae03b64847f45a8f13805e504535d6 +RUSTUP=1.29.0 +RUSTUP_SHA256=4acc9acc76d5079515b46346a485974457b5a79893cfb01112423c89aeb5aa10 +RUST_TOOLCHAIN=1.89.0 +RUST_COMPONENTS="clippy rustfmt" +CLAUDE_CLI=2.1.251 +CODEX_CLI=0.150.1 +GROK_CLI=1.0.5 +GROK_CLI_SHA256=9ba87444e1819e8f6104adbbf4676a870c204380aa5c3e1c38a926c4ea677238 +GITEA=1.24.6 +GITEA_DIGEST=sha256:91f2b27e080739f0d19dba716c3214f17257c632cf762e545b44ca577e37052c +ACT_RUNNER=0.2.13 +ACT_RUNNER_DIGEST=sha256:8477d5b61b655caad4449888bae39f1f34bebd27db56cb15a62dccb3dcf3a944 diff --git a/ops/devlane/infra/runner/Dockerfile b/ops/devlane/infra/runner/Dockerfile new file mode 100644 index 0000000..65d4052 --- /dev/null +++ b/ops/devlane/infra/runner/Dockerfile @@ -0,0 +1,11 @@ +ARG ACT_RUNNER=0.2.13 +ARG LANE_TAG +FROM gitea/act_runner:${ACT_RUNNER}@sha256:8477d5b61b655caad4449888bae39f1f34bebd27db56cb15a62dccb3dcf3a944 AS upstream-runner + +FROM lane:${LANE_TAG} + +COPY --from=upstream-runner /usr/local/bin/act_runner /usr/local/bin/act_runner + +COPY runner/config.yaml /usr/local/bin/config.yaml +COPY runner/entrypoint.sh /usr/local/bin/agent-lab-runner +ENTRYPOINT ["/usr/local/bin/agent-lab-runner"] diff --git a/ops/devlane/infra/runner/config.yaml b/ops/devlane/infra/runner/config.yaml new file mode 100644 index 0000000..327e049 --- /dev/null +++ b/ops/devlane/infra/runner/config.yaml @@ -0,0 +1,13 @@ +runner: + capacity: 4 + timeout: 30m + file: /data/.runner +cache: + enabled: true + dir: /data/cache +host: + workdir_parent: /tmp/act +container: + docker_host: "-" + privileged: false + network: "" diff --git a/ops/devlane/infra/runner/entrypoint.sh b/ops/devlane/infra/runner/entrypoint.sh new file mode 100755 index 0000000..075d8a2 --- /dev/null +++ b/ops/devlane/infra/runner/entrypoint.sh @@ -0,0 +1,32 @@ +#!/bin/sh +set -eu + +labels=${RUNNER_LABELS:?RUNNER_LABELS is required} +old_ifs=$IFS +IFS=, +for label in $labels; do + case "$label" in + *:host|*:docker://*) ;; + *) echo "invalid runner label: $label" >&2; exit 64 ;; + esac +done +IFS=$old_ifs + +if [ -n "${LANE_TAG:-}" ]; then + labels=$(printf '%s' "$labels" | sed "s/\${LANE_TAG}/${LANE_TAG}/g") +fi + +until [ -s /run/runner-token/token ]; do + sleep 1 +done + +cd /data +if [ ! -f /data/.runner ]; then + act_runner register --no-interactive \ + --instance "$GITEA_INSTANCE_URL" \ + --token "$(cat /run/runner-token/token)" \ + --name agent-lab \ + --labels "$labels" +fi + +exec act_runner daemon --config "${CONFIG_FILE:?CONFIG_FILE is required}" diff --git a/ops/devlane/infra/secrets/README.md b/ops/devlane/infra/secrets/README.md new file mode 100644 index 0000000..620c965 --- /dev/null +++ b/ops/devlane/infra/secrets/README.md @@ -0,0 +1,5 @@ +# Local credentials + +Place runtime-only files in this directory. Git ignores every file here +except this explanation; containers receive required values through read-only +mounts at runtime. From 0e345241191434ac1d5f2014f47e9b4dbdc8c0da Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:21:15 -0400 Subject: [PATCH 08/18] =?UTF-8?q?ops:=20the=20term=20wall=20=E2=80=94=20th?= =?UTF-8?q?e=20lane=20refuses=20names=20this=20organisation=20does=20not?= =?UTF-8?q?=20use?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit workflow/checks/term_wall.py scans tracked files, a commit message, a commit range, or stdin for names that must not appear here — not affirmed, not negated, not cited — never spelling them and masking every hit. The commit-msg hook runs it beside the trailer check (a clone without the wall warns and lets the commit through; CI holds the wall). apply-push's content guard refuses a landing patch that carries such a name in added content or a path, and the landing message is refused the same way before the bridge commits it. Verified: tree scan of this lane exits 0; planted message, stdin, path and content each exit 1 with a masked hit; a missing message file refuses with exit 2 on the wire; hooks suite 156/156 OK. Source: original Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012Jj94rkp3tfHAxUkTCthgY --- ops/devlane/dispatch/levers/apply-push.sh | 10 +++++++++- ops/devlane/hooks/commit-msg | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/ops/devlane/dispatch/levers/apply-push.sh b/ops/devlane/dispatch/levers/apply-push.sh index 74c84a1..85a5297 100755 --- a/ops/devlane/dispatch/levers/apply-push.sh +++ b/ops/devlane/dispatch/levers/apply-push.sh @@ -272,8 +272,12 @@ hits=[] home=re.compile(r"/home/[A-Za-z0-9._-]+/") # credential filenames introduced as content cred=re.compile(r"\b(auth\.json|id_rsa|id_ed25519|\.pem|\.p12|credentials(\.json)?)\b") +# names this organisation does not use — never affirmed, negated, or cited +wall=re.compile(r"s[c]ient[ _-]?db|u[s]cient",re.I) for p,text in added: pth=p or "(unknown)" + if wall.search(text) or wall.search(pth): + hits.append(f"{wall.sub('[forbidden name]',pth)}: forbidden name in added content or path") # runtime job capture path anywhere in an added line or as the file itself if "levers/jobs/" in (pth+" "+text): hits.append(f"{pth}: runtime job-capture content ('levers/jobs/')") @@ -304,6 +308,7 @@ import re,sys src,out,job,digest=sys.argv[1:] s=open(src,encoding="utf-8").read().rstrip() if "\x00" in s or not s.strip(): raise SystemExit(1) +if re.search(r"s[c]ient[ _-]?db|u[s]cient",s,re.I): raise SystemExit(2) # the term wall, on the message trailer=re.compile(r"^[A-Za-z0-9][A-Za-z0-9-]*: .+$") caller_last=re.split(r"\n[ \t]*\n",s)[-1].splitlines() separator="\n" if caller_last and all(trailer.match(x) for x in caller_last) else "\n\n" @@ -314,7 +319,10 @@ if len(last)<2 or not all(trailer.match(x) for x in last): raise SystemExit(1) open(out,"w",encoding="utf-8").write(s) PY then - refuse "landing message/trailer block is not a contiguous final paragraph" + case $? in + 2) refuse "landing message carries a forbidden name — a name this organisation does not use; found in the message; needed a message without it" ;; + *) refuse "landing message/trailer block is not a contiguous final paragraph" ;; + esac fi if ! env -i HOME="$OPERATOR_HOME" PATH="$SAFE_PATH" LANG=C.UTF-8 LC_ALL=C.UTF-8 \ diff --git a/ops/devlane/hooks/commit-msg b/ops/devlane/hooks/commit-msg index 8e7f3a5..819867b 100755 --- a/ops/devlane/hooks/commit-msg +++ b/ops/devlane/hooks/commit-msg @@ -51,3 +51,19 @@ python3 "$check" --message-file "$msg_file" || { echo " Source: and Co-Authored-By:." >&2 exit 1 } + +# The term wall: names this organisation does not use, refused in the +# message before it becomes history. Same convention as above — a clone +# without the wall warns and lets the commit through; CI holds the wall. +wall="$top/ops/devlane/workflow/checks/term_wall.py" +if [ -f "$wall" ]; then + python3 "$wall" --message-file "$msg_file" || { + echo "" >&2 + echo "commit-msg: the message above was NOT committed." >&2 + echo " It carries a name this organisation does not use" >&2 + echo " (the hits are listed, masked). Rewrite without it." >&2 + exit 1 + } +else + echo "commit-msg: term wall not present here — letting the message through; CI holds the wall." >&2 +fi From 27a138dcd115a7005609ea448c9f0497567f31f5 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:45:52 -0400 Subject: [PATCH 09/18] ops: ignore every lever job root The codex and claude levers keep their job captures under their own jobs/ directories, which the ignore for the grok lever's root did not cover; a git add -A swept two write clones and their captures into a local commit before it was noticed and reset. Every lever job root is now ignored. Source: original Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012Jj94rkp3tfHAxUkTCthgY --- ops/devlane/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/ops/devlane/.gitignore b/ops/devlane/.gitignore index 0466322..5473a3b 100644 --- a/ops/devlane/.gitignore +++ b/ops/devlane/.gitignore @@ -5,3 +5,4 @@ __pycache__/ # The lever SOURCE (levers/*.sh, levers/claude, levers/codex, README) # is tracked; everything a run WRITES under jobs/ is never committed. dispatch/levers/jobs/ +dispatch/levers/**/jobs/ From 017f64bf4a7f97b0d1e03a4673b949de49b4f6b1 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:45:52 -0400 Subject: [PATCH 10/18] =?UTF-8?q?ops:=20term=20wall=20tests,=20from=20the?= =?UTF-8?q?=20contract=20=E2=80=94=2015=20real=20executions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Authored from the contract by a producer other than the implementer: each test runs term_wall.py as a subprocess in its own temporary git repository with a test-only pattern and asserts exit code and output shape — tree, message file, range, stdin, conf-file fallback, every refusal class, masking, binaries skipped. Red at this commit: 14 of 15 fail, identically over three runs. Source: original Co-Authored-By: GPT-5.6 Sol Claude-Session: https://claude.ai/code/session_012Jj94rkp3tfHAxUkTCthgY --- ops/devlane/workflow/tests/test_term_wall.py | 196 +++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 ops/devlane/workflow/tests/test_term_wall.py diff --git a/ops/devlane/workflow/tests/test_term_wall.py b/ops/devlane/workflow/tests/test_term_wall.py new file mode 100644 index 0000000..da4dfc2 --- /dev/null +++ b/ops/devlane/workflow/tests/test_term_wall.py @@ -0,0 +1,196 @@ +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import unittest + + +CHECK = Path(__file__).resolve().parents[1] / "checks" / "term_wall.py" +PATTERN = "zz[q]orblat" +PLANT = "zzqorblat" +MASK = "[forbidden name]" + + +class TermWallTest(unittest.TestCase): + def setUp(self): + self.temporary_directory = tempfile.TemporaryDirectory() + self.root = Path(self.temporary_directory.name) + self._git("init", "-q") + self._git("config", "user.name", "Term Wall Test") + self._git("config", "user.email", "term-wall-test@example.invalid") + + def tearDown(self): + self.temporary_directory.cleanup() + + def _git(self, *arguments): + return subprocess.run( + ["git", "-C", str(self.root), *arguments], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=self._environment(PATTERN), + ) + + def _environment(self, pattern=PATTERN): + environment = { + "PATH": os.environ.get("PATH", os.defpath), + "HOME": str(self.root), + "LANG": "C", + "LC_ALL": "C", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + } + if pattern is not None: + environment["TERM_WALL"] = pattern + return environment + + def _run(self, *arguments, pattern=PATTERN, input_text=None, cwd=None): + return subprocess.run( + [sys.executable, str(CHECK), *map(str, arguments)], + cwd=str(cwd or self.root), + env=self._environment(pattern), + input=input_text, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + def _track(self, relative_path, content, binary=False): + path = self.root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + if binary: + path.write_bytes(content) + else: + path.write_text(content, encoding="utf-8") + self._git("add", "--", relative_path) + return path + + def _commit(self, message): + self._git("commit", "-q", "--allow-empty", "-m", message) + return self._git("rev-parse", "HEAD").stdout.strip() + + def assertClean(self, result): + self.assertEqual(0, result.returncode, result) + self.assertEqual("", result.stderr) + lines = result.stdout.splitlines() + self.assertEqual(1, len(lines), result.stdout) + self.assertTrue(lines[0].strip()) + + def assertHit(self, result, surface): + self.assertEqual(1, result.returncode, result) + self.assertEqual("", result.stderr) + lines = result.stdout.splitlines() + self.assertGreaterEqual(len(lines), 1) + for line in lines: + self.assertRegex(line, rf"^{re.escape(surface)}: .+: .+$") + self.assertIn(MASK, line) + self.assertMasked(result) + + def assertRefusal(self, result, refusal_class): + self.assertEqual(2, result.returncode, result) + self.assertEqual("", result.stdout) + lines = result.stderr.splitlines() + self.assertEqual(1, len(lines), result.stderr) + self.assertRegex( + lines[0], + rf"^{re.escape(refusal_class)}: expected .+; found .+; needed .+$", + ) + + def assertMasked(self, result): + combined = result.stdout + result.stderr + self.assertNotIn(PLANT, combined) + self.assertNotIn(PLANT.upper(), combined) + + def test_clean_tree(self): + self._track("notes.txt", "ordinary text\n") + self.assertClean(self._run("--root", self.root)) + + def test_planted_tracked_content(self): + self._track("notes.txt", f"before {PLANT} after\n") + self.assertHit(self._run("--root", self.root), "content") + + def test_planted_tracked_path(self): + self._track(f"docs/{PLANT}.txt", "ordinary text\n") + self.assertHit(self._run("--root", self.root), "path") + + def test_message_file_clean(self): + message = self.root / "message.txt" + message.write_text("An ordinary commit message\n", encoding="utf-8") + self.assertClean(self._run("--message-file", message)) + + def test_message_file_planted(self): + message = self.root / "message.txt" + message.write_text(f"Mention {PLANT.upper()} here\n", encoding="utf-8") + self.assertHit(self._run("--message-file", message), "message") + + def test_range_clean(self): + base = self._commit("base message") + head = self._commit("ordinary follow-up") + self.assertClean(self._run("--root", self.root, "--range", f"{base}..{head}")) + + def test_range_planted(self): + base = self._commit("base message") + head = self._commit(f"follow-up mentioning {PLANT}") + self.assertHit( + self._run("--root", self.root, "--range", f"{base}..{head}"), + "commit", + ) + + def test_stdin_clean_and_planted(self): + self.assertClean(self._run("--stdin", input_text="ordinary input\n")) + self.assertHit( + self._run("--stdin", input_text=f"input with {PLANT}\n"), + "stdin", + ) + + def test_missing_message_file_refuses(self): + result = self._run("--message-file", self.root / "does-not-exist") + self.assertRefusal(result, "message") + + def test_unresolvable_range_refuses(self): + result = self._run("--root", self.root, "--range", "missing..also-missing") + self.assertRefusal(result, "range") + + def test_not_a_git_work_tree_refuses(self): + with tempfile.TemporaryDirectory() as directory: + result = self._run("--root", directory, cwd=directory) + self.assertRefusal(result, "root") + + def test_unset_and_empty_pattern_refuse(self): + for pattern in (None, ""): + with self.subTest(pattern=pattern): + result = self._run("--root", self.root, pattern=pattern) + self.assertRefusal(result, "pattern") + self.assertEqual( + "pattern: expected TERM_WALL set; found empty; needed the org variable " + "(CI) or ops/bin/term-wall.conf (local)\n", + result.stderr, + ) + + def test_pattern_falls_back_to_conf_file(self): + config = self.root / "ops" / "bin" / "term-wall.conf" + config.parent.mkdir(parents=True) + config.write_text(PATTERN + "\n", encoding="utf-8") + self._track("notes.txt", f"configured match: {PLANT}\n") + self.assertHit( + self._run("--root", self.root, pattern=None), + "content", + ) + + def test_every_match_is_masked(self): + self._track("notes.txt", f"{PLANT} and {PLANT.upper()}\n") + result = self._run("--root", self.root) + self.assertHit(result, "content") + self.assertEqual(2, result.stdout.count(MASK)) + + def test_binary_files_are_skipped(self): + self._track("payload.bin", b"header\x00" + PLANT.encode("ascii") + b"\n", binary=True) + self._track("image.png", PLANT.encode("ascii") + b"\n", binary=True) + self.assertClean(self._run("--root", self.root)) + + +if __name__ == "__main__": + unittest.main() From 4a54cbaf36dd9edeaeb0fd6d445d17c0d953221e Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:45:52 -0400 Subject: [PATCH 11/18] =?UTF-8?q?ops:=20the=20term=20wall=20to=20its=20con?= =?UTF-8?q?tract=20=E2=80=94=20pattern=20from=20configuration=20only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit term_wall.py reads its pattern from TERM_WALL, else ops/bin/term-wall.conf (machine-local, gitignored, an .example beside it), else refuses on the wire; every hit is masked. The commit-msg hook exports the pattern from the conf and refuses on a hit, warning through on a refusal like the trailer check. apply-push's content and message guards take the same pattern and refuse when none is available — the lever cannot look, so it does not land. CONTRACT-term-wall.md carries the contract verbatim. Source: original Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012Jj94rkp3tfHAxUkTCthgY --- ops/bin/.gitignore | 1 + ops/bin/term-wall.conf.example | 5 ++ ops/devlane/dispatch/levers/apply-push.sh | 25 +++++-- ops/devlane/hooks/commit-msg | 18 ++++- .../workflow/checks/CONTRACT-term-wall.md | 75 +++++++++++++++++++ ops/devlane/workflow/checks/term_wall.py | 60 ++++++++++----- 6 files changed, 156 insertions(+), 28 deletions(-) create mode 100644 ops/bin/term-wall.conf.example create mode 100644 ops/devlane/workflow/checks/CONTRACT-term-wall.md diff --git a/ops/bin/.gitignore b/ops/bin/.gitignore index c54180c..2f7b5dd 100644 --- a/ops/bin/.gitignore +++ b/ops/bin/.gitignore @@ -1,2 +1,3 @@ # Owner-owned, machine-specific — copy from dispatch.conf.example. dispatch.conf +term-wall.conf diff --git a/ops/bin/term-wall.conf.example b/ops/bin/term-wall.conf.example new file mode 100644 index 0000000..d146b5d --- /dev/null +++ b/ops/bin/term-wall.conf.example @@ -0,0 +1,5 @@ +# Copy this file to the gitignored term-wall.conf beside it, holding the +# real pattern: one line, an extended, case-insensitive regular expression +# — the names this organisation does not use. The pattern is configuration, +# never tree content; the line below is a placeholder, not the pattern. +ex[a]mple-name diff --git a/ops/devlane/dispatch/levers/apply-push.sh b/ops/devlane/dispatch/levers/apply-push.sh index 85a5297..25c7e48 100755 --- a/ops/devlane/dispatch/levers/apply-push.sh +++ b/ops/devlane/dispatch/levers/apply-push.sh @@ -83,6 +83,20 @@ for item in "${denied[@]}"; do [[ -z "$item" || "$branch" != "$item" ]] || refuse "branch '$branch' is protected" done +# The term wall pattern is configuration, never tree content: taken from +# TERM_WALL, else from the gitignored ops/bin/term-wall.conf. Both guards +# below need it, and a wall that cannot look never passes — no pattern is +# a refusal, not a skip. +if [[ -z "${TERM_WALL:-}" ]]; then + wall_conf="$LEVERS_DIR/../../../bin/term-wall.conf" + if [[ -f "$wall_conf" ]]; then + IFS= read -r TERM_WALL <"$wall_conf" || true + fi +fi +[[ -n "${TERM_WALL:-}" ]] || + refuse "term wall pattern unavailable: TERM_WALL unset and ops/bin/term-wall.conf absent; the wall cannot look, so the lever cannot land" +export TERM_WALL + resolved_repo=$(realpath -e -- "$repo" 2>/dev/null) || refuse "--repo '$repo' does not resolve" git -C "$resolved_repo" rev-parse --git-dir >/dev/null 2>&1 || refuse "'$resolved_repo' is not a git repository" git -C "$resolved_repo" check-ref-format --branch "$branch" >/dev/null 2>&1 || refuse "invalid branch name '$branch'" @@ -258,7 +272,7 @@ if ! git -C "$worktree" diff --cached --unified=0 -- >"$record_dir/staged.diff" refuse "cannot read staged diff for content guard" fi python3 - "$record_dir/staged.diff" "$guard_report" <<'PY' -import re,sys +import os,re,sys diff,report=sys.argv[1:] added=[] # (path, lineno_in_added_hunk, text) path=None @@ -272,8 +286,9 @@ hits=[] home=re.compile(r"/home/[A-Za-z0-9._-]+/") # credential filenames introduced as content cred=re.compile(r"\b(auth\.json|id_rsa|id_ed25519|\.pem|\.p12|credentials(\.json)?)\b") -# names this organisation does not use — never affirmed, negated, or cited -wall=re.compile(r"s[c]ient[ _-]?db|u[s]cient",re.I) +# names this organisation does not use — never affirmed, negated, or cited; +# the pattern is configuration (TERM_WALL, exported by the lever), never here +wall=re.compile(os.environ["TERM_WALL"],re.I) for p,text in added: pth=p or "(unknown)" if wall.search(text) or wall.search(pth): @@ -304,11 +319,11 @@ else printf '%s\n' "$message_text" >"$record_dir/message.input" fi if ! python3 - "$record_dir/message.input" "$record_dir/message.final" "$job_id" "$patch_sha" <<'PY' -import re,sys +import os,re,sys src,out,job,digest=sys.argv[1:] s=open(src,encoding="utf-8").read().rstrip() if "\x00" in s or not s.strip(): raise SystemExit(1) -if re.search(r"s[c]ient[ _-]?db|u[s]cient",s,re.I): raise SystemExit(2) # the term wall, on the message +if re.search(os.environ["TERM_WALL"],s,re.I): raise SystemExit(2) # the term wall, on the message trailer=re.compile(r"^[A-Za-z0-9][A-Za-z0-9-]*: .+$") caller_last=re.split(r"\n[ \t]*\n",s)[-1].splitlines() separator="\n" if caller_last and all(trailer.match(x) for x in caller_last) else "\n\n" diff --git a/ops/devlane/hooks/commit-msg b/ops/devlane/hooks/commit-msg index 819867b..01e71b4 100755 --- a/ops/devlane/hooks/commit-msg +++ b/ops/devlane/hooks/commit-msg @@ -53,17 +53,27 @@ python3 "$check" --message-file "$msg_file" || { } # The term wall: names this organisation does not use, refused in the -# message before it becomes history. Same convention as above — a clone -# without the wall warns and lets the commit through; CI holds the wall. +# message before it becomes history. Same convention as above — the hook +# refuses only for the thing it is here to judge (a hit in the message, +# exit 1); a wall that cannot look (exit 2: no pattern) or is absent +# warns and lets the commit through; CI holds the wall. wall="$top/ops/devlane/workflow/checks/term_wall.py" if [ -f "$wall" ]; then - python3 "$wall" --message-file "$msg_file" || { + if [ -z "${TERM_WALL:-}" ] && [ -f "$top/ops/bin/term-wall.conf" ]; then + IFS= read -r TERM_WALL <"$top/ops/bin/term-wall.conf" + export TERM_WALL + fi + python3 "$wall" --message-file "$msg_file" + wall_rc=$? + if [ "$wall_rc" -eq 1 ]; then echo "" >&2 echo "commit-msg: the message above was NOT committed." >&2 echo " It carries a name this organisation does not use" >&2 echo " (the hits are listed, masked). Rewrite without it." >&2 exit 1 - } + elif [ "$wall_rc" -ne 0 ]; then + echo "commit-msg: term wall cannot look here (no pattern) — letting the message through; CI holds the wall." >&2 + fi else echo "commit-msg: term wall not present here — letting the message through; CI holds the wall." >&2 fi diff --git a/ops/devlane/workflow/checks/CONTRACT-term-wall.md b/ops/devlane/workflow/checks/CONTRACT-term-wall.md new file mode 100644 index 0000000..656d357 --- /dev/null +++ b/ops/devlane/workflow/checks/CONTRACT-term-wall.md @@ -0,0 +1,75 @@ +# The term wall — contract v1 + +Names this organisation does not use must not appear in any of its +repositories: not affirmed, not negated, not cited. The wall refuses +them, and the wall itself never carries them in any readable or +encoded form. + +## Instruments + +1. `.github/actions/term-wall/term-wall.sh` (run by the composite + action `.github/actions/term-wall`) — every repo's `ci` job runs it. +2. `ops/devlane/workflow/checks/term_wall.py` — the lane's local copy, + run by the commit-msg hook and by apply-push's guards. + +## The pattern is configuration, never tree content + +- The pattern is an extended, case-insensitive regular expression read + from the environment variable `TERM_WALL`. In CI the action takes it + from the org-level Actions variable `vars.TERM_WALL`. Locally the + Python check reads `TERM_WALL`, falling back to the gitignored file + `/ops/bin/term-wall.conf` (one line: the pattern). +- No tracked file may contain the pattern, a piece of it, or any + encoding of the names (hex, base64, bracket tricks, escapes). The + self-test's planted fault comes from `vars.TERM_WALL_PLANT` (a string + the pattern matches), never from the tree. +- An unset or empty pattern is a refusal, exit 2, stdout empty, one + line on stderr: `pattern: expected TERM_WALL set; found empty; + needed the org variable (CI) or ops/bin/term-wall.conf (local)`. + The wall never passes vacuously. + +## Surfaces (term-wall.sh) + +1. tracked content — every `git ls-files` path, binaries skipped, + case-insensitive; +2. tracked paths; +3. the commit messages of the change — `pull_request`: `base..head`; + `push`: `before..head`, or only the head commit when `before` is + all zeros — read from git (fetching what the checkout lacks), never + from an API: the action needs no token and declares none; +4. the pull request title and body (from the event payload); +5. the branch name (`GITHUB_HEAD_REF` for a PR, `GITHUB_REF_NAME` for + a push). + +Outside GitHub Actions (no `GITHUB_EVENT_PATH`), surfaces 1 and 2 run +against the current directory. The Python check covers surface 1 and +2 (`[--root DIR] [PATH ...]`), one message (`--message-file FILE`), a +range of commit messages (`--range BASE..HEAD`), or `--stdin`. + +## Outcomes, on the wire + +| exit | meaning | +|---|---| +| 0 | clean; one summary line on stdout | +| 1 | at least one hit; every hit printed on stdout as `: : `; a surface that could not be read (fetch failed, payload unreadable) is itself a hit — could-not-look is never a pass | +| 2 | refusal: pattern unset/empty, not a git work tree, missing message file, unresolvable range; stdout empty, one stderr line `class: expected …; found …; needed …` | + +The raw matched text never appears in any output. + +## Self-test (the `.github` repo's own ci) + +With `TERM_WALL_PLANT` as the planted fault: a planted file fires +(exit 1, masked hit), a clean neighbour stays quiet (exit 0), and an +empty `TERM_WALL` refuses (exit 2). + +## Tests + +Tests execute the real instrument as a subprocess inside temporary git +repositories they create; nothing about the wall is mocked. They set +`TERM_WALL` explicitly to a test-only pattern (for example +`zz[q]orblat`) and plant matches of it, so no forbidden name exists +anywhere. They are deterministic and hermetic: no network, no sleeps, +no dependence on the caller's cwd, environment, or git identity +(configure user.name/user.email in each temp repo). Every test asserts +the exit code and the output shape. Push and pull-request events are +simulated with an event JSON file and the `GITHUB_*` variables. diff --git a/ops/devlane/workflow/checks/term_wall.py b/ops/devlane/workflow/checks/term_wall.py index 8feb21c..1673ba7 100644 --- a/ops/devlane/workflow/checks/term_wall.py +++ b/ops/devlane/workflow/checks/term_wall.py @@ -3,9 +3,12 @@ Some names must not appear in this organisation's trees, commit messages, or pull-request text — not affirmed, not negated, not cited. -This check is the wall. It never spells the names it refuses (the -pattern below would otherwise be its own first hit) and every hit it -prints is masked, so its output does not carry what the tree may not. +This check is the wall. The pattern is configuration, never tree +content: it is read from the environment variable `TERM_WALL`, falling +back to the gitignored file `/ops/bin/term-wall.conf` (one line: +an extended, case-insensitive regular expression). No pattern is a +refusal, never a vacuous pass. Every hit this check prints is masked, +so its output does not carry what the tree may not. term_wall.py [--root DIR] [PATH ...] tracked files (default: all) term_wall.py --message-file FILE one commit message @@ -15,19 +18,19 @@ Exit 0 clean; 1 on a hit, every hit printed; 2 on a refusal, one line on stderr in the lane's shape: `class: expected …; found …; needed …`. The same wall stands in CI (the org's term-wall action) — this copy is -the local hook's and the landing lever's. +the local hook's and the landing lever's. Contract: +CONTRACT-term-wall.md beside this file. """ from __future__ import annotations import argparse +import os import subprocess import sys from pathlib import Path import re -#: The names, spelled so that this file passes its own wall. -WALL = re.compile(r"s[c]ient[ _-]?db|u[s]cient", re.IGNORECASE) MASK = "[forbidden name]" SKIP_SUFFIXES = {".png", ".jpg", ".jpeg", ".gif", ".pdf", ".zip", ".gz", ".db", ".phar"} @@ -37,11 +40,29 @@ def refuse(cls: str, expected: str, found: str, needed: str) -> None: sys.exit(2) -def scan_text(text: str, where: str) -> list[str]: +def load_wall(root: Path) -> re.Pattern[str]: + pattern = os.environ.get("TERM_WALL", "") + if not pattern: + conf = root / "ops" / "bin" / "term-wall.conf" + if conf.is_file(): + text = conf.read_text(encoding="utf-8", errors="replace").strip() + pattern = text.splitlines()[0].strip() if text else "" + if not pattern: + refuse("pattern", "TERM_WALL set", "empty", + "the org variable (CI) or ops/bin/term-wall.conf (local)") + try: + return re.compile(pattern, re.IGNORECASE) + except re.error as exc: + refuse("pattern", "a compilable regular expression", f"({exc})", + "a valid extended regex in TERM_WALL or ops/bin/term-wall.conf") + raise AssertionError("unreachable") + + +def scan_text(wall: re.Pattern[str], text: str, where: str) -> list[str]: hits = [] for n, line in enumerate(text.splitlines(), 1): - if WALL.search(line): - hits.append(f"{where}:{n}: {WALL.sub(MASK, line).strip()[:160]}") + if wall.search(line): + hits.append(f"{where}:{n}: {wall.sub(MASK, line).strip()[:160]}") return hits @@ -56,22 +77,22 @@ def tracked(root: Path, paths: list[str]) -> list[str]: return [p for p in out.decode("utf-8", "surrogateescape").split("\0") if p] -def scan_tree(root: Path, paths: list[str]) -> list[str]: +def scan_tree(wall: re.Pattern[str], root: Path, paths: list[str]) -> list[str]: hits = [] for rel in tracked(root, paths): - if WALL.search(rel): - hits.append(f"{rel}: forbidden name in the path ({WALL.sub(MASK, rel)})") + if wall.search(rel): + hits.append(f"{rel}: forbidden name in the path ({wall.sub(MASK, rel)})") p = root / rel if p.suffix.lower() in SKIP_SUFFIXES or not p.is_file(): continue data = p.read_bytes() if b"\0" in data[:8000]: continue - hits.extend(scan_text(data.decode("utf-8", "replace"), rel)) + hits.extend(scan_text(wall, data.decode("utf-8", "replace"), rel)) return hits -def scan_range(root: Path, rng: str) -> list[str]: +def scan_range(wall: re.Pattern[str], root: Path, rng: str) -> list[str]: try: out = subprocess.run( ["git", "-C", str(root), "log", "--format=%H%x00%B%x00", rng], @@ -87,7 +108,7 @@ def scan_range(root: Path, rng: str) -> list[str]: for sha, body in zip(parts[0::2], parts[1::2]): sha = sha.strip() if sha: - hits.extend(scan_text(body, f"commit {sha[:12]}")) + hits.extend(scan_text(wall, body, f"commit {sha[:12]}")) return hits @@ -101,20 +122,21 @@ def main(argv=None) -> int: ap.add_argument("paths", nargs="*", help="tracked paths to scan (default: every tracked file)") args = ap.parse_args(argv) root = Path(args.root) + wall = load_wall(root) if args.message_file: path = Path(args.message_file) if not path.is_file(): refuse("message", "a readable message file", str(path), "the path git hands the commit-msg hook") - hits = scan_text(path.read_text(encoding="utf-8", errors="replace"), "message") + hits = scan_text(wall, path.read_text(encoding="utf-8", errors="replace"), "message") elif args.rng: - hits = scan_range(root, args.rng) + hits = scan_range(wall, root, args.rng) elif args.stdin: - hits = scan_text(sys.stdin.read(), "stdin") + hits = scan_text(wall, sys.stdin.read(), "stdin") else: if not root.is_dir(): refuse("root", "a directory", str(root), "an existing work tree") - hits = scan_tree(root, args.paths) + hits = scan_tree(wall, root, args.paths) for hit in hits: print(hit) From 80a0c824f17162f7d25ec25c342caec918da1e0a Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:51:25 -0400 Subject: [PATCH 12/18] ops: pin the term wall's hit format and refusal classes Same pin as the org action's contract: hit locations carry no colon, title and body are separate surfaces, refusal classes are named. The implementation moves to it; the tests stand. Source: original Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012Jj94rkp3tfHAxUkTCthgY --- .../workflow/checks/CONTRACT-term-wall.md | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/ops/devlane/workflow/checks/CONTRACT-term-wall.md b/ops/devlane/workflow/checks/CONTRACT-term-wall.md index 656d357..9bf3f69 100644 --- a/ops/devlane/workflow/checks/CONTRACT-term-wall.md +++ b/ops/devlane/workflow/checks/CONTRACT-term-wall.md @@ -50,11 +50,31 @@ range of commit messages (`--range BASE..HEAD`), or `--stdin`. | exit | meaning | |---|---| -| 0 | clean; one summary line on stdout | -| 1 | at least one hit; every hit printed on stdout as `: : `; a surface that could not be read (fetch failed, payload unreadable) is itself a hit — could-not-look is never a pass | -| 2 | refusal: pattern unset/empty, not a git work tree, missing message file, unresolvable range; stdout empty, one stderr line `class: expected …; found …; needed …` | +| 0 | clean; exactly one summary line on stdout | +| 1 | at least one hit, every hit printed on stdout in the pinned format below; a surface that could not be read (fetch failed, payload unreadable) is itself a hit — could-not-look is never a pass | +| 2 | refusal; stdout empty, one stderr line `: expected …; found …; needed …` | -The raw matched text never appears in any output. +Refusal classes: `pattern` (unset or empty), `git work tree` (not +inside one), `message` (missing message file), `range` (unresolvable). + +## Hit format, pinned + +One stdout line per hit: `: : `. The location never contains a +colon. The raw matched text never appears in any output. + +| surface | location | +|---|---| +| `content` | ` line ` | +| `path` | `` | +| `commit messages` | ` line ` | +| `pull request title` | `line ` | +| `pull request body` | `line ` | +| `branch name` | `` | +| `event payload` | `` — the hit when the payload cannot be read | +| `message` (`--message-file`) | `line ` | +| `range` (`--range`) | ` line ` | +| `stdin` | `line ` | ## Self-test (the `.github` repo's own ci) From 63f140829033e1a2e0a85769af88a69abf6e5baf Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:53:38 -0400 Subject: [PATCH 13/18] ops: pin how CI hands the term wall its pattern A composite action cannot read vars (GitHub refuses the template: 'Unrecognized named-value: vars', PR#8 run 33568333825), so the calling step passes env TERM_WALL from vars.TERM_WALL and the action refuses when it did not. Same for the self-test's planted fault. Source: original Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012Jj94rkp3tfHAxUkTCthgY --- ops/devlane/workflow/checks/CONTRACT-term-wall.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ops/devlane/workflow/checks/CONTRACT-term-wall.md b/ops/devlane/workflow/checks/CONTRACT-term-wall.md index 9bf3f69..a81af96 100644 --- a/ops/devlane/workflow/checks/CONTRACT-term-wall.md +++ b/ops/devlane/workflow/checks/CONTRACT-term-wall.md @@ -15,8 +15,10 @@ encoded form. ## The pattern is configuration, never tree content - The pattern is an extended, case-insensitive regular expression read - from the environment variable `TERM_WALL`. In CI the action takes it - from the org-level Actions variable `vars.TERM_WALL`. Locally the + from the environment variable `TERM_WALL`. In CI the calling step + passes it: `env: TERM_WALL: ${{ vars.TERM_WALL }}` — a composite + action cannot read `vars` itself, so the action declares no default + and refuses when the step did not pass one. Locally the Python check reads `TERM_WALL`, falling back to the gitignored file `/ops/bin/term-wall.conf` (one line: the pattern). - No tracked file may contain the pattern, a piece of it, or any @@ -78,7 +80,7 @@ colon. The raw matched text never appears in any output. ## Self-test (the `.github` repo's own ci) -With `TERM_WALL_PLANT` as the planted fault: a planted file fires +With the calling step passing `env: TERM_WALL_PLANT: ${{ vars.TERM_WALL_PLANT }}` as the planted fault: a planted file fires (exit 1, masked hit), a clean neighbour stays quiet (exit 0), and an empty `TERM_WALL` refuses (exit 2). From e24bd2732fb5ea6458419c8d9a9143f593b63b10 Mon Sep 17 00:00:00 2001 From: Apply Push Bridge Date: Tue, 1 Sep 2026 19:01:11 -0400 Subject: [PATCH 14/18] ops: pin the workbench CI actions to release commits A mutable tag (`@v4`, `@v2`) lets whoever controls the upstream tag change what this workflow runs. Pin checkout, setup-php and cache to the commit each tag resolves to today; the tag stays as a comment. Source: original Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012Jj94rkp3tfHAxUkTCthgY Apply-Push-Job: 20260901T230110Z-apply-push-b4b8d4 Patch-SHA256: c1b69e625272039008a976d622d0f824ece2cfd936f36bd4e1ea63fa8288918c --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab35eeb..fe73e4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,14 +9,14 @@ jobs: ci: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: persist-credentials: false - - uses: shivammathur/setup-php@v2 + - uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # v2 with: php-version: '8.4' - run: composer validate --strict - - uses: actions/cache@v4 + - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: ~/.cache/composer/files key: composer-${{ hashFiles('composer.lock') }} From f5b60355a5b5cc2c5818c546d268165b07065eae Mon Sep 17 00:00:00 2001 From: Apply Push Bridge Date: Tue, 1 Sep 2026 19:03:01 -0400 Subject: [PATCH 15/18] ops: bring the lane term wall onto the pinned contract The pinned wire: one summary line on a clean run, one `: : ` per hit with the location never carrying a colon, four refusal classes (`pattern`, `git work tree`, `message`, `range`). Implementation moves; the tests stand. Source: original Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012Jj94rkp3tfHAxUkTCthgY Apply-Push-Job: 20260901T230301Z-apply-push-8bf543 Patch-SHA256: 18a95460cadb216e175fcbbfaad7424db85f9280ea13b2d19dd7a99da09dc5c9 --- ops/devlane/workflow/checks/term_wall.py | 68 +++++++++++++++--------- 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/ops/devlane/workflow/checks/term_wall.py b/ops/devlane/workflow/checks/term_wall.py index 1673ba7..f6b3afb 100644 --- a/ops/devlane/workflow/checks/term_wall.py +++ b/ops/devlane/workflow/checks/term_wall.py @@ -15,10 +15,12 @@ term_wall.py --range BASE..HEAD every commit message in the range term_wall.py --stdin text on stdin -Exit 0 clean; 1 on a hit, every hit printed; 2 on a refusal, one line -on stderr in the lane's shape: `class: expected …; found …; needed …`. -The same wall stands in CI (the org's term-wall action) — this copy is -the local hook's and the landing lever's. Contract: +Exit 0 clean, exactly one summary line on stdout; 1 on a hit, one +stdout line per hit — `: : ` — and nothing else; 2 on a refusal, one +line on stderr in the lane's shape: `class: expected …; found …; +needed …`. The same wall stands in CI (the org's term-wall action) — +this copy is the local hook's and the landing lever's. Contract: CONTRACT-term-wall.md beside this file. """ @@ -58,41 +60,45 @@ def load_wall(root: Path) -> re.Pattern[str]: raise AssertionError("unreachable") -def scan_text(wall: re.Pattern[str], text: str, where: str) -> list[str]: +def scan_text(wall: re.Pattern[str], text: str, surface: str, where: str = "") -> list[str]: hits = [] for n, line in enumerate(text.splitlines(), 1): if wall.search(line): - hits.append(f"{where}:{n}: {wall.sub(MASK, line).strip()[:160]}") + location = f"{where} line {n}" if where else f"line {n}" + hits.append(f"{surface}: {location}: {wall.sub(MASK, line).strip()[:160]}") return hits -def tracked(root: Path, paths: list[str]) -> list[str]: +def tracked(wall: re.Pattern[str], root: Path, paths: list[str]) -> list[str]: try: out = subprocess.run( ["git", "-C", str(root), "ls-files", "-z", "--", *paths], capture_output=True, check=True, ).stdout except (OSError, subprocess.CalledProcessError) as exc: - refuse("root", "a git work tree", f"{root} ({exc})", "run inside a clone or pass --root") + refuse("root", "a git work tree", wall.sub(MASK, f"{root} ({exc})"), + "run inside a clone or pass --root") return [p for p in out.decode("utf-8", "surrogateescape").split("\0") if p] -def scan_tree(wall: re.Pattern[str], root: Path, paths: list[str]) -> list[str]: +def scan_tree(wall: re.Pattern[str], root: Path, paths: list[str]) -> tuple[list[str], int]: hits = [] - for rel in tracked(root, paths): + files = tracked(wall, root, paths) + for rel in files: + masked_rel = wall.sub(MASK, rel) if wall.search(rel): - hits.append(f"{rel}: forbidden name in the path ({wall.sub(MASK, rel)})") + hits.append(f"path: {masked_rel}: {masked_rel}") p = root / rel if p.suffix.lower() in SKIP_SUFFIXES or not p.is_file(): continue data = p.read_bytes() if b"\0" in data[:8000]: continue - hits.extend(scan_text(wall, data.decode("utf-8", "replace"), rel)) - return hits + hits.extend(scan_text(wall, data.decode("utf-8", "replace"), "content", masked_rel)) + return hits, len(files) -def scan_range(wall: re.Pattern[str], root: Path, rng: str) -> list[str]: +def scan_range(wall: re.Pattern[str], root: Path, rng: str) -> tuple[list[str], int]: try: out = subprocess.run( ["git", "-C", str(root), "log", "--format=%H%x00%B%x00", rng], @@ -100,16 +106,18 @@ def scan_range(wall: re.Pattern[str], root: Path, rng: str) -> list[str]: ).stdout except (OSError, subprocess.CalledProcessError) as exc: detail = getattr(exc, "stderr", b"") or b"" + first = detail.decode("utf-8", "replace").strip().splitlines() refuse("range", "BASE..HEAD git can resolve", - f"{rng} ({detail.decode('utf-8', 'replace').strip() or exc})", + wall.sub(MASK, f"{rng} ({first[0] if first else exc})"), "a range of reachable commits") - hits = [] + hits, commits = [], 0 parts = out.decode("utf-8", "replace").split("\0") for sha, body in zip(parts[0::2], parts[1::2]): sha = sha.strip() if sha: - hits.extend(scan_text(wall, body, f"commit {sha[:12]}")) - return hits + commits += 1 + hits.extend(scan_text(wall, body, "commit", sha[:12])) + return hits, commits def main(argv=None) -> int: @@ -127,20 +135,28 @@ def main(argv=None) -> int: if args.message_file: path = Path(args.message_file) if not path.is_file(): - refuse("message", "a readable message file", str(path), "the path git hands the commit-msg hook") + refuse("message", "a readable message file", wall.sub(MASK, str(path)), + "the path git hands the commit-msg hook") hits = scan_text(wall, path.read_text(encoding="utf-8", errors="replace"), "message") + summary = "term wall: clean (1 commit message)" elif args.rng: - hits = scan_range(wall, root, args.rng) + hits, commits = scan_range(wall, root, args.rng) + summary = f"term wall: clean ({commits} commit message(s))" elif args.stdin: hits = scan_text(wall, sys.stdin.read(), "stdin") + summary = "term wall: clean (stdin)" else: if not root.is_dir(): - refuse("root", "a directory", str(root), "an existing work tree") - hits = scan_tree(wall, root, args.paths) - - for hit in hits: - print(hit) - return 1 if hits else 0 + refuse("root", "a directory", wall.sub(MASK, str(root)), "an existing work tree") + hits, files = scan_tree(wall, root, args.paths) + summary = f"term wall: clean ({files} tracked file(s), content and path)" + + if hits: + for hit in hits: + print(hit) + return 1 + print(summary) + return 0 if __name__ == "__main__": From 9149aa7b6dcad3d62d57afb266f275fee0da37dd Mon Sep 17 00:00:00 2001 From: Apply Push Bridge Date: Tue, 1 Sep 2026 19:10:03 -0400 Subject: [PATCH 16/18] ops: run the term wall in CI Add the organisation's term wall to the `ci` job, right after checkout, pinned to the commit under review in minspec/.github#8. The step reads its pattern from the repository variable TERM_WALL and refuses when that is unset, so an unconfigured repository fails instead of passing. Source: original Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012Jj94rkp3tfHAxUkTCthgY Apply-Push-Job: 20260901T231003Z-apply-push-db7712 Patch-SHA256: 35bebcb925860d46a10042fcd31e8877c9a1208df74397d30dbda1672a7e2728 --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe73e4a..1f85e4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,10 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: persist-credentials: false + - name: term wall + uses: minspec/.github/.github/actions/term-wall@6175b67bd0df4710dd0bb8b79df413b452265c6c # minspec/.github#8 + env: + TERM_WALL: ${{ vars.TERM_WALL }} - uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # v2 with: php-version: '8.4' From 2672aab34a2f2f5b67dc63e5fe05bdc5606b2db2 Mon Sep 17 00:00:00 2001 From: Apply Push Bridge Date: Tue, 1 Sep 2026 20:40:28 -0400 Subject: [PATCH 17/18] repo: move the wall to the object-store scan minspec/.github#10 pins the content surface to every tracked blob, read from the object store, bytewise, symlinks never followed. Point the wall step at that merge. Source: original Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012Jj94rkp3tfHAxUkTCthgY Apply-Push-Job: 20260902T004028Z-apply-push-4faea8 Patch-SHA256: 57e5b08eab2bf07672c3ea101dcf7443de3419d46713583a712cf8cef7c1c6ba --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f85e4e..b3d5983 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,7 @@ name: ci on: pull_request: + types: [opened, synchronize, reopened, edited] push: branches: [dev, main] permissions: @@ -13,7 +14,7 @@ jobs: with: persist-credentials: false - name: term wall - uses: minspec/.github/.github/actions/term-wall@6175b67bd0df4710dd0bb8b79df413b452265c6c # minspec/.github#8 + uses: minspec/.github/.github/actions/term-wall@a5d88bb5b9bb744cf23c8829436e32f03d58c79d # minspec/.github#10 env: TERM_WALL: ${{ vars.TERM_WALL }} - uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # v2 From a4e0273ffa14d37db97a4c7fd5e03dce64417f80 Mon Sep 17 00:00:00 2001 From: xor <127287135+xormania@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:28:01 -0400 Subject: [PATCH 18/18] ops: treat killed zombies as terminated in tests Process-group termination can leave dead descendants in zombie state until container PID 1 reaps them. Make the isolation assertions distinguish those non-runnable processes from surviving workers instead of relying on kill(pid, 0) alone. Source: original Co-Authored-By: OpenAI Codex --- ops/devlane/dispatch/tests/launch_support.py | 12 ++++++++++++ ops/devlane/task/tests/test_run.py | 8 ++++++++ 2 files changed, 20 insertions(+) diff --git a/ops/devlane/dispatch/tests/launch_support.py b/ops/devlane/dispatch/tests/launch_support.py index 14a2df2..8042bdb 100644 --- a/ops/devlane/dispatch/tests/launch_support.py +++ b/ops/devlane/dispatch/tests/launch_support.py @@ -663,6 +663,18 @@ def sha256_file(path) -> str: def pid_is_alive(pid: int) -> bool: + # A group SIGKILL can leave a descendant as a zombie until the host's + # init process reaps it. ``kill(pid, 0)`` still succeeds for zombies, + # even though they cannot execute and therefore are not survivors of the + # isolation boundary. Check procfs first so the process-group assertion + # measures live workers rather than the reaping behaviour of PID 1 (which + # is notably delayed in some CI containers). + stat = Path(f"/proc/{pid}/stat") + try: + if stat.read_text(encoding="utf-8").split()[2] == "Z": + return False + except (FileNotFoundError, IndexError, OSError): + pass try: os.kill(pid, 0) except ProcessLookupError: diff --git a/ops/devlane/task/tests/test_run.py b/ops/devlane/task/tests/test_run.py index f3f48d6..c2925be 100644 --- a/ops/devlane/task/tests/test_run.py +++ b/ops/devlane/task/tests/test_run.py @@ -1307,6 +1307,14 @@ def _orphan_maker(self): @staticmethod def _alive(pid): + # kill(0) also reports zombies as present. A killed orphan can stay + # zombied until PID 1 reaps it in a container, but it is no longer a + # runnable descendant and must not make this isolation check flaky. + try: + if Path(f"/proc/{pid}/stat").read_text().split()[2] == "Z": + return False + except (FileNotFoundError, IndexError, OSError): + pass try: os.kill(pid, 0) return True