diff --git a/README.md b/README.md index d556fa4..fdadd14 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Every write is checked against the files that worker was allowed to touch, and a - **Claude Code ≥ 2.1.219.** Compound V runs on the native Workflow runtime and on native hook events (`PreToolUse`, `UserPromptSubmit`, `PostCompact`, `Stop`). Older versions lack them. The floor is checked at session start: the `SessionStart` banner reads `claude --version` and appends one warning line when the running version is below it. - **One ambient cost.** The lane-guard hook runs on every `Write`/`Edit`/`Bash` call. What it costs depends on the machine — measure it on yours; the recipe is in [AGENTS.md](AGENTS.md). -- **One project setting.** `worktree.baseRef` has to be `head` for jobs that depend on each other. It is a native Claude Code setting, not a Compound V one — see Install. +- **One project setting.** `worktree.baseRef` should be `head` for jobs that depend on each other: without it a dependent job's agent runs in the main checkout, and dependent jobs that would have run in parallel are serialized so their writes stay attributable. It is a native Claude Code setting, not a Compound V one — see Install. ## Install ``` @@ -26,7 +26,7 @@ Every write is checked against the files that worker was allowed to touch, and a - **Antigravity:** install the `agy` CLI → log in Then the one setting. `worktree.baseRef` is a **native Claude Code project setting** in the project's `.claude/settings.json`, with two values: `fresh` (the default) and `head`. It is -project-wide: `head` branches every worktree from the current `HEAD`, your own `--worktree` sessions included. A job that depends on another needs it, or its worktree cannot see that job. +project-wide: `head` branches every worktree from the current `HEAD`, your own `--worktree` sessions included. A job that depends on another needs it, or its worktree cannot see that job — without it such a job runs its agent in the main checkout instead, and a wave carrying more than one of those is split so each runs alone (one before-image cannot attribute two concurrent writers). So the setting buys parallelism, not correctness: the run still completes without it, one job at a time. ```json { "worktree": { "baseRef": "head" } } diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 844ce14..12ccfc6 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -177,7 +177,7 @@ At runtime the same fail-closed posture applies to individual commands: *"no cla **Fix:** add `{"worktree": {"baseRef": "head"}}` to the project's `.claude/settings.json` (merge — do not overwrite `permissions`/`hooks`/`env` or any other existing key). `/v:init` offers to make this edit for you; see [`v-init.md`](commands/v-init.md) Step 4d. This is a native, project-wide Claude Code setting (not a Compound V config key, and not the same file as `.claude/compound-v.json`) — the only two legal values are `"fresh"` (default) and `"head"`. -**Related:** the finalizer now takes its record of where a job ran from the emitter's own gate receipt rather than trusting the manifest's `isolation` label (finding 89), so a direct-mode dependent job is not refused at integration on that account alone. What `baseRef: head` buys is isolation, not survival — without it, several dependent jobs in the same wave share one checkout and can collide with each other's edits. +**Related:** the finalizer now takes its record of where a job ran from the emitter's own gate receipt rather than trusting the manifest's `isolation` label (finding 89), so a direct-mode dependent job is not refused at integration on that account alone. What `baseRef: head` buys is **parallelism**, not survival. Two or more dependent jobs sharing one checkout cannot be attributed — a single before-image cannot separate concurrent writers, so each job's diff carries the other's files and both are BLOCKED for out-of-lane writes. The emitter no longer allows that shape: a wave carrying 2+ downgraded jobs is split so each runs alone, and emit says on stderr which jobs it serialized and that this setting restores full parallelism. So without the setting a dependent wave still completes, just one job at a time. ## The lane guard never denies anything diff --git a/scripts/compound-v-emit-workflow.py b/scripts/compound-v-emit-workflow.py index 5deb9af..4512d75 100644 --- a/scripts/compound-v-emit-workflow.py +++ b/scripts/compound-v-emit-workflow.py @@ -1813,6 +1813,83 @@ def _worktree_base_is_head(repo_root): return False +def _agent_isolation_downgraded(job, abs_repo_root): + """True when the manifest asked for ``worktree`` but the AGENT will run in the + main checkout anyway — the `depends_on` rule, unless `worktree.baseRef: head`. + + Only a CLAUDE job can be downgraded this way. An external backend (codex, cursor, + antigravity, …) owns its own worktree: `job_entry` pins its manifest isolation to + `direct`, and the gate still runs in worktree mode over the worker's tree via the + `externalBackend` branch of the emitted script. Such a job is fully attributable + and must NOT be serialized — treating it as downgraded would cost parallelism for + nothing. + + This is the exact NEGATION of the `agent_isolation` expression in `job_entry`, + written as one predicate rather than a second copy of the positive condition: + the agent runs in the main checkout whenever it does not get a real worktree. A + manifest-`direct` claude job is therefore counted too — it is also a main-tree + writer, and two of those in one wave is the same unattributable shape (invariant + 7 makes that unreachable through a validated manifest, but `build_plan` does not + validate, so the predicate should not depend on it). + """ + if (job.get("backend") or "claude") != "claude": + return False + gets_real_worktree = ( + (job.get("isolation") or "direct") == "worktree" + and (not (job.get("depends_on") or []) + or _worktree_base_is_head(abs_repo_root)) + ) + return not gets_real_worktree + + +def _serialize_unattributable_waves(waves, abs_repo_root): + """Split a wave that would run TWO OR MORE downgraded jobs at once in ONE tree. + + A downgraded job runs its agent in the main checkout, and direct mode attributes + its writes with a single before-image snapshot. One such job per wave is exactly + what that snapshot handles, and the comment on `agent_isolation` says so. Two is + a different thing: one before-image cannot separate two concurrent writers, so + each job's diff contains the other's files and BOTH are BLOCKED for out-of-lane + writes — a run that is guaranteed to fail before any code is judged. + + Observed on a seven-job run in a project with no `.claude/settings.json`: wave 2 + ran `task-1` ∥ `task-2`, both `isolation: worktree`, both `depends_on` the wave-0 + job; each blocked carrying the other's entire lane. + + The validator REQUIRES `isolation: worktree` for parallel jobs and + partition-reviewer verifies it, so the manifest passes an invariant the run then + does not honour. Rather than emit that run, give each downgraded job its own + wave: a wave is already a barrier, so more barriers is strictly safer, and one + writer at a time is the case attribution can actually handle. Jobs that were NOT + downgraded keep their real worktrees and stay together. + + Returns ``(waves, notes)``; ``notes`` is empty when nothing was re-ordered. + """ + out, notes = [], [] + for wave in waves: + downgraded = [j for j in wave if _agent_isolation_downgraded(j, abs_repo_root)] + if len(downgraded) < 2: + out.append(wave) + continue + # Partition by id, not by dict equality: `j not in downgraded` compares job + # dicts by VALUE, which is only safe while topo_waves rejects duplicate ids. + _down_ids = {j.get("id") for j in downgraded} + others = [j for j in wave if j.get("id") not in _down_ids] + if others: + out.append(others) + for job in downgraded: + out.append([job]) + notes.append( + "wave with %s concurrent main-checkout jobs (%s) serialized: their " + "manifest `isolation: worktree` is downgraded because they carry " + "depends_on and this project has no `worktree.baseRef: \"head\"` in " + ".claude/settings.json, and one before-image cannot attribute two " + "concurrent writers. Set that key to get real worktrees and full " + "parallelism." % (len(downgraded), ", ".join( + str(j.get("id")) for j in downgraded))) + return out, notes + + def build_plan(manifest, run_dir, repo_root, python_bin, self_path, scope_check, fastpath, workers_dir, recall=True, recall_results_root=None, recall_engine=None): @@ -1851,6 +1928,13 @@ def build_plan(manifest, run_dir, repo_root, python_bin, self_path, max_parallel = manifest.get("max_parallel") or 4 jobs = manifest.get("jobs") or [] waves = topo_waves(jobs, max_parallel) + # A wave whose agents were all downgraded into the SAME checkout cannot be + # attributed (see _serialize_unattributable_waves). Serialize it here rather + # than emit a run that is guaranteed to BLOCK, and tell the operator why — + # the downgrade was previously invisible until the gate failed two waves later. + waves, _isolation_notes = _serialize_unattributable_waves(waves, abs_repo_root) + for _note in _isolation_notes: + sys.stderr.write("compound-v: %s\n" % _note) # A `test_contract` block is what makes resolution POSSIBLE at all. Without # one, `resolve-tests` fails closed and writes no file — and the worker # scripts reject a `--test-contract-file` that does not exist (exit 2). So the @@ -2177,6 +2261,12 @@ def job_entry(job): "retry": retry_config(manifest), "escalation": escalation_map(), "waves": [[job_entry(j) for j in wave] for wave in waves], + # WHY THE WAVE PLAN MAY NOT MATCH THE PARTITION MAP. A stderr line is gone + # the moment emit finishes, and `dispatch.workflow.js` is what gets committed + # — an auditor comparing the reviewed partition (N waves) against the run + # (N+k) would otherwise have no explanation for the difference. Empty on + # every run that was not re-shaped. + "isolation_notes": list(_isolation_notes), } @@ -6042,6 +6132,75 @@ def selftest(): _check("worktree.baseRef: head — a dependent worktree job gets a REAL worktree", _br_b2.get("agent_isolation") == "worktree") + # TWO downgraded jobs in ONE wave are unattributable: a single before-image + # cannot separate two concurrent writers in the same checkout, so both jobs + # BLOCK carrying each other's lane. Observed live before this guard existed. + _pw_repo = os.path.join(tmp, "pw-repo"); os.makedirs(_pw_repo) + _pw_man = {"run_id": "pw", "jobs": [ + {"id": "a", "isolation": "worktree", "write_allowed": ["a/**"]}, + {"id": "b", "isolation": "worktree", "depends_on": ["a"], "write_allowed": ["b/**"]}, + {"id": "c", "isolation": "worktree", "depends_on": ["a"], "write_allowed": ["c/**"]}]} + _pw_man["_manifest_path"] = os.path.join(_pw_repo, "manifest.yaml") + _pw_plan = build_plan(_with_body(_pw_man), os.path.join(tmp, "pw-run"), _pw_repo, + "/usr/bin/python3", os.path.abspath(__file__), + SCOPE_CHECK_DEFAULT, FASTPATH_DEFAULT, tmp) + _pw_waves = [[e["id"] for e in w] for w in _pw_plan["waves"]] + _check("no baseRef: two downgraded dependents never share a wave", + all(len([i for i in w if i in ("b", "c")]) <= 1 for w in _pw_waves), + str(_pw_waves)) + _check("no baseRef: serializing does not drop a job", + sorted(i for w in _pw_waves for i in w if i in ("b", "c")) == ["b", "c"], + str(_pw_waves)) + # With real worktrees each job is attributable, so parallelism is kept. + os.makedirs(os.path.join(_pw_repo, ".claude")) + with open(os.path.join(_pw_repo, ".claude", "settings.json"), "w") as fh: + fh.write('{"worktree": {"baseRef": "head"}}') + _pw_plan2 = build_plan(_with_body(_pw_man), os.path.join(tmp, "pw-run"), _pw_repo, + "/usr/bin/python3", os.path.abspath(__file__), + SCOPE_CHECK_DEFAULT, FASTPATH_DEFAULT, tmp) + _pw_waves2 = [[e["id"] for e in w] for w in _pw_plan2["waves"]] + _check("worktree.baseRef: head — the two dependents STAY in one wave", + any(sorted(i for i in w if i in ("b", "c")) == ["b", "c"] for w in _pw_waves2), + str(_pw_waves2)) + _nd = os.path.join(tmp, "nd"); os.makedirs(_nd) + _check("_agent_isolation_downgraded: worktree + depends_on + no baseRef", + _agent_isolation_downgraded( + {"isolation": "worktree", "depends_on": ["a"]}, _nd) is True) + _check("_agent_isolation_downgraded: a job that GETS a real worktree is not one", + _agent_isolation_downgraded({"isolation": "worktree"}, _nd) is False + and _agent_isolation_downgraded( + {"isolation": "worktree", "depends_on": ["a"]}, _pw_repo) is False) + # A manifest-`direct` claude job is a main-tree writer too. Invariant 7 makes + # two of them in one parallel wave unreachable through a VALIDATED manifest, + # but build_plan does not validate, so the predicate does not lean on that. + _check("_agent_isolation_downgraded: a manifest-direct claude job counts", + _agent_isolation_downgraded({"isolation": "direct"}, _nd) is True) + # An external backend owns its own worktree and is attributable, so it is + # never "downgraded" and must keep its parallelism. + _check("_agent_isolation_downgraded: an external backend is NOT downgraded", + _agent_isolation_downgraded( + {"backend": "codex", "isolation": "worktree", + "depends_on": ["a"]}, _nd) is False) + _pw_man_ext = {"run_id": "pwx", "jobs": [ + {"id": "a", "isolation": "worktree", "write_allowed": ["a/**"]}, + {"id": "b", "backend": "codex", "isolation": "worktree", + "depends_on": ["a"], "write_allowed": ["b/**"], "model": "gpt-5.5"}, + {"id": "c", "backend": "codex", "isolation": "worktree", + "depends_on": ["a"], "write_allowed": ["c/**"], "model": "gpt-5.5"}]} + _pw_ext_repo = os.path.join(tmp, "pwx-repo"); os.makedirs(_pw_ext_repo) + _pw_man_ext["_manifest_path"] = os.path.join(_pw_ext_repo, "manifest.yaml") + _pwx_workers = os.path.join(tmp, "workers-pwx"); os.makedirs(_pwx_workers, exist_ok=True) + with open(os.path.join(_pwx_workers, "compound-v-run-codex-worker.sh"), "w") as _fh: + _fh.write("#!/bin/sh\nexit 0\n") + _pw_ext_plan = build_plan(_with_body(_pw_man_ext), os.path.join(tmp, "pwx-run"), + _pw_ext_repo, "/usr/bin/python3", + os.path.abspath(__file__), + SCOPE_CHECK_DEFAULT, FASTPATH_DEFAULT, _pwx_workers) + _pw_ext_waves = [[e["id"] for e in w] for w in _pw_ext_plan["waves"]] + _check("external-backend dependents keep their wave (no needless serializing)", + any(sorted(i for i in w if i in ("b", "c")) == ["b", "c"] + for w in _pw_ext_waves), str(_pw_ext_waves)) + try: topo_waves([{"id": "a", "depends_on": ["b"]}, {"id": "b", "depends_on": ["a"]}], 2)