diff --git a/agents/conductors/profiling/AGENTS.md b/agents/conductors/profiling/AGENTS.md index cff64ef..cae0021 100644 --- a/agents/conductors/profiling/AGENTS.md +++ b/agents/conductors/profiling/AGENTS.md @@ -23,6 +23,7 @@ prompt, PyAutoMind `issued/profiling_agent.md`. | `ingest` | Which probe JSONs aren't in the vram tables yet, and which results have no pin? | table-update rows, pin list, baseline + dashboard steps | | `ingest --axis compile` | Which warm compile rows are unpinned, and which have drifted from their pin? | drifted rows (with pinned vs observed), unpinned keys, confirm/classify/re-pin steps | | `triage` | What do the pinned-drift findings mean? | per-finding classification: stale pin → re-pin here; library regression → `bug/` via intake | +| `triage --axis compile` | What does the compile drift MEAN, and who owns it? | per-finding classification: cache / autotune / host-load / library regression, expected-recompile, new-machine, new-cell | ``` pyauto-brain profiling # campaign, local tier @@ -42,9 +43,8 @@ bucketed by **hardware**, with `mixed_precision` a separate field. The two vocabularies do not interchange, so the compile axis maps tiers itself rather than reusing `TIER_CONFIGS`. -`--axis compile` serves `campaign` (coverage) and `ingest` (warm-pin drift); -`triage` rejects it with exit 5 until drift classification lands, so a compile -flag can never silently return a runtime answer. +`--axis compile` serves all three modes: `campaign` (coverage), `ingest` +(warm-pin drift) and `triage` (what the drift means). **Drift is deliberately hard to trigger.** A row counts only if it is *newer* than its pin, at least `2.0x` the pinned value, **and** at least `1.0 s` above it @@ -105,11 +105,44 @@ as malformed — only a record missing *some* of its key fields is corruption. flagged by integration tests is hygiene's `perf` mode, not profiling's. - **vs build** — campaigns are not releases; `profile.yml`'s on-release runs stay CI/Build territory. +- **release-validation script cost is NOT ours.** The compile axis was + originally proposed (PyAutoMind, 2026-07-14) to cover the release-validation + heavy scripts blowing the 300 s cap. That was declined on the hygiene boundary + above — script-suite cost is the developer loop, and it had already been moved + out of this agent's staged future modes once. Recorded here so the question is + not re-opened a third time. + +### Classifying compile drift + +`triage --axis compile` answers *who owns this*, which is the whole reason the +axis exists — the persistent compilation cache and `--xla_gpu_autotune_level=0` +are **settings**, so they can stop applying with nothing failing. + +| classification | signal | actionable | +|---|---|---| +| `cache-regression` | warm compile has returned to its own **cold** scale | yes — config/stack, never the library | +| `autotune-regression` | GPU compile up ≥10× with no cold-scale match | yes — check the flag actually reaches XLA | +| `host-load` | the measuring host's 1m load average was high | no — re-measure idle first | +| `library-regression` | growth on an unchanged key with no other explanation | yes — route to `bug/` via intake | +| `expected-recompile` | the key differs from a pin only by `jax_version` | no — cache keys include the version, so one recompile is by design | +| `new-machine` / `new-precision` / `new-cell` | the key is simply unpinned | no — pin it | + +The cold-scale comparison is what makes `cache-regression` a *measurement* +rather than a guess: 25 of 32 cell/transform keys in the corpus carry both a +warm and a cold row, so the yardstick is real data from the same machine. + +Two categories the design deliberately does NOT have: drift caused by a +`jax_version` bump or a changed host never reaches `triage` at all, because +those are different comparability keys and `ingest` reports them as *unpinned* +rather than drifted. They are classified here as bookkeeping so nothing +vanishes, but they are never regressions. ## Future modes (staged in the founding prompt) -JAX compilation-time profiling of likelihood functions. (Hunting -generally-slow functions flagged by integration tests moved to the hygiene -conductor's `perf` mode — that is developer-loop cost, not modelling speed.) A read-only profiling *faculty* (opine on regressions / optimization targets) splits out only on demonstrated consult demand. + +(Two things that were once staged here have moved. Hunting generally-slow +functions flagged by integration tests is the hygiene conductor's `perf` mode — +developer-loop cost, not modelling speed. JAX compilation-time profiling of +likelihood functions is **built**, and is the `--axis compile` surface above.) diff --git a/agents/conductors/profiling/_profiling.py b/agents/conductors/profiling/_profiling.py index 0aff8f6..f67c433 100644 --- a/agents/conductors/profiling/_profiling.py +++ b/agents/conductors/profiling/_profiling.py @@ -75,6 +75,18 @@ COMPILE_DRIFT_RATIO = 2.0 COMPILE_DRIFT_FLOOR_S = 1.0 +# A warm compile at >= this fraction of its own cold cost has effectively stopped +# being warm. Not 1.0: a cache miss need not reproduce the cold time exactly. +CACHE_REVERT_FRACTION = 0.5 +# A GPU compile up this much with no cold-scale match looks like autotune, whose +# pathological case was a 17x cold-probe cost. +AUTOTUNE_RATIO = 10.0 +# 1m load average at which a host is too busy for its compile timing to be trusted. +HOST_LOAD_SUSPECT = 2.0 + +# Classifications that need a human to do something. The rest are bookkeeping. +ACTIONABLE_CLASSIFICATIONS = ("cache-regression", "autotune-regression", "library-regression") + def workspace_root(explicit: str | None = None) -> Path: if explicit: @@ -423,7 +435,7 @@ def ingest_compile(ws: Path) -> dict[str, Any]: if pin is None: if key not in seen: seen.add(key) - unpinned.append({"record": f"{rel}[{idx}]", "pin": pin_key_str(key)}) + unpinned.append({"record": f"{rel}[{idx}]", "pin": pin_key_str(key), "_key": key}) continue # Only rows NEWER than the pin can be drift. Every warm row predating # the pin is the history the pin was chosen over — flagging those @@ -450,6 +462,11 @@ def ingest_compile(ws: Path) -> dict[str, Any]: "observed_s": got, "ratio": round(ratio, 2), "tag": rec.get("tag"), + "_key": key, + # Absent on records written before host_state existed; the + # classifier treats None as "cannot rule host load in or out" + # rather than as "the host was idle". + "host_load": (rec.get("host_state") or {}).get("load_avg_1m"), } ) @@ -551,6 +568,178 @@ def ingest(ws: Path) -> dict[str, Any]: } +def _cold_reference(ws: Path) -> dict[tuple, float]: + """Slowest cold compile per (comparability-minus-cache_state, cell, transform). + + The yardstick for "has the cache stopped being hit": a warm row that has + climbed back to its own cold scale is the regression this arc exists for. + Slowest rather than mean — the alarm should need the warm row to reach the + full cold cost, not merely an average a fast cold run drags down. + """ + out: dict[tuple, float] = {} + fields = [f for f in PIN_FIELDS if f != "cache_state"] + for _rel, _idx, rec in load_compile_corpus(ws): + if rec.get("cache_state") != "cold": + continue + got = rec.get("compile_s") + if not isinstance(got, (int, float)): + continue + key = tuple(rec.get(f) for f in fields) + out[key] = max(out.get(key, 0.0), float(got)) + return out + + +def _classify_drift(row: dict[str, Any], cold: dict[tuple, float]) -> dict[str, Any]: + key = dict(zip(PIN_FIELDS, row["_key"])) + fields = [f for f in PIN_FIELDS if f != "cache_state"] + cold_ref = cold.get(tuple(key.get(f) for f in fields)) + observed = row["observed_s"] + + if cold_ref and observed >= CACHE_REVERT_FRACTION * cold_ref: + return { + "classification": "cache-regression", + "evidence": ( + f"warm {observed}s has returned to its own cold scale ({cold_ref}s) — " + "the persistent cache is not being hit" + ), + "action": ( + "config/stack, NOT the library: check jax_compilation_cache_dir is set and " + "writable, and that nothing overwrites XLA_FLAGS (PyAutoNerves#127)" + ), + } + + if str(key.get("hardware", "")).startswith("local_gpu") and row["ratio"] >= AUTOTUNE_RATIO: + return { + "classification": "autotune-regression", + "evidence": ( + f"GPU compile up {row['ratio']}x with no cold-scale match — the shape of " + "--xla_gpu_autotune_level=0 not reaching XLA" + ), + "action": ( + "verify the flag actually reaches XLA first; the 2026-07-15 A/B was " + "invalidated for two months by XLA_FLAGS being clobbered at import" + ), + } + + if row.get("host_load") is not None and row["host_load"] >= HOST_LOAD_SUSPECT: + return { + "classification": "host-load", + "evidence": ( + f"1m load average {row['host_load']} on the measuring host — compile runs on " + "the host cores, and load alone has produced 7x errors in this corpus" + ), + "action": "NOT a regression until re-measured on an idle host; re-run warm", + } + + return { + "classification": "library-regression", + "evidence": ( + f"warm {observed}s vs pinned {row['pinned_s']}s ({row['ratio']}x) on an unchanged " + "key, with no cache, autotune or host-load explanation" + ), + "action": ( + "file a bug/ prompt via intake against the library owning the likelihood — " + "profiling classifies and routes, it never debugs the library here" + ), + } + + +def _classify_unpinned(key: tuple, pins: list[dict[str, Any]]) -> dict[str, Any]: + """An unpinned key differing from a pinned one in exactly ONE field is + explained by that field, not by a missing measurement.""" + parts = dict(zip(PIN_FIELDS, key)) + for pin in pins: + differing = [f for f in PIN_FIELDS if pin.get(f) != parts.get(f)] + if len(differing) != 1: + continue + field = differing[0] + if field == "jax_version": + return { + "classification": "expected-recompile", + "evidence": ( + f"same cell/transform pinned at jax {pin.get('jax_version')}; cache keys " + "include the jax version, so a bump recompiles once BY DESIGN" + ), + "action": "re-pin at the new version — this is not drift", + } + if field in ("hardware", "hostname"): + return { + "classification": "new-machine", + "evidence": f"same cell/transform pinned on {pin.get(field)}", + "action": "pin it; compile times are never comparable across machines", + } + if field == "mixed_precision": + return { + "classification": "new-precision", + "evidence": "same cell/transform pinned at the other precision", + "action": "pin it", + } + return { + "classification": "new-cell", + "evidence": "no pin shares this cell/transform", + "action": "pin it: `update_pins.py --write`", + } + + +def triage_compile(ws: Path) -> dict[str, Any]: + """Classify what `ingest --axis compile` found, and say what to do about it. + + Phases 1-2 make compile drift visible; this makes it actionable. The + classification IS the deliverable — profiling records and routes, it never + adjudicates library correctness inside the profiling repo. + """ + ing = ingest_compile(ws) + if ing.get("pins", 0) == 0: + return { + "agent": "profiling", + "mode": "triage", + "axis": "compile", + "findings": [], + "counts": {}, + "next_action": ing.get("next_action"), + } + + cold = _cold_reference(ws) + pins = load_pins(ws) + findings: list[dict[str, Any]] = [] + + for row in ing["drifted"]: + findings.append( + { + "finding": row["pin"], + "observed_s": row["observed_s"], + "pinned_s": row["pinned_s"], + **_classify_drift(row, cold), + } + ) + for row in ing["unpinned"]: + findings.append({"finding": row["pin"], **_classify_unpinned(row["_key"], pins)}) + + counts: dict[str, int] = {} + for f in findings: + counts[f["classification"]] = counts.get(f["classification"], 0) + 1 + actionable = [f for f in findings if f["classification"] in ACTIONABLE_CLASSIFICATIONS] + + return { + "agent": "profiling", + "mode": "triage", + "axis": "compile", + "pins": ing["pins"], + "findings": findings, + "counts": counts, + "policy": ( + "Every classification is made INSIDE one comparability key. A jax_version " + "bump is an expected recompile, never drift. Library findings are routed to " + "bug/ via intake and never debugged here." + ), + "next_action": ( + "no compile findings — warm compile is where the pins say it is" + if not findings + else f"{len(findings)} finding(s); {len(actionable)} needing action" + ), + } + + # --------------------------------------------------------------------------- # triage # --------------------------------------------------------------------------- @@ -606,6 +795,15 @@ def triage(ws: Path) -> dict[str, Any]: # --------------------------------------------------------------------------- +def _strip_internal(obj): + """Drop `_`-prefixed plumbing (e.g. raw key tuples) from emitted decisions.""" + if isinstance(obj, dict): + return {k: _strip_internal(v) for k, v in obj.items() if not k.startswith("_")} + if isinstance(obj, list): + return [_strip_internal(v) for v in obj] + return obj + + def emit_human(d: dict[str, Any]) -> None: print(f"== ProfilingDecision ({d['mode']}) ==") if d.get("error"): @@ -685,6 +883,17 @@ def emit_human(d: dict[str, Any]) -> None: print("Steps:") for s in d["steps"]: print(f" - {s}") + elif d["mode"] == "triage" and d.get("axis") == "compile": + print(f"Compile pins: {d.get('pins', 0)}") + print(f"Findings: {len(d['findings'])}") + for c, n in sorted(d.get("counts", {}).items()): + print(f" {c}: {n}") + for f in d["findings"]: + print(f" [{f['classification']}] {f['finding']}") + print(f" evidence: {f['evidence']}") + print(f" -> {f['action']}") + if d.get("policy"): + print(f"Policy: {d['policy']}") elif d["mode"] == "triage": print(f"Observed: {d.get('observed')}") print(f"Findings: {len(d['findings'])}") @@ -711,17 +920,6 @@ def main(argv=None) -> int: # ingest/triage own the compile axis in later phases of the arc (pins, then # drift classification). Refusing now is deliberate: a mode that silently # ignored --axis would report runtime findings under a compile flag. - # triage owns the compile axis in phase 3 (drift CLASSIFICATION). Refusing - # is deliberate: a mode that silently ignored --axis would report runtime - # findings under a compile flag. - if a.axis == "compile" and a.mode == "triage": - print( - "profiling: --axis compile is not implemented for 'triage' yet " - "(campaign + ingest only; classification lands next)", - file=sys.stderr, - ) - return 5 - ws = workspace_root(a.workspace) if not ws.is_dir(): print(f"profiling: workspace not found: {ws}", file=sys.stderr) @@ -732,8 +930,9 @@ def main(argv=None) -> int: elif a.mode == "ingest": d = ingest_compile(ws) if a.axis == "compile" else ingest(ws) else: - d = triage(ws) + d = triage_compile(ws) if a.axis == "compile" else triage(ws) + d = _strip_internal(d) print(json.dumps(d, indent=2)) if a.as_json else emit_human(d) return 0 diff --git a/tests/test_profiling_conductor.py b/tests/test_profiling_conductor.py index 6b19b37..47ecafb 100644 --- a/tests/test_profiling_conductor.py +++ b/tests/test_profiling_conductor.py @@ -242,16 +242,13 @@ def test_bad_tier_is_an_error(tmp_path): # --------------------------------------------------------------------------- -def test_compile_axis_is_refused_for_triage(tmp_path): - """Better a usage error than runtime findings reported under a compile flag. - - `ingest` gained the axis with the pins; `triage` classifies drift and lands - with phase 3. - """ +def test_every_mode_serves_the_compile_axis(tmp_path): + """The arc is closed: campaign, ingest and triage all answer --axis compile.""" ws = _workspace(tmp_path) - r = _run(["triage", "--axis", "compile"], ws) - assert r.returncode == 5, f"{r.stdout}{r.stderr}" - assert "not implemented" in r.stderr + for mode in ("campaign", "ingest", "triage"): + r = _run([mode, "--axis", "compile", "--json"], ws) + assert r.returncode == 0, f"{mode}: {r.stdout}{r.stderr}" + assert json.loads(r.stdout)["axis"] == "compile" def test_missing_workspace_exits_4(tmp_path): @@ -435,13 +432,6 @@ def test_absent_pins_file_says_so_rather_than_reporting_all_clear(tmp_path): assert "update_pins.py" in d["next_action"] -def test_triage_still_refuses_the_compile_axis(tmp_path): - ws = _workspace(tmp_path) - r = _run(["triage", "--axis", "compile"], ws) - assert r.returncode == 5 - assert "not implemented" in r.stderr - - def test_brain_comparability_key_matches_the_workspace_definition(tmp_path): """The Brain mirrors pins.py rather than importing it (importing the workspace would drag the JAX stack in), so pin the two together.""" @@ -468,3 +458,152 @@ def test_brain_comparability_key_matches_the_workspace_definition(tmp_path): assert _profiling.COMPARABILITY_FIELDS == found["COMPARABILITY_FIELDS"] assert _profiling.CELL_FIELDS == found["CELL_FIELDS"] + + +# --------------------------------------------------------------------------- +# triage --axis compile (classification) +# --------------------------------------------------------------------------- + + +def _triage(ws): + r = _run(["triage", "--axis", "compile", "--json"], ws) + assert r.returncode == 0, r.stderr + return json.loads(r.stdout) + + +def test_warm_returning_to_its_cold_scale_is_a_cache_regression(tmp_path): + """The alarm the whole arc exists for.""" + ws = _workspace(tmp_path, { + "local_cpu/mge.json": [ + _record(cache_state="cold", compile_s=117.0, hostname="laptop", + transform="vag", timestamp="2026-06-01T00:00:00"), + _warm(compile_s=110.0), + ], + }) + _pinned(ws, [_pin(compile_s=2.3)]) + d = _triage(ws) + + assert d["counts"] == {"cache-regression": 1} + f = d["findings"][0] + assert "cold scale" in f["evidence"] + assert "NOT the library" in f["action"] + + +def test_growth_with_no_cold_scale_match_routes_to_the_library(tmp_path): + """Not everything slow is the cache; what is left over is a bug/ candidate.""" + ws = _workspace(tmp_path, { + "local_cpu/mge.json": [ + _record(cache_state="cold", compile_s=117.0, hostname="laptop", + transform="vag", timestamp="2026-06-01T00:00:00"), + _warm(compile_s=8.0), # 3.5x the pin, nowhere near 117s + ], + }) + _pinned(ws, [_pin(compile_s=2.3)]) + d = _triage(ws) + + assert d["counts"] == {"library-regression": 1} + assert "intake" in d["findings"][0]["action"] + assert "never debugs the library here" in d["findings"][0]["action"] + + +def test_a_busy_host_is_not_a_regression(tmp_path): + """Compile runs on host cores; load alone has produced 7x errors here.""" + ws = _workspace(tmp_path, { + "local_cpu/mge.json": [ + _warm(compile_s=8.0, host_state={"cpu_count": 8, "load_avg_1m": 14.0}), + ], + }) + _pinned(ws, [_pin(compile_s=2.3)]) + d = _triage(ws) + + assert d["counts"] == {"host-load": 1} + assert "NOT a regression until re-measured" in d["findings"][0]["action"] + + +def test_a_big_gpu_jump_reads_as_autotune(tmp_path): + ws = _workspace(tmp_path, { + "local_gpu_NVIDIA_A100_80GB_PCIe/mge.json": [ + _warm(compile_s=50.0, hardware="local_gpu_NVIDIA_A100_80GB_PCIe"), + ], + }) + _pinned(ws, [_pin(compile_s=2.3, hardware="local_gpu_NVIDIA_A100_80GB_PCIe")]) + d = _triage(ws) + + assert d["counts"] == {"autotune-regression": 1} + assert "XLA_FLAGS" in d["findings"][0]["action"] + + +def test_a_jax_bump_is_an_expected_recompile_not_a_regression(tmp_path): + """Cache keys include the jax version, so a bump recompiles once BY DESIGN.""" + ws = _workspace(tmp_path, { + "local_cpu/mge.json": [_warm(compile_s=117.0, jax_version="0.11.0")], + }) + _pinned(ws, [_pin(compile_s=2.3, jax_version="0.10.2")]) + d = _triage(ws) + + assert d["counts"] == {"expected-recompile": 1} + f = d["findings"][0] + assert "BY DESIGN" in f["evidence"] + assert "not drift" in f["action"] + + +def test_a_new_machine_is_classified_as_such(tmp_path): + ws = _workspace(tmp_path, { + "local_cpu/mge.json": [_warm(compile_s=9.0, hostname="euclid-ral-compute-22")], + }) + _pinned(ws, [_pin(compile_s=2.3, hostname="laptop")]) + d = _triage(ws) + + assert d["counts"] == {"new-machine": 1} + assert "never comparable across machines" in d["findings"][0]["action"] + + +def test_an_unrelated_cell_is_simply_new(tmp_path): + ws = _workspace(tmp_path, { + "local_cpu/mge.json": [_warm(compile_s=9.0, model_type="pixelization")], + }) + _pinned(ws, [_pin(compile_s=2.3, model_type="mge")]) + d = _triage(ws) + assert d["counts"] == {"new-cell": 1} + + +def test_bookkeeping_classifications_do_not_count_as_actionable(tmp_path): + ws = _workspace(tmp_path, { + "local_cpu/mge.json": [_warm(compile_s=117.0, jax_version="0.11.0")], + }) + _pinned(ws, [_pin(compile_s=2.3, jax_version="0.10.2")]) + d = _triage(ws) + assert "1 finding(s); 0 needing action" in d["next_action"] + + +def test_a_clean_corpus_reports_no_findings(tmp_path): + ws = _workspace(tmp_path, {"local_cpu/mge.json": [_warm(compile_s=2.3)]}) + _pinned(ws, [_pin(compile_s=2.3)]) + d = _triage(ws) + assert d["findings"] == [] + assert "no compile findings" in d["next_action"] + + +def test_no_pins_says_so_rather_than_reporting_clean(tmp_path): + ws = _workspace(tmp_path, {"local_cpu/mge.json": [_warm()]}) + d = _triage(ws) + assert d["findings"] == [] + assert "update_pins.py" in d["next_action"] + + +def test_internal_plumbing_is_not_emitted(tmp_path): + """The raw key tuples are implementation detail, not part of the decision.""" + ws = _workspace(tmp_path, {"local_cpu/mge.json": [_warm(compile_s=117.0)]}) + _pinned(ws, [_pin(compile_s=2.3)]) + for mode in ("ingest", "triage"): + r = _run([mode, "--axis", "compile", "--json"], ws) + assert "_key" not in r.stdout, mode + + +def test_triage_writes_nothing_to_the_workspace(tmp_path): + ws = _workspace(tmp_path, {"local_cpu/mge.json": [_warm(compile_s=117.0)]}) + _pinned(ws, [_pin(compile_s=2.3)]) + before = {p: p.stat().st_mtime_ns for p in ws.rglob("*") if p.is_file()} + _triage(ws) + after = {p: p.stat().st_mtime_ns for p in ws.rglob("*") if p.is_file()} + assert before == after, "the conductor reasons and delegates; it never edits the workspace"