diff --git a/agents/faculties/samplers/AGENTS.md b/agents/faculties/samplers/AGENTS.md index 0907f23..940a0ef 100644 --- a/agents/faculties/samplers/AGENTS.md +++ b/agents/faculties/samplers/AGENTS.md @@ -25,14 +25,26 @@ samplers.sh [--json] archive (autofit_workspace_developer/searches) integration (autofit_workspace_test/scripts/searches) promoted (PyAutoFit autofit/non_linear/search//) + -> the findings maturation lane (below): + experiment probes (autolens_workspace_developer/searches_minimal) + experiment findings (the same tier's *_findings.md, name + verdict) + mature (autolens_profiling — one row per + sampler/dataset_class/model_type cell) -> the latest minimal-tier benchmark table (output/comparison.txt) -> tier gaps: prototyped-never-promoted, promoted-never-integration-tested ``` Surfaces are resolved as sibling checkouts (`PYAUTO_FIT`, -`PYAUTO_FIT_DEVELOPER`, `PYAUTO_FIT_TEST` override); absent ones are +`PYAUTO_FIT_DEVELOPER`, `PYAUTO_FIT_TEST`, `PYAUTO_LENS_DEVELOPER`, +`PYAUTO_PROFILING` override); absent ones are reported, never fatal. Exit codes: `0` digest · `4` no surface · `5` usage. +A mature-tier cell is read from its leaf's `run_search(sampler=, +dataset_class=, model_type=)` declaration, **not** from its path — the two +disagree on the live tree (the `cluster/searches/*/mge.py` leaves declare the +`group` dataset class), and the declaration is what the runner acts on. The +lane tiers are inventory only; they feed no `gaps` rule. + ## Judgment: matching sampler to likelihood | Situation | Reach for | Why | @@ -111,9 +123,10 @@ and promote what matures: A conductor planning search-validation work should name the target tier up front; `ship` routes all three repos through the normal workspace flow. -(Surface gap, filed: this faculty's SamplerSurface scan reads only the -autofit-side tiers — it does not yet see the autolens experiment/mature -tiers above.) +The SamplerSurface scan covers tiers 1 and 2 directly (see above), so the +digest answers "where does this search x likelihood sit in the lane — +experimented, matured, or neither". Tier 3 is user-facing prose and is read, +not inventoried. ## Where the knowledge lives (pointers, not copies) diff --git a/agents/faculties/samplers/_samplers.py b/agents/faculties/samplers/_samplers.py index b626639..845e8f4 100755 --- a/agents/faculties/samplers/_samplers.py +++ b/agents/faculties/samplers/_samplers.py @@ -6,15 +6,19 @@ prototypes, the removed-sampler archive, the workspace_test integration scripts) and the PyAutoFit search catalogue — plus the latest minimal-tier benchmark outputs, and flags tier gaps (prototyped but never promoted, -promoted but never integration-tested). The consulting agent reads the digest -and reasons with AGENTS.md's judgment tables; this script never writes, -never runs a sampler, and never edits anything. +promoted but never integration-tested). It also inventories the **findings +maturation lane** (AGENTS.md "Judgment: the maturation lane"): the experiment +tier's probes and findings docs, and the mature tier's +(sampler x dataset_class x model_type) cell matrix. The consulting agent +reads the digest and reasons with AGENTS.md's judgment tables; this script +never writes, never runs a sampler, and never edits anything. Exit codes: 0 digest · 4 no surface found · 5 usage. """ from __future__ import annotations import argparse +import ast import json import sys from pathlib import Path @@ -74,6 +78,105 @@ def tier_promoted(autofit: Path) -> list[str]: return out +# -------------------------------------------------------------------------- +# The findings maturation lane (AGENTS.md "Judgment: the maturation lane") +# -------------------------------------------------------------------------- +# +# Distinct from sampler promotion (minimal -> PyAutoFit), this lane validates an +# already-promoted search on a new likelihood class: experiment tier (hand-rolled +# probes + findings docs) -> mature tier (first-class `af` search cells). Named +# as module constants so callers and tests can refer to a tier without +# re-spelling an instance fact. +SURFACE_LENS_DEVELOPER = "autolens_workspace_developer" +SURFACE_PROFILING = "autolens_profiling" +TIER_LENS_PROBES = "experiment probes (autolens searches_minimal)" +TIER_LENS_FINDINGS = "experiment findings (autolens searches_minimal)" +TIER_LENS_MATURE = "mature (autolens_profiling searches cells)" + + +def tier_lens_probes(lens_developer: Path) -> list[str]: + """Experiment tier: the runnable probes, same flat shape as the autofit + minimal tier (so `_py_stems` — which already drops `_`-prefixed helpers — + is reused verbatim).""" + return _py_stems(lens_developer / "searches_minimal") + + +def _first_heading(path: Path) -> str: + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if line.startswith("# "): + return line[2:].strip() + return "" + + +def tier_lens_findings(lens_developer: Path) -> list[str]: + """Experiment tier: the campaign findings docs as `name — first heading`. + + The heading carries the verdict ("... YES — on every mesh, once the + regularization axis is handled"), which is the whole reason a conductor + consults this tier; a doc with no heading degrades to its bare name. + """ + root = lens_developer / "searches_minimal" + if not root.is_dir(): + return [] + out = [] + for doc in sorted(root.glob("*_findings.md")): + heading = _first_heading(doc) + out.append(f"{doc.stem} — {heading}" if heading else doc.stem) + return out + + +def _declared_cell(leaf: Path) -> tuple[str, str, str] | None: + """Read the `run_search(sampler=, dataset_class=, model_type=)` declaration. + + The declaration — NOT the path — is the cell's identity. The two genuinely + disagree on the live tree: every `cluster/searches/*/mge.py` leaf declares + `dataset_class="group"` while its siblings declare `"cluster"`, and `group` + is a legitimate dataset class (each such leaf passes an explicit + `default_instrument`). Parsing the path would silently mislabel them. + """ + try: + tree = ast.parse(leaf.read_text(encoding="utf-8", errors="replace")) + except SyntaxError: + return None + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + name = func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", None) + if name != "run_search": + continue + kw = { + k.arg: k.value.value + for k in node.keywords + if k.arg and isinstance(k.value, ast.Constant) + and isinstance(k.value.value, str) + } + if {"sampler", "dataset_class", "model_type"} <= kw.keys(): + return kw["sampler"], kw["dataset_class"], kw["model_type"] + return None + + +def tier_lens_mature(profiling: Path) -> list[str]: + """Mature tier: the (sampler x dataset_class x model_type) cell matrix. + + Walks `scripts//searches//.py`, skipping the + `misc/` framework directory and `_`-prefixed helpers. A leaf with no + parsable declaration falls back to its path shape rather than vanishing. + """ + scripts = profiling / "scripts" + if not scripts.is_dir(): + return [] + cells = set() + for leaf in scripts.glob("*/searches/*/*.py"): + if leaf.stem.startswith("_") or leaf.parent.parent.parent.name == "misc": + continue + declared = _declared_cell(leaf) + if declared is None: + declared = (leaf.parent.name, leaf.parent.parent.parent.name, leaf.stem) + cells.add("/".join(declared)) + return sorted(cells) + + def benchmarks(developer: Path) -> dict: out_dir = developer / "searches_minimal" / "output" result = {"comparison": None, "summaries": []} @@ -105,7 +208,7 @@ def gaps(minimal, integration, promoted) -> list[str]: return out -def digest(autofit, developer, test) -> dict: +def digest(autofit, developer, test, lens_developer=None, profiling=None) -> dict: d = { "surfaces_present": [], "tiers": {}, @@ -133,6 +236,15 @@ def digest(autofit, developer, test) -> dict: d["surfaces_present"].append("PyAutoFit") promoted = tier_promoted(autofit) d["tiers"]["promoted (autofit/non_linear/search)"] = promoted + if lens_developer and lens_developer.is_dir(): + d["surfaces_present"].append(SURFACE_LENS_DEVELOPER) + d["tiers"][TIER_LENS_PROBES] = tier_lens_probes(lens_developer) + d["tiers"][TIER_LENS_FINDINGS] = tier_lens_findings(lens_developer) + if profiling and profiling.is_dir(): + d["surfaces_present"].append(SURFACE_PROFILING) + d["tiers"][TIER_LENS_MATURE] = tier_lens_mature(profiling) + # The lane tiers are inventory only: `gaps` stays keyed on the autofit + # promotion tiers, so adding them introduces no new judgment. if minimal or promoted: d["gaps"] = gaps(minimal, integration, promoted) return d @@ -167,15 +279,23 @@ def main(argv=None) -> int: help="autofit_workspace_developer checkout") ap.add_argument("--test", default="", help="autofit_workspace_test checkout") + ap.add_argument("--lens-developer", default="", dest="lens_developer", + help="autolens_workspace_developer checkout " + "(lane experiment tier)") + ap.add_argument("--profiling", default="", + help="autolens_profiling checkout (lane mature tier)") ap.add_argument("--json", action="store_true", dest="as_json") a = ap.parse_args(argv) autofit = Path(a.autofit) if a.autofit else None developer = Path(a.developer) if a.developer else None test = Path(a.test) if a.test else None - d = digest(autofit, developer, test) + lens_developer = Path(a.lens_developer) if a.lens_developer else None + profiling = Path(a.profiling) if a.profiling else None + d = digest(autofit, developer, test, lens_developer, profiling) if not d["surfaces_present"]: print("samplers: no sampler surface found (PyAutoFit / " - "autofit_workspace_developer / autofit_workspace_test absent)", + "autofit_workspace_developer / autofit_workspace_test / " + "autolens_workspace_developer / autolens_profiling absent)", file=sys.stderr) return 4 print(json.dumps(d, indent=2)) if a.as_json else emit_human(d) diff --git a/agents/faculties/samplers/samplers.sh b/agents/faculties/samplers/samplers.sh index 6ad83c3..985fa83 100755 --- a/agents/faculties/samplers/samplers.sh +++ b/agents/faculties/samplers/samplers.sh @@ -7,7 +7,9 @@ # tiers (searches_minimal prototypes, the removed-sampler archive, the # workspace_test integration scripts) plus the PyAutoFit search catalogue and # the latest minimal-tier benchmark table, with tier-gap findings (prototyped -# but never promoted; promoted but never integration-tested). The consulting +# but never promoted; promoted but never integration-tested). It also covers +# the findings maturation lane: the experiment tier's probes + findings docs +# and the mature tier's (sampler x dataset_class x model_type) cells. The consulting # agent reads the digest and reasons with AGENTS.md's judgment tables # (sampler<->likelihood match, gradient/JAX constraints, initialization # chaining). Read-only: never runs a sampler, never writes, never dispatches. @@ -25,9 +27,15 @@ source "$HERE/../../_common.sh" autofit="$(_resolve_dir PYAUTO_FIT PyAutoFit 2>/dev/null || true)" developer="$(_resolve_dir PYAUTO_FIT_DEVELOPER autofit_workspace_developer 2>/dev/null || true)" test_ws="$(_resolve_dir PYAUTO_FIT_TEST autofit_workspace_test 2>/dev/null || true)" +# The findings maturation lane's two tiers (AGENTS.md "Judgment: the maturation +# lane") — present-if-checked-out, exactly like the autofit-side surfaces. +lens_developer="$(_resolve_dir PYAUTO_LENS_DEVELOPER autolens_workspace_developer 2>/dev/null || true)" +profiling="$(_resolve_dir PYAUTO_PROFILING autolens_profiling 2>/dev/null || true)" exec python3 "$HERE/_samplers.py" \ ${autofit:+--autofit "$autofit"} \ ${developer:+--developer "$developer"} \ ${test_ws:+--test "$test_ws"} \ + ${lens_developer:+--lens-developer "$lens_developer"} \ + ${profiling:+--profiling "$profiling"} \ "$@" diff --git a/tests/test_samplers_surface.py b/tests/test_samplers_surface.py new file mode 100644 index 0000000..84d5685 --- /dev/null +++ b/tests/test_samplers_surface.py @@ -0,0 +1,130 @@ +"""Contract tests for the SamplerSurface's findings-maturation-lane tiers. + +Hermetic: every test fabricates a temp checkout and drives ``_samplers.py`` +directly with explicit ``--`` paths, so nothing depends on which sibling repos +happen to be cloned. The faculty is read-only — asserted in test_never_writes — +and these tests name no repositories (tenant firewall), taking tier labels and +surface names from the module's own constants instead. +""" + +import json +import subprocess +import sys +from pathlib import Path + +BRAIN_HOME = Path(__file__).resolve().parents[1] +FACULTY = BRAIN_HOME / "agents" / "faculties" / "samplers" +sys.path.insert(0, str(FACULTY)) + +import _samplers # noqa: E402 + + +def _digest(*args): + result = subprocess.run( + [sys.executable, str(FACULTY / "_samplers.py"), "--json", *args], + capture_output=True, text=True, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + +def _make_experiment(root: Path) -> Path: + """One checkout with two probes, one private helper (must not be listed), + a findings doc with a verdict heading, and one with no heading at all.""" + minimal = root / "searches_minimal" + minimal.mkdir(parents=True) + (minimal / "alpha_probe.py").write_text("# probe\n") + (minimal / "beta_probe.py").write_text("# probe\n") + (minimal / "_helper.py").write_text("# shared helper, not a probe\n") + (minimal / "alpha_findings.md").write_text( + "# Does alpha work on the ringed mesh? YES — once damping is handled.\n" + "\nbody\n" + ) + (minimal / "headless_findings.md").write_text("no heading here\n") + return root + + +def _cell(root: Path, dataset_dir, sampler, model_type, declared=None): + """Write a leaf at scripts//searches//.py. + + ``declared`` overrides the dataset_class the leaf *declares*, so a fixture + can reproduce the live divergence between path and declaration. + """ + leaf = root / "scripts" / dataset_dir / "searches" / sampler + leaf.mkdir(parents=True, exist_ok=True) + (leaf / f"{model_type}.py").write_text( + "from searches._runner import run_search\n\n" + "run_search(\n" + f' sampler="{sampler}",\n' + f' dataset_class="{declared or dataset_dir}",\n' + f' model_type="{model_type}",\n' + ' default_instrument="hst",\n' + ")\n" + ) + + +def _make_mature(root: Path) -> Path: + _cell(root, "imaging", "nautilus", "mge") + # the live divergence: a leaf under one directory declaring another class + _cell(root, "cluster", "nautilus", "mge", declared="group") + # framework helpers under misc/ are not cells + misc = root / "scripts" / "misc" / "searches" + misc.mkdir(parents=True) + (misc / "_runner.py").write_text("def run_search(**kw): ...\n") + (misc / "sweep.py").write_text("# driver, not a cell\n") + # a private helper inside a real sampler dir is not a cell either + (root / "scripts" / "imaging" / "searches" / "nautilus" / "_shared.py").write_text("x = 1\n") + return root + + +def test_experiment_tier_lists_probes_and_findings_verdicts(tmp_path): + d = _digest("--lens-developer", str(_make_experiment(tmp_path))) + assert _samplers.SURFACE_LENS_DEVELOPER in d["surfaces_present"] + assert d["tiers"][_samplers.TIER_LENS_PROBES] == ["alpha_probe", "beta_probe"] + findings = d["tiers"][_samplers.TIER_LENS_FINDINGS] + assert findings[0].startswith("alpha_findings — Does alpha work") + # a doc with no heading degrades to its bare name rather than an empty row + assert findings[1] == "headless_findings" + + +def test_mature_tier_reads_the_declaration_not_the_path(tmp_path): + d = _digest("--profiling", str(_make_mature(tmp_path))) + assert _samplers.SURFACE_PROFILING in d["surfaces_present"] + cells = d["tiers"][_samplers.TIER_LENS_MATURE] + # the cluster/ leaf declares `group`, and the declaration wins — parsing the + # path instead would mislabel it and silently collide with a real cell + assert cells == ["nautilus/group/mge", "nautilus/imaging/mge"] + + +def test_mature_tier_falls_back_to_path_when_undeclared(tmp_path): + root = _make_mature(tmp_path) + undeclared = root / "scripts" / "interferometer" / "searches" / "emcee" + undeclared.mkdir(parents=True) + (undeclared / "delaunay.py").write_text("# no run_search call at all\n") + cells = _digest("--profiling", str(root))["tiers"][_samplers.TIER_LENS_MATURE] + assert "emcee/interferometer/delaunay" in cells + + +def test_lane_tiers_add_no_gaps(tmp_path): + """Inventory only — `gaps` stays keyed on the autofit promotion tiers.""" + root = _make_mature(_make_experiment(tmp_path)) + d = _digest("--lens-developer", str(root), "--profiling", str(root)) + assert d["gaps"] == [] + + +def test_absent_lane_checkouts_are_not_fatal(tmp_path): + result = subprocess.run( + [sys.executable, str(FACULTY / "_samplers.py"), "--json", + "--lens-developer", str(tmp_path / "nope"), + "--profiling", str(tmp_path / "also-nope")], + capture_output=True, text=True, + ) + assert result.returncode == 4 # no surface, reported cleanly + assert "Traceback" not in result.stderr + + +def test_never_writes(tmp_path): + root = _make_mature(_make_experiment(tmp_path)) + before = sorted(str(p) for p in root.rglob("*")) + _digest("--lens-developer", str(root), "--profiling", str(root)) + assert sorted(str(p) for p in root.rglob("*")) == before