Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 35 additions & 2 deletions benchmarks/lib/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,20 @@
"""Shared benchmark library — retrieval, fusion, and PG database helpers."""
"""Shared benchmark library — retrieval, fusion, and PG database helpers.

``BenchmarkDB`` is re-exported lazily (PEP 562 module ``__getattr__``)
rather than imported eagerly at the top: ``bench_db.py`` hard-imports
``psycopg``/``psycopg_pool``/``pgvector`` at module scope (issue #282),
and every other consumer in this package already defers that same import
inside a function for exactly this reason (see the "deferred: module
hard-imports pgvector/psycopg/psycopg_pool at top level; hoisting would
break installs without it" comments in ``ablation_runner.py``,
``bench_db.py``, ``_xb_drivers.py``, ``longitudinal_runner.py``,
``llm_head_to_head/pilot.py``). Eagerly importing it here at package-init
time — as this module did before — made every other submodule
(``benchmarks.lib.verification_report`` included) unimportable on an
install without the Postgres extras, since Python always runs a
package's ``__init__.py`` before any of its submodules.
"""

from benchmarks.lib.bench_db import BenchmarkDB
from benchmarks.lib.fusion import (
wrrf_fuse,
QualityZone,
Expand All @@ -17,3 +31,22 @@
"assess_quality_zone",
"enforce_chunk_limit",
]


def __getattr__(name: str):
"""Lazily resolve ``BenchmarkDB`` on first access (PEP 562).

Precondition: ``name`` is an attribute looked up on this module that
was not found among its eagerly-bound names above.
Postcondition: returns ``bench_db.BenchmarkDB`` for
``name == "BenchmarkDB"`` — raising whatever ``ImportError`` psycopg's
own absence produces, deferred until actually needed — else raises
``AttributeError`` (the standard module-attribute-not-found contract).
"""
if name == "BenchmarkDB":
from benchmarks.lib.bench_db import ( # noqa: PLC0415 — deferred: module hard-imports pgvector/psycopg/psycopg_pool at top level; hoisting would break installs without it
BenchmarkDB,
)

return BenchmarkDB
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
74 changes: 46 additions & 28 deletions benchmarks/lib/verification_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,34 +100,50 @@

@dataclass
class ExpResult:
"""Data only — deliberately no methods. mutmut's mutation generator
categorically excludes the body of any `@dataclass`-decorated class
(`mutmut/mutation/file_mutation.py:236`), so logic placed on methods
here would carry zero mutation coverage no matter how the test loader
names the module (issue #262 3rd pass; issue #282). `exp_result_name` /
`exp_result_threshold` / `exp_result_verdict` below carry the same
logic as free functions instead.
"""

exp_id: str
found: bool = False
observed: float | None = None
raw_path: str | None = None
detail_md: str = ""
ci95: tuple[float, float] | None = None

@property
def name(self) -> str:
return HYPOTHESES[self.exp_id]["name"]

@property
def threshold(self) -> str:
return HYPOTHESES[self.exp_id]["threshold"]

@property
def verdict(self) -> str:
if not self.found or self.observed is None:
return "INCONCLUSIVE"
h = HYPOTHESES[self.exp_id]
op, val = h["pass_op"], h["pass_value"]
cmp = {
">=": self.observed >= val,
"<": self.observed < val,
">": self.observed > val,
"<=": self.observed <= val,
}
return "PASS" if cmp.get(op, False) else "FAIL"

def exp_result_name(result: "ExpResult") -> str:
return HYPOTHESES[result.exp_id]["name"]


def exp_result_threshold(result: "ExpResult") -> str:
return HYPOTHESES[result.exp_id]["threshold"]


def exp_result_verdict(result: "ExpResult") -> str:
if not result.found or result.observed is None:
return "INCONCLUSIVE"
h = HYPOTHESES[result.exp_id]
op, val = h["pass_op"], h["pass_value"]
cmp = {
">=": result.observed >= val,
"<": result.observed < val,
">": result.observed > val,
"<=": result.observed <= val,
}
# §12 note: `cmp.get(op, False)`'s default is read only when `op` is not
# one of the four keys above (an unrecognized pass_op). A mutant that
# swaps the default to `None` (or omits it, same effect) is EQUIVALENT:
# both `False` and `None` are falsy, so `"PASS" if ... else "FAIL"`
# resolves identically either way — only a *truthy* default (e.g. `True`)
# changes the outcome, and that mutant is killed by
# test_verdict_unrecognized_operator_defaults_to_fail.
return "PASS" if cmp.get(op, False) else "FAIL"


# source: contract stated in the _bootstrap_ci docstring ("None if <10
Expand Down Expand Up @@ -300,20 +316,20 @@ def _render_section(r: ExpResult) -> str:
h = HYPOTHESES[r.exp_id]
if not r.found:
return (
f"## {r.exp_id} — {r.name}\n\n"
f"## {r.exp_id} — {exp_result_name(r)}\n\n"
f"- **Hypothesis:** {h['claim']}\n"
f"- **Threshold:** {r.threshold}\n"
f"- **Threshold:** {exp_result_threshold(r)}\n"
f"- **Result:** `// TODO: not yet run`\n"
)
obs_str = f"{r.observed:.4f}" if r.observed is not None else "N/A"
ci_str = f"\n- **95% CI:** [{r.ci95[0]:.4f}, {r.ci95[1]:.4f}]" if r.ci95 else ""
parts = [
f"## {r.exp_id} — {r.name}",
f"## {r.exp_id} — {exp_result_name(r)}",
"",
f"- **Hypothesis:** {h['claim']}",
f"- **Threshold:** {r.threshold}",
f"- **Threshold:** {exp_result_threshold(r)}",
f"- **Observed:** {obs_str}{ci_str}",
f"- **Verdict:** **{r.verdict}**",
f"- **Verdict:** **{exp_result_verdict(r)}**",
f"- **Source:** `{r.raw_path}`",
]
if r.detail_md:
Expand Down Expand Up @@ -341,8 +357,10 @@ def _render_summary(results: list[ExpResult]) -> str:
obs, verdict = "// TODO: not yet run", "—"
else:
obs = f"{r.observed:.4f}" if r.observed is not None else "N/A"
verdict = r.verdict
lines.append(f"| {r.exp_id} | {claim} | {r.threshold} | {obs} | {verdict} |")
verdict = exp_result_verdict(r)
lines.append(
f"| {r.exp_id} | {claim} | {exp_result_threshold(r)} | {obs} | {verdict} |"
)
return "\n".join(lines) + "\n"


Expand Down
97 changes: 60 additions & 37 deletions fuzz/replay_corpus.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,43 +42,66 @@

@dataclass(frozen=True)
class Harness:
"""One fuzz harness and the corpus directory that feeds it."""
"""One fuzz harness and the corpus directory that feeds it.

Data only — deliberately no methods. mutmut's mutation generator
categorically excludes the body of any `@dataclass`-decorated class
(`mutmut/mutation/file_mutation.py:236`), so logic placed on methods
here would carry zero mutation coverage no matter how the test loader
names the module (issue #262 3rd pass; issue #282).
`harness_module_path` / `harness_corpus_path` / `harness_consume` /
`harness_inputs` below carry the same logic as free functions instead.
"""

name: str

def module_path(self) -> Path:
return FUZZ_DIR / f"{self.name}.py"

def corpus_path(self) -> Path:
return CORPUS_DIR / self.name

def consume(self) -> Callable[[bytes], None]:
"""Import the harness and hand back its single-iteration entry point."""
spec = importlib.util.spec_from_file_location(self.name, self.module_path())
if spec is None or spec.loader is None:
raise RuntimeError(f"{self.module_path()}: not importable")
module = importlib.util.module_from_spec(spec)
# Registered before exec: @dataclass (and typing constructs
# generally) resolve annotations via sys.modules[cls.__module__],
# which is None for a module loaded from a path and never inserted.
sys.modules[spec.name] = module
spec.loader.exec_module(module)
consume = getattr(module, "consume", None)
if consume is None:
raise RuntimeError(
f"{self.module_path()}: no consume(data: bytes) —"
" every harness must expose one so its property is runnable"
" without atheris"
)
return consume

def inputs(self) -> Iterator[tuple[Path, bytes]]:
directory = self.corpus_path()
if not directory.is_dir():
return
for path in sorted(directory.iterdir()):
if path.is_file():
yield path, path.read_bytes()

def harness_module_path(harness: "Harness") -> Path:
return FUZZ_DIR / f"{harness.name}.py"


def harness_corpus_path(harness: "Harness") -> Path:
return CORPUS_DIR / harness.name


def harness_consume(harness: "Harness") -> Callable[[bytes], None]:
"""Import the harness and hand back its single-iteration entry point."""
module_path = harness_module_path(harness)
spec = importlib.util.spec_from_file_location(harness.name, module_path)
# Defensive guard, structurally unreachable for this call site: module_path
# always ends in ".py", and spec_from_file_location always attaches a
# SourceFileLoader for that suffix (verified empirically: it only returns
# spec=None for an unrecognized extension, never spec-with-loader=None for
# a .py path, existing or not). mutmut's `or`->`and` mutant (mutmut_8) and
# its RuntimeError(None) mutant (mutmut_11) both survive because no
# reachable input can enter this branch -- documented equivalent per
# issue #282's mutation-testing sweep rather than chased with a contrived
# test that fakes an unsupported loader.
if spec is None or spec.loader is None:
raise RuntimeError(f"{module_path}: not importable")
module = importlib.util.module_from_spec(spec)
# Registered before exec: @dataclass (and typing constructs
# generally) resolve annotations via sys.modules[cls.__module__],
# which is None for a module loaded from a path and never inserted.
sys.modules[spec.name] = module
spec.loader.exec_module(module)
consume = getattr(module, "consume", None)
if consume is None:
raise RuntimeError(
f"{module_path}: no consume(data: bytes) —"
" every harness must expose one so its property is runnable"
" without atheris"
)
return consume


def harness_inputs(harness: "Harness") -> Iterator[tuple[Path, bytes]]:
directory = harness_corpus_path(harness)
if not directory.is_dir():
return
for path in sorted(directory.iterdir()):
if path.is_file():
yield path, path.read_bytes()


def discover() -> list[Harness]:
Expand All @@ -97,9 +120,9 @@ def discover() -> list[Harness]:

def replay(harness: Harness) -> int:
"""Run one harness over its corpus. Returns the number of inputs run."""
consume = harness.consume()
consume = harness_consume(harness)
count = 0
for path, data in harness.inputs():
for path, data in harness_inputs(harness):
try:
consume(data)
except Exception as error:
Expand All @@ -125,7 +148,7 @@ def main(argv: list[str] | None = None) -> int:

if args.list:
for harness in harnesses:
print(f"{harness.name}: {sum(1 for _ in harness.inputs())} input(s)")
print(f"{harness.name}: {sum(1 for _ in harness_inputs(harness))} input(s)")
return 0

total = 0
Expand Down
65 changes: 44 additions & 21 deletions mcp_server/core/ablation.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,30 +111,53 @@ class Mechanism(Enum):

@dataclass
class AblationConfig:
"""Configuration specifying which mechanisms are enabled/disabled."""
"""Configuration specifying which mechanisms are enabled/disabled.

Data only — mutmut skips the body of any `@dataclass`-decorated class
(`mutmut/mutation/file_mutation.py:236`; issue #262 3rd pass, #282).
`ablation_config_is_enabled/disable/enable/disable_all_except` below
carry the logic as free functions instead.
"""

disabled: set[str] = field(default_factory=set)

def is_enabled(self, mechanism: Mechanism | str) -> bool:
"""Check if a mechanism is enabled."""
name = mechanism.value if isinstance(mechanism, Mechanism) else mechanism
return name not in self.disabled

def disable(self, mechanism: Mechanism | str) -> "AblationConfig":
"""Return new config with mechanism disabled."""
name = mechanism.value if isinstance(mechanism, Mechanism) else mechanism
return AblationConfig(disabled=self.disabled | {name})

def enable(self, mechanism: Mechanism | str) -> "AblationConfig":
"""Return new config with mechanism enabled."""
name = mechanism.value if isinstance(mechanism, Mechanism) else mechanism
return AblationConfig(disabled=self.disabled - {name})

def disable_all_except(self, *mechanisms: Mechanism) -> "AblationConfig":
"""Disable all mechanisms except the specified ones."""
keep = {m.value for m in mechanisms}
all_mechs = {m.value for m in Mechanism}
return AblationConfig(disabled=all_mechs - keep)

def ablation_config_is_enabled(
config: "AblationConfig", mechanism: Mechanism | str
) -> bool:
"""Check if a mechanism is enabled."""
name = mechanism.value if isinstance(mechanism, Mechanism) else mechanism
return name not in config.disabled


def ablation_config_disable(
config: "AblationConfig", mechanism: Mechanism | str
) -> "AblationConfig":
"""Return new config with mechanism disabled."""
name = mechanism.value if isinstance(mechanism, Mechanism) else mechanism
return AblationConfig(disabled=config.disabled | {name})


def ablation_config_enable(
config: "AblationConfig", mechanism: Mechanism | str
) -> "AblationConfig":
"""Return new config with mechanism enabled."""
name = mechanism.value if isinstance(mechanism, Mechanism) else mechanism
return AblationConfig(disabled=config.disabled - {name})


def ablation_config_disable_all_except(
config: "AblationConfig", *mechanisms: Mechanism
) -> "AblationConfig":
"""Disable all mechanisms except the specified ones.

``config`` is unread — kept for a uniform ``(config, ...)`` call shape;
the result always replaces ``disabled`` wholesale (matches the
pre-extraction method's own behavior of ignoring prior state).
"""
keep = {m.value for m in mechanisms}
all_mechs = {m.value for m in Mechanism}
return AblationConfig(disabled=all_mechs - keep)


# -- Ablation Results ---------------------------------------------------------
Expand Down
23 changes: 15 additions & 8 deletions mcp_server/core/attentional_control.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,14 +92,21 @@ class AttentionAllocation:
capacity: int
overflow: int = 0

def as_dict(self) -> dict:
return {
"focus": self.focus,
"weights": [round(w, 4) for w in self.weights],
"entropy": round(self.entropy, 4),
"capacity": self.capacity,
"overflow": self.overflow,
}

def attention_allocation_as_dict(allocation: "AttentionAllocation") -> dict:
"""A free function, not a method: mutmut categorically excludes the
body of any `@dataclass`-decorated class (`mutmut/mutation/
file_mutation.py:236`), so logic placed on `AttentionAllocation`
methods would carry zero mutation coverage no matter how the test
loader names the module (issue #262 3rd pass; issue #282).
"""
return {
"focus": allocation.focus,
"weights": [round(w, 4) for w in allocation.weights],
"entropy": round(allocation.entropy, 4),
"capacity": allocation.capacity,
"overflow": allocation.overflow,
}


# ── Relevance scoring (top-down) ────────────────────────────────────────────
Expand Down
Loading